saret/supermarket-scraping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added specific searching for the download url of non-full promotions and prices files. Changed return value of get_download_url accordingly.

Browse Source
This commit is contained in:
KorenLazar
2021-08-17 13:06:42 +03:00
parent 3770352d04
commit cffdd84086
4 changed files with 36 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
chains/cerberus_web_client.py
View File

@@ -1,4 +1,6 @@
import json
import re
import requests
from supermarket_chain import SupermarketChain
@@ -17,7 +19,13 @@ class CerberusWebClient:
ajax_dir_payload: dict = {'iDisplayLength': 100000, 'sSearch': category.name.replace('s', '')}
s: requests.Response = session.post(hostname + "/file/ajax_dir", data=ajax_dir_payload)
s_json: dict = json.loads(s.text)
suffix: str = next(d['name'] for d in s_json['aaData'] if f'-{store_id:03d}-20' in d['name'])
if category in [SupermarketChain.XMLFilesCategory.Promos, SupermarketChain.XMLFilesCategory.Prices]:
filter_func = lambda d, id: f'-{id:03d}-20' in d['name'] and not re.search('full', d['name'], re.IGNORECASE)
if not any(filter_func(d, store_id) for d in s_json['aaData']):
return "" # Could not find non-full Prices/Promos file
else:
filter_func = lambda d, id: f'-{id:03d}-20' in d['name']
suffix: str = next(d['name'] for d in s_json['aaData'] if filter_func(d, store_id))
download_url: str = hostname + "/file/d/" + suffix
return download_url