saret/supermarket-scraping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Minor changes

Browse Source
This commit is contained in:
KorenLazar
2021-08-18 11:32:04 +03:00
parent 87b6fbe2b0
commit 90cab0a2e1
2 changed files with 6 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
promotion.py
View File

@@ -104,9 +104,9 @@ class Promotion:
return self.promotion_id == other.promotion_id
def write_promotions_to_csv(promotions: List[Promotion], output_filename: str) -> None:
def write_promotions_to_table(promotions: List[Promotion], output_filename: str) -> None:
"""
This function writes a promotions table to a given CSV or XLSX output file.
This function writes a List of promotions to a csv or xlsx output file.
:param promotions: A given list of promotions
:param output_filename: A given file to write to
@@ -313,7 +313,7 @@ def main_latest_promos(store_id: int, output_filename, chain: SupermarketChain,
promotions: List[Promotion] = get_available_promos(chain, store_id, load_xml, load_promos)
promotions.sort(key=lambda promo: (max(promo.update_date.date(), promo.start_date.date()), promo.start_date -
promo.end_date), reverse=True)
write_promotions_to_csv(promotions, output_filename)
write_promotions_to_table(promotions, output_filename)
def log_promos_by_name(store_id: int, chain: SupermarketChain, promo_name: str, load_prices: bool, load_promos: bool):
@@ -332,7 +332,6 @@ def log_promos_by_name(store_id: int, chain: SupermarketChain, promo_name: str,
logging.info(promo.repr_ltr())
# TODO: change to returning list of Items
def get_all_null_items_in_promos(chain, store_id) -> List[str]:
"""
This function finds all items appearing in the chain's promotions file but not in the chain's prices file.

6
tests/test_scraping.py
View File

@@ -47,12 +47,12 @@ MIN_NUM_OF_PROMOS = 3
def test_searching_for_download_urls():
"""
Test that get_download_url of each chain returns the correct download url for each category:
Test that get_download_url of each chain returns the correct download url for each category in every chain.
"""
session = requests.Session()
for chain_name, chain in tqdm(chain_dict.items(), desc='chains'):
logging.info(f'Finding download url in chain {chain_name}')
logging.info(f'Checking download urls in chain {chain_name}')
store_id: int = valid_store_id_by_chain(chain_name)
_test_download_url_helper(chain, store_id, chain.XMLFilesCategory.PromosFull, r'promo[s]?full', session)
@@ -64,9 +64,9 @@ def test_searching_for_download_urls():
def _test_download_url_helper(chain: SupermarketChain, store_id: int, category: SupermarketChain.XMLFilesCategory,
regex_pat: str, session: requests.session):
download_url: str = chain.get_download_url(store_id, category, session)
logging.debug(download_url)
if not download_url: # Not found non-full Promos/Prices file
return
logging.debug(download_url)
assert re.search(regex_pat, download_url, re.IGNORECASE), f'Invalid {category.name} url in {repr(type(chain))}'
if category in [chain.XMLFilesCategory.Prices, chain.XMLFilesCategory.Promos]:
assert not re.search('full', download_url, re.IGNORECASE), \