saret/supermarket-scraping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

has fixed has_expired validation, added start_date and has_started validation, changed promotions sorting to max(update_date,start_date),

Browse Source
This commit is contained in:
KorenLazar
2020-12-07 13:48:16 +02:00
parent 50dd160a06
commit dfca61c793
1 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
main.py
View File

@@ -36,15 +36,20 @@ class Promotion:
It contains only part of the available information in Shufersal's data. It contains only part of the available information in Shufersal's data.
""" """
def __init__(self, content: str, end_date: datetime, update_date: datetime, code_items: List[str]): def __init__(self, content: str, start_date: datetime, end_date: datetime, update_date: datetime,
code_items: List[str]):
self.content: str = content self.content: str = content
self.start_date = start_date
self.end_date: datetime = end_date self.end_date: datetime = end_date
self.update_date: datetime = update_date self.update_date: datetime = update_date
self.code_items: List[str] = code_items self.code_items: List[str] = code_items
def __str__(self): def __str__(self):
title = self.content
dates_range = f"Between {self.start_date.date()} and {self.end_date.date()}"
update_line = f"Updated at {self.update_date.date()}"
items = '\n'.join(str(item) for item in self.code_items) items = '\n'.join(str(item) for item in self.code_items)
return f"*** {self.content} until {self.end_date.date()} update in {self.update_date} ***\n{items}\n" return '\n'.join([title, dates_range, update_line, items]) + '\n'
def get_download_url(store_id: int, cat_id: int) -> str: def get_download_url(store_id: int, cat_id: int) -> str:
@@ -102,6 +107,7 @@ def get_available_promos(store_id: int, load_xml: bool) -> List[Promotion]:
for cur_promo in bs_promos.find_all("Promotion"): for cur_promo in bs_promos.find_all("Promotion"):
cur_promo = Promotion( cur_promo = Promotion(
content=cur_promo.find('PromotionDescription').text, content=cur_promo.find('PromotionDescription').text,
start_date=datetime.strptime(cur_promo.find('PromotionStartDate').text, '%Y-%m-%d'),
end_date=datetime.strptime(cur_promo.find('PromotionEndDate').text, '%Y-%m-%d'), end_date=datetime.strptime(cur_promo.find('PromotionEndDate').text, '%Y-%m-%d'),
update_date=datetime.strptime(cur_promo.find('PromotionUpdateDate').text, '%Y-%m-%d %H:%M'), update_date=datetime.strptime(cur_promo.find('PromotionUpdateDate').text, '%Y-%m-%d %H:%M'),
code_items=[items_dict.get(item.find('ItemCode').text) for item in cur_promo.find_all('Item') code_items=[items_dict.get(item.find('ItemCode').text) for item in cur_promo.find_all('Item')
@@ -114,9 +120,12 @@ def get_available_promos(store_id: int, load_xml: bool) -> List[Promotion]:
def is_valid_promo(promo: Promotion): def is_valid_promo(promo: Promotion):
not_expired = promo.end_date > datetime.now() today_date = datetime.now()
not_expired = promo.end_date.date() >= today_date.date()
has_started = promo.start_date <= today_date
has_products = len(promo.code_items) > 0 has_products = len(promo.code_items) > 0
return not_expired and has_products and not any(product in promo.content for product in PRODUCTS_TO_IGNORE) in_promo_ignore_list = any(product in promo.content for product in PRODUCTS_TO_IGNORE)
return not_expired and has_started and has_products and not in_promo_ignore_list
def create_items_dict(store_id: int, load_xml) -> Dict: def create_items_dict(store_id: int, load_xml) -> Dict:
@@ -146,7 +155,7 @@ def main_latest_promos(store_id: int, load_xml: bool):
""" """
promotions = get_available_promos(store_id, load_xml) promotions = get_available_promos(store_id, load_xml)
promotions.sort(key=lambda promo: promo.update_date, reverse=True) promotions.sort(key=lambda promo: max(promo.update_date, promo.start_date), reverse=True)
logger.info('\n'.join(str(promotion) for promotion in promotions)) logger.info('\n'.join(str(promotion) for promotion in promotions))