From 5aa4cd734d6af07670e13c78f9863dee9e252fc4 Mon Sep 17 00:00:00 2001 From: KorenLazar Date: Sat, 6 Feb 2021 15:57:05 +0200 Subject: [PATCH] changed chains' members to be 'immutable static' --- co_op.py | 23 +++++++------------- promotion.py | 1 + rami_levi.py | 52 ++++++-------------------------------------- shufersal.py | 29 ++---------------------- supermarket_chain.py | 43 ++++++++++++++++++++++-------------- zol_vebegadol.py | 28 +++++------------------- 6 files changed, 50 insertions(+), 126 deletions(-) diff --git a/co_op.py b/co_op.py index 900676a..81f0eba 100644 --- a/co_op.py +++ b/co_op.py @@ -8,16 +8,12 @@ from supermarket_chain import SupermarketChain class CoOp(SupermarketChain): - - promotion_tag_name = 'Sale' - promotion_update_tag_name = 'PriceUpdateDate' - date_format = '%Y/%m/%d' - date_hour_format = '%Y/%m/%d %H:%M:%S' - item_tag_name = 'Product' - - @property - def update_date_format(self): - return CoOp.date_hour_format + _promotion_tag_name = 'Sale' + _promotion_update_tag_name = 'PriceUpdateDate' + _date_format = '%Y/%m/%d' + _date_hour_format = '%Y/%m/%d %H:%M:%S' + _update_date_format = '%Y/%m/%d %H:%M:%S' + _item_tag_name = 'Product' @staticmethod def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: @@ -26,14 +22,11 @@ class CoOp(SupermarketChain): req_res: requests.Response = requests.get(url) soup = BeautifulSoup(req_res.text, features='lxml') suffix: str = soup.find('a', href=lambda value: value and category.name.replace('s', '') in value - and f'-{store_id:03d}-20' in value).attrs['href'] - down_url = prefix + suffix + and f'-{store_id:03d}-20' in value).attrs['href'] + down_url: str = prefix + suffix print(down_url) return down_url - class XMLFilesCategory(SupermarketChain.XMLFilesCategory): - All, Promos, PromosFull, Prices, PricesFull, Stores = range(6) - def __repr__(self): return 'CoOp' diff --git a/promotion.py b/promotion.py index 118e400..2b2ac18 100644 --- a/promotion.py +++ b/promotion.py @@ -132,6 +132,7 @@ def main_latest_promos(store_id: int, load_xml: bool, logger, chain: Supermarket :param chain: The name of the requested supermarket chain :param store_id: A given store id :param load_xml: A boolean representing whether to load an existing prices xml file + :param load_promos: A boolean representing whether to load an existing promos xml file :param logger: A given logger """ diff --git a/rami_levi.py b/rami_levi.py index 9106843..6f4b345 100644 --- a/rami_levi.py +++ b/rami_levi.py @@ -1,66 +1,28 @@ import json -from typing import Dict, List import requests -from bs4.element import Tag -from item import Item from supermarket_chain import SupermarketChain class RamiLevi(SupermarketChain): - @property - def promotion_tag_name(self): - return 'Promotion' - - @property - def promotion_update_tag_name(self): - return 'PromotionUpdateDate' - - @property - def date_format(self): - return '%Y-%m-%d' - - @property - def date_hour_format(self): - return '%Y-%m-%d %H:%M:%S' - - @property - def update_date_format(self): - return '%Y-%m-%d %H:%M' - - @property - def item_tag_name(self): - return 'Item' + _date_hour_format = '%Y-%m-%d %H:%M:%S' @staticmethod def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: - hostname = "https://publishedprices.co.il" + hostname: str = "https://publishedprices.co.il" # Post the payload to the site to log in session.post(hostname + "/login/user", data={'username': 'ramilevi'}) # Scrape the data - ajax_dir_payload = {'iDisplayLength': 100000, 'sSearch': category.name.replace('s', '')} - s = session.post(hostname + "/file/ajax_dir", data=ajax_dir_payload) - s_json = json.loads(s.text) - suffix = next(d['name'] for d in s_json['aaData'] if f'-{store_id:03d}-20' in d['name']) + ajax_dir_payload: dict = {'iDisplayLength': 100000, 'sSearch': category.name.replace('s', '')} + s: requests.Response = session.post(hostname + "/file/ajax_dir", data=ajax_dir_payload) + s_json: dict = json.loads(s.text) + suffix: str = next(d['name'] for d in s_json['aaData'] if f'-{store_id:03d}-20' in d['name']) - download_url = hostname + "/file/d/" + suffix + download_url: str = hostname + "/file/d/" + suffix print(download_url) return download_url - @staticmethod - def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]: - items = list() - for item in promo.find_all('Item'): - item_code = item.find('ItemCode').text - full_item_info = items_dict.get(item_code) - if full_item_info: - items.append(full_item_info) - return items - - class XMLFilesCategory(SupermarketChain.XMLFilesCategory): - All, Promos, PromosFull, Prices, PricesFull, Stores = range(6) - def __repr__(self): return 'RamiLevi' diff --git a/shufersal.py b/shufersal.py index 5656fbc..dfc4bfd 100644 --- a/shufersal.py +++ b/shufersal.py @@ -1,22 +1,10 @@ -from typing import Dict, List import requests from bs4 import BeautifulSoup -from bs4.element import Tag -from item import Item from supermarket_chain import SupermarketChain class ShuferSal(SupermarketChain): - promotion_tag_name = 'Promotion' - promotion_update_tag_name = 'PromotionUpdateDate' - date_format = '%Y-%m-%d' - date_hour_format = '%Y-%m-%d %H:%M' - item_tag_name = 'Item' - - @property - def update_date_format(self): - return ShuferSal.date_hour_format @staticmethod def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: @@ -24,23 +12,10 @@ class ShuferSal(SupermarketChain): if SupermarketChain.is_valid_store_id(int(store_id)): url += f"&storeId={store_id}" req_res: requests.Response = requests.get(url) - soup = BeautifulSoup(req_res.text, features='lxml') - down_url = soup.find('a', text="לחץ להורדה")['href'] + soup: BeautifulSoup = BeautifulSoup(req_res.text, features='lxml') + down_url: str = soup.find('a', text="לחץ להורדה")['href'] print(down_url) return down_url - class XMLFilesCategory(SupermarketChain.XMLFilesCategory): - All, Prices, PricesFull, Promos, PromosFull, Stores = range(6) - def __repr__(self): return 'Shufersal' - - @staticmethod - def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]: - items = list() - for item in promo.find_all('Item'): - item_code = item.find('ItemCode').text - full_item_info = items_dict.get(item_code) - if full_item_info: - items.append(full_item_info) - return items diff --git a/supermarket_chain.py b/supermarket_chain.py index cd989c3..4520b86 100644 --- a/supermarket_chain.py +++ b/supermarket_chain.py @@ -15,36 +15,42 @@ class SupermarketChain: A class representing a supermarket chain. """ - @abstractmethod class XMLFilesCategory(Enum): """ An enum class of different XML files produced by a supermarket chain """ - pass + All, Prices, PricesFull, Promos, PromosFull, Stores = range(6) + + _promotion_tag_name = 'Promotion' + _promotion_update_tag_name = 'PromotionUpdateDate' + _date_format = '%Y-%m-%d' + _date_hour_format = '%Y-%m-%d %H:%M' + _update_date_format = '%Y-%m-%d %H:%M' + _item_tag_name = 'Item' @property - @abstractmethod - def promotion_tag_name(self): pass + def promotion_tag_name(self): + return type(self)._promotion_tag_name @property - @abstractmethod - def promotion_update_tag_name(self): pass + def promotion_update_tag_name(self): + return type(self)._promotion_update_tag_name @property - @abstractmethod - def date_format(self): pass + def date_format(self): + return type(self)._date_format @property - @abstractmethod - def date_hour_format(self): pass + def date_hour_format(self): + return type(self)._date_hour_format @property - @abstractmethod - def update_date_format(self): pass + def update_date_format(self): + return type(self)._update_date_format @property - @abstractmethod - def item_tag_name(self): pass + def item_tag_name(self): + return type(self)._item_tag_name @staticmethod def is_valid_store_id(store_id: int) -> bool: @@ -81,7 +87,6 @@ class SupermarketChain: pass @staticmethod - @abstractmethod def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]: """ This method returns a list of the items that participate in a given promotion. @@ -89,7 +94,13 @@ class SupermarketChain: :param promo: A given promotion :param items_dict: A given dictionary of products """ - pass + items = list() + for item in promo.find_all('Item'): + item_code = item.find('ItemCode').text + full_item_info = items_dict.get(item_code) + if full_item_info: + items.append(full_item_info) + return items @staticmethod def get_null_items(promo: Tag, items_dict: Dict[str, Item]) -> List[str]: diff --git a/zol_vebegadol.py b/zol_vebegadol.py index 3879a57..3297c0a 100644 --- a/zol_vebegadol.py +++ b/zol_vebegadol.py @@ -1,9 +1,6 @@ import json -from typing import Dict, List import requests -from bs4.element import Tag -from item import Item from supermarket_chain import SupermarketChain @@ -11,13 +8,8 @@ class ZolVebegadol(SupermarketChain): def __repr__(self): return 'Zol-Vebegadol' - class XMLFilesCategory(SupermarketChain.XMLFilesCategory): - All, Promos, PromosFull, Prices, PricesFull, Stores = range(6) - - promotion_tag_name = 'Promotion' - promotion_update_tag_name = 'PromotionUpdateDate' - date_format = '%Y-%m-%d' - date_hour_format = '%Y-%m-%d %H:%M:%S' + _date_hour_format = '%Y-%m-%d %H:%M:%S' + _update_date_format = '%Y-%m-%d %H:%M:%S' item_tag_name = 'Item' @property @@ -25,23 +17,13 @@ class ZolVebegadol(SupermarketChain): return ZolVebegadol.date_hour_format @staticmethod - def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session:requests.Session) -> str: + def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: prefix = "http://zolvebegadol.binaprojects.com" url = prefix + "/MainIO_Hok.aspx" - req_res: requests.Response = requests.get(url) + req_res: requests.Response = session.get(url) jsons_files = json.loads(req_res.text) suffix = next(cur_json["FileNm"] for cur_json in jsons_files if f'-{store_id:03d}-20' in cur_json["FileNm"] and category.name.replace('s', '') in cur_json["FileNm"]) - down_url = '/'.join([prefix, "Download", suffix]) + down_url: str = '/'.join([prefix, "Download", suffix]) print(down_url) return down_url - - @staticmethod - def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]: - items = list() - for item in promo.find_all('Item'): - item_code = item.find('ItemCode').text - full_item_info = items_dict.get(item_code) - if full_item_info: - items.append(full_item_info) - return items