changed chains' members to be 'immutable static'

This commit is contained in:
KorenLazar
2021-02-06 15:57:05 +02:00
parent 3a57edf5af
commit 5aa4cd734d
6 changed files with 50 additions and 126 deletions

View File

@@ -8,16 +8,12 @@ from supermarket_chain import SupermarketChain
class CoOp(SupermarketChain): class CoOp(SupermarketChain):
_promotion_tag_name = 'Sale'
promotion_tag_name = 'Sale' _promotion_update_tag_name = 'PriceUpdateDate'
promotion_update_tag_name = 'PriceUpdateDate' _date_format = '%Y/%m/%d'
date_format = '%Y/%m/%d' _date_hour_format = '%Y/%m/%d %H:%M:%S'
date_hour_format = '%Y/%m/%d %H:%M:%S' _update_date_format = '%Y/%m/%d %H:%M:%S'
item_tag_name = 'Product' _item_tag_name = 'Product'
@property
def update_date_format(self):
return CoOp.date_hour_format
@staticmethod @staticmethod
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
@@ -27,13 +23,10 @@ class CoOp(SupermarketChain):
soup = BeautifulSoup(req_res.text, features='lxml') soup = BeautifulSoup(req_res.text, features='lxml')
suffix: str = soup.find('a', href=lambda value: value and category.name.replace('s', '') in value suffix: str = soup.find('a', href=lambda value: value and category.name.replace('s', '') in value
and f'-{store_id:03d}-20' in value).attrs['href'] and f'-{store_id:03d}-20' in value).attrs['href']
down_url = prefix + suffix down_url: str = prefix + suffix
print(down_url) print(down_url)
return down_url return down_url
class XMLFilesCategory(SupermarketChain.XMLFilesCategory):
All, Promos, PromosFull, Prices, PricesFull, Stores = range(6)
def __repr__(self): def __repr__(self):
return 'CoOp' return 'CoOp'

View File

@@ -132,6 +132,7 @@ def main_latest_promos(store_id: int, load_xml: bool, logger, chain: Supermarket
:param chain: The name of the requested supermarket chain :param chain: The name of the requested supermarket chain
:param store_id: A given store id :param store_id: A given store id
:param load_xml: A boolean representing whether to load an existing prices xml file :param load_xml: A boolean representing whether to load an existing prices xml file
:param load_promos: A boolean representing whether to load an existing promos xml file
:param logger: A given logger :param logger: A given logger
""" """

View File

@@ -1,66 +1,28 @@
import json import json
from typing import Dict, List
import requests import requests
from bs4.element import Tag
from item import Item
from supermarket_chain import SupermarketChain from supermarket_chain import SupermarketChain
class RamiLevi(SupermarketChain): class RamiLevi(SupermarketChain):
@property _date_hour_format = '%Y-%m-%d %H:%M:%S'
def promotion_tag_name(self):
return 'Promotion'
@property
def promotion_update_tag_name(self):
return 'PromotionUpdateDate'
@property
def date_format(self):
return '%Y-%m-%d'
@property
def date_hour_format(self):
return '%Y-%m-%d %H:%M:%S'
@property
def update_date_format(self):
return '%Y-%m-%d %H:%M'
@property
def item_tag_name(self):
return 'Item'
@staticmethod @staticmethod
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
hostname = "https://publishedprices.co.il" hostname: str = "https://publishedprices.co.il"
# Post the payload to the site to log in # Post the payload to the site to log in
session.post(hostname + "/login/user", data={'username': 'ramilevi'}) session.post(hostname + "/login/user", data={'username': 'ramilevi'})
# Scrape the data # Scrape the data
ajax_dir_payload = {'iDisplayLength': 100000, 'sSearch': category.name.replace('s', '')} ajax_dir_payload: dict = {'iDisplayLength': 100000, 'sSearch': category.name.replace('s', '')}
s = session.post(hostname + "/file/ajax_dir", data=ajax_dir_payload) s: requests.Response = session.post(hostname + "/file/ajax_dir", data=ajax_dir_payload)
s_json = json.loads(s.text) s_json: dict = json.loads(s.text)
suffix = next(d['name'] for d in s_json['aaData'] if f'-{store_id:03d}-20' in d['name']) suffix: str = next(d['name'] for d in s_json['aaData'] if f'-{store_id:03d}-20' in d['name'])
download_url = hostname + "/file/d/" + suffix download_url: str = hostname + "/file/d/" + suffix
print(download_url) print(download_url)
return download_url return download_url
@staticmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items
class XMLFilesCategory(SupermarketChain.XMLFilesCategory):
All, Promos, PromosFull, Prices, PricesFull, Stores = range(6)
def __repr__(self): def __repr__(self):
return 'RamiLevi' return 'RamiLevi'

View File

@@ -1,22 +1,10 @@
from typing import Dict, List
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from bs4.element import Tag
from item import Item
from supermarket_chain import SupermarketChain from supermarket_chain import SupermarketChain
class ShuferSal(SupermarketChain): class ShuferSal(SupermarketChain):
promotion_tag_name = 'Promotion'
promotion_update_tag_name = 'PromotionUpdateDate'
date_format = '%Y-%m-%d'
date_hour_format = '%Y-%m-%d %H:%M'
item_tag_name = 'Item'
@property
def update_date_format(self):
return ShuferSal.date_hour_format
@staticmethod @staticmethod
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
@@ -24,23 +12,10 @@ class ShuferSal(SupermarketChain):
if SupermarketChain.is_valid_store_id(int(store_id)): if SupermarketChain.is_valid_store_id(int(store_id)):
url += f"&storeId={store_id}" url += f"&storeId={store_id}"
req_res: requests.Response = requests.get(url) req_res: requests.Response = requests.get(url)
soup = BeautifulSoup(req_res.text, features='lxml') soup: BeautifulSoup = BeautifulSoup(req_res.text, features='lxml')
down_url = soup.find('a', text="לחץ להורדה")['href'] down_url: str = soup.find('a', text="לחץ להורדה")['href']
print(down_url) print(down_url)
return down_url return down_url
class XMLFilesCategory(SupermarketChain.XMLFilesCategory):
All, Prices, PricesFull, Promos, PromosFull, Stores = range(6)
def __repr__(self): def __repr__(self):
return 'Shufersal' return 'Shufersal'
@staticmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items

View File

@@ -15,36 +15,42 @@ class SupermarketChain:
A class representing a supermarket chain. A class representing a supermarket chain.
""" """
@abstractmethod
class XMLFilesCategory(Enum): class XMLFilesCategory(Enum):
""" """
An enum class of different XML files produced by a supermarket chain An enum class of different XML files produced by a supermarket chain
""" """
pass All, Prices, PricesFull, Promos, PromosFull, Stores = range(6)
_promotion_tag_name = 'Promotion'
_promotion_update_tag_name = 'PromotionUpdateDate'
_date_format = '%Y-%m-%d'
_date_hour_format = '%Y-%m-%d %H:%M'
_update_date_format = '%Y-%m-%d %H:%M'
_item_tag_name = 'Item'
@property @property
@abstractmethod def promotion_tag_name(self):
def promotion_tag_name(self): pass return type(self)._promotion_tag_name
@property @property
@abstractmethod def promotion_update_tag_name(self):
def promotion_update_tag_name(self): pass return type(self)._promotion_update_tag_name
@property @property
@abstractmethod def date_format(self):
def date_format(self): pass return type(self)._date_format
@property @property
@abstractmethod def date_hour_format(self):
def date_hour_format(self): pass return type(self)._date_hour_format
@property @property
@abstractmethod def update_date_format(self):
def update_date_format(self): pass return type(self)._update_date_format
@property @property
@abstractmethod def item_tag_name(self):
def item_tag_name(self): pass return type(self)._item_tag_name
@staticmethod @staticmethod
def is_valid_store_id(store_id: int) -> bool: def is_valid_store_id(store_id: int) -> bool:
@@ -81,7 +87,6 @@ class SupermarketChain:
pass pass
@staticmethod @staticmethod
@abstractmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]: def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
""" """
This method returns a list of the items that participate in a given promotion. This method returns a list of the items that participate in a given promotion.
@@ -89,7 +94,13 @@ class SupermarketChain:
:param promo: A given promotion :param promo: A given promotion
:param items_dict: A given dictionary of products :param items_dict: A given dictionary of products
""" """
pass items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items
@staticmethod @staticmethod
def get_null_items(promo: Tag, items_dict: Dict[str, Item]) -> List[str]: def get_null_items(promo: Tag, items_dict: Dict[str, Item]) -> List[str]:

View File

@@ -1,9 +1,6 @@
import json import json
from typing import Dict, List
import requests import requests
from bs4.element import Tag
from item import Item
from supermarket_chain import SupermarketChain from supermarket_chain import SupermarketChain
@@ -11,13 +8,8 @@ class ZolVebegadol(SupermarketChain):
def __repr__(self): def __repr__(self):
return 'Zol-Vebegadol' return 'Zol-Vebegadol'
class XMLFilesCategory(SupermarketChain.XMLFilesCategory): _date_hour_format = '%Y-%m-%d %H:%M:%S'
All, Promos, PromosFull, Prices, PricesFull, Stores = range(6) _update_date_format = '%Y-%m-%d %H:%M:%S'
promotion_tag_name = 'Promotion'
promotion_update_tag_name = 'PromotionUpdateDate'
date_format = '%Y-%m-%d'
date_hour_format = '%Y-%m-%d %H:%M:%S'
item_tag_name = 'Item' item_tag_name = 'Item'
@property @property
@@ -28,20 +20,10 @@ class ZolVebegadol(SupermarketChain):
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str: def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
prefix = "http://zolvebegadol.binaprojects.com" prefix = "http://zolvebegadol.binaprojects.com"
url = prefix + "/MainIO_Hok.aspx" url = prefix + "/MainIO_Hok.aspx"
req_res: requests.Response = requests.get(url) req_res: requests.Response = session.get(url)
jsons_files = json.loads(req_res.text) jsons_files = json.loads(req_res.text)
suffix = next(cur_json["FileNm"] for cur_json in jsons_files if f'-{store_id:03d}-20' in cur_json["FileNm"] suffix = next(cur_json["FileNm"] for cur_json in jsons_files if f'-{store_id:03d}-20' in cur_json["FileNm"]
and category.name.replace('s', '') in cur_json["FileNm"]) and category.name.replace('s', '') in cur_json["FileNm"])
down_url = '/'.join([prefix, "Download", suffix]) down_url: str = '/'.join([prefix, "Download", suffix])
print(down_url) print(down_url)
return down_url return down_url
@staticmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items