saret/supermarket-scraping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

changed chains' members to be 'immutable static'

Browse Source
This commit is contained in:
KorenLazar
2021-02-06 15:57:05 +02:00
parent 3a57edf5af
commit 5aa4cd734d
6 changed files with 50 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
co_op.py
View File

@@ -8,16 +8,12 @@ from supermarket_chain import SupermarketChain
class CoOp(SupermarketChain):
promotion_tag_name = 'Sale'
promotion_update_tag_name = 'PriceUpdateDate'
date_format = '%Y/%m/%d'
date_hour_format = '%Y/%m/%d %H:%M:%S'
item_tag_name = 'Product'
@property
def update_date_format(self):
return CoOp.date_hour_format
_promotion_tag_name = 'Sale'
_promotion_update_tag_name = 'PriceUpdateDate'
_date_format = '%Y/%m/%d'
_date_hour_format = '%Y/%m/%d %H:%M:%S'
_update_date_format = '%Y/%m/%d %H:%M:%S'
_item_tag_name = 'Product'
@staticmethod
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
@@ -26,14 +22,11 @@ class CoOp(SupermarketChain):
req_res: requests.Response = requests.get(url)
soup = BeautifulSoup(req_res.text, features='lxml')
suffix: str = soup.find('a', href=lambda value: value and category.name.replace('s', '') in value
and f'-{store_id:03d}-20' in value).attrs['href']
down_url = prefix + suffix
and f'-{store_id:03d}-20' in value).attrs['href']
down_url: str = prefix + suffix
print(down_url)
return down_url
class XMLFilesCategory(SupermarketChain.XMLFilesCategory):
All, Promos, PromosFull, Prices, PricesFull, Stores = range(6)
def __repr__(self):
return 'CoOp'

1
promotion.py
View File

@@ -132,6 +132,7 @@ def main_latest_promos(store_id: int, load_xml: bool, logger, chain: Supermarket
:param chain: The name of the requested supermarket chain
:param store_id: A given store id
:param load_xml: A boolean representing whether to load an existing prices xml file
:param load_promos: A boolean representing whether to load an existing promos xml file
:param logger: A given logger
"""

52
rami_levi.py
View File

@@ -1,66 +1,28 @@
import json
from typing import Dict, List
import requests
from bs4.element import Tag
from item import Item
from supermarket_chain import SupermarketChain
class RamiLevi(SupermarketChain):
@property
def promotion_tag_name(self):
return 'Promotion'
@property
def promotion_update_tag_name(self):
return 'PromotionUpdateDate'
@property
def date_format(self):
return '%Y-%m-%d'
@property
def date_hour_format(self):
return '%Y-%m-%d %H:%M:%S'
@property
def update_date_format(self):
return '%Y-%m-%d %H:%M'
@property
def item_tag_name(self):
return 'Item'
_date_hour_format = '%Y-%m-%d %H:%M:%S'
@staticmethod
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
hostname = "https://publishedprices.co.il"
hostname: str = "https://publishedprices.co.il"
# Post the payload to the site to log in
session.post(hostname + "/login/user", data={'username': 'ramilevi'})
# Scrape the data
ajax_dir_payload = {'iDisplayLength': 100000, 'sSearch': category.name.replace('s', '')}
s = session.post(hostname + "/file/ajax_dir", data=ajax_dir_payload)
s_json = json.loads(s.text)
suffix = next(d['name'] for d in s_json['aaData'] if f'-{store_id:03d}-20' in d['name'])
ajax_dir_payload: dict = {'iDisplayLength': 100000, 'sSearch': category.name.replace('s', '')}
s: requests.Response = session.post(hostname + "/file/ajax_dir", data=ajax_dir_payload)
s_json: dict = json.loads(s.text)
suffix: str = next(d['name'] for d in s_json['aaData'] if f'-{store_id:03d}-20' in d['name'])
download_url = hostname + "/file/d/" + suffix
download_url: str = hostname + "/file/d/" + suffix
print(download_url)
return download_url
@staticmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items
class XMLFilesCategory(SupermarketChain.XMLFilesCategory):
All, Promos, PromosFull, Prices, PricesFull, Stores = range(6)
def __repr__(self):
return 'RamiLevi'

29
shufersal.py
View File

@@ -1,22 +1,10 @@
from typing import Dict, List
import requests
from bs4 import BeautifulSoup
from bs4.element import Tag
from item import Item
from supermarket_chain import SupermarketChain
class ShuferSal(SupermarketChain):
promotion_tag_name = 'Promotion'
promotion_update_tag_name = 'PromotionUpdateDate'
date_format = '%Y-%m-%d'
date_hour_format = '%Y-%m-%d %H:%M'
item_tag_name = 'Item'
@property
def update_date_format(self):
return ShuferSal.date_hour_format
@staticmethod
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
@@ -24,23 +12,10 @@ class ShuferSal(SupermarketChain):
if SupermarketChain.is_valid_store_id(int(store_id)):
url += f"&storeId={store_id}"
req_res: requests.Response = requests.get(url)
soup = BeautifulSoup(req_res.text, features='lxml')
down_url = soup.find('a', text="לחץ להורדה")['href']
soup: BeautifulSoup = BeautifulSoup(req_res.text, features='lxml')
down_url: str = soup.find('a', text="לחץ להורדה")['href']
print(down_url)
return down_url
class XMLFilesCategory(SupermarketChain.XMLFilesCategory):
All, Prices, PricesFull, Promos, PromosFull, Stores = range(6)
def __repr__(self):
return 'Shufersal'
@staticmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items

43
supermarket_chain.py
View File

@@ -15,36 +15,42 @@ class SupermarketChain:
A class representing a supermarket chain.
"""
@abstractmethod
class XMLFilesCategory(Enum):
"""
An enum class of different XML files produced by a supermarket chain
"""
pass
All, Prices, PricesFull, Promos, PromosFull, Stores = range(6)
_promotion_tag_name = 'Promotion'
_promotion_update_tag_name = 'PromotionUpdateDate'
_date_format = '%Y-%m-%d'
_date_hour_format = '%Y-%m-%d %H:%M'
_update_date_format = '%Y-%m-%d %H:%M'
_item_tag_name = 'Item'
@property
@abstractmethod
def promotion_tag_name(self): pass
def promotion_tag_name(self):
return type(self)._promotion_tag_name
@property
@abstractmethod
def promotion_update_tag_name(self): pass
def promotion_update_tag_name(self):
return type(self)._promotion_update_tag_name
@property
@abstractmethod
def date_format(self): pass
def date_format(self):
return type(self)._date_format
@property
@abstractmethod
def date_hour_format(self): pass
def date_hour_format(self):
return type(self)._date_hour_format
@property
@abstractmethod
def update_date_format(self): pass
def update_date_format(self):
return type(self)._update_date_format
@property
@abstractmethod
def item_tag_name(self): pass
def item_tag_name(self):
return type(self)._item_tag_name
@staticmethod
def is_valid_store_id(store_id: int) -> bool:
@@ -81,7 +87,6 @@ class SupermarketChain:
pass
@staticmethod
@abstractmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
"""
This method returns a list of the items that participate in a given promotion.
@@ -89,7 +94,13 @@ class SupermarketChain:
:param promo: A given promotion
:param items_dict: A given dictionary of products
"""
pass
items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items
@staticmethod
def get_null_items(promo: Tag, items_dict: Dict[str, Item]) -> List[str]:

28
zol_vebegadol.py
View File

@@ -1,9 +1,6 @@
import json
from typing import Dict, List
import requests
from bs4.element import Tag
from item import Item
from supermarket_chain import SupermarketChain
@@ -11,13 +8,8 @@ class ZolVebegadol(SupermarketChain):
def __repr__(self):
return 'Zol-Vebegadol'
class XMLFilesCategory(SupermarketChain.XMLFilesCategory):
All, Promos, PromosFull, Prices, PricesFull, Stores = range(6)
promotion_tag_name = 'Promotion'
promotion_update_tag_name = 'PromotionUpdateDate'
date_format = '%Y-%m-%d'
date_hour_format = '%Y-%m-%d %H:%M:%S'
_date_hour_format = '%Y-%m-%d %H:%M:%S'
_update_date_format = '%Y-%m-%d %H:%M:%S'
item_tag_name = 'Item'
@property
@@ -25,23 +17,13 @@ class ZolVebegadol(SupermarketChain):
return ZolVebegadol.date_hour_format
@staticmethod
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session:requests.Session) -> str:
def get_download_url(store_id: int, category: SupermarketChain.XMLFilesCategory, session: requests.Session) -> str:
prefix = "http://zolvebegadol.binaprojects.com"
url = prefix + "/MainIO_Hok.aspx"
req_res: requests.Response = requests.get(url)
req_res: requests.Response = session.get(url)
jsons_files = json.loads(req_res.text)
suffix = next(cur_json["FileNm"] for cur_json in jsons_files if f'-{store_id:03d}-20' in cur_json["FileNm"]
and category.name.replace('s', '') in cur_json["FileNm"])
down_url = '/'.join([prefix, "Download", suffix])
down_url: str = '/'.join([prefix, "Download", suffix])
print(down_url)
return down_url
@staticmethod
def get_items(promo: Tag, items_dict: Dict[str, Item]) -> List[Item]:
items = list()
for item in promo.find_all('Item'):
item_code = item.find('ItemCode').text
full_item_info = items_dict.get(item_code)
if full_item_info:
items.append(full_item_info)
return items