saret/supermarket-scraping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added new requirements to requirements.txt

Browse Source
This commit is contained in:
KorenLazar
2021-08-17 09:35:20 +03:00
parent 63fec1490c
commit 3770352d04
1 changed files with 2 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/test_scraping.py
View File

@@ -8,11 +8,7 @@ import pandas as pd
from chains.bareket import Bareket from chains.bareket import Bareket
from chains.co_op import CoOp from chains.co_op import CoOp
from chains.dor_alon import DorAlon from chains.dor_alon import DorAlon
from chains.freshmarket import Freshmarket
from chains.hazi_hinam import HaziHinam
from chains.keshet import Keshet from chains.keshet import Keshet
from chains.maayan2000 import Maayan2000
from chains.mahsaneiHashook import MahsaneiHashook
from chains.shuk_hayir import ShukHayir from chains.shuk_hayir import ShukHayir
from chains.stop_market import StopMarket from chains.stop_market import StopMarket
from chains.tiv_taam import TivTaam from chains.tiv_taam import TivTaam
@@ -51,8 +47,6 @@ MIN_NUM_OF_PROMOS = 3
def test_scraping(): def test_scraping():
filename = 'temp.xlsx' filename = 'temp.xlsx'
for chain_name, chain in tqdm(chain_dict.items(), desc='chains'): for chain_name, chain in tqdm(chain_dict.items(), desc='chains'):
if chain_name in [repr(Maayan2000), repr(ZolVebegadol), repr(ShukHayir)]:
continue
logging.info(f'Test scraping promotions from {chain_name}') logging.info(f'Test scraping promotions from {chain_name}')
if chain_name == repr(DorAlon): if chain_name == repr(DorAlon):
store_id = 501 store_id = 501
@@ -73,12 +67,12 @@ def test_scraping():
output_filename=filename, output_filename=filename,
chain=chain, chain=chain,
load_promos=False, load_promos=False,
load_xml=False load_xml=False,
) )
df = pd.read_excel(filename) df = pd.read_excel(filename)
except Exception as e: except Exception as e:
logging.error(e) logging.error(e)
logging.info(f"Failed loading excel of {chain_name}") logging.error(f"Failed loading excel of {chain_name}")
raise raise
assert df.shape[0] > MIN_NUM_OF_PROMOS and df.shape[1] == PROMOTION_COLS_NUM, f"Failed scraping {chain_name}" assert df.shape[0] > MIN_NUM_OF_PROMOS and df.shape[1] == PROMOTION_COLS_NUM, f"Failed scraping {chain_name}"