79 lines
2.1 KiB
Python
79 lines
2.1 KiB
Python
import logging
|
|
import os
|
|
|
|
import pytest
|
|
from tqdm import tqdm
|
|
import pandas as pd
|
|
|
|
from chains.bareket import Bareket
|
|
from chains.co_op import CoOp
|
|
from chains.dor_alon import DorAlon
|
|
from chains.keshet import Keshet
|
|
from chains.shuk_hayir import ShukHayir
|
|
from chains.stop_market import StopMarket
|
|
from chains.tiv_taam import TivTaam
|
|
from chains.zol_vebegadol import ZolVebegadol
|
|
from promotion import PROMOTION_COLS_NUM, main_latest_promos
|
|
from supermarket_chain import SupermarketChain
|
|
from chains import (
|
|
bareket,
|
|
mahsaneiHashook,
|
|
dor_alon,
|
|
freshmarket,
|
|
hazi_hinam,
|
|
keshet,
|
|
stop_market,
|
|
tiv_taam,
|
|
shufersal,
|
|
co_op,
|
|
victory,
|
|
yohananof,
|
|
zol_vebegadol,
|
|
rami_levi,
|
|
osher_ad,
|
|
maayan2000,
|
|
shuk_hayir,
|
|
king_store,
|
|
shefa_birkat_hashem,
|
|
)
|
|
|
|
pytest.main(args=['-s', os.path.abspath(__file__)])
|
|
|
|
chain_dict = {repr(chain): chain() if callable(chain) else None for chain in SupermarketChain.__subclasses__()}
|
|
|
|
MIN_NUM_OF_PROMOS = 3
|
|
|
|
|
|
def test_scraping():
|
|
filename = 'temp.xlsx'
|
|
for chain_name, chain in tqdm(chain_dict.items(), desc='chains'):
|
|
logging.info(f'Test scraping promotions from {chain_name}')
|
|
if chain_name == repr(DorAlon):
|
|
store_id = 501
|
|
elif chain_name in [repr(Keshet), repr(TivTaam), repr(Bareket), repr(ZolVebegadol)]:
|
|
store_id = 2
|
|
elif chain_name == repr(CoOp):
|
|
store_id = 202
|
|
elif chain_name == repr(ShukHayir):
|
|
store_id = 4
|
|
elif chain_name == repr(StopMarket):
|
|
store_id = 5
|
|
else:
|
|
store_id = 1
|
|
|
|
try:
|
|
main_latest_promos(
|
|
store_id=store_id,
|
|
output_filename=filename,
|
|
chain=chain,
|
|
load_promos=False,
|
|
load_xml=False,
|
|
)
|
|
df = pd.read_excel(filename)
|
|
except Exception as e:
|
|
logging.error(e)
|
|
logging.error(f"Failed loading excel of {chain_name}")
|
|
raise
|
|
|
|
assert df.shape[0] > MIN_NUM_OF_PROMOS and df.shape[1] == PROMOTION_COLS_NUM, f"Failed scraping {chain_name}"
|