saret/DH
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: saret:f592236971543ffdb480057251940f17f044cf32
Branches Tags
saret:master
..
compare: saret:218a3d813578f191fb994cd6f65016eb57a4983e
Branches Tags
saret:master

2 Commits
f592236971 ... 218a3d8135

Author SHA1 Message Date
1kamma
218a3d8135 finished scrapping all the data 2023-04-12 22:05:16 +03:00
1kamma
df548fa29d update folders 2023-04-12 12:34:56 +03:00
10 changed files with 90847 additions and 22885 deletions
Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1 +1,2 @@
venv/*
logs/*

130
.ipynb_checkpoints/project_notebook-checkpoint.ipynb Normal file
View File

@@ -0,0 +1,130 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Imported modules"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import scrapping\n",
"import psycopg2\n",
"from psycopg2.extras import execute_batch\n",
"import re\n",
"import logging\n",
"import datetime\n",
"import multiprocessing\n",
"import os\n",
"logging.basicConfig(\n",
" level=logging.INFO, filename=f'{datetime.datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")}.log', filemode='w',\n",
" format='%(name)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')\n",
"# from psycopg2.extensions import register_adapter\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
"connection = conn.cursor()\n",
"with open('project_list') as f:\n",
" for project in f.read().split('\\n'):\n",
" # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
" try:\n",
" scrap = scrapping.get_raw_english_texts_of_project(project)\n",
" except Exception:\n",
" logging.error(f\"Error in {project}:{Exception}\")\n",
" try:\n",
" execute_batch(\n",
" connection,\n",
" \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n",
" conn.commit()\n",
" except Exception:\n",
" with open(project, 'w') as f:\n",
" f.write(json.dumps(scrap))\n",
" logging.error(f\"Error in {project}:{Exception}\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
"connection = conn.cursor()\n",
"logging.basicConfig(\n",
" level=logging.INFO, filename=f'./now.log',\n",
" format='%(name)s - %(levelname)s - %(message)s')\n",
"logging.info('Start')\n",
"with open('project_list', encoding='unicode-escape') as f:\n",
" # project = f.read().split('\\n')[2]\n",
" for project in f.read().split('\\n'):\n",
" # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
" if os.path.exists(project):\n",
" with open(project) as f:\n",
" logging.info(f'reading {project}')\n",
" scrap = json.loads(f.read())\n",
" # scrap = json.loads(project)\n",
" execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n",
" conn.commit()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# with open('cams') as file:\n",
"conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
"connection = conn.cursor()\n",
"scrap = scrapping.get_raw_english_texts_of_project('saao')\n",
"execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n",
"conn.commit()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

5
2023-04-12_01-39-28.log
View File

@@ -1,5 +0,0 @@
root - INFO - http://oracc.iaas.upenn.edu/ario/Q006653/html
root - INFO - http://oracc.iaas.upenn.edu/ario/Q006654/html
root - INFO - http://oracc.iaas.upenn.edu/ario/Q006655/html
root - INFO - http://oracc.iaas.upenn.edu/ario/Q007129/html
root - INFO - http://oracc.iaas.upenn.edu/ario/Q007130/html

22399
2023-04-12_01-43-06.log
View File

File diff suppressed because it is too large Load Diff

447
2023-04-12_12-01-00.log
View File

@@ -1,447 +0,0 @@
root - INFO - Start
root - INFO - Start
root - INFO - reading adsd
root - INFO - reading aemw
root - INFO - reading akklove
root - INFO - reading asbp
root - INFO - reading atae
root - INFO - reading babcity
root - INFO - reading blms
root - INFO - reading btmao
root - INFO - reading btto
root - INFO - reading cams
root - INFO - reading ccpo
root - INFO - reading cdli
root - INFO - reading ckst
root - INFO - reading cmawro
root - INFO - reading contrib
root - INFO - reading dcclt
root - INFO - reading dccmt
root - INFO - reading doc
root - INFO - reading dsst
root - INFO - reading ecut
root - INFO - reading eisl
root - INFO - reading epsd2
root - INFO - reading etcsri
root - INFO - reading glass
root - INFO - reading hbtin
root - INFO - reading lacost
root - INFO - reading lovelyrics
root - INFO - reading neo
root - INFO - reading nere
root - INFO - reading nimrud
root - INFO - reading obel
root - INFO - reading obmc
root - INFO - reading obta
root - INFO - reading ogsl
root - INFO - reading oimea
root - INFO - reading pnao
root - INFO - reading qcat
root - INFO - reading riao
root - INFO - reading ribo
root - INFO - reading rimanum
root - INFO - reading rinap
root - INFO - reading saao
root - INFO - reading suhu
root - INFO - reading tcma
root - INFO - reading tsae
root - INFO - reading xcat
sroot - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224395/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224403/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224417/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224431/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224433/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224447/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224485/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224487/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P224587/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313416/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313417/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313425/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313427/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313435/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313437/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313439/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313447/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313458/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313487/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313491/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313497/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313502/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313505/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313509/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313511/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313523/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313527/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313543/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313551/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313559/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313571/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313598/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313600/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313614/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313623/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313626/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313627/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313629/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313644/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313648/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313660/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313677/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313684/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313699/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313719/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313722/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313726/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313742/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313748/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313755/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313759/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313762/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313807/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313815/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313832/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313854/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313864/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313871/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313874/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313876/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313878/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313879/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313885/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313897/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313904/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313907/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313915/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313919/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313923/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313938/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313947/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313974/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P313975/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314001/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314003/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314022/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314026/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314030/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314032/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314048/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314051/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314054/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314056/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314134/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314144/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314157/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314211/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314223/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314227/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314232/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314238/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314243/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314248/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314257/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314260/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314272/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314273/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314275/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314282/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314287/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314297/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P314316/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334036/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334037/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334038/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334039/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334040/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334041/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334042/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334043/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334044/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334045/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334046/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334047/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334048/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334049/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334050/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334051/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334053/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334054/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334055/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334056/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334078/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334079/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334080/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334081/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334082/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334083/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334100/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334101/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334102/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334113/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334118/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334120/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334124/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334125/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334127/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334135/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334136/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334141/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334142/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334143/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334144/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334158/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334160/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334165/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334166/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334170/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334171/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334172/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334173/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334174/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334175/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334176/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334190/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334193/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334194/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334195/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334209/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334210/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334211/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334212/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334213/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334214/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334271/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334272/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334273/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334284/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334288/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334298/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334314/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334317/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334328/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334329/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334330/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334331/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334332/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334333/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334334/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334335/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334336/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334359/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334372/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334384/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334385/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334390/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334394/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334396/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334397/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334403/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334422/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334432/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334435/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334442/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334443/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334444/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334445/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334496/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334499/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334512/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334519/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334520/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334567/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334568/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334586/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334587/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334588/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334592/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334598/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334610/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334621/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334631/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334632/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334634/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334643/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334644/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334658/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334665/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334667/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334676/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334687/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334689/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334693/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334699/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334709/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334715/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334718/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334721/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334727/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334728/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334729/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334739/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334773/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334774/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334776/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334789/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334791/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334792/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334794/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334804/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334807/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334810/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334820/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334826/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334830/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334834/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334849/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334864/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334895/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334903/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334904/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334910/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334912/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334918/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334922/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P334923/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P336167/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P336172/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P336595/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P336596/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P336597/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P393855/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P393866/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/P428858/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa01/X010028/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P237185/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P240211/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P296062/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P314346/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P334814/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336039/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336040/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336126/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336216/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336217/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336218/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336220/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P336317/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/P500551/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa02/Q009186/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P223388/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P237351/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P238051/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P238321/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P238357/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P313430/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P313818/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P334919/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P334925/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P334926/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P334929/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P334930/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P334931/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P334932/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336128/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336130/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336144/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336149/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336150/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336151/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336158/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336161/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336175/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336225/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336226/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336234/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336242/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336243/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336244/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336245/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336291/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336306/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336599/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336600/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336601/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336602/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336603/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336604/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336605/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336606/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336607/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336608/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336609/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P336796/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P337164/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P338360/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P338383/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P338404/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P338675/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/P338681/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/Q009249/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa03/Q009250/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236925/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236926/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236927/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236928/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236929/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236937/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236943/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236955/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236956/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P236960/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237009/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237018/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237027/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237030/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237053/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237081/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237119/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237127/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237128/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237168/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237173/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237190/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237191/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237203/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237208/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237209/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237210/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237212/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237213/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237222/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237224/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237231/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237355/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237358/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237360/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237361/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237362/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237363/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237364/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237365/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237366/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237367/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237369/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237370/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237371/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237372/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237373/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237374/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237376/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237377/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237378/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237380/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237383/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237386/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237387/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237405/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237410/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237412/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237413/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237416/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237417/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237423/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237435/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237443/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237449/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237461/html
root - INFO - http://oracc.museum.upenn.edu/saao/saa04/P237479/html

2300
2023-04-12_22-00-38.log Normal file
View File

File diff suppressed because it is too large Load Diff

2300
missing_list.txt Normal file
View File

File diff suppressed because it is too large Load Diff

86115
project_notebook.ipynb
View File

File diff suppressed because it is too large Load Diff

31
scrapping.py
View File

@@ -1,7 +1,7 @@
import json
from typing import Dict, List
import requests
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, ResultSet
import os
from pathlib import Path
import re
@@ -31,10 +31,23 @@ def _load_json_from_path(json_path: str) -> Dict:
if os.stat(json_path).st_size != 0: # If the file is not empty:
return json.load(json_file)
def _download_data_from_website(url: str) -> ResultSet:
try:
res = requests.get(url)
soup = BeautifulSoup(res.text, "html.parser")
return soup.find_all("span", {"class": "cell"})
except Exception as e:
logging.error(e)
return ResultSet()
def _clean_raw_text(results: ResultSet)->str:
return " ".join(["".join([content if isinstance(content, str) else content.text
for content in result.contents]) for result in results]).replace('\n', ' ')
def get_raw_english_texts_of_project(project_dirname: str, oracc_site: str = 'oracc.museum.upenn.edu') -> List[Dict]:
raw_jsons = list()
all_paths = glob.glob(f'jsons_unzipped/{project_dirname}/**/corpusjson/*.json', recursive=True)
all_paths = glob.glob(f'jsons_unzipped/{project_dirname}/**/corpusjson/*.json', recursive=True) + glob.glob(
f'jsons_unzipped/{project_dirname}/corpusjson/*.json', recursive=True)
# path = Path(os.path.join(JSONS_DIR, project_dirname, 'catalogue.json'))
# if not os.path.isfile(path):
# return raw_jsons
@@ -43,8 +56,11 @@ def get_raw_english_texts_of_project(project_dirname: str, oracc_site: str = 'or
# for member in d.get('members').values():
for filename in all_paths:
cur_json = _load_json_from_path(filename)
project_name = cur_json['project']
try:
project_name = cur_json['project']
except TypeError:
logging.error(f"Error in {filename}")
continue
# # Skip in case we are in saa project and the current sub project is not in neo-assyrian
# if project_dirname == "saao" and project_name[-2:] not in SUB_PROJECTS_IN_NEO_ASS: # TODO: validate
# continue
@@ -55,12 +71,7 @@ def get_raw_english_texts_of_project(project_dirname: str, oracc_site: str = 'or
# print(url)
logging.info(url)
try:
res = requests.get(url)
soup = BeautifulSoup(res.text, "html.parser")
results = soup.find_all("span", {"class": "cell"})
raw_text = " ".join(["".join([content if isinstance(content, str) else content.text
for content in result.contents]) for result in results])
raw_text = raw_text.replace('\n', ' ')
raw_text = _clean_raw_text(_download_data_from_website(url))
if raw_text:
raw_jsons.append({
"id_text": cur_json['textid'],