saret/DH
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

almost done with scraping

Browse Source
This commit is contained in:
Benny Saret
2023-04-12 12:06:49 +03:00
parent 15c9b56fd0
commit faab62e768
11 changed files with 3176 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3062
2023-04-12_01-43-06.log
View File

File diff suppressed because it is too large Load Diff

48
2023-04-12_12-01-00.log Normal file
View File

@@ -0,0 +1,48 @@
root - INFO - Start
root - INFO - Start
root - INFO - reading adsd
root - INFO - reading aemw
root - INFO - reading akklove
root - INFO - reading asbp
root - INFO - reading atae
root - INFO - reading babcity
root - INFO - reading blms
root - INFO - reading btmao
root - INFO - reading btto
root - INFO - reading cams
root - INFO - reading ccpo
root - INFO - reading cdli
root - INFO - reading ckst
root - INFO - reading cmawro
root - INFO - reading contrib
root - INFO - reading dcclt
root - INFO - reading dccmt
root - INFO - reading doc
root - INFO - reading dsst
root - INFO - reading ecut
root - INFO - reading eisl
root - INFO - reading epsd2
root - INFO - reading etcsri
root - INFO - reading glass
root - INFO - reading hbtin
root - INFO - reading lacost
root - INFO - reading lovelyrics
root - INFO - reading neo
root - INFO - reading nere
root - INFO - reading nimrud
root - INFO - reading obel
root - INFO - reading obmc
root - INFO - reading obta
root - INFO - reading ogsl
root - INFO - reading oimea
root - INFO - reading pnao
root - INFO - reading qcat
root - INFO - reading riao
root - INFO - reading ribo
root - INFO - reading rimanum
root - INFO - reading rinap
root - INFO - reading saao
root - INFO - reading suhu
root - INFO - reading tcma
root - INFO - reading tsae
root - INFO - reading xcat

BIN
__pycache__/scrapping.cpython-39.pyc
View File

Binary file not shown.

82
project_list
View File

@@ -1,47 +1,47 @@
#adsd adsd
#aemw aemw
#akklove akklove
#amgg#### amgg####
#ario ario
#armep armep
#arrim arrim
#asbp asbp
#atae atae
#babcity babcity
#blms blms
#btmao btmao
#btto btto
#cams cams
#caspo############## #caspo##############
#ccpo ccpo
#cdli cdli
#ckst ckst
#cmawro cmawro
#contrib contrib
#contrib/amarna #contrib/amarna
#contrib/lambert #contrib/lambert
#ctij ctij
#dcclt dcclt
#dccmt dccmt
#doc doc
#dsst dsst
#ecut ecut
#eisl eisl
#epsd2 epsd2
#etcsri etcsri
#glass glass
#hbtin hbtin
#lacost lacost
#lovelyrics lovelyrics
#neo neo
#nere nere
#nimrud nimrud
#obel obel
#obmc obmc
#obta obta
#ogsl ogsl
#oimea oimea
#pnao pnao
qcat qcat
riao riao
ribo ribo

24
project_notebook.ipynb
View File

@@ -57,7 +57,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -67,12 +67,15 @@
" level=logging.INFO, filename=f'./now.log',\n", " level=logging.INFO, filename=f'./now.log',\n",
" format='%(name)s - %(levelname)s - %(message)s')\n", " format='%(name)s - %(levelname)s - %(message)s')\n",
"logging.info('Start')\n", "logging.info('Start')\n",
"with open('project_list') as f:\n", "with open('project_list', encoding='unicode-escape') as f:\n",
" # project = f.read().split('\\n')[2]\n", " # project = f.read().split('\\n')[2]\n",
" for project in f.read().split('\\n'):\n", " for project in f.read().split('\\n'):\n",
" # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n", " # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
" if os.path.exists(project):\n", " if os.path.exists(project):\n",
" scrap = json.load(project)\n", " with open(project) as f:\n",
" logging.info(f'reading {project}')\n",
" scrap = json.loads(f.read())\n",
" # scrap = json.loads(project)\n",
" execute_batch(\n", " execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n", " connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n", " scrap)\n",
@@ -81,17 +84,18 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 25, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"with open('cams') as file:\n", "# with open('cams') as file:\n",
" conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n", "conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
" connection = conn.cursor()\n", "connection = conn.cursor()\n",
" execute_batch(\n", "scrap = scrapping.get_raw_english_texts_of_project('saao')\n",
"execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n", " connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" json.load(file))\n", " scrap)\n",
" conn.commit()\n" "conn.commit()\n"
] ]
}, },
{ {

1
rinap Normal file
View File

File diff suppressed because one or more lines are too long

1
saao Normal file
View File

File diff suppressed because one or more lines are too long

1
suhu Normal file
View File

File diff suppressed because one or more lines are too long

1
tcma Normal file
View File

@@ -0,0 +1 @@
[]

1
tsae Normal file
View File

@@ -0,0 +1 @@
[]

1
xcat Normal file
View File

@@ -0,0 +1 @@
[]