almost done with scraping
This commit is contained in:
File diff suppressed because it is too large
Load Diff
48
2023-04-12_12-01-00.log
Normal file
48
2023-04-12_12-01-00.log
Normal file
@@ -0,0 +1,48 @@
|
||||
root - INFO - Start
|
||||
root - INFO - Start
|
||||
root - INFO - reading adsd
|
||||
root - INFO - reading aemw
|
||||
root - INFO - reading akklove
|
||||
root - INFO - reading asbp
|
||||
root - INFO - reading atae
|
||||
root - INFO - reading babcity
|
||||
root - INFO - reading blms
|
||||
root - INFO - reading btmao
|
||||
root - INFO - reading btto
|
||||
root - INFO - reading cams
|
||||
root - INFO - reading ccpo
|
||||
root - INFO - reading cdli
|
||||
root - INFO - reading ckst
|
||||
root - INFO - reading cmawro
|
||||
root - INFO - reading contrib
|
||||
root - INFO - reading dcclt
|
||||
root - INFO - reading dccmt
|
||||
root - INFO - reading doc
|
||||
root - INFO - reading dsst
|
||||
root - INFO - reading ecut
|
||||
root - INFO - reading eisl
|
||||
root - INFO - reading epsd2
|
||||
root - INFO - reading etcsri
|
||||
root - INFO - reading glass
|
||||
root - INFO - reading hbtin
|
||||
root - INFO - reading lacost
|
||||
root - INFO - reading lovelyrics
|
||||
root - INFO - reading neo
|
||||
root - INFO - reading nere
|
||||
root - INFO - reading nimrud
|
||||
root - INFO - reading obel
|
||||
root - INFO - reading obmc
|
||||
root - INFO - reading obta
|
||||
root - INFO - reading ogsl
|
||||
root - INFO - reading oimea
|
||||
root - INFO - reading pnao
|
||||
root - INFO - reading qcat
|
||||
root - INFO - reading riao
|
||||
root - INFO - reading ribo
|
||||
root - INFO - reading rimanum
|
||||
root - INFO - reading rinap
|
||||
root - INFO - reading saao
|
||||
root - INFO - reading suhu
|
||||
root - INFO - reading tcma
|
||||
root - INFO - reading tsae
|
||||
root - INFO - reading xcat
|
||||
Binary file not shown.
82
project_list
82
project_list
@@ -1,47 +1,47 @@
|
||||
#adsd
|
||||
#aemw
|
||||
#akklove
|
||||
#amgg####
|
||||
#ario
|
||||
#armep
|
||||
#arrim
|
||||
#asbp
|
||||
#atae
|
||||
#babcity
|
||||
#blms
|
||||
#btmao
|
||||
#btto
|
||||
#cams
|
||||
adsd
|
||||
aemw
|
||||
akklove
|
||||
amgg####
|
||||
ario
|
||||
armep
|
||||
arrim
|
||||
asbp
|
||||
atae
|
||||
babcity
|
||||
blms
|
||||
btmao
|
||||
btto
|
||||
cams
|
||||
#caspo##############
|
||||
#ccpo
|
||||
#cdli
|
||||
#ckst
|
||||
#cmawro
|
||||
#contrib
|
||||
ccpo
|
||||
cdli
|
||||
ckst
|
||||
cmawro
|
||||
contrib
|
||||
#contrib/amarna
|
||||
#contrib/lambert
|
||||
#ctij
|
||||
#dcclt
|
||||
#dccmt
|
||||
#doc
|
||||
#dsst
|
||||
#ecut
|
||||
#eisl
|
||||
#epsd2
|
||||
#etcsri
|
||||
#glass
|
||||
#hbtin
|
||||
#lacost
|
||||
#lovelyrics
|
||||
#neo
|
||||
#nere
|
||||
#nimrud
|
||||
#obel
|
||||
#obmc
|
||||
#obta
|
||||
#ogsl
|
||||
#oimea
|
||||
#pnao
|
||||
ctij
|
||||
dcclt
|
||||
dccmt
|
||||
doc
|
||||
dsst
|
||||
ecut
|
||||
eisl
|
||||
epsd2
|
||||
etcsri
|
||||
glass
|
||||
hbtin
|
||||
lacost
|
||||
lovelyrics
|
||||
neo
|
||||
nere
|
||||
nimrud
|
||||
obel
|
||||
obmc
|
||||
obta
|
||||
ogsl
|
||||
oimea
|
||||
pnao
|
||||
qcat
|
||||
riao
|
||||
ribo
|
||||
|
||||
@@ -57,7 +57,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -67,31 +67,35 @@
|
||||
" level=logging.INFO, filename=f'./now.log',\n",
|
||||
" format='%(name)s - %(levelname)s - %(message)s')\n",
|
||||
"logging.info('Start')\n",
|
||||
"with open('project_list') as f:\n",
|
||||
"with open('project_list', encoding='unicode-escape') as f:\n",
|
||||
" # project = f.read().split('\\n')[2]\n",
|
||||
" for project in f.read().split('\\n'):\n",
|
||||
" # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
|
||||
" if os.path.exists(project):\n",
|
||||
" scrap = json.load(project)\n",
|
||||
" execute_batch(\n",
|
||||
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
|
||||
" scrap)\n",
|
||||
" conn.commit()\n"
|
||||
" with open(project) as f:\n",
|
||||
" logging.info(f'reading {project}')\n",
|
||||
" scrap = json.loads(f.read())\n",
|
||||
" # scrap = json.loads(project)\n",
|
||||
" execute_batch(\n",
|
||||
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
|
||||
" scrap)\n",
|
||||
" conn.commit()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with open('cams') as file:\n",
|
||||
" conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
|
||||
" connection = conn.cursor()\n",
|
||||
" execute_batch(\n",
|
||||
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
|
||||
" json.load(file))\n",
|
||||
" conn.commit()\n"
|
||||
"# with open('cams') as file:\n",
|
||||
"conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
|
||||
"connection = conn.cursor()\n",
|
||||
"scrap = scrapping.get_raw_english_texts_of_project('saao')\n",
|
||||
"execute_batch(\n",
|
||||
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
|
||||
" scrap)\n",
|
||||
"conn.commit()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user