saret/DH
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

almost done with scraping

Browse Source
This commit is contained in:
Benny Saret
2023-04-12 12:06:49 +03:00
parent 15c9b56fd0
commit faab62e768
11 changed files with 3176 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
project_notebook.ipynb
View File

@@ -57,7 +57,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -67,31 +67,35 @@
" level=logging.INFO, filename=f'./now.log',\n",
" format='%(name)s - %(levelname)s - %(message)s')\n",
"logging.info('Start')\n",
"with open('project_list') as f:\n",
"with open('project_list', encoding='unicode-escape') as f:\n",
" # project = f.read().split('\\n')[2]\n",
" for project in f.read().split('\\n'):\n",
" # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
" if os.path.exists(project):\n",
" scrap = json.load(project)\n",
" execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n",
" conn.commit()\n"
" with open(project) as f:\n",
" logging.info(f'reading {project}')\n",
" scrap = json.loads(f.read())\n",
" # scrap = json.loads(project)\n",
" execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n",
" conn.commit()\n"
]
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"with open('cams') as file:\n",
" conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
" connection = conn.cursor()\n",
" execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" json.load(file))\n",
" conn.commit()\n"
"# with open('cams') as file:\n",
"conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
"connection = conn.cursor()\n",
"scrap = scrapping.get_raw_english_texts_of_project('saao')\n",
"execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n",
"conn.commit()\n"
]
},
{