saret/DH
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update exceptions in projects

Browse Source
This commit is contained in:
Benny Saret
2023-04-12 08:24:03 +03:00
parent 52f7b9abf0
commit 1236d91d9a
1 changed files with 15 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
project_notebook.ipynb
View File

@@ -51,7 +51,10 @@
"with open('project_list') as f:\n", "with open('project_list') as f:\n",
" for project in f.read().split('\\n'):\n", " for project in f.read().split('\\n'):\n",
" # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n", " # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
" try:\n",
" scrap = scrapping.get_raw_english_texts_of_project(project)\n", " scrap = scrapping.get_raw_english_texts_of_project(project)\n",
" except Exception:\n",
" logging.error(f\"Error in {project}:{Exception}\")\n",
" try:\n", " try:\n",
" execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n", " execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
" conn.commit()\n", " conn.commit()\n",
@@ -76,17 +79,25 @@
"with open('project_list') as f:\n", "with open('project_list') as f:\n",
" project = f.read().split('\\n')[2]\n", " project = f.read().split('\\n')[2]\n",
" scrap = scrapping.get_raw_english_texts_of_project(project)\n", " scrap = scrapping.get_raw_english_texts_of_project(project)\n",
" execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n", " execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" scrap)\n",
" conn.commit()\n" " conn.commit()\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 44, "execution_count": 25,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"\n" "with open('cams') as file:\n",
" conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
" connection = conn.cursor()\n",
" execute_batch(\n",
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
" json.load(file))\n",
" conn.commit()\n"
] ]
}, },
{ {