From 1236d91d9a18ae910afa0092f501d711e30d1c6e Mon Sep 17 00:00:00 2001 From: 1kamma Date: Wed, 12 Apr 2023 08:24:03 +0300 Subject: [PATCH] update exceptions in projects --- project_notebook.ipynb | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/project_notebook.ipynb b/project_notebook.ipynb index 62281c3..55fbda1 100644 --- a/project_notebook.ipynb +++ b/project_notebook.ipynb @@ -51,7 +51,10 @@ "with open('project_list') as f:\n", " for project in f.read().split('\\n'):\n", " # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n", - " scrap = scrapping.get_raw_english_texts_of_project(project)\n", + " try:\n", + " scrap = scrapping.get_raw_english_texts_of_project(project)\n", + " except Exception:\n", + " logging.error(f\"Error in {project}:{Exception}\")\n", " try:\n", " execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n", " conn.commit()\n", @@ -76,17 +79,25 @@ "with open('project_list') as f:\n", " project = f.read().split('\\n')[2]\n", " scrap = scrapping.get_raw_english_texts_of_project(project)\n", - " execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n", + " execute_batch(\n", + " connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n", + " scrap)\n", " conn.commit()\n" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ - "\n" + "with open('cams') as file:\n", + " conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n", + " connection = conn.cursor()\n", + " execute_batch(\n", + " connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n", + " json.load(file))\n", + " conn.commit()\n" ] }, {