writing data

project_notebook.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -18,12 +18,18 @@
     "import scrapping\n",
     "import psycopg2\n",
     "from psycopg2.extras import execute_batch\n",
-    "# from psycopg2.extensions import register_adapter"
+    "import re\n",
+    "import logging\n",
+    "import datetime\n",
+    "logging.basicConfig(\n",
+    "    level=logging.INFO, filename=f'{datetime.datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")}.log', filemode='w',\n",
+    "    format='%(name)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')\n",
+    "# from psycopg2.extensions import register_adapter\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -32,9 +38,14 @@
     "with open('project_list') as f:\n",
     "    for project in f.read().split('\\n'):\n",
     "        scrap = scrapping.get_raw_english_texts_of_project(project)\n",
-    "        connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
-    "        execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
-    "        conn.commit()"
+    "        # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
+    "        try:\n",
+    "            execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
+    "            conn.commit()\n",
+    "        except Exception:\n",
+    "            with open(project, 'w') as f:\n",
+    "                f.write(json.dumps(scrap))\n",
+    "            logging.error(f\"Error in {project}:{Exception}\")"
    ]
   },
   {