updates in oracc link; reading project that failed

doc

@@ -0,0 +1 @@
+[]

epsd2

@@ -0,0 +1 @@
+[]

lovelyrics

@@ -0,0 +1 @@
+[]

neo

@@ -0,0 +1 @@
+[]

nimrud

@@ -0,0 +1 @@
+[]

obel

@@ -0,0 +1 @@
+[{"id_text": "P345452", "project_name": "obel", "raw_text": "Pure barge of the heavens, you are authoritative all on your own. Father Nanna, lord of Ur. Father Nanna, lord of the Eki\u0161nu\u014bal. Father Nanna, lord Dilimbabbar. Lord Nanna, foremost son of Enlil. When you float, when you float, When you appear authoritatively before your father, before Enlil, Father Nanna, when you appear authoritatively, when you raise your chest, When you appear authoritatively in your barge which is floating through the midst of heavens, Father Nanna, you, when you ride to your pure shrine, Father Nanna, when you float like a boat in a flood wave, When you float, when you float, you, when you float, When you float, when you pour out beer, you, when you float, When you pour out beer in a joyful mood, you, when you float, Father Nanna, when you tend to the ur cows and \u0161ar cows, Your father (Enlil) looks upon you with joyful eyes, and tends to you truly. Behold, he shines forth for the king; Enlil entrusted the sceptre of a lengthy reign to your hands. When you take care of lord Nudimmud, ... Having filled water into the .. canal ... Having filled water into the .. canal ... Having filled water into the Tigris, it is Nanna's. Having filled water into the Euphrates, it is Nanna's. Having filled water into canal and ditch for purification, they are Nanna's. Having filled the great marsh and the small marsh with water, they are Nanna's. An er\u0161ema song of Suen."}, {"id_text": "P355693", "project_name": "obel", "raw_text": "Oh my brother! ... Oh my brother! ... Oh my brother, son of Ga\u0161anmah! I lament for my brother, I lament, I lament in every way. I lament, the song of youthfulness  I lament, in crying for the ... man She makes the woes plentyful, she makes the woes plentiful, standing up she makes the woes plentiful, young man, your mother makes the woes plentiful, your mother, Ninhursa\u014b, makes the woes plentiful. Our Princess in the Emah, the princess makes the woes plentiful. Atutur, the minister with hair hanging down makes the woes plentiful. My brother, you mother makes the woes plentiful. The palace of Ke\u0161 makes the woes plentiful. The brickwork of Iri\u0161ar makes the woes plentiful. The Emah of Adab makes the woes plentiful. The brickwork of Adab makes the woes plentiful. 'Where shall my son be handed over?' she is saying. 'Where shall my son, the Foolish One, be handed over?' she is saying. 'Where shall my son, the one I love, be handed over?' she is saying. The spouse calls out to her man. My brother, rise from your bed, may your mother rejoice over you. Your mother, Ga\u0161anhursa\u014b, may your mother rejoice over you. The en-priest, the lord, the great ruler of Adab may he rejoice over you. A\u0161irgi, the lord of Ke\u0161, may he rejoice over you. Atutur, in mourning, may she rejoice over you. Damgalnuna, of the Ema\u1e2b, the princess, may she rejoice over you. Lisin, the one of liver and heart, may she rejoice over you. ... ... ... ... ... Let me hear your sweet lips,  let me hear your sweet voice let (my) heart be close to your good looks. Young man, do not let your mother, sit in tears,  do not let your mother, Ninhursa\u014b, sit moaning, do not let Our Princess, sit (witnessing) your pain,  do not let them do \"ua!\" Rise from your bed! Foolish One, do not let them do \"ua!\" Rise from your bed! The brother replied to his sister: My release, my sister, my release, Our Princess, my release, my sister, my release, Oh sister, do not speak so much, I am not one who can see. Our Princess, do not speak so much, I am not one who can see, My mother, Ga\u0161anmah, do not speak so much, I am not one who can see. In my bed, the dust of the netherworld, the ... lie with me. In my sleep, terror, the enemy sits with me. My sister, when I lie down and when I do not rise, my mother is the one who is anguished(?) over me, may I loosen the silah. Ga\u0161anhursa\u014b is the one who is anguished over me, may I loosen the simlah. My sister, stand up, give me my share, the estate of my father. My father made the woes over me plentiful, that be my share. Let my mother let her hair hang down for me, so that my ribs may lay down. May the bride whom my father (chose for me) measure grain for me, so that I may listen to it. Acquire a bed for me, (and recite) \"Its spirit is blown off.\" Set up the throne, seat the silah. Place the clothes on the throne, cover the simlah. Make funerary offerings, turn, accept them for me. Pour water into the libation pipe, and stir in the dust of the netherworld. Pour out the hot soup, let me drink its radiance. My sister, alas! Where ...? Our Princess ... Tears ... ... ... ... ... ..."}]

obta

@@ -0,0 +1 @@
+[]

ogsl

@@ -0,0 +1 @@
+[]

oimea

@@ -0,0 +1 @@
+[]

pnao

@@ -0,0 +1 @@
+[]

project_list

@@ -20,28 +20,28 @@
 #contrib
 #contrib/amarna
 #contrib/lambert
-ctij
-dcclt
-dccmt
-doc
-dsst
-ecut
-eisl
-epsd2
-etcsri
-glass
-hbtin
-lacost
-lovelyrics
-neo
-nere
-nimrud
-obel
-obmc
-obta
-ogsl
-oimea
-pnao
+#ctij
+#dcclt
+#dccmt
+#doc
+#dsst
+#ecut
+#eisl
+#epsd2
+#etcsri
+#glass
+#hbtin
+#lacost
+#lovelyrics
+#neo
+#nere
+#nimrud
+#obel
+#obmc
+#obta
+#ogsl
+#oimea
+#pnao
 qcat
 riao
 ribo

project_notebook.ipynb

@@ -21,6 +21,7 @@
     "import re\n",
     "import logging\n",
     "import datetime\n",
+    "import os\n",
     "logging.basicConfig(\n",
     "    level=logging.INFO, filename=f'{datetime.datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")}.log', filemode='w',\n",
     "    format='%(name)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')\n",
@@ -29,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 29,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -56,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -68,11 +69,14 @@
     "logging.info('Start')\n",
     "with open('project_list') as f:\n",
     "    # project = f.read().split('\\n')[2]\n",
-    "    scrap = scrapping.get_raw_english_texts_of_project('contrib/amarna')\n",
-    "    execute_batch(\n",
-    "        connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
-    "        scrap)\n",
-    "    conn.commit()\n"
+    "    for project in f.read().split('\\n'):\n",
+    "        # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
+    "        if os.path.exists(project):\n",
+    "            scrap = json.load(project)\n",
+    "            execute_batch(\n",
+    "                connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
+    "                scrap)\n",
+    "            conn.commit()\n"
    ]
   },
   {

qcat

@@ -0,0 +1 @@
+[]

riao

@@ -0,0 +1 @@
+[]

scrapping.py

@@ -32,7 +32,7 @@ def _load_json_from_path(json_path: str) -> Dict:
             return json.load(json_file)
 
 
-def get_raw_english_texts_of_project(project_dirname: str) -> List[Dict]:
+def get_raw_english_texts_of_project(project_dirname: str, oracc_site: str = 'oracc.museum.upenn.edu') -> List[Dict]:
     raw_jsons = list()
     all_paths = glob.glob(f'jsons_unzipped/{project_dirname}/**/corpusjson/*.json', recursive=True)
     # path = Path(os.path.join(JSONS_DIR, project_dirname, 'catalogue.json'))
@@ -51,7 +51,7 @@ def get_raw_english_texts_of_project(project_dirname: str) -> List[Dict]:
 
         # id_text = member.get('id_text', "") + member.get('id_composite', "")
         # html_dir = "/".join(path.parts[1:-1])
-        url = f"http://oracc.iaas.upenn.edu/{project_name}/{cur_json['textid']}/html"
+        url = f"http://{oracc_site}/{project_name}/{cur_json['textid']}/html"
         # print(url)
         logging.info(url)
         try: