trying to scrape

.gitignore

@@ -0,0 +1 @@
+venv/*

data.jsonl

@@ -0,0 +1 @@
+[{"id_text": "X102611", "project_name": "adsd/adart1", "raw_text": "Month II, (the 1st of which was identical with) the 30th (of the preceding month), [sunset to moonset:] 12\u00b0 [...] Night of the 8th, first part of the night, Mercury was [... above \u03b7 Geminorum ...] Night of the 11th, beginning of the night, the moon was [...] in front of [\u03b1 Virginis ...] The 14th, moonset to sunrise: 8\u00b0 50'. Night of the 1[5th, ...] The 16th, Sirius\u2019 last appearance. Night of the 17th, ... [...] 1 2/3 cubits [...] Capricorni [...] That [month,] the equivalent (for 1 shekel of silver was): ba[rley, ...] [...] ... [...]  [...] ... on the ... [... Mars was ...] in the middle of the month, in Cancer. Th[at month, ...] That month, Paini ... [...] Month VI, (the 1st of which was identical with) the 30th (of the preceding month, sunset to moonset:) 9\u00b0 30'; mist, when I watched [I did not see the moon ... Night of the 5th, beginning of the night, the moon was] 1 cubit 4 fingers behind \u03b1 Scorpii. Night of the 6th, [...] Night of the 9th, beginning of the night,  the moon was [... behind \u03b2 Capricorni ...] The 12th, moonset to sunrise: 11\u00b0. The 13th, 2\u00b0 [... Night of the 15th, last part of the night, the moon was] [...] below \u03b1 Arietis [...] last part of the night,, the moon was [...] in front of [...]  [... year 5]0, Antiochus, the great king [...]"}, {"id_text": "X102612", "project_name": "adsd/adart1", "raw_text": "[...] from within it were killed ... [...] [... of? S]eleucus seized the fields in Babylonia, they surrounded and [...] ... [...] [...] the guard which they had put on this fortress [...] [...] (holes) [Month IX, the 1st (of which followed the 30th of the preceding month), ... Ni]ght of the 3rd, the moon was 1 cubit behind \u03b4 Capricorni. Night of the 7th, beginning of the night, the moon was 2 cubits in front of \u03b7 Piscium. Night of the 8th, beginning of the night, the moon was [...] below \u03b2 Arietis. [... Night of the 10th, beginning of the night, the moon was] 8 fingers [...] \u03b7 Tauri. Night of the 11th, beginning of the night, the moon was 5 cubits above \u03b1 Tauri. Night of the 12th, beginning of the ni[ght, the moon was] 6 fingers [...] \u03b2 Tauri. The 12th, moonset to sunrise: 7\u00b0. Night of the 13th, moonrise to sunset: 9\u00b0 [...] [... The 13th, sunrise] to moonset: [nn] (expected); clouds, I did not watch. Night of the 14th, sunset to moonrise: 3\u00b0 20'; measured (despite) clouds; last part of the night, the moon was 1 2/3 cubits below \u03b2 Geminorum. Ni[ght of the 15th, last part of the] night, the moon was 1 1/2 cubits in front of \u03b4 Cancri. Night of the 16th, last part of the night, the moon was [... below \u03b5 Leonis, the moon] being [nn] fingers back to the west. Night of the 18th, last part of the night, the moon was 1 1/2 cubits in front of \u03d1 Leonis, 1 1/2 cubits below Saturn, the moon being 1/2 cubit back to the west. Around the 18th, when Saturn became stationary to the east, [it became stationary] 1/2 cubit in front of [... Night of the 19th, last part of the night, the moon was 1] cubit 8 fingers [in front of \u03b2 Virginis,] it stood 1 cubit in front of Mars to the west. Night of the 20th, last part of the night, the moon was 1 1/2 cubits in front of \u03b3 Virginis. The 20th, Mercury\u2019s first appearance in the west in Capricorn; it was bright and high, sunset to setting of Mercury: 15\u00b0, (ideal) first appearance on the 19th or 18th. [... The 2]2nd, Venus\u2019 first appearance in the west in Capricorn; sunset to setting of Venus: 10\u00b0, (ideal) first appearance on the 20th. Night of the 23rd, last part of the night, the moon was [nn] cubits below \u03b1 Librae, it stood 2 1/2 cubits behind Jupiter to the east; around [...] [... Night] of the 25th, last part of the night, the moon was 1 cubit 8 fingers behind \u03b1 Scorpii. Night of the 26th, last part of the night, the moon was [...] behind \u03d1 Ophiuchi [...] visible [... The 2]8th, solar eclipse [...] [... d]ates, 4 p\u0101nu 1 s\u016btu; mustard, 1 kur 2 p\u0101nu 3 s\u016btu; cress, 5 s\u016btu 3 qa; sesame, 1 p\u0101nu 1 s\u016btu; wool, [...] ... [...] [... the 20th, Mercury\u2019s] first appearance [in the west in Capri]corn; Saturn was in Leo; Ma[rs ...]  [...] ... [...] [...] sunset to moonrise: [x]+2\u00b0 30' [...] [...] clouds crossed the sky [...] [...] 9\u00b0 ... [...] [... around the 23rd, when Mars became stationary to the west, it became stationary ... behind \u03c1 Le]onis, 1 cubit behind Sa[turn ...] [...] Mars was above ... [...] [... min]as. At that time, Jupi[ter ...] [...] ... [...] [...] ... [...] (holes)"}, {"id_text": "X102613", "project_name": "adsd/adart1", "raw_text": "[...] last part of the night, Jupiter was 2 fingers above Venus [... Ni]ght of the 2nd, the moon was 1/2 cubit in front of \u03b1 Scorpii [...] [...] ... The 3rd, [equinox? ... Ni]ght of the 6th, beginning of the night, the moon was 2 1/2 cubits Line truncated

project_list

@@ -0,0 +1,54 @@
+adsd
+aemw
+akklove
+amgg
+ario
+armep
+arrim
+asbp
+atae
+babcity
+blms
+btmao
+btto
+cams
+caspo
+ccpo
+cdli
+ckst
+cmawro
+contrib
+contrib/amarna
+contrib/lambert
+ctij
+dcclt
+dccmt
+doc
+dsst
+ecut
+eisl
+epsd2
+etcsri
+glass
+hbtin
+lacost
+lovelyrics
+neo
+nere
+nimrud
+obel
+obmc
+obta
+ogsl
+oimea
+pnao
+qcat
+riao
+ribo
+rimanum
+rinap
+saao
+suhu
+tcma
+tsae
+xcat

project_notebook.ipynb

@@ -0,0 +1,70 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Imported modules"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import scrapping\n",
+    "import psycopg2\n",
+    "from psycopg2.extras import execute_batch\n",
+    "# from psycopg2.extensions import register_adapter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
+    "connection = conn.cursor()\n",
+    "with open('project_list') as f:\n",
+    "    for project in f.read().split('\\n'):\n",
+    "        scrap = scrapping.get_raw_english_texts_of_project(project)\n",
+    "        connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
+    "        execute_batch(connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
+    "        conn.commit()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scrape.py

@@ -0,0 +1,9 @@
+import urllib3
+import bs4
+import psycopg2
+
+# def connect_postgres():
+#     conn = psycopg2.connect("dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'")
+#     return conn
+
+def 

scrapping.py

@@ -0,0 +1,227 @@
+import json
+from typing import Dict, List
+import requests
+from bs4 import BeautifulSoup
+import os
+from pathlib import Path
+import re
+import glob
+import logging
+# parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# os.chdir(parent_dir)
+logging.basicConfig(filename='scrape.log', level=logging.INFO)
+JSONS_DIR = "jsons_unzipped"
+
+SUB_PROJECTS_IN_NEO_ASS = ["01", "05", "06", "07", "09", "11", "12", "14", "15", "16"]
+
+CORPUS_DIRNAME = "corpusjson"
+
+
+def _load_json_from_path(json_path: str) -> Dict:
+    """
+    This helper function loads a json from a given path, with the exception of empty files.
+
+    :param json_path: A given string representing a path to a json file.
+    :return: The json file (if it is a valid json file).
+    """
+    with open(json_path, "r", encoding='utf-8') as json_file:
+        if os.stat(json_path).st_size != 0:  # If the file is not empty:
+            return json.load(json_file)
+
+
+def get_raw_english_texts_of_project(project_dirname: str) -> List[Dict]:
+    raw_jsons = list()
+    all_paths = glob.glob(f'jsons_unzipped/{project_dirname}/**/corpusjson/*.json', recursive=True)
+    # path = Path(os.path.join(JSONS_DIR, project_dirname, 'catalogue.json'))
+    # if not os.path.isfile(path):
+    #     return raw_jsons
+    # d = _load_json_to_dict(str(path))
+    # if d and d.get('members'):
+    # for member in d.get('members').values():
+    for filename in all_paths:
+        cur_json = _load_json_from_path(filename)
+        project_name = cur_json['project']
+
+        # # Skip in case we are in saa project and the current sub project is not in neo-assyrian
+        # if project_dirname == "saao" and project_name[-2:] not in SUB_PROJECTS_IN_NEO_ASS:  # TODO: validate
+        #     continue
+
+        # id_text = member.get('id_text', "") + member.get('id_composite', "")
+        # html_dir = "/".join(path.parts[1:-1])
+        url = f"http://oracc.iaas.upenn.edu/{project_name}/{cur_json['textid']}/html"
+        # print(url)
+        logging.info(url)
+        try:
+            res = requests.get(url)
+            soup = BeautifulSoup(res.text, "html.parser")
+            results = soup.find_all("span", {"class": "cell"})
+            raw_text = " ".join(["".join([content if isinstance(content, str) else content.text
+                                        for content in result.contents]) for result in results])
+            raw_text = raw_text.replace('\n', ' ')
+            if raw_text:
+                raw_jsons.append({
+                    "id_text": cur_json['textid'],
+                    "project_name": project_name,
+                    "raw_text": raw_text,
+                })
+        except Exception as e:
+            logging.error(e)
+    return raw_jsons
+
+
+def num_words_in_english(jsonl_file):
+    words_counter = 0
+    with open(jsonl_file, "r", encoding='utf-8') as f_in:
+        for line in f_in:
+            cur_json = json.loads(line)
+            if cur_json["project_name"].startswith("saa"):
+                cur_json["raw_text"] = re.sub(r'\([^)]*\)', '', cur_json["raw_text"])
+                words_counter += len(cur_json["raw_text"].split())
+    print(words_counter)
+
+
+def get_raw_text_akk_from_html(id_text, project_name):
+    # else:  # If the file doesn't exist in the jsons -> look for it online
+    url = f'http://oracc.iaas.upenn.edu/{project_name}/{id_text}/html'
+    res = requests.get(url)
+    if res.status_code != 200:
+        print("******STATUS CODE IS NOT 200***********")
+        return ""
+    soup = BeautifulSoup(res.text, "html.parser")
+    # print(f"Check this out:\n{url}")
+    raw_text = _get_raw_text_html(soup)
+    # cur_json = {"id_text": id_text, "project_name": project_name, "raw_text": raw_text}
+    return raw_text
+
+
+def _get_raw_text_html1(soup):
+    words = soup.find_all("a", class_="cbd")
+    return ' '.join(tag.text for tag in soup.find_all('p', class_='tt'))
+    return ' '.join([word.text for word in words])
+
+
+def _get_raw_text_html(soup):
+    tags = soup.find_all('span', class_=lambda value: value and value.startswith('w '))
+    signs = list()
+    for tag in tags:
+        temp_tag = tag.find('a')
+        if temp_tag:
+            tag = temp_tag
+        for sign in tag.contents:
+            if isinstance(sign, str):
+                signs.append(sign)
+            elif sign.name == 'span':
+                signs.append(sign.text)
+            elif sign.name == 'sup':
+                signs.append("{" + sign.text + "}")
+    return ' '.join(signs)
+
+
+def get_raw_akk_texts_of_project(project_dirname: str) -> List[Dict]:
+    """
+    This function parses the raw texts of a project in ORACC.
+
+    :param project_dirname: A given string representing the path to the project's directory.
+    :return: A list of jsons containing the raw texts of the given project and basic metadata.
+    """
+    raw_jsons = list()
+    all_paths = glob.glob(f'jsons_unzipped/{project_dirname}/**/corpusjson/*.json', recursive=True)
+
+    for filename in all_paths:
+        cur_json = _load_json_from_path(filename)
+
+        try:
+            project_name = cur_json['project']
+            sents_dicts = cur_json['cdl'][0]['cdl'][-1]['cdl']
+        except Exception as e:
+            print(f"In file {filename} failed because of {e}")
+            continue
+
+        raw_text = get_raw_akk_text_from_json(sents_dicts)
+        raw_jsons.append({
+            "id_text": cur_json['textid'],
+            "project_name": project_name,
+            "raw_text": raw_text,
+        })
+
+    # if not texts_jsons or not texts_jsons.get('members'):
+    #     return raw_jsons
+    # for member in texts_jsons.get('members').values():  # Iterate over different tablets:
+    #     project_name = member['project'].split("/")[-1]
+    #
+    #     # Skip in case we are in saa project and the current sub project is not in neo-assyrian
+    #     if project_dirname == "saao" and project_name[-2:] not in SUB_PROJECTS_IN_NEO_ASS:  # TODO: validate
+    #         continue
+    #
+    #     id_text = member.get("id_text", "") + member.get("id_composite", "")
+    #     json_file_path = os.path.join(JSONS_DIR, Path(member['project']), CORPUS_DIRNAME, f'{id_text}.json')
+    #
+    #     if os.path.isfile(json_file_path):  # If file exists in the jsons
+    #         d = _load_json_to_dict(json_file_path)
+    #
+    #         try:
+    #             sents_dicts = d['cdl'][0]['cdl'][-1]['cdl']
+    #         except Exception as e:
+    #             print(f"In file {json_file_path} failed because of {e}")
+    #             continue
+    #
+    #         raw_text = " ".join([_get_raw_text(sent_dict['cdl']) for sent_dict in sents_dicts if _is_sent(sent_dict)])
+    #         cur_json = {
+    #             "id_text": id_text,
+    #             "project_name": project_name,
+    #             "raw_text": raw_text,
+    #         }
+    #         raw_jsons.append(cur_json)
+
+    return raw_jsons
+
+
+def get_raw_akk_text_from_json(sents_dicts):
+    return " ".join([_get_raw_text(sent_dict['cdl']) for sent_dict in sents_dicts if _is_sent(sent_dict)])
+
+
+def _get_raw_text(json_dict: dict) -> str:
+    """
+    This function gets the raw text of a given transliterated tablet in ORACC recursively.
+    It appends each instance in the tablet only once (even if there are multiple possible meanings).
+
+    :param json_dict: A given dictionary representing some portion of the words in the tablet.
+    :return: The aforementioned raw text.
+    """
+    previous_ref: str = ""
+    raw_texts = list()
+    for d in json_dict:
+        if _is_sent(d):  # If node represents a sentence -> call recursively to the inner dictionary
+            raw_texts.extend(_get_raw_text(d['cdl']).split())
+        elif _is_word(d):  # If node represents a word
+            if previous_ref != d.get('ref'):  # If encountered new instance:
+                cur_text = d['frag'] if d.get('frag') else d['f']['form']
+                raw_texts.append(cur_text + _get_addition(d))
+                previous_ref = d.get('ref')
+
+    return " ".join(raw_texts)
+
+
+def _is_sent(d: Dict) -> bool:
+    return d.get('node') == 'c'
+
+
+def _is_word(d: Dict) -> bool:
+    return d.get('node') == 'l'
+
+
+def _get_addition(d: Dict) -> str:
+    """
+    This function looks for an asterisk or a question mark in a dictionary representing a word in a tablet from ORACC.
+
+    :param d: A given dictionary as described above.
+    :return An asterisk or a question mark if one of the word's signs has one, otherwise an empty string.
+    """
+    has_signs_dicts = 'f' in d and 'gdl' in d.get('f')
+    if has_signs_dicts:
+        for sign_dict in d['f']['gdl']:
+            if 'gdl_collated' in sign_dict:  # If cur sign has an asterisk
+                return "*"
+            if 'queried' in sign_dict:  # If cur sign has a question mark
+                return "?"
+    return ""