almost done with scraping
This commit is contained in:
saret committed
1 parent
15c9b56fd0
commit
faab62e768
11 files changed
+3167
-47
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,48 @@
|
||||
root - INFO - Start
|
||||
root - INFO - Start
|
||||
root - INFO - reading adsd
|
||||
root - INFO - reading aemw
|
||||
root - INFO - reading akklove
|
||||
root - INFO - reading asbp
|
||||
root - INFO - reading atae
|
||||
root - INFO - reading babcity
|
||||
root - INFO - reading blms
|
||||
root - INFO - reading btmao
|
||||
root - INFO - reading btto
|
||||
root - INFO - reading cams
|
||||
root - INFO - reading ccpo
|
||||
root - INFO - reading cdli
|
||||
root - INFO - reading ckst
|
||||
root - INFO - reading cmawro
|
||||
root - INFO - reading contrib
|
||||
root - INFO - reading dcclt
|
||||
root - INFO - reading dccmt
|
||||
root - INFO - reading doc
|
||||
root - INFO - reading dsst
|
||||
root - INFO - reading ecut
|
||||
root - INFO - reading eisl
|
||||
root - INFO - reading epsd2
|
||||
root - INFO - reading etcsri
|
||||
root - INFO - reading glass
|
||||
root - INFO - reading hbtin
|
||||
root - INFO - reading lacost
|
||||
root - INFO - reading lovelyrics
|
||||
root - INFO - reading neo
|
||||
root - INFO - reading nere
|
||||
root - INFO - reading nimrud
|
||||
root - INFO - reading obel
|
||||
root - INFO - reading obmc
|
||||
root - INFO - reading obta
|
||||
root - INFO - reading ogsl
|
||||
root - INFO - reading oimea
|
||||
root - INFO - reading pnao
|
||||
root - INFO - reading qcat
|
||||
root - INFO - reading riao
|
||||
root - INFO - reading ribo
|
||||
root - INFO - reading rimanum
|
||||
root - INFO - reading rinap
|
||||
root - INFO - reading saao
|
||||
root - INFO - reading suhu
|
||||
root - INFO - reading tcma
|
||||
root - INFO - reading tsae
|
||||
root - INFO - reading xcat
|
||||
Binary file not shown.
+41
-41
@@ -1,47 +1,47 @@
|
||||
#adsd
|
||||
#aemw
|
||||
#akklove
|
||||
#amgg####
|
||||
#ario
|
||||
#armep
|
||||
#arrim
|
||||
#asbp
|
||||
#atae
|
||||
#babcity
|
||||
#blms
|
||||
#btmao
|
||||
#btto
|
||||
#cams
|
||||
adsd
|
||||
aemw
|
||||
akklove
|
||||
amgg####
|
||||
ario
|
||||
armep
|
||||
arrim
|
||||
asbp
|
||||
atae
|
||||
babcity
|
||||
blms
|
||||
btmao
|
||||
btto
|
||||
cams
|
||||
#caspo##############
|
||||
#ccpo
|
||||
#cdli
|
||||
#ckst
|
||||
#cmawro
|
||||
#contrib
|
||||
ccpo
|
||||
cdli
|
||||
ckst
|
||||
cmawro
|
||||
contrib
|
||||
#contrib/amarna
|
||||
#contrib/lambert
|
||||
#ctij
|
||||
#dcclt
|
||||
#dccmt
|
||||
#doc
|
||||
#dsst
|
||||
#ecut
|
||||
#eisl
|
||||
#epsd2
|
||||
#etcsri
|
||||
#glass
|
||||
#hbtin
|
||||
#lacost
|
||||
#lovelyrics
|
||||
#neo
|
||||
#nere
|
||||
#nimrud
|
||||
#obel
|
||||
#obmc
|
||||
#obta
|
||||
#ogsl
|
||||
#oimea
|
||||
#pnao
|
||||
ctij
|
||||
dcclt
|
||||
dccmt
|
||||
doc
|
||||
dsst
|
||||
ecut
|
||||
eisl
|
||||
epsd2
|
||||
etcsri
|
||||
glass
|
||||
hbtin
|
||||
lacost
|
||||
lovelyrics
|
||||
neo
|
||||
nere
|
||||
nimrud
|
||||
obel
|
||||
obmc
|
||||
obta
|
||||
ogsl
|
||||
oimea
|
||||
pnao
|
||||
qcat
|
||||
riao
|
||||
ribo
|
||||
|
||||
+10
-6
@@ -57,7 +57,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -67,12 +67,15 @@
|
||||
" level=logging.INFO, filename=f'./now.log',\n",
|
||||
" format='%(name)s - %(levelname)s - %(message)s')\n",
|
||||
"logging.info('Start')\n",
|
||||
"with open('project_list') as f:\n",
|
||||
"with open('project_list', encoding='unicode-escape') as f:\n",
|
||||
" # project = f.read().split('\\n')[2]\n",
|
||||
" for project in f.read().split('\\n'):\n",
|
||||
" # connection = connection.execute(\"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s)\", scrap)\n",
|
||||
" if os.path.exists(project):\n",
|
||||
" scrap = json.load(project)\n",
|
||||
" with open(project) as f:\n",
|
||||
" logging.info(f'reading {project}')\n",
|
||||
" scrap = json.loads(f.read())\n",
|
||||
" # scrap = json.loads(project)\n",
|
||||
" execute_batch(\n",
|
||||
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
|
||||
" scrap)\n",
|
||||
@@ -81,16 +84,17 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with open('cams') as file:\n",
|
||||
"# with open('cams') as file:\n",
|
||||
"conn = psycopg2.connect(\"dbname='dh' user='dh' host='dh.saret.tk' password='qwerty'\")\n",
|
||||
"connection = conn.cursor()\n",
|
||||
"scrap = scrapping.get_raw_english_texts_of_project('saao')\n",
|
||||
"execute_batch(\n",
|
||||
" connection, \"insert into raw_texts values (%(id_text)s, %(project_name)s, %(raw_text)s) ON CONFLICT DO NOTHING\",\n",
|
||||
" json.load(file))\n",
|
||||
" scrap)\n",
|
||||
"conn.commit()\n"
|
||||
]
|
||||
},
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
[{"id_text": "Q003414", "project_name": "rinap/rinap1", "raw_text": "Precious scion of Baltil (A\u0161\u0161ur), beloved of the god(dess) [(DN and) \u0160\u0113]r\u016ba, ..., creation of the goddess Ninmena, who [(...)] ... for the dominion of the lands, (...) who grew up to be king, ... [(...)] governor, [(...)] ..., the one who increases voluntary offerings for ..., ... [(...)] of emblems, (5) powerful male, light of all of his people, lord of [(...) all] rulers ..., the one who overwhelms his foes, valiant man, the one who destroys [(...)] enemies, who cuts (straight) through interlocking mountains like a (taut) string and ... [...] ..."}, {"id_text": "Q003415", "project_name": "rinap/rinap1", "raw_text": "warrio[r ... who made ... bow down at his] feet [..., who] pu[t ... to the sword (lit. \u201cweapon\u201d), ...] circumsp[ect ...], (No translation possible)"}, {"id_text": "Q003416", "project_name": "rinap/rinap1", "raw_text": "[... he] made [...] kiss [his feet ... mo]untains [... in/of] battle [... he (a god) made my weapon/rule greater than all of those/the kings who] sit on (royal) daises, (5) [... ci]rcumspect [..., ... exalted lio]n-dragon, [...] inhabited world."}, {"id_text": "Q003417", "project_name": "rinap/rinap1", "raw_text": "At the beginning of my reign, in my first pal\u00fb, in the fifth month after I sat in greatness on the throne of kingship, (the god) A\u0161\u0161ur, my lord, encouraged me and [I marched] against (the Aramean tribes) \u1e2aamar\u0101nu (\u1e2aamr\u0101nu), Lu\u1e2bu\u02be\u0101tu, \u1e2aatallu, Rubb\u00fb, Rapiqu, \u1e2a\u012br\u0101nu, (5) Rabbi-ilu, Na\u1e63\u012bru, Gul\u016bsu, Nab\u0101tu, Li\u02beta\u02beu, Ra\u1e2b\u012bqu, Kap\u012bri, Rummul\u012btu (Rummul\u016btu), Adil\u00ea, Gibr\u00ea, Ub\u016bdu, Gur\u016bmu, "}, {"id_text": "Q003418", "project_name": "rinap/rinap1", "raw_text": "[... I] adorned them (statues of the gods) and they (the gods) went (back) to their land. I rebuilt th[ose] cities. I built a city on top of a tell (lit. \u201ca heaped-up ruin mound\u201d) called [\u1e2aumut]. I b[uilt (and) co]mpleted (it) from its foundations to its parapets. [Inside (it), I founded] a palace for my royal residence. I named it K\u0101r-A\u0161\u0161ur, set up the weapon of (the god) A\u0161\u0161ur, my lord, therein, (and) settled the people [of (foreign) lands] conquered by me therein. [I] imposed upon them [tax (and) tribute], (and) considered them as inhabitants of Assyria. I dug out the Patti-[Enlil] canal, [which] had lain abandoned for a very long time and [...], and I made an abundance of water gurgle through it. I exercised authority over [... from] the cities D\u016br-(Kuri)galzu, Sippar of the god \u0160ama\u0161, [..., the (tribes) Na]sikku, Naq(i)ru, (and) Tan\u00ea, the city Kala\u02bein, the \u0160umandar (Sumandar) canal, [the city Pa]\u1e63\u012btu of the (tribe) Dun\u0101nu, the land Qirbutu (Qirbit), [... the (tribe) Adi]l\u00ea, the land B\u016bdu, the city Pa\u1e2b\u1e2baz, the land Qin-Nippur, (and) the cities [of Kar]dunia\u0161 (Babylonia) as far as the Uqn\u00fb River, [which are on the shore of the Lo]wer [Sea]. I anne[xed] (those areas) to Assyria (and) placed a eunuch of mine as [provincial governor over them]. From their sheep levy, [which] I take [annually], I apportion[ed] 240 sheep as a gift to (the god) A\u0161\u0161ur, my lord. [From] those [Ara]means whom I deported, (10) [I distribut]ed (and) settled [... thousand to the province of] the turt\u0101nu, 10,000 (to) the province of the palace herald, [...] thousand (to) the province of the chief cupbearer, [... thousand (to) the province of the land] Bar\u1e2balzu, (and) 5,000 (to) the province of the land Mazamua (Zamua). I united them, [considered them] as inhabitants of [Assyria, (and) imposed] the yoke of (the god) A\u0161\u0161ur, my lord, [upon them] as Assyrians. (As for) the abandoned settlements on the periphery of my [land] that had bec[ome] desolated [during the reign(s) of (previous) kings], my [ancestor]s,"}, {"id_text": "Q003419", "project_name": "rinap/rinap1", "raw_text": "I restored and [...] Assyria to [...]. I built a city [... Inside (it), I founded] a palace for [my] royal residence. [...]. I named it [..., set up] the weapon of (the god) A\u0161\u0161ur, my lord, th[erein, (and) settled the people of (foreign) lands conquered by me therein. I imposed upon them tax (and) tribute], (and) considered them as inhabitants of Assyria. [I fashioned (on a stele) my royal] image [and the image(s) of the great gods, my lords, inscribed on it the mighty victories (lit. \u201cvictory and might\u201d)] (5) that I had [achieved again and again] over (foreign) lands in the name of (the god) A\u0161\u0161ur, my lord, (and) [I erected (it) in ...]. I received as his payment [...], 10 talents of gold (measured) by the heavy (standard), 1,000 talents of [silver, ...]. In my second pal\u00fb, (the god) A\u0161\u0161ur, [my] lor[d, encouraged me and I marched againLine truncated
|
||||
@@ -0,0 +1 @@
|
||||
[{"id_text": "Q003414", "project_name": "rinap/rinap1", "raw_text": "Precious scion of Baltil (A\u0161\u0161ur), beloved of the god(dess) [(DN and) \u0160\u0113]r\u016ba, ..., creation of the goddess Ninmena, who [(...)] ... for the dominion of the lands, (...) who grew up to be king, ... [(...)] governor, [(...)] ..., the one who increases voluntary offerings for ..., ... [(...)] of emblems, (5) powerful male, light of all of his people, lord of [(...) all] rulers ..., the one who overwhelms his foes, valiant man, the one who destroys [(...)] enemies, who cuts (straight) through interlocking mountains like a (taut) string and ... [...] ..."}, {"id_text": "Q003415", "project_name": "rinap/rinap1", "raw_text": "warrio[r ... who made ... bow down at his] feet [..., who] pu[t ... to the sword (lit. \u201cweapon\u201d), ...] circumsp[ect ...], (No translation possible)"}, {"id_text": "Q003416", "project_name": "rinap/rinap1", "raw_text": "[... he] made [...] kiss [his feet ... mo]untains [... in/of] battle [... he (a god) made my weapon/rule greater than all of those/the kings who] sit on (royal) daises, (5) [... ci]rcumspect [..., ... exalted lio]n-dragon, [...] inhabited world."}, {"id_text": "Q003417", "project_name": "rinap/rinap1", "raw_text": "At the beginning of my reign, in my first pal\u00fb, in the fifth month after I sat in greatness on the throne of kingship, (the god) A\u0161\u0161ur, my lord, encouraged me and [I marched] against (the Aramean tribes) \u1e2aamar\u0101nu (\u1e2aamr\u0101nu), Lu\u1e2bu\u02be\u0101tu, \u1e2aatallu, Rubb\u00fb, Rapiqu, \u1e2a\u012br\u0101nu, (5) Rabbi-ilu, Na\u1e63\u012bru, Gul\u016bsu, Nab\u0101tu, Li\u02beta\u02beu, Ra\u1e2b\u012bqu, Kap\u012bri, Rummul\u012btu (Rummul\u016btu), Adil\u00ea, Gibr\u00ea, Ub\u016bdu, Gur\u016bmu, "}, {"id_text": "Q003418", "project_name": "rinap/rinap1", "raw_text": "[... I] adorned them (statues of the gods) and they (the gods) went (back) to their land. I rebuilt th[ose] cities. I built a city on top of a tell (lit. \u201ca heaped-up ruin mound\u201d) called [\u1e2aumut]. I b[uilt (and) co]mpleted (it) from its foundations to its parapets. [Inside (it), I founded] a palace for my royal residence. I named it K\u0101r-A\u0161\u0161ur, set up the weapon of (the god) A\u0161\u0161ur, my lord, therein, (and) settled the people [of (foreign) lands] conquered by me therein. [I] imposed upon them [tax (and) tribute], (and) considered them as inhabitants of Assyria. I dug out the Patti-[Enlil] canal, [which] had lain abandoned for a very long time and [...], and I made an abundance of water gurgle through it. I exercised authority over [... from] the cities D\u016br-(Kuri)galzu, Sippar of the god \u0160ama\u0161, [..., the (tribes) Na]sikku, Naq(i)ru, (and) Tan\u00ea, the city Kala\u02bein, the \u0160umandar (Sumandar) canal, [the city Pa]\u1e63\u012btu of the (tribe) Dun\u0101nu, the land Qirbutu (Qirbit), [... the (tribe) Adi]l\u00ea, the land B\u016bdu, the city Pa\u1e2b\u1e2baz, the land Qin-Nippur, (and) the cities [of Kar]dunia\u0161 (Babylonia) as far as the Uqn\u00fb River, [which are on the shore of the Lo]wer [Sea]. I anne[xed] (those areas) to Assyria (and) placed a eunuch of mine as [provincial governor over them]. From their sheep levy, [which] I take [annually], I apportion[ed] 240 sheep as a gift to (the god) A\u0161\u0161ur, my lord. [From] those [Ara]means whom I deported, (10) [I distribut]ed (and) settled [... thousand to the province of] the turt\u0101nu, 10,000 (to) the province of the palace herald, [...] thousand (to) the province of the chief cupbearer, [... thousand (to) the province of the land] Bar\u1e2balzu, (and) 5,000 (to) the province of the land Mazamua (Zamua). I united them, [considered them] as inhabitants of [Assyria, (and) imposed] the yoke of (the god) A\u0161\u0161ur, my lord, [upon them] as Assyrians. (As for) the abandoned settlements on the periphery of my [land] that had bec[ome] desolated [during the reign(s) of (previous) kings], my [ancestor]s,"}, {"id_text": "Q003419", "project_name": "rinap/rinap1", "raw_text": "I restored and [...] Assyria to [...]. I built a city [... Inside (it), I founded] a palace for [my] royal residence. [...]. I named it [..., set up] the weapon of (the god) A\u0161\u0161ur, my lord, th[erein, (and) settled the people of (foreign) lands conquered by me therein. I imposed upon them tax (and) tribute], (and) considered them as inhabitants of Assyria. [I fashioned (on a stele) my royal] image [and the image(s) of the great gods, my lords, inscribed on it the mighty victories (lit. \u201cvictory and might\u201d)] (5) that I had [achieved again and again] over (foreign) lands in the name of (the god) A\u0161\u0161ur, my lord, (and) [I erected (it) in ...]. I received as his payment [...], 10 talents of gold (measured) by the heavy (standard), 1,000 talents of [silver, ...]. In my second pal\u00fb, (the god) A\u0161\u0161ur, [my] lor[d, encouraged me and I marched againLine truncated
|
||||
@@ -0,0 +1 @@
|
||||
[{"id_text": "Q006206", "project_name": "suhu", "raw_text": "[...] ... [...] his [...] and chariots [...] weapons [... \u0160ama\u0161-r]\u0113\u0161a-u\u1e63ur, governor of the land [of S\u016b\u1e2bu and the land of Ma]ri [...] ... [...] ... [...] my [...] and the town K\u0101r-Nab\u00fb was closed [...] ... in the well [...] the living [...] [...] ... [...] ... to/for [...] went up to the water-meadow [... cour]tiers and mayors [...] when they crossed over [in] boats [...] flocks (and) goods [...] weighty [boo]ty [...] I/he caused to enter. [... in]side the land of S\u016b\u1e2bu, the water-meadow ... until now I/he did not inhabit. Four hundred of the Tu\u02bem\u0101nu came and rose up against the town Ribani\u0161. I had gone to the New City for the festival and when in the town Baqa I heard (of the uprising), I crossed over (the river) to the landside with the palace troops who were with me and I pursued them. When I crossed over (the river), I defeated them at Qaqqaru-arad\u0101tu. I killed 350 soldiers among them (and) let the remainder go free to (spread the news of my) glory. I, \u0160ama\u0161-r\u0113\u0161a-u\u1e63ur, governor of the land of S\u016b\u1e2bu and the land of Mari \u2014 (when) the canal of the land of S\u016b\u1e2bu became old, I called up numerous workmen from the town \u1e2aurz\u0113 as far as the town Y\u0101bi\u02be and I realigned (it over a distance of) one thousand reeds. I made that canal twenty-two cubits wide. I took down the bridge at the entrance to the canal so that I could send a boat of twenty-five cubits through it. I (and) my nobles went inside (the boat). When the boat reached the opening of the canal at \u00c9.SAL.A, I brought (it) out (of the canal). I planted the date palm trees which are <in> the courtyard of the palace of the town Ribani\u0161. I set up the throne and footstool which are in the town Ribani\u0161. I built a (new) town and named it \u0100l-gabb\u0101ri-b\u0101n\u00ee. I set up (statues of) the deities Adad, Apla-Adad, \u0160ala, (and) Mad\u0101nu; (iii 3) I settled them in a pleasant abode inside my town \u0100l-gabb\u0101ri-b\u0101n\u00ee. With regard to anyone in the future who comes forward, approaches this stele, effaces (my) inscribed name, and inscribes his (own) name, may the great gods of heaven and netherworld remove (the mention of) his name, his seed, his offspring, (and) his progeny from the mouth(s) of the people! May he not accomplish any task that I have done! May (this) curse reach the one who transgresses against these words (and) does not fear them! I planted the date palm trees which are in the courtyard of the palace of the city Ra\u02beil. I planted the date palm trees which are in the courtyard of the palace of the town K\u0101r-Nab\u00fb. I planted (an area which could be seeded by) two kurru, three p\u0101nu, and two s\u016btu of dates in the district of the town Y\u0101duru, in the town Ukal\u0101ya, (for the growing of) date palm trees. I constructed a new weir along the old one over (a length of) three hundred and sixty cubits and thus the water in the canal of the land of S\u016b\u1e2bu was plentiful. From the weir as far as \u00c9.SAL.LE.E, I planted all the willows which are along my canal so that carrying baskets for (use in connection with) the canal could be woven with them and so that I could cut beams from them and make (them) into .... The willows which I planted along the canal all belong to the palace; the people of S\u016b\u1e2bu may not bring a claim about (them). (This) document was sealed in the thirteenth year of \u0160ama\u0161-r\u0113\u0161a-u\u1e63ur, governor of the land of S\u016b\u1e2bu and the land of Mari. I, \u0160ama\u0161-r\u0113\u0161a-u\u1e63ur, governor of the land of S\u016b\u1e2bu and the land of Mari, brought down from the mountain of the people of \u1e2aab\u1e2bu the bees which gather honey \u2014 which none from among my forefathers had seen or brought down to the land of S\u016b\u1e2bu \u2014 and I established them in the gardens of the town \u0100l-gabb\u0101ri-b\u0101n\u00ee. They (now) collect honey and wax (there). I know how (to separate) honey and wax by melting (the combs) and (my) gardeners (also) know how (to do it). Anyone in the future who comes forward should ask the elders of his land: \u201cIs it true that \u0160ama\u0161-r\u0113\u0161a-u\u1e63ur, the governor of the land of S\u016b\u1e2bu, introduced honey-bees into the land of S\u016b\u1e2bu?\u201d [Image of the goddess] I\u0161tar. Image of the god Adad. Image of \u0160ama\u0161-r\u0113\u0161a-u\u1e63ur, governor of the land of S\u016b\u1e2bu and the land of Mari. I have established one q\u00fb of bread (and) one q\u00fb of fine beer as the regular offerings for this stele. Anyone who protects the palace may eat (it). [Image of (the goddess) Anat.]"}, {"id_text": "Q006207", "project_name": "suhu", "raw_text": "[I, \u0160ama\u0161-r\u0113\u0161a-u\u1e63ur, governor of the land of S\u016b\u1e2bu Line truncated
|
||||
Reference in new issue
Block a user