Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
fix(xmlupload): prevent crash with incremental option (DEV-811) (#197)
* actual bugfix
* write e2e test for incremental xmlupload
* delete unused files. tmp folder should be empty in the VCS
* create folder testdata/tmp during setUp()
  • Loading branch information
jnussbaum committed Jun 1, 2022
1 parent d3efde8 commit cccb5e8
Show file tree
Hide file tree
Showing 10 changed files with 65 additions and 158 deletions.
3 changes: 1 addition & 2 deletions .gitignore
Expand Up @@ -66,8 +66,7 @@ lists.json
out.json
id2iri_*
**/~$*.*
testdata/tmp/_*.json
testdata/tmp/_*.xml
testdata/tmp/
testdata/test-list.json

# for testing in development
Expand Down
3 changes: 3 additions & 0 deletions knora/dsplib/utils/xml_upload.py
Expand Up @@ -880,6 +880,9 @@ def xml_upload(input_file: str, server: str, user: str, password: str, imgdir: s
# temporarily remove circular references, but only if not an incremental upload
if not incremental:
resources, stashed_xml_texts, stashed_resptr_props = remove_circular_references(resources, verbose)
else:
stashed_xml_texts = dict()
stashed_resptr_props = dict()

sipi_server = Sipi(sipi, con.get_token())

Expand Down
33 changes: 28 additions & 5 deletions test/e2e/test_tools.py
@@ -1,6 +1,8 @@
"""This test class tests the basic functionalities of dsp-tools"""
import json
import unittest
import os
import datetime

from knora.dsplib.utils import excel_to_json_lists
from knora.dsplib.utils.excel_to_json_lists import list_excel2json
Expand All @@ -16,6 +18,9 @@
class TestTools(unittest.TestCase):
server = 'http://0.0.0.0:3333'
user = 'root@example.com'
password = 'test'
imgdir = '.'
sipi = 'http://0.0.0.0:1024'
test_onto_file = 'testdata/test-onto.json'
test_list_file = 'testdata/test-list.json'
test_data_file = 'testdata/test-data.xml'
Expand All @@ -24,6 +29,7 @@ def setUp(self) -> None:
"""Is executed before each test"""
excel_to_json_lists.list_of_lists = []
excel_to_json_lists.cell_names = []
os.makedirs('testdata/tmp', exist_ok=True)

def tearDown(self) -> None:
"""Is executed after each test"""
Expand Down Expand Up @@ -173,19 +179,36 @@ def test_xml_upload(self) -> None:
xml_upload(input_file=self.test_data_file,
server=self.server,
user=self.user,
password='test',
imgdir='.',
sipi='http://0.0.0.0:1024',
password=self.password,
imgdir=self.imgdir,
sipi=self.sipi,
verbose=False,
validate_only=False,
incremental=False)

def test_id_to_iri(self) -> None:
mapping_file = ''
for mapping in [x for x in os.scandir('.') if x.name.startswith('id2iri_test-data_mapping_')]:
delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(mapping.stat().st_mtime_ns / 1000000000)
if delta.seconds < 15:
mapping_file = mapping.name

id_to_iri(xml_file='testdata/test-id2iri-data.xml',
json_file='testdata/test-id2iri-mapping.json',
json_file=mapping_file,
out_file='testdata/tmp/_test-id2iri-replaced.xml',
verbose=False)

xml_upload(
input_file='testdata/tmp/_test-id2iri-replaced.xml',
server=self.server,
user=self.user,
password=self.password,
imgdir=self.imgdir,
sipi=self.sipi,
verbose=False,
validate_only=False,
incremental=True
)


if __name__ == '__main__':
unittest.main()
4 changes: 4 additions & 0 deletions test/unittests/test_excel_to_json_lists.py
Expand Up @@ -11,6 +11,10 @@

class TestExcelToJSONList(unittest.TestCase):

def setUp(self) -> None:
"""Is executed before each test"""
os.makedirs('testdata/tmp', exist_ok=True)

def test_excel2jsonlist(self) -> None:
# check that the output file was created
excelfolder = "testdata/lists"
Expand Down
4 changes: 4 additions & 0 deletions test/unittests/test_excel_to_resource.py
Expand Up @@ -9,6 +9,10 @@

class TestExcelToResource(unittest.TestCase):

def setUp(self) -> None:
"""Is executed before each test"""
os.makedirs('testdata/tmp', exist_ok=True)

def test_excel2json(self) -> None:
in_file = "testdata/Resources.xlsx"
out_file = "testdata/tmp/_out_res.json"
Expand Down
5 changes: 5 additions & 0 deletions test/unittests/test_id_to_iri.py
@@ -1,6 +1,7 @@
"""Unit tests for id to iri mapping"""

import unittest
import os

from lxml import etree

Expand All @@ -10,6 +11,10 @@
class TestIdToIri(unittest.TestCase):
out_file = 'testdata/tmp/_test-id2iri-replaced.xml'

def setUp(self) -> None:
"""Is executed before each test"""
os.makedirs('testdata/tmp', exist_ok=True)

def test_invalid_xml_file_name(self):
with self.assertRaises(SystemExit) as cm:
id_to_iri(xml_file='test.xml',
Expand Down
51 changes: 18 additions & 33 deletions testdata/test-id2iri-data.xml
@@ -1,20 +1,19 @@
<?xml version='1.0' encoding='utf-8'?>

<knora xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="https://dasch.swiss/schema"
xsi:schemaLocation="https://dasch.swiss/schema https://raw.githubusercontent.com/dasch-swiss/dsp-tools/main/knora/dsplib/schemas/data.xsd"
shortcode="082E"
default-ontology="rosetta">
<knora xmlns="https://dasch.swiss/schema"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://dasch.swiss/schema ../knora/dsplib/schemas/data.xsd"
shortcode="4123"
default-ontology="testonto">

<!-- :permissions see https://docs.dasch.swiss/DSP-API/05-internals/design/api-admin/administration/#permissions -->
<permissions id="res-default">
<allow group="UnknownUser">V</allow>
<allow group="UnknownUser">RV</allow>
<allow group="KnownUser">V</allow>
<allow group="Creator">CR</allow>
<allow group="ProjectAdmin">CR</allow>
</permissions>
<permissions id="res-restricted">
<allow group="UnknownUser">V</allow>
<allow group="KnownUser">V</allow>
<allow group="Creator">CR</allow>
<allow group="ProjectAdmin">CR</allow>
Expand All @@ -26,39 +25,25 @@
<allow group="ProjectAdmin">CR</allow>
</permissions>
<permissions id="prop-restricted">
<allow group="UnknownUser">V</allow>
<allow group="KnownUser">V</allow>
<allow group="Creator">CR</allow>
<allow group="ProjectAdmin">CR</allow>
</permissions>

<!-- :Image2D -->
<resource label="Deluge2" restype=":Image2D" id="img_obj_1" permissions="res-default">
<bitstream>Images/Danby-deluge.jpg</bitstream>
<text-prop name=":hasTitle">
<text permissions="prop-default" encoding="utf8">The Deluge</text>
<resource label="TestThing2Incremental"
restype=":TestThing2"
id="TestThing2Incremental">
<text-prop name=":hasText">
<text encoding="xml">
Text with XML standoff link to resource <a class="salsah-link" href="IRI:obj_0000:IRI">obj_0000</a>
and <a class="salsah-link" href="IRI:obj_0011:IRI">obj_0011</a>
</text>
</text-prop>
<resptr-prop name=":hasCreator">
<resptr permissions="prop-default">person_2</resptr>
<resptr-prop name=":hasTestThing">
<resptr permissions="prop-default">obj_0001</resptr>
<resptr permissions="prop-default">obj_0011</resptr>
</resptr-prop>
<date-prop name=":hasDate">
<date permissions="prop-default">GREGORIAN:CE:1877:CE:1879</date>
</date-prop>
<text-prop name=":hasDescription">
<text permissions="prop-default" encoding="utf8">Oil paint on canvas.</text>
</text-prop>
<text-prop name=":hasCopyright">
<text permissions="prop-default" encoding="utf8">The Deluge</text>
</text-prop>
<resptr-prop name=":inInstitution">
<resptr permissions="prop-default">institution_2</resptr>
</resptr-prop>
<geoname-prop name=":hasLocation">
<geoname permissions="prop-default" comment="London">6695209</geoname>
</geoname-prop>
<uri-prop name=":hasExternalLink">
<uri permissions="prop-default">https://www.tate.org.uk/art/artworks/danby-the-deluge-t01337</uri>
</uri-prop>
</resource>


</knora>
4 changes: 2 additions & 2 deletions testdata/test-id2iri-mapping.json
@@ -1,4 +1,4 @@
{
"person_2": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg",
"institution_2": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ"
"obj_0001": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg",
"obj_0011": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ"
}
58 changes: 0 additions & 58 deletions testdata/test-id2iri-replaced.xml

This file was deleted.

58 changes: 0 additions & 58 deletions testdata/tmp/_test-id2iri-replaced.xml

This file was deleted.

0 comments on commit cccb5e8

Please sign in to comment.