fix(xmlupload): prevent crash with incremental option (DEV-811)

.gitignore

@@ -66,8 +66,7 @@ lists.json
 out.json
 id2iri_*
 **/~$*.*
-testdata/tmp/_*.json
-testdata/tmp/_*.xml
+testdata/tmp/
 testdata/test-list.json
 
 # for testing in development

knora/dsplib/utils/xml_upload.py

@@ -880,6 +880,9 @@ def xml_upload(input_file: str, server: str, user: str, password: str, imgdir: s
     # temporarily remove circular references, but only if not an incremental upload
     if not incremental:
         resources, stashed_xml_texts, stashed_resptr_props = remove_circular_references(resources, verbose)
+    else:
+        stashed_xml_texts = dict()
+        stashed_resptr_props = dict()
 
     sipi_server = Sipi(sipi, con.get_token())
 

test/e2e/test_tools.py

@@ -1,6 +1,8 @@
 """This test class tests the basic functionalities of dsp-tools"""
 import json
 import unittest
+import os
+import datetime
 
 from knora.dsplib.utils import excel_to_json_lists
 from knora.dsplib.utils.excel_to_json_lists import list_excel2json
@@ -16,6 +18,9 @@
 class TestTools(unittest.TestCase):
     server = 'http://0.0.0.0:3333'
     user = 'root@example.com'
+    password = 'test'
+    imgdir = '.'
+    sipi = 'http://0.0.0.0:1024'
     test_onto_file = 'testdata/test-onto.json'
     test_list_file = 'testdata/test-list.json'
     test_data_file = 'testdata/test-data.xml'
@@ -24,6 +29,7 @@ def setUp(self) -> None:
         """Is executed before each test"""
         excel_to_json_lists.list_of_lists = []
         excel_to_json_lists.cell_names = []
+        os.makedirs('testdata/tmp', exist_ok=True)
 
     def tearDown(self) -> None:
         """Is executed after each test"""
@@ -173,19 +179,36 @@ def test_xml_upload(self) -> None:
         xml_upload(input_file=self.test_data_file,
                    server=self.server,
                    user=self.user,
-                   password='test',
-                   imgdir='.',
-                   sipi='http://0.0.0.0:1024',
+                   password=self.password,
+                   imgdir=self.imgdir,
+                   sipi=self.sipi,
                    verbose=False,
                    validate_only=False,
                    incremental=False)
 
-    def test_id_to_iri(self) -> None:
+        mapping_file = ''
+        for mapping in [x for x in os.scandir('.') if x.name.startswith('id2iri_test-data_mapping_')]:
+            delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(mapping.stat().st_mtime_ns / 1000000000)
+            if delta.seconds < 15:
+                mapping_file = mapping.name
+
         id_to_iri(xml_file='testdata/test-id2iri-data.xml',
-                  json_file='testdata/test-id2iri-mapping.json',
+                  json_file=mapping_file,
                   out_file='testdata/tmp/_test-id2iri-replaced.xml',
                   verbose=False)
 
+        xml_upload(
+            input_file='testdata/tmp/_test-id2iri-replaced.xml',
+            server=self.server,
+            user=self.user,
+            password=self.password,
+            imgdir=self.imgdir,
+            sipi=self.sipi,
+            verbose=False,
+            validate_only=False,
+            incremental=True
+        )
+
 
 if __name__ == '__main__':
     unittest.main()

test/unittests/test_excel_to_json_lists.py

@@ -11,6 +11,10 @@
 
 class TestExcelToJSONList(unittest.TestCase):
 
+    def setUp(self) -> None:
+        """Is executed before each test"""
+        os.makedirs('testdata/tmp', exist_ok=True)
+
     def test_excel2jsonlist(self) -> None:
         # check that the output file was created
         excelfolder = "testdata/lists"

test/unittests/test_excel_to_resource.py

@@ -9,6 +9,10 @@
 
 class TestExcelToResource(unittest.TestCase):
 
+    def setUp(self) -> None:
+        """Is executed before each test"""
+        os.makedirs('testdata/tmp', exist_ok=True)
+
     def test_excel2json(self) -> None:
         in_file = "testdata/Resources.xlsx"
         out_file = "testdata/tmp/_out_res.json"

test/unittests/test_id_to_iri.py

@@ -1,6 +1,7 @@
 """Unit tests for id to iri mapping"""
 
 import unittest
+import os
 
 from lxml import etree
 
@@ -10,6 +11,10 @@
 class TestIdToIri(unittest.TestCase):
     out_file = 'testdata/tmp/_test-id2iri-replaced.xml'
 
+    def setUp(self) -> None:
+        """Is executed before each test"""
+        os.makedirs('testdata/tmp', exist_ok=True)
+
     def test_invalid_xml_file_name(self):
         with self.assertRaises(SystemExit) as cm:
             id_to_iri(xml_file='test.xml',

testdata/test-id2iri-data.xml

@@ -1,20 +1,19 @@
 <?xml version='1.0' encoding='utf-8'?>
 
-<knora xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-       xmlns="https://dasch.swiss/schema"
-       xsi:schemaLocation="https://dasch.swiss/schema https://raw.githubusercontent.com/dasch-swiss/dsp-tools/main/knora/dsplib/schemas/data.xsd"
-       shortcode="082E"
-       default-ontology="rosetta">
+<knora xmlns="https://dasch.swiss/schema"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="https://dasch.swiss/schema ../knora/dsplib/schemas/data.xsd"
+       shortcode="4123"
+       default-ontology="testonto">
 
     <!-- :permissions see https://docs.dasch.swiss/DSP-API/05-internals/design/api-admin/administration/#permissions -->
     <permissions id="res-default">
-        <allow group="UnknownUser">V</allow>
+        <allow group="UnknownUser">RV</allow>
         <allow group="KnownUser">V</allow>
         <allow group="Creator">CR</allow>
         <allow group="ProjectAdmin">CR</allow>
     </permissions>
     <permissions id="res-restricted">
-        <allow group="UnknownUser">V</allow>
         <allow group="KnownUser">V</allow>
         <allow group="Creator">CR</allow>
         <allow group="ProjectAdmin">CR</allow>
@@ -26,39 +25,25 @@
         <allow group="ProjectAdmin">CR</allow>
     </permissions>
     <permissions id="prop-restricted">
-        <allow group="UnknownUser">V</allow>
         <allow group="KnownUser">V</allow>
         <allow group="Creator">CR</allow>
         <allow group="ProjectAdmin">CR</allow>
     </permissions>
 
-    <!-- :Image2D -->
-    <resource label="Deluge2" restype=":Image2D" id="img_obj_1" permissions="res-default">
-        <bitstream>Images/Danby-deluge.jpg</bitstream>
-        <text-prop name=":hasTitle">
-            <text permissions="prop-default" encoding="utf8">The Deluge</text>
+    <resource label="TestThing2Incremental"
+              restype=":TestThing2"
+              id="TestThing2Incremental">
+        <text-prop name=":hasText">
+            <text encoding="xml">
+                Text with XML standoff link to resource <a class="salsah-link" href="IRI:obj_0000:IRI">obj_0000</a>
+                and <a class="salsah-link" href="IRI:obj_0011:IRI">obj_0011</a>
+            </text>
         </text-prop>
-        <resptr-prop name=":hasCreator">
-            <resptr permissions="prop-default">person_2</resptr>
+        <resptr-prop name=":hasTestThing">
+            <resptr permissions="prop-default">obj_0001</resptr>
+            <resptr permissions="prop-default">obj_0011</resptr>
         </resptr-prop>
-        <date-prop name=":hasDate">
-            <date permissions="prop-default">GREGORIAN:CE:1877:CE:1879</date>
-        </date-prop>
-        <text-prop name=":hasDescription">
-            <text permissions="prop-default" encoding="utf8">Oil paint on canvas.</text>
-        </text-prop>
-        <text-prop name=":hasCopyright">
-            <text permissions="prop-default" encoding="utf8">The Deluge</text>
-        </text-prop>
-        <resptr-prop name=":inInstitution">
-            <resptr permissions="prop-default">institution_2</resptr>
-        </resptr-prop>
-        <geoname-prop name=":hasLocation">
-            <geoname permissions="prop-default" comment="London">6695209</geoname>
-        </geoname-prop>
-        <uri-prop name=":hasExternalLink">
-            <uri permissions="prop-default">https://www.tate.org.uk/art/artworks/danby-the-deluge-t01337</uri>
-        </uri-prop>
     </resource>
 
+
 </knora>

testdata/test-id2iri-mapping.json

@@ -1,4 +1,4 @@
 {
-    "person_2": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg",
-    "institution_2": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ"
+    "obj_0001": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg",
+    "obj_0011": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ"
 }