From cccb5e8698eb529f22b8260a2b011e8ed30b171c Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum <39048939+jnussbaum@users.noreply.github.com> Date: Wed, 1 Jun 2022 16:51:16 +0200 Subject: [PATCH] fix(xmlupload): prevent crash with incremental option (DEV-811) (#197) * actual bugfix * write e2e test for incremental xmlupload * delete unused files. tmp folder should be empty in the VCS * create folder testdata/tmp during setUp() --- .gitignore | 3 +- knora/dsplib/utils/xml_upload.py | 3 ++ test/e2e/test_tools.py | 33 ++++++++++-- test/unittests/test_excel_to_json_lists.py | 4 ++ test/unittests/test_excel_to_resource.py | 4 ++ test/unittests/test_id_to_iri.py | 5 ++ testdata/test-id2iri-data.xml | 51 +++++++------------ testdata/test-id2iri-mapping.json | 4 +- testdata/test-id2iri-replaced.xml | 58 ---------------------- testdata/tmp/_test-id2iri-replaced.xml | 58 ---------------------- 10 files changed, 65 insertions(+), 158 deletions(-) delete mode 100644 testdata/test-id2iri-replaced.xml delete mode 100644 testdata/tmp/_test-id2iri-replaced.xml diff --git a/.gitignore b/.gitignore index 2cc11d992..284f01fc8 100644 --- a/.gitignore +++ b/.gitignore @@ -66,8 +66,7 @@ lists.json out.json id2iri_* **/~$*.* -testdata/tmp/_*.json -testdata/tmp/_*.xml +testdata/tmp/ testdata/test-list.json # for testing in development diff --git a/knora/dsplib/utils/xml_upload.py b/knora/dsplib/utils/xml_upload.py index 708770046..fa512a962 100644 --- a/knora/dsplib/utils/xml_upload.py +++ b/knora/dsplib/utils/xml_upload.py @@ -880,6 +880,9 @@ def xml_upload(input_file: str, server: str, user: str, password: str, imgdir: s # temporarily remove circular references, but only if not an incremental upload if not incremental: resources, stashed_xml_texts, stashed_resptr_props = remove_circular_references(resources, verbose) + else: + stashed_xml_texts = dict() + stashed_resptr_props = dict() sipi_server = Sipi(sipi, con.get_token()) diff --git a/test/e2e/test_tools.py b/test/e2e/test_tools.py index 6c67a2506..d4557d450 100644 --- a/test/e2e/test_tools.py +++ b/test/e2e/test_tools.py @@ -1,6 +1,8 @@ """This test class tests the basic functionalities of dsp-tools""" import json import unittest +import os +import datetime from knora.dsplib.utils import excel_to_json_lists from knora.dsplib.utils.excel_to_json_lists import list_excel2json @@ -16,6 +18,9 @@ class TestTools(unittest.TestCase): server = 'http://0.0.0.0:3333' user = 'root@example.com' + password = 'test' + imgdir = '.' + sipi = 'http://0.0.0.0:1024' test_onto_file = 'testdata/test-onto.json' test_list_file = 'testdata/test-list.json' test_data_file = 'testdata/test-data.xml' @@ -24,6 +29,7 @@ def setUp(self) -> None: """Is executed before each test""" excel_to_json_lists.list_of_lists = [] excel_to_json_lists.cell_names = [] + os.makedirs('testdata/tmp', exist_ok=True) def tearDown(self) -> None: """Is executed after each test""" @@ -173,19 +179,36 @@ def test_xml_upload(self) -> None: xml_upload(input_file=self.test_data_file, server=self.server, user=self.user, - password='test', - imgdir='.', - sipi='http://0.0.0.0:1024', + password=self.password, + imgdir=self.imgdir, + sipi=self.sipi, verbose=False, validate_only=False, incremental=False) - def test_id_to_iri(self) -> None: + mapping_file = '' + for mapping in [x for x in os.scandir('.') if x.name.startswith('id2iri_test-data_mapping_')]: + delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(mapping.stat().st_mtime_ns / 1000000000) + if delta.seconds < 15: + mapping_file = mapping.name + id_to_iri(xml_file='testdata/test-id2iri-data.xml', - json_file='testdata/test-id2iri-mapping.json', + json_file=mapping_file, out_file='testdata/tmp/_test-id2iri-replaced.xml', verbose=False) + xml_upload( + input_file='testdata/tmp/_test-id2iri-replaced.xml', + server=self.server, + user=self.user, + password=self.password, + imgdir=self.imgdir, + sipi=self.sipi, + verbose=False, + validate_only=False, + incremental=True + ) + if __name__ == '__main__': unittest.main() diff --git a/test/unittests/test_excel_to_json_lists.py b/test/unittests/test_excel_to_json_lists.py index 0a8659d0c..eac378654 100644 --- a/test/unittests/test_excel_to_json_lists.py +++ b/test/unittests/test_excel_to_json_lists.py @@ -11,6 +11,10 @@ class TestExcelToJSONList(unittest.TestCase): + def setUp(self) -> None: + """Is executed before each test""" + os.makedirs('testdata/tmp', exist_ok=True) + def test_excel2jsonlist(self) -> None: # check that the output file was created excelfolder = "testdata/lists" diff --git a/test/unittests/test_excel_to_resource.py b/test/unittests/test_excel_to_resource.py index 175c455e2..3cda7a782 100644 --- a/test/unittests/test_excel_to_resource.py +++ b/test/unittests/test_excel_to_resource.py @@ -9,6 +9,10 @@ class TestExcelToResource(unittest.TestCase): + def setUp(self) -> None: + """Is executed before each test""" + os.makedirs('testdata/tmp', exist_ok=True) + def test_excel2json(self) -> None: in_file = "testdata/Resources.xlsx" out_file = "testdata/tmp/_out_res.json" diff --git a/test/unittests/test_id_to_iri.py b/test/unittests/test_id_to_iri.py index cf8278cd1..581c75d0f 100644 --- a/test/unittests/test_id_to_iri.py +++ b/test/unittests/test_id_to_iri.py @@ -1,6 +1,7 @@ """Unit tests for id to iri mapping""" import unittest +import os from lxml import etree @@ -10,6 +11,10 @@ class TestIdToIri(unittest.TestCase): out_file = 'testdata/tmp/_test-id2iri-replaced.xml' + def setUp(self) -> None: + """Is executed before each test""" + os.makedirs('testdata/tmp', exist_ok=True) + def test_invalid_xml_file_name(self): with self.assertRaises(SystemExit) as cm: id_to_iri(xml_file='test.xml', diff --git a/testdata/test-id2iri-data.xml b/testdata/test-id2iri-data.xml index c1a9d24ac..86b24918d 100644 --- a/testdata/test-id2iri-data.xml +++ b/testdata/test-id2iri-data.xml @@ -1,20 +1,19 @@ - + - V + RV V CR CR - V V CR CR @@ -26,39 +25,25 @@ CR - V V CR CR - - - Images/Danby-deluge.jpg - - The Deluge + + + + Text with XML standoff link to resource obj_0000 + and obj_0011 + - - person_2 + + obj_0001 + obj_0011 - - GREGORIAN:CE:1877:CE:1879 - - - Oil paint on canvas. - - - The Deluge - - - institution_2 - - - 6695209 - - - https://www.tate.org.uk/art/artworks/danby-the-deluge-t01337 - + diff --git a/testdata/test-id2iri-mapping.json b/testdata/test-id2iri-mapping.json index a317b56fc..f492d9194 100644 --- a/testdata/test-id2iri-mapping.json +++ b/testdata/test-id2iri-mapping.json @@ -1,4 +1,4 @@ { - "person_2": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg", - "institution_2": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ" + "obj_0001": "http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg", + "obj_0011": "http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ" } diff --git a/testdata/test-id2iri-replaced.xml b/testdata/test-id2iri-replaced.xml deleted file mode 100644 index 666a04caa..000000000 --- a/testdata/test-id2iri-replaced.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - V - V - CR - CR - - - V - V - CR - CR - - - V - V - CR - CR - - - V - V - CR - CR - - - - - Images/Danby-deluge.jpg - - The Deluge - - - http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg - - - GREGORIAN:CE:1877:CE:1879 - - - Oil paint on canvas. - - - The Deluge - - - http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ - - - 6695209 - - - https://www.tate.org.uk/art/artworks/danby-the-deluge-t01337 - - - - diff --git a/testdata/tmp/_test-id2iri-replaced.xml b/testdata/tmp/_test-id2iri-replaced.xml deleted file mode 100644 index 666a04caa..000000000 --- a/testdata/tmp/_test-id2iri-replaced.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - V - V - CR - CR - - - V - V - CR - CR - - - V - V - CR - CR - - - V - V - CR - CR - - - - - Images/Danby-deluge.jpg - - The Deluge - - - http://rdfh.ch/082E/ylRvrg7tQI6aVpcTJbVrwg - - - GREGORIAN:CE:1877:CE:1879 - - - Oil paint on canvas. - - - The Deluge - - - http://rdfh.ch/082E/JK63OpYWTDWNYVOYFN7FdQ - - - 6695209 - - - https://www.tate.org.uk/art/artworks/danby-the-deluge-t01337 - - - -