From 6e3e7ca4f603be3f656e0caede7eefbea2790383 Mon Sep 17 00:00:00 2001 From: Lukas Rosenthaler Date: Tue, 20 Apr 2021 23:23:00 +0200 Subject: [PATCH] fix: fix import ontology from salsah-export (DSP-1532) (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Small fixes for dealing with ontolpogies from salsah-export * fix: typo in test data * Small fixes * Searching for bug in CI * … * Debugging CI… * Changes for passing CI tests Co-authored-by: Balduin Landolt --- knora/dsp_tools.py | 8 +++++++- knora/dsplib/models/helpers.py | 1 - knora/dsplib/models/langstring.py | 5 +++-- knora/dsplib/models/listnode.py | 10 ++++++++++ knora/dsplib/models/ontology.py | 19 +++++++++--------- knora/dsplib/models/propertyclass.py | 3 ++- knora/dsplib/models/sipi.py | 11 ++++++----- knora/dsplib/utils/knora-schema-lists.json | 11 +++++++++-- knora/dsplib/utils/knora-schema.json | 2 +- knora/dsplib/utils/onto_create_lists.py | 23 +++++++++++++--------- knora/dsplib/utils/onto_process_excel.py | 3 ++- knora/dsplib/utils/onto_validate.py | 3 ++- knora/mylist.json | 0 test/test_propertyclass.py | 5 ++++- testdata/test-data.xml | 2 +- 15 files changed, 71 insertions(+), 35 deletions(-) create mode 100644 knora/mylist.json diff --git a/knora/dsp_tools.py b/knora/dsp_tools.py index 773f276d5..18f0a659f 100644 --- a/knora/dsp_tools.py +++ b/knora/dsp_tools.py @@ -81,7 +81,13 @@ def program(args): if args.validate: validate_list(args.datamodelfile) else: - create_lists(args.datamodelfile, args.dump) + create_lists(input_file=args.datamodelfile, + lists_file=args.listfile, + server=args.server, + user=args.user, + password=args.password, + verbose=args.verbose, + dump=args.dump) else: if args.validate: validate_ontology(args.datamodelfile) diff --git a/knora/dsplib/models/helpers.py b/knora/dsplib/models/helpers.py index a37fb7ac2..a499780ec 100644 --- a/knora/dsplib/models/helpers.py +++ b/knora/dsplib/models/helpers.py @@ -397,7 +397,6 @@ def reduce_iri(self, iristr: str, ontoname: Optional[str] = None) -> str: knora_api = self.prefix_from_iri("http://api.knora.org/ontology/knora-api/v2#") salsah_gui = self.prefix_from_iri("http://api.knora.org/ontology/salsah-gui/v2#") - #if self.__is_iri(iristr): if IriTest.test(iristr): iristr = self.get_prefixed_iri(iristr) tmp = iristr.split(':') diff --git a/knora/dsplib/models/langstring.py b/knora/dsplib/models/langstring.py index f322e9522..8c307266b 100644 --- a/knora/dsplib/models/langstring.py +++ b/knora/dsplib/models/langstring.py @@ -193,7 +193,7 @@ def toJsonObj(self): if self._simplestring is not None: return self._simplestring else: - return list(map(lambda a: {'language': a[0].value, 'value': a[1]}, self._langstrs.items())) + return list(map(lambda a: {'language': a[0].value, 'value': a[1] if a[1] else "-"}, self._langstrs.items())) def toJsonLdObj(self): if self.isEmpty(): @@ -201,7 +201,8 @@ def toJsonLdObj(self): if self._simplestring is not None: return self._simplestring else: - return list(map(lambda a: {'@language': a[0].value, '@value': a[1]}, self._langstrs.items())) + return [{'@language': a[0].value, '@value': a[1]} for a in self._langstrs.items()] + #return list(map(lambda a: {'@language': a[0].value, '@value': a[1]}, self._langstrs.items())) @classmethod def fromJsonLdObj(cls, obj: Optional[Union[List[Dict[str, str]], str]]) -> 'LangString': diff --git a/knora/dsplib/models/listnode.py b/knora/dsplib/models/listnode.py index e805eee45..563ff1cd4 100644 --- a/knora/dsplib/models/listnode.py +++ b/knora/dsplib/models/listnode.py @@ -445,6 +445,16 @@ def toJsonObj(self, action: Actions, listIri: str = None) -> Any: tmp['comments'] = self._comments.toJsonObj() if self._name is not None and 'name' in self._changed: tmp['name'] = self._name + # + # temporary fix for bug in dsp-api which prevents labels from having + # escaped double-quotes in the string, e.g. "this \"label\" not works"! + # The double quotes will be replaced by single quotes... + # + if tmp.get('labels'): + print(tmp['labels']) + tmp['labels'] = [{'language': ele['language'], 'value': ele['value'].replace('"', "'")} for ele in tmp['labels']] + #tmp['labels'] = {k: v.replace('"', "'") for k, v in tmp['labels'].items()} + # End of FIX return tmp def create(self) -> 'ListNode': diff --git a/knora/dsplib/models/ontology.py b/knora/dsplib/models/ontology.py index 8f3822aea..00f57bd6a 100644 --- a/knora/dsplib/models/ontology.py +++ b/knora/dsplib/models/ontology.py @@ -432,19 +432,20 @@ def createDefinitionFileObj(self): return ontology - def print(self) -> None: + def print(self, short: bool = True) -> None: print('Ontology Info:') print(' Id: {}'.format(self._id)) print(' Label: {}'.format(self._label)) print(' Name: {}'.format(self._name)) print(' Project: {}'.format(self._project)) print(' LastModificationDate: {}'.format(str(self._lastModificationDate))) - print(' Property Classes:') - if self._property_classes: - for pc in self._property_classes: - pc.print(4) - print(' Resource Classes:') - if self._resource_classes: - for rc in self._resource_classes: - rc.print(4) + if not short: + print(' Property Classes:') + if self._property_classes: + for pc in self._property_classes: + pc.print(4) + print(' Resource Classes:') + if self._resource_classes: + for rc in self._resource_classes: + rc.print(4) diff --git a/knora/dsplib/models/propertyclass.py b/knora/dsplib/models/propertyclass.py index 126b5eb8e..a6b0b2833 100644 --- a/knora/dsplib/models/propertyclass.py +++ b/knora/dsplib/models/propertyclass.py @@ -311,7 +311,8 @@ def resolve_propref(resref: str): tmp = resref.split(':') if len(tmp) > 1: if tmp[0]: - return {"@id": resref} # fully qualified name in the form "prefix:name" +# return {"@id": resref} # fully qualified name in the form "prefix:name" + return {"@id": self._context.get_qualified_iri(resref)} # fully qualified name in the form "prefix:name" else: return {"@id": self._context.prefix_from_iri(self._ontology_id) + ':' + tmp[1]} # ":name" in current ontology else: diff --git a/knora/dsplib/models/sipi.py b/knora/dsplib/models/sipi.py index 5cb0d3e8f..cf6606b10 100644 --- a/knora/dsplib/models/sipi.py +++ b/knora/dsplib/models/sipi.py @@ -25,11 +25,12 @@ def on_api_error(self, res): def upload_bitstream(self, filepath): print(f"filepath=${os.path.basename(filepath)} (${filepath})") - files = { - 'file': (os.path.basename(filepath), open(filepath, 'rb')), - } - req = requests.post(self.sipiserver + "/upload?token=" + self.token, - files=files) + with open(filepath, 'rb') as bitstreamfile: + files = { + 'file': (os.path.basename(filepath), bitstreamfile), + } + req = requests.post(self.sipiserver + "/upload?token=" + self.token, + files=files) self.on_api_error(req) res = req.json() return res diff --git a/knora/dsplib/utils/knora-schema-lists.json b/knora/dsplib/utils/knora-schema-lists.json index 8f05e7f8a..83446212f 100644 --- a/knora/dsplib/utils/knora-schema-lists.json +++ b/knora/dsplib/utils/knora-schema-lists.json @@ -76,8 +76,15 @@ }, "required": ["shortcode", "lists"], "additionalProperties": false + }, + "prefixes": { + "type": "object", + "patternProperties": { + "^[\\w-]+$": { "format" : "uri" } + }, + "additionalProperties": false } }, - "required": ["project"], + "required": ["prefixes", "project"], "additionalProperties": false -} \ No newline at end of file +} diff --git a/knora/dsplib/utils/knora-schema.json b/knora/dsplib/utils/knora-schema.json index 7c2446fc2..36373ad5b 100644 --- a/knora/dsplib/utils/knora-schema.json +++ b/knora/dsplib/utils/knora-schema.json @@ -329,7 +329,7 @@ }, "width": { "type": "string", - "pattern": "^[0-9]*%$" + "pattern": "^[0-9]*%?$" }, "wrap": { "type": "string", diff --git a/knora/dsplib/utils/onto_create_lists.py b/knora/dsplib/utils/onto_create_lists.py index a9ca23149..f84b97d68 100644 --- a/knora/dsplib/utils/onto_create_lists.py +++ b/knora/dsplib/utils/onto_create_lists.py @@ -8,7 +8,8 @@ from ..models.listnode import ListNode from .onto_commons import list_creator, validate_list_from_excel, json_list_from_excel -def create_lists (input_file: str, output_file: str, server: str, user: str, password: str, verbose: bool) -> bool: + +def create_lists(input_file: str, lists_file: str, server: str, user: str, password: str, verbose: bool, dump: bool = False) -> bool: current_dir = os.path.dirname(os.path.realpath(__file__)) # let's read the schema for the data model definition @@ -55,6 +56,9 @@ def create_lists (input_file: str, output_file: str, server: str, user: str, pas con = Connection(server) con.login(user, password) + if dump: + con.start_logging() + # -------------------------------------------------------------------------- # let's read the prefixes of external ontologies that may be used # @@ -84,16 +88,17 @@ def create_lists (input_file: str, output_file: str, server: str, user: str, pas con=con, project=project, label=rootnode['labels'], - comment=rootnode.get('comments'), + #comment=rootnode.get('comments'), name=rootnode['name'] ).create() - listnodes = list_creator(con, project, root_list_node, rootnode['nodes']) - listrootnodes[rootnode['name']] = { - "id": root_list_node.id, - "nodes": listnodes - } + if rootnode.get('nodes') is not None: + listnodes = list_creator(con, project, root_list_node, rootnode['nodes']) + listrootnodes[rootnode['name']] = { + "id": root_list_node.id, + "nodes": listnodes + } - with open(output_file, 'w', encoding="utf-8") as fp: + with open(lists_file, 'w', encoding="utf-8") as fp: json.dump(listrootnodes, fp, indent=3, sort_keys=True) - print("The definitions of the node-id's can be found in \"{}}\"!".format(output_file)) + print(f"The definitions of the node-id's can be found in \"{lists_file}\"!") return True diff --git a/knora/dsplib/utils/onto_process_excel.py b/knora/dsplib/utils/onto_process_excel.py index 1f41d67ab..2b352ab68 100644 --- a/knora/dsplib/utils/onto_process_excel.py +++ b/knora/dsplib/utils/onto_process_excel.py @@ -40,4 +40,5 @@ def list_excel2json(excelpath: str, with open(os.path.join(current_dir, 'knora-schema-lists.json')) as s: schema = json.load(s) validate(jsonobj, schema) - json.dump(jsonobj, open(outfile, "w"), indent=4) + with open(outfile, "w") as outfile: + json.dump(jsonobj, outfile, indent=4) diff --git a/knora/dsplib/utils/onto_validate.py b/knora/dsplib/utils/onto_validate.py index 8da21e780..d81049a5a 100644 --- a/knora/dsplib/utils/onto_validate.py +++ b/knora/dsplib/utils/onto_validate.py @@ -68,7 +68,8 @@ def validate_ontology_from_string(jsonstr: str, exceldir: Optional[str] = None) newlists.append(rootnode) datamodel["project"]["lists"] = newlists - json.dump(datamodel, open("gaga.json", "w"), indent=4) + with open("gaga.json", "w") as outfile: + json.dump(datamodel, outfile, indent=4) # validate the data model definition in order to be sure that it is correct validate(datamodel, schema) diff --git a/knora/mylist.json b/knora/mylist.json new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_propertyclass.py b/test/test_propertyclass.py index 7d177f254..1d00c8293 100644 --- a/test/test_propertyclass.py +++ b/test/test_propertyclass.py @@ -10,7 +10,7 @@ class TestPropertyClass(unittest.TestCase): project = "http://rdfh.ch/projects/0001" - onto_name = 'propclass-test' + onto_name = 'propclass-test-a' onto_label = 'propclass_test_ontology' onto: Ontology @@ -30,6 +30,9 @@ def setUp(self) -> None: self.con = Connection('http://0.0.0.0:3333') self.con.login('root@example.com', 'test') + ontos = Ontology.getAllOntologies(self.con) + for onto in ontos: + onto.print(True) # # Create a test ontology # diff --git a/testdata/test-data.xml b/testdata/test-data.xml index d0b006227..89535779a 100644 --- a/testdata/test-data.xml +++ b/testdata/test-data.xml @@ -112,7 +112,7 @@ id="obj_0003" permissions="res-default"> - This is a Compoundthong + This is a Compoundthing