From 06d071a6d47cd1002610c70b076319236bdab0db Mon Sep 17 00:00:00 2001 From: irinaschubert Date: Tue, 10 Aug 2021 14:51:27 +0200 Subject: [PATCH] feat(excel-lists): create multilanguage json lists from excel files (DSP-1580) (#75) * add docstring to main * integrate code from prep repo into dsp-tools * update documentation * reference folder directly in ontology * Update BUILD.bazel * add test data and setup and teardown methods for unit tests * update tests, update bazel files, eliminate duplicated code * update .gitignore * reformat code --- .gitignore | 9 +- docs/dsp-tools-create.md | 22 +- docs/dsp-tools-excel.md | 74 ++- docs/dsp-tools-usage.md | 27 +- docs/index.md | 6 +- knora/dsp_tools.py | 157 +++--- knora/dsplib/models/permission.py | 8 +- knora/dsplib/utils/BUILD.bazel | 72 +-- knora/dsplib/utils/excel_to_json_lists.py | 340 ++++++++++++ knora/dsplib/utils/expand_all_lists.py | 35 ++ .../dsplib/utils/knora-schema-lists-only.json | 75 +++ knora/dsplib/utils/language-codes-3b2_csv.csv | 185 +++++++ knora/dsplib/utils/onto_commons.py | 115 ---- knora/dsplib/utils/onto_create_lists.py | 157 +++--- knora/dsplib/utils/onto_create_ontology.py | 391 ++++++-------- knora/dsplib/utils/onto_get.py | 11 +- knora/dsplib/utils/onto_process_excel.py | 44 -- knora/dsplib/utils/onto_validate.py | 98 ++-- knora/dsplib/utils/xml_upload.py | 4 +- knora/mylist.json | 0 test/BUILD.bazel | 1 + test/test_tools.py | 109 +++- testdata/BUILD.bazel | 7 +- .../{anything.json => anything-onto.json} | 0 testdata/error-onto.json | 492 ------------------ testdata/list-as-excel.xlsx | Bin 9870 -> 0 bytes testdata/lists/Beschreibung_de.xlsx | Bin 0 -> 6438 bytes testdata/lists/description_en.xlsx | Bin 0 -> 6334 bytes testdata/test-onto.json | 7 +- 29 files changed, 1228 insertions(+), 1218 deletions(-) create mode 100644 knora/dsplib/utils/excel_to_json_lists.py create mode 100644 knora/dsplib/utils/expand_all_lists.py create mode 100644 knora/dsplib/utils/knora-schema-lists-only.json create mode 100644 knora/dsplib/utils/language-codes-3b2_csv.csv delete mode 100644 knora/dsplib/utils/onto_commons.py delete mode 100644 knora/dsplib/utils/onto_process_excel.py delete mode 100644 knora/mylist.json rename testdata/{anything.json => anything-onto.json} (100%) delete mode 100644 testdata/error-onto.json delete mode 100644 testdata/list-as-excel.xlsx create mode 100644 testdata/lists/Beschreibung_de.xlsx create mode 100644 testdata/lists/description_en.xlsx diff --git a/.gitignore b/.gitignore index 1190daeaa..7206f750b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,10 +37,6 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt - - - - # Environments .env .venv @@ -64,7 +60,10 @@ venv.bak/ .mypy_cache/ .idea .vscode -/knora/lists.json + +# created files +lists.json +out.json # bazel /bazel-* diff --git a/docs/dsp-tools-create.md b/docs/dsp-tools-create.md index ae975897c..e754224cd 100644 --- a/docs/dsp-tools-create.md +++ b/docs/dsp-tools-create.md @@ -275,12 +275,18 @@ Here is an example on how to build a taxonomic structure in JSON: { "name": "my_list", "labels": {"en": "Disciplines of the Humanities"}, - "comments": {"en": "This ist is just a silly example", "fr": "un example un peu fou"}, + "comments": { + "en": "This is just an example.", + "fr": "C'est un example." + }, "nodes": [ { "name": "node_1_1", "labels": {"en": "Performing arts"}, - "comments": {"en": "Arts that are events", "de": "Künste mit performativem Character"}, + "comments": { + "en": "Arts that are events", + "de": "Künste mit performativem Character" + }, "nodes": [ { "name": "node_2_2", @@ -340,17 +346,17 @@ Here is an example on how to build a taxonomic structure in JSON: ``` #### Lists from Excel -A list can also be imported from an Excel sheet. The Excel sheet must have the following format (currently only a single -language is supported): +A list can be directly imported from an Excel sheet. The Excel sheet must have the following format: ![img-list-example.png](assets/images/img-list-example.png) In such a case, the Excel file can directly be referenced in the list definition by defining a special list node: ```json { - "name": "fromexcel", + "name": "List-from-excel", "labels": { - "en": "Fromexcel" + "en": "List from an Excel file", + "de": "Liste von einer Excel-Datei" }, "nodes": { "file": "excel-list.xlsx", @@ -1066,9 +1072,7 @@ Example for a resource definition: { "name": "Schule", "super": "Resource", - "labels": { - "de": "Schule" - }, + "labels": {"de": "Schule"}, "cardinalities": [ { "propname": ":schulcode", diff --git a/docs/dsp-tools-excel.md b/docs/dsp-tools-excel.md index 72578c9f2..3e2b6194e 100644 --- a/docs/dsp-tools-excel.md +++ b/docs/dsp-tools-excel.md @@ -11,7 +11,73 @@ create a list from an Excel file. ## Create a DSP-conform XML file from an Excel file [not yet implemented] -## Create flat or hierarchical lists from an Excel file -Lists or controlled vocabularies are sets of fixed terms that are used to characterize objects. Hierarchical lists -correspond to classifications or taxonomies. With dsp-tools a list can be created from an Excel file. The expected -format of the Excel file is described [here](./dsp-tools-create.md#lists-from-excel). +## Create a list from one or several Excel files +With dsp-tools a JSON list can be created from one or several Excel files. The list can then be inserted into a JSON ontology +and uploaded to a DSP server. The expected format of the Excel files is described [here](./dsp-tools-create.md#lists-from-excel). +It is possible to create multilingual lists. In this case, a separate Excel file has to be created for each language. The data +has to be in the first worksheet of the Excel file(s). It is important that all the Excel lists have the same structure. So, +the translation(s) of a label in one Excel sheet has to be in the exact same cell (i.e. with the same cell index) in its own +Excel sheet. + +Only Excel files with file extension `.xlsx` are considered. All Excel files have to be located in the same directory. When +calling the `excel` command, this folder is provided as an argument to the call. The language of the labels has to be provided in +the Excel file's file name after an underline and before the file extension, p.ex. `liste_de.xlsx` would be considered a list with +German (`de`) labels, `list_en.xlsx` a list with English (`en`) labels. The language has to be a valid ISO 639-1 or ISO +639-2 language code. + +The following example shows how to create a JSON list from two Excel files which are in a directory called `lists`. The output is +written to the file `list.json`. + +```bash +dsp-tools excel lists list.json +``` + +The two Excel files `liste_de.xlsx` and `list_en.xlsx` are located in a folder called `lists`. `liste_de.xlsx` contains German +labels for the list, `list_en.xlsx` contains the English labels. + +``` +lists + |__ liste_de.xlsx + |__ list_en.xlsx +``` + +For each list node, the `label`s are read from the Excel files. The language code, provided in the file name, is then used for +the labels. As node `name`, a simplified version of the English label is taken if English is one of the available languages. If +English is not available, one of the other languages is chosen (which one depends on the representation of the file order). If +there are two node names with the same name, an incrementing number is appended to the `name`. + +```JSON +{ + "name": "sand", + "labels": { + "de": "Sand", + "en": "sand" + }, + "nodes": [ + { + "name": "fine-sand", + "labels": { + "de": "Feinsand", + "en": "fine sand" + } + }, + { + "name": "medium-sand", + "labels": { + "de": "Mittelsand", + "en": "medium sand" + } + }, + { + "name": "coarse-sand", + "labels": { + "de": "Grobsand", + "en": "coarse sand" + } + } + ] +}, ... +``` + +After the creation of the list, a validation against the JSON schema for lists is performed. An error message ist printed out if +the list is not valid. Furthermore, it is checked that no two nodes are the same. diff --git a/docs/dsp-tools-usage.md b/docs/dsp-tools-usage.md index b7a6c91ca..c867674f0 100644 --- a/docs/dsp-tools-usage.md +++ b/docs/dsp-tools-usage.md @@ -94,19 +94,26 @@ dsp-tools xmlupload -s https://api.dsl.server.org -u root@example.com -p test -S The description of the expected XML format can be found [here](./dsp-tools-xmlupload.md). -## Convert an Excel file into a JSON file that is compatible with dsp-tools +## Create a JSON list file from one or several Excel files ```bash -dsp-tools excel [options] excel_file.xlsx output_file.json +dsp-tools excel [option] folder_with_excel_files output_file.json ``` -The following options are available: +The following option is available: -- `-S` | `--sheet` _sheetname_: name of the Excel worksheet to use (default: Tabelle1) -- `-s` | `--shortcode` _shortcode_: shortcode of the project (required) -- `-l` | `--listname` _listname_: name to be used for the list and the list definition file (required) -- `-L` | `--label` _label_: label to be used for the list (required) -- `-x` | `--lang` _lang_: language used for the list labels and commentaries (default: en) -- `-v` | `--verbose`: If set, some information about the progress is printed to the console. +- `-l` | `--listname` _listname_: name to be used for the list (filename before last occurrence of `_` is used if omitted) + +The command is used to create a JSON list file from one or several Excel files. It is possible to create multilingual lists. +Therefore, an Excel file for each language has to be provided. The data has to be in the first worksheet of the Excel +file and all Excel files have to be in the same directory. When calling the `excel` command, this directory has to be provided +as an argument to the call. + +The following example shows how to create a JSON list from Excel files in a directory called `lists`. + +```bash +dsp-tools excel lists list.json +``` -The description of the expected Excel format can be found [here](./dsp-tools-create.md#lists-from-excel). +The description of the expected Excel format can be found [here](./dsp-tools-create.md#lists-from-excel). More information +about the usage of this command can be found [here](./dsp-tools-excel.md#create-a-list-from-one-or-several-excel-files). diff --git a/docs/index.md b/docs/index.md index a2038c116..cbde50829 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,6 +20,6 @@ dsp-tools helps you with the following tasks: writes it into a JSON file. - [`dsp-tools xmlupload`](./dsp-tools-usage.md#upload-data-to-a-dsp-server) uploads data from a provided XML file (bulk data import). -- [`dsp-tools excel`](./dsp-tools-usage.md#convert-an-excel-file-into-a-json-file-that-is-compatible-with-dsp-tools) - converts an Excel file into a JSON and/or XML file in order to use it with `dsp-tools create` or `dsp-tools xmlupload` - (not yet implemented) or converts a list from an Excel file into a JSON file which than can be used in an ontology. +- [`dsp-tools excel`](./dsp-tools-usage.md#create-a-json-list-file-from-one-or-several-excel-files) + creates a JSON or XML file from one or several Excel files. The created data can then be uploaded to a DSP server with + `dsp-tools create`. diff --git a/knora/dsp_tools.py b/knora/dsp_tools.py index 98b6b05b0..2d4a1ce6f 100644 --- a/knora/dsp_tools.py +++ b/knora/dsp_tools.py @@ -2,6 +2,7 @@ The code in this file handles the arguments passed by the user from the command line and calls the requested actions. """ import argparse +import datetime import os import sys @@ -12,8 +13,8 @@ from dsplib.utils.onto_create_lists import create_lists from dsplib.utils.onto_create_ontology import create_ontology from dsplib.utils.onto_get import get_ontology -from dsplib.utils.onto_process_excel import list_excel2json -from dsplib.utils.onto_validate import validate_list, validate_ontology +from dsplib.utils.excel_to_json_lists import list_excel2json, validate_list_with_schema +from dsplib.utils.onto_validate import validate_ontology from dsplib.utils.xml_upload import xml_upload @@ -27,57 +28,61 @@ def program(args: list) -> None: Returns: None """ - version = pkg_resources.require("dsp-tools")[0].version + version = pkg_resources.require('dsp-tools')[0].version + now = datetime.datetime.now() # parse the arguments of the command line parser = argparse.ArgumentParser( - description=f"dsp-tools (Version {version}) DaSCH Service Platform data modelling tools (© 2021 by DaSCH).") - - subparsers = parser.add_subparsers(title="Subcommands", description='Valid subcommands are', help='sub-command help') - - parser_create = subparsers.add_parser('create', help='Create ontologies, lists etc.') - parser_create.set_defaults(action="create") - parser_create.add_argument("-s", "--server", type=str, default="http://0.0.0.0:3333", help="URL of the DSP server") - parser_create.add_argument("-u", "--user", default="root@example.com", help="Username for DSP server") - parser_create.add_argument("-p", "--password", default="test", help="The password for login") - parser_create.add_argument("-V", "--validate", action='store_true', - help="Do only validation of JSON, no upload of the ontology") - parser_create.add_argument("-L", "--listfile", type=str, default="lists.json", help="Name of list node informationfile") - parser_create.add_argument("-l", "--lists", action='store_true', help="Only create the lists") - parser_create.add_argument("-v", "--verbose", action="store_true", help="Verbose feedback") - parser_create.add_argument("-d", "--dump", action="store_true", help="dump test files for DSP-API requests") - parser_create.add_argument("datamodelfile", help="path to data model file") - - parser_get = subparsers.add_parser('get', help='Get project/ontology information from server') - parser_get.set_defaults(action="get") - parser_get.add_argument("-u", "--user", default="root@example.com", help="Username for DSP server") - parser_get.add_argument("-p", "--password", default="test", help="The password for login") - parser_get.add_argument("-s", "--server", type=str, default="http://0.0.0.0:3333", help="URL of the DSP server") - parser_get.add_argument("-P", "--project", type=str, help="Shortcode, shortname or iri of project", required=True) - parser_get.add_argument("-v", "--verbose", action="store_true", help="Verbose feedback") - parser_get.add_argument("datamodelfile", help="path to data model file", default="onto.json") - - parser_upload = subparsers.add_parser('xmlupload', help='Upload data from XML file to server') - parser_upload.set_defaults(action="xmlupload") - parser_upload.add_argument("-s", "--server", type=str, default="http://0.0.0.0:3333", help="URL of the DSP server") - parser_upload.add_argument("-u", "--user", type=str, default="root@example.com", help="Username for DSP server") - parser_upload.add_argument("-p", "--password", type=str, default="test", help="The password for login") - parser_upload.add_argument("-V", "--validate", action='store_true', help="Do only validation of XML, no upload of the data") - parser_upload.add_argument("-i", "--imgdir", type=str, default=".", help="Path to folder containing the images") - parser_upload.add_argument("-S", "--sipi", type=str, default="http://0.0.0.0:1024", help="URL of SIPI server") - parser_upload.add_argument("-v", "--verbose", action="store_true", help="Verbose feedback") - parser_upload.add_argument("xmlfile", help="path to xml file containing the data", default="data.xml") - - parser_excel_lists = subparsers.add_parser('excel', help='Create lists JSON from excel files') - parser_excel_lists.set_defaults(action="excel") - parser_excel_lists.add_argument("-S", "--sheet", type=str, help="Name of excel sheet to be used", default="Tabelle1") - parser_excel_lists.add_argument("-s", "--shortcode", type=str, help="Shortcode of project", default="4123") - parser_excel_lists.add_argument("-l", "--listname", type=str, help="Name of list to be created", default="my_list") - parser_excel_lists.add_argument("-L", "--label", type=str, help="Label of list to be created", default="MyList") - parser_excel_lists.add_argument("-x", "--lang", type=str, help="Language for label", default="en") - parser_excel_lists.add_argument("-v", "--verbose", action="store_true", help="Verbose feedback") - parser_excel_lists.add_argument("excelfile", help="Path to the excel file containing the list data", default="lists.xlsx") - parser_excel_lists.add_argument("outfile", help="Path to the output JSON file containing the list data", default="list.json") + description=f'dsp-tools (Version {version}) DaSCH Service Platform data modelling tools (© {now.year} by DaSCH).') + + subparsers = parser.add_subparsers(title='Subcommands', description='Valid subcommands are', help='sub-command help') + + parser_create = subparsers.add_parser('create', help='Upload an ontology and/or list(s) from a JSON file to the DaSCH ' + 'Service Platform') + parser_create.set_defaults(action='create') + parser_create.add_argument('-s', '--server', type=str, default='http://0.0.0.0:3333', help='URL of the DSP server') + parser_create.add_argument('-u', '--user', default='root@example.com', help='Username for DSP server') + parser_create.add_argument('-p', '--password', default='test', help='The password for login') + parser_create.add_argument('-V', '--validate', action='store_true', help='Do only validation of JSON, no upload of the ' + 'ontology') + parser_create.add_argument('-L', '--listfile', type=str, default='lists.json', help='Name of list node informationfile') + parser_create.add_argument('-l', '--lists', action='store_true', help='Upload only the list(s)') + parser_create.add_argument('-v', '--verbose', action='store_true', help='Verbose feedback') + parser_create.add_argument('-d', '--dump', action='store_true', help='dump test files for DSP-API requests') + parser_create.add_argument('datamodelfile', help='path to data model file') + + parser_get = subparsers.add_parser('get', help='Get the ontology (data model) of a project from the DaSCH Service Platform.') + parser_get.set_defaults(action='get') + parser_get.add_argument('-u', '--user', default='root@example.com', help='Username for DSP server') + parser_get.add_argument('-p', '--password', default='test', help='The password for login') + parser_get.add_argument('-s', '--server', type=str, default='http://0.0.0.0:3333', help='URL of the DSP server') + parser_get.add_argument('-P', '--project', type=str, help='Shortcode, shortname or iri of project', required=True) + parser_get.add_argument('-v', '--verbose', action='store_true', help='Verbose feedback') + parser_get.add_argument('datamodelfile', help='Path to the file the ontology should be written to', default='onto.json') + + parser_upload = subparsers.add_parser('xmlupload', help='Upload data from an XML file to the DaSCH Service Platform.') + parser_upload.set_defaults(action='xmlupload') + parser_upload.add_argument('-s', '--server', type=str, default='http://0.0.0.0:3333', help='URL of the DSP server') + parser_upload.add_argument('-u', '--user', type=str, default='root@example.com', help='Username for DSP server') + parser_upload.add_argument('-p', '--password', type=str, default='test', help='The password for login') + parser_upload.add_argument('-V', '--validate', action='store_true', help='Do only validation of XML, no upload of the data') + parser_upload.add_argument('-i', '--imgdir', type=str, default='.', help='Path to folder containing the images') + parser_upload.add_argument('-S', '--sipi', type=str, default='http://0.0.0.0:1024', help='URL of SIPI server') + parser_upload.add_argument('-v', '--verbose', action='store_true', help='Verbose feedback') + parser_upload.add_argument('xmlfile', help='path to xml file containing the data', default='data.xml') + + parser_excel_lists = subparsers.add_parser('excel', help='Create a JSON list from one or multiple Excel files. The JSON ' + 'list can be integrated into a JSON ontology. If the list should ' + 'contain multiple languages, an Excel file has to be used for ' + 'each language. The filenames should contain the language as ' + 'label, p.ex. liste_de.xlsx, list_en.xlsx. The language is then ' + 'taken from the filename. Only files with extension .xlsx are ' + 'considered.') + parser_excel_lists.set_defaults(action='excel') + parser_excel_lists.add_argument('-l', '--listname', type=str, help='Name of the list to be created (filename is taken if ' + 'omitted)', default=None) + parser_excel_lists.add_argument('excelfolder', help='Path to the folder containing the Excel file(s)', default='lists') + parser_excel_lists.add_argument('outfile', help='Path to the output JSON file containing the list data', default='list.json') args = parser.parse_args(args) @@ -85,31 +90,53 @@ def program(args: list) -> None: parser.print_help(sys.stderr) exit(0) - if args.action == "create": + if args.action == 'create': if args.lists: if args.validate: - validate_list(args.datamodelfile) + validate_list_with_schema(args.datamodelfile) else: - create_lists(input_file=args.datamodelfile, lists_file=args.listfile, server=args.server, user=args.user, - password=args.password, verbose=args.verbose, dump=args.dump) + create_lists(input_file=args.datamodelfile, + lists_file=args.listfile, + server=args.server, + user=args.user, + password=args.password, + verbose=args.verbose, + dump=args.dump) else: if args.validate: validate_ontology(args.datamodelfile) else: - create_ontology(input_file=args.datamodelfile, lists_file=args.listfile, server=args.server, user=args.user, - password=args.password, verbose=args.verbose, dump=args.dump if args.dump else False) - elif args.action == "get": - get_ontology(projident=args.project, outfile=args.datamodelfile, server=args.server, user=args.user, - password=args.password, verbose=args.verbose) - elif args.action == "xmlupload": - xml_upload(input_file=args.xmlfile, server=args.server, user=args.user, password=args.password, imgdir=args.imgdir, - sipi=args.sipi, verbose=args.verbose, validate_only=args.validate) - elif args.action == "excel": - list_excel2json(excelpath=args.excelfile, sheetname=args.sheet, shortcode=args.shortcode, listname=args.listname, - label=args.label, lang=args.lang, outfile=args.outfile, verbose=args.verbose) + create_ontology(input_file=args.datamodelfile, + lists_file=args.listfile, + server=args.server, + user=args.user, + password=args.password, + verbose=args.verbose, + dump=args.dump if args.dump else False) + elif args.action == 'get': + get_ontology(projident=args.project, + outfile=args.datamodelfile, + server=args.server, + user=args.user, + password=args.password, + verbose=args.verbose) + elif args.action == 'xmlupload': + xml_upload(input_file=args.xmlfile, + server=args.server, + user=args.user, + password=args.password, + imgdir=args.imgdir, + sipi=args.sipi, + verbose=args.verbose, + validate_only=args.validate) + elif args.action == 'excel': + list_excel2json(listname=args.listname, + excelfolder=args.excelfolder, + outfile=args.outfile) def main(): + """Main entry point of the program as referenced in setup.py""" program(sys.argv[1:]) diff --git a/knora/dsplib/models/permission.py b/knora/dsplib/models/permission.py index b9e0c9bd7..f40fbabfd 100644 --- a/knora/dsplib/models/permission.py +++ b/knora/dsplib/models/permission.py @@ -1,10 +1,8 @@ -from enum import Enum, unique -from typing import List, Set, Dict, Tuple, Optional, Any, Union, Type -from pystrict import strict import re +from enum import Enum, unique +from typing import List, Dict, Optional, Union -from dsplib.models.group import Group -from dsplib.models.helpers import BaseError +from pystrict import strict @unique diff --git a/knora/dsplib/utils/BUILD.bazel b/knora/dsplib/utils/BUILD.bazel index 1806a24e9..c1ee8ae18 100644 --- a/knora/dsplib/utils/BUILD.bazel +++ b/knora/dsplib/utils/BUILD.bazel @@ -5,60 +5,55 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("@knora_py_deps//:requirements.bzl", "requirement") py_library( - name = "onto_commons", + name = "excel_to_json_lists", visibility = ["//visibility:public"], - srcs = ["onto_commons.py"], + srcs = ["excel_to_json_lists.py"], deps = [ - "//knora/dsplib/models:helpers", - "//knora/dsplib/models:listnode", - "//knora/dsplib/models:project", - ], - imports = [".."], + requirement("jsonschema"), + requirement("openpyxl") + ] ) py_library( - name = "onto_create_lists", + name = "expand_all_lists", visibility = ["//visibility:public"], - srcs = ["onto_create_lists.py"], + srcs = ["expand_all_lists.py"], deps = [ - "//knora/dsplib/models:helpers", - "//knora/dsplib/models:connection", - "//knora/dsplib/models:listnode", - "//knora/dsplib/models:project", - ":onto_commons", - requirement("jsonschema"), - ], - imports = [".", ".."], + "//knora/dsplib/utils:excel_to_json_lists" + ] ) py_library( - name = "onto_validate", + name = "onto_create_lists", visibility = ["//visibility:public"], - srcs = ["onto_validate.py"], + srcs = ["onto_create_lists.py"], deps = [ - ":onto_commons", + "//knora/dsplib/models:connection", + "//knora/dsplib/models:listnode", + "//knora/dsplib/models:project", + ":expand_all_lists", + ":onto_validate" ], imports = [".", ".."], ) - py_library( name = "onto_create_ontology", visibility = ["//visibility:public"], srcs = ["onto_create_ontology.py"], deps = [ + "//knora/dsplib/models:connection", + "//knora/dsplib/models:group", "//knora/dsplib/models:helpers", "//knora/dsplib/models:langstring", - "//knora/dsplib/models:group", - "//knora/dsplib/models:user", "//knora/dsplib/models:ontology", + "//knora/dsplib/models:project", "//knora/dsplib/models:propertyclass", "//knora/dsplib/models:resourceclass", - "//knora/dsplib/models:connection", - "//knora/dsplib/models:listnode", - "//knora/dsplib/models:project", - ":onto_commons", - requirement("jsonschema"), + "//knora/dsplib/models:user", + ":expand_all_lists", + ":onto_create_lists", + ":onto_validate" ], imports = [".", ".."], ) @@ -69,10 +64,20 @@ py_library( srcs = ["onto_get.py"], deps = [ "//knora/dsplib/models:connection", - "//knora/dsplib/models:project", "//knora/dsplib/models:listnode", "//knora/dsplib/models:ontology", - requirement("jsonschema"), + "//knora/dsplib/models:project" + ], + imports = [".", ".."], +) + +py_library( + name = "onto_validate", + visibility = ["//visibility:public"], + srcs = ["onto_validate.py"], + deps = [ + ":expand_all_lists", + requirement("jsonschema") ], imports = [".", ".."], ) @@ -84,14 +89,11 @@ py_library( deps = [ "//knora/dsplib/models:connection", "//knora/dsplib/models:group", + "//knora/dsplib/models:permission", "//knora/dsplib/models:project", "//knora/dsplib/models:resource", - "//knora/dsplib/models:value", - "//knora/dsplib/models:permission", "//knora/dsplib/models:sipi", - "//knora/dsplib/models:listnode", - "//knora/dsplib/models:ontology", - requirement("jsonschema"), + requirement("lxml") ], imports = [".", ".."], ) diff --git a/knora/dsplib/utils/excel_to_json_lists.py b/knora/dsplib/utils/excel_to_json_lists.py new file mode 100644 index 000000000..400b51057 --- /dev/null +++ b/knora/dsplib/utils/excel_to_json_lists.py @@ -0,0 +1,340 @@ +"""This module handles all the operations which are used for the creation of JSON lists form Excel files.""" +import csv +import glob +import json +import os +import re +import unicodedata +from typing import List + +import jsonschema +from jsonschema import validate +from openpyxl import load_workbook + +list_of_lists = [] +cell_names = [] + + +def get_values_from_excel(excelfiles: List[str], base_file: str, parentnode: {}, row: int, col: int, preval: List[str]) -> int: + """ + This function calls itself recursively to go through the Excel files. It extracts the cell values and creates the JSON list + file. + + Args: + base_file: File name of the base file + excelfiles: List of Excel files with the values in different languages + parentnode: Name(s) of the parent node(s) of the actual node + row: The index of the actual row of the Excel sheet + col: The index of the actual column of the Excel sheet + preval: List of previous values, needed to check the consistency of the list hierarchy + + Returns: + int: Row index for the next loop (actual row index minus 1) + """ + nodes = [] + currentnode = {} + wb = load_workbook(filename=base_file, read_only=True) + worksheet = wb.worksheets[0] + cell = worksheet.cell(column=col, row=row) + + if col > 1: + # append the cell value of the parent node (which is one value to the left of the actual cell) to the list of previous + # values + preval.append(worksheet.cell(column=col - 1, row=row).value) + + while cell.value: + # check if all predecessors in row (values to the left) are consistent with the values in preval list + for idx, val in enumerate(preval[:-1]): + if val != worksheet.cell(column=idx + 1, row=row).value: + print(f'Inconsistency in Excel list: {val} not equal to {worksheet.cell(column=idx + 1, row=row).value}') + quit() + + # loop through the row until the last (furthest right) value is found + if worksheet.cell(column=col + 1, row=row).value: + row = get_values_from_excel(excelfiles=excelfiles, base_file=base_file, parentnode=currentnode, col=col + 1, row=row, + preval=preval) + + # if value was last in row (no further values to the right), it's a node, continue here + else: + # check if there are duplicate nodes (i.e. identical rows), quit the program if so + new_check_list = preval.copy() + new_check_list.append(cell.value) + + list_of_lists.append(new_check_list) + + if check_list_for_duplicates(list_of_lists): + print('There is at least one duplicate node in the list. Found duplicate:', cell.value) + quit() + + # create a simplified version of the cell value and use it as name of the node + cellname = simplify_name(cell.value) + cell_names.append(cellname) + + # append a number (p.ex. node-name-2) if there are list nodes with identical names + if check_list_for_duplicates(cell_names): + n = cell_names.count(cellname) + if n > 1: + cellname = cellname + '-' + str(n) + + labels_dict = {} + + # read label values from the other Excel files (other languages) + for filename_other_lang in excelfiles: + wb_other_lang = load_workbook(filename=filename_other_lang, read_only=True) + ws_other_lang = wb_other_lang.worksheets[0] + + lang = os.path.splitext(filename_other_lang)[0].split('_')[-1] + labels_dict[lang] = ws_other_lang.cell(column=col, row=row).value + + # create current node from extracted cell values and append it to the nodes list + currentnode = {'name': cellname, 'labels': labels_dict} + + nodes.append(currentnode) + + print(f'Added list node: {cell.value} ({cellname})') + + # go one row down and repeat loop if there is a value + row += 1 + cell = worksheet.cell(column=col, row=row) + + if col > 1: + preval.pop() + + # add the new nodes to the parentnode + parentnode['nodes'] = nodes + + return row - 1 + + +def make_json_list_from_excel(rootnode: {}, excelfiles: List[str]) -> None: + """ + Reads Excel files and makes a JSON list file from them. The JSON can then be used in an ontology that is uploaded to the + DaSCH Service Platform. + + Args: + rootnode: The root node of the JSON list + excelfiles: A list with all the Excel files to be processed + + Returns: + None + + """ + # Define starting point in Excel file + startrow = 1 + startcol = 1 + + # Check if English file is available and take it as base file, take last one from list of Excel files if English is not + # available. The node names are later derived from the labels of the base file. + base_file = '' + + for filename in excelfiles: + base_file = filename + if '_en.xlsx' in os.path.basename(filename): + base_file = filename + break + + get_values_from_excel(excelfiles=excelfiles, base_file=base_file, parentnode=rootnode, row=startrow, col=startcol, preval=[]) + + +def check_list_for_duplicates(list_to_check: list) -> bool: + """ + Checks if the given list contains any duplicate items. + + Args: + list_to_check: A list of items to be checked for duplicates + + Returns: + True if there is a duplicate, false otherwise + + """ + has_duplicates = False + + for item in list_to_check: + if list_to_check.count(item) > 1: + has_duplicates = True + + return has_duplicates + + +def simplify_name(value: str) -> str: + """ + Simplifies a given value in order to use it as node name + + Args: + value: The value to be simplified + + Returns: + str: The simplified value + + """ + simplified_value = str(value).lower() + + # normalize characters (p.ex. ä becomes a) + simplified_value = unicodedata.normalize('NFKD', simplified_value) + + # replace forward slash and whitespace with a dash + simplified_value = re.sub('[/\\s]+', '-', simplified_value) + + # delete all characters which are not letters, numbers or dashes + simplified_value = re.sub('[^A-Za-z0-9\\-]+', '', simplified_value) + + return simplified_value + + +def check_language_code(lang_code: str) -> bool: + """ + Checks if a given language code is valid. The code is valid if it is listed in language-codes-3b2_csv.csv. This + file provides all ISO 639-1 and ISO 639-2 language codes. + + Args: + lang_code: the language code to be checked + + Returns: + True if valid, False if not + + """ + current_dir = os.path.dirname(os.path.realpath(__file__)) + with open(os.path.join(current_dir, 'language-codes-3b2_csv.csv'), 'r') as language_codes_file: + language_codes = csv.reader(language_codes_file, delimiter=',') + for row in language_codes: + if lang_code in row: + return True + return False + + +def make_root_node_from_args(excelfiles: List[str], listname_from_args: str) -> dict: + """ + Creates the root node for the JSON list + + Args: + excelfiles: List of Excel files (names) to be checked + listname_from_args: Listname from arguments provided by the user via the command line + + Returns: + dict: The root node of the list as dictionary (JSON) + + """ + rootnode_labels_dict = {} + listname = listname_from_args + listname_en = '' + + for filename in excelfiles: + basename = os.path.basename(filename) + label, lang_code = os.path.splitext(basename)[0].rsplit('_', 1) + + # check if language code is valid + if not check_language_code(lang_code): + print('Invalid language code is used. Only language codes from ISO 639-1 and ISO 639-2 are accepted.') + quit() + + rootnode_labels_dict[lang_code] = label + listname = label + + if '_en.xlsx' in filename: + listname_en = label + + # if an english list is available use its label as listname + if listname_en: + listname = listname_en + + # if the user provided a listname use it + if listname_from_args: + listname = listname_from_args + + rootnode = {'name': listname, 'labels': rootnode_labels_dict} + + return rootnode + + +def validate_list_with_schema(json_list: str) -> bool: + """ + This function checks if a list is valid according to the schema. + + Args: + json_list (json): the json list to be validated + + Returns: + True if the list passed validation, False otherwise + + """ + current_dir = os.path.dirname(os.path.realpath(__file__)) + with open(os.path.join(current_dir, 'knora-schema-lists-only.json')) as schema: + list_schema = json.load(schema) + + try: + validate(instance=json_list, schema=list_schema) + except jsonschema.exceptions.ValidationError as err: + print(err) + return False + print('List passed schema validation.') + return True + + +def prepare_list_creation(excelfolder: str, listname: str): + """ + Gets the excelfolder parameter and checks the validity of the files. It then makes the root node for the list. + + Args: + excelfolder: path to the folder containing the Excel file(s) + listname: name of the list to be created + + Returns: + rootnode (dict): The rootnode of the list as a dictionary + excel_files (List[str]): list of the Excel files to process + """ + # reset the global variables before list creation starts + global cell_names + global list_of_lists + + list_of_lists = [] + cell_names = [] + + # check if the given folder parameter is actually a folder + if not os.path.isdir(excelfolder): + print(excelfolder, 'is not a directory.') + exit() + + # create a list with all excel files from the path provided by the user + excel_files = [filename for filename in glob.iglob(f'{excelfolder}/*.xlsx') if + not os.path.basename(filename).startswith('~$')] + + # check if all excel_files are actually files + print('Found the following files:') + for file in excel_files: + print(file) + if not os.path.isfile(file): + print(file, 'is not a valid file.') + exit() + + # create root node of list + rootnode = make_root_node_from_args(excel_files, listname) + + return rootnode, excel_files + + +def list_excel2json(listname: str, excelfolder: str, outfile: str): + """ + Takes the arguments from the command line, checks folder and files and starts the process of list creation. + + Args: + listname: name of the list to be created, file name is taken if omitted + excelfolder: path to the folder containing the Excel file(s) + outfile: path to the JSON file the output is written into + + Return: + None + """ + # get the Excel files from the folder and crate the rootnode of the list + rootnode, excel_files = prepare_list_creation(excelfolder, listname) + + # create the list from the Excel files + make_json_list_from_excel(rootnode, excel_files) + + # validate created list with schema + if validate_list_with_schema(json.loads(json.dumps(rootnode, indent=4))): + # write final list to JSON file if list passed validation + with open(outfile, 'w', encoding='utf-8') as fp: + json.dump(rootnode, fp, indent=4, sort_keys=False, ensure_ascii=False) + print('List was created successfully and written to file:', outfile) + else: + print('List is not valid according to schema.') diff --git a/knora/dsplib/utils/expand_all_lists.py b/knora/dsplib/utils/expand_all_lists.py new file mode 100644 index 000000000..04789f55c --- /dev/null +++ b/knora/dsplib/utils/expand_all_lists.py @@ -0,0 +1,35 @@ +from typing import List, Dict + +from knora.dsplib.utils.excel_to_json_lists import prepare_list_creation, make_json_list_from_excel + + +def expand_lists_from_excel(data_model: Dict) -> List[str]: + """ + Gets all lists from an ontology and expands them to json if they are only referenced via an Excel file + + Args: + data_model: The data model (json) the lists are read from + + Returns: + A list of all expanded lists. It can be added to the root node of an ontology as list section. + """ + # create and add lists from Excel references to the ontology + lists = data_model['project'].get('lists') + new_lists = [] + if lists is not None: + for rootnode in lists: + # check if the folder parameter is used + if rootnode.get('nodes') is not None and isinstance(rootnode['nodes'], dict) and rootnode['nodes'].get( + 'folder') is not None: + # get the Excel files from the folder and crate the rootnode of the list + excel_folder = rootnode['nodes']['folder'] + rootnode, excel_files = prepare_list_creation(excel_folder, rootnode.get('name')) + + # create the list from the Excel files + make_json_list_from_excel(rootnode, excel_files) + + new_lists.append(rootnode) + else: + new_lists.append(rootnode) + + return new_lists diff --git a/knora/dsplib/utils/knora-schema-lists-only.json b/knora/dsplib/utils/knora-schema-lists-only.json new file mode 100644 index 000000000..f01b56b1a --- /dev/null +++ b/knora/dsplib/utils/knora-schema-lists-only.json @@ -0,0 +1,75 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema-list-only#", + "$id": "http://knora.org/pyknora/ontology/knora-schema-list-only.json", + "title": "Knora JSON schema for lists only", + "description": "JSON schema for lists used in Knora ontologies", + "definitions": { + "label": { + "type": "object", + "patternProperties": { + "^(en|de|fr|it)": { + "type": "string" + } + }, + "additionalProperties": false + }, + "comment": { + "type": "object", + "patternProperties": { + "^(en|de|fr|it)": { + "type": "string" + } + }, + "additionalProperties": false + }, + "node": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "labels": { + "$ref": "#/definitions/label" + }, + "comments": { + "$ref": "#/definitions/comment" + }, + "nodes": { + "type": "array", + "items": { + "$ref": "#/definitions/node" + } + } + }, + "required": [ + "name", + "labels" + ], + "additionalProperties": false + } + }, + + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "labels": { + "$ref": "#/definitions/label" + }, + "comments": { + "$ref": "#/definitions/comment" + }, + "nodes": { + "type": "array", + "items": { + "$ref": "#/definitions/node" + } + } + }, + "required": [ + "name", + "nodes" + ], + "additionalProperties": false +} diff --git a/knora/dsplib/utils/language-codes-3b2_csv.csv b/knora/dsplib/utils/language-codes-3b2_csv.csv new file mode 100644 index 000000000..791eed085 --- /dev/null +++ b/knora/dsplib/utils/language-codes-3b2_csv.csv @@ -0,0 +1,185 @@ +ISO-639-2,ISO-639-1,English +aar,aa,Afar +abk,ab,Abkhazian +afr,af,Afrikaans +aka,ak,Akan +alb,sq,Albanian +amh,am,Amharic +ara,ar,Arabic +arg,an,Aragonese +arm,hy,Armenian +asm,as,Assamese +ava,av,Avaric +ave,ae,Avestan +aym,ay,Aymara +aze,az,Azerbaijani +bak,ba,Bashkir +bam,bm,Bambara +baq,eu,Basque +bel,be,Belarusian +ben,bn,Bengali +bih,bh,Bihari languages +bis,bi,Bislama +bos,bs,Bosnian +bre,br,Breton +bul,bg,Bulgarian +bur,my,Burmese +cat,ca,Catalan; Valencian +cha,ch,Chamorro +che,ce,Chechen +chi,zh,Chinese +chu,cu,Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic +chv,cv,Chuvash +cor,kw,Cornish +cos,co,Corsican +cre,cr,Cree +cze,cs,Czech +dan,da,Danish +div,dv,Divehi; Dhivehi; Maldivian +dut,nl,Dutch; Flemish +dzo,dz,Dzongkha +eng,en,English +epo,eo,Esperanto +est,et,Estonian +ewe,ee,Ewe +fao,fo,Faroese +fij,fj,Fijian +fin,fi,Finnish +fre,fr,French +fry,fy,Western Frisian +ful,ff,Fulah +geo,ka,Georgian +ger,de,German +gla,gd,Gaelic; Scottish Gaelic +gle,ga,Irish +glg,gl,Galician +glv,gv,Manx +gre,el,"Greek, Modern (1453-)" +grn,gn,Guarani +guj,gu,Gujarati +hat,ht,Haitian; Haitian Creole +hau,ha,Hausa +heb,he,Hebrew +her,hz,Herero +hin,hi,Hindi +hmo,ho,Hiri Motu +hrv,hr,Croatian +hun,hu,Hungarian +ibo,ig,Igbo +ice,is,Icelandic +ido,io,Ido +iii,ii,Sichuan Yi; Nuosu +iku,iu,Inuktitut +ile,ie,Interlingue; Occidental +ina,ia,Interlingua (International Auxiliary Language Association) +ind,id,Indonesian +ipk,ik,Inupiaq +ita,it,Italian +jav,jv,Javanese +jpn,ja,Japanese +kal,kl,Kalaallisut; Greenlandic +kan,kn,Kannada +kas,ks,Kashmiri +kau,kr,Kanuri +kaz,kk,Kazakh +khm,km,Central Khmer +kik,ki,Kikuyu; Gikuyu +kin,rw,Kinyarwanda +kir,ky,Kirghiz; Kyrgyz +kom,kv,Komi +kon,kg,Kongo +kor,ko,Korean +kua,kj,Kuanyama; Kwanyama +kur,ku,Kurdish +lao,lo,Lao +lat,la,Latin +lav,lv,Latvian +lim,li,Limburgan; Limburger; Limburgish +lin,ln,Lingala +lit,lt,Lithuanian +ltz,lb,Luxembourgish; Letzeburgesch +lub,lu,Luba-Katanga +lug,lg,Ganda +mac,mk,Macedonian +mah,mh,Marshallese +mal,ml,Malayalam +mao,mi,Maori +mar,mr,Marathi +may,ms,Malay +mlg,mg,Malagasy +mlt,mt,Maltese +mon,mn,Mongolian +nau,na,Nauru +nav,nv,Navajo; Navaho +nbl,nr,"Ndebele, South; South Ndebele" +nde,nd,"Ndebele, North; North Ndebele" +ndo,ng,Ndonga +nep,ne,Nepali +nno,nn,"Norwegian Nynorsk; Nynorsk, Norwegian" +nob,nb,"Bokmål, Norwegian; Norwegian Bokmål" +nor,no,Norwegian +nya,ny,Chichewa; Chewa; Nyanja +oci,oc,Occitan (post 1500) +oji,oj,Ojibwa +ori,or,Oriya +orm,om,Oromo +oss,os,Ossetian; Ossetic +pan,pa,Panjabi; Punjabi +per,fa,Persian +pli,pi,Pali +pol,pl,Polish +por,pt,Portuguese +pus,ps,Pushto; Pashto +que,qu,Quechua +roh,rm,Romansh +rum,ro,Romanian; Moldavian; Moldovan +run,rn,Rundi +rus,ru,Russian +sag,sg,Sango +san,sa,Sanskrit +sin,si,Sinhala; Sinhalese +slo,sk,Slovak +slv,sl,Slovenian +sme,se,Northern Sami +smo,sm,Samoan +sna,sn,Shona +snd,sd,Sindhi +som,so,Somali +sot,st,"Sotho, Southern" +spa,es,Spanish; Castilian +srd,sc,Sardinian +srp,sr,Serbian +ssw,ss,Swati +sun,su,Sundanese +swa,sw,Swahili +swe,sv,Swedish +tah,ty,Tahitian +tam,ta,Tamil +tat,tt,Tatar +tel,te,Telugu +tgk,tg,Tajik +tgl,tl,Tagalog +tha,th,Thai +tib,bo,Tibetan +tir,ti,Tigrinya +ton,to,Tonga (Tonga Islands) +tsn,tn,Tswana +tso,ts,Tsonga +tuk,tk,Turkmen +tur,tr,Turkish +twi,tw,Twi +uig,ug,Uighur; Uyghur +ukr,uk,Ukrainian +urd,ur,Urdu +uzb,uz,Uzbek +ven,ve,Venda +vie,vi,Vietnamese +vol,vo,Volapük +wel,cy,Welsh +wln,wa,Walloon +wol,wo,Wolof +xho,xh,Xhosa +yid,yi,Yiddish +yor,yo,Yoruba +zha,za,Zhuang; Chuang +zul,zu,Zulu diff --git a/knora/dsplib/utils/onto_commons.py b/knora/dsplib/utils/onto_commons.py deleted file mode 100644 index 5db312ca5..000000000 --- a/knora/dsplib/utils/onto_commons.py +++ /dev/null @@ -1,115 +0,0 @@ -from typing import List, Set, Dict, Tuple, Optional -from openpyxl import load_workbook, worksheet - -from ..models.connection import Connection -from ..models.project import Project -from ..models.listnode import ListNode -from ..models.langstring import Languages, LangString -from ..models.helpers import BaseError - -from pprint import pprint - -def list_creator(con: Connection, project: Project, parent_node: ListNode, nodes: List[dict]): - nodelist = [] - for node in nodes: - newnode = ListNode( - con=con, - project=project, - label=node["labels"], - comments=node.get("comments"), - name=node["name"], - parent=parent_node - ).create() - if node.get('nodes') is not None: - subnodelist = list_creator(con, project, newnode, node['nodes']) - nodelist.append({newnode.name: {"id": newnode.id, 'nodes': subnodelist}}) - else: - nodelist.append({newnode.name: {"id": newnode.id}}) - return nodelist - - -def validate_list_from_excel(filepath: str, - sheetname: str, - startrow: int = 1, - startcol: int = 1, - verbose: bool = False): - - def analyse_level(ws: worksheet, level: int, row: int, col: int, preval: List[str]) ->int: - cell = ws.cell(column=col, row=row) - if col > 1: - preval.append(ws.cell(column=col - 1, row=row).value) - while cell.value: - for idx, val in enumerate(preval[:-1]): - if val != ws.cell(column=idx + 1, row=row).value: - raise BaseError(f"Inconsistency in Excel list! {val} not equal {ws.cell(column=idx + 1, row=row).value}") - if ws.cell(column=col + 1, row=row).value: - row = analyse_level(ws=ws, level=level + 1, col=col + 1, row=row, preval=preval) - if not ws.cell(column=col, row=row).value: - if col > 1: - preval.pop() - return row - else: - if verbose: - print(f"Node on level={level}, value={cell.value} ok...") - row += 1 - cell = ws.cell(column=col, row=row) - if col > 1: - preval.pop() - return row - 1 - - wb = load_workbook(filename=filepath, read_only=True) - ws = wb[sheetname] - tmp = [] - analyse_level(ws=ws, level=1, row=startrow, col=startcol, preval=tmp) - - -def json_list_from_excel(rootnode: {}, filepath: str, sheetname: str, startrow: int = 1, startcol: int = 1): - - names: Set[str] = set() - - def analyse_level(ws: worksheet, parentnode: {}, row: int, col: int, preval: List[str]) ->int: - nodes: [] = [] - currentnode: {} - - cell = ws.cell(column=col, row=row) - if col > 1: - preval.append(ws.cell(column=col - 1, row=row).value) - while cell.value: - for idx, val in enumerate(preval[:-1]): - if val != ws.cell(column=idx + 1, row=row).value: - raise BaseError(f"Inconsistency in Excel list! {val} not equal {ws.cell(column=idx + 1, row=row).value}") - if ws.cell(column=col + 1, row=row).value: - row = analyse_level(ws=ws, parentnode=currentnode, col=col + 1, row=row, preval=preval) - if not ws.cell(column=col, row=row).value: - if col > 1: - preval.pop() - parentnode["nodes"] = nodes - return row - else: - tmpstr = cell.value - tmpstr = tmpstr.split(" ") - tmpstr = [w.title() for w in tmpstr] - tmpstr = "".join(tmpstr) - tmpstr = tmpstr[0].lower() + tmpstr[1:] - while tmpstr in names: - tmpstr = tmpstr + "_" - names.add(tmpstr) - currentnode = { - "name": tmpstr, - "labels": {"en": cell.value} - } - nodes.append(currentnode) - print(f"Adding list node: value={cell.value}") - row += 1 - cell = ws.cell(column=col, row=row) - if col > 1: - preval.pop() - parentnode["nodes"] = nodes - return row - 1 - - wb = load_workbook(filename=filepath, read_only=True) - ws = wb[sheetname] - tmp = [] - analyse_level(ws=ws, parentnode=rootnode, row=startrow, col=startcol, preval=tmp) - - diff --git a/knora/dsplib/utils/onto_create_lists.py b/knora/dsplib/utils/onto_create_lists.py index f84b97d68..f62ccbb95 100644 --- a/knora/dsplib/utils/onto_create_lists.py +++ b/knora/dsplib/utils/onto_create_lists.py @@ -1,104 +1,101 @@ -import os import json -from jsonschema import validate +from typing import List -from ..models.helpers import Actions, BaseError, Context, Cardinality +from .expand_all_lists import expand_lists_from_excel +from .onto_validate import validate_ontology from ..models.connection import Connection -from ..models.project import Project from ..models.listnode import ListNode -from .onto_commons import list_creator, validate_list_from_excel, json_list_from_excel +from ..models.project import Project + + +def list_creator(con: Connection, project: Project, parent_node: ListNode, nodes: List[dict]): + """ + Creates the list on the DSP server + Args: + con: The connection to the DSP server + project: The project which the lists should be added + parent_node: The root node of the list + nodes: List of nodes the list is made of -def create_lists(input_file: str, lists_file: str, server: str, user: str, password: str, verbose: bool, dump: bool = False) -> bool: - current_dir = os.path.dirname(os.path.realpath(__file__)) + Returns: + The list of all nodes with their names and respective IDs + """ + nodelist = [] + for node in nodes: + new_node = ListNode(con=con, project=project, label=node["labels"], comments=node.get("comments"), name=node["name"], + parent=parent_node).create() + if node.get('nodes') is not None: + subnode_list = list_creator(con, project, new_node, node['nodes']) + nodelist.append({new_node.name: {"id": new_node.id, 'nodes': subnode_list}}) + else: + nodelist.append({new_node.name: {"id": new_node.id}}) + return nodelist - # let's read the schema for the data model definition - with open(os.path.join(current_dir, 'knora-schema-lists.json')) as s: - schema = json.load(s) - # read the data model definition + +def create_lists(input_file: str, lists_file: str, server: str, user: str, password: str, verbose: bool, dump: bool = False): + """ + Creates the lists on the DSP server + + Args: + input_file: Path to the json data model file + lists_file: Output file for the list node names and their respective IRI + server: URL of the DSP server + user: Username (e-mail) for the DSP server, has to have the permissions to create an ontology + password: Password of the user + verbose: Verbose output if True + dump: ??? + + Returns: + list_root_nodes: Dictionary of node names and their respective IRI + """ + # read the ontology from the input file with open(input_file) as f: - datamodel = json.load(f) + onto_json_str = f.read() - # - # now let's see if there are any lists defined as reference to excel files - # - lists = datamodel["project"].get('lists') - if lists is not None: - newlists: [] = [] - for rootnode in lists: - if rootnode.get("nodes") is not None and isinstance(rootnode["nodes"], dict) and rootnode["nodes"].get("file") is not None: - newroot = { - "name": rootnode.get("name"), - "labels": rootnode.get("labels"), - "comments": rootnode.get("comments") - } - startrow = 1 if rootnode["nodes"].get("startrow") is None else rootnode["nodes"]["startrow"] - startcol = 1 if rootnode["nodes"].get("startcol") is None else rootnode["nodes"]["startcol"] - json_list_from_excel(rootnode=newroot, - filepath=rootnode["nodes"]["file"], - sheetname=rootnode["nodes"]["worksheet"], - startrow=startrow, - startcol=startcol) - newlists.append(newroot) - else: - newlists.append(rootnode) - datamodel["project"]["lists"] = newlists - - # validate the data model definition in order to be sure that it is correct - validate(datamodel, schema) + data_model = json.loads(onto_json_str) - if verbose: - print("Data model is syntactically correct and passed validation!") + # expand all lists referenced in the list section of the data model + new_lists = expand_lists_from_excel(data_model) + + # add the newly created lists from Excel to the ontology + data_model['project']['lists'] = new_lists + + # validate the ontology + if validate_ontology(data_model): + pass + else: + quit() - # # Connect to the DaSCH Service Platform API - # con = Connection(server) con.login(user, password) if dump: con.start_logging() - # -------------------------------------------------------------------------- - # let's read the prefixes of external ontologies that may be used - # - context = Context(datamodel["prefixes"]) - - # -------------------------------------------------------------------------- - # Let's get the project which must exist - # - project = Project( - con=con, - shortcode=datamodel["project"]["shortcode"], - ).read() + # get the project which must exist + project = Project(con=con, shortcode=data_model['project']['shortcode'], ).read() assert project is not None - # -------------------------------------------------------------------------- - # now let's create the lists - # + # create the lists if verbose: - print("Creating lists...") - lists = datamodel["project"].get('lists') - listrootnodes = {} + print('Create lists...') + + lists = data_model['project'].get('lists') + list_root_nodes = {} if lists is not None: for rootnode in lists: - if verbose is not None: - print(" Creating list:" + rootnode['name']) - root_list_node = ListNode( - con=con, - project=project, - label=rootnode['labels'], - #comment=rootnode.get('comments'), - name=rootnode['name'] - ).create() + if verbose: + print(' Create list:' + rootnode['name']) + root_list_node = ListNode(con=con, project=project, label=rootnode['labels'], # comment=rootnode.get('comments'), + name=rootnode['name']).create() if rootnode.get('nodes') is not None: - listnodes = list_creator(con, project, root_list_node, rootnode['nodes']) - listrootnodes[rootnode['name']] = { - "id": root_list_node.id, - "nodes": listnodes - } - - with open(lists_file, 'w', encoding="utf-8") as fp: - json.dump(listrootnodes, fp, indent=3, sort_keys=True) - print(f"The definitions of the node-id's can be found in \"{lists_file}\"!") - return True + list_nodes = list_creator(con, project, root_list_node, rootnode['nodes']) + list_root_nodes[rootnode['name']] = {'id': root_list_node.id, 'nodes': list_nodes} + + with open(lists_file, 'w', encoding='utf-8') as fp: + json.dump(list_root_nodes, fp, indent=3, sort_keys=True) + print(f'The IRI for the created nodes can be found in {lists_file}.') + + return list_root_nodes diff --git a/knora/dsplib/utils/onto_create_ontology.py b/knora/dsplib/utils/onto_create_ontology.py index 27d784d25..b1d73b9e0 100644 --- a/knora/dsplib/utils/onto_create_ontology.py +++ b/knora/dsplib/utils/onto_create_ontology.py @@ -1,31 +1,32 @@ -import os -from typing import List, Set, Dict, Tuple, Optional +"""This module handles the ontology creation and upload to a DSP server. This includes the creation and upload of lists.""" import json -from jsonschema import validate +from typing import Dict, List, Optional, Set -from ..models.helpers import Actions, BaseError, Context, Cardinality -from ..models.langstring import Languages, LangStringParam, LangString +from .expand_all_lists import expand_lists_from_excel +from .onto_create_lists import create_lists +from .onto_validate import validate_ontology from ..models.connection import Connection -from ..models.project import Project -from ..models.listnode import ListNode from ..models.group import Group -from ..models.user import User +from ..models.helpers import BaseError, Cardinality, Context +from ..models.langstring import LangString from ..models.ontology import Ontology +from ..models.project import Project from ..models.propertyclass import PropertyClass from ..models.resourceclass import ResourceClass +from ..models.user import User -from .onto_commons import list_creator, validate_list_from_excel, json_list_from_excel - -from pprint import pprint def login(server: str, user: str, password: str) -> Connection: """ - Make a login and return the active Connection. + Logs in and returns the active connection + + Args: + server: URL of the DSP server to connect to + user: Username (e-mail) + password: Password of the user - :param server: URl of the server to connect to - :param user: A valid username - :param password: The password - :return: Connection instance + Return: + Connection instance """ con = Connection(server) con.login(user, password) @@ -39,167 +40,97 @@ def create_ontology(input_file: str, password: str, verbose: bool, dump: bool) -> bool: - with open(input_file) as f: - jsonstr = f.read() - - con = login(server=server, user=user, password=password) - datapath = os.path.dirname(input_file) - create_ontology_from_string(con=con, - jsonstr=jsonstr, - exceldir=datapath, - lists_file=lists_file, - verbose=verbose, - dump=dump) - -def create_ontology_from_string(con: Connection, - jsonstr: str, - exceldir: Optional[str], - lists_file: Optional[str], - verbose: bool, - dump: bool) -> bool: - current_dir = os.path.dirname(os.path.realpath(__file__)) - - # let's read the schema for the data model definition - with open(os.path.join(current_dir, 'knora-schema.json')) as s: - schema = json.load(s) + """ + Creates the ontology from a json input file on a DSP server + Args: + input_file: The input json file from which the ontology should be created + lists_file: The file which the output (list node ID) is written to + server: The DSP server which the ontology should be created on + user: The user which the ontology should be created with + password: The password for the user + verbose: Prints some more information + dump: Dumps test files (json) for DSP API requests if True - # read the data model definition - datamodel = json.loads(jsonstr) + Returns: + True if successful + """ + # read the ontology from the input file + with open(input_file) as f: + onto_json_str = f.read() - # - # now let's see if there are any lists defined as reference to excel files - # - lists = datamodel["project"].get('lists') - if lists is not None: - newlists: [] = [] - for rootnode in lists: - if rootnode.get("nodes") is not None and isinstance(rootnode["nodes"], dict) and rootnode["nodes"].get("file") is not None: - newroot = { - "name": rootnode.get("name"), - "labels": rootnode.get("labels"), - } - if rootnode.get("comments") is not None: - newroot["comments"] = rootnode["comments"] + data_model = json.loads(onto_json_str) - startrow = 1 if rootnode["nodes"].get("startrow") is None else rootnode["nodes"]["startrow"] - startcol = 1 if rootnode["nodes"].get("startcol") is None else rootnode["nodes"]["startcol"] - # - # determine where to find the excel file... - # - excelpath = rootnode["nodes"]["file"] - if excelpath[0] != '/' and exceldir is not None: - excelpath = os.path.join(exceldir, excelpath) + # expand all lists referenced in the list section of the data model + new_lists = expand_lists_from_excel(data_model) - json_list_from_excel(rootnode=newroot, - filepath=excelpath, - sheetname=rootnode["nodes"]["worksheet"], - startrow=startrow, - startcol=startcol) - newlists.append(newroot) - else: - newlists.append(rootnode) - datamodel["project"]["lists"] = newlists + # add the newly created lists from Excel to the ontology + data_model['project']['lists'] = new_lists - # validate the data model definition in order to be sure that it is correct - validate(datamodel, schema) + # validate the ontology + if validate_ontology(data_model): + pass + else: + quit() - if verbose: - print("Data model is syntactically correct and passed validation!") + # make the connection to the server + con = login(server=server, + user=user, + password=password) if dump: con.start_logging() - # -------------------------------------------------------------------------- - # let's read the prefixes of external ontologies that may be used - # - context = Context(datamodel["prefixes"]) + # read the prefixes of external ontologies that may be used + context = Context(data_model["prefixes"]) - # -------------------------------------------------------------------------- - # Let's create the project... - # + # create or update the project project = None try: - # we try to read the project to see if it's existing.... - project = Project( - con=con, - shortcode=datamodel["project"]["shortcode"], - ).read() - # - # we got it, update the project data if necessary... - # - if project.shortname != datamodel["project"]["shortname"]: - project.shortname = datamodel["project"]["shortname"] - if project.longname != datamodel["project"]["longname"]: - project.longname == datamodel["project"]["longname"] - project.description = datamodel["project"].get("descriptions") - project.keywords = set(datamodel["project"].get("keywords")) - nproject = project.update() - if nproject is not None: - project = nproject + # try to read the project to check if it exists + project = Project(con=con, shortcode=data_model["project"]["shortcode"]).read() + + # update the project with data from data_model + if project.shortname != data_model["project"]["shortname"]: + project.shortname = data_model["project"]["shortname"] + if project.longname != data_model["project"]["longname"]: + project.longname == data_model["project"]["longname"] + project.description = data_model["project"].get("descriptions") + project.keywords = set(data_model["project"].get("keywords")) + updated_project = project.update() + if updated_project is not None: + project = updated_project if verbose: print("Modified project:") project.print() except: - # - # The project doesn't exist yet – let's create it - # + # create the project if it does not exist try: - project = Project( - con=con, - shortcode=datamodel["project"]["shortcode"], - shortname=datamodel["project"]["shortname"], - longname=datamodel["project"]["longname"], - description=LangString(datamodel["project"].get("descriptions")), - keywords=set(datamodel["project"].get("keywords")), - selfjoin=False, - status=True - ).create() + project = Project(con=con, + shortcode=data_model["project"]["shortcode"], + shortname=data_model["project"]["shortname"], + longname=data_model["project"]["longname"], + description=LangString(data_model["project"].get("descriptions")), + keywords=set(data_model["project"].get("keywords")), + selfjoin=False, + status=True).create() except BaseError as err: - print("Creating project failed: " + err.message) + print("Creating project failed: ", err.message) return False if verbose: print("Created project:") project.print() assert project is not None - # -------------------------------------------------------------------------- - # now let's create the lists - # - if verbose: - print("Creating lists...") - lists = datamodel["project"].get('lists') - listrootnodes = {} - if lists is not None: - for rootnode in lists: - if verbose is not None: - print(" Creating list:" + rootnode['name']) - root_list_node = ListNode( - con=con, - project=project, - label=rootnode['labels'], - comments=rootnode.get('comments'), - name=rootnode['name'] - ).create() - listnodes = list_creator(con, project, root_list_node, rootnode['nodes']) - listrootnodes[rootnode['name']] = { - "id": root_list_node.id, - "nodes": listnodes - } - - if lists_file is not None: - with open(lists_file, 'w', encoding="utf-8") as fp: - json.dump(listrootnodes, fp, indent=3, sort_keys=True) - print("The definitions of the node-id's can be found in \"{}\"!".format('lists.json')) + # create the lists + list_root_nodes = create_lists(input_file, lists_file, server, user, password, verbose) - # -------------------------------------------------------------------------- - # now let's add the groups (if there are groups defined...) - # + # create the groups if verbose: - print("Adding groups...") + print("Create groups...") new_groups = {} - groups = datamodel["project"].get('groups') + groups = data_model["project"].get('groups') if groups is not None: for group in groups: try: @@ -210,21 +141,19 @@ def create_ontology_from_string(con: Connection, status=group["status"] if group.get("status") is not None else True, selfjoin=group["selfjoin"] if group.get("selfjoin") is not None else False).create() except BaseError as err: - print("Creating group failed: " + err.message) + print("Creating group has failed: ", err.message) return False new_groups[new_group.name] = new_group if verbose: print("Groups:") - new_group.print() - #project.set_default_permissions(new_group.id) - # -------------------------------------------------------------------------- - # now let's add the users (if there are users defined...) - # + new_group.print() # project.set_default_permissions(new_group.id) + + # create the users if verbose: - print("Adding users...") + print("Create users...") all_groups: List[Group] = [] all_projects: List[Project] = [] - users = datamodel["project"].get('users') + users = data_model["project"].get('users') if users is not None: for user in users: sysadmin = False @@ -256,10 +185,8 @@ def create_ontology_from_string(con: Connection, project_infos: Dict[str, bool] = {} for projectname in user["projects"]: - # - # now we determine the project memberships of the user + # determine the project memberships of the user # projectname has the form [projectname]:"member"|"admin" (projectname omitted = current project) - # tmp = projectname.split(':') assert len(tmp) == 2 if tmp[0]: @@ -276,21 +203,21 @@ def create_ontology_from_string(con: Connection, project_infos[in_project.id] = True else: project_infos[in_project.id] = False - user_existing = False; + user_existing = False tmp_user = None try: - tmp_user = User(con, username=user["username"]).read() + tmp_user = User(con, + username=user["username"]).read() except BaseError as err: pass if tmp_user is None: try: - tmp_user = User(con, email=user["email"]).read() + tmp_user = User(con, + email=user["email"]).read() except BaseError as err: pass if tmp_user: - # - # The user is already in the database – let's update its settings - # + # if the user exists already, update his settings if tmp_user.username != user["username"]: tmp_user.username = user["username"] if tmp_user.email != user["email"]: @@ -311,12 +238,11 @@ def create_ontology_from_string(con: Connection, tmp_user.update() except BaseError as err: tmp_user.print() - print("Updating user failed: " + err.message) + print("Updating user failed:", err.message) return False - # - # now we update group and project membership - # Note: we do NOT remove any mambership here, we just add! - # + + # update group and project membership + # Note: memberships are NOT removed here, just added tmp_in_groups = tmp_user.in_groups add_groups = group_ids - tmp_in_groups for g in add_groups: @@ -329,57 +255,47 @@ def create_ontology_from_string(con: Connection, continue User.addToProject(p[0], p[1]) else: - # - # The user does not exist yet, let's create a new one - # + # if the user does not exist yet, create him try: - new_user = User( - con=con, - username=user["username"], - email=user["email"], - givenName=user["givenName"], - familyName=user["familyName"], - password=user["password"], - status=user["status"] if user.get("status") is not None else True, - lang=user["lang"] if user.get("lang") is not None else "en", - sysadmin=sysadmin, - in_projects=project_infos, - in_groups=group_ids - ).create() + new_user = User(con=con, + username=user["username"], + email=user["email"], + givenName=user["givenName"], + familyName=user["familyName"], + password=user["password"], + status=user["status"] if user.get("status") is not None else True, + lang=user["lang"] if user.get("lang") is not None else "en", + sysadmin=sysadmin, + in_projects=project_infos, + in_groups=group_ids).create() except BaseError as err: - print("Creating user failed: " + err.message) + print("Creating user failed:", err.message) return False if verbose: print("New user:") new_user.print() - # -------------------------------------------------------------------------- - # now let's create the ontologies - # - ontologies = datamodel["project"]["ontologies"] + # create the ontologies + if verbose: + print("Create ontologies...") + ontologies = data_model["project"]["ontologies"] for ontology in ontologies: - newontology = Ontology( - con=con, - project=project, - label=ontology["label"], - name=ontology["name"] - ).create() + newontology = Ontology(con=con, + project=project, + label=ontology["label"], + name=ontology["name"]).create() last_modification_date = newontology.lastModificationDate if verbose: print("Created empty ontology:") newontology.print() - # - # add prefixes defined in json file... - # + # add the prefixes defined in the json file for prefix, iri in context: if not prefix in newontology.context: s = iri.iri + ("#" if iri.hashtag else "") newontology.context.add_context(prefix, s) - # - # First we create the empty resource classes - # + # create the empty resource classes resclasses = ontology["resources"] newresclasses: Dict[str, ResourceClass] = {} for resclass in resclasses: @@ -392,54 +308,46 @@ def create_ontology_from_string(con: Connection, if rescomment is not None: rescomment = LangString(rescomment) try: - last_modification_date, newresclass = ResourceClass( - con=con, - context=newontology.context, - ontology_id=newontology.id, - name=resname, - superclasses=super_classes, - label=reslabel, - comment=rescomment - ).create(last_modification_date) + last_modification_date, newresclass = ResourceClass(con=con, + context=newontology.context, + ontology_id=newontology.id, + name=resname, + superclasses=super_classes, + label=reslabel, + comment=rescomment).create(last_modification_date) newontology.lastModificationDate = last_modification_date except BaseError as err: - print("Creating resource class failed: " + err.message) + print("Creating resource class failed:", err.message) exit(105) newresclasses[newresclass.id] = newresclass if verbose is not None: print("New resource class:") newresclass.print() - # - # Then we create the property classes - # + # create the property classes propclasses = ontology["properties"] newpropclasses: Dict[str, ResourceClass] = {} for propclass in propclasses: propname = propclass.get("name") proplabel = LangString(propclass.get("labels")) - # # get the super-property/ies if defined. Valid forms are: # - "prefix:superproperty" : fully qualified name of property in another ontology. The prefix has to # be defined in the prefixes part. # - "superproperty" : Use of super-property defined in the knora-api ontology - # if omitted, automatically "knora-api:hasValue" is assumed - # + # if omitted, "knora-api:hasValue" is assumed if propclass.get("super") is not None: super_props = list(map(lambda a: a if ':' in a else "knora-api:" + a, propclass["super"])) else: super_props = ["knora-api:hasValue"] - # - # now we get the "object" if defined. Valid forms are: + # get the "object" if defined. Valid forms are: # - "prefix:object_name" : fully qualified object. The prefix has to be defined in the prefixes part. # - ":object_name" : The object is defined in the current ontology. # - "object_name" : The object is defined in "knora-api" - # if propclass.get("object") is not None: tmp = propclass["object"].split(':') if len(tmp) > 1: if tmp[0]: - object = propclass["object"] # fully qualified name + object = propclass["object"] # fully qualified name else: newontology.print() object = newontology.name + ':' + tmp[1] @@ -455,38 +363,34 @@ def create_ontology_from_string(con: Connection, gui_element = propclass.get("gui_element") gui_attributes = propclass.get("gui_attributes") if gui_attributes is not None and gui_attributes.get("hlist") is not None: - gui_attributes['hlist'] = "<" + listrootnodes[gui_attributes['hlist']]["id"] + ">" + gui_attributes['hlist'] = "<" + list_root_nodes[gui_attributes['hlist']]["id"] + ">" propcomment = propclass.get("comment") if propcomment is not None: propcomment = LangString(propcomment) else: propcomment = "no comment given" try: - last_modification_date, newpropclass = PropertyClass( - con=con, - context=newontology.context, - label=proplabel, - name=propname, - ontology_id=newontology.id, - superproperties=super_props, - object=object, - subject=subject, - gui_element="salsah-gui:" + gui_element, - gui_attributes=gui_attributes, - comment=propcomment - ).create(last_modification_date) + last_modification_date, newpropclass = PropertyClass(con=con, + context=newontology.context, + label=proplabel, + name=propname, + ontology_id=newontology.id, + superproperties=super_props, + object=object, + subject=subject, + gui_element="salsah-gui:" + gui_element, + gui_attributes=gui_attributes, + comment=propcomment).create(last_modification_date) newontology.lastModificationDate = last_modification_date except BaseError as err: - print("Creating property class failed: " + err.message) + print("Creating property class failed:", err.message) return False newpropclasses[newpropclass.id] = newpropclass if verbose: print("New property class:") newpropclass.print() - # # Add cardinalities - # switcher = { "1": Cardinality.C_1, "0-1": Cardinality.C_0_1, @@ -506,9 +410,10 @@ def create_ontology_from_string(con: Connection, else: propid = "knora-api:" + cardinfo["propname"] gui_order = cardinfo.get('gui_order') - last_modification_date = rc.addProperty(property_id=propid, - cardinality=cardinality, - gui_order=gui_order, - last_modification_date=last_modification_date) + last_modification_date = rc.addProperty( + property_id=propid, + cardinality=cardinality, + gui_order=gui_order, + last_modification_date=last_modification_date) newontology.lastModificationDate = last_modification_date return True diff --git a/knora/dsplib/utils/onto_get.py b/knora/dsplib/utils/onto_get.py index 5365ecc66..81c115b38 100644 --- a/knora/dsplib/utils/onto_get.py +++ b/knora/dsplib/utils/onto_get.py @@ -1,12 +1,11 @@ -from typing import List, Set, Dict, Tuple, Optional, Any, Union - import json import re +from typing import Dict from ..models.connection import Connection -from ..models.project import Project from ..models.listnode import ListNode from ..models.ontology import Ontology +from ..models.project import Project def get_ontology(projident: str, outfile: str, server: str, user: str, password: str, verbose: bool) -> bool: @@ -48,11 +47,7 @@ def get_ontology(projident: str, outfile: str, server: str, user: str, password: projectobj["ontologies"].append(ontology.createDefinitionFileObj()) prefixes.update(ontology.context.get_externals_used()) - umbrella = { - "prefixes": prefixes, - "project": projectobj - } + umbrella = {"prefixes": prefixes, "project": projectobj} with open(outfile, 'w', encoding='utf8') as outfile: json.dump(umbrella, outfile, indent=3, ensure_ascii=False) - diff --git a/knora/dsplib/utils/onto_process_excel.py b/knora/dsplib/utils/onto_process_excel.py deleted file mode 100644 index 2b352ab68..000000000 --- a/knora/dsplib/utils/onto_process_excel.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -from typing import List, Set, Dict, Tuple, Optional -import json -from jsonschema import validate - -from ..models.helpers import Actions, BaseError, Context, Cardinality -from .onto_commons import list_creator, validate_list_from_excel, json_list_from_excel - - -def list_excel2json(excelpath: str, - sheetname: str, - shortcode: str, - listname: str, - label: str, - lang: str, - outfile: str, - verbose: bool): - current_dir = os.path.dirname(os.path.realpath(__file__)) - langs = ["en", "de", "fr", "it"] - - if lang not in langs: - raise BaseError(f"Language '{lang}' not supported!") - - rootnode = { - "name": listname, - "labels": { - lang: label - } - } - - json_list_from_excel(rootnode, excelpath, sheetname) - jsonobj = { - "project": { - "shortcode": shortcode, - "lists": [ - rootnode - ] - } - } - with open(os.path.join(current_dir, 'knora-schema-lists.json')) as s: - schema = json.load(s) - validate(jsonobj, schema) - with open(outfile, "w") as outfile: - json.dump(jsonobj, outfile, indent=4) diff --git a/knora/dsplib/utils/onto_validate.py b/knora/dsplib/utils/onto_validate.py index d81049a5a..9ce0fed93 100644 --- a/knora/dsplib/utils/onto_validate.py +++ b/knora/dsplib/utils/onto_validate.py @@ -1,79 +1,51 @@ -import os import json +import os +from typing import Union, Dict + +import jsonschema from jsonschema import validate -from typing import List, Set, Dict, Tuple, Optional, Any, Union, NewType -from .onto_commons import validate_list_from_excel, json_list_from_excel +from ..utils.expand_all_lists import expand_lists_from_excel -from pprint import pprint -def validate_list(input_file: str) -> None: - current_dir = os.path.dirname(os.path.realpath(__file__)) +def validate_ontology(input_file_or_json: Union[str, Dict, os.PathLike]) -> bool: + """ + Validates an ontology against the knora schema - # let's read the schema for the data model definition - with open(os.path.join(current_dir, 'knora-schema-lists.json')) as s: - schema = json.load(s) - # read the data model definition - with open(input_file) as f: - datamodel = json.load(f) + Args: + input_file_or_json: the ontology to be validated, can either be a file or a json string (dict) - # validate the data model definition in order to be sure that it is correct - validate(datamodel, schema) - print("Data model is syntactically correct and passed validation!") + Returns: + True if ontology passed validation, False otherwise + """ + data_model = '' + if isinstance(input_file_or_json, dict): + data_model = input_file_or_json + elif os.path.isfile(input_file_or_json): + with open(input_file_or_json) as f: + onto_json_str = f.read() + data_model = json.loads(onto_json_str) + else: + print('Input is not valid.') + quit() -def validate_ontology(input_file: str) -> None: - with open(input_file) as f: - jsonstr = f.read() - datapath = os.path.dirname(input_file) - validate_ontology_from_string(jsonstr, datapath) + # expand all lists referenced in the list section of the data model + new_lists = expand_lists_from_excel(data_model) + # add the newly created lists from Excel to the ontology + data_model['project']['lists'] = new_lists -def validate_ontology_from_string(jsonstr: str, exceldir: Optional[str] = None) -> None: + # validate the data model against the schema current_dir = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(current_dir, 'knora-schema.json')) as s: schema = json.load(s) - datamodel = json.loads(jsonstr) - - # - # now let's see if there are any lists defined as reference to excel files - # - lists = datamodel["project"].get('lists') - if lists is not None: - newlists: [] = [] - for rootnode in lists: - if rootnode.get("nodes") is not None and isinstance(rootnode["nodes"], dict) and rootnode["nodes"].get("file") is not None: - newroot = { - "name": rootnode.get("name"), - "labels": rootnode.get("labels") - } - if rootnode.get("comments") is not None: - newroot["comments"] = rootnode["comments"] - startrow = 1 if rootnode["nodes"].get("startrow") is None else rootnode["nodes"]["startrow"] - startcol = 1 if rootnode["nodes"].get("startcol") is None else rootnode["nodes"]["startcol"] - # - # determine where to find the excel file... - # - excelpath = rootnode["nodes"]["file"] - if excelpath[0] != '/' and exceldir is not None: - excelpath = os.path.join(exceldir, excelpath) - json_list_from_excel(rootnode=newroot, - filepath=excelpath, - sheetname=rootnode["nodes"]["worksheet"], - startrow=startrow, - startcol=startcol) - newlists.append(newroot) - else: - newlists.append(rootnode) - datamodel["project"]["lists"] = newlists - - with open("gaga.json", "w") as outfile: - json.dump(datamodel, outfile, indent=4) - - # validate the data model definition in order to be sure that it is correct - validate(datamodel, schema) - - print("Data model is syntactically correct and passed validation!") - + try: + validate(instance=data_model, schema=schema) + except jsonschema.exceptions.ValidationError as err: + print(err) + return False + print('Data model is syntactically correct and passed validation.') + return True diff --git a/knora/dsplib/utils/xml_upload.py b/knora/dsplib/utils/xml_upload.py index 51b870be5..de4b689f1 100644 --- a/knora/dsplib/utils/xml_upload.py +++ b/knora/dsplib/utils/xml_upload.py @@ -1,8 +1,8 @@ """ -The code in this file handles the import of XML data into the DSP platform. +This module handles the import of XML data into the DSP platform. """ import os -from typing import List, Dict, Optional, Union +from typing import Dict, List, Optional, Union from lxml import etree diff --git a/knora/mylist.json b/knora/mylist.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/BUILD.bazel b/test/BUILD.bazel index ed746cd1b..ef499ebfe 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -132,6 +132,7 @@ py_test( "//knora/dsplib/utils:onto_validate", "//knora/dsplib/utils:onto_create_ontology", "//knora/dsplib/utils:xml_upload", + "//knora/dsplib/utils:excel_to_json_lists" ], data = [ "//testdata:testdata", diff --git a/test/test_tools.py b/test/test_tools.py index 0356a3a81..096cc9913 100644 --- a/test/test_tools.py +++ b/test/test_tools.py @@ -1,51 +1,104 @@ +"""This test class tests the basic functionalities of dsp-tools""" import json import unittest -from dsplib.utils.onto_create_ontology import create_ontology -from dsplib.utils.onto_get import get_ontology -from dsplib.utils.onto_validate import validate_ontology +from knora.dsplib.utils import excel_to_json_lists +from knora.dsplib.utils.excel_to_json_lists import list_excel2json +from knora.dsplib.utils.onto_create_ontology import create_ontology +from knora.dsplib.utils.onto_get import get_ontology +from knora.dsplib.utils.onto_validate import validate_ontology from knora.dsplib.utils.xml_upload import xml_upload class TestTools(unittest.TestCase): + def setUp(self) -> None: + """Is executed before each test""" + excel_to_json_lists.list_of_lists = [] + excel_to_json_lists.cell_names = [] + + def tearDown(self) -> None: + """Is executed after each test""" + excel_to_json_lists.list_of_lists = [] + excel_to_json_lists.cell_names = [] def test_get(self): - with open('testdata/anything.json') as f: - jsonstr = f.read() - refobj = json.loads(jsonstr) - - get_ontology(projident="anything", - outfile="_anything.json", - server="http://0.0.0.0:3333", - user="root@example.com", - password="test", + with open('testdata/anything-onto.json') as f: + onto_json_str = f.read() + anything_onto = json.loads(onto_json_str) + + get_ontology(projident='anything', + outfile='_anything-onto.json', + server='http://0.0.0.0:3333', + user='root@example.com', + password='test', verbose=True) - with open('_anything.json') as f: - jsonstr = f.read() - jsonobj = json.loads(jsonstr) + with open('_anything-onto.json') as f: + onto_json_str = f.read() + anything_onto_out = json.loads(onto_json_str) + + self.assertEqual(anything_onto['project']['shortcode'], anything_onto_out['project']['shortcode']) + self.assertEqual(anything_onto['project']['shortname'], anything_onto_out['project']['shortname']) + self.assertEqual(anything_onto['project']['longname'], anything_onto_out['project']['longname']) + + for list in anything_onto['project']['lists']: + list_name = list.get('name') + if list_name == 'otherTreeList': + other_tree_list = list + elif list_name == 'notUsedList': + not_used_list = list + elif list_name == 'treelistroot': + tree_list_root = list + + for list in anything_onto_out['project']['lists']: + list_name = list.get('name') + print(list.get('name')) + if list_name == 'otherTreeList': + other_tree_list_out = list + elif list_name == 'notUsedList': + not_used_list_out = list + elif list_name == 'treelistroot': + tree_list_root_out = list + + self.assertEqual(other_tree_list.get('labels'), other_tree_list_out.get('labels')) + self.assertEqual(other_tree_list.get('comments'), other_tree_list_out.get('comments')) + self.assertEqual(other_tree_list.get('nodes'), other_tree_list_out.get('nodes')) + + self.assertEqual(not_used_list.get('labels'), not_used_list_out.get('labels')) + self.assertEqual(not_used_list.get('comments'), not_used_list_out.get('comments')) + self.assertEqual(not_used_list.get('nodes'), not_used_list_out.get('nodes')) + + self.assertEqual(tree_list_root.get('labels'), tree_list_root_out.get('labels')) + self.assertEqual(tree_list_root.get('comments'), tree_list_root_out.get('comments')) + self.assertEqual(tree_list_root.get('nodes'), tree_list_root_out.get('nodes')) + + # TODO fix this test + # self.assertEqual(anything_onto['project']['ontologies'], anything_onto_out['project']['ontologies']) - self.assertEqual(refobj["project"]["shortcode"], jsonobj["project"]["shortcode"]) + def test_excel(self): + list_excel2json(listname='my_test_list', + excelfolder='testdata/lists', + outfile='_lists-out.json') - def test_validate_onto(self): + def test_validate_ontology(self): validate_ontology('testdata/test-onto.json') - def test_create_onto(self): + def test_create_ontology(self): create_ontology(input_file='testdata/test-onto.json', lists_file='lists-out.json', - server="http://0.0.0.0:3333", - user="root@example.com", - password="test", + server='http://0.0.0.0:3333', + user='root@example.com', + password='test', verbose=True, dump=True) - def test_xmlupload(self): - xml_upload(input_file="testdata/test-data.xml", - server="http://0.0.0.0:3333", - user="root@example.com", - password="test", - imgdir="testdata/bitstreams", - sipi="http://0.0.0.0:1024", + def test_xml_upload(self): + xml_upload(input_file='testdata/test-data.xml', + server='http://0.0.0.0:3333', + user='root@example.com', + password='test', + imgdir='testdata/bitstreams', + sipi='http://0.0.0.0:1024', verbose=True, validate_only=False) diff --git a/testdata/BUILD.bazel b/testdata/BUILD.bazel index 75998a19a..a2a6d1caf 100644 --- a/testdata/BUILD.bazel +++ b/testdata/BUILD.bazel @@ -8,10 +8,11 @@ filegroup( name = "testdata", visibility = ["//visibility:public"], srcs = [ - "anything.json", - "list-as-excel.xlsx", + "anything-onto.json", + "lists/description_en.xlsx", + "lists/Beschreibung_de.xlsx", "test-data.xml", - "test-onto.json", + "test-onto.json" ], ) diff --git a/testdata/anything.json b/testdata/anything-onto.json similarity index 100% rename from testdata/anything.json rename to testdata/anything-onto.json diff --git a/testdata/error-onto.json b/testdata/error-onto.json deleted file mode 100644 index cda80a736..000000000 --- a/testdata/error-onto.json +++ /dev/null @@ -1,492 +0,0 @@ -{ - "prefixes": { - "foaf": "http://xmlns.com/foaf/0.1/", - "dcterms": "http://purl.org/dc/terms/" - }, - "project": { - "shortcode": "4123", - "shortname": "tp", - "longname": "test project", - "descriptions": { - "en": "A systematic test project", - "de": "Ein systematisches Testprojekt" - }, - "keywords": [ - "test", - "testing" - ], - "lists": [ - { - "name": "testlist", - "labels": { - "en": "Testlist" - }, - "nodes": [ - { - "name": "a", - "labels": { - "en": "a_label" - } - }, - { - "name": "b", - "labels": { - "en": "b_label" - }, - "nodes": [ - { - "name": "b1", - "labels": { - "en": "b1_label" - } - }, - { - "name": "b2", - "labels": { - "en": "b2_label" - } - } - ] - }, - { - "name": "c", - "labels": { - "en": "c_label" - } - } - ] - }, - { - "name": "fromexcel", - "labels": { - "en": "Fromexcel" - }, - "nodes": { - "file": "list-as-excel.xlsx", - "worksheet": "Tabelle1" - } - } - ], - "groups": [ - { - "name": "testgroup", - "description": "Test group", - "selfjoin": false, - "status": true - } - ], - "users": [ - { - "username": "tester", - "email": "tester@test.org", - "givenName": "Testing", - "familyName": "tester", - "password": "test0815", - "lang": "en", - "groups": [ - ":testgroup" - ], - "projects": [ - ":admin", - "anything:member" - ] - } - ], - "ontologies": [ - { - "name": "testonto", - "label": "Test ontology", - "properties": [ - { - "name": "hasText", - "super": [ - "hasValue" - ], - "object": "TextValue", - "labels": {"en": "Text"}, - "gui_element": "SimpleText", - "gui_attributes": { - "maxlength": "255", - "size": 80 - } - }, - { - "name": "hasRichtext", - "super": [ - "hasValue" - ], - "object": "TextValue", - "labels": {"en": "Text"}, - "gui_element": "Richtext" - }, - { - "name": "hasUri", - "super": [ - "hasValue" - ], - "object": "UriValue", - "labels": {"en": "URI"}, - "gui_element": "SimpleText", - "gui_attributes": { - "maxlength": "255", - "size": 80 - } - }, - { - "name": "hasBoolean", - "super": [ - "hasValue" - ], - "object": "BooleanValue", - "labels": {"en": "Boolean value"}, - "gui_element": "**Checkbox**" - }, - { - "name": "hasDate", - "super": [ - "hasValue" - ], - "object": "DateValue", - "labels": {"en": "Date"}, - "gui_element": "Date" - }, - { - "name": "hasInteger", - "super": [ - "hasValue" - ], - "object": "IntValue", - "labels": {"en": "Integer"}, - "gui_element": "Spinbox", - "gui_attributes": { - "max": -1.0, - "min": 0.0 - } - }, - { - "name": "hasDecimal", - "super": [ - "hasValue" - ], - "object": "DecimalValue", - "labels": {"en": "Decimal number"}, - "gui_element": "SimpleText", - "gui_attributes": { - "maxlength": "255", - "size": 80 - } - }, - { - "name": "hasGeometry", - "super": [ - "hasValue" - ], - "object": "GeomValue", - "labels": { "en": "Geometry" }, - "gui_element": "Geometry" - }, - { - "name": "hasGeoname", - "super": [ - "hasValue" - ], - "object": "GeonameValue", - "labels": {"en": "Geoname"}, - "gui_element": "Geonames" - }, - { - "name": "hasInterval", - "super": [ - "hasValue" - ], - "object": "IntervalValue", - "labels": {"en": "Time interval"}, - "gui_element": "Interval" - }, - { - "name": "hasColor", - "super": [ - "hasValue" - ], - "object": "ColorValue", - "labels": {"en": "Color"}, - "gui_element": "Colorpicker" - }, - { - "name": "hasListItem", - "super": [ - "hasValue" - ], - "object": "ListValue", - "labels": {"en": "List element"}, - "gui_element": "List", - "gui_attributes": { - "hlist": "testlist" - } - }, - { - "name": "hasTestRegion", - "super": [ - "hasLinkTo" - ], - "object": "Region", - "labels": {"en": "has region"}, - "gui_element": "Searchbox" - }, - { - "name": "hasTestThing2", - "super": [ - "hasLinkTo" - ], - "object": ":TestThing2", - "labels": {"en": "Another thing"}, - "gui_element": "Searchbox" - } - ], - "resources": [ - { - "name": "TestThing", - "super": "Resource", - "labels": { - "en": "TestThing" - }, - "comments": { - "en": "A thing to test things", - "de": "Ein Ding um allerlei Dinge zu testen." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1-n" - }, - { - "propname": ":hasRichtext", - "gui_order": 2, - "cardinality": "0-n" - }, - { - "propname": ":hasUri", - "gui_order": 3, - "cardinality": "0-n" - }, - { - "propname": ":hasBoolean", - "gui_order": 4, - "cardinality": "1" - }, - { - "propname": ":hasDate", - "gui_order": 5, - "cardinality": "0-n" - }, - { - "propname": ":hasInteger", - "gui_order": 6, - "cardinality": "0-n" - }, - { - "propname": ":hasDecimal", - "gui_order": 7, - "cardinality": "0-n" - }, - { - "propname": ":hasGeometry", - "gui_order": 8, - "cardinality": "0-n" - }, - { - "propname": ":hasGeoname", - "gui_order": 9, - "cardinality": "0-n" - }, - { - "propname": ":hasInterval", - "gui_order": 10, - "cardinality": "0-n" - }, - { - "propname": ":hasColor", - "gui_order": 11, - "cardinality": "0-n" - }, - { - "propname": ":hasListItem", - "gui_order": 12, - "cardinality": "0-n" - }, - { - "propname": ":hasTestRegion", - "gui_order": 13, - "cardinality": "0-n" - }, - { - "propname": ":hasTestThing2", - "gui_order": 14, - "cardinality": "0-n" - } - ] - }, - { - "name": "TestThing2", - "super": "Resource", - "labels": { - "en": "Another Test Thing" - }, - "comments": { - "en": "Another thing for testing things." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "CompoundThing", - "super": "Resource", - "labels": { - "en": "A Compound Thing" - }, - "comments": { - "en": "A thing for testing compound things." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "ImageThing", - "super": "StillImageRepresentation", - "labels": { - "en": "An Image Thing" - }, - "comments": { - "en": "An image thing for testing image things." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "AudioThing", - "super": "AudioRepresentation", - "labels": { - "en": "An Audio Thing" - }, - "comments": { - "en": "An audio thing for testing audio things." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "MovieThing", - "super": "MovingImageRepresentation", - "labels": { - "en": "An Movie Thing" - }, - "comments": { - "en": "An movie thing for testing moving image things." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "DocumentThing", - "super": "DocumentRepresentation", - "labels": { - "en": "A Document Thing" - }, - "comments": { - "en": "A second things for testing different things." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "ZipThing", - "super": "DocumentRepresentation", - "labels": { - "en": "A ZIP Thing" - }, - "comments": { - "en": "A things for testing ZIPS." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "TextThing", - "super": "TextRepresentation", - "labels": { - "en": "A Text Thing" - }, - "comments": { - "en": "A things for testing TEXTS." - }, - "cardinalities": [ - { - "propname": ":hasText", - "gui_order": 1, - "cardinality": "1" - } - ] - }, - { - "name": "PartOfThing", - "super": "StillImageRepresentation", - "labels": { - "en": "A Thing having a partOf and seqnum property" - }, - "comments": { - "en": "A thing for testing partOf and seqnum properties." - }, - "cardinalities": [ - { - "propname": "isPartOf", - "gui_order": 1, - "cardinality": "1" - }, - { - "propname": "seqnum", - "gui_order": 2, - "cardinality": "1" - } - ] - } - ] - } - ] - } -} diff --git a/testdata/list-as-excel.xlsx b/testdata/list-as-excel.xlsx deleted file mode 100644 index 7d72c1a08f9fbfbcf82a1783b07922baddd4c6fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9870 zcmeHt1y>x~()Iwsf?IGI+#zUif=jRj*C2yCgS!pxPLKo%4#C|Wf&~T$EZ$6g+RE~9PjCSU03-ka00bBvWSZ;4006O1000~S60DB6 zoh=w-3pP-7w+A`tvAWqrGF2AeQf`~8gcu20}hSz0i7>YuK&$Ucdn^jvu6qMxq})O^EW^t#dU zCPlu@iN1jo`qmhBi5(K#ioQ6yjL_W1OQyg@R8M zc_qI;OG1f{modPnU|_J}MTgS9F^QM$#|gx5Pb9A%a4)cSnoc{!Fb6w;)ZQWcD+4(BH5==fjE z!M_c?G+sfen;jK$D03Gwa0y+E#T1iweJRrnRQ2(bUBIY~%B3b=Y^Ni^R3!?8llE=( zxf_~a6pY#(0A8$dl}2FW3Q*O#mO)Z(9GnpuX&sZL97Z; zhQbf>gUeJ>lgAR3m}9IOM94V#B#>u9Y5v-M3R+7>SLHBJu{Zl=kWclz?{^c%-uccX zf80S63FUjW_azN)z{$w$Yq{5eHSqkJNbQX|pT#GmEJuDycLNjaj#G(u?Kt;dEQ%Qe zuPM2(uQ)^_wXWjmkb;Y21Jk-E++3*_`UTP9{~($k!lXM?kYxF3%6yIgXkufgG*cQO@*V?Xi0u|5=lw;MSf zfkQIdFCz<(hR4C)+x=d$kAj}=gwrS-Kfs)Y`(pEoo`x_a_amjmS>qQ7m*ptCGl!+q zIO))%&J5Rl^ScRpvsV^SQ(sS!&S1Ln&?r8Cfsh^GPCju;h%76J<_?(Ho&qvmbxK|! zowVGUD!7kYb>`lnZjhA7lNECdc|pJwAp_Fnz6|kB8#!ssYKN;}vD+br_W2X$;j=^2 zZ-f2N<;Ru&PnEgY*CEUU_Rv0m%0sq7t5*Ba7Wk&#%5&0NBJ8>Fw+*2TM6a8Jak)c?6_u(6rH5E- zf3qAkXFhrRJ9KC)){ObPC9PV&Ja`z*!}a1~WJiBzQM)J;5jK-S#n{H+t#vV^{&PlQ zmMi90`b7>Dz1H53Ga1m2s|$75h(Xj1pd4}OV6k>7aT}hTH54^8mcu<238z8kS>CAa zk-Ta3ia4vZCA3z9qB_K|+Ssn8=$rwok~O1whLLIPFT&%q0_In=N4ZZ}JGpRz^j|^4 z<(P7LE}rBaPn<>EF?1#hYXx)QxBH$j@B?kjBLUYOXjMH*f%Y3? zYxEbVf?3x)l;K?VJY0J#l|uZ#%~J_O3PpIDy_eKy3{xQ zUFxc0HeG&@A>->osxu1eWs`mNc7z8tsgiGLYAL{VCK8TQE?|LX6yqYW(ikNVnO%m; zF)Lp11bM1Erb_UrnjZDey#^`Sp-Gq7tdc3t6A$-cU6jP1?W1d?(^qzz3a%2DBh#DD zyqupKQ!?f*&`NC~sk$4C3B`bZeG-wERccDDvO{j{xlYRgvh?3g~} zd%l<+uAt`1lqd5b3rl)yb{RxZHETXBXbi<*T6N`vH=RHj4u9Q5n?%9sPV?b`0K->3 zI`}zG<3@G9Xj`$!)MD@ml(yd>d+2R=oU+Q2;yu)SCH0cu^8$u@}io}uQ z>BhwjOdXrFybGzr)Bv_9o_PM*bhJu=ZgZP^AcG7Egdr}GNs!_*=v;&%K7GptUsd~{ zaNUZvLEuc2V9@eMgUZ}^EudJB>jU0y&Qwn!X90EVr6k!PQ*N2<0@@1pB3FwM+s^1f z#^T6HTH&Er`Eat7lR)zg~Q#*`g^;8 ztUYOJd&qvd1ERwHk&K3c(HE13h&y<%<^YrZ_f#In+raRI2)H~YO%p2{%hbI}aypGT z7nkK@Z8Vms(dERG)BUheGK9n_f9T4SdYsp<|ZV?>B z*>w#DiA8Vv@mdT1pqcZ#H`Sctp1JaKdwCmv#wT>*vE>F`c7D^nNAhyKX`{>G9C64` z+tnCHtit~^rNVBE592Jy6wM~sli&CAvAM4)w;jkU`ljo0?i`bkY+p&gg#MgQb4yr@ zfL&9?$}QSgm|6I9G6sm`Rd+8}>OvKDDoq3^_SFIYFo+Lj{piO`FU}DJ@x5oYizK{o_das|7c`ez84&UWdeotL_NTNAnFy0-Jjg@T;VC;N$FYfn z@R#3`FeGJi%-tLXuT2gbOae+0)aim*TPd)uA6`Gsbskb1l`CiRoMphJ8L;@7qkfJAFTxi zmsvg@r{Lo+t*IW;sVH6X$p&r8CqfyKv#HdR$-Y)E=()UK*+Y&D9m^A^VvR* zST=)5{j+e_64oD*$z}QG8K3ncLE;)ZOkemf(?<5obWvqL>n9-%TuOb$l>fSnfGYk< zKfp-Y)0+cGwP(i$yIX3Pe6D{Fg3??Pb9j*Uv6+Xb5oln zj3%0kRmxS{JFQg#4n(!``%;0v*y>TCU3Ucev70`!uk6j-f{=d`$bI*A9G*?v4NdNN zl}LDxOF&4c9U=c}j;KxF@-m<)-yo1uF20)^%uuS#vzI5H(b-A#Js9vxt< z2}(&f>cVbhm@xvU9K!%`V<}~^l~_&P0;FQw^&|#i=$2U1q+;DT6WL+9^W`j*i@SW! zk9eRF=o(qcONrT6oR)^?smMNkfM~`a&=H5 z;_I{&B_>4SWpOx)^&1KF@kRp z$%bjgP8qTdV&Sw?PL$#5MNZDTi6S-H(_ADKV{Si)!!ip{$a4*GBbCTtC5(q-igWxx ziY>1Dj1>+zm27um>q#o5S8}jQMT|TTs}gc7{)&ZHU&<-WC}jv3UbJ(hM@^LycZ(v- z;ouDfetGjMZHC+!sLL_KS2}}qqxP1-b@p@VXf18hG+u)C zg3|$sA-sk7Rnv;+=Oy4S;hJB3)M}f;C%gk*oK6-BjmbR0ms4y`2ca9F&wKX`?{R|k zm?t+Yd&_MUGrmrwDV0qv^`_<+bM?(OpQm*Ppuf`+9*b(jhAG*5t6`+~(xtq3>j}!< zPLpXSrEA!y*DmBrUlBYnp3#Mx`NtC+E2LMvoGP0NelseYKeMwv{zRWj`RZlci%ds7 zKXj_b+jlifp|Q>`MP{p`Gw}6mmpq=R7aNO{2lBC<1<5c*zE;bqShchRIHsZRJM1S| zLN!;%B&QmSdp8vu!&)s)*+mUapaRcjmo)D(=sml93-%oF+0B%X8t~ecimJn2yC~wm zcxV66S@khwa}W)cZjj$BQ-ga;E$X4Y!jfly<5YXIC7G`Rq-$qDo|im0j9+Md!WwgM z-9xM+j3c(wrmNh-1fG%ZARci81U8Z8d&ZYMTVW2U&E*}nzU}r29>^Zmti?gWJ+!Z^5!mGVOFYBI|dM&8*#uZw5%r^ltApW zImD`yMf{_mdqtVW3W&8j1NnndQsTbtKz#8{(;R@FG7?4I+&g*Jm^fjL(y|LD&=ZZX zqBtYKxjIL9q}5#Ra^#wmNfD5-1 zSF-}|oqTX!h6kTlIj*E4HN2oNSR%n`K-R(1%JQW{{}|~qQbM08zVjld8VN)fF1yK~ zo3K4Za$j@ef1FrGNYZrB65 z-RZ+A%B`l&`0o@dnFT$>ii`CxFi|H@2AL-ld%ts)z7Qa;RS@V7=s1$F30zh;3!20^ z^f(Dz7%Wvo?5QfB3aeln9?`LJvBw|CMZ64gVhl#7Spz5f|r{bvBy8s#%x2M+-Fk^=w)zq{X`0hp6H2n2Rw`|a|Z56(!N zk6x3)gd9?BkTNd^j(zs?!-FFP(S4nv z%ZR=;JJ$u5T+j6-llxC<*fT3_h}Lii^}P6*Ka2SIk_n|t=?4|)r4A%$rjk?gim
NeSQD%+732@+`rLoQtoVKplnjAy`HZgaGb+)*M4TVCF?Z1; z*L`K8PH6fymc+3=hN?1dLY`4kGKm^TR@3j{jv)Kgs*Z~&o)xXSf^AEMdXh75wog{q zD^XT~D#|n)gDx$2{>jb#!Ihy&i`milu54G9H*L$siG3=xPK|7Rv0sY?>zhStKdKWf za@5|6&_}K7q`5^ozF^X?LId-I(sE>+{d-7efO*K8EEPUultES@E&qk7rl94JU0$ zVcE8aN=O9zQAS(fp4=tO;#-^V2Gq|o<#1q+^^vo%Noi@oq?&eex1G*>wD zsygNLX{DG9ReLIOKsrTnLQfRxUU+b>)bERW9ZP+$#j)@we(%gS*x@VLoLmP|pgV-WcgN1qesMX7>M4virD`MlbufWEvj{y^JgQ|D;?NdbVmB<*OzBXNO`-NAr1*M~ zXT_1w8qP?IxOlqegJo@$V;4{HGq8@=_$Q>wQb{!{ZPnV3l4|e3Iz39a9} z`e-&;=m2ADzy!|(_}TkbJf0Dh^7HHVu9Qcne4u6QR+2`1PoO(vtQ@KnvO1g$sPK=Z zd!;sG+^|{rw!2J!rf%axMR6NiEUk+0I$Ex%`5Y-+`#f}Ld_ctkTu)}GSVE>kxVK_@ z&f|NzYiKVdjw+;^MCQtgjW9*Xc1xw^+&Gdd7y5{LqJ6Fk1<|h*<2LBT?3EEbZD52y zFjcK^svmVDe4fLB+c9!_#HYb9UM@Y=q%Wz)4Ax;|701+|8CYB@i8OBpS`MSB5Vke& zdv&ff`Ia?O{C0oSz<<`cLdGir+FsusM-24b#i}iL)gqJtm0PzskhLg2PjwZCek3mD z$U6f!KstqNL+2dDRUuVb=;!QLn|ix$f4(k)=*`Q9L^U%53iZWT9?A7#&Zf?{A z2l1(7VzqO^-E01-?w>=lX;e|mzSUL=2@}3HjWkTG=$1fjJ0G)KYFAD&{Pgqgph#L3 z;?edvl!k)GWhlSq*X>@(@{lO|t|q`SmkgAB|0wv+4uIfD%YW+|hc3!*Q}Nh=78luh zzg6v@&Hi{l8bLYyPbt5@ANu&vUEAf}_IgQ(2H~>daS@`55fr4#>$=_OQ((6B>b^JM@eyLh5tAb!OTd}$@9$T1xG$;S1qi>- z1zqeq#xYQ=6Xm(2c`uf+EtxS&C`8pozfH@eFqzpH#+O7=abOMxP4~X=8aRE+zwO)r zhQp-t-0~#7o~U;x5ZtB(11jsUzxS@`Em`HgB21xoTr4ShMbnjyY@4;*;+>Er*-;e= zLi>>sLCCOgOn;*J?3&O6QqS<7u)Spm{6N~W3u1Sl9C;g^ZbFIw?jv}c4tmIk6SQ6X zS>Bl-=aeoEy!RH}ak@6)y}P?|_1cOYQA=X#UY=hYosgBMjT z1*?V}t?llOi>h30oS(cq+MAYKZ1@Q&O0&&q>#YgQr0O9a!B!*GNe8xbtg48S0}0nm zXp88GE5w$oR<<4!m#w+#b0jj-6i?}^wHtF<+uLu>%Yo!E`#f156Js%6teO)RlIPuyRb%)HY?l zxq|`QQ)@b*9Jh>kAr;U8+~U0ipO0q`9tipI=hUu0UwngGa8@x{t)5XtD1G2&Ed7D+ zfo$mCGq=P-U)*IL6ZlS#3WfEU>^HG9R(7p@dj31?6|$l zh_~o8hc@Z_5@I@r#KH`bBbAl)vAVUhcfha`pWFO07t)(g{ogWfhxt;zo#vM2+I=ej zQY=a>tMVk=N~Orge`zMd4MMq7Pfk@ljOZ z5UqFim{evnent7^f;m$fhIDf;6DjLlvdgPxR?5NQ-I z;vuST2E4KLcefedFdWl8E7p@~Hs+osiJ5Xq;CAxCOKvL?70j|l(|y;4qW9qkA*M>G z%&i#yhVjW+i~f5Kc~0yCtycs53KkclgJ&fpM`mqN3kCyD@xC~8Z{$p=*5H*luXf)h ziu|iBtFBM1NPkp6%E#gX=HJ!N(BA%k+V?nRe;gU{owoBotCDaR=#Yb`3|o4LsJcoX zv4>_Qz`?`Jg#Q_>Jfw$cFw4yD3qF%gZc+7q4e?<1&2?COlTaP&_vf_4UclrCU6XBv z!cq}jBMqPC8_Ft)6(;*mW1wPcPuY*Fmo`S*?D4&cmBC8E=AlhViGy6TV%nefw2H*7 z{a-{O67J3swP0*EN-CZ3kJ(X^A$tcy)m(XS6V{6h9=OW_2(P}pao;7-ENL!xng$%R zoab^s&r^PLfz;>mv}mx642_>=%O;$!-aIQeKVdW}qrlRtV>MlRn@CrTW*KD{#!zz^ z!TOc?RaHbmB?bTh_W|^w$l(098l^DAo(M7kfCV517&w|* zJ8^RSbBzSRM;y4jlqGaPyST7J_Ixl`yz)K;a?2#@_hi@cG@|cYGjfNsGnD9@obU!# zqA5^V%gW#09#DUR4I6>V`)0w`M+F~hIxYIoXP#V~AJw;xU%J+Kb@t1JuXYrcDlo6h z;L&f<(7})b`$-t1*2W0X8BAAc5-Rhy2x?t?u&xqwU}#`!-MsT}0^08uiEg5Ckh%pK z?y+%yBuWv8?A84i;*}&LQq@~ElokFVj$fB_?nXbXaY)CT&I^}(ErDz(U^J)cqY0?d z%=s`&X5e$(plHW~gR|#+z76fm=yQ|ks&{ljeK^-NyQVsbdqPY!xJdFSU1rnbuTmONXj;{02;lEN@sY2^%$fhgTp%@!FTEDpfH? zy2W$CWB!M>FW6*7uY9gijZx?*(ZQ))d@-wpnIE^CLNZi*%<9NLmwcILkIIeQvnDJE z*U#Z$NyAXkz*=FfrecgkNcKU2c>s7^G=On5krbz`^D&i-%S#paM_n&<-8zCKShXBG zN2b`XDxZ(S+X7pJ`l7!+d!Mv@RN-{4hBZAf0P}6-iIEloqDda0+%5hzCW$i&QT#OG zjsJV`M)vl9tNq>3SIkZ>eAI7VS@f%m1)lgU*A2wOG#^eAtmph@ar(8lVGjtr0kK_N zwBxu`0 zd>v7VE8)#Nv0MfadIC;L!657=PiZax)bS~ z5zN|3-nKG5D(cO2HFI@yt({%SU|b?k#K`P|_WNbq1@g3R>^sc0saFt1nqeP!VtiNl zQvbqbHQR{>ssJt-?f-<0axu2^+?j`r(u=Zj=+ha^9A~KbIKQzD<>(h(6-Jv7RJ^)d z_xBHT)#Vw?%xN`MDDiR|3NQEsnGUc$3RhhP^n>Fi&o|9`-ahZa$cp~bD0+2K>jOpS zYZ;O*ebwz35pu(wWjmbgz7^LcuRo%#iJVZTY)9s2_sk+c7znvT&rX)Z&CNu zSeydl5SAa{)tCnR$t(dqy zB9=4^lXl)h^Rj*qDs~cXqquhh>!VeD^IIkhi>^f}rsqY6ruQZim1=CTS8c%lbJT=4 z%p$cmd-g;NyK1STAayS&3h-S*x_sd-wr z)1}%?+9Z2Os1QXfWHC{8mAAM5!iQZ;h07DZvfYh$S1MVg8p(Y$giM+M0MRe0oGnak zOgaCx=KkAVwW|xWg$d&O?7h72srRXhjiokFP?~Uh6WP)VsP%)FU9FZfVl2TqeRao8 zPS2*l;gdY9I|gS)5*{asix;{)(m#3$OVWCx-UnO(!+f4puz%@dQ#gY{Z(aCbi_|XB zwh7UPLY2W#r1Swvo|vJhyE@3=bjVPQmmfGZ)2n~xy%_N1QDJvRuK3GK_oRgKv{aOK zC619qoo$XnZVaT;_^JmV65?KEZG}L^R^t+60Xg3I-+pW=5#1b$jBGm3t#kJ%oX$N~ zA#YN5i^^0S*Ga1crb`ycrz#vztXknKy6Ds=)5+9I72K#yTsSH_s;*GtuxbgkBxq*J z(*?EE*{pCAepJxcRHPk|^7)cW53_0tYNSZi?yseP@nm1!baAscsR3<2d~QD5KxaQ& zpwLwL6)EYcPJkB|o4O7UFiK|br!>msr{-nihQKMRGI=SlDxDRlP$Ku1dj3Fjmu1r^ z(M|H%t&dDSb>OMGVlj*mf16-n)*ScQJ^XR1xkPVMouD$qVLT1)_TK5`p)ls_Qb+U~ z@e;e@zF>7;`e|u)u)2+3^VTibehsz08DQx2r1L>jTLszB&xtX?f}FX;abQlVMSk7U zT8E0k@SBozyB|l_d)l9(>2HN%y6`TjdQITQdhQX2(BcOtvF_xkFI>PvYF3J`0b4`_ zRc2OgUpNd;%I0LXq0lyJvfLuxl#vYwEBnUcteU_m^zyd7+Nj-K?i0g(;T~L1Yidzz zXknAoyD8obd6yWCSvc!vW(U!72#F9+sQN=TDYxW3F;Sb2llrUP3>lC6o*~AJq4co* zqTNkcjB;!zt76FkTlJ>xLq2N$1-7)%LsZJA%pmqKy7cr5se0v}RCY6%N5-9H=(xj&t3TFQsWuem{T{x4S*XvN~|7Jt3Pdd?2(k5f;J5O7D-x z(XOk=`++f@mwJlBg)?W#brps3-)e4&oU4;>Ba!ab&$$ z?>pwlGWd4I1B3#owpQdu>{own^*!r~CvZd+8taW$>x$?8wdK$ikMV0uUcbK#zq}N+ zhttBEan#20+w!u}lXBImX8~zDqQ-tzS;u-tKBnzwLkrJrP@4&7S_u85x0L5zA}4Mu zWV312DrUMcW($)ci;j~aA8@weOYFgEk2uJ)Wg`szXoLcgJXi1E+l*&G48+_S=p(=$ zVUYFR0QgX+ysV2w+g$iw^dQ?VXG}`Xs1nG2Yhq@28~fU;d_U#%Y{2VkP0Uu8dSz~tM7=w3V}*J$;NCiZEh|3x^is(8kn`@Ou_TTmG>QrU7*YTL zg>e%uaCdZ9MNfdPH$z1L-r0?$vGS+F^AwRvLL1!oFP7C(PEgsu_Yt- zvrN4m?i|HaSHuN+gH(YP+sQ=9q;seL?oWP7gCHSKgt zG%w1v?ircJanrO~c(OgSC6^g|Hu}z6|EnI_ylpp&-IvLfs`WAzO>aG_$Fsq8xr7m< zjo~yDXppa3dEy%!&BegQ2C7ZiK?eUr_3q7J5xvV>H6BnFpU=mi~E9d21^Y; zeIN9<+$?U%ZhVaEhD?Pe=FxJ>K1GlByxi={;aDW`0cmXO`dH8XTx2)v4SsqUvoPb- zZI^9;b>!LfjTGHYu)g!HxNv~vJ%+d(3Ld`aAu@8>1ay7)lFM{m&uAch z_NY8Ok}@)VefQ;ApQBP3$^Dau0h=OPqQ|Un=03%PhsF%RvM>ZyZz?6aM^y0&!S-jR zcjU!H3@(uT0ut_A6)t{UI4YQ#VWK!qm>*OqvppfF%t|Qr4g#fGWhqdREh;qIE{^wv zBF<*X&0LP<#^s?}1!D_Q`=c%?%}%B|w?A>jO)?zuPhe?m)5EB`pJfsBLbQ{%3mwZF z>j5&ac6i^8NGJyLU8S56g+RuNsJw%}*5GkH3!35av?@k^SPW~#DJXHhx zo-W(xW`gs}&>1p$o`h8iRO}vXY&K%-LEL+lXbTd`L3?IO?&0o;@5$BYTGuyxXkb{& zDaYoH4uuT9#n;c(MbOESMQM?}vLH)s!2)3|`lq8F76jG3SZ39S=twwY^>|~k1z`>L zpn(y`DB-X$i)o>Tb;f11&&`HPEvdnt+|{1kOM@cnfrHyI=O#{p^`!ESGP;q`4W!lf z>}Ia)_eh&Rg`B6RSgXbZ8RUn;bSg)cAHUUm9BK2|tiww$@u=C)V{4scp(Pjg()l)` zKFO`-YdxpHRc!8TlFN>|cNyv03*y(~fjh=5Cl`;DpBsGjBn7qbPxrqeV_kF(ho{!y z!`D9VY~1#*8SRx06eP!PAMF^O4qldhbz#Z+JgHwN-u5F>c(-aqq@dMP3XgqwKggE3 znZ<+s_%-hOPkp{B7KJV01=BWuPYqUw4_UR%H?FJ7b-BJZY5MCBCe4z=@0XCeTl zw%b#>)lhVO>ISxy;KO9vGtQ;+byX7O+jI_;g4b6+cQ}2n1P}F@hnnlA;T*SYTFsZV zcdhXmRDuhL=+1f&tq1WhYjm*OIT(3QWGlhdfnN6Pe1Y7;lM@*NA7nzJd>Q9%UzB69i z{kqKeQU=*Z96R~SIA@9Do`1qlJ`{{)L)x@%E=7WE%%C0xPU#ivB>Pw_4WH z>wA-Ct$U42CB|t&CSBV#1>e#*IBKF-YuefSO zwiJ0!q-qmiwab`3p*~mQk5Xx3ZK5P|va^AON*K&OTVTlof@Hxq4yj=?K`(6jM@UVS z(PHiGcZ^X*NhsE&gAAqyo;Y0*F9p@ial0=99_6FBQ-6;JO^OVcjN>Hbw~~<9+w=#s zg^%U3H8&@@Z<<2*J`Y5ZyTlt7xa(Etx}@tU1s4+V{Jj zToslX-Uj+a!04cf8XGQ#7-dxcc!CT@wG8%;`_QLhHhc|IeM;pknTyz`9|TR}SgAQI zE4j-ElRL;1!a;643YZ-jyeLNmD{>}euY)FgI2QAk(ObU`3c506tl5d#JT`BzY7<|n zzIwM~X{#rj!iLt9i8D8SWyo85eju~zlcG)nD@*4=YWM_m8sP|z+yjcS2JQw@JH{>X zYVQ;FAQ|MV#y!5Ifx8gqyK$E@S2yuxWvZWXq?d|L0mOUhN<}Bi2AWZfT>y#l{o*J@3!&B7 z&VzdRkhFXsXJ|b~y6YJ8^AU$h2*;?u_p1Zfk+idQHnnv&Q1`Gmb<+ErVU#8RE7y@e zy^eSUd(7>ura=-bZJELj+CootPat0owI=FZa-+ih5Jx0YA)iE%KuT2nkm$DPQ^P}C(0 z##gT9XZI7uxi<_1kNG=?A5tEe@C9brJ|=59Po=0++F7s)brER|ESf zimdToDY}C6;4ifzOqzP$=)!t!Lz6?6Kl@`X%_)#khrKcC?Ctst+NArwVwC&MBX0kI z>$~v%X?cq`r-M`GT^Xt_c%fRj)FumF*vL)A@C@^I^;&|k{JvrA1^CdjC&@OpjumqN z(^^#fTm@KD-9_hoS&+8%q*%y0E$L*`C6qk5GFY1ZgJqWs_PL z_w&K~VB>aI)_Vc3Xby47_r@8%Ih^J&EWO0NQ{qIB?F??tFEua1q zcsH{B9Yv0~Q~oQ!{S$sS^!y!8f-ubg!2cbA{)xUDGX9QUL7@ML9RCF0joy9-VQ92ZJ<9I{P^|w{@n0RwpDcH$^WRy55&tM4e#U=I z?te1eou+^q+hG05@y# ANdN!< literal 0 HcmV?d00001 diff --git a/testdata/lists/description_en.xlsx b/testdata/lists/description_en.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..c7a88d3aa1c84d38d7b703aad2e36854f2e6f4d6 GIT binary patch literal 6334 zcmaJ_1yq#ZzMX+lxf!7cZ{h_E@6BEtM>G!`_|o~{lC36^4hN6rw!F2_ zY_@bg$bLHXzTu;I7vnqvLeh+^oe~HD5FmB2a5PtUadd)l zn>)HdxjgOd{+l}Xu`0HqQus1E*O+N7^$v6(y`xm{Jro6#N@>KHX(nEg7f*%)2fk z6gSYBzN~3DbZE6W4x$w69G}l z@{Rv{@up5rf2%!n_!DL~4?bwoC!1kqzQ`N@!BrFS2+f<5M7!C*8Jt1=4Fn~DFCeam zhjs$DB_IOF%q28aT&s>aFg?1bFsWf(Gi!B}bX!0(Z~!-P0985DFikpNYn=U5M7ybP zPB+s3JJoXDfPS$l)74>Md90hCW)*)95FAd++F@3+*HJ=$b@uY-1iOXWQ7`4~Y7B{4 z_P--5aYsH%JY+SVcHAD0F1BWlj<$a*CPr<@0a;Az0m+y7D4aaj7&BNY{+zYQYfmDB zQ-T=>>6>;I){_1#cT0B<_xhQy8O#f0Nf=o@HiLmVjv;*QKX#pGJG82ZqAaoZyfMBh zerQy4TgidZ*eHU_#s+J#QP0OWpSbdoQut6b57(a3%yPpeCIrk4D8@b*sxsS$gYX(| zJPYabv=kUEpVLBADe&@|iq9Vkv+QAe6|cAp8iyrFpRGObd;O#jBRl3pv-stCy`K%* z!?t1Ba^2p*$nb04Y=@B)&-M5o1>;eDLtxruK3j{FnLTENTwR*@cUjSXqNr!LWdeRQ zpjkl5REl&0ieEDQGK1g6`hSe!6Ve#)LwCF|-<{~Y>nlD;fMJ9gA`XF}t@X-w3RVr0 zUKb8vc{B7}wCwlMwfx!fkj))u_3T%raa~f{$vHfY(}%hIyX`(rKEZBnx@}7)da?04 zL=Vz1EIRp%pH~ceQE`&+n#N}eejlqIm|M5_I`3Zc5_(p00KGezq+Dl@yo%WeC94_A`Hx_4FdOuzo7Z1KPIyiQLNY)e58em!mBm9J_I6w4oD4yEb+_(t@h}AF9 zc8D;*ZB)QE!1N($zPc&JRd_6;^Z?7gQ7RZ0>dc~V9K*yNScNQiHQC)Svh;6JsE6Ls9&*g3J&{0NoK z6g_`!B#5E7)Cvgv~1)GJ-oX|GT_ zmAOWf3^urmc`<+|3DuNu661BV*TZcdt;8qF0djru7q`}wiLMXCM%P?tzw`Dfp3JhT zlC@}gL}w{Y7^J;pN|!EDc%^tWxnhg2`;53&zq%AVyty^B9VHJjuMH3+RR8Nt)$?d+dg8jiSsRqldz zEm7uJIuNGC&oCv+3D&X?Y+b+M*{!2Cwge2H9CuT;bX1WJ|C}5bF3O!vnqbN;w<>5j zT}QgZ`~XxJ(R0M>~J zsx56hK5&^FSIo-k+t_qaljfE1za0JHZ0poqnq3zXjaJ#QQy;y(&3kOJE82(aZAUF` zXH(oFllg@|L%}UpduE?~Evt)YDV#)v53WVeA>)y<^GMvj>$vf+fPPF2y~Dff|&Ob()0UpM^T z?^%0aj|Q2d{pRb@<&+z{h@AS~>y_CS5Tge+5kfV>-E2m6SH(M(cYfA(JOvT~-kwa0d8sD2YbC`a1UG+MBPoKY(o_}O_ov`1ad_=|P24sE&S2Du?FfKh` z7)LFJWeZu#v-p7nJJ2aAh)UkdT{-p0UlEj5G)tx;XON3$^rIpdq}q#T97vFe^~90$ zT|sv#h-38cNdSlhQ>`z{k2(+Gk&845q!9@Qdat10T}xm@PQ>1t7$CqN zWt8*(0q~=KsnZa5XYK2^63QIM+;JHV(|1gqHx`yAH*okJR_l^aj}u*I^CB4P*6W(ki;4TX9Ui zEi2W{gClIuPCrFa|H)yuc*gKMg?k31jxE==xU~7r`qgTy!^)QV8BOtK*bUG@@yVr$ zh|XYuz@e}ql@H?}k@-S@lG;)C%g&xu>VQ?1tETn!90ma+oeM~Gm%$f41+3+0V0yI{ zv&8NX0=|o#qI(7+bQ!A}#F;4smz=c!u$WteH}0VLd{IckG`UOP{JSVYWwXP*AF;U7!P;3wmABklu?vcoB1LHZ*D!v=aITG2t{wc>FJUd zM#TrD_7tMB=xC6r*hWIB_s6MN@GV-vq4lC>+0 zs37Lei+C8RTRe&iwabyuU?pUiH7k_Imo3*|5)GqaB|??mkV(*gpvf+4Q`q=XUi&GM zB6~x(6vz(i{!p%)M#5fW21RH8T*XJt&Tq1+s@p`u-e9&0DW@`pQb69Cuih>1#we>Y zBY~8on66Sla^FI^pA|kSX2mAPZ?^Zz^;^KIpwoF!nbex7X{BbRJXSw6D)XbURBUU* zq!?9bo0c4)4sBl|s(?t4Z~D#h}iKNZrLZJbdo{ zIsDpqPh|qnSZT_o%ZVjG>=aZOdk0aPFCHZ=GDSp0c)?d)sI78FmVh`&=iT!qR6e+Q z-0T0Nf>I&^&FdVx@Mx^`eyziM#ulro(gT?EASEQlMj&h-wVu|hbc9Fnz*E{MH$59G zpucfZo6*XZ}H2ruPUMSFa{lo1LUzSCmwpR=qoon zl3zg1O(X-8DLM4+OAQ3a8Zi3Vwu2-Yf|8U$8yiy0I=ormyx%$(j#HUqS9jX&!|dWP zv|=I_og=kBK$Q$f*uXUOTCXQdBj({sD~|c1BHa$8CZ^L|7AMV8dIOk0$5MofMK0Pf zc9n$9s!;?cJ`|Q%`!=|CWd?wxS;T)Z36>B(#<1bna2D!ceRDsxqbA*@@M?qTw!+^h zfA(Y`)kVVr0C4`Lu&b9H^smx>)iJO|@DTg&RN{L-<1F5`f^S#fsopNcp6xb<8n}b1 zEi4LQ=bIj`&Re0O%;eGT;SkY=os6A<>g1JnN07H^^Ri-*a61zjuU<*wkZA9SS`=Qx zsYHvsXwAJX=fPKx@#$KKxnry)Wi@m-^rTyqKJ#IuDz1l9;JZ<~Ran+;*{L(;k<-)W zhr7I}k$IagYc8r)lJYR#E&qxCsc{h*=AH>myP7xvnrO}Dl+3JwK9Z<7NU%!bBb(}4 zxfX`mnW=a31f!7PCG#fv;=F!l0nUL0jLq|HQ_s;@vAM7LQW&|h`e7$zi)ZrF+3GI9 z6o<0)luBG^dfu5{u&t%rIDwf{Al$kM01P{uXb!s*uXxvHcn5F3u4vQK9p)=l4C1yY zIr4oyn%O%LlKM6p(X6j*6WVX*^ASa>}Y@{Qi_~#tSiF#$Ckwh;?knn{YjPi8QHJ zX}W7Mlsgg1v=eJY8~1No{STEUj$a&Eo)Wijein|wIjfv*eOIT)_URdLZvP#eOae|T z@{-1@>@d&il``&N1*1#8?D0~{5}J~{$)rv$?O7yNPvOhFb-g>UpprD;VH+!4kL?N$ z3)nxU)fM@%^cF)Q<(JSwqC^w;XTLF;-BYvoOtPW4Yw^ z`P~BWF4(-2>(GI%THCTs%f_$9&19f7$1rQJfmxulJ+teo!Gn7#`?yJT?rm@j>1bq@ zv-|1e@GCdZngDiKS84-^{`h>j$ItVUfRd0}$-KcHImi@K5mVNbGkoCh{PU z1pg%<`;+H(Ui3SU05U~F^8A?|{Yh}Ucle#a2J7Ed{6{D8C(G@f{dX2ZWYUlP8~?G- z|H*KB=lY$&mEae{zxS~}p|_pkcc>b2jr|v&P*=KxG!Xy*6Zr~6J}Xva-vj{s522@# AZU6uP literal 0 HcmV?d00001 diff --git a/testdata/test-onto.json b/testdata/test-onto.json index 35b019275..8caa5cce5 100644 --- a/testdata/test-onto.json +++ b/testdata/test-onto.json @@ -57,13 +57,12 @@ ] }, { - "name": "fromexcel", + "name": "my-list-from-excel", "labels": { - "en": "Fromexcel" + "en": "My list from Excel" }, "nodes": { - "file": "list-as-excel.xlsx", - "worksheet": "Tabelle1" + "folder": "testdata/lists" } } ],