fix: catch network interruptions during onto creation (DEV-1073) (#210)

 - If there are temporary problems during the ontology creation process (like e.g. short connectivity interruption), dsp-tools will several times retry instead of giving up immediately. 
 - The console output during the onto creation is now much more informative.
 - No more "lists.json" file is written during each onto creation.
 - No more doubling: the onto isn't validated two times any more, before being created on the DSP server, and the Excel file references in the “lists” section aren’t expanded two times any more.
 - Thorough code refactoring of the onto creation process.

docs/dsp-tools-create.md

@@ -363,12 +363,12 @@ The groups that were created here are then available in the XML file in the
 
 A group definition has the following elements:
 
-- _name_: name of the group, mandatory
-- _descriptions_: description of the group with language tags in the form `"descriptions": {"<lang>": "<string>", ...}` (
-  currently "en", "de", "fr", "it", and "rm" are supported), mandatory
-- _selfjoin_: true if users are allowed to join the group themselves, false if an administrator has to add the users,
-  optional
-- _status_: true if the group is active, false if the group is inactive, optional
+- _name_ (mandatory): name of the group
+- _descriptions_ (mandatory): description of the group with language tags in the form `"descriptions": {"<lang>": 
+  "<string>", ...}` (currently "en", "de", "fr", "it", and "rm" are supported)
+- _selfjoin_ (optional): true if users are allowed to join the group themselves, false (default) if an administrator has 
+  to add them
+- _status_ (optional): true (default) if the group is active, false if the group is inactive
 
 Example:
 
@@ -399,13 +399,16 @@ This object contains user definitions. A user has the following elements:
 - _familyName_: surname of the user
 - _password_: password of the user
 - _lang_: the default language of the user: "en", "de", "fr", "it" (optional, default: "en")
-- _groups_: List of groups the user belongs to. The name of the group has to be provided with the project's shortname,
-  p.ex. "shortname:editors". The project defined in the same ontology file has no name, so only ":editors" is required
-  if the user belongs to the group "editors". (optional)
-- _projects_: List of projects the user belongs to. The project name has to be followed by a ":" and either "member"
-  or "admin". This indicates if the new user has admin rights in the given project or is an ordinary
-  user. `myproject:admin` would add the user as admin to the project "myproject". The given project defined in the same
-  ontology file has no name, so only ":admin"or ":member" is required. (optional)
+- _groups_ (optional): List of groups the user belongs to. The group names must be provided in one of the following forms:
+    - `other_project_shortname:groupname`
+    - `:groupname` (for groups defined in the current ontology file)
+    - `SystemAdmin` (the most powerful group, built-in into DSP)
+- _projects_ (optional): List of projects the user belongs to. The project name has to be followed by a `:` and either 
+  `member` or `admin`. This indicates if the new user has admin rights in the given project or is an ordinary
+  user. `myproject:admin` would add the user as admin to the project `myproject`. The project defined in the same
+  ontology file can be omitted, so only `:admin` or `:member` is enough.
+    - If _projects_ is omitted, the user won't be part in any project.
+- _status_ (optional): true (default) if the user is active, false if the user is deleted/inactive
 
 Example:
 
@@ -420,12 +423,14 @@ Example:
       "password": "biz1234",
       "lang": "en",
       "groups": [
-        ":biz-editors"
+        ":biz-editors",
+        "SystemAdmin"
       ],
       "projects": [
         ":admin",
         "otherProject:member"
-      ]
+      ], 
+      "status": true
     }
   ]
 }

docs/dsp-tools-usage.md

@@ -35,10 +35,10 @@ The following options are available:
 - `-s` | `--server` _server_: URL of the DSP server (default: 0.0.0.0:3333)
 - `-u` | `--user` _username_: username used for authentication with the DSP API (default: root@example.com)
 - `-p` | `--password` _password_: password used for authentication with the DSP API (default: test)
-- `-V` | `--validate`: If set, only the validation of the JSON file is performed.
-- `-l` | `--lists`: If set, only the lists are created using a [simplified schema](./dsp-tools-create.md#lists). Please
-  note that in this case the project must already exist.
-- `-v` | `--verbose`: If set, some information about the progress is printed to the console.
+- `-V` | `--validate-only`: If set, only the validation of the JSON file is performed.
+- `-l` | `--lists-only`: If set, only the lists are created. Please note that in this case the project must already exist.
+- `-v` | `--verbose`: If set, more information about the progress is printed to the console.
+- `-d` | `--dump`: If set, dump test files for DSP-API requests.
 
 The command is used to read the definition of a data model (provided in a JSON file) and create it on the DSP server.
 The following example shows how to load the ontology defined in `data_model_definition.json` onto the DSP

knora/dsp_tools.py

@@ -11,9 +11,9 @@
 from knora.dsplib.utils.excel_to_json_resources import resources_excel2json
 from knora.dsplib.utils.id_to_iri import id_to_iri
 from knora.dsplib.utils.onto_create_lists import create_lists
-from knora.dsplib.utils.onto_create_ontology import create_ontology
+from knora.dsplib.utils.onto_create_ontology import create_project
 from knora.dsplib.utils.onto_get import get_ontology
-from knora.dsplib.utils.onto_validate import validate_ontology
+from knora.dsplib.utils.onto_validate import validate_project
 from knora.dsplib.utils.xml_upload import xml_upload
 
 
@@ -55,12 +55,10 @@ def program(user_args: list[str]) -> None:
     parser_create.add_argument('-s', '--server', type=str, default=default_localhost, help=url_text)
     parser_create.add_argument('-u', '--user', default=default_user, help=username_text)
     parser_create.add_argument('-p', '--password', default=default_pw, help=password_text)
-    parser_create.add_argument('-V', '--validate', action='store_true',
+    parser_create.add_argument('-V', '--validate-only', action='store_true',
                                help='Do only validation of JSON, no upload of the '
                                     'ontology')
-    parser_create.add_argument('-L', '--listfile', type=str, default='lists-only.json',
-                               help='Name of list node informationfile')
-    parser_create.add_argument('-l', '--lists', action='store_true', help='Upload only the list(s)')
+    parser_create.add_argument('-l', '--lists-only', action='store_true', help='Upload only the list(s)')
     parser_create.add_argument('-v', '--verbose', action='store_true', help=verbose_text)
     parser_create.add_argument('-d', '--dump', action='store_true', help='dump test files for DSP-API requests')
     parser_create.add_argument('datamodelfile', help='path to data model file')
@@ -142,31 +140,27 @@ def program(user_args: list[str]) -> None:
         exit(0)
 
     if args.action == 'create':
-        if args.lists:
-            if args.validate:
+        if args.lists_only:
+            if args.validate_only:
                 validate_list_with_schema(args.datamodelfile)
             else:
                 create_lists(input_file=args.datamodelfile,
-                             lists_file=args.listfile,
                              server=args.server,
                              user=args.user,
                              password=args.password,
-                             verbose=args.verbose,
                              dump=args.dump)
         else:
-            if args.validate:
-                if validate_ontology(args.datamodelfile):
+            if args.validate_only:
+                if validate_project(args.datamodelfile):
+                    print('Data model is syntactically correct and passed validation.')
                     exit(0)
-                else:
-                    exit(1)
             else:
-                create_ontology(input_file=args.datamodelfile,
-                                lists_file=args.listfile,
-                                server=args.server,
-                                user_mail=args.user,
-                                password=args.password,
-                                verbose=args.verbose,
-                                dump=args.dump if args.dump else False)
+                create_project(input_file=args.datamodelfile,
+                               server=args.server,
+                               user_mail=args.user,
+                               password=args.password,
+                               verbose=args.verbose,
+                               dump=args.dump if args.dump else False)
     elif args.action == 'get':
         get_ontology(project_identifier=args.project,
                      outfile=args.datamodelfile,

knora/dsplib/models/group.py

@@ -240,23 +240,18 @@ def delete(self):
         return Group.fromJsonObj(self._con, result['group'])
 
     @staticmethod
-    def getAllGroups(con: Connection) -> Optional[list[Group]]:
-        try:
-            result = con.get(Group.ROUTE)
-            return [Group.fromJsonObj(con, group_item) for group_item in result["groups"]]
-        except BaseError:
-            # return None if no groups are found or an error happened
-            return None
+    def getAllGroups(con: Connection) -> list[Group]:
+        result = con.get(Group.ROUTE)
+        return [Group.fromJsonObj(con, group_item) for group_item in result["groups"]]
 
     @staticmethod
     def getAllGroupsForProject(con: Connection, proj_shortcode: str) -> Optional[list[Group]]:
-        all_groups: Optional[list[Group]] = Group.getAllGroups(con)
-        if all_groups:
-            project_groups = []
-            for group in all_groups:
-                if group.project == "http://rdfh.ch/projects/" + proj_shortcode:
-                    project_groups.append(group)
-            return project_groups
+        all_groups = Group.getAllGroups(con)
+        project_groups = []
+        for group in all_groups:
+            if group.project == "http://rdfh.ch/projects/" + proj_shortcode:
+                project_groups.append(group)
+        return project_groups
 
     def createDefinitionFileObj(self):
         group = {

knora/dsplib/models/project.py

@@ -446,7 +446,8 @@ def read(self) -> Project:
         if result is not None:
             return Project.fromJsonObj(self._con, result['project'])
         else:
-            return None  # Todo: throw exception
+            raise BaseError(f"ERROR: Could not read project '{self.shortname}' ({self.shortcode}) with IRI {self._id} "
+                            f"from DSP server.")
 
     def update(self) -> Project:
         """