diff --git a/.gitignore b/.gitignore index 894a44cc0..b22bff296 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,4 @@ venv.bak/ # mypy .mypy_cache/ +.idea diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000..94a25f7f4 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 1a14ccdd5..313041ec5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,512 @@ # knora-py -A Python library and tools for the Knora-API +knora-py is a toolbox to create data models (ontologies) and for mass upload of data into the Knora framework. + +The famework consists of +- ```knora.py``` Python modules for accessing Knora using the API (ontology creation, data import/export etc.) +- ```create_ontology.py``` A program to create an ontology out of a simple JSON description +- ```knoraConsole.py``` A graphical console application + +## Content +- [creating an ontology](#create-ontology.py) +- [Bulk data import](#bulk-data-import) + +## create_ontology.py +This script reads a JSON file containing the data model (ontology) definition, +connects to the Knora server and creates the data model. +usage: + +```bash +python3 create_ontology.py data_model_definition.json +``` +It supports the foloowing options: + +- _"-s server" | "--server server"_: The URl of the Knora server [default: localhost:3333] +- _"-u username" | "--user username"_: Username to log into Knora [default: root@example.com] +- _"-p password" | "--password password"_: The password for login to the Knora server [default: test] +- _"-v" | "--validate"_: If this flag is set, only the validation of the json is run +- _"-l" | "--lists"_: Only create the lists using [simplyfied schema](#json-for-lists). Please note + that in this case the project __must__ exist. + +## JSON ontology definition format + +The JSON file contains a first object an object with the ```prefixes``` for +external ontologies that are being used, followed by the definition of +the project wic h includes all resources and properties: + +### Prefixes + +```json +{ + "prefixes": { + "foaf": "http://xmlns.com/foaf/0.1/", + "dcterms": "http://purl.org/dc/terms/" + }, + "project": {…}, + +} +``` + +### Project data +The project definitions requires + +- _"shortcode"_: A hexadecimal string in the range between "0000" and "FFFF" uniquely identifying the project. +- _"shortname"_: A short name (string) +- a _"longname"_: A longer string giving the full name for the project +- _descriptions_: Strings describing the projects content. These + descriptions can be supplied in several languages (currently _"en"_, _"de"_, _"fr"_ and _"it"_ are supported). + The descriptions have to be given as JSON object with the language as key + and the description as value. At least one description in one language is required. +- _keywords_: An array of keywords describing the project. +- _lists_: The definition of flat or hierarchical list (thesauri, controlled vocabularies) +- _ontology_: The definition of the data model (ontology) + +This a project definition lokks like follows: + +```json +"project": { + "shortcode": "0809", + "shortname": "test" + "longname": "Test Example", + "descriptions": { + "en": "This is a simple example project with no value.", + "de": "Dies ist ein einfaches, wertloses Beispielproject" + } + "keywords": ["example", "senseless"], + "lists": […], + "ontology": {…} +} +``` + +### Lists +A List consists of a root node identifing the list and an array of subnodes. +Each subnode may contain again subnodes (hierarchical list). +A node has the following elements: + +- _name_: Name of the node. Should be unique for the given list +- _labels_: Language dependent labels +- _comments_: language dependent comments (optional) +- _nodes_: Array of subnodes (optional – leave out if there are no subnodes) + +The _lists_ object contains an array of lists. Here an example: + +```json + "lists": [ + { + "name": "orgtpye", + "labels": { "de": "Organisationsart", "en": "Organization Type" }, + "nodes": [ + { + "name": "business", + "labels": { "en": "Commerce", "de": "Handel" }, + "comments": { "en": "no comment", "de": "kein Kommentar" }, + "nodes": [ + { + "name": "transport", + "labels": { "en": "Transportation", "de": "Transport" } + }, + { + "name": "finances", + "labels": { "en": "Finances", "de": "Finanzen" } + } + ] + }, + { + "name": "society", + "labels": { "en": "Society", "de": "Gesellschaft" } + } + ] + } + ] +``` +the _lists_ element is optional. + +## Ontology + +The _ontology_ object contains the definition of the data model. The ontology has +the following elemens: + +- _name_: The name of the ontology. This has to be a CNAME conformant name that can be use as prefix! +- _label_: Human readable and understandable name of the ontology +- _resources_: Array defining the resources (entities) of the data model + +```json + "ontology": { + "name": "teimp", + "label": "Test import ontology", + "resources": […] + } +``` + +### Resources +The resource classes are the primary entities of the data model. A resource class +is a template for the representation of a real object that is represented in +the DaSCh database. A resource class defines properties (aka _data fields_). For each of +these properties a data type as well as the cardinality have to defined. + +A resource consists of the following definitions: + +- _name_: A name for the resource +- _label_: The string displayed of the resource is being accessed +- _super_: A resource class is always derived from an other resource. The + most generic resource class Knora offers is _"Resource"_. The following + parent predefined resources are provided by knora: + - _Resource_: A generic "thing" that represents an item from the reral world + - _StillImageRepresentation_: An object that is connected to a still image + - _TextRepresentation_: An object that is connected to an (external) text (Not Yet Implemented) + - _AudioRepresentation_: An object representing audio data (Not Yet Implemented) + - _DDDRepresentation_: An object representing a 3d representation (Not Yet Implemented) + - _DocumentRepresentation_: An object representing a opaque document (e.g. a PDF) + - _MovingImageRepresentation_: An object representing a moving image (video, film) + - _Annotation_: A predefined annotation object. It has the following properties + defined: + - _hasComment_ (1-n), _isAnnotationOf_ (1) + - _LinkObj_: An resource class linking together several other, generic, resource classes. The class + has the following properties: _hasComment_ (1-n), _hasLinkTo_ (1-n) + - _Region_: Represents a simple region. The class has the following properties: + _hasColor_ (1), _isRegionOf_ (1) _hasGeometry_ (1), _isRegionOf_ (1), _hasComment_ (0-n) + + However, a resource my be derived from a resource class in another ontology within the same project or + from another resource class in the same ontology. In this case the reference + has to have the form _prefix_:_resourceclassname_. +- _labels_: Language dependent, human readable names +- _comments_: Language dependend comments (optional) +- _properties_: Array of property definition for this resource class. + +Example: + +```json + "resources": [ + { + "name": "person", + "super": "Resource", + "labels": { "en": "Person", "de": "Person" }, + "comments": { + "en": "Represents a human being", + "de": "Repräsentiert eine Person/Menschen" + }, + "properties": […] + } +``` + +#### Properties +Properties are the definition of the data fields a resource class may or must have. +The properties object has the following fields: + +- _name_: A name for the property +- _super_: A property has to be derived from at least one base property. The most generic base property + Knora offers is _hasValue_. In addition the property may by als a subproperty of + properties defined in external ontologies. In this case the qualified name including + the prefix has to be given. + The following base properties are definied by Knora: + - _hasValue_: This is the most generic base. + - _hasLinkTo_: This value represents a link to another resource. You have to indicate the + the "_object_" as a prefixed IRI that identifies the resource class this link points to. + - _hasColor_: Defines a color value (_ColorValue_) + - _hasComment_: Defines a "standard" comment + - _hasGeometry_: Defines a geometry value (a JSON describing a polygon, circle or rectangle), see _ColorValue_ + - _isPartOf_: A special variant of _hasLinkTo_. It says that an instance of the given resource class + is an integral part of another resource class. E.g. a "page" is a prt of a "book". + - _isRegionOf_: A special variant of _hasLinkTo_. It means that the given resource class + is a "region" of another resource class. This is typically used to describe regions + of interest in images. + - _isAnnotationOf_: A special variant of _hasLinkTo_. It denotes the given resource class + as an annotation to another resource class. + - _seqnum_: An integer that is used to define a sequence number in an ordered set of + instances. +- _object_: The "object" defines the type of the value that the property will store. + The following object types are allowed: + - _TextValue_: Represents a text that may contain standoff markup + - _ColorValue_: A string in the form "#rrggbb" (standard web color format) + - _DateValue_: represents a date. It is a string having the format "_calendar":"start":"end" + - _calender_ is either _GREGORIAN_ or _JULIAN_ + - _start_ has the form _yyyy_-_mm_-_dd_. If only the year is given, the precision + is to the year, of only the year and month are given, the precision is to a month. + - _end_ is optional if the date represents a clearely defined period or uncertainty. + In total, a DateValue has the following form: "GREGORIAN:1925:1927-03-22" + which means antime in between 1925 and the 22nd March 1927. + - _DecimalValue_: a number with decimal point + - _GeomValue_: Represents a geometrical shape as JSON. + - _GeonameValue_: Represents a location ID in geonames.org + - _IntValue_: Represents an integer value + - _BooleanValue_: Represents a Boolean ("true" or "false) + - _UriValue_: : Represents an URI + - _IntervalValue_: Represents a time-interval + - _ListValue_: Represents a node of a (possibly hierarchical) list +- _labels_: Language dependent, human readable names +- _gui_element_: The gui_element is – strictly seen – not part of the data. It gives the + generic GUI a hint about how the property should be presented to the used. Each gui_element + may have associated gui_attributes which contain further hints. + There are the following gui_elements available: + - _Colorpicker_: The only GUI element for _ColorValue_. Let's You pick a color. It requires the attribute "ncolors=integer" + - _Date_: The only GUI element for _DateValue_. A date picker gui. No attributes + - _Geometry_: Not Yet Implemented. + - _Geonames_: The only GUI element for _GeonameValue_. Interfaces with geonames.org and allows to select a location + - _Interval_: Not Yet Implemented. + - _List_: A list of values. The Attribute "hlist=" is mandatory! + - _Pulldown_: A GUI element for _ListValue_. Pulldown for list values. Works also for hierarchical lists. The Attribute "hlist=" is mandatory! + - _Radio_: A GUI element for _ListValue_. A set of radio buttons. The Attribute "hlist=" is mandatory! + - _SimpleText_: A GUI element for _TextValue_. A simple text entry box (one line only). The attributes "maxlength=integer" and "size=integer" are optional. + - _Textarea_: A GUI element for _TextValue_. Presents a multiline textentry box. Optional attributes are "cols=integer", "rows=integer", "width=percent" and "wrap=soft|hard". + - _Richtext_: A GUI element for _TextValue_. Provides a richtext editor. + - _Searchbox_: Must be used with _hasLinkTo_ properties. Allows to search and enter a resource that the given resource should link to. The Attribute "numprops=integer" + indicates how many properties of the found resources should be indicated. It's mandatory! + - _Slider_: A GUI element for _DecimalValue_. Provides a slider to select a decimal value. The attributes "max=decimal" and "min=decimal" are mandatory! + - _Spinbox_: A GUI element for _IntegerValue_. A text field with and "up"- and "down"-button for increment/decrement. The attributes "max=decimal" and "min=decimal" are optional. + - _Checkbox_: A GUI element for _BooleanValue_. + - _Fileupload_: not yet documented! +- _gui_attributes_: See above +- _cardinality_: The cardinality indicates how often a given property may occur. The possible values + are: + - "1": Exactly once (mandatory one value and only one) + - "0-1": The value may be omitted, but can occur only once + - "1-n": At least one value must be present. But multiple values may be present. + - "0-n": The value may be omitted, but may also occur multiple times. + +### A complete example for a full ontology + +```json +{ + "prefixes": { + "foaf": "http://xmlns.com/foaf/0.1/", + "dcterms": "http://purl.org/dc/terms/" + }, + "project": { + "shortcode": "0170", + "shortname": "teimp", + "longname": "Test Import", + "descriptions": { + "en": "This is a project for testing the creation of ontologies and data", + "de": "Dies ist ein Projekt, um die Erstellung von Ontologien und Datenimport zu testen" + }, + "keywords": ["test", "import"], + "lists": [ + { + "name": "orgtpye", + "labels": { + "de": "Roganisationsart", + "en": "Organization Type" + }, + "nodes": [ + { + "name": "business", + "labels": { + "en": "Commerce", + "de": "Handel" + }, + "comments": { + "en": "no comment", + "de": "kein Kommentar" + }, + "nodes": [ + { + "name": "transport", + "labels": { + "en": "Transportation", + "de": "Transport" + } + }, + { + "name": "finances", + "labels": { + "en": "Finances", + "de": "Finanzen" + } + } + ] + }, + { + "name": "society", + "labels": { + "en": "Society", + "de": "Gesellschaft" + } + } + ] + } + ], + "ontology": { + "name": "teimp", + "label": "Test import ontology", + "resources": [ + { + "name": "person", + "super": "Resource", + "labels": { + "en": "Person", + "de": "Person" + }, + "comments": { + "en": "Represents a human being", + "de": "Repräsentiert eine Person/Menschen" + }, + "properties": [ + { + "name": "firstname", + "super": ["hasValue", "foaf:givenName"], + "object": "TextValue", + "labels": { + "en": "Firstname", + "de": "Vorname" + }, + "gui_element": "SimpleText", + "gui_attributes": ["size=24", "maxlength=32"], + "cardinality": "1" + }, + { + "name": "lastname", + "super": ["hasValue", "foaf:familyName"], + "object": "TextValue", + "labels": { + "en": "Lastname", + "de": "Nachname" + }, + "gui_element": "SimpleText", + "gui_attributes": ["size=24", "maxlength=64"], + "cardinality": "1" + }, + { + "name": "member", + "super": ["hasLinkTo"], + "object": "teimp:organization", + "labels": { + "en": "member of", + "de": "Mitglied von" + }, + "gui_element": "Searchbox", + "cardinality": "0-n" + } + ] + }, + { + "name": "organization", + "super": "Resource", + "labels": { + "en": "Organization", + "de": "Organisation" + }, + "comments": { + "en": "Denotes an organizational unit", + "de": "Eine Institution oder Trägerschaft" + }, + "properties": [ + { + "name": "name", + "super": ["hasValue"], + "object": "TextValue", + "labels": { + "en": "Name", + "de": "Name" + }, + "gui_element": "SimpleText", + "gui_attributes": ["size=64", "maxlength=64"], + "cardinality": "1-n" + }, + { + "name": "orgtype", + "super": ["hasValue"], + "object": "ListValue", + "labels": { + "en": "Organizationtype", + "de": "Organisationstyp" + }, + "comments": { + "en": "Type of organization", + "de": "Art der Organisation" + }, + "gui_element": "Pulldown", + "gui_attributes": ["hlist=orgtype"], + "cardinality": "1-n" + } + ] + } + ] + } + } +} +``` + +## JSON for lists + +The JSON schema for uploading hierarchical lists only is simplyfied: +```json +{ + "project": { + "shortcode": "abcd", + "lists": [] + } +} +``` +The definition of the lists is the same as in the full upload of an ontology! + +### A full example for creating lists only +The following JSON definition assumes that there is a project with the shortcode _0808_. + +```json +{ + "project": { + "shortcode": "0808", + "lists": [ + { + "name": "test1", + "labels": { + "de": "TEST1" + }, + "nodes": [ + { + "name": "A", + "labels": { + "de": "_A_" + } + }, + { + "name": "B", + "labels": { + "de": "_B_" + }, + "nodes": [ + { + "name": "BA", + "labels": { + "de": "_BA_" + } + }, + { + "name": "BB", + "labels": { + "de": "_BB_" + } + } + ] + }, + { + "name": "C", + "labels": { + "de": "_C_" + } + } + ] + } + ] + } +} +``` + +## Bulk data import +In order to make a bulk data import, a properly formatted XML file has to be created. The python module "knora.py" contains +classes and methods to facilitate the creation of such a XML file. + +## Requirements + +To install the requirements: + +```bash +$ pip3 install -r requirements.txt +``` + + +To generate a "requirements" file (usually requirements.txt), that you commit with your project, do: + +```bash +$ pip3 freeze > requirements.txt +``` + diff --git a/knora/__init__.py b/knora/__init__.py new file mode 100644 index 000000000..846e8299f --- /dev/null +++ b/knora/__init__.py @@ -0,0 +1 @@ +name = "knora" \ No newline at end of file diff --git a/knora/create_ontology.py b/knora/create_ontology.py new file mode 100644 index 000000000..963259551 --- /dev/null +++ b/knora/create_ontology.py @@ -0,0 +1,231 @@ +from typing import List, Set, Dict, Tuple, Optional +from pprint import pprint +import argparse +import json +from jsonschema import validate +from knora import KnoraError, knora + + +# parse the arguments of the command line +parser = argparse.ArgumentParser() +parser.add_argument("ontofile", help="path to ontology file") +parser.add_argument("-s", "--server", type=str, default="http://0.0.0.0:3333", help="URL of the Knora server") +parser.add_argument("-u", "--user", default="root@example.com", help="Username for Knora") +parser.add_argument("-p", "--password", default="test", help="The password for login") +parser.add_argument("-v", "--validate", action='store_true', help="Do only validation of JSON, no upload of the ontology") +parser.add_argument("-l", "--lists", action='store_true', help="Only create the lists") + +args = parser.parse_args() + + +def list_creator(con: knora, proj_iri: str, list_iri: str, parent_iri: str, nodes: List[dict]): + nodelist = [] + for node in nodes: + node_id = con.create_list_node( + name=node["name"], + project_iri=proj_iri, + labels=node["labels"], + comments=node.get("comments"), + parent_iri=parent_iri + ) + if node.get('nodes') is not None: + subnodelist = list_creator(con, proj_iri, list_iri, node_id, node['nodes']) + nodelist.append({node["name"]: {"id": node_id, 'nodes': subnodelist}}) + else: + nodelist.append({node["name"]: {"id": node_id}}) + return nodelist + + +# let's read the schema for the ontology definition +if args.lists: + with open('knora-schema-lists.json') as s: + schema = json.load(s) +else: + with open('knora-schema.json') as s: + schema = json.load(s) + +# read the ontology definition +with open(args.ontofile) as f: + ontology = json.load(f) + +# validate the ontology definition in order to be sure that it is correct +validate(ontology, schema) +print("Ontology is syntactically correct and passed validation!") + +if args.validate: + exit(0) + +# create the knora connection object +con = knora(args.server, args.user, args.password, ontology.get("prefixes")) + +# bulk_templ = con.create_schema(ontology["project"]["shortcode"], ontology["project"]["ontology"]["name"]) + +if not args.lists: + # create or update the project + try: + project = con.get_project(ontology["project"]["shortcode"]) + except KnoraError as err: + proj_iri = con.create_project( + shortcode=ontology["project"]["shortcode"], + shortname=ontology["project"]["shortname"], + longname=ontology["project"]["longname"], + descriptions=ontology["project"]["descriptions"], + keywords=ontology["project"]["keywords"] + ) + print("New project created: IRI: " + proj_iri) + else: + print("Updating existing project!") + print("Old project data:") + pprint(project) + proj_iri = con.update_project( + shortcode=ontology["project"]["shortcode"], + shortname=ontology["project"]["shortname"], + longname=ontology["project"]["longname"], + descriptions=ontology["project"]["descriptions"], + keywords=ontology["project"]["keywords"] + ) + project = con.get_project(ontology["project"]["shortcode"]) + print("New project data:") + pprint(project) +else: + project = con.get_project(ontology["project"]["shortcode"]) + proj_iri = project["id"] + +# now let's create the lists +lists = ontology["project"].get('lists') +listrootnodes = {} +if lists is not None: + for rootnode in lists: + rootnode_iri = con.create_list_node( + project_iri=proj_iri, + name=rootnode['name'], + labels=rootnode['labels'], + comments=rootnode.get('comments') + ) + listnodes = list_creator(con, proj_iri, rootnode_iri, rootnode_iri, rootnode['nodes']) + listrootnodes[rootnode['name']] = { + "id": rootnode_iri, + "nodes": listnodes + } + + +with open('lists.json', 'w', encoding="utf-8") as fp: + json.dump(listrootnodes, fp, indent=3, sort_keys=True) + +if args.lists: + print("The definitions of the node-id's can be found in \"lists.json\"!") + exit(0) + +# now we add the users if existing +users = ontology["project"].get('users') +if users is not None: + for user in users: + user_iri = con.create_user(username=user["username"], + email=user["email"], + givenName=user["givenName"], + familyName=user["familyName"], + password="password", + lang=user["lang"] if user.get("lang") is not None else "en") + con.add_user_to_project(user_iri, proj_iri) + +# now we start creating the ontology +# first we assemble the ontology IRI +onto_iri = args.server + "/ontology/" + ontology["project"]["shortcode"]\ + + "/" + ontology["project"]["ontology"]["name"] + "/v2" + +# test, if the ontolgy already exists. if so, let's delete it! +ontos = con.get_project_ontologies(ontology["project"]["shortcode"]) +if ontos is not None: + for onto in ontos: + if onto['iri'] == onto_iri: + con.delete_ontology(onto_iri, onto['moddate']) +onto_data = con.create_ontology( + onto_name=ontology["project"]["ontology"]["name"], + project_iri=proj_iri, + label=ontology["project"]["ontology"]["label"] +) + +onto_iri = onto_data['onto_iri'] +last_onto_date = onto_data['last_onto_date'] + +# let's create the resources +resource_ids = {} + +for resource in ontology["project"]["ontology"]["resources"]: + result = con.create_res_class( + onto_iri=onto_iri, + onto_name=ontology["project"]["ontology"]["name"], + last_onto_date=last_onto_date, + class_name=resource["name"], + super_class=resource["super"] if ':' in resource["super"] else "knora-api:" + resource["super"], + labels=resource["labels"] + ) + last_onto_date = result["last_onto_date"] + resource_ids[resource["name"]] = result["class_iri"] + +pprint(resource_ids) + +# let's create the properties +property_ids = {} +for resource in ontology["project"]["ontology"]["resources"]: + for prop in resource["properties"]: + guiattrs = prop.get("gui_attributes") + if guiattrs is not None: + new_guiattrs = [] + for guiattr in guiattrs: + parts = guiattr.split("=") + if parts[0] == "hlist": + new_guiattrs.append("hlist=<" + listrootnodes[parts[1]]["id"] + ">") + else: + new_guiattrs.append(guiattr) + guiattrs = new_guiattrs + + if prop.get("super") is not None: + super_props = list(map(lambda a: a if ':' in a else "knora-api:" + a, prop["super"])) + else: + super_props = ["knora-api:hasValue"] + + if prop.get("object") is not None: + object = prop["object"] if ':' in prop["object"] else "knora-api:" + prop["object"] + else: + object = None + + if prop.get("subject") is not None: + psubject = prop["subject"] + else: + psubject = ontology["project"]["ontology"]["name"] + ':' + resource["name"] + + result = con.create_property( + onto_iri=onto_iri, + onto_name=ontology["project"]["ontology"]["name"], + last_onto_date=last_onto_date, + prop_name=prop["name"], + super_props=super_props, + labels=prop["labels"], + gui_element="salsah-gui:" + prop["gui_element"], + gui_attributes=guiattrs, + subject=psubject, + object=object, + comments=prop.get("comments") + ) + last_onto_date = result["last_onto_date"] + property_ids[prop["name"]] = result['prop_iri'] + +# add the cardinalities +for resource in ontology["project"]["ontology"]["resources"]: + for prop in resource["properties"]: + print("=======>" + prop["name"] + "...") + + result = con.create_cardinality( + onto_iri=onto_iri, + onto_name=ontology["project"]["ontology"]["name"], + last_onto_date=last_onto_date, + class_iri=ontology["project"]["ontology"]["name"] + ':' + resource["name"], + prop_iri=ontology["project"]["ontology"]["name"] + ':' + prop["name"], + occurrence=prop["cardinality"] + ) + last_onto_date = result["last_onto_date"] + +con = None # force logout by deleting the connection object. + + diff --git a/knora/knora.py b/knora/knora.py new file mode 100755 index 000000000..48a3a9075 --- /dev/null +++ b/knora/knora.py @@ -0,0 +1,1131 @@ +from typing import List, Set, Dict, Tuple, Optional +from urllib.parse import quote_plus +from rdflib import Graph +from lxml import etree +import requests +import json +import urllib +import pprint +import validators +import re + + +# TODO: recheck all the documentation of this file +""" + Properties in knora-api: + + - :hasValue + - :hasColor + - :hasComment + - :hasGeometry + - :hasLinkTo + - :isPartOf + - :isRegionOf + - :isAnnotationOf + - :seqnum + + Classes in knora-api: + + - :Resource + - :StillImageRepresentation + - :TextRepresentation + - :AudioRepresentation + - :DDDRepresentation + - :DocumentRepresentation + - :MovingImageRepresentation + - :Annotation -> :hasComment, :isAnnotationOf, :isAnnotationOfValue + - :LinkObj -> :hasComment, :hasLinkTo, :hasLinkToValue + - :LinkValue [reification node] + - :Region -> :hasColor, :isRegionOf, :hasGeometry, :isRegionOfValue, :hasComment + + For lists: + + - :ListNode -> :hasSubListNode, :listNodePosition, :listNodeName, :isRootNode, :hasRootNode, :attachedToProject + + Values in knora-api: + + - :Value + - :TextValue -> :SimpleText, :TextArea + - :ColorValue -> :Colorpicker + - :DateValue -> :Date + - :DecimalValue -> :SimpleText + - :GeomValue -> :Geometry + - :GeonameValue -> :Geonames + - :IntValue -> :SimpleText, :Spinbox, :Slider + - :BooleanValue -> :Checkbox + - :UriValue -> :SimpleText + - :IntervalValue + - :ListValue -> :Pulldown + + GUI elements + + - :Colorpicker -> ncolors=integer + - :Date + - :Geometry + - :Geonames + - :Interval + - :List -> hlist(required)= + - :Pulldown -> hlist(required)= + - :Radio -> hlist(required)= + - :Richtext + - :Searchbox -> numprops=integer + - :SimpleText -> maxlength=integer, size=integer + - :Slider -> max(required)=decimal, min(required)=decimal + - :Spinbox -> max=decimal, min=decimal + - :Textarea -> cols=integer, rows=integer, width=percent, wrap=string(soft|hard) + - :Checkbox + - :Fileupload + +""" +class KnoraError(Exception): + """Handles errors happening in this file""" + + def __init__(self, message): + self.message = message + + +class knora: + """ + This is the main class which holds all the methods for communication with the Knora backend. + """ + + def __init__(self, server: str, email: str, password: str, prefixes: Dict[str,str] = None): + """ + Constructor requiring the server address, the user and password of KNORA + :param server: Adress of the server, e.g http://data.dasch.swiss + :param user: Username for Knora e.g., root@example.com + :param password: The password, e.g. test + """ + self.server = server + self.prefixes = prefixes + + credentials = { + "email": email, + "password": password + } + jsondata = json.dumps(credentials) + + req = requests.post(self.server + '/v2/authentication', + headers={'Content-Type': 'application/json; charset=UTF-8'}, + data=jsondata) + self.on_api_error(req) + + result = req.json() + self.token = result["token"] + + def __del__(self): + req = requests.delete(self.server + '/v2/authentication', + headers={'Authorization': 'Bearer ' + self.token}) + result = req.json() + + pprint.pprint(result) + + + + def on_api_error(self, res): + """ + Method to check for any API errors + :param res: The input to check, usually JSON format + :return: Possible KnoraError that is being raised + """ + + if (res.status_code != 200): + raise KnoraError("KNORA-ERROR: status code=" + str(res.status_code) + "\nMessage:" + res.text) + + if 'error' in res: + raise KnoraError("KNORA-ERROR: API error: " + res.error) + + def get_existing_projects(self, full: bool = False): + """Returns a list of existing projects + + :return: List of existing projects + """ + + req = requests.get(self.server + '/admin/projects', + headers={'Authorization': 'Bearer ' + self.token}) + self.on_api_error(req) + result = req.json() + + if 'projects' not in result: + raise KnoraError("KNORA-ERROR:\n Request got no projects!") + else: + if full: + return result['projects'] + else: + return list(map(lambda a: a['id'], result['projects'])) + + def get_project(self, shortcode: str) -> dict: + """Returns project data of given project + + :param shortcode: Shortcode of object + :return: JSON containing the project information + """ + + url = self.server + '/admin/projects/shortcode/' + shortcode + req = requests.get(url, headers={'Authorization': 'Bearer ' + self.token}) + self.on_api_error(req) + + result = req.json() + + return result["project"] + + def project_exists(self, proj_iri: str): + """Checks if a given project exists + + :return: Boolean + """ + + projects = self.get_existing_projects() + return proj_iri in projects + + def create_project( + self, + shortcode: str, + shortname: str, + longname: str, + descriptions: Optional[Dict[str, str]] = None, + keywords: Optional[List[str]] = None, + logo: Optional[str] = None) -> str: + """ + Create a new project + + :param shortcode: Dedicated shortcode of project + :param shortname: Short name of the project (e.g acronym) + :param longname: Long name of project + :param descriptions: Dict of the form {lang: descr, …} for the description of the project [Default: None] + :param keywords: List of keywords + :param logo: Link to the project logo [default: None] + :return: Project IRI + """ + + descriptions = list(map(lambda p: {"language": p[0], "value": p[1]}, descriptions.items())) + + project = { + "shortname": shortname, + "shortcode": shortcode, + "longname": longname, + "description": descriptions, + "keywords": keywords, + "logo": logo, + "status": True, + "selfjoin": False + } + + jsondata = json.dumps(project) + + req = requests.post(self.server + "/admin/projects", + headers={'Content-Type': 'application/json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + self.on_api_error(req) + + res = req.json() + return res["project"]["id"] + + def update_project( + self, + shortcode: str, + shortname: Optional[str] = None, + longname: Optional[str] = None, + descriptions: Optional[Dict[str, str]] = None, + keywords: Optional[List[str]] = None, + logo: Optional[str] = None) -> str: + """ + + :param shortcode: + :param shortname: + :param longname: + :param descriptions: + :param keywords: + :param logo: + :return: + """ + + descriptions = list(map(lambda p: {"language": p[0], "value": p[1]}, descriptions.items())) + + project = { + "longname": longname, + "description": descriptions, + "keywords": keywords, + "logo": logo, + "status": True, + "selfjoin": False + } + + jsondata = json.dumps(project) + url = self.server + '/admin/projects/iri/' + quote_plus("http://rdfh.ch/projects/" + shortcode) + + req = requests.put(url, + headers={'Content-Type': 'application/json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + self.on_api_error(req) + + res = req.json() + return res['project']['id'] + + def get_users(self): + """ + Get a list of all users + + :return: + """ + url = self.server + '/admin/users' + req = requests.get(url, headers={'Authorization': 'Bearer ' + self.token}) + + self.on_api_error(req) + res = req.json() + return res['users'] + + def get_user(self, user_iri: str): + """ + Get a list of all users + + :return: + """ + url = self.server + '/admin/users/' + quote_plus(user_iri) + req = requests.get(url, headers={'Authorization': 'Bearer ' + self.token}) + + self.on_api_error(req) + res = req.json() + return res['user'] + + def create_user(self, + username: str, + email: str, + givenName: str, + familyName: str, + password: str, + lang: str = "en"): + """ + Create a new user + + :param username: The username for login purposes (must be unique) + :param email: The email address of the user + :param givenName: The given name (surname, "Vorname", ...) + :param familyName: The family name + :param password: The password for the user + :param lang: language code, either "en", "de", "fr", "it" [default: "en"] + :return: The user ID as IRI + """ + + userinfo = { + "username": username, + "email": email, + "givenName": givenName, + "familyName": familyName, + "password": password, + "status": True, + "lang": lang, + "systemAdmin": False + } + + jsondata = json.dumps(userinfo) + url = self.server + '/admin/users' + + req = requests.post(url, + headers={'Content-Type': 'application/json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + + self.on_api_error(req) + res = req.json() + + return res['user']['id'] + + def add_user_to_project(self, user_iri: str, project_iri: str): + print("USER: " + user_iri) + print("PROJECT: " + project_iri) + url = self.server + '/admin/users/iri/' + quote_plus(user_iri) + '/project-memberships/' + quote_plus(project_iri) + print(url) + req = requests.post(url, headers={'Authorization': 'Bearer ' + self.token}) + self.on_api_error(req) + return None + + def get_existing_ontologies(self): + """ + + :return: Returns the metadata of all existing ontologies on v2/ontologies + """ + + req = requests.get(self.server + '/v2/ontologies/metadata', + headers={'Authorization': 'Bearer ' + self.token}) + result = req.json() + + if not '@graph' in result: + raise KnoraError("KNORA-ERROR:\n Request got no graph!") + else: + names = list(map(lambda a: a['@id'], result['@graph'])) + return names + + def get_project_ontologies(self, project_code: str) -> Optional[dict]: + """ + + :param project_code: + :return: + """ + + proj = quote_plus("http://rdfh.ch/projects/" + project_code) + req = requests.get(self.server + "/v2/ontologies/metadata/" + proj, + headers={'Authorization': 'Bearer ' + self.token}) + self.on_api_error(req) + result = req.json() + + if '@graph' in result: # multiple ontologies + ontos = list(map(lambda a: { + 'iri': a['@id'], + 'label': a['rdfs:label'], + 'moddate': a.get('knora-api:lastModificationDate') + }, result['@graph'])) + return ontos + elif '@id' in result: # single ontology + return [{ + 'iri': result['@id'], + 'label': result['rdfs:label'], + 'moddate': result.get('knora-api:lastModificationDate') + }] + else: + return None + + def ontology_exists(self, onto_iri: str): + """ + Checks if an ontology exists + + :param onto_iri: The possible ontology iri + :return: boolean + """ + + ontos = self.get_existing_ontologies() + + return onto_iri in ontos + + def get_ontology_lastmoddate(self, onto_iri: str): + """ + Retrieves the lastModificationDate of a Ontology + + :param onto_iri: The ontology to retrieve the lastModificationDate from. + :return: The lastModificationDate if it exists. Else, this method returns a dict with (id, None). If the ontology does not exist, it return None. + """ + + req = requests.get(self.server + '/v2/ontologies/metadata', + headers={'Authorization': 'Bearer ' + self.token}) + result = req.json() + + all_ontos = {} + + for onto in result['@graph']: + if 'knora-api:lastModificationDate' in onto: + all_ontos.__setitem__(onto['@id'], onto['knora-api:lastModificationDate']) + else : + all_ontos.__setitem__(onto['@id'], None) + + return all_ontos[onto_iri] + + def create_ontology(self, + onto_name: str, + project_iri: str, + label: str) -> Dict[str, str]: + """ + Create a new ontology + + :param onto_name: Name of the omntology + :param project_iri: IRI of the project + :param label: A label property for this ontology + :return: Dict with "onto_iri" and "last_onto_date" + """ + + ontology = { + "knora-api:ontologyName": onto_name, + "knora-api:attachedToProject": { + "@id": project_iri + }, + "rdfs:label": label, + "@context": { + "rdfs": 'http://www.w3.org/2000/01/rdf-schema#', + "knora-api": 'http://api.knora.org/ontology/knora-api/v2#' + } + } + + jsondata = json.dumps(ontology) + + req = requests.post(self.server + "/v2/ontologies", + headers={'Content-Type': 'application/json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + + self.on_api_error(req) + + res = req.json() + #TODO: return also ontology name + return {"onto_iri": res['@id'], "last_onto_date": res['knora-api:lastModificationDate']} + + def delete_ontology(self, onto_iri: str, last_onto_date = None): + """ + A method to delete an ontology from /v2/ontologies + + :param onto_iri: The ontology to delete + :param last_onto_date: the lastModificationDate of an ontology. None by default + :return: + """"" #TODO: add return documentation + url = self.server + "/v2/ontologies/" + urllib.parse.quote_plus(onto_iri) + req = requests.delete(url, + params={"lastModificationDate": last_onto_date}, + headers={'Authorization': 'Bearer ' + self.token}) + self.on_api_error(req) + res = req.json() + return req + + def get_ontology_graph(self, + shortcode: str, + name: str): + """ + Returns the turtle definition of the ontology. + + :param shortcode: Shortcode of the project + :param name: Name of the ontology + :return: + """ + url = self.server + "/ontology/" + shortcode + "/" + name + "/v2" + turtle = requests.get(url, + headers={"Accept": "text/turtle", + 'Authorization': 'Bearer ' + self.token}) + self.on_api_error(turtle) + return turtle.text + + def create_res_class(self, + onto_iri: str, + onto_name: str, + last_onto_date: str, + class_name: str, + super_class: List[str], + labels: Dict[str, str], + comments: Optional[Dict[str, str]] = None) -> Dict[str, str]: + """Creates a knora resource class + + :param onto_iri: IRI of the ontology + :param onto_name: Name of the ontology + :param last_onto_date: Last modification date as returned by last call + :param class_name: Name of the class to be created + :param super_class: List of super classes + :param labels: Dict with labels in the form { lang: labeltext } + :param comments: Dict with comments in the form { lang: commenttext } + :return: Dict with "class_iri" and "last_onto_date" + """ + + # + # using map and iterable to get the proper format + # + labels = list(map(lambda p: {"@language": p[0], "@value": p[1]}, labels.items())) + + if not comments: + comments = {"en": "none"} + + # + # using map and iterable to get the proper format + # + comments = list(map(lambda p: {"@language": p[0], "@value": p[1]}, comments.items())) + + res_class = { + "@id": onto_iri, + "@type": "owl:Ontology", + "knora-api:lastModificationDate": last_onto_date, + "@graph": [{ + "@id": onto_name + ":" + class_name, + "@type": "owl:Class", + "rdfs:label": labels, + "rdfs:comment": comments, + "rdfs:subClassOf": { + "@id": super_class + } + }], + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "knora-api": "http://api.knora.org/ontology/knora-api/v2#", + "owl": "http://www.w3.org/2002/07/owl#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + onto_name: onto_iri + "#" + } + } + + jsondata = json.dumps(res_class, indent=3, separators=(',', ': ')) + + req = requests.post(self.server + "/v2/ontologies/classes", + headers={'Content-Type': 'application/json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + self.on_api_error(req) + + res = req.json() + return {"class_iri": res['@graph'][0]['@id'], "last_onto_date": res['knora-api:lastModificationDate']} + + def create_property( + self, + onto_iri: str, + onto_name: str, + last_onto_date: str, + prop_name: str, + super_props: List[str], + labels: Dict[str, str], + gui_element: str, + gui_attributes: List[str] = None, + subject: Optional[str] = None, + object: Optional[str] = None, + comments: Optional[Dict[str, str]] = None + ) -> Dict[str, str]: + """Create a Knora property + + :param onto_iri: IRI of the ontology + :param onto_name: Name of the Ontology (prefix) + :param last_onto_date: Last modification date as returned by last call + :param prop_name: Name of the property + :param super_props: List of super-properties + :param labels: Dict with labels in the form { lang: labeltext } + :param gui_element: Valid GUI-Element + :param gui_attributes: Valid GUI-Attributes (or None + :param subject: Full name (prefix:name) of subject resource class + :param object: Full name (prefix:name) of object resource class + :param comments: Dict with comments in the form { lang: commenttext } + :return: Dict with "prop_iri" and "last_onto_date" keys + """ + # + # using map and iterable to get the proper format + # + labels = list(map(lambda p: {"@language": p[0], "@value": p[1]}, labels.items())) + + + if not comments: + comments = {"en": "none"} + + # + # using map and iterable to get the proper format + # + comments = list(map(lambda p: {"@language": p[0], "@value": p[1]}, comments.items())) + + additional_context = {} + for sprop in super_props: + pp = sprop.split(':') + if pp[0] != "knora-api": + additional_context[pp[0]] = self.prefixes[pp[0]] + + # + # using map and iterable to get the proper format + # + super_props = list(map(lambda x: {"@id": x}, super_props)) + if len(super_props) == 1: + super_props = super_props[0] + + propdata = { + "@id": onto_name + ":" + prop_name, + "@type": "owl:ObjectProperty", + "rdfs:label": labels, + "rdfs:comment": comments, + "rdfs:subPropertyOf": super_props, + "salsah-gui:guiElement": { + "@id": gui_element + } + } + if subject: + propdata["knora-api:subjectType"] = { + "@id": subject + } + + if object: + propdata["knora-api:objectType"] = { + "@id": object + } + + if gui_attributes: + propdata["salsah-gui:guiAttribute"] = gui_attributes + + property = { + "@id": onto_iri, + "@type": "owl:Ontology", + "knora-api:lastModificationDate": last_onto_date, + "@graph": [ + propdata + ], + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "knora-api": "http://api.knora.org/ontology/knora-api/v2#", + "salsah-gui": "http://api.knora.org/ontology/salsah-gui/v2#", + "owl": "http://www.w3.org/2002/07/owl#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + onto_name: onto_iri + "#" + } + } + property["@context"].update(additional_context) + jsondata = json.dumps(property, indent=3, separators=(',', ': ')) + + req = requests.post(self.server + "/v2/ontologies/properties", + headers={'Content-Type': 'application/json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + self.on_api_error(req) + + res = req.json() + return {"prop_iri": res['@graph'][0]['@id'], "last_onto_date": res['knora-api:lastModificationDate']} + + def create_cardinality( + self, + onto_iri: str, + onto_name: str, + last_onto_date: str, + class_iri: str, + prop_iri: str, + occurrence: str + ) -> Dict[str, str]: + """Add a property with a given cardinality to a class + + :param onto_iri: IRI of the ontology + :param onto_name: Name of the ontology (prefix) + :param last_onto_date: Last modification date as returned by last call + :param class_iri: IRI of the class to which the property will be added + :param prop_iri: IRI of the property that should be added + :param occurrence: Occurrence: "1", "0-1", "0-n" or "1-n" + :return: Dict with "last_onto_date" key + """ + switcher = { + "1": ("owl:cardinality", 1), + "0-1": ("owl:maxCardinality", 1), + "0-n": ("owl:minCardinality", 0), + "1-n": ("owl:minCardinality", 1) + } + occurrence = switcher.get(occurrence) + if not occurrence: + KnoraError("KNORA-ERROR:\n Invalid occurrence!") + + cardinality = { + "@id": onto_iri, + "@type": "owl:Ontology", + "knora-api:lastModificationDate": last_onto_date, + "@graph": [{ + "@id": class_iri, + "@type": "owl:Class", + "rdfs:subClassOf": { + "@type": "owl:Restriction", + occurrence[0]: occurrence[1], + "owl:onProperty": { + "@id": prop_iri + } + } + }], + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "knora-api": "http://api.knora.org/ontology/knora-api/v2#", + "owl": "http://www.w3.org/2002/07/owl#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + onto_name: onto_iri + "#" + } + } + + jsondata = json.dumps(cardinality, indent=3, separators=(',', ': ')) + + req = requests.post(self.server + "/v2/ontologies/cardinalities", + headers={'Content-Type': 'application/ld+json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + self.on_api_error(req) + + res = req.json() + + return {"last_onto_date": res["knora-api:lastModificationDate"]} + + def create_list_node(self, + project_iri: str, + labels: Dict[str, str], + comments: Optional[Dict[str, str]] = None, + name: Optional[str] = None, + parent_iri: Optional[str] = None) -> str: + """ + Creates a new list node. If there is no parent, a root node is created + + :param project_iri: IRI of the project + :param labels: Dict in the form {lang: label, …} giving the label(s) + :param comments: Dict in the form {lang: comment, …} giving the comment(s) + :param name: Name of the list node + :param parent_iri: None for root node (or omit), otherwise IRI of parent node + :return: IRI of list node + """ + + # + # using map and iterable to get the proper format + # + labels = list(map(lambda p: {"language": p[0], "value": p[1]}, labels.items())) + + + listnode = { + "projectIri": project_iri, + "labels": labels, + } + + # + # using map and iterable to get the proper format + # + if comments is not None: + listnode["comments"] = list(map(lambda p: {"language": p[0], "value": p[1]}, comments.items())) + else: + listnode["comments"] = [] + + if name is not None: + listnode["name"] = name + + if parent_iri is not None: + listnode["parentNodeIri"] = parent_iri + url = self.server + "/admin/lists/" + quote_plus(parent_iri) + else: + url = self.server + "/admin/lists" + + jsondata = json.dumps(listnode, indent=3, separators=(',', ': ')) + + + req = requests.post(url, + headers={'Content-Type': 'application/json; charset=UTF-8', + 'Authorization': 'Bearer ' + self.token}, + data=jsondata) + self.on_api_error(req) + + res = req.json() + + if parent_iri is not None: + return res['nodeinfo']['id'] + else: + return res['list']['listinfo']['id'] + + def get_lists(self, shortcode: str): + """ + Get the lists belonging to a certain project identified by its shortcode + :param shortcode: Project shortcode + + :return: JSON with the lists + """ + url = self.server + "/admin/lists?projectIri=" + quote_plus("http://rdfh.ch/projects/" + shortcode) + req = requests.get(url, headers={'Authorization': 'Bearer ' + self.token}) + self.on_api_error(req) + return req.json() + + def get_complete_list(self, list_iri: str): + """ + Get all the data (nodes) of a specific list + + :param list_iri: IRI of the list + :return: JSON containing the list info including all nodes + """ + url = self.server + "/admin/lists/" + quote_plus(list_iri) + req = requests.get(url, headers={'Authorization': 'Bearer ' + self.token}) + self.on_api_error(req) + return req.json() + + def list_creator(self, children: List): + """ + internal Helper function + + :param children: + :return: + """ + if len(children) == 0: + res = list(map(lambda a: {"name": a["name"], "id": a["id"]}, children)) + else: + res = list(map(lambda a: {"name": a["name"], "id": a["id"], "nodes": self.list_creator(a["children"])}, children)) + return res + + def create_schema(self, shortcode: str, shortname: str): + """ + This method extracts the ontology from the ontology information it gets from Knora. It + gets the ontology information as n3-data using the Knora API and concerts into a convenient + python dict that can be used for further processing. It is required by the bulk import ptrocessing + routines. + + :param shortcode: Shortcode of the project + :param shortname: Short name of the ontolopgy + :return: Dict with a simple description of the ontology + """ + turtle = self.get_ontology_graph(shortcode, shortname) + g = Graph() + g.parse(format='n3', data=turtle) + sparql=""" + SELECT ?res ?prop ?superprop ?otype ?guiele ?attr ?card ?cardval + WHERE { + ?res a owl:Class . + ?res rdfs:subClassOf ?restriction . + ?restriction a owl:Restriction . + ?restriction owl:onProperty ?prop . + ?restriction ?card ?cardval . + ?prop a owl:ObjectProperty . + ?prop knora-api:objectType ?otype . + ?prop salsah-gui:guiElement ?guiele . + ?prop rdfs:subPropertyOf ?superprop . + OPTIONAL { ?prop salsah-gui:guiAttribute ?attr } . + FILTER(?card = owl:cardinality || ?card = owl:maxCardinality || ?card = owl:minCardinality) + } + ORDER BY ?res ?prop + """ + qres = g.query(sparql) + + resources = {} + resclass = '' + propname = '' + link_otypes = [] + propcnt = 0 + propindex= {} # we have to keep the order of the properties as given in the ontology.... + for row in qres: + nresclass = row.res.toPython() + nresclass = nresclass[nresclass.find('#') + 1:] + if resclass != nresclass: + resclass = nresclass + resources[resclass] = [] + propcnt = 0 + superprop = row.superprop.toPython() + superprop = superprop[superprop.find('#') + 1:] + if superprop == 'hasLinkToValue': # we ignore this one.... + continue + npropname = row.prop.toPython() + npropname = npropname[npropname.find('#') + 1:] + attr = row.attr.toPython() if row.attr is not None else None + if attr is not None: + attr = attr.split('=') + if propname == npropname: + if attr is not None: + propcnt -= 1 + if resources[resclass][propcnt]["attr"] is not None: # TODO: why is this necessary??? + resources[resclass][propcnt]["attr"][attr[0]] = attr[1].strip('<>') + continue + else: + propname = npropname + objtype = row.otype.toPython() + objtype = objtype[objtype.find('#') + 1:] + card = row.card.toPython() + card = card[card.find('#') + 1:] + guiele = row.guiele.toPython() + guiele = guiele[guiele.find('#') + 1:] + resources[resclass].append({ + "propname": propname, + "otype": objtype, + "superpro": superprop, + "guiele": guiele, + "attr": {attr[0]: attr[1].strip('<>')} if attr is not None else None, + "card": card, + "cardval": row.cardval.toPython() + }) + if superprop == "hasLinkTo": + link_otypes.append(objtype) + propindex[propname] = propcnt + propcnt += 1 + listdata = {} + lists = self.get_lists(shortcode) + lists = lists["lists"] + for list in lists: + tmp = self.get_complete_list(list["id"]) + clist = tmp["list"] + listdata[clist["listinfo"]["name"]] = { + "id": clist["listinfo"]["id"], + "nodes": self.list_creator(clist["children"]) + } + schema = { + "shortcode": shortcode, + "ontoname": shortname, + "lists": listdata, + "resources": resources, + "link_otypes": link_otypes + } + return schema + + +class BulkImport: + def __init__(self, schema: Dict): + self.schema = schema + self.proj_prefix = 'p' + schema['shortcode'] + '-' + schema["ontoname"] + self.proj_iri = "http://api.knora.org/ontology/" + schema['shortcode'] + "/" + schema["ontoname"] + "/xml-import/v1#" + self.xml_prefixes = { + None: self.proj_iri, + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + self.proj_prefix: self.proj_iri, + "knoraXmlImport": "http://api.knora.org/ontology/knoraXmlImport/v1#" + } + self.root = etree.Element('{http://api.knora.org/ontology/knoraXmlImport/v1#}resources', nsmap=self.xml_prefixes) + + def new_xml_element(self, tag: str, options: Dict = None, value: str = None): + tagp = tag.split(':') + if len(tagp) > 1: + fulltag = '{' + self.xml_prefixes.get(tagp[0]) + '}' + tagp[1] + else: + fulltag = tagp[0] + if options is None: + ele = etree.Element(fulltag) + else: + ele = etree.Element(fulltag, options) + if value is not None: + ele.text = value + return ele + + + def write_xml(self, filename: str): + # print(etree.tostring(self.root, pretty_print=True, xml_declaration=True, encoding='utf-8')) + f = open(filename, "wb") + f.write(etree.tostring(self.root, pretty_print=True, xml_declaration=True, encoding='utf-8')) + f.close() + + def add_resource(self, resclass: str, id: str, label: str, properties: Dict): + """ + + :param resclass: + :param id: + :param label: + :param properties: + :return: + """ + + def find_list_node_id(nodename: str, nodes: List): + """ + Finds a list node ID from the nodename in a (eventually hierarchical) list of nodes + + :param nodename: Name of the node + :param nodes: List of nodes + :return: the id of the list node (an IRI) + """ + for node in nodes: + if node["name"] == nodename: + return node["id"] + if node.get("nodes") is not None and len(node.get("nodes")) > 0: + node_id = find_list_node_id(nodename, node["nodes"]) + if node_id is not None: + return node_id + return None + + + def process_properties(propinfo: Dict, valuestr: any): + """ + Processes a property in order to generate the approptiate XML for V1 bulk import. + + :param pname: property name + :param valuestr: value of the property + :return: Tuple with xml options and processed value: (xmlopt, val) + """ + switcher = { + 'TextValue': {'knoraType': 'richtext_value'}, + 'ColorValue': {'knoraType': 'color_value'}, + 'DateValue': {'knoraType': 'date_value'}, + 'DecimalValue': {'knoraType': 'decimal_value'}, + 'GeomValue': {'knoraType': 'geom_value'}, + 'GeonameValue': {'knoraType': 'geoname_value'}, + 'IntValue': {'knoraType': 'int_value'}, + 'BooleanValue': {'knoraType': 'boolean_value'}, + 'UriValue': {'knoraType': 'uri_value'}, + 'IntervalValue': {'knoraType': 'interval_value'}, + 'ListValue': {'knoraType': 'hlist_value'}, + 'LinkValue': {'knoraType': 'link_value'} + } + for link_otype in self.schema["link_otypes"]: + switcher[link_otype] = {'knoraType': 'link_value'} + xmlopt = switcher.get(propinfo["otype"]) + if xmlopt is None: + raise KnoraError("Did not find " + propinfo["otype"] + " in switcher!") + if xmlopt['knoraType'] == 'link_value': + xmlopt['target'] = str(valuestr) + if validators.url(str(valuestr)): + xmlopt['linkType'] = 'iri' + else: + xmlopt['linkType'] = 'ref' + value = None + elif propinfo["otype"] == 'ListValue': + if validators.url(str(valuestr)): + # it's a full IRI identifying the node + value = valuestr + else: + # it's only a node name. First let's get the node list from the ontology schema + list_id = propinfo["attr"]["hlist"] + for listname in self.schema["lists"]: + if self.schema["lists"][listname]["id"] == list_id: + nodes = self.schema["lists"][listname]["nodes"] + value = find_list_node_id(str(valuestr), nodes) + if value == 'http://rdfh.ch/lists/0808/X6bb-JerQyu5ULruCGEO0w': + print("BANG!") + exit(0) + elif propinfo["otype"] == 'DateValue': + # processing and validating date format + res = re.match('(GREGORIAN:|JULIAN:)?(\d{4})?(-\d{1,2})?(-\d{1,2})?(:\d{4})?(-\d{1,2})?(-\d{1,2})?', str(valuestr)) + if res is None: + raise KnoraError("Invalid date format! " + str(valuestr)) + dp = res.groups() + calendar = 'GREGORIAN:' if dp[0] is None else dp[0] + y1 = None if dp[1] is None else int(dp[1].strip('-: ')) + m1 = None if dp[2] is None else int(dp[2].strip('-: ')) + d1 = None if dp[3] is None else int(dp[3].strip('-: ')) + y2 = None if dp[4] is None else int(dp[4].strip('-: ')) + m2 = None if dp[5] is None else int(dp[5].strip('-: ')) + d2 = None if dp[6] is None else int(dp[6].strip('-: ')) + if y1 is None: + raise KnoraError("Invalid date format! " + str(valuestr)) + if y2 is not None: + date1 = y1*10000; + if m1 is not None: + date1 += m1*100 + if d1 is not None: + date1 += d1 + date2 = y2 * 10000; + if m2 is not None: + date2 += m2 * 100 + if d1 is not None: + date2 += d2 + if date1 > date2: + y1, y2 = y2, y1 + m1, m2 = m2, m1 + d1, d2 = d2, d1 + value = calendar + str(y1) + if m1 is not None: + value += f'-{m1:02d}' + if d1 is not None: + value += f'-{d1:02d}' + if y2 is not None: + value += f':{y2:04d}' + if m2 is not None: + value += f'-{m2:02d}' + if d2 is not None: + value += f'-{d2:02d}' + else: + value = str(valuestr) + return xmlopt, value + + if self.schema["resources"].get(resclass) is None: + raise KnoraError('Resource class is not defined in ontlogy!') + resnode = self.new_xml_element(self.proj_prefix + ':' + resclass, {'id': str(id)}) + + labelnode = self.new_xml_element('knoraXmlImport:label') + labelnode.text = str(label) + resnode.append(labelnode) + + for prop_info in self.schema["resources"][resclass]: + # first we check if the cardinality allows to add this property + if properties.get(prop_info["propname"]) is None: # this property-value is missing + if prop_info["card"] == 'cardinality'\ + and prop_info["cardval"] == 1: + raise KnoraError(resclass + " requires exactly one " + prop_info["propname"] + "-value: none supplied!") + if prop_info["card"] == 'minCardinality'\ + and prop_info["cardval"] == 1: + raise KnoraError(resclass + " requires at least one " + prop_info["propname"] + "-value: none supplied!") + continue + if type(properties[prop_info["propname"]]) is list: + if len(properties[prop_info["propname"]]) > 1: + if prop_info["card"] == 'maxCardinality' \ + and prop_info["cardval"] == 1: + raise KnoraError(resclass + " allows maximal one " + prop_info["propname"] + "-value: several supplied!") + if prop_info["card"] == 'cardinality'\ + and prop_info["cardval"] == 1: + raise KnoraError(resclass + " requires exactly one " + prop_info["propname"] + "-value: several supplied!") + for p in properties[prop_info["propname"]]: + xmlopt, value = process_properties(prop_info, p) + pnode = self.new_xml_element(self.proj_prefix + ':' + prop_info["propname"], xmlopt, value) + resnode.append(pnode) + else: + xmlopt, value = process_properties(prop_info, properties[prop_info["propname"]]) + if xmlopt['knoraType'] == 'link_value': + pnode = self.new_xml_element(self.proj_prefix + ':' + prop_info["propname"]) + pnode.append(self.new_xml_element(self.proj_prefix + ':' + prop_info["otype"], xmlopt, value)) + else: + pnode = self.new_xml_element(self.proj_prefix + ':' + prop_info["propname"], xmlopt, value) + resnode.append(pnode) + self.root.append(resnode) + + diff --git a/knora/knoraConsole.py b/knora/knoraConsole.py new file mode 100644 index 000000000..c622c4ab6 --- /dev/null +++ b/knora/knoraConsole.py @@ -0,0 +1,294 @@ +from typing import List, Set, Dict, Tuple, Optional +from knora import KnoraError, knora +import wx +from pprint import pprint + + +class KnoraConsole(wx.Frame): + """ + Main Window for Knora console + """ + + def __init__(self, *args, **kw): + super(KnoraConsole, self).__init__(*args, **kw) + + # create a menu bar + self.makeMenuBar() + + # and a status bar + self.CreateStatusBar() + self.SetStatusText("Knora Console") + + self.nb = wx.Notebook(self) + + self.up = UserPanel(self.nb) + self.nb.InsertPage(index=0, page=self.up, text="User") + self.con = None + + + def makeMenuBar(self): + """ + A menu bar is composed of menus, which are composed of menu items. + This method builds a set of menus and binds handlers to be called + when the menu item is selected. + """ + + # Make a file menu with Hello and Exit items + fileMenu = wx.Menu() + # The "\t..." syntax defines an accelerator key that also triggers + # the same event + connectItem = fileMenu.Append(wx.ID_OPEN, "&Open connection...\tCtrl-O", + "Connect to server") + disconnectItem = fileMenu.Append(wx.ID_CLOSE, "&Close connection...\tCtrl-C", + "Disconnect from server") + fileMenu.AppendSeparator() + # When using a stock ID we don't need to specify the menu item's + # label + exitItem = fileMenu.Append(wx.ID_EXIT) + + # Now a help menu for the about item + helpMenu = wx.Menu() + aboutItem = helpMenu.Append(wx.ID_ABOUT) + + # Make the menu bar and add the two menus to it. The '&' defines + # that the next letter is the "mnemonic" for the menu item. On the + # platforms that support it those letters are underlined and can be + # triggered from the keyboard. + menuBar = wx.MenuBar() + menuBar.Append(fileMenu, "&Connection") + menuBar.Append(helpMenu, "&Help") + + # Give the menu bar to the frame + self.SetMenuBar(menuBar) + + # Finally, associate a handler function with the EVT_MENU event for + # each of the menu items. That means that when that menu item is + # activated then the associated handler function will be called. + self.Bind(wx.EVT_MENU, self.onConnect, connectItem) + self.Bind(wx.EVT_MENU, self.onDisconnect, disconnectItem) + self.Bind(wx.EVT_MENU, self.onExit, exitItem) + self.Bind(wx.EVT_MENU, self.onAbout, aboutItem) + + def onExit(self, event): + """Close the frame, terminating the application.""" + self.con = None + self.Close(True) + + def onConnect(self, event): + """Say hello to the user.""" + dialog = OpenConnectionDialog(self) + if dialog.GetReturnCode() == wx.ID_OK: + self.con = dialog.get_res() + self.up.set_connection(self.con) + self.up.update(self.con) + + def onDisconnect(self, event): + wx.MessageBox("Disconnect from server") + + def onAbout(self, event): + """Display an About Dialog""" + wx.MessageBox("Knora Console", + "Knora Console, a tool for setting up Knora", + wx.OK | wx.ICON_INFORMATION) + + +class OpenConnectionDialog(wx.Dialog): + """ + This open a dialog which allows the user to select a server and to + give the username and password + """ + + def __init__(self, *args, **kw): + super(OpenConnectionDialog, self).__init__(*args, **kw, + title="Open connection...", + style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER) + + topsizer = wx.BoxSizer(wx.VERTICAL) + + panel1 = wx.Panel(self) + l0 = wx.StaticText(panel1, label="Server: ") + server = wx.TextCtrl(panel1, name="server", value="http://0.0.0.0:3333", size=wx.Size(200, -1)) + + l1 = wx.StaticText(panel1, label="Username: ") + username = wx.TextCtrl(panel1, name="username", value="root@example.com", size=wx.Size(200, -1)) + l2 = wx.StaticText(panel1, label="Password: ") + password = wx.TextCtrl(panel1, name="password", value="test", size=wx.Size(200, -1), style=wx.TE_PASSWORD) + gsizer = wx.GridSizer(cols=2) + gsizer.Add(l0, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.ALL, border=3) + gsizer.Add(server, wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.ALL, border=3) + gsizer.Add(l1, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.ALL, border=3) + gsizer.Add(username, wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.ALL, border=3) + gsizer.Add(l2, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.ALL, border=3) + gsizer.Add(password, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.ALL, border=3) + gsizer.SetSizeHints(panel1) + panel1.SetSizer(gsizer) + panel1.SetAutoLayout(1) + gsizer.Fit(panel1) + + topsizer.Add(panel1, flag=wx.EXPAND | wx.ALL, border=5) + + bsizer = self.CreateStdDialogButtonSizer(wx.OK | wx.CANCEL) + topsizer.Add(bsizer, flag=wx.EXPAND | wx.ALL, border=5) + + self.SetSizerAndFit(topsizer) + + self.ShowModal() + if self.GetReturnCode() == wx.ID_OK: + server_str = server.GetLineText(0) + username_str = username.GetLineText(0) + password_str = password.GetLineText(0) + self.con = knora(server_str, username_str, password_str) + else: + print("CANCEL PRESSED") + + def get_res(self): + return self.con + + + +class UserPanel(wx.Panel): + """ + User tab + """ + def __init__(self, *args, **kw): + super(UserPanel, self).__init__(*args, **kw) + + self.con = None + self.ids = [] + + topsizer = wx.BoxSizer(wx.VERTICAL) + + self.listctl = wx.ListCtrl(self, name="Users:", + style=wx.LC_REPORT | wx.LC_SINGLE_SEL | wx.LC_HRULES) + self.listctl.AppendColumn("Username", width=wx.LIST_AUTOSIZE) + self.listctl.AppendColumn("Lastname", width=wx.LIST_AUTOSIZE) + self.listctl.AppendColumn("Firstname", width=wx.LIST_AUTOSIZE) + self.listctl.AppendColumn("Email", width=wx.LIST_AUTOSIZE) + + topsizer.Add(self.listctl, proportion=1, flag=wx.EXPAND | wx.ALL, border=5) + + bottomsizer = wx.BoxSizer(wx.HORIZONTAL) + self.edit_button = wx.Button(parent=self, label="edit") + self.edit_button.Bind(wx.EVT_BUTTON, self.start_entry) + self.new_button = wx.Button(parent=self, label="new") + bottomsizer.Add(self.edit_button, proportion=1, flag=wx.EXPAND | wx.ALIGN_CENTER | wx.ALL, border=3) + bottomsizer.Add(self.new_button, proportion=1, flag=wx.EXPAND | wx.ALIGN_CENTER | wx.ALL, border=3) + + topsizer.Add(bottomsizer, proportion=0, flag=wx.EXPAND) + self.SetAutoLayout(1) + self.SetSizerAndFit(topsizer) + + def set_connection(self, con: knora): + self.con = con + + def update(self, con: knora): + users = con.get_users() + self.listctl.DeleteAllItems() + for user in users: + self.listctl.Append((user['username'], user['familyName'], user['givenName'], user['email'])) + self.ids.append(user['id']) + self.listctl.SetColumnWidth(0, -1) + self.listctl.SetColumnWidth(1, -1) + self.listctl.SetColumnWidth(2, -1) + self.listctl.SetColumnWidth(3, -1) + self.listctl.Select(0) + + def start_entry(self, event): + ue = UserEntryDialog(self.con, self.ids[self.listctl.GetFirstSelected()], self) + #ue = UserEntryDialog(self) + + +class UserEntryDialog(wx.Dialog): + def __init__(self, con: knora = None, iri: str = None, *args, **kw): + super(UserEntryDialog, self).__init__(*args, **kw, + title="User Entry", + style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER) + + user_info = con.get_user(iri) + existing_projects = con.get_existing_projects(full=True) + pprint(user_info) + pprint(existing_projects) + + topsizer = wx.BoxSizer(wx.VERTICAL) + panel1 = wx.Panel(self) + + gsizer = wx.FlexGridSizer(cols=2) + + username_l = wx.StaticText(panel1, label="Username: ") + username = wx.TextCtrl(panel1, name="Username", value=user_info['username'], size=wx.Size(200, -1)) + gsizer.Add(username_l, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + gsizer.Add(username, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + + password_l = wx.StaticText(panel1, label="Password: ") + password = wx.TextCtrl(panel1, name="password", value="test", size=wx.Size(200, -1), style=wx.TE_PASSWORD) + gsizer.Add(password_l, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + gsizer.Add(password, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + + lastname_1 = wx.StaticText(panel1, label="Lastname: ") + lastname = wx.TextCtrl(panel1, name="lastname", value=user_info['familyName'], size=wx.Size(200, -1)) + gsizer.Add(lastname_1, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + gsizer.Add(lastname, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + + firstname_l = wx.StaticText(panel1, label="Firstname: ") + firstname = wx.TextCtrl(panel1, name="firstname", value=user_info['givenName'], size=wx.Size(200, -1)) + gsizer.Add(firstname_l, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + gsizer.Add(firstname, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + + langswitcher = { + "en": 0, + "de": 1, + "fr": 2, + "it": 3 + } + language_l = wx.StaticText(panel1, label="Language: ") + language = wx.Choice(panel1, choices=["en", "de", "fr", "it"]) + language.SetSelection(langswitcher[user_info['lang']]) + gsizer.Add(language_l, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + gsizer.Add(language, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + + status_l = wx.StaticText(panel1, label="Status: ") + status = wx.CheckBox(panel1, label="active") + status.SetValue(user_info['status']) + gsizer.Add(status_l, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + gsizer.Add(status, flag=wx.ALIGN_CENTER_VERTICAL | wx.ALL, border=3) + + projects_l = wx.StaticText(panel1, label="Projects: ") + plist = list(map(lambda a: a['shortname'] + ' (' + a['shortcode'] + ')', user_info['projects'])) + + projects = wx.CheckListBox(panel1, choices=plist) + for i in range(len(plist)): + projects.Check(i) + projsizer = wx.BoxSizer(wx.VERTICAL) + projsizer.Add(projects, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.GROW | wx.ALL) + projs = list(map(lambda a: a['shortname'] + ' (' + a['shortcode'] + ')', existing_projects)) + projlist = wx.Choice(panel1, choices=projs) + projsizer.Add(projlist, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.GROW | wx.ALL) + + gsizer.Add(projects_l, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.GROW | wx.ALL, border=3) + gsizer.Add(projsizer, flag=wx.ALIGN_CENTER_VERTICAL | wx.EXPAND | wx.GROW | wx.ALL, border=3) + + gsizer.SetSizeHints(panel1) + panel1.SetSizer(gsizer) + panel1.SetAutoLayout(1) + gsizer.Fit(panel1) + + topsizer.Add(panel1, flag=wx.EXPAND | wx.ALL, border=5) + + bsizer = self.CreateStdDialogButtonSizer(wx.OK | wx.CANCEL) + topsizer.Add(bsizer, flag=wx.EXPAND | wx.ALL, border=5) + + self.SetSizerAndFit(topsizer) + self.ShowModal() + + + + +if __name__ == '__main__': + # When this module is run (not imported) then create the app, the + # frame, show it, and start the event loop. + + app = wx.App() + frm = KnoraConsole(None, title='Knora Console V0.1.1 Beta', size=wx.Size(800, 600)) + frm.Show() + app.MainLoop() + print("Bye Bye") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..e582d90ba --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +certifi==2018.11.29 +chardet==3.0.4 +decorator==4.3.0 +idna==2.8 +isodate==0.6.0 +jsonschema==2.6.0 +lxml==4.3.0 +pprint==0.1 +pyparsing==2.3.1 +rdflib==4.2.2 +requests==2.21.0 +six==1.12.0 +urllib3==1.24.1 +validators==0.12.4 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..f71aee586 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name='knora', + version='0.0.1', + description='A Python library and tools for the Knora-API', + url='https://github.com/dhlab-basel/knora-py', + author='Lukas Rosenthaler', + author_email='lukas.rosenthaler@unibas.ch', + license='GPLv3', + zip_safe=False, + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GPLv3 License", + "Operating System :: OS Independent", + ], +)