test: add all supported file extensions to testdata and docs (DEV-1187)…

… (#213)
dasch-swiss · Aug 15, 2022 · 72000b3 · 72000b3
1 parent 6849737
commit 72000b3
Show file tree

Hide file tree

Showing 42 changed files with 392 additions and 23 deletions.
diff --git a/docs/dsp-tools-create-ontologies.md b/docs/dsp-tools-create-ontologies.md
@@ -797,13 +797,13 @@ The following base resources can be used as super-resource:
 
 - `Resource`: A generic resource representing an item from the real world. This is the most general case, to be 
 used in all cases when your resource is none of the special cases below.
-- `StillImageRepresentation`: A resource representing an image
-- `TextRepresentation`: A resource representing a text
+- `ArchiveRepresentation`: A resource representing an archive file (e.g. ZIP)
 - `AudioRepresentation`: A resource representing an audio file
 - `DDDRepresentation`: A resource representing a 3-D representation (not yet implemented)
 - `DocumentRepresentation`: A resource representing an opaque document (e.g. a PDF)
 - `MovingImageRepresentation`: A resource representing a video
-- `ArchiveRepresentation`: A resource representing an archive file (e.g. ZIP)
+- `StillImageRepresentation`: A resource representing an image
+- `TextRepresentation`: A resource representing a text
 
 
 ### Cardinalities

diff --git a/docs/dsp-tools-xmlupload.md b/docs/dsp-tools-xmlupload.md
@@ -265,6 +265,19 @@ Note:
 - There is only _one_ `<bitstream>` element allowed per representation!
 - The `<bitstream>` element must be the first element!
 
+Supported file extensions:
+
+| Representation              | Supported formats                      |
+| --------------------------- |----------------------------------------| 
+| `ArchiveRepresentation`     | ZIP, TAR, GZ, Z, TAR.GZ, TGZ, GZIP, 7Z | 
+| `AudioRepresentation`       | MP3, MP4, WAV                          | 
+| `DocumentRepresentation`    | PDF, DOC, DOCX, XLS, XLSX, PPT, PPTX   | 
+| `MovingImageRepresentation` | MP4                                    | 
+| `StillImageRepresentation`  | JPG, JPEG, PNG, TIF, TIFF, JP2         | 
+| `TextRepresentation`        | TXT, CSV, XML, XSL, XSD                | 
+
+For more details, please consult the [API docs](https://docs.dasch.swiss/latest/DSP-API/01-introduction/file-formats/).
+
 Attributes:
 
 - `permissions` : Permission ID (optional, but if omitted, users who are lower than a `ProjectAdmin` have no permissions at all, not even view rights)

diff --git a/test/e2e/test_resource.py b/test/e2e/test_resource.py
@@ -52,13 +52,13 @@ def test_Resource_create(self) -> None:
 
         thing_picture = factory.get_resclass_type('anything:ThingPicture')
         sipi = Sipi('http://0.0.0.0:1024', self.con.get_token())
-        img = sipi.upload_bitstream('testdata/bitstreams/TEMP11.TIF')
+        img = sipi.upload_bitstream('testdata/bitstreams/test.tif')
         file_ref = img['uploadedFiles'][0]['internalFilename']
         res_perm = Permissions({PermissionValue.M: ["knora-admin:UnknownUser", "knora-admin:KnownUser"],
                                 PermissionValue.CR: ["knora-admin:Creator", "knora-admin:ProjectAdmin"]})
 
         resource_bitstream = {
-            'value': 'testdata/bitstreams/TEMP11.TIF',
+            'value': 'testdata/bitstreams/test.tif',
             'internal_file_name': file_ref,
             'permissions': res_perm
         }

diff --git a/test/e2e/test_tools.py b/test/e2e/test_tools.py
@@ -22,7 +22,7 @@ class TestTools(unittest.TestCase):
     sipi = 'http://0.0.0.0:1024'
     test_project_file = 'testdata/test-project-systematic.json'
     test_project_minimal_file = 'testdata/test-project-minimal.json'
-    test_data_file = 'testdata/test-data.xml'
+    test_data_file = 'testdata/test-data-systematic.xml'
 
     def setUp(self) -> None:
         """Is executed before each test"""
@@ -197,7 +197,7 @@ def test_xml_upload(self) -> None:
         self.assertTrue(result)
 
         mapping_file = ''
-        for mapping in [x for x in os.scandir('.') if x.name.startswith('id2iri_test-data_mapping_')]:
+        for mapping in [x for x in os.scandir('.') if x.name.startswith('id2iri_test-data-systematic_mapping_')]:
             delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(mapping.stat().st_mtime_ns / 1000000000)
             if delta.seconds < 15:
                 mapping_file = mapping.name

diff --git a/test/unittests/test_xmlupload.py b/test/unittests/test_xmlupload.py
@@ -34,7 +34,7 @@ def test_convert_ark_v0_to_resource_iri(self) -> None:
 
     def test_remove_circular_references(self) -> None:
         # create a list of XMLResources from the test data file
-        tree = _parse_xml_file('testdata/test-data.xml')
+        tree = _parse_xml_file('testdata/test-data-systematic.xml')
         resources = [XMLResource(x, 'testonto') for x in tree.getroot() if x.tag == "resource"]
 
         # get the purged resources and the stashes from the function to be tested

diff --git a/testdata/bitstreams/README.txt b/testdata/bitstreams/README.txt
@@ -0,0 +1,32 @@
+Rights/Licenses of the test data in this directory
+**************************************************
+
+test.7z     DaSCH: Johannes Nussbaum
+test.csv    DaSCH: rosetta/Documents/RosettaCategoryList.csv
+test.doc    DaSCH: rosetta/Documents/RosettaTestProject.docx
+test.docx   DaSCH: rosetta/Documents/RosettaTestProject.docx
+test.gz	    DaSCH: Johannes Nussbaum
+test.gzip   Python Package joblib: /test/data/joblib_0.11.0_pickle_py36_np111.pkl.gzip
+test.jp2    Public Domain: www.loc.gov/item/00279382687-ms (Library of Congress Collection of Manuscripts in St. Catherine's Monastery, Mt. Sinai)
+test.jpeg   https://filesamples.com/samples/image/jpeg/sample_640%C3%97426.jpeg
+test.jpg    Public Domain: https://en.wikipedia.org/wiki/Haiku#/media/File:Basho_Horohoroto.jpg
+test.mp3    DaSCH: Johannes Nussbaum
+test.mp4    https://filesamples.com/samples/video/mp4/sample_960x540.mp4
+test.pdf    DaSCH: rosetta/Documents/RosettaTestProject.docx
+test.png    DaSCH: Johannes Nussbaum
+test.ppt    DaSCH: Johannes Nussbaum
+test.pptx   DaSCH: Johannes Nussbaum
+test.tar    DaSCH: Johannes Nussbaum
+test.tar.gz DaSCH: Johannes Nussbaum
+test.tgz    DaSCH: Johannes Nussbaum
+test.tif    DaSCH: Lukas Rosenthaler
+test.tiff   DaSCH: Lukas Rosenthaler
+test.txt    DaSCH: Johannes Nussbaum
+test.wav    DaSCH: Johannes Nussbaum
+test.xls    DaSCH: Johannes Nussbaum
+test.xlsx   DaSCH: Johannes Nussbaum
+test.xml    DaSCH: Johannes Nussbaum
+test.xsd    DaSCH: Johannes Nussbaum
+test.xsl    DaSCH: Johannes Nussbaum
+test.z      DaSCH: Johannes Nussbaum
+test.zip    DaSCH: Johannes Nussbaum
diff --git a/testdata/bitstreams/TEMP11.TIF b/testdata/bitstreams/TEMP11.TIF
diff --git a/testdata/bitstreams/TEMP12.TIF b/testdata/bitstreams/TEMP12.TIF
diff --git a/testdata/bitstreams/TEMP13.TIF b/testdata/bitstreams/TEMP13.TIF
diff --git a/testdata/bitstreams/TEMP14.TIF b/testdata/bitstreams/TEMP14.TIF
diff --git a/testdata/bitstreams/TEMP15.TIF b/testdata/bitstreams/TEMP15.TIF
diff --git a/testdata/bitstreams/clara.wav b/testdata/bitstreams/clara.wav
diff --git a/testdata/bitstreams/test.7z b/testdata/bitstreams/test.7z
diff --git a/testdata/bitstreams/test.csv b/testdata/bitstreams/test.csv
@@ -1 +1,13 @@
-Performing arts;;;Performing arts;Music;;Performing arts;Music;Chamber music;Performing arts;Music;Church music;Performing arts;Music;Conducting;Performing arts;Music;Conducting;ChoirsPerforming arts;Music;Conducting;OrchestrasPerforming arts;Music;Music history;Performing arts;Music;Musictheory;Performing arts;Music;Musicology;Performing arts;Music;Jazz;Performing arts;Music;Pop/Rock/Blues;Dance;;;Dance;Choreography;;Theatre;;;Theatre;Acting;;Theatre;Directing;;Theatre;Playwriting;;Theatre;Scenography;;Movies/television;;;Movies/television;Animation;;Movies/television;Live action;;Visual arts;;;Visual arts;Fine Arts;;Visual arts;Fine Arts;Drawing;Visual arts;Fine Arts;Painting;Visual arts;Fine Arts;Photography;Visual arts;Applied arts;;Visual arts;Applied arts;Animation;Visual arts;Applied arts;Architecture;Visual arts;Applied arts;Decorative arts;History;;;History;Ancient history;;History;Modern history;;Languages and literature;;;Languages and literature;Linguistics;;Languages and literature;Linguistics;Grammer;Languages and literature;Linguistics;Etymology;Languages and literature;Linguistics;Phonetics;Languages and literature;Linguistics;Semantics;Languages and literature;Literature;;Languages and literature;Literature;Fiction;Languages and literature;Literature;Non-fiction;Languages and literature;Literature;Theory of literature;Philosophy;;;Philosophy;Aesthetics;;Philosophy;Applied philosophy;;Philosophy;Epistemology;;Philosophy;Epistemology;Justification;Philosophy;Epistemology;Reasoning;Philosophy;Metaphysics;;Philosophy;Metaphysics;Determinisn and free will;Philosophy;Metaphysics;Ontology;Philosophy;Metaphysics;Philosophy of mind;Philosophy;Metaphysics;Teleology;
+artwork;;
+vehicles;;
+nature;;
+nature;humans;
+nature;animals;
+nature;animals;mammals
+nature;animals;insects
+nature;animals;birds
+nature;animals;amphibians
+nature;animals;reptiles
+nature;plants;
+nature;weather;
+nature;physics;
diff --git a/testdata/bitstreams/test.doc b/testdata/bitstreams/test.doc
diff --git a/testdata/bitstreams/test.docx b/testdata/bitstreams/test.docx
diff --git a/testdata/bitstreams/test.gz b/testdata/bitstreams/test.gz
diff --git a/testdata/bitstreams/test.gzip b/testdata/bitstreams/test.gzip
diff --git a/testdata/bitstreams/test.jp2 b/testdata/bitstreams/test.jp2
diff --git a/testdata/bitstreams/test.jpeg b/testdata/bitstreams/test.jpeg
diff --git a/testdata/bitstreams/test.jpg b/testdata/bitstreams/test.jpg
diff --git a/testdata/bitstreams/test.mp3 b/testdata/bitstreams/test.mp3
diff --git a/testdata/bitstreams/Dummy.mp4 → testdata/bitstreams/test.mp4 b/testdata/bitstreams/Dummy.mp4 → testdata/bitstreams/test.mp4
diff --git a/testdata/bitstreams/test.pdf b/testdata/bitstreams/test.pdf
diff --git a/testdata/bitstreams/test.png b/testdata/bitstreams/test.png
diff --git a/testdata/bitstreams/test.ppt b/testdata/bitstreams/test.ppt
diff --git a/testdata/bitstreams/test.pptx b/testdata/bitstreams/test.pptx
diff --git a/testdata/bitstreams/test.tar b/testdata/bitstreams/test.tar
diff --git a/testdata/bitstreams/test.tar.gz b/testdata/bitstreams/test.tar.gz
diff --git a/testdata/bitstreams/test.tgz b/testdata/bitstreams/test.tgz
diff --git a/testdata/bitstreams/test.tif b/testdata/bitstreams/test.tif
diff --git a/testdata/bitstreams/test.tiff b/testdata/bitstreams/test.tiff
diff --git a/testdata/bitstreams/test.txt b/testdata/bitstreams/test.txt
@@ -0,0 +1,5 @@
+Test text file
+--------------
+
+This is a sample text file without much content.	
+It contains the special characters àö¨`ô^.
diff --git a/testdata/bitstreams/test.wav b/testdata/bitstreams/test.wav
diff --git a/testdata/bitstreams/test.xls b/testdata/bitstreams/test.xls
diff --git a/testdata/bitstreams/test.xlsx b/testdata/bitstreams/test.xlsx
diff --git a/testdata/bitstreams/test.xml b/testdata/bitstreams/test.xml
@@ -0,0 +1,21 @@
+<?xml version='1.0' encoding='utf-8'?>
+
+<knora xmlns="https://dasch.swiss/schema"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="https://dasch.swiss/schema ../knora/dsplib/schemas/data.xsd">
+
+    <!-- this is a comment -->
+    <permissions id="res-default">
+        <allow group="UnknownUser">RV</allow>
+        <allow group="KnownUser">V</allow>
+    </permissions>
+
+    <resource label="test xml file"
+              restype=":TestFile"
+              id="test_xml_file">
+        <text-prop name=":hasText">
+            <text encoding="utf8">This is a test XML file</text>
+        </text-prop>
+    </resource>
+
+</knora>
diff --git a/testdata/bitstreams/test.xsd b/testdata/bitstreams/test.xsd
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xs:schema xmlns="https://dasch.swiss/schema"
+           xmlns:xs="http://www.w3.org/2001/XMLSchema"
+           targetNamespace="https://dasch.swiss/schema"
+           elementFormDefault="qualified">
+
+    <!-- complete document -->
+    <xs:element name="knora">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element name="permissions" type="permissions_type" minOccurs="0" maxOccurs="unbounded"/>
+                <xs:choice maxOccurs="unbounded">
+                    <xs:element name="region" type="region_type" minOccurs="0" maxOccurs="unbounded">
+                        <xs:unique name="UniqueNameAttr_Region">
+                            <xs:selector xpath=".//*"/>
+                            <xs:field xpath="@name"/>
+                        </xs:unique>
+                    </xs:element>
+                    <xs:element name="link" type="link_type" minOccurs="0" maxOccurs="unbounded">
+                        <xs:unique name="UniqueNameAttr_Link">
+                            <xs:selector xpath=".//*"/>
+                            <xs:field xpath="@name"/>
+                        </xs:unique>
+                    </xs:element>
+                </xs:choice>
+            </xs:sequence>
+            <xs:attribute name="default-ontology" type="xs:string"/>
+            <xs:attribute name="shortcode" type="knorashortcode_type"/>
+        </xs:complexType>
+        <xs:key name="permissionsId">
+            <xs:selector xpath="./permissions"/>
+            <xs:field xpath="@id"/>
+        </xs:key>
+    </xs:element>
+</xs:schema>
+
diff --git a/testdata/bitstreams/test.xsl b/testdata/bitstreams/test.xsl
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="https://www.w3.org/1999/XSL/Transform">
+<xsl:template match="/">
+  <html>
+  <body>
+    <h2>Title</h2>
+    <xsl:for-each select="Example">
+    <a>
+    <xsl:attribute name="href">
+    <xsl:value-of select="Something"/>
+    </xsl:attribute>
+    <xsl:value-of select="anotherthing"/>
+    </a><p/>
+    </xsl:for-each>
+  </body>
+  </html>
+</xsl:template>
+</xsl:stylesheet>
diff --git a/testdata/bitstreams/test.z b/testdata/bitstreams/test.z
diff --git a/testdata/bitstreams/test.zip b/testdata/bitstreams/test.zip