diff --git a/knora-ontologies/knora-base.ttl b/knora-ontologies/knora-base.ttl index 9046fbed1f..2ad4b5fc01 100644 --- a/knora-ontologies/knora-base.ttl +++ b/knora-ontologies/knora-base.ttl @@ -33,7 +33,7 @@ :attachedToProject knora-admin:SystemProject ; - :ontologyVersion "knora-base v9" . + :ontologyVersion "knora-base v10" . @@ -1792,7 +1792,7 @@ rdfs:subClassOf :FileValue , [ rdf:type owl:Restriction ; owl:onProperty :pageCount ; - owl:cardinality "1"^^xsd:nonNegativeInteger + owl:maxCardinality "1"^^xsd:nonNegativeInteger ] , [ rdf:type owl:Restriction ; owl:onProperty :dimX ; diff --git a/sipi/scripts/file_info.lua b/sipi/scripts/file_info.lua index 0c172cf72e..069590bc74 100644 --- a/sipi/scripts/file_info.lua +++ b/sipi/scripts/file_info.lua @@ -35,7 +35,21 @@ local IMAGE_JPG = "image/jpeg" local APPLICATION_XML = "application/xml" local TEXT_XML = "text/xml" local TEXT_PLAIN = "text/plain" +local AUDIO_MP3 = "audio/mpeg" +local AUDIO_MP4 = "audio/mp4" +local AUDIO_WAV = "audio/x-wav" local APPLICATION_PDF = "application/pdf" +local APPLICATION_DOC = "application/msword" +local APPLICATION_DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +local APPLICATION_XLS = "application/vnd.ms-excel" +local APPLICATION_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +local APPLICATION_PPT = "application/vnd.ms-powerpoint" +local APPLICATION_PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation" +local APPLICATION_ZIP = "application/zip" +local APPLICATION_TAR = "application/x-tar" +local APPLICATION_ISO = "application/x-iso9660-image" +local APPLICATION_GZIP = "application/gzip" + local image_mime_types = { IMAGE_JP2, @@ -44,6 +58,12 @@ local image_mime_types = { IMAGE_JPG } +local audio_mime_types = { + AUDIO_MP3, + AUDIO_MP4, + AUDIO_WAV +} + local text_mime_types = { TEXT_PLAIN, APPLICATION_XML, @@ -51,7 +71,23 @@ local text_mime_types = { } local document_mime_types = { - APPLICATION_PDF + APPLICATION_PDF, + APPLICATION_TAR, + APPLICATION_ZIP, + APPLICATION_ISO, + APPLICATION_GZIP, + APPLICATION_DOC, + APPLICATION_DOCX, + APPLICATION_XLS, + APPLICATION_XLSX, + APPLICATION_PPT, + APPLICATION_PPTX +} + +local audio_extensions = { + "mp3", + "mp4", + "wav" } local text_extensions = { @@ -63,16 +99,37 @@ local text_extensions = { } local document_extensions = { - "pdf" + "pdf", + "zip", + "tar", + "iso", + "gz", + "doc", + "docx", + "xls", + "xlsx", + "ppt", + "pptx" } -function make_image_file_info() +function make_image_file_info(extension) return { media_type = IMAGE, - extension = "jp2" + extension = extension } end +function make_audio_file_info(extension) + if not table.contains(audio_extensions, extension) then + return nil + else + return { + media_type = AUDIO, + extension = extension + } + end +end + function make_text_file_info(extension) if not table.contains(text_extensions, extension) then return nil @@ -105,12 +162,15 @@ end -- a table containing "media_type" and "extension", or false if no supported media type was found. ------------------------------------------------------------------------------- function get_file_info(filename, mimetype) + local extension = filename:match("^.+%.([^.]+)$") if extension == nil then return nil elseif table.contains(image_mime_types, mimetype) then - return make_image_file_info() + return make_image_file_info(extension) + elseif table.contains(audio_mime_types, mimetype) then + return make_audio_file_info(extension) elseif table.contains(text_mime_types, mimetype) then return make_text_file_info(extension) elseif table.contains(document_mime_types, mimetype) then diff --git a/sipi/scripts/store.lua b/sipi/scripts/store.lua index 27dad5e338..93b592873f 100644 --- a/sipi/scripts/store.lua +++ b/sipi/scripts/store.lua @@ -22,52 +22,145 @@ require "send_response" require "jwt" --- Buffer the response (helps with error handling). +---------------------------------------- +-- Extract the full filename form a path +---------------------------------------- +function get_file_name(path) + local str = path + local temp = "" + local result = "" + + -- Get file name + extension until first forward slash (/) and then break + for i = str:len(), 1, -1 do + if str:sub(i,i) ~= "/" then + temp = temp..str:sub(i,i) + else + break + end + end -local success, error_msg = server.setBuffer() + -- Reverse order of full file name + for j = temp:len(), 1, -1 do + result = result..temp:sub(j,j) + end + + return result +end +---------------------------------------- + +-------------------------------------------------------------------------------- +-- Get the extension of a string determined by a dot . at the end of the string. +-------------------------------------------------------------------------------- +function get_file_extension(path) + local str = path + local temp = "" + local result = "" + + for i = str:len(), 1, -1 do + if str:sub(i,i) ~= "." then + temp = temp..str:sub(i,i) + else + break + end + end + + -- Reverse order of full file name + for j = temp:len(), 1, -1 do + result = result..temp:sub(j,j) + end + + return result +end +-------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- Get the basename of a string determined by removing the extension +-------------------------------------------------------------------------------- +function get_file_basename(path) + local str = path + local temp = "" + local result = "" + local pfound = false + + for i = str:len(), 1, -1 do + if str:sub(i,i) ~= "." then + if pfound then temp = temp..str:sub(i,i) end + else + pfound = true + end + end + if pfound then + -- Reverse order of full file name + for j = temp:len(), 1, -1 do + result = result..temp:sub(j,j) + end + else + result = str + end + + return result +end +------------------------------------------------------------------------------- + + +---------------------------------------------------- +-- Check if a directory exists. If not, create it -- +---------------------------------------------------- +function check_create_dir(path) + local exists + success, exists = server.fs.exists(path) + if not success then + return success, "server.fs.exists() failed: " .. exists + end + if not exists then + success, error_msg = server.fs.mkdir(path, 511) + if not success then + return success, "server.fs.mkdir() failed: " .. error_msg + end + end + return true, "OK" +end +---------------------------------------------------- + +-- Buffer the response (helps with error handling). +local success, error_msg = server.setBuffer() if not success then send_error(500, "server.setBuffer() failed: " .. error_msg) return end +-- -- Check that this request is really from Knora and that the user has permission -- to store the file. - +-- local token = get_knora_token() - if token == nil then return end - local knora_data = token["knora-data"] - if knora_data == nil then send_error(403, "No knora-data in token") return end - if knora_data["permission"] ~= "StoreFile" then send_error(403, "Token does not grant permission to store file") return end +-- get token filename local token_filename = knora_data["filename"] - if token_filename == nil then send_error(401, "Token does not specify a filename") return end +-- get token prefix local token_prefix = knora_data["prefix"] - if token_prefix == nil then send_error(401, "Token does not specify a prefix") return end - --- Check that the permanent storage directory exists. - local prefix = server.post["prefix"] if prefix ~= token_prefix then @@ -75,45 +168,40 @@ if prefix ~= token_prefix then return end -local storage_dir = config.imgroot .. "/" .. prefix .. "/" -local exists -success, exists = server.fs.exists(storage_dir) +-- Check that original file storage directory exists +local originals_dir = config.imgroot .. "/originals/" .. prefix .. "/" +success, msg = check_create_dir(config.imgroot .. "/originals/") if not success then - send_error(500, "server.fs.exists() failed: " .. exists) + send_error(500, msg) return end - -if not exists then - success, error_msg = server.fs.mkdir(storage_dir, 511) - - if not success then - send_error(500, "server.fs.mkdir() failed: " .. error_msg) - return - end +success, msg = check_create_dir(originals_dir) +if not success then + send_error(500, msg) + return end --- Get the submitted filename. - +-- +-- Get the submitted filename and check consistency. +-- if server.post == nil then send_error(400, PARAMETERS_INCORRECT) return end - local filename = server.post["filename"] - if filename == nil then send_error(400, PARAMETERS_INCORRECT) return end - if filename ~= token_filename then send_error(401, "Incorrect filename in token") return end +-- -- Construct the path of that file under the temp directory. - +-- local hashed_filename success, hashed_filename = helper.filename_hash(filename) if not success then @@ -121,23 +209,30 @@ if not success then return end -local source_path = config.imgroot .. "/tmp/" .. hashed_filename - +-- -- Make sure the source file is readable. - +-- +local source_path = config.imgroot .. "/tmp/" .. hashed_filename local readable success, readable = server.fs.is_readable(source_path) if not success then send_error(500, "server.fs.is_readable() failed: " .. readable) return end - if not readable then send_error(400, source_path .. " not readable") return end --- Move the temporary file to the permanent storage directory. +-- +-- Move the temporary files to the permanent storage directory. +-- +local storage_dir = config.imgroot .. "/" .. prefix .. "/" +success, msg = check_create_dir(storage_dir) +if not success then + send_error(500, msg) + return +end local destination_path = storage_dir .. hashed_filename success, error_msg = server.fs.moveFile(source_path, destination_path) @@ -146,6 +241,61 @@ if not success then return end +-- +-- Move sidecarfile if it exists +-- +local originals_dir = config.imgroot .. "/originals/" .. prefix .. "/" +success, msg = check_create_dir(originals_dir) +if not success then + send_error(500, msg) + return +end +local hashed_sidecar = get_file_basename(hashed_filename) .. ".info" +local source_sidecar = config.imgroot .. "/tmp/" .. hashed_sidecar +success, readable = server.fs.is_readable(source_sidecar) +if not success then + send_error(500, "server.fs.is_readable() failed: " .. readable) + return +end +if readable then + -- first we read the file into a string + local f = io.open(source_sidecar) + local jsonstr = f:read("*a") + f:close() + success, sidecar = server.json_to_table(jsonstr) + if not success then + send_error(500, "server.json_to_table() failed: " .. sidecar) + return + end + + -- copy sidecar to IIIF directory for this project + local destination_sidecar = storage_dir .. hashed_sidecar + success, error_msg = server.fs.copyFile(source_sidecar, destination_sidecar) + if not success then + send_error(500, "server.fs.copyFile() failed: " .. error_msg) + return + end + + -- copy sidecar file to originals directory + local destination2_sidecar = originals_dir .. hashed_sidecar + success, error_msg = server.fs.moveFile(source_sidecar, destination2_sidecar) + if not success then + send_error(500, "server.fs.moveFile() failed: " .. error_msg) + return + end + + -- move the original file to the originals directory + local source_original = config.imgroot .. "/tmp/" .. sidecar["originalInternalFilename"] + local destination_original = originals_dir .. sidecar["originalInternalFilename"] + success, error_msg = server.fs.moveFile(source_original, destination_original) + if not success then + send_error(500, "server.fs.moveFile() failed: " .. error_msg) + return + end + +end + + server.log("store.lua: moved " .. source_path .. " to " .. destination_path, server.loglevel.LOG_DEBUG) local result = { diff --git a/sipi/scripts/upload.lua b/sipi/scripts/upload.lua index b2c24b0442..53cb86c957 100644 --- a/sipi/scripts/upload.lua +++ b/sipi/scripts/upload.lua @@ -25,38 +25,42 @@ require "jwt" require "clean_temp_dir" require "util" --- Buffer the response (helps with error handling). +-------------------------------------------------------------------------- +-- Calculate the SHA256 checksum of a file using the operating system tool +-------------------------------------------------------------------------- +function file_checksum(path) + local handle = io.popen("/usr/bin/sha256sum " .. path) + local checksum_orig = handle:read("*a") + handle:close() + return string.match(checksum_orig, "%w*") +end +-------------------------------------------------------------------------- -local success, error_msg = server.setBuffer() +-- Buffer the response (helps with error handling). +local success, error_msg = server.setBuffer() if not success then send_error(500, "server.setBuffer() failed: " .. error_msg) return end -- Check for a valid JSON Web Token from Knora. - local token = get_knora_token() - if token == nil then return end -- Check that the temp folder is created local tmpFolder = config.imgroot .. '/tmp/' - local exists success, exists = server.fs.exists(tmpFolder) - if not success then send_error(500, "server.fs.exists() failed: " .. exists) return end - if not exists then local error_msg success, error_msg = server.fs.mkdir(tmpFolder, 511) - if not success then send_error(500, "server.fs.mkdir() failed: " .. error_msg) return @@ -68,26 +72,26 @@ local file_upload_data = {} -- Process the uploaded files. for file_index, file_params in pairs(server.uploads) do + -- -- Check that the file's MIME type is supported. - + -- local mime_info success, mime_info = server.file_mimetype(file_index) - if not success then send_error(500, "server.file_mimetype() failed: " .. tostring(mime_info)) return end - local mime_type = mime_info["mimetype"] - if mime_type == nil then send_error(400, "Could not determine MIME type of uploaded file") return end + -- + -- get some more MIME type related information + -- local original_filename = file_params["origname"] local file_info = get_file_info(original_filename, mime_type) - if file_info == nil then send_error(400, "Unsupported MIME type: " .. tostring(mime_type)) return @@ -96,52 +100,69 @@ for file_index, file_params in pairs(server.uploads) do -- Make a random filename for the temporary file. local uuid62 success, uuid62 = server.uuid62() - if not success then send_error(500, "server.uuid62() failed: " .. uuid62) return end - if server.secure then - protocol = 'https://' + + -- Construct response data about the file that was uploaded. + local media_type = file_info["media_type"] + + -- Add a subdirectory path if necessary. + local tmp_storage_filename + if media_type == IMAGE then + tmp_storage_filename = uuid62 .. ".jp2" else - protocol = 'http://' + tmp_storage_filename = uuid62 .. "." .. file_info["extension"] end - - local tmp_storage_filename = uuid62 .. "." .. file_info["extension"] - - -- Add a subdirectory path if necessary. local hashed_tmp_storage_filename success, hashed_tmp_storage_filename = helper.filename_hash(tmp_storage_filename) - if not success then send_error(500, "helper.filename_hash() failed: " .. tostring(hashed_tmp_storage_filename)) return end + -- filename for sidecar file + local tmp_storage_sidecar = uuid62 .. ".info" + local hashed_tmp_storage_sidecar + success, hashed_tmp_storage_sidecar = helper.filename_hash(tmp_storage_sidecar) + if not success then + send_error(500, "helper.filename_hash() failed: " .. tostring(hashed_tmp_storage_sidecar)) + return + end + + -- filename for original file copy + local tmp_storage_original = uuid62 .. "." .. file_info["extension"] .. ".orig" + local hashed_tmp_storage_original + success, hashed_tmp_storage_original = helper.filename_hash(tmp_storage_original) + if not success then + send_error(500, "helper.filename_hash() failed: " .. tostring(hashed_tmp_storage_original)) + return + end + local tmp_storage_file_path = config.imgroot .. '/tmp/' .. hashed_tmp_storage_filename + local tmp_storage_sidecar_path = config.imgroot .. '/tmp/' .. hashed_tmp_storage_sidecar + local tmp_storage_original_path = config.imgroot .. '/tmp/' .. hashed_tmp_storage_original -- Create a IIIF base URL for the converted file. local tmp_storage_url = get_external_protocol() .. "://" .. get_external_hostname() .. ":" .. get_external_port() .. '/tmp/' .. tmp_storage_filename - -- Construct response data about the file that was uploaded. - - local media_type = file_info["media_type"] - local this_file_upload_data = {} - this_file_upload_data["internalFilename"] = tmp_storage_filename - this_file_upload_data["originalFilename"] = original_filename - this_file_upload_data["temporaryUrl"] = tmp_storage_url - this_file_upload_data["fileType"] = media_type - file_upload_data[file_index] = this_file_upload_data + -- Copy original file also to tmp + success, error_msg = server.copyTmpfile(file_index, tmp_storage_original_path) + if not success then + send_error(500, "server.copyTmpfile() failed for " .. tostring(tmp_storage_original_path) .. ": " .. tostring(error_msg)) + return + end -- Is this an image file? if media_type == IMAGE then + -- -- Yes. Create a new Lua image object. This reads the image into an - -- internal in-memory representation independent of the original - -- image format. + -- internal in-memory representation independent of the original image format. + -- local uploaded_image - success, uploaded_image = SipiImage.new(file_index) - + success, uploaded_image = SipiImage.new(file_index, {original = original_filename, hash = "sha256"}) if not success then send_error(500, "SipiImage.new() failed: " .. tostring(uploaded_image)) return @@ -150,44 +171,75 @@ for file_index, file_params in pairs(server.uploads) do -- Check that the file extension is correct for the file's MIME type. local check success, check = uploaded_image:mimetype_consistency(mime_type, original_filename) - if not success then send_error(500, "upload.lua: uploaded_image:mimetype_consistency() failed: " .. check) return end - if not check then send_error(400, MIMETYPES_INCONSISTENCY) return end -- Convert the image to JPEG 2000 format. - success, error_msg = uploaded_image:write(tmp_storage_file_path) - if not success then send_error(500, "uploaded_image:write() failed for " .. tostring(tmp_storage_file_path) .. ": " .. tostring(error_msg)) return end - server.log("upload.lua: wrote image file to " .. tmp_storage_file_path, server.loglevel.LOG_DEBUG) else -- It's not an image file. Just move it to its temporary storage location. - success, error_msg = server.copyTmpfile(file_index, tmp_storage_file_path) - if not success then send_error(500, "server.copyTmpfile() failed for " .. tostring(tmp_storage_file_path) .. ": " .. tostring(error_msg)) return end - server.log("upload.lua: wrote non-image file to " .. tmp_storage_file_path, server.loglevel.LOG_DEBUG) end + + -- + -- Calculate checksum of original file + -- + local checksum_original = file_checksum(tmp_storage_original_path) + + -- + -- Calculate checksum of derivative file + -- + local checksum_derivative = file_checksum(tmp_storage_file_path) + + -- + -- prepare and write sidecar file + -- + local sidecar_data = { + originalFilename = original_filename, + checksumOriginal = checksum_original, + originalInternalFilename = hashed_tmp_storage_original, + internalFilename = tmp_storage_filename, + checksumDerivative = checksum_derivative + } + local success, jsonstr = server.table_to_json(sidecar_data) + if not success then + send_error(500, "Couldn't create json string!") + return + end + sidecar = io.open(tmp_storage_sidecar_path, "w") + sidecar:write(jsonstr) + sidecar:close() + + local this_file_upload_data = { + internalFilename = tmp_storage_filename, + originalFilename = original_filename, + temporaryUrl = tmp_storage_url, + fileType = media_type, + sidecarFile = tmp_storage_sidecar, + checksumOriginal = checksum_orig, + checksumDerivative = checksum_derivative + } + file_upload_data[file_index] = this_file_upload_data end -- Clean up old temporary files. clean_temp_dir() - -- Return the file upload data in the response. local response = {} response["uploadedFiles"] = file_upload_data diff --git a/test_data/test_route/files/minimal.zip b/test_data/test_route/files/minimal.zip new file mode 100644 index 0000000000..30389846f8 Binary files /dev/null and b/test_data/test_route/files/minimal.zip differ diff --git a/test_data/test_route/files/test.zip b/test_data/test_route/files/test.zip new file mode 100644 index 0000000000..b42b309919 Binary files /dev/null and b/test_data/test_route/files/test.zip differ diff --git a/third_party/versions.bzl b/third_party/versions.bzl index 4371ac1904..d51558b146 100644 --- a/third_party/versions.bzl +++ b/third_party/versions.bzl @@ -8,9 +8,9 @@ METRICS_VERSION = "4.0.1" # SIPI - digest takes precedence! SIPI_REPOSITORY = "daschswiss/sipi" -SIPI_VERSION = "3.0.3" +SIPI_VERSION = "3.1.0" SIPI_IMAGE = SIPI_REPOSITORY + ":" + SIPI_VERSION -SIPI_IMAGE_DIGEST = "sha256:54d1cef3b923d87fa0feaf259b44e6b683a5e94392bc4b94b35685cffc7a9ca1" +SIPI_IMAGE_DIGEST = "sha256:51774e98160af255e4b542d9f1944afe43265001b20b365003848820384ea299" # Jena Fuseki - digest takes precedence! FUSEKI_REPOSITORY = "daschswiss/apache-jena-fuseki" diff --git a/webapi/src/main/resources/application.conf b/webapi/src/main/resources/application.conf index 0a419e76dc..749d1dfb0d 100644 --- a/webapi/src/main/resources/application.conf +++ b/webapi/src/main/resources/application.conf @@ -385,10 +385,21 @@ app { } image-mime-types = ["image/tiff", "image/jpeg", "image/png", "image/jp2", "image/jpx"] - document-mime-types = ["application/pdf"] + document-mime-types = [ + "application/pdf", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/zip", + "application/x-tar", + "application/x-iso9660-image", + ] text-mime-types = ["application/xml", "text/xml", "text/csv", "text/plain"] movie-mime-types = [] - sound-mime-types = [] + sound-mime-types = ["audio/mpeg", "audio/mp4", "audio/x-wav", "audio/vnd.wav"] } ark { diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/ConstructResponseUtilV2.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/ConstructResponseUtilV2.scala index e22932e5b4..60a05d5224 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/ConstructResponseUtilV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/ConstructResponseUtilV2.scala @@ -1074,7 +1074,7 @@ object ConstructResponseUtilV2 { DocumentFileValueContentV2( ontologySchema = InternalSchema, fileValue = fileValue, - pageCount = valueObject.requireIntObject(OntologyConstants.KnoraBase.PageCount.toSmartIri), + pageCount = valueObject.maybeIntObject(OntologyConstants.KnoraBase.PageCount.toSmartIri), dimX = valueObject.maybeIntObject(OntologyConstants.KnoraBase.DimX.toSmartIri), dimY = valueObject.maybeIntObject(OntologyConstants.KnoraBase.DimY.toSmartIri), comment = valueCommentOption diff --git a/webapi/src/main/scala/org/knora/webapi/messages/util/ValueUtilV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/util/ValueUtilV1.scala index 04ae657c07..fb2314c443 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/util/ValueUtilV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/util/ValueUtilV1.scala @@ -157,7 +157,10 @@ class ValueUtilV1(private val settings: KnoraSettingsImpl) { "text/xml" -> "XML", "text/csv" -> "CSV", "application/zip" -> "ZIP", - "application/x-compressed-zip" -> "ZIP" + "application/x-compressed-zip" -> "ZIP", + "audio/x-wav" -> "AUDIO", + "audio/mp4" -> "AUDIO", + "audio/mpeg" -> "AUDIO" ), { key: String => s"Unknown MIME type: $key" } @@ -847,7 +850,7 @@ class ValueUtilV1(private val settings: KnoraSettingsImpl) { internalFilename = predicates(OntologyConstants.KnoraBase.InternalFilename).literals.head, originalFilename = predicates.get(OntologyConstants.KnoraBase.OriginalFilename).map(_.literals.head), projectShortcode = projectShortcode, - pageCount = predicates(OntologyConstants.KnoraBase.PageCount).literals.head.toInt, + pageCount = predicates.get(OntologyConstants.KnoraBase.PageCount).flatMap(_.literals.headOption.map(_.toInt)), dimX = predicates.get(OntologyConstants.KnoraBase.DimX).flatMap(_.literals.headOption.map(_.toInt)), dimY = predicates.get(OntologyConstants.KnoraBase.DimY).flatMap(_.literals.headOption.map(_.toInt)) )) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala index 68c41337ab..a76241569c 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala @@ -1584,7 +1584,7 @@ case class DocumentFileValueV1(internalMimeType: String, originalFilename: Option[String] = None, originalMimeType: Option[String] = None, projectShortcode: String, - pageCount: Int, + pageCount: Option[Int], dimX: Option[Int], dimY: Option[Int]) extends FileValueV1 { diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala b/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala index b770b485fa..f9bac2f921 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala @@ -3250,7 +3250,7 @@ object StillImageFileValueContentV2 extends ValueContentReaderV2[StillImageFileV */ case class DocumentFileValueContentV2(ontologySchema: OntologySchema, fileValue: FileValueV2, - pageCount: Int, + pageCount: Option[Int], dimX: Option[Int], dimY: Option[Int], comment: Option[String] = None) @@ -3269,7 +3269,7 @@ case class DocumentFileValueContentV2(ontologySchema: OntologySchema, projectADM: ProjectADM, settings: KnoraSettingsImpl, schemaOptions: Set[SchemaOption]): JsonLDValue = { - val fileUrl: String = s"${settings.externalSipiBaseUrl}/${projectADM.shortcode}/${fileValue.internalFilename}" + val fileUrl: String = s"${settings.externalSipiBaseUrl}/${projectADM.shortcode}/${fileValue.internalFilename}/file" targetSchema match { case ApiV2Simple => toJsonLDValueInSimpleSchema(fileUrl) @@ -3283,10 +3283,12 @@ case class DocumentFileValueContentV2(ontologySchema: OntologySchema, OntologyConstants.KnoraApiV2Complex.DocumentFileValueHasDimY -> JsonLDInt(definedDimY) } + val maybePageCountStatement: Option[(IRI, JsonLDInt)] = pageCount.map { definedPageCount => + OntologyConstants.KnoraApiV2Complex.DocumentFileValueHasPageCount -> JsonLDInt(definedPageCount) + } + JsonLDObject( - toJsonLDObjectMapInComplexSchema(fileUrl) ++ Map( - OntologyConstants.KnoraApiV2Complex.DocumentFileValueHasPageCount -> JsonLDInt(pageCount) - ) ++ maybeDimXStatement ++ maybeDimYStatement + toJsonLDObjectMapInComplexSchema(fileUrl) ++ maybeDimXStatement ++ maybeDimYStatement ++ maybePageCountStatement ) } } @@ -3347,8 +3349,7 @@ object DocumentFileValueContentV2 extends ValueContentReaderV2[DocumentFileValue DocumentFileValueContentV2( ontologySchema = ApiV2Complex, fileValue = fileValueWithSipiMetadata.fileValue, - pageCount = fileValueWithSipiMetadata.sipiFileMetadata.pageCount - .getOrElse(throw SipiException("Sipi did not return a page count")), + pageCount = fileValueWithSipiMetadata.sipiFileMetadata.pageCount, dimX = fileValueWithSipiMetadata.sipiFileMetadata.width, dimY = fileValueWithSipiMetadata.sipiFileMetadata.height, comment = getComment(jsonLDObject) @@ -3380,7 +3381,7 @@ case class TextFileValueContentV2(ontologySchema: OntologySchema, projectADM: ProjectADM, settings: KnoraSettingsImpl, schemaOptions: Set[SchemaOption]): JsonLDValue = { - val fileUrl: String = s"${settings.externalSipiBaseUrl}/${projectADM.shortcode}/${fileValue.internalFilename}" + val fileUrl: String = s"${settings.externalSipiBaseUrl}/${projectADM.shortcode}/${fileValue.internalFilename}/file" targetSchema match { case ApiV2Simple => toJsonLDValueInSimpleSchema(fileUrl) diff --git a/webapi/src/main/scala/org/knora/webapi/package.scala b/webapi/src/main/scala/org/knora/webapi/package.scala index 5524a3dbbf..0396ad51b2 100644 --- a/webapi/src/main/scala/org/knora/webapi/package.scala +++ b/webapi/src/main/scala/org/knora/webapi/package.scala @@ -25,7 +25,7 @@ package object webapi { * The version of `knora-base` and of the other built-in ontologies that this version of Knora requires. * Must be the same as the object of `knora-base:ontologyVersion` in the `knora-base` ontology being used. */ - val KnoraBaseVersion: String = "knora-base v9" + val KnoraBaseVersion: String = "knora-base v10" /** * `IRI` is a synonym for `String`, used to improve code readability. diff --git a/webapi/src/main/scala/org/knora/webapi/routing/RouteUtilV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/RouteUtilV1.scala index 862cbae5c1..c5ca3ae6ee 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/RouteUtilV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/RouteUtilV1.scala @@ -266,7 +266,17 @@ object RouteUtilV1 { * MIME types used in Sipi to store document files. */ private val documentMimeTypes: Set[String] = Set( - "application/pdf" + "application/pdf", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/zip", + "application/x-tar", + "application/x-iso9660-image", + "application/gzip" ) /** @@ -306,8 +316,7 @@ object RouteUtilV1 { originalFilename = fileMetadataResponse.originalFilename, originalMimeType = fileMetadataResponse.originalMimeType, projectShortcode = projectShortcode, - pageCount = fileMetadataResponse.pageCount.getOrElse( - throw SipiException(s"Sipi did not return the page count of the document")), + pageCount = fileMetadataResponse.pageCount, dimX = fileMetadataResponse.width, dimY = fileMetadataResponse.height ) diff --git a/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala b/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala index 60dad47083..62eadd17fb 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala @@ -69,12 +69,6 @@ class SipiConnector extends Actor with ActorLogging { private val httpClient: CloseableHttpClient = HttpClients.custom.setDefaultRequestConfig(sipiRequestConfig).build - // Sipi's /knora.json route only returns the correct original filename for images. - private val internalMimeTypesForWhichSipiReturnsTheCorrectOriginalFilename: Set[String] = Set( - "image/jpx", - "image/jp2" - ) - override def receive: Receive = { case getFileMetadataRequest: GetFileMetadataRequest => try2Message(sender(), getFileMetadata(getFileMetadataRequest), log) @@ -134,10 +128,7 @@ class SipiConnector extends Actor with ActorLogging { sipiResponse: SipiKnoraJsonResponse = sipiResponseStr.parseJson.convertTo[SipiKnoraJsonResponse] } yield GetFileMetadataResponse( - originalFilename = - if (internalMimeTypesForWhichSipiReturnsTheCorrectOriginalFilename.contains(sipiResponse.internalMimeType)) - sipiResponse.originalFilename - else None, + originalFilename = sipiResponse.originalFilename, originalMimeType = sipiResponse.originalMimeType, internalMimeType = sipiResponse.internalMimeType, width = sipiResponse.width, diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala index 68f8427dbf..a174206d47 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala @@ -32,7 +32,8 @@ object RepositoryUpdatePlan { PluginForKnoraBaseVersion(versionNumber = 6, plugin = new NoopPlugin), // PR 1206 PluginForKnoraBaseVersion(versionNumber = 7, plugin = new NoopPlugin), // PR 1403 PluginForKnoraBaseVersion(versionNumber = 8, plugin = new UpgradePluginPR1615(featureFactoryConfig)), - PluginForKnoraBaseVersion(versionNumber = 9, plugin = new UpgradePluginPR1746(featureFactoryConfig, log)) + PluginForKnoraBaseVersion(versionNumber = 9, plugin = new UpgradePluginPR1746(featureFactoryConfig, log)), + PluginForKnoraBaseVersion(versionNumber = 10, plugin = new NoopPlugin) // PR 1808 ) /** diff --git a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/addValueVersion.scala.txt b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/addValueVersion.scala.txt index 967856b8dd..0b71321e46 100644 --- a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/addValueVersion.scala.txt +++ b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/addValueVersion.scala.txt @@ -268,7 +268,6 @@ DELETE { case documentFileValue: DocumentFileValueV1 => { ?newValue knora-base:internalFilename """@documentFileValue.internalFilename""" . ?newValue knora-base:internalMimeType """@documentFileValue.internalMimeType""" . - ?newValue knora-base:pageCount @documentFileValue.pageCount . @documentFileValue.dimX match { case Some(definedDimX) => { @@ -286,6 +285,14 @@ DELETE { case None => {} } + @documentFileValue.pageCount match { + case Some(definedPageCount) => { + ?newValue knora-base:pageCount @definedPageCount . + } + + case None => {} + } + @documentFileValue.originalFilename match { case Some(definedOriginalFilename) => { ?newValue knora-base:originalFilename """@definedOriginalFilename""" . diff --git a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/generateInsertStatementsForCreateValue.scala.txt b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/generateInsertStatementsForCreateValue.scala.txt index ce858371bb..e610b088c1 100644 --- a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/generateInsertStatementsForCreateValue.scala.txt +++ b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v1/generateInsertStatementsForCreateValue.scala.txt @@ -223,8 +223,7 @@ case documentFileValue: DocumentFileValueV1 => { <@newValueIri> knora-base:internalFilename """@documentFileValue.internalFilename""" ; - knora-base:internalMimeType """@documentFileValue.internalMimeType""" ; - knora-base:pageCount @documentFileValue.pageCount . + knora-base:internalMimeType """@documentFileValue.internalMimeType""" . @documentFileValue.dimX match { case Some(definedDimX) => { @@ -242,6 +241,14 @@ case None => {} } + @documentFileValue.pageCount match { + case Some(definedPageCount) => { + <@newValueIri> knora-base:pageCount @definedPageCount . + } + + case None => {} + } + @documentFileValue.originalFilename match { case Some(definedOriginalFilename) => { <@newValueIri> knora-base:originalFilename """@definedOriginalFilename""" . diff --git a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/generateInsertStatementsForValueContent.scala.txt b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/generateInsertStatementsForValueContent.scala.txt index 65a0d4efb7..0c9ae4cc50 100644 --- a/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/generateInsertStatementsForValueContent.scala.txt +++ b/webapi/src/main/twirl/org/knora/webapi/messages/twirl/queries/sparql/v2/generateInsertStatementsForValueContent.scala.txt @@ -180,11 +180,9 @@ } case documentFileValue: DocumentFileValueContentV2 => { - <@newValueIri> knora-base:pageCount @documentFileValue.pageCount . - @documentFileValue.dimX match { case Some(definedDimX) => { - <@newValueIri> knora-base:dimX @documentFileValue.dimX . + <@newValueIri> knora-base:dimX @definedDimX . } case None => {} @@ -192,7 +190,15 @@ @documentFileValue.dimY match { case Some(definedDimY) => { - <@newValueIri> knora-base:dimY @documentFileValue.dimY . + <@newValueIri> knora-base:dimY @definedDimY . + } + + case None => {} + } + + @documentFileValue.pageCount match { + case Some(definedPageCount) => { + <@newValueIri> knora-base:pageCount @definedPageCount . } case None => {} diff --git a/webapi/src/test/scala/org/knora/webapi/it/v1/KnoraSipiIntegrationV1ITSpec.scala b/webapi/src/test/scala/org/knora/webapi/it/v1/KnoraSipiIntegrationV1ITSpec.scala index 9a2b5dd39f..513d069bf2 100644 --- a/webapi/src/test/scala/org/knora/webapi/it/v1/KnoraSipiIntegrationV1ITSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/it/v1/KnoraSipiIntegrationV1ITSpec.scala @@ -79,6 +79,7 @@ class KnoraSipiIntegrationV1ITSpec private val gravsearchTemplateIri = new MutableTestIri private val pdfResourceIri = new MutableTestIri + private val zipResourceIri = new MutableTestIri private val minimalPdfOriginalFilename = "minimal.pdf" private val pathToMinimalPdf = s"test_data/test_route/files/$minimalPdfOriginalFilename" @@ -90,6 +91,12 @@ class KnoraSipiIntegrationV1ITSpec private val testPdfWidth = 2480 private val testPdfHeight = 3508 + private val minimalZipOriginalFilename = "minimal.zip" + private val pathToMinimalZip = s"test_data/test_route/files/$minimalZipOriginalFilename" + + private val testZipOriginalFilename = "test.zip" + private val pathToTestZip = s"test_data/test_route/files/$testZipOriginalFilename" + /** * Adds the IRI of a XSL transformation to the given mapping. * @@ -726,6 +733,7 @@ class KnoraSipiIntegrationV1ITSpec ) val uploadedPdfFile: SipiUploadResponseEntry = pdfUploadResponse.uploadedFiles.head + uploadedPdfFile.originalFilename should ===(minimalPdfOriginalFilename) // Create a resource for the PDF file. val createDocumentResourceParams = JsObject( @@ -808,5 +816,95 @@ class KnoraSipiIntegrationV1ITSpec assert(nx == testPdfWidth) assert(ny == testPdfHeight) } + + "create a resource with a Zip file attached" in { + // Upload the Zip file to Sipi. + val zipUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToMinimalZip, mimeType = MediaTypes.`application/zip`)) + ) + + val uploadedZipFile: SipiUploadResponseEntry = zipUploadResponse.uploadedFiles.head + uploadedZipFile.originalFilename should ===(minimalZipOriginalFilename) + + // Create a resource for the Zip file. + val createDocumentResourceParams = JsObject( + Map( + "restype_id" -> JsString("http://www.knora.org/ontology/0001/anything#ThingDocument"), + "label" -> JsString("Zip file"), + "project_id" -> JsString("http://rdfh.ch/projects/0001"), + "properties" -> JsObject(), + "file" -> JsString(uploadedZipFile.internalFilename) + ) + ) + + // Send the JSON in a POST request to the Knora API server. + val createDocumentResourceRequest: HttpRequest = Post( + baseApiUrl + "/v1/resources", + HttpEntity(ContentTypes.`application/json`, createDocumentResourceParams.compactPrint)) ~> addCredentials( + BasicHttpCredentials(userEmail, password)) + + val createDocumentResourceResponseJson: JsObject = getResponseJson(createDocumentResourceRequest) + + // get the IRI of the document file resource + val resourceIri: String = createDocumentResourceResponseJson.fields.get("res_id") match { + case Some(JsString(res_id: String)) => res_id + case _ => throw InvalidApiJsonException("member 'res_id' was expected") + } + + zipResourceIri.set(resourceIri) + + // Request the document resource from the Knora API server. + val documentResourceRequest = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(resourceIri, "UTF-8")) ~> addCredentials( + BasicHttpCredentials(userEmail, password)) + + val documentResourceResponse: JsObject = getResponseJson(documentResourceRequest) + val locdata = documentResourceResponse.fields("resinfo").asJsObject.fields("locdata").asJsObject + val zipUrl = + locdata.fields("path").asInstanceOf[JsString].value.replace("http://0.0.0.0:1024", baseInternalSipiUrl) + + // Request the file from Sipi. + val sipiGetRequest = Get(zipUrl) ~> addCredentials(BasicHttpCredentials(userEmail, password)) + checkResponseOK(sipiGetRequest) + } + + "change the Zip file attached to a resource" in { + // Upload the file to Sipi. + val sipiUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToTestZip, mimeType = MediaTypes.`application/zip`)) + ) + + val uploadedFile: SipiUploadResponseEntry = sipiUploadResponse.uploadedFiles.head + uploadedFile.originalFilename should ===(testZipOriginalFilename) + + // JSON describing the new file to Knora. + val knoraParams = JsObject( + Map( + "file" -> JsString(s"${uploadedFile.internalFilename}") + ) + ) + + // Send the JSON in a PUT request to the Knora API server. + val knoraPutRequest = Put( + baseApiUrl + "/v1/filevalue/" + URLEncoder.encode(zipResourceIri.get, "UTF-8"), + HttpEntity(ContentTypes.`application/json`, knoraParams.compactPrint)) ~> addCredentials( + BasicHttpCredentials(userEmail, password)) + + checkResponseOK(knoraPutRequest) + + // Request the document resource from the Knora API server. + val documentResourceRequest = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(zipResourceIri.get, "UTF-8")) ~> addCredentials( + BasicHttpCredentials(userEmail, password)) + + val documentResourceResponse: JsObject = getResponseJson(documentResourceRequest) + val locdata = documentResourceResponse.fields("resinfo").asJsObject.fields("locdata").asJsObject + val zipUrl = + locdata.fields("path").asInstanceOf[JsString].value.replace("http://0.0.0.0:1024", baseInternalSipiUrl) + + // Request the file from Sipi. + val sipiGetRequest = Get(zipUrl) ~> addCredentials(BasicHttpCredentials(userEmail, password)) + checkResponseOK(sipiGetRequest) + } } } diff --git a/webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala b/webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala index 98862d156f..29829d5ddc 100644 --- a/webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala @@ -61,9 +61,14 @@ class KnoraSipiIntegrationV2ITSpec private val stillImageResourceIri = new MutableTestIri private val stillImageFileValueIri = new MutableTestIri - private val pdfResourceIri = new MutableTestIri private val pdfValueIri = new MutableTestIri + private val xmlResourceIri = new MutableTestIri + private val xmlValueIri = new MutableTestIri + private val csvResourceIri = new MutableTestIri + private val csvValueIri = new MutableTestIri + private val zipResourceIri = new MutableTestIri + private val zipValueIri = new MutableTestIri private val marblesOriginalFilename = "marbles.tif" private val pathToMarbles = s"test_data/test_route/images/$marblesOriginalFilename" @@ -93,17 +98,17 @@ class KnoraSipiIntegrationV2ITSpec private val csv2OriginalFilename = "spam.csv" private val pathToCsv2 = s"test_data/test_route/files/$csv2OriginalFilename" - private val csvResourceIri = new MutableTestIri - private val csvValueIri = new MutableTestIri - private val xml1OriginalFilename = "test1.xml" private val pathToXml1 = s"test_data/test_route/files/$xml1OriginalFilename" private val xml2OriginalFilename = "test2.xml" private val pathToXml2 = s"test_data/test_route/files/$xml2OriginalFilename" - private val xmlResourceIri = new MutableTestIri - private val xmlValueIri = new MutableTestIri + private val minimalZipOriginalFilename = "minimal.zip" + private val pathToMinimalZip = s"test_data/test_route/files/$minimalZipOriginalFilename" + + private val testZipOriginalFilename = "test.zip" + private val pathToTestZip = s"test_data/test_route/files/$testZipOriginalFilename" /** * Represents the information that Knora returns about an image file value that was created. @@ -126,7 +131,7 @@ class KnoraSipiIntegrationV2ITSpec */ case class SavedDocument(internalFilename: String, url: String, - pageCount: Int, + pageCount: Option[Int], width: Option[Int], height: Option[Int]) @@ -226,7 +231,7 @@ class KnoraSipiIntegrationV2ITSpec validationFun = stringFormatter.toSparqlEncodedString ) - val pageCount: Int = savedValue.requireInt(OntologyConstants.KnoraApiV2Complex.DocumentFileValueHasPageCount) + val pageCount: Option[Int] = savedValue.maybeInt(OntologyConstants.KnoraApiV2Complex.DocumentFileValueHasPageCount) val dimX: Option[Int] = savedValue.maybeInt(OntologyConstants.KnoraApiV2Complex.DocumentFileValueHasDimX) val dimY: Option[Int] = savedValue.maybeInt(OntologyConstants.KnoraApiV2Complex.DocumentFileValueHasDimY) @@ -549,9 +554,14 @@ class KnoraSipiIntegrationV2ITSpec val savedDocument: SavedDocument = savedValueToSavedDocument(savedValueObj) assert(savedDocument.internalFilename == uploadedFile.internalFilename) - assert(savedDocument.pageCount == 1) + assert(savedDocument.pageCount.contains(1)) assert(savedDocument.width.contains(minimalPdfWidth)) assert(savedDocument.height.contains(minimalPdfHeight)) + + // Request the permanently stored file from Sipi. + println(s"PDF URL is ${savedDocument.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)}") + val sipiGetFileRequest = Get(savedDocument.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) } "change a PDF file value" in { @@ -602,9 +612,13 @@ class KnoraSipiIntegrationV2ITSpec val savedDocument: SavedDocument = savedValueToSavedDocument(savedValue) assert(savedDocument.internalFilename == uploadedFile.internalFilename) - assert(savedDocument.pageCount == 1) + assert(savedDocument.pageCount.contains(1)) assert(savedDocument.width.contains(testPdfWidth)) assert(savedDocument.height.contains(testPdfHeight)) + + // Request the permanently stored file from Sipi. + val sipiGetFileRequest = Get(savedDocument.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) } "create a resource with a CSV file" in { @@ -672,6 +686,10 @@ class KnoraSipiIntegrationV2ITSpec val savedTextFile: SavedTextFile = savedValueToSavedTextFile(savedValueObj) assert(savedTextFile.internalFilename == uploadedFile.internalFilename) + + // Request the permanently stored file from Sipi. + val sipiGetFileRequest = Get(savedTextFile.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) } "change a CSV file value" in { @@ -722,6 +740,10 @@ class KnoraSipiIntegrationV2ITSpec val savedTextFile: SavedTextFile = savedValueToSavedTextFile(savedValue) assert(savedTextFile.internalFilename == uploadedFile.internalFilename) + + // Request the permanently stored file from Sipi. + val sipiGetFileRequest = Get(savedTextFile.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) } "not create a resource with a still image file that's actually a text file" in { @@ -827,6 +849,10 @@ class KnoraSipiIntegrationV2ITSpec val savedTextFile: SavedTextFile = savedValueToSavedTextFile(savedValueObj) assert(savedTextFile.internalFilename == uploadedFile.internalFilename) + + // Request the permanently stored file from Sipi. + val sipiGetFileRequest = Get(savedTextFile.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) } "change an XML file value" in { @@ -877,6 +903,137 @@ class KnoraSipiIntegrationV2ITSpec val savedTextFile: SavedTextFile = savedValueToSavedTextFile(savedValue) assert(savedTextFile.internalFilename == uploadedFile.internalFilename) + + // Request the permanently stored file from Sipi. + val sipiGetFileRequest = Get(savedTextFile.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) + } + + "create a resource with a Zip file" in { + // Upload the file to Sipi. + val sipiUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToMinimalZip, mimeType = MediaTypes.`application/zip`)) + ) + + val uploadedFile: SipiUploadResponseEntry = sipiUploadResponse.uploadedFiles.head + uploadedFile.originalFilename should ===(minimalZipOriginalFilename) + + // Ask Knora to create the resource. + + val jsonLdEntity = + s"""{ + | "@type" : "anything:ThingDocument", + | "knora-api:hasDocumentFileValue" : { + | "@type" : "knora-api:DocumentFileValue", + | "knora-api:fileValueHasFilename" : "${uploadedFile.internalFilename}" + | }, + | "knora-api:attachedToProject" : { + | "@id" : "http://rdfh.ch/projects/0001" + | }, + | "rdfs:label" : "test thing", + | "@context" : { + | "rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + | "knora-api" : "http://api.knora.org/ontology/knora-api/v2#", + | "rdfs" : "http://www.w3.org/2000/01/rdf-schema#", + | "xsd" : "http://www.w3.org/2001/XMLSchema#", + | "anything" : "http://0.0.0.0:3333/ontology/0001/anything/v2#" + | } + |}""".stripMargin + + val request = Post(s"$baseApiUrl/v2/resources", HttpEntity(RdfMediaTypes.`application/ld+json`, jsonLdEntity)) ~> addCredentials( + BasicHttpCredentials(anythingUserEmail, password)) + val responseJsonDoc: JsonLDDocument = getResponseJsonLD(request) + zipResourceIri.set(responseJsonDoc.body.requireIDAsKnoraDataIri.toString) + + // Get the resource from Knora. + val knoraGetRequest = Get(s"$baseApiUrl/v2/resources/${URLEncoder.encode(zipResourceIri.get, "UTF-8")}") + val resource: JsonLDDocument = getResponseJsonLD(knoraGetRequest) + assert( + resource.requireTypeAsKnoraTypeIri.toString == "http://0.0.0.0:3333/ontology/0001/anything/v2#ThingDocument") + + // Get the new file value from the resource. + + val savedValues: JsonLDArray = getValuesFromResource( + resource = resource, + propertyIriInResult = OntologyConstants.KnoraApiV2Complex.HasDocumentFileValue.toSmartIri + ) + + val savedValue: JsonLDValue = if (savedValues.value.size == 1) { + savedValues.value.head + } else { + throw AssertionException(s"Expected one file value, got ${savedValues.value.size}") + } + + val savedValueObj: JsonLDObject = savedValue match { + case jsonLDObject: JsonLDObject => jsonLDObject + case other => throw AssertionException(s"Invalid value object: $other") + } + + zipValueIri.set(savedValueObj.requireIDAsKnoraDataIri.toString) + + val savedDocument: SavedDocument = savedValueToSavedDocument(savedValueObj) + assert(savedDocument.internalFilename == uploadedFile.internalFilename) + assert(savedDocument.pageCount.isEmpty) + + // Request the permanently stored file from Sipi. + val sipiGetFileRequest = Get(savedDocument.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) + } + + "change a Zip file value" in { + // Upload the file to Sipi. + val sipiUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToTestZip, mimeType = MediaTypes.`application/zip`)) + ) + + val uploadedFile: SipiUploadResponseEntry = sipiUploadResponse.uploadedFiles.head + uploadedFile.originalFilename should ===(testZipOriginalFilename) + + // Ask Knora to update the value. + + val jsonLdEntity = + s"""{ + | "@id" : "${zipResourceIri.get}", + | "@type" : "anything:ThingDocument", + | "knora-api:hasDocumentFileValue" : { + | "@id" : "${zipValueIri.get}", + | "@type" : "knora-api:DocumentFileValue", + | "knora-api:fileValueHasFilename" : "${uploadedFile.internalFilename}" + | }, + | "@context" : { + | "rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + | "knora-api" : "http://api.knora.org/ontology/knora-api/v2#", + | "rdfs" : "http://www.w3.org/2000/01/rdf-schema#", + | "xsd" : "http://www.w3.org/2001/XMLSchema#", + | "anything" : "http://0.0.0.0:3333/ontology/0001/anything/v2#" + | } + |}""".stripMargin + + val request = Put(s"$baseApiUrl/v2/values", HttpEntity(RdfMediaTypes.`application/ld+json`, jsonLdEntity)) ~> addCredentials( + BasicHttpCredentials(anythingUserEmail, password)) + val responseJsonDoc: JsonLDDocument = getResponseJsonLD(request) + zipValueIri.set(responseJsonDoc.body.requireIDAsKnoraDataIri.toString) + + // Get the resource from Knora. + val knoraGetRequest = Get(s"$baseApiUrl/v2/resources/${URLEncoder.encode(zipResourceIri.get, "UTF-8")}") + val resource = getResponseJsonLD(knoraGetRequest) + + // Get the new file value from the resource. + val savedValue: JsonLDObject = getValueFromResource( + resource = resource, + propertyIriInResult = OntologyConstants.KnoraApiV2Complex.HasDocumentFileValue.toSmartIri, + expectedValueIri = zipValueIri.get + ) + + val savedDocument: SavedDocument = savedValueToSavedDocument(savedValue) + assert(savedDocument.internalFilename == uploadedFile.internalFilename) + assert(savedDocument.pageCount.isEmpty) + + // Request the permanently stored file from Sipi. + val sipiGetFileRequest = Get(savedDocument.url.replace("http://0.0.0.0:1024", baseInternalSipiUrl)) + checkResponseOK(sipiGetFileRequest) } } }