From 61d2e868d71cf167241ab8bdbb8ddd817c06d7a6 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 21 Jan 2021 13:21:06 +0100 Subject: [PATCH] fix(HttpTriplestoreConnector): Always parse triplestore responses as UTF-8. (#1789) --- .../http/HttpTriplestoreConnector.scala | 12 +- .../v2/MetadataResponderV2Spec.scala | 173 +++++++++++++----- 2 files changed, 130 insertions(+), 55 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala index 54801e51ab..ea4670e678 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/http/HttpTriplestoreConnector.scala @@ -22,6 +22,7 @@ package org.knora.webapi.store.triplestore.http import java.io.BufferedInputStream import java.net.URI import java.nio.file.{Files, Path, Paths, StandardCopyOption} +import java.nio.charset.StandardCharsets import java.util import akka.actor.{Actor, ActorLogging, ActorSystem, Status} @@ -604,7 +605,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat val responseTry: Try[String] = Try { maybeResponse = Some(queryHttpClient.execute(targetHost, httpGet, context)) - EntityUtils.toString(maybeResponse.get.getEntity) + EntityUtils.toString(maybeResponse.get.getEntity, StandardCharsets.UTF_8) } maybeResponse.foreach(_.close()) @@ -712,7 +713,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat val responseTry: Try[String] = Try { maybeResponse = Some(queryHttpClient.execute(targetHost, httpGet, context)) - EntityUtils.toString(maybeResponse.get.getEntity) + EntityUtils.toString(maybeResponse.get.getEntity, StandardCharsets.UTF_8) } maybeResponse.foreach(_.close()) @@ -1008,7 +1009,8 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat val statusCategory: Int = statusCode / 100 if (statusCategory != 2) { - Option(response.getEntity).map(responseEntity => EntityUtils.toString(responseEntity)) match { + Option(response.getEntity) + .map(responseEntity => EntityUtils.toString(responseEntity, StandardCharsets.UTF_8)) match { case Some(responseEntityStr) => log.error(s"Triplestore responded with HTTP code $statusCode: $responseEntityStr") throw TriplestoreResponseException(s"Triplestore responded with HTTP code $statusCode: $responseEntityStr") @@ -1043,7 +1045,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat case None => "" case Some(responseEntity) => - EntityUtils.toString(responseEntity) + EntityUtils.toString(responseEntity, StandardCharsets.UTF_8) } } @@ -1055,7 +1057,7 @@ class HttpTriplestoreConnector extends Actor with ActorLogging with Instrumentat case Some(responseEntity: HttpEntity) => NamedGraphDataResponse( - turtle = EntityUtils.toString(responseEntity) + turtle = EntityUtils.toString(responseEntity, StandardCharsets.UTF_8) ) } } diff --git a/webapi/src/test/scala/org/knora/webapi/responders/v2/MetadataResponderV2Spec.scala b/webapi/src/test/scala/org/knora/webapi/responders/v2/MetadataResponderV2Spec.scala index bfcba11e63..d5e51d6c58 100644 --- a/webapi/src/test/scala/org/knora/webapi/responders/v2/MetadataResponderV2Spec.scala +++ b/webapi/src/test/scala/org/knora/webapi/responders/v2/MetadataResponderV2Spec.scala @@ -42,56 +42,129 @@ class MetadataResponderV2Spec extends CoreSpec() with ImplicitSender { private val timeout = 10.seconds private val metadataContent = - s""" - |@prefix dsp-repo: . - |@prefix knora-base: . - |@prefix knora-admin: . - |@prefix owl: . - |@prefix rdf: . - |@prefix rdfs: . - |@prefix xml: . - |@prefix xsd: . - |@prefix foaf: . - |@prefix vowl: . - |@prefix prov: . - |@prefix dc: . - |@prefix dct: . - |@prefix locn: . - |@prefix vcard: . - |@prefix schema: . - |@prefix skos: . - |@prefix unesco6: . - |@base . - | - | rdf:type dsp-repo:Project . - | dsp-repo:hasName "Bernoulli-Euler Online (BEOL)" . - | dsp-repo:hasDescription "The project Bernoulli-Euler Online (BEOL) integrates the two edition projects Basler Edition der Bernoulli-Briefwechsel (BEBB) and Leonhardi Euleri Opera Omnia (LEOO) into one digital platform available on the web. In addition, Jacob Bernoulli's scientific notebook Meditationes - a document of outstanding significance for the history of mathematics at its turning point around 1700 - is published for the first time in its entirety on the BEOL platform as a region-based multilayer interactive digital edition providing access to facsimiles, transcriptions, translations, indices, and commentaries. Besides being an edition platform, BEOL is a virtual research environment for the study of early modern mathematics and science as a data graph using sophisticated analysis tools. Currently BEOL is connected to two third-party repositories: The Newton Project and the Briefportal Leibniz, initiating the formation of a network of digital editions of the early modern scientific correspondence data.The goal of BEOL is thus twofold: it focuses on the mathematics influenced by the Bernoulli dynasty and Leonhard Euler and undertakes a methodological effort to present these materials to the public and researchers in a highly functional way." . - | dsp-repo:hasKeywords "mathematics" . - | dsp-repo:hasKeywords "science" . - | dsp-repo:hasKeywords "history of science" . - | dsp-repo:hasKeywords "history of mathematics" . - | dsp-repo:hasKeywords "Bernoulli" . - | dsp-repo:hasKeywords "Euler" . - | dsp-repo:hasKeywords "Newton" . - | dsp-repo:hasKeywords "Leibniz" . - | dsp-repo:hasCategories "mathematics" . - | dsp-repo:hasStartDate "2016.07" . - | dsp-repo:hasEndDate "2020.01" . - | dsp-repo:hasTemporalCoverage "17th century and 18th century CE" . - | dsp-repo:hasSpatialCoverage "Europe" . - | dsp-repo:hasSpatialCoverage "Russia" . - | dsp-repo:hasSpatialCoverage "France" . - | dsp-repo:hasSpatialCoverage "Switzerland" . - | dsp-repo:hasSpatialCoverage "Germany" . - | dsp-repo:hasSpatialCoverage "Italy" . - | dsp-repo:hasSpatialCoverage "England" . - | dsp-repo:hasFunder "Schweizerischer Nationalfonds (SNSF)" . - | dsp-repo:hasURL "https://beol.dasch.swiss/" . - | dsp-repo:hasDateCreated "09.2017" . - | dsp-repo:hasDateModified "04.2020" . - | dsp-repo:hasShortcode "0801" . - | dsp-repo:hasAlternateName "beol" . - |""".stripMargin + s"""@base . + |@prefix dsp-repo: . + |@prefix prov: . + |@prefix rdf: . + |@prefix schema: . + |@prefix xsd: . + | + |<082D-dataset-001> a ; + | "Smart technologies transform farming today. Think of autonomous tractors and weeding robots, animals and underground infrastructures with inbuilt sensors, and drones or satellites offering image analysis from the air. There are many opportunities arising from this evolution in terms of increased productivity, profitability, and sustainability. Critical issues, in turn, include techno-dependency, vulnerability and privacy. This project contributes empirical evidence to the discussion of the opportunities and risks associated with smart farming. It starts from the assumption that smart farming – i.e. the management of agricultural practices and processes through techniques of data gathering, transfer and analysis – is never neutral, but shaped by all kinds of decisions and judgements built into the systems from their very elaboration. The project thus studies ‘what lies behind’ novel smart-farming solutions, looking at where, by whom and how they are produced and subsequently diffused as exemplars to follow. It does so through the investigation of two high-profile initiatives in Switzerland: Swiss Future Farm: An experimental and demonstration farm, set up on Agroscope’s test site in Tänikon (TG) as a joint public–private initiative, to be inaugurated on 21 September 2018. AgroFly’s (now with Aero41) sprayer-drone project (Sierre, VS): The first authorized drone system in Europe for the automated application of pesticides. Test sites are distributed in Switzerland and beyond. This project claims that specific initiatives, anchored in specific sites of experimentation, play a fundamental role in assembling the actor networks that carry forward the evolutions around smart farming. Grounded in Actor Network Theory, the project unpacks the chain of mediations through which relevant actors, ideas and things connect and interact in the co-production of the studied initiatives, and investigates the relationships and mechanisms that tie them to each other and to other sites and initiatives. There are four interrelated research objectives that run through this investigation, relating to the collaborative, expertise-related, discursive, and spatial dimensions of learning and exemplification: to analyse the practices of collaboration through which novel smart-farming solutions are developed and stabilized for more normalized use through the studied sites of experimentation. to examine what and how specific forms of expertise become authorized to act in the different stages and chains of mediation associated with the studied sites. to investigate what and how specific discourses, expectations and beliefs surrounding smart technologies crystallize in and emerge from the studied initiatives. to explore the forces and mechanisms through which the studied sites are tied to other places and projects in the circuits of learning and exemplification that shape the field of smart farming. To pursue these objectives, the project follows three methodological pathways: textual analysis of literature and reports, non-participant observation, and semi-structured interviews. This approach is exploratory in ambition and scope and leads to further, follow-up research proposals on Big Data in agriculture. On these grounds, the project studies, informs and debates innovation in the agricultural sector. It highlights the factors that favour or hinder technology innovation, diffusion and adaptation, and fosters greater understanding about the desirability or non-desirability of this. User beneficiaries include farmers and farming associations, technology "^^xsd:string ; + | "Restricted"^^xsd:string ; + | "Klauser FR and Pauschinger D (2020) Big Data in Agriculture: The Making of Smart Farms, SNF Project Data Set."^^xsd:string ; + | "English"^^xsd:string, + | "French"^^xsd:string, + | "German"^^xsd:string ; + | [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "creativecommons.org" ] ; + | schema:url "https://creativecommons.org/licenses/" ] ; + | [ a prov:Attribution ; + | "Senior Research Fellow"^^xsd:string ; + | prov:agent <082D-person-002> ], + | [ a prov:Attribution ; + | "PI"^^xsd:string ; + | prov:agent <082D-person-001> ] ; + | "Finished"^^xsd:string ; + | "Dataset of Big Data in Agriculture: The Making of Smart Farms"^^xsd:string ; + | "Text"^^xsd:string ; + | <082D-project> . + | + | a ; + | true . + | + |<082D-grant-001> a ; + | <082D-organization-001> ; + | "Digital Lives"^^xsd:string ; + | "FN 10DL1A_183037"^^xsd:string ; + | [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "snf.ch" ] ; + | schema:url "http://www.snf.ch/de/foerderung/projekte/digital-lives/Seiten/default.aspx" ] . + | + |<082D-person-002> a ; + | [ a schema:PostalAddress ; + | schema:addressLocality "Neuchâtel"^^xsd:string ; + | schema:postalCode "2000"^^xsd:string ; + | schema:streetAddress "Espace Tilo-Frey 1"^^xsd:string ] ; + | "dennis.pauschinger@unine.ch" ; + | "Pauschinger"^^xsd:string ; + | ( "Dennis"^^xsd:string ) ; + | "Postdoc "^^xsd:string ; + | <082D-organization-002> . + | + |<082D-project> a ; + | <082D-person-001> ; + | ; + | \"\"\"Smart technologies transform farming today. Think of autonomous tractors and weeding robots, animals and underground infrastructures with inbuilt sensors, and drones or satellites offering image analysis from the air. There are many opportunities arising from this evolution in terms of increased productivity, profitability, and sustainability. Critical issues, in turn, include techno-dependency, vulnerability and privacy. + |This project contributes empirical evidence to the discussion of the opportunities and risks associated with smart farming. It starts from the assumption that smart farming – i.e. the management of agricultural practices and processes through techniques of data gathering, transfer and analysis – is never neutral, but shaped by all kinds of decisions and judgements built into the systems from their very elaboration. The project thus studies ‘what lies behind’ novel smart-farming solutions, looking at where, by whom and how they are produced and subsequently diffused as exemplars to follow. It does so through the investigation of two high-profile initiatives in Switzerland: + |Swiss Future Farm: An experimental and demonstration farm, set up on Agroscope’s test site in Tänikon (TG) as a joint public–private initiative, to be inaugurated on 21 September 2018. + |AgroFly’s (now with Aero41) sprayer-drone project (Sierre, VS): The first authorized drone system in Europe for the automated application of pesticides. Test sites are distributed in Switzerland and beyond. + |This project claims that specific initiatives, anchored in specific sites of experimentation, play a fundamental role in assembling the actor networks that carry forward the evolutions around smart farming. Grounded in Actor Network Theory, the project unpacks the chain of mediations through which relevant actors, ideas and things connect and interact in the co-production of the studied initiatives, and investigates the relationships and mechanisms that tie them to each other and to other sites and initiatives. There are four interrelated research objectives that run through this investigation, relating to the collaborative, expertise-related, discursive, and spatial dimensions of learning and exemplification: + |to analyse the practices of collaboration through which novel smart-farming solutions are developed and stabilized for more normalized use through the studied sites of experimentation. + |to examine what and how specific forms of expertise become authorized to act in the different stages and chains of mediation associated with the studied sites. + |to investigate what and how specific discourses, expectations and beliefs surrounding smart technologies crystallize in and emerge from the studied initiatives. + |to explore the forces and mechanisms through which the studied sites are tied to other places and projects in the circuits of learning and exemplification that shape the field of smart farming. + |To pursue these objectives, the project follows three methodological pathways: textual analysis of literature and reports, non-participant observation, and semi-structured interviews. This approach is exploratory in ambition and scope and leads to further, follow-up research proposals on Big Data in agriculture. On these grounds, the project studies, informs and debates innovation in the agricultural sector. It highlights the factors that favour or hinder technology innovation, diffusion and adaptation, and fosters greater understanding about the desirability or non-desirability of this. User beneficiaries include farmers and farming associations, technology \"\"\"^^xsd:string ; + | [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "SKOS UNESCO Nomenclature" ] ; + | schema:url "http://skos.um.es/unesco6/540305" ] ; + | "2020-06-30"^^xsd:date ; + | <082D-organization-001> ; + | <082D-grant-001> ; + | "Agriculture"^^xsd:string ; + | "Big Data in Agriculture: The Making of Smart Famrs"^^xsd:string ; + | "Klauser, FR.: Surveillance Farms: Towards a Research Agenda on Big Data in Agriculture, Surveillance & Society, 16(3), 370-378, https://doi.org/10.24908/ss.v16i3.12594, 2020. "^^xsd:string, + | "Pauschinger, D. and Klauser, F.: Räume des Experimentierens: Die Einführung von Sprühdrohnen in der digitalen Landwirtschaft, Geogr. Helv., 75, 325–336, https://doi.org/10.5194/gh-75-325-2020, 2020. "^^xsd:string ; + | "082D"^^xsd:string ; + | [ a schema:Place ; + | schema:url [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "Geonames" ; + | schema:url "https://www.geonames.org/2658434/switzerland.html" ] ] ] ; + | "2018-12-01"^^xsd:date ; + | [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "dainst.org" ] ; + | schema:url "https://chronontology.dainst.org/period/8bGPuf5syqCD" ] ; + | [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "dasch.swiss" ] ; + | schema:url "https://test.dasch.swiss" ] . + | + |<082D-organization-001> a ; + | "Swiss National Science Foundation"^^xsd:string . + | + |<082D-organization-002> a ; + | [ a schema:PostalAddress ; + | schema:addressLocality "Neuchâtel"^^xsd:string ; + | schema:postalCode "2000"^^xsd:string ; + | schema:streetAddress "Espace Tilo-Frey 1"^^xsd:string ] ; + | "Secretariat.Geographie@unine.ch " ; + | "Université de Neuchâtel, Faculté des lettres et sciences humaines"^^xsd:string ; + | [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "unine.ch" ] ; + | schema:url "http://www.unine.ch/geographie/home.html" ] . + | + |<082D-person-001> a ; + | [ a schema:PostalAddress ; + | schema:addressLocality "Neuchâtel"^^xsd:string ; + | schema:postalCode "2000"^^xsd:string ; + | schema:streetAddress "Espace Tilo-Frey 1"^^xsd:string ] ; + | "francisco.klauser@unine.ch" ; + | "Klauser"^^xsd:string ; + | ( "Francisco R."^^xsd:string ) ; + | "Full professor of political geography"^^xsd:string ; + | <082D-organization-002> ; + | [ a schema:URL ; + | schema:propertyID [ a schema:PropertyValue ; + | schema:propertyID "orcid.org" ] ; + | schema:url "https://orcid.org/0000-0003-3383-3570." ] . + | + |""".stripMargin // Parse the request to an RdfModel. private val requestModel: RdfModel = rdfFormatUtil.parseToRdfModel(