DSP-756 Tests failing because Knora version header and route are inco…

…rrect (#1748)
dasch-swiss · Nov 3, 2020 · 060b627 · 060b627
1 parent 37decd9
commit 060b627
Show file tree

Hide file tree

Showing 29 changed files with 308 additions and 90 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -60,7 +60,7 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@v1
         with:
-          fetch-depth: 1
+          fetch-depth: 50
       - uses: joschi/setup-jdk@v2
         with:
           java-version: '11' # The OpenJDK version to make available on the path

diff --git a/Makefile b/Makefile
@@ -231,6 +231,7 @@ test-webapi: docker-build ## runs all dsp-api tests.
 .PHONY: test-unit
 test-unit: docker-build ## runs the dsp-api unit tests.
 	bazel test \
+	//webapi/src/test/scala/org/knora/webapi/contributors/... \
 	//webapi/src/test/scala/org/knora/webapi/http/... \
 	//webapi/src/test/scala/org/knora/webapi/messages/... \
 	//webapi/src/test/scala/org/knora/webapi/other/... \

diff --git a/WORKSPACE b/WORKSPACE
@@ -261,3 +261,13 @@ http_archive(
 # load further dependencies of this rule
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 rules_pkg_dependencies()
+
+#
+# download rules_stamp - stamping helper
+#
+http_archive(
+    name = "ecosia_rules_stamp",
+    url = "https://github.com/ecosia/rules_stamp/archive/48d5ef2bc0d93bd65fddddbe02f3ae410e25169d.tar.gz",
+    strip_prefix = "rules_stamp-48d5ef2bc0d93bd65fddddbe02f3ae410e25169d",
+    sha256 = "36d7ea381bfb2520f9353299b162434b25c77365d3c9e9459195c536da5e837d",
+)
diff --git a/docs/05-internals/development/bazel.md b/docs/05-internals/development/bazel.md
@@ -71,3 +71,46 @@ section in the Bazel documentation.
     ```bash
     $ bazel run //webapi:main_library_repl
     ```
+
+## Build stamping
+
+By default, Bazel tries not to include anything about the system state in build
+outputs. However, released binaries and libraries often want to include
+something like the version they were built at or the branch or tag they came
+from.
+
+To reconcile this, Bazel has an option called the *workspace status command*.
+This command is run outside of any sandboxes on the local machine, so it can
+access anything about your source control, OS, or anything else you might want
+to include. It then dumps its output into `bazel-out/volatile-status.txt`, which
+you can use (and certain language rulesets provide support for accessing from code).
+
+Our *workspace status command* is defined in `//tools/buildstamp/get_workspace_status`.
+To use it on every bazel command, we need to supply it to each Bazel invocation,
+which is done by the following line found in `.bazelrc`:
+```
+build --workspace_status_command=tools/buildstamp/get_workspace_status --stamp=yes
+```
+
+Any line added to `.bazelrc` is invoked on each corresponding command.
+
+The `//tools/buildstamp/get_workspace_status` emits additional values
+to `bazel-out/volatile-status.txt` whereas `BUILD_TIMESTAMP` is emitted by
+Bazel itself:
+
+```
+BUILD_SCM_REVISION 2d6df6c8fe2d56e3712eb26763f9727916a60164
+BUILD_SCM_STATUS Modified
+BUILD_SCM_TAG v13.0.0-rc.21-17-g2d6df6c-dirty
+BUILD_TIMESTAMP 1604401028
+```
+
+The value of `BUILD_SCM_TAG` is used in `//webapi/src/main/scala/org/knora/webapi/http/version/versioninfo`,
+which emits a JAR containing `VersionInfo.scala`. This file is generated based on
+`VersionInfoTemplate.scala` found in the same Bazel package.
+
+In short, the `versioninfo` target producing the JAR library depends on
+the `version_info_with_build_tag` target which emits the `VersionInfo.scala`
+file which has the `{BUILD_TAG}` variable replaced by the current value of
+`BUILD_SCM_TAG`. In an intermediary step, the `version_info_without_build_tag`
+target, replaces variables coming from `//third_party:versions.bzl`.
diff --git a/third_party/dependencies.bzl b/third_party/dependencies.bzl
@@ -149,13 +149,13 @@ def dependencies():
     )
 
 ALL_WEBAPI_MAIN_DEPENDENCIES = [
-    "//tools/version_info",
     "//webapi/src/main/scala/org/knora/webapi",
     "//webapi/src/main/scala/org/knora/webapi/app",
     "//webapi/src/main/scala/org/knora/webapi/core",
     "//webapi/src/main/scala/org/knora/webapi/exceptions",
     "//webapi/src/main/scala/org/knora/webapi/http/handler",
     "//webapi/src/main/scala/org/knora/webapi/http/version",
+    "//webapi/src/main/scala/org/knora/webapi/http/version/versioninfo",
     "//webapi/src/main/scala/org/knora/webapi/instrumentation",
     "//webapi/src/main/scala/org/knora/webapi/messages",
     "//webapi/src/main/scala/org/knora/webapi/responders",

diff --git a/tools/version_info/add_build_tag.sh b/tools/version_info/add_build_tag.sh
diff --git a/webapi/BUILD.bazel b/webapi/BUILD.bazel
@@ -39,12 +39,12 @@ scala_library(
         "@maven//:org_slf4j_log4j_over_slf4j",
     ],
     deps = [
-        "//tools/version_info",
         "//webapi/src/main/scala/org/knora/webapi",
         "//webapi/src/main/scala/org/knora/webapi/core",
         "//webapi/src/main/scala/org/knora/webapi/exceptions",
         "//webapi/src/main/scala/org/knora/webapi/http/handler",
         "//webapi/src/main/scala/org/knora/webapi/http/version",
+        "//webapi/src/main/scala/org/knora/webapi/http/version/versioninfo",
         "//webapi/src/main/scala/org/knora/webapi/instrumentation",
         "//webapi/src/main/scala/org/knora/webapi/messages",
         "//webapi/src/main/scala/org/knora/webapi/responders",

diff --git a/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala b/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala
@@ -1,3 +1,22 @@
+/*
+ * Copyright © 2015-2020 the contributors (see Contributors.md).
+ *
+ *  This file is part of the DaSCH Service Platform.
+ *
+ *  Knora is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  Knora is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public
+ *  License along with Knora.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 package org.knora.webapi.app
 
 import akka.actor.SupervisorStrategy._
@@ -14,7 +33,7 @@ import kamon.Kamon
 import org.knora.webapi.core.LiveActorMaker
 import org.knora.webapi.exceptions.{InconsistentTriplestoreDataException, SipiException, UnexpectedMessageException, UnsupportedValueException}
 import org.knora.webapi.http.handler
-import org.knora.webapi.http.version.ServerVersion.addServerHeader
+import org.knora.webapi.http.version.ServerVersion
 import org.knora.webapi.messages.admin.responder.KnoraRequestADM
 import org.knora.webapi.messages.app.appmessages._
 import org.knora.webapi.messages.store.StoreRequest
@@ -395,7 +414,7 @@ class ApplicationActor extends Actor with Stash with LazyLogging with AroundDire
      * The FIRST matching route is used for handling a request.
      */
     private val apiRoutes: Route = logDuration {
-        addServerHeader {
+        ServerVersion.addServerHeader {
             handleErrors {
                 CorsDirectives.cors(CorsSettings(system)) {
                     handleErrors {

diff --git a/webapi/src/main/scala/org/knora/webapi/app/BUILD.bazel b/webapi/src/main/scala/org/knora/webapi/app/BUILD.bazel
@@ -18,7 +18,6 @@ scala_library(
         "//webapi/src/main/scala/org/knora/webapi/routing",
         "//webapi/src/main/scala/org/knora/webapi/settings",
         "//webapi/src/main/scala/org/knora/webapi/store",
-        "//webapi/src/main/scala/org/knora/webapi/util",
         "//webapi/src/main/scala/org/knora/webapi/util/cache",
         "@maven//:ch_megard_akka_http_cors_2_12",
         "@maven//:com_github_swagger_akka_http_swagger_akka_http_2_12",

diff --git a/webapi/src/main/scala/org/knora/webapi/app/Main.scala b/webapi/src/main/scala/org/knora/webapi/app/Main.scala
@@ -1,7 +1,7 @@
 /*
- * Copyright © 2015-2018 the contributors (see Contributors.md).
+ * Copyright © 2015-2020 the contributors (see Contributors.md).
  *
- *  This file is part of Knora.
+ *  This file is part of the DaSCH Service Platform.
  *
  *  Knora is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU Affero General Public License as published

diff --git a/webapi/src/main/scala/org/knora/webapi/http/version/BUILD.bazel b/webapi/src/main/scala/org/knora/webapi/http/version/BUILD.bazel
@@ -7,7 +7,7 @@ scala_library(
     srcs = glob(["*.scala"]),
     unused_dependency_checker_mode = "warn",
     deps = [
-        "//tools/version_info",
+        "//webapi/src/main/scala/org/knora/webapi/http/version/versioninfo",
         "@maven//:com_typesafe_akka_akka_actor_2_12",
         "@maven//:com_typesafe_akka_akka_http_2_12",
         "@maven//:com_typesafe_akka_akka_http_core_2_12",

diff --git a/webapi/src/main/scala/org/knora/webapi/http/version/ServerVersion.scala b/webapi/src/main/scala/org/knora/webapi/http/version/ServerVersion.scala
@@ -1,7 +1,7 @@
 /*
- * Copyright © 2015-2018 the contributors (see Contributors.md).
+ * Copyright © 2015-2020 the contributors (see Contributors.md).
  *
- *  This file is part of Knora.
+ *  This file is part of the DaSCH Service Platform.
  *
  *  Knora is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU Affero General Public License as published
@@ -22,19 +22,21 @@ package org.knora.webapi.http.version
 import akka.http.scaladsl.model.headers.Server
 import akka.http.scaladsl.server.Directives.respondWithHeader
 import akka.http.scaladsl.server.Route
+import org.knora.webapi.http.version.versioninfo.VersionInfo
 
 /**
  * This object provides methods that can be used to add the [[Server]] header
  * to an [[akka.http.scaladsl.model.HttpResponse]].
  */
 object ServerVersion {
 
-    private val ServerVersionString = s"${VersionInfo.name}/${VersionInfo.version}" + " " + s"akka-http/${VersionInfo.akkaHttpVersion}"
-    private val ServerVersionHeader = `Server`(ServerVersionString)
+    private val ApiNameAndVersion = s"${VersionInfo.name}/${VersionInfo.webapiVersion}"
+    private val AkkaNameAndVersion = s"akka-http/${VersionInfo.akkaHttpVersion}"
+    private val AllProducts = ApiNameAndVersion + " " + AkkaNameAndVersion
 
-    def addServerHeader(route: Route): Route = respondWithHeader(ServerVersionHeader) {
+    def serverVersionHeader(): Server = Server(products = AllProducts)
+
+    def addServerHeader(route: Route): Route = respondWithHeader(serverVersionHeader) {
         route
     }
-
-    def getServerVersionHeader(): Server = ServerVersionHeader
 }
diff --git a/tools/version_info/BUILD.bazel → ...bapi/http/version/versioninfo/BUILD.bazel b/tools/version_info/BUILD.bazel → ...bapi/http/version/versioninfo/BUILD.bazel
@@ -3,16 +3,25 @@ package(default_visibility = ["//visibility:public"])
 # bring our version constants in
 load("//third_party:versions.bzl", "SCALA_VERSION", "AKKA_VERSION", "AKKA_HTTP_VERSION", "SIPI_VERSION", "FUSEKI_VERSION")
 load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+load("@ecosia_rules_stamp//:defs.bzl", "stamp")
 
 filegroup(
     name = "srcs",
     srcs = glob(["**"]),
 )
 
+# the stamp rule extracts the stamped values (from //tools/buildstamp)
+# into a separate file (see 'bazel-out/darwin-fastbuild/bin/tools/version_info/STAMP_BUILD_SCM_TAG'),
+# onto which other rules can then depend
+stamp(
+    name = "build_scm_tag_stamp",
+    stamp_keys = ["BUILD_SCM_TAG"],
+)
+
 # Generates a scala source file that includes version information
 # The generated intermediate file is missing the BUILD_TAG, which is
-# add by the version_info genrule
-load("//tools/version_info:gen_version_info.bzl", "gen_version_info")
+# added by the 'version_info_with_build_tag' genrule
+load("//webapi/src/main/scala/org/knora/webapi/http/version/versioninfo:gen_version_info.bzl", "gen_version_info")
 gen_version_info(
     name = "version_info_without_build_tag",
     scala_version = SCALA_VERSION,
@@ -25,14 +34,16 @@ gen_version_info(
 # Adds the missing build tag and emits the final scala object
 genrule(
     name = "version_info_with_build_tag",
-    srcs = [":version_info_without_build_tag"],
+    srcs = [
+        ":build_scm_tag_stamp",
+        ":version_info_without_build_tag"
+    ],
     outs = ["version_info.scala"],
-    local = 1,
     tools = [":add_build_tag.sh"],
-    cmd = '$(location add_build_tag.sh) bazel-out/volatile-status.txt "$<" > "$@"',
+    cmd = '$(location add_build_tag.sh) $(location build_scm_tag_stamp) $(location version_info_without_build_tag) > "$@"',
 )
 
 scala_library(
-    name = "version_info",
+    name = "versioninfo",
     srcs = [":version_info_with_build_tag"],
 )
diff --git a/tools/version_info/VersionInfoTemplate.scala → ...ion/versioninfo/VersionInfoTemplate.scala b/tools/version_info/VersionInfoTemplate.scala → ...ion/versioninfo/VersionInfoTemplate.scala
@@ -1,7 +1,7 @@
 /*
- * Copyright © 2015-2018 the contributors (see Contributors.md).
+ * Copyright © 2015-2020 the contributors (see Contributors.md).
  *
- *  This file is part of Knora.
+ *  This file is part of the DaSCH Service Platform.
  *
  *  Knora is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU Affero General Public License as published
@@ -17,7 +17,7 @@
  *  License along with Knora.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package org.knora.webapi.http.version
+package org.knora.webapi.http.version.versioninfo
 
 import scala.Predef._
 
@@ -26,7 +26,7 @@ case object VersionInfo {
 
     val name: String = "webapi"
 
-    val version: String = "{BUILD_TAG}"
+    val webapiVersion: String = "{BUILD_TAG}"
 
     val scalaVersion: String = "{SCALA_VERSION}"
 
@@ -38,7 +38,7 @@ case object VersionInfo {
 
     override val toString: String = {
         "name: %s, version: %s, scalaVersion: %s, akkaHttpVersion: %s, sipiVersion: %s, jenaFusekiVersion: %s".format(
-            name, version, scalaVersion, akkaHttpVersion, sipiVersion, jenaFusekiVersion
+            name, webapiVersion, scalaVersion, akkaHttpVersion, sipiVersion, jenaFusekiVersion
         )
     }
-}
+}
diff --git a/webapi/src/main/scala/org/knora/webapi/http/version/versioninfo/add_build_tag.sh b/webapi/src/main/scala/org/knora/webapi/http/version/versioninfo/add_build_tag.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# get the tag from volatile-status.txt
+EXTRACTED_TAG=$(cat "$1")
+
+# exchange the placeholder 'BUILD_TAG' with the 'EXTRACTED_TAG' extracted in the previous step
+sed "s/{BUILD_TAG}/$EXTRACTED_TAG/" "$2"
diff --git a/tools/version_info/gen_version_info.bzl → .../version/versioninfo/gen_version_info.bzl b/tools/version_info/gen_version_info.bzl → .../version/versioninfo/gen_version_info.bzl
@@ -4,7 +4,7 @@ content during the analysis phase.
 """
 
 # Label of the template file to use.
-_TEMPLATE = "//tools/version_info:VersionInfoTemplate.scala"
+_TEMPLATE = "//webapi/src/main/scala/org/knora/webapi/http/version/versioninfo:VersionInfoTemplate.scala"
 
 def _gen_version_info_impl(ctx):
     ctx.actions.expand_template(

diff --git a/webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel b/webapi/src/main/scala/org/knora/webapi/messages/BUILD.bazel
@@ -13,26 +13,25 @@ scala_library(
         "//webapi/src/main/scala/org/knora/webapi/settings",
         "//webapi/src/main/scala/org/knora/webapi/util",
         "//webapi/src/main/scala/org/knora/webapi/util/cache",
+        "@maven//:com_apicatalog_titanium_json_ld",
         "@maven//:com_google_gwt_gwt_servlet",
         "@maven//:com_ibm_icu_icu4j",
         "@maven//:com_sksamuel_diff_diff",
         "@maven//:com_typesafe_akka_akka_actor_2_12",
         "@maven//:com_typesafe_akka_akka_http_2_12",
         "@maven//:com_typesafe_akka_akka_http_core_2_12",
         "@maven//:com_typesafe_akka_akka_http_spray_json_2_12",
-        "@maven//:com_apicatalog_titanium_json_ld",
-        "@maven//:javax_json_javax_json_api",
-        "@maven//:org_glassfish_jakarta_json",
         "@maven//:com_typesafe_akka_akka_stream_2_12",
         "@maven//:com_typesafe_play_twirl_api_2_12",
         "@maven//:com_typesafe_scala_logging_scala_logging_2_12",
-        "@maven//:org_apache_jena_apache_jena_libs",
         "@maven//:commons_io_commons_io",
         "@maven//:commons_validator_commons_validator",
         "@maven//:io_spray_spray_json_2_12",
+        "@maven//:javax_json_javax_json_api",
         "@maven//:net_sf_saxon_Saxon_HE",
         "@maven//:org_apache_commons_commons_lang3",
         "@maven//:org_apache_commons_commons_text",
+        "@maven//:org_apache_jena_apache_jena_libs",
         "@maven//:org_eclipse_rdf4j_rdf4j_client",
         "@maven//:org_jodd_jodd",
         "@maven//:org_scala_lang_modules_scala_java8_compat_2_12",

diff --git a/webapi/src/main/scala/org/knora/webapi/routing/BUILD.bazel b/webapi/src/main/scala/org/knora/webapi/routing/BUILD.bazel
@@ -7,11 +7,11 @@ scala_library(
     srcs = glob(["**/*.scala"]),
     unused_dependency_checker_mode = "warn",
     deps = [
-        "//tools/version_info",
         "//webapi/src/main/scala/org/knora/webapi",
         "//webapi/src/main/scala/org/knora/webapi/annotation",
         "//webapi/src/main/scala/org/knora/webapi/exceptions",
         "//webapi/src/main/scala/org/knora/webapi/http/status",
+        "//webapi/src/main/scala/org/knora/webapi/http/version/versioninfo",
         "//webapi/src/main/scala/org/knora/webapi/instrumentation",
         "//webapi/src/main/scala/org/knora/webapi/messages",
         "//webapi/src/main/scala/org/knora/webapi/settings",
@@ -30,14 +30,13 @@ scala_library(
         "@maven//:com_typesafe_config",
         "@maven//:com_typesafe_play_twirl_api_2_12",
         "@maven//:com_typesafe_scala_logging_scala_logging_2_12",
-        "@maven//:org_apache_jena_apache_jena_libs",
         "@maven//:commons_validator_commons_validator",
         "@maven//:io_spray_spray_json_2_12",
         "@maven//:io_swagger_swagger_annotations",
         "@maven//:io_swagger_swagger_jaxrs",
         "@maven//:io_swagger_swagger_models",
         "@maven//:javax_ws_rs_jsr311_api",
-        "@maven//:org_eclipse_rdf4j_rdf4j_client",
+        "@maven//:org_apache_jena_apache_jena_libs",
         "@maven//:org_scala_lang_modules_scala_xml_2_12",
         "@maven//:org_slf4j_slf4j_api",
     ],

diff --git a/webapi/src/main/scala/org/knora/webapi/routing/VersionRoute.scala b/webapi/src/main/scala/org/knora/webapi/routing/VersionRoute.scala
@@ -23,7 +23,7 @@ import akka.http.scaladsl.model._
 import akka.http.scaladsl.server.Directives.{get, path}
 import akka.http.scaladsl.server.Route
 import akka.util.Timeout
-import org.knora.webapi.http.version.VersionInfo
+import org.knora.webapi.http.version.versioninfo.VersionInfo
 import spray.json.{JsObject, JsString}
 
 import scala.concurrent.duration._
@@ -77,7 +77,7 @@ trait VersionCheck {
 
         VersionCheckResult(
             name = "version",
-            webapi = VersionInfo.version,
+            webapi = VersionInfo.webapiVersion,
             scala = VersionInfo.scalaVersion,
             akkaHttp = VersionInfo.akkaHttpVersion,
             sipi = sipiVersion,