fix (sipi): bump sipi and add missing config (memory leak) (DEV-237) (#…

…1959) Co-authored-by: irinaschubert <irina.schubert@dasch.swiss>
dasch-swiss · Feb 25, 2022 · 9f6df4c · 9f6df4c
1 parent 7a186ba
commit 9f6df4c
Show file tree

Hide file tree

Showing 9 changed files with 187 additions and 41 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,4 +1,47 @@
-===REMOVE===
-Important! Please follow the new guidelines for naming Pull Requests: https://docs.dasch.swiss/developers/dsp/contribution/#pull-request-guidelines
-===REMOVE===
-resolves DEV-
+<!-- Important! Please follow the new guidelines for naming Pull Requests: https://docs.dasch.swiss/developers/dsp/contribution/#pull-request-guidelines-->
+
+Resolves DEV-
+
+## PR Checklist
+
+Please check if your PR fulfills the following requirements:
+
+- [ ] Tests for the changes have been added (for bug fixes / features)
+- [ ] Docs have been added / updated (for bug fixes / features)
+
+
+## PR Type
+
+What kind of change does this PR introduce?
+
+<!-- Please check the one that applies to this PR using "x". -->
+
+- [ ] Bugfix
+- [ ] Feature
+- [ ] Code style update (formatting, local variables)
+- [ ] Refactoring (no functional changes, no api changes)
+- [ ] Build related changes
+- [ ] CI related changes
+- [ ] Documentation content changes
+- [ ] Other... Please describe:
+
+
+## What is the current behavior?
+<!-- Please describe the current behavior that you are modifying, or link to a relevant issue. -->
+
+Issue Number: N/A
+
+
+## What is the new behavior?
+
+
+## Does this PR introduce a breaking change?
+
+- [ ] Yes
+- [ ] No
+
+
+<!-- If this PR contains a breaking change, please describe the impact and migration path for existing applications below. -->
+
+
+## Other information
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,6 +1,9 @@
 name: CI
 
 on:
+  schedule:
+    # run every midnight (on main) so that the first compile of the day from main is cached
+    - cron: '0 0 * * *'
   push:
   release:
     types: [published]
@@ -35,6 +38,11 @@ jobs:
         with:
           java-version: '11' # The OpenJDK version to make available on the path
           architecture: 'x64' # defaults to 'x64'
+      - name: add docker compose v2
+        run: |
+          mkdir -p ~/.docker/cli-plugins/
+          curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
       - name: install requirements
         run: sudo apt-get install ca-certificates-java expect
       - name: Coursier cache
@@ -77,6 +85,11 @@ jobs:
         with:
           java-version: '11' # The OpenJDK version to make available on the path
           architecture: 'x64' # defaults to 'x64'
+      - name: add docker compose v2
+        run: |
+          mkdir -p ~/.docker/cli-plugins/
+          curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
       - name: install requirements
         run: sudo apt-get install ca-certificates-java expect unzip
       - name: Coursier cache
@@ -94,7 +107,6 @@ jobs:
 
 # ------------------------------------------
 # ------------------------------------------
-
   upgrade-integration-tests:
     name: Upgrade Integration Tests
     runs-on: ubuntu-latest
@@ -119,11 +131,31 @@ jobs:
         with:
           java-version: '11' # The OpenJDK version to make available on the path
           architecture: 'x64' # defaults to 'x64'
+      - name: add docker compose v2
+        run: |
+          mkdir -p ~/.docker/cli-plugins/
+          curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
       - name: Install dependencies
         run: |
           sudo apt-get install ca-certificates-java expect
+      - name: Coursier cache
+        uses: coursier/cache-action@v6
       - name: test repository upgrade
         run: make test-repository-upgrade
+      - name: dump docker logs on failure using different shell
+        # if: failure()
+        uses: jwalton/gh-docker-logs@v1
+        with:
+          shell: '/bin/sh'
+      - name: Cleanup before cache
+        shell: bash
+        run: |
+          rm -rf "$HOME/.ivy2/local" || true
+          find $HOME/Library/Caches/Coursier/v1        -name "ivydata-*.properties" -delete || true
+          find $HOME/.ivy2/cache                       -name "ivydata-*.properties" -delete || true
+          find $HOME/.cache/coursier/v1                -name "ivydata-*.properties" -delete || true
+          find $HOME/.sbt                              -name "*.lock"               -delete || true
 
   docs-build-test:
     name: Build Docs Testrun
@@ -133,6 +165,11 @@ jobs:
         uses: actions/checkout@v1
         with:
           fetch-depth: 1
+      - name: add docker compose v2
+        run: |
+          mkdir -p ~/.docker/cli-plugins/
+          curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -194,6 +231,11 @@ jobs:
         with:
           java-version: '11' # The OpenJDK version to make available on the path
           architecture: 'x64' # defaults to 'x64'
+      - name: add docker compose v2
+        run: |
+          mkdir -p ~/.docker/cli-plugins/
+          curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
       - name: install requirements
         run: sudo apt-get install ca-certificates-java expect unzip
       - name: Install dependencies
@@ -241,6 +283,11 @@ jobs:
         uses: actions/checkout@v1
         with:
           fetch-depth: 1
+      - name: add docker compose v2
+        run: |
+          mkdir -p ~/.docker/cli-plugins/
+          curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
+          chmod +x ~/.docker/cli-plugins/docker-compose
       - name: install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/Makefile b/Makefile
@@ -102,9 +102,9 @@ env-file: ## write the env file used by knora-stack.
 
 .PHONY: stack-up
 stack-up: docker-build env-file ## starts the knora-stack: fuseki, sipi, redis, api.
-	docker-compose -f docker-compose.yml up -d db
+	@docker compose -f docker-compose.yml up -d db
 	$(CURRENT_DIR)/webapi/scripts/wait-for-db.sh
-	docker-compose -f docker-compose.yml up -d
+	@docker compose -f docker-compose.yml up -d
 	$(CURRENT_DIR)/webapi/scripts/wait-for-knora.sh
 
 .PHONY: stack-up-fast
@@ -118,7 +118,7 @@ stack-up-ci: docker-build env-file print-env-file ## starts the knora-stack usin
 
 .PHONY: stack-restart
 stack-restart: stack-up ## re-starts the knora-stack: fuseki, sipi, redis, api.
-	@docker-compose -f docker-compose.yml restart
+	@docker compose -f docker-compose.yml restart
 
 .PHONY: stack-restart-api
 stack-restart-api: ## re-starts the api. Usually used after loading data into fuseki.
@@ -127,69 +127,69 @@ stack-restart-api: ## re-starts the api. Usually used after loading data into fu
 
 .PHONY: stack-logs
 stack-logs: ## prints out and follows the logs of the running knora-stack.
-	@docker-compose -f docker-compose.yml logs -f
+	@docker compose -f docker-compose.yml logs -f
 
 .PHONY: stack-logs-db
 stack-logs-db: ## prints out and follows the logs of the 'db' container running in knora-stack.
-	@docker-compose -f docker-compose.yml logs -f db
+	@docker compose -f docker-compose.yml logs -f db
 
 .PHONY: stack-logs-db-no-follow
 stack-logs-db-no-follow: ## prints out the logs of the 'db' container running in knora-stack.
-	docker-compose -f docker-compose.yml logs db
+	@docker-compose -f docker-compose.yml logs db
 
 .PHONY: stack-logs-sipi
 stack-logs-sipi: ## prints out and follows the logs of the 'sipi' container running in knora-stack.
-	@docker-compose -f docker-compose.yml logs -f sipi
+	@docker compose -f docker-compose.yml logs -f sipi
 
 .PHONY: stack-logs-sipi-no-follow
 stack-logs-sipi-no-follow: ## prints out the logs of the 'sipi' container running in knora-stack.
-	@docker-compose -f docker-compose.yml logs sipi
+	@docker compose -f docker-compose.yml logs sipi
 
 .PHONY: stack-logs-redis
 stack-logs-redis: ## prints out and follows the logs of the 'redis' container running in knora-stack.
-	@docker-compose -f docker-compose.yml logs -f redis
+	@docker compose -f docker-compose.yml logs -f redis
 
 .PHONY: stack-logs-api
 stack-logs-api: ## prints out and follows the logs of the 'api' container running in knora-stack.
-	@docker-compose -f docker-compose.yml logs -f api
+	@docker compose -f docker-compose.yml logs -f api
 
 .PHONY: stack-logs-api-no-follow
 stack-logs-api-no-follow: ## prints out the logs of the 'api' container running in knora-stack.
-	docker-compose -f docker-compose.yml logs api
+	@docker compose -f docker-compose.yml logs api
 
 .PHONY: stack-health
 stack-health:
 	curl -f 0.0.0.0:3333/health
 
 .PHONY: stack-status
 stack-status:
-	docker-compose -f docker-compose.yml ps
+	@docker compose -f docker-compose.yml ps
 
 .PHONY: stack-down
 stack-down: ## stops the knora-stack.
-	docker-compose -f docker-compose.yml down
+	@docker compose -f docker-compose.yml down
 
 .PHONY: stack-down-delete-volumes
 stack-down-delete-volumes: ## stops the knora-stack and deletes any created volumes (deletes the database!).
-	docker-compose -f docker-compose.yml down --volumes
+	@docker compose -f docker-compose.yml down --volumes
 
 .PHONY: stack-config
 stack-config: env-file
-	docker-compose -f docker-compose.yml config
+	@docker compose -f docker-compose.yml config
 
 ## stack without api
 .PHONY: stack-without-api
 stack-without-api: stack-up ## starts the knora-stack without knora-api: fuseki, sipi, redis.
-	@docker-compose -f docker-compose.yml stop api
+	@docker compose -f docker-compose.yml stop api
 
 .PHONY: stack-without-api-and-sipi
 stack-without-api-and-sipi: stack-up ## starts the knora-stack without knora-api and sipi: fuseki, redis.
-	@docker-compose -f docker-compose.yml stop api
-	@docker-compose -f docker-compose.yml stop sipi
+	@docker compose -f docker-compose.yml stop api
+	@docker compose -f docker-compose.yml stop sipi
 
 .PHONY: stack-db-only
 stack-db-only: env-file docker-build-knora-jena-fuseki-image  ## starts only fuseki.
-	docker-compose -f docker-compose.yml up -d db
+	@docker compose -f docker-compose.yml up -d db
 	$(CURRENT_DIR)/webapi/scripts/wait-for-db.sh
 
 #################################

diff --git a/WORKSPACE b/WORKSPACE
@@ -56,9 +56,9 @@ http_archive(
 # Docker                            #
 #####################################
 
-rules_docker_version = "0.20.0"  # 12.10.2021
+rules_docker_version = "0.22.0"  # 06.12.2021
 
-rules_docker_version_sha256 = "92779d3445e7bdc79b961030b996cb0c91820ade7ffa7edca69273f404b085d5"
+rules_docker_version_sha256 = "59536e6ae64359b716ba9c46c39183403b01eabfbd57578e84398b4829ca499a"
 
 http_archive(
     name = "io_bazel_rules_docker",

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -26,13 +26,13 @@ services:
       - ${LOCAL_HOME}/sipi/server:/sipi/server:delegated
     networks:
       - knora-net
-    restart: unless-stopped
     environment:
       - SIPI_EXTERNAL_PROTOCOL=http
       - SIPI_EXTERNAL_HOSTNAME=localhost
       - SIPI_EXTERNAL_PORT=1024
       - SIPI_WEBAPI_HOSTNAME=api
       - SIPI_WEBAPI_PORT=3333
+    # entrypoint: [ "valgrind", "--leak-check=yes", "/sipi/sipi" ] ## uncomment to run SIPI under valgrind
     command: --config=/sipi/config/sipi.knora-docker-config.lua
 
   api:

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -29,7 +29,7 @@ object Dependencies {
         akkaHttpVersion := "10.2.8",
         jenaVersion := "4.4.0",
         metricsVersion := "4.0.1",
-        sipiImage := "daschswiss/sipi:v3.3.1",
+        sipiImage := "daschswiss/sipi:3.3.4",
         fusekiImage := "daschswiss/apache-jena-fuseki:2.0.8"
     )
 

diff --git a/sipi/config/sipi.knora-docker-config.lua b/sipi/config/sipi.knora-docker-config.lua
@@ -1,5 +1,5 @@
--- * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
--- * SPDX-License-Identifier: Apache-2.0
+-- Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+-- SPDX-License-Identifier: Apache-2.0
 
 --
 -- ATTENTION: This configuration file should only be used for integration testing. It has additional routes defined!!!
@@ -22,6 +22,34 @@ sipi = {
     --
     port = 1024,
 
+    --
+    -- Number of threads to use
+    --
+    nthreads = 8,
+
+    --
+    -- SIPI is using libjpeg to generate the JPEG images. libjpeg requires a quality value which
+    -- corresponds to the compression rate. 100 is (almost) no compression and best quality, 0
+    -- would be full compression and no quality. Reasonable values are between 30 and 95...
+    --
+    jpeg_quality = 60,
+
+    --
+    -- For scaling images, SIPI offers two methods. The value "high" offers best quality using expensive
+    -- algorithms: bilinear interpolation, if downscaling the image is first scaled up to an integer
+    -- multiple of the requires size, and then downscaled using averaging. This results in the best
+    -- image quality. "medium" uses bilinear interpolation but does not do upscaling before
+    -- downscaling. If scaling quality is set to "low", then just a lookup table and nearest integer
+    -- interpolation is being used to scale the images.
+    -- Recognized values are: "high", "medium", "low".
+    --
+    scaling_quality = {
+        jpeg = "medium",
+        tiff = "high",
+        png = "high",
+        j2k = "high"
+    },
+
     --
     -- Number of seconds a connection (socket) remains open
     --
@@ -83,17 +111,17 @@ sipi = {
     --
     -- if the cache becomes full, the given percentage of file space is marked for reuase
     --
-    cache_hysteresis = 0.1,
+    cache_hysteresis = 0.15,
 
     --
     -- Path to the directory where the scripts for the routes defined below are to be found
     --
     scriptdir = '/sipi/scripts',
 
     ---
-    --- Size of the thumbnails
+    --- Size of the thumbnails (to be used within Lua)
     ---
-    thumb_size = 'pct:4',
+    thumb_size = '!128,128',
 
     --
     -- Path to the temporary directory

diff --git a/third_party/versions.bzl b/third_party/versions.bzl
@@ -8,8 +8,7 @@ METRICS_VERSION = "4.0.1"
 
 # SIPI - digest takes precedence!
 SIPI_REPOSITORY = "daschswiss/sipi"
-SIPI_VERSION = "3.3.1"
-SIPI_IMAGE_DIGEST = "sha256:67a0e8c16a67914f2765a1c7906e781383b835cfe72cd19c763b7bc9eb7a38a5"
+SIPI_IMAGE_DIGEST = "sha256:235afabf427d4e8619d10f3858ab63b6c26809ef27184f69951e098f005faa02"  # 3.3.4
 
 # Jena Fuseki - digest takes precedence!
 FUSEKI_REPOSITORY = "daschswiss/apache-jena-fuseki"