Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(upgrade): Update repository on startup (#1643)
  • Loading branch information
Benjamin Geer committed May 18, 2020
1 parent dedb00b commit 0127dca
Show file tree
Hide file tree
Showing 99 changed files with 1,883 additions and 11,672 deletions.
66 changes: 9 additions & 57 deletions .github/workflows/main.yml
Expand Up @@ -232,36 +232,6 @@ jobs:
df -h
docker system df
upgrade-unit-tests:
name: Upgrade Unit Tests
needs: compile
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
with:
fetch-depth: 1
- name: cache maven artifacts
uses: actions/cache@v1
with:
path: ~/.ivy2
key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
restore-keys: |
${{ runner.OS }}-build-${{ env.cache-name }}-
${{ runner.OS }}-build-
${{ runner.OS }}-
- name: Disk Free
run: |
df -h
docker system df
docker system prune --all --force --volumes
df -h
- name: run upgrade tests
run: sbt "upgrade/test"
- name: Disk Free After
run: |
df -h
docker system df
upgrade-integration-tests:
name: Upgrade Integration Tests
needs: compile
Expand Down Expand Up @@ -297,32 +267,8 @@ jobs:
docker system df
docker system prune --all --force --volumes
df -h
- name: build all docker images
run: |
make build-all-scala
make build-all-images
- name: start and initialize graphdb-se
run: |
docker network create knora-net
docker run --rm -d --name=db --network=knora-net -p 127.0.0.1:7200:7200 -e GDB_HEAP_SIZE=$GDB_HEAP_SIZE daschswiss/knora-graphdb-se -Dgraphdb.license.file=/graphdb/graphdb.license
sleep 5
make init-db-test-minimal
- name: unzip test data
run: unzip $GITHUB_WORKSPACE/test-data/v7.0.0/v7.0.0-knora-test.trig.zip -d $GITHUB_WORKSPACE/test-data/v7.0.0/
- name: empty repository
run: $GITHUB_WORKSPACE/upgrade/graphdb-se/empty-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200
- name: load v7.0.0 data
run: $GITHUB_WORKSPACE/upgrade/graphdb-se/upload-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 $GITHUB_WORKSPACE/test-data/v7.0.0/v7.0.0-knora-test.trig
- name: dump repository data
run: |
mkdir -p $GITHUB_WORKSPACE/upgrade/out
$GITHUB_WORKSPACE/upgrade/graphdb-se/dump-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 $GITHUB_WORKSPACE/upgrade/out/dump.trig
- name: run upgrade from inside docker
run: docker run --rm --network=knora-net -v $GITHUB_WORKSPACE/upgrade/out:/upgrade/out daschswiss/knora-upgrade /upgrade/out/dump.trig /upgrade/out/dump-upgraded.trig
- name: empty repository
run: $GITHUB_WORKSPACE/upgrade/graphdb-se/empty-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200
- name: load upgraded data
run: $GITHUB_WORKSPACE/upgrade/graphdb-se/upload-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 $GITHUB_WORKSPACE/upgraded/out/dump-upgraded.trig
- name: test repository update
run: cd $GITHUB_WORKSPACE && make test-repository-update
- name: Disk Free After
run: |
df -h
Expand Down Expand Up @@ -364,7 +310,13 @@ jobs:
# publish only for develop and tags
publish:
name: Publish to Dockerhub
needs: [api-unit-tests, api-e2e-tests, api-integration-tests, upgrade-unit-tests, upgrade-integration-tests, docs-build-test]
needs: [
api-unit-tests,
api-e2e-tests,
api-integration-tests,
upgrade-integration-tests,
docs-build-test
]
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags')
steps:
Expand Down
111 changes: 21 additions & 90 deletions KnoraBuild.sbt
Expand Up @@ -5,6 +5,7 @@ import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd}
import org.knora.Dependencies
import sbt.Keys.version

import scala.language.postfixOps
import scala.sys.process.Process

//////////////////////////////////////
Expand Down Expand Up @@ -226,6 +227,7 @@ lazy val knoraGraphDbSe: Project = knoraModule("knora-graphdb-se")
)
)


//////////////////////////////////////
// Knora's custom GraphDB-Free
//////////////////////////////////////
Expand Down Expand Up @@ -265,6 +267,7 @@ lazy val knoraGraphdbFree: Project = knoraModule("knora-graphdb-free")
)
)


//////////////////////////////////////
// Knora's custom Sipi
//////////////////////////////////////
Expand Down Expand Up @@ -343,94 +346,6 @@ lazy val knoraAssets: Project = knoraModule("knora-assets")
)
)

//////////////////////////////////////
// Knora upgrade scripts
//////////////////////////////////////

lazy val upgradeCommonSettings = Seq(
name := "upgrade"
)

lazy val upgrade: Project = knoraModule("upgrade")
.dependsOn(webapi)
.enablePlugins(JavaAppPackaging, DockerPlugin)
.settings(
upgradeCommonSettings,
Dependencies.upgradeLibraryDependencies,
// use jars (and not class directory) for run, test, console
exportJars := true,
unmanagedResourceDirectories in Compile += (rootBaseDir.value / "knora-ontologies"),

// add content of knora-ontologies to jar
mappings in (Compile, packageBin) ++= Seq (
(rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl") -> "knora-ontologies/knora-admin.ttl",
(rootBaseDir.value / "knora-ontologies" / "knora-base.ttl") -> "knora-ontologies/knora-base.ttl",
(rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl") -> "knora-ontologies/salsah-gui.ttl",
(rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl") -> "knora-ontologies/standoff-data.ttl",
(rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl") -> "knora-ontologies/standoff-onto.ttl",
),
// contentOf("salsah1/src/main/resources").toMap.mapValues("config/" + _)
// (rootBaseDir.value / "knora-ontologies") -> "knora-ontologies",

)
.settings(
scalacOptions ++= Seq("-feature", "-unchecked", "-deprecation", "-Yresolve-term-conflict:package"),
logLevel := Level.Info,
run / fork := true,
run / javaOptions ++= upgradeJavaRunOptions,
Compile / run / mainClass := Some("org.knora.upgrade.Main"),
Test / fork := true,
Test / javaOptions ++= upgradeJavaTestOptions,
Test / parallelExecution := false,
/* show full stack traces and test case durations */
Test / testOptions += Tests.Argument("-oDF"),
)
.settings(
// Skip packageDoc and packageSrc task on stage
Compile / packageDoc / mappings := Seq(),
Compile / packageSrc / mappings := Seq(),

Universal / mappings ++= {
// copy the different folders
directory("upgrade/graphdb-se")
},

// add dockerCommands used to create the image
// docker:stage, docker:publishLocal, docker:publish, docker:clean
dockerRepository := Some("dhlabbasel"),
dockerUpdateLatest := true,
maintainer := "400790+subotic@users.noreply.github.com",
Docker / dockerCommands := Seq(
Cmd("FROM", "adoptopenjdk/openjdk11:alpine-jre"),
Cmd("LABEL", s"""MAINTAINER="${maintainer.value}""""),
Cmd("RUN apk update && apk upgrade && apk add bash"),

Cmd("ENV", """KNORA_UPGRADE_DOCKER="true""""),
Cmd("COPY", "opt/docker", "/upgrade"),
Cmd("WORKDIR", "/upgrade"),
ExecCmd("ENTRYPOINT", "/upgrade/bin/upgrade"),
),
)

lazy val upgradeJavaRunOptions = Seq(
// "-showversion",
"-Xms1G",
"-Xmx1G"
// "-verbose:gc",
//"-XX:+UseG1GC",
//"-XX:MaxGCPauseMillis=500",
//"-XX:MaxMetaspaceSize=4096m"
)

lazy val upgradeJavaTestOptions = Seq(
// "-showversion",
"-Xms1G",
"-Xmx1G"
// "-verbose:gc",
//"-XX:+UseG1GC",
//"-XX:MaxGCPauseMillis=500",
//"-XX:MaxMetaspaceSize=4096m"
)

//////////////////////////////////////
// SALSAH1 (./salsah1)
Expand Down Expand Up @@ -533,13 +448,12 @@ lazy val javaTestOptions = Seq(
)


lazy val HeadlessTest = config("headless") extend (Test)
lazy val HeadlessTest = config("headless") extend Test
lazy val javaHeadlessTestOptions = Seq(
"-Dconfig.resource=headless-testing.conf"
) ++ javaTestOptions



//////////////////////////////////////
// WEBAPI (./webapi)
//////////////////////////////////////
Expand Down Expand Up @@ -601,6 +515,23 @@ lazy val webapi = knoraModule("webapi")
inConfig(FusekiIt)(Defaults.testTasks ++ Seq(forkOptions := Defaults.forkOptionsTask.value)),
inConfig(EmbeddedJenaTDBTest)(Defaults.testTasks ++ Seq(forkOptions := Defaults.forkOptionsTask.value))
)

.settings(
exportJars := true,
unmanagedResourceDirectories in Compile += (rootBaseDir.value / "knora-ontologies"),

// add content of knora-ontologies to jar
mappings in (Compile, packageBin) ++= Seq (
(rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl") -> "knora-ontologies/knora-admin.ttl",
(rootBaseDir.value / "knora-ontologies" / "knora-base.ttl") -> "knora-ontologies/knora-base.ttl",
(rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl") -> "knora-ontologies/salsah-gui.ttl",
(rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl") -> "knora-ontologies/standoff-data.ttl",
(rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl") -> "knora-ontologies/standoff-onto.ttl",
),
// contentOf("salsah1/src/main/resources").toMap.mapValues("config/" + _)
// (rootBaseDir.value / "knora-ontologies") -> "knora-ontologies",
)

.settings(
scalacOptions ++= Seq("-feature", "-unchecked", "-deprecation", "-Yresolve-term-conflict:package"),

Expand Down
30 changes: 16 additions & 14 deletions Makefile
Expand Up @@ -24,7 +24,7 @@ docs-build: ## build the docs

.PHONY: build-all-scala
build-all-scala: ## build all scala projects
@sbt webapi/universal:stage knora-graphdb-se/universal:stage knora-graphdb-free/universal:stage knora-sipi/universal:stage salsah1/universal:stage upgrade/universal:stage knora-assets/universal:stage webapi_test/universal:stage webapi_it/universal:stage
@sbt webapi/universal:stage knora-graphdb-se/universal:stage knora-graphdb-free/universal:stage knora-sipi/universal:stage salsah1/universal:stage knora-assets/universal:stage webapi_test/universal:stage webapi_it/universal:stage

## knora-api
.PHONY: build-knora-api-image
Expand Down Expand Up @@ -83,15 +83,6 @@ build-knora-salsah1-image: build-all-scala ## build and publish knora-salsah1 do
publish-knora-salsah1-image: build-knora-salsah1-image ## publish knora-salsah1 image to Dockerhub
docker push $(KNORA_SALSAH1_IMAGE)

## knora-upgrade
.PHONY: build-knora-upgrade-image
build-knora-upgrade-image: build-all-scala ## build and publish knora-upgrade docker image locally
docker build -t $(KNORA_UPGRADE_IMAGE) -t $(REPO_PREFIX)/$(KNORA_UPGRADE_REPO):latest -f docker/knora-upgrade.dockerfile upgrade/target/universal

.PHONY: publish-knora-upgrade-image
publish-knora-upgrade-image: build-knora-upgrade-image ## publish knora-upgrade image to Dockerhub
docker push $(KNORA_UPGRADE_IMAGE)

## knora-assets
.PHONY: build-knora-assets-image
build-knora-assets-image: build-all-scala ## build and publish knora-assets docker image locally
Expand All @@ -103,10 +94,10 @@ publish-knora-assets-image: build-knora-assets-image ## publish knora-assets ima

## all images
.PHONY: build-all-images
build-all-images: build-knora-api-image build-knora-graphdb-se-image build-knora-graphdb-free-image build-knora-sipi-image build-knora-salsah1-image build-knora-upgrade-image build-knora-assets-image ## build all Docker images
build-all-images: build-knora-api-image build-knora-graphdb-se-image build-knora-graphdb-free-image build-knora-sipi-image build-knora-salsah1-image build-knora-assets-image ## build all Docker images

.PHONY: publish-all-images
publish-all-images: publish-knora-api-image publish-knora-graphdb-se-image publish-knora-graphdb-free-image publish-knora-sipi-image publish-knora-salsah1-image publish-knora-upgrade-image publish-knora-assets-image ## publish all Docker images
publish-all-images: publish-knora-api-image publish-knora-graphdb-se-image publish-knora-graphdb-free-image publish-knora-sipi-image publish-knora-salsah1-image publish-knora-assets-image ## publish all Docker images

#################################
## Docker-Compose targets
Expand Down Expand Up @@ -164,6 +155,7 @@ stack-restart: stack-up ## re-starts the knora-stack: graphdb, sipi, redis, api,
.PHONY: stack-restart-api
stack-restart-api: ## re-starts the api. Usually used after loading data into GraphDB.
docker-compose -f docker/knora.docker-compose.yml restart api
@$(CURRENT_DIR)/webapi/scripts/wait-for-knora.sh

.PHONY: stack-logs
stack-logs: ## prints out and follows the logs of the running knora-stack.
Expand Down Expand Up @@ -343,12 +335,23 @@ test-js-lib-integration: clean-local-tmp stack-without-api ## run knora-api-js-l
@$(MAKE) -f $(THIS_FILE) init-db-test
@sleep 15
@$(MAKE) -f $(THIS_FILE) stack-restart-api
sleep 15
@$(MAKE) -f $(THIS_FILE) stack-logs-api-no-follow
@git clone --single-branch --depth 1 https://github.com/dasch-swiss/knora-api-js-lib.git $(CURRENT_DIR)/.tmp/js-lib
$(MAKE) -C $(CURRENT_DIR)/.tmp/js-lib npm-install
$(MAKE) -C $(CURRENT_DIR)/.tmp/js-lib test

.PHONY: test-repository-update
test-repository-update: stack-without-api
@sleep 15
@$(MAKE) -f $(THIS_FILE) init-db-test-minimal
@rm -rf /tmp/knora-test-data/v7.0.0/
@mkdir -p /tmp/knora-test-data/v7.0.0/
@unzip $(CURRENT_DIR)/test-data/v7.0.0/v7.0.0-knora-test.trig.zip -d /tmp/knora-test-data/v7.0.0/
$(CURRENT_DIR)/webapi/scripts/graphdb-empty-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200
$(CURRENT_DIR)/webapi/scripts/graphdb-upload-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 /tmp/knora-test-data/v7.0.0/v7.0.0-knora-test.trig
@$(MAKE) -f $(THIS_FILE) stack-restart-api
@$(MAKE) -f $(THIS_FILE) stack-logs-api-no-follow

.PHONY: init-db-test
init-db-test: ## initializes the knora-test repository
@echo $@
Expand Down Expand Up @@ -413,7 +416,6 @@ info: ## print out all variables
@echo "KNORA_GRAPHDB_FREE_IMAGE: \t $(KNORA_GRAPHDB_FREE_IMAGE)"
@echo "KNORA_SIPI_IMAGE: \t\t $(KNORA_SIPI_IMAGE)"
@echo "KNORA_ASSETS_IMAGE: \t\t $(KNORA_ASSETS_IMAGE)"
@echo "KNORA_UPGRADE_IMAGE: \t\t $(KNORA_UPGRADE_IMAGE)"
@echo "KNORA_SALSAH1_IMAGE: \t\t $(KNORA_SALSAH1_IMAGE)"
@echo "KNORA_GDB_LICENSE: \t\t $(KNORA_GDB_LICENSE)"
@echo "KNORA_GDB_IMPORT: \t\t $(KNORA_GDB_IMPORT)"
Expand Down

0 comments on commit 0127dca

Please sign in to comment.