Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
build: BAZEL to SBT migration (DEV-508) (#2002)
* bring sbt configuration back

* wip: update dependencies

* fix deprecation warnings

* add ZIO + update some deps + cleanup

* add missing deps from current setup

* fix compilation error

* fix more compilation errors

* remove deps not needed for compilation

* build: add running build and test through SBT on Github

* bring back all deps

* fixing paths to test files (ongoing)

* fixing paths to test files (ongoing)

* wip: fix test

* add building of docker images on github CI

* wip: fix more tests

* wip: fix more tests

* fix sbt info regarding docker image

* wip: fix more tests

* fix bazel build

* ignore server version test

* fix test

* wip: fix more tests

* ignored 3 tests

* fix more tests

* tweak corespec

* tweak base specs

* tweak cache spec

* fix cache spec

* fix rejeccting route test

* fix cache spec

* fix sbt

* fix client test data (ongoing)

* fix client test data (ongoing)

* cleanup github ci

* remove redis

* fix bazel

* build dsp-api docker image through sbt

* fix buildinfo a bit

* fix buildinfo

* cleanup buildinfo

* review changes

* removed too much

* cleanup

Co-authored-by: Ivan Subotic <400790+subotic@users.noreply.github.com>
  • Loading branch information
mpro7 and subotic committed Feb 23, 2022
1 parent f1f8005 commit 38faa9e
Show file tree
Hide file tree
Showing 157 changed files with 1,081 additions and 4,810 deletions.
142 changes: 32 additions & 110 deletions .github/workflows/main.yml
Expand Up @@ -7,13 +7,18 @@ on:

jobs:
compile:
name: Build Everything
name: Build and Test
runs-on: ubuntu-latest
steps:
- name: checkout source
uses: actions/checkout@v1
with:
fetch-depth: 15
- name: Setup JDK
uses: actions/setup-java@v2
with:
distribution: temurin
java-version: 11
- name: get current time
uses: josStorer/get-current-time@v2
id: current-time
Expand All @@ -32,15 +37,24 @@ jobs:
architecture: 'x64' # defaults to 'x64'
- name: install requirements
run: sudo apt-get install ca-certificates-java expect
- name: run build everything
run: bazel build //...
- name: Coursier cache
uses: coursier/cache-action@v6
- name: Build and Test
run: make test
- name: Cleanup before cache
shell: bash
run: |
rm -rf "$HOME/.ivy2/local" || true
find $HOME/Library/Caches/Coursier/v1 -name "ivydata-*.properties" -delete || true
find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete || true
find $HOME/.cache/coursier/v1 -name "ivydata-*.properties" -delete || true
find $HOME/.sbt -name "*.lock" -delete || true
# ------------------------------------------
# ------------------------------------------
# Test client-test-data
client-test-data-tests:
name: Test client-test-data
needs: compile
name: Test client-test-data
runs-on: ubuntu-latest
steps:
- name: checkout source
Expand All @@ -64,114 +78,25 @@ jobs:
java-version: '11' # The OpenJDK version to make available on the path
architecture: 'x64' # defaults to 'x64'
- name: install requirements
run: sudo apt-get install ca-certificates-java expect redis-tools unzip
run: sudo apt-get install ca-certificates-java expect unzip
- name: Coursier cache
uses: coursier/cache-action@v6
- name: generate api-client-test-data
run: make client-test-data
- name: Cleanup before cache
shell: bash
run: |
rm -rf "$HOME/.ivy2/local" || true
find $HOME/Library/Caches/Coursier/v1 -name "ivydata-*.properties" -delete || true
find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete || true
find $HOME/.cache/coursier/v1 -name "ivydata-*.properties" -delete || true
find $HOME/.sbt -name "*.lock" -delete || true
# ------------------------------------------
# ------------------------------------------

api-unit-tests:
name: API Unit Tests
needs: compile
runs-on: ubuntu-latest
steps:
- name: checkout source
uses: actions/checkout@v1
with:
fetch-depth: 15
- name: get current time
uses: josStorer/get-current-time@v2
id: current-time
with:
format: YYYY-MM-DD
utcOffset: "+02:00"
- name: use daily bazel disk cache
uses: actions/cache@v2
with:
path: ~/.bazel_disk_cache
key: ${{ runner.OS }}-bazel_disk_cache-${{ steps.current-time.outputs.formattedTime }}
- name: setup java version
uses: joschi/setup-jdk@v2
with:
java-version: '11' # The OpenJDK version to make available on the path
architecture: 'x64' # defaults to 'x64'
- name: install requirements
run: sudo apt-get install ca-certificates-java expect
- name: use daily bazel disk cache
uses: actions/cache@v2
with:
path: ~/.bazel_disk_cache
key: ${{ runner.OS }}-bazel_disk_cache-${{ steps.date.outputs.date }} }}
- name: run API unit tests
run: make test-unit

api-e2e-tests:
name: API E2E Tests
needs: compile
runs-on: ubuntu-latest
steps:
- name: checkout source
uses: actions/checkout@v1
with:
fetch-depth: 15
- name: get current time
uses: josStorer/get-current-time@v2
id: current-time
with:
format: YYYY-MM-DD
utcOffset: "+02:00"
- name: use daily bazel disk cache
uses: actions/cache@v2
with:
path: ~/.bazel_disk_cache
key: ${{ runner.OS }}-bazel_disk_cache-${{ steps.current-time.outputs.formattedTime }}
- name: setup java version
uses: joschi/setup-jdk@v2
with:
java-version: '11' # The OpenJDK version to make available on the path
architecture: 'x64' # defaults to 'x64'
- name: install requirements
run: sudo apt-get install ca-certificates-java expect redis-tools unzip
- name: run API E2E tests
run: make test-e2e

api-integration-tests:
name: API Integration Tests
needs: compile
runs-on: ubuntu-latest
steps:
- name: checkout source
uses: actions/checkout@v1
with:
fetch-depth: 15
- name: get current time
uses: josStorer/get-current-time@v2
id: current-time
with:
format: YYYY-MM-DD
utcOffset: "+02:00"
- name: use daily bazel disk cache
uses: actions/cache@v2
with:
path: ~/.bazel_disk_cache
key: ${{ runner.OS }}-bazel_disk_cache-${{ steps.current-time.outputs.formattedTime }}
- name: setup java version
uses: joschi/setup-jdk@v2
with:
java-version: '11' # The OpenJDK version to make available on the path
architecture: 'x64' # defaults to 'x64'
- name: Install dependencies
run: |
sudo apt-get install ca-certificates-java expect
- name: run API integration tests
run: |
make test-docker
make test-it
upgrade-integration-tests:
name: Upgrade Integration Tests
needs: compile
runs-on: ubuntu-latest
steps:
- name: checkout source
Expand Down Expand Up @@ -202,7 +127,6 @@ jobs:

docs-build-test:
name: Build Docs Testrun
needs: compile
runs-on: ubuntu-latest
steps:
- name: checkout source
Expand Down Expand Up @@ -242,9 +166,7 @@ jobs:
publish:
name: Publish (on release only)
needs: [
api-unit-tests,
api-e2e-tests,
api-integration-tests,
compile,
client-test-data-tests,
upgrade-integration-tests,
docs-build-test
Expand Down Expand Up @@ -273,7 +195,7 @@ jobs:
java-version: '11' # The OpenJDK version to make available on the path
architecture: 'x64' # defaults to 'x64'
- name: install requirements
run: sudo apt-get install ca-certificates-java expect redis-tools unzip
run: sudo apt-get install ca-certificates-java expect unzip
- name: Install dependencies
run: |
python -m pip install --upgrade pip
Expand Down
35 changes: 8 additions & 27 deletions .gitignore
@@ -1,37 +1,22 @@
/.project
.ijwb
.vscode
.bloop
.bsp
**/.idea
**/*.iml
**/*.ipr
**/*.iws
/logs/
/bin/
/info.aduna.logging.dir_IS_UNDEFINED/
.cache
.classpath
.docker
.ensime
.ensime_cache
.env
/.tmp/
native
**/.DS_Store
site/
docs/env
docs/_build
docs/_format_docu
docs/_format_docu_v2
docs/project
salsah1/project
webapi/project
webapi/_fuseki/run
webapi-it
webapi-test
upgrade/project
upgrade/target
triplestores/graphdb-se-7
idea.vmoptions
sipi/test
bazel-*

Expand All @@ -50,18 +35,14 @@ bazel-*
*.bak
.sbtrc

**/venv
**/__pycache__
dump.rdb
/site/
dependencies.txt
/client-test-data.zip
/db_prod_dump.trig
/db_staging_dump.trig
/db_test_dump.trig
/.vscode
/cleandeps.sh
/.metals
/sipi/images/082E/*
/sipi/images/originals/082E/*
/dependencies.bzl

.idea/
.metals/
metals.sbt

.tmp/
42 changes: 9 additions & 33 deletions Makefile
Expand Up @@ -39,7 +39,6 @@ docs-clean: ## cleans the project directory

.PHONY: build
build: docker-build ## build all targets (excluding docs)
@bazel build //...

.PHOBY: check-for-outdated-deps
check-for-outdated-deps: ## check for outdated maven dependencies
Expand All @@ -55,7 +54,7 @@ buildifier: ## format Bazel WORKSPACE and BUILD.bazel files

.PHONY: docker-build-knora-api-image
docker-build-knora-api-image: # build and publish knora-api docker image locally
@bazel run //docker/knora-api:image
@sbt "webapi / Docker / publishLocal"

.PHONY: docker-publish-knora-api-image
docker-publish-knora-api-image: # publish knora-api image to Dockerhub
Expand Down Expand Up @@ -201,38 +200,15 @@ stack-db-only: env-file docker-build-knora-jena-fuseki-image ## starts only fus
test-docker: docker-build ## runs Docker image tests
bazel test //docker/...

.PHONY: test-webapi
test-webapi: docker-build ## runs all dsp-api tests.
bazel test //webapi/...

.PHONY: test-unit
test-unit: docker-build ## runs the dsp-api unit tests.
bazel test \
//webapi/src/test/scala/org/knora/webapi/http/... \
//webapi/src/test/scala/org/knora/webapi/messages/... \
//webapi/src/test/scala/org/knora/webapi/other/... \
//webapi/src/test/scala/org/knora/webapi/responders/... \
//webapi/src/test/scala/org/knora/webapi/routing/... \
//webapi/src/test/scala/org/knora/webapi/store/... \
//webapi/src/test/scala/org/knora/webapi/util/... \

.PHONY: test-e2e
test-e2e: docker-build ## runs the dsp-api e2e tests.
bazel test //webapi/src/test/scala/org/knora/webapi/e2e/...

.PHONY: client-test-data
client-test-data: docker-build ## runs the dsp-api e2e tests and generates client test data.
docker-compose -f docker-compose.yml up -d redis
$(CURRENT_DIR)/webapi/scripts/clear-client-test-data.sh
bazel test --cache_test_results=no //webapi/src/test/scala/org/knora/webapi/e2e/... --action_env=KNORA_WEBAPI_COLLECT_CLIENT_TEST_DATA=true
$(CURRENT_DIR)/webapi/scripts/dump-client-test-data.sh

.PHONY: test-it
test-it: docker-build ## runs the dsp-api integration tests.
bazel test //webapi/src/test/scala/org/knora/webapi/it/...
client-test-data: export KNORA_WEBAPI_COLLECT_CLIENT_TEST_DATA := true
client-test-data: build ## runs the dsp-api e2e tests and generates client test data.
$(CURRENT_DIR)/webapi/scripts/zap-client-test-data.sh
sbt -v "webapi/testOnly *E2ESpec *R2RSpec"
$(CURRENT_DIR)/webapi/scripts/zip-client-test-data.sh

.PHONY: test-repository-upgrade
test-repository-upgrade: init-db-test-minimal ## runs DB upgrade integration test
test-repository-upgrade: build init-db-test-minimal ## runs DB upgrade integration test
@rm -rf $(CURRENT_DIR)/.tmp/knora-test-data/v7.0.0/
@mkdir -p $(CURRENT_DIR)/.tmp/knora-test-data/v7.0.0/
@unzip $(CURRENT_DIR)/test_data/v7.0.0/v7.0.0-knora-test.trig.zip -d $(CURRENT_DIR)/.tmp/knora-test-data/v7.0.0/
Expand All @@ -245,8 +221,8 @@ test-repository-upgrade: init-db-test-minimal ## runs DB upgrade integration tes
@$(MAKE) -f $(THIS_FILE) stack-up

.PHONY: test
test: docker-build ## runs all test targets.
bazel test //webapi/...
test: build ## runs all tests
sbt -v +test

#################################
## Database Management
Expand Down
1 change: 1 addition & 0 deletions README.md
Expand Up @@ -53,6 +53,7 @@ Each developer machine should have the following prerequisites installed:
* Docker Desktop: https://www.docker.com/products/docker-desktop
* Homebrew (on macOS): https://brew.sh
* [OpenJDK](https://adoptopenjdk.net) 11
* [sbt](https://www.scala-sbt.org/)
* [Bazel](https://bazel.build)

#### Java Adoptopenjdk 11
Expand Down

0 comments on commit 38faa9e

Please sign in to comment.