Skip to content

Commit

Permalink
feature: add retrying of bind at startup (#1644)
Browse files Browse the repository at this point in the history
  • Loading branch information
subotic committed Jun 10, 2020
1 parent f984c2e commit f93677d
Show file tree
Hide file tree
Showing 71 changed files with 1,752 additions and 563 deletions.
1,101 changes: 1,101 additions & 0 deletions .editorconfig

Large diffs are not rendered by default.

46 changes: 35 additions & 11 deletions Makefile
Expand Up @@ -141,6 +141,8 @@ endif
## knora stack
.PHONY: stack-up
stack-up: build-all-images env-file ## starts the knora-stack: graphdb, sipi, redis, api, salsah1.
docker-compose -f docker/knora.docker-compose.yml up -d db
$(CURRENT_DIR)/webapi/scripts/wait-for-db.sh
docker-compose -f docker/knora.docker-compose.yml up -d

.PHONY: stack-up-ci
Expand All @@ -165,6 +167,10 @@ stack-logs: ## prints out and follows the logs of the running knora-stack.
stack-logs-db: ## prints out and follows the logs of the 'db' container running in knora-stack.
docker-compose -f docker/knora.docker-compose.yml logs -f db

.PHONY: stack-logs-db-no-follow
stack-logs-db-no-follow: ## prints out the logs of the 'db' container running in knora-stack.
docker-compose -f docker/knora.docker-compose.yml logs db

.PHONY: stack-logs-sipi
stack-logs-sipi: ## prints out and follows the logs of the 'sipi' container running in knora-stack.
docker-compose -f docker/knora.docker-compose.yml logs -f sipi
Expand All @@ -191,7 +197,11 @@ stack-logs-salsah1: ## prints out and follows the logs of the 'salsah1' containe

.PHONY: stack-health
stack-health:
curl 0.0.0.0:3333/health
curl -f 0.0.0.0:3333/health

.PHONY: stack-status
stack-status:
docker-compose -f docker/knora.docker-compose.yml ps

.PHONY: stack-down
stack-down: ## stops the knora-stack.
Expand All @@ -212,8 +222,25 @@ stack-without-api-and-sipi: stack-up ## starts the knora-stack without knora-api
docker-compose -f docker/knora.docker-compose.yml stop api
docker-compose -f docker/knora.docker-compose.yml stop sipi

.PHONY: test-only
test-only: stack-without-api init-db-test-unit ## runs only the supplied tests, e.g., make test-only TARGET="*.CORSSupportE2ESpec".
@echo $@ # print target name
docker run --rm \
-v /tmp:/tmp:delegated \
-v $(CURRENT_DIR):/src:delegated \
-v $(HOME)/.ivy2:/root/.ivy2:delegated \
--name=api \
-e KNORA_WEBAPI_TRIPLESTORE_HOST=db \
-e KNORA_WEBAPI_SIPI_EXTERNAL_HOST=sipi \
-e KNORA_WEBAPI_SIPI_INTERNAL_HOST=sipi \
-e KNORA_WEBAPI_CACHE_SERVICE_REDIS_HOST=redis \
-e SBT_OPTS="-Xms2048M -Xmx2048M -Xss6M" \
--network=docker_knora-net \
daschswiss/scala-sbt sbt "webapi/testOnly $(TARGET)"

.PHONY: test-unit
test-unit: stack-without-api init-db-test-unit ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest').
@echo $@ # print target name
docker run --rm \
-v /tmp:/tmp \
-v $(CURRENT_DIR):/src \
Expand All @@ -228,10 +255,8 @@ test-unit: stack-without-api init-db-test-unit ## runs the unit tests (equivalen
daschswiss/scala-sbt sbt 'webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest'

.PHONY: test-unit-ci
test-unit-ci: stack-without-api ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting.
test-unit-ci: stack-without-api init-db-test-unit ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting.
@echo $@ # print target name
@sleep 5
@$(MAKE) -f $(THIS_FILE) init-db-test-unit
docker run --rm \
-v /tmp:/tmp \
-v $(CURRENT_DIR):/src \
Expand All @@ -247,6 +272,7 @@ test-unit-ci: stack-without-api ## runs the unit tests (equivalent to 'sbt webap

.PHONY: test-e2e
test-e2e: stack-without-api init-db-test-unit ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest').
@echo $@ # print target name
docker run --rm \
-v /tmp:/tmp \
-v $(CURRENT_DIR):/src \
Expand All @@ -261,10 +287,8 @@ test-e2e: stack-without-api init-db-test-unit ## runs the e2e tests (equivalent
daschswiss/scala-sbt sbt 'webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest'

.PHONY: test-e2e-ci
test-e2e-ci: stack-without-api ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting.
test-e2e-ci: stack-without-api init-db-test-unit ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting.
@echo $@ # print target name
@sleep 5
@$(MAKE) -f $(THIS_FILE) init-db-test-unit
docker run --rm \
-v /tmp:/tmp \
-v $(CURRENT_DIR):/src \
Expand All @@ -280,6 +304,7 @@ test-e2e-ci: stack-without-api ## runs the e2e tests (equivalent to 'sbt webapi/

.PHONY: test-it
test-it: stack-without-api init-db-test-unit ## runs the integration tests (equivalent to 'sbt webapi/it').
@echo $@
docker run --rm \
-v /tmp:/tmp \
-v $(CURRENT_DIR):/src \
Expand All @@ -294,10 +319,8 @@ test-it: stack-without-api init-db-test-unit ## runs the integration tests (equi
daschswiss/scala-sbt sbt 'webapi/it:test'

.PHONY: test-it-ci
test-it-ci: stack-without-api ## runs the integration tests (equivalent to 'sbt webapi/it:test') with code-coverage reporting.
test-it-ci: stack-without-api init-db-test-unit ## runs the integration tests (equivalent to 'sbt webapi/it:test') with code-coverage reporting.
@echo $@ # print target name
@sleep 5
@$(MAKE) -f $(THIS_FILE) init-db-test-unit
docker run --rm \
-v /tmp:/tmp \
-v $(CURRENT_DIR):/src \
Expand All @@ -312,7 +335,8 @@ test-it-ci: stack-without-api ## runs the integration tests (equivalent to 'sbt
daschswiss/scala-sbt sbt coverage webapi/it:test webapi/coverageReport

.PHONY: test-all
test-all: stack-without-api ## runs the all tests (equivalent to 'sbt webapi/test').
test-all: stack-without-api init-db-test-unit ## runs the all tests (equivalent to 'sbt webapi/test').
@echo $@
# docker build -t webapi-test -f docker/knora-api-test.dockerfile webapi/build/test/target/universal
docker run --rm \
-v /tmp:/tmp \
Expand Down
2 changes: 1 addition & 1 deletion docker/knora-api-it.dockerfile
Expand Up @@ -11,6 +11,6 @@ WORKDIR /webapi-it

EXPOSE 3333

ENTRYPOINT ["bin/webapi-it]
ENTRYPOINT ["bin/webapi-it"]

LABEL MAINTAINER="400790+subotic@users.noreply.github.com"
3 changes: 3 additions & 0 deletions docker/knora-api.dockerfile
Expand Up @@ -18,6 +18,9 @@ COPY stage /webapi

WORKDIR /webapi

# check every minute
HEALTHCHECK --interval=1m --timeout=1s CMD curl -f http://localhost:3333/health || exit 1

EXPOSE 3333
#EXPOSE 10001

Expand Down
17 changes: 7 additions & 10 deletions project/Dependencies.scala
Expand Up @@ -44,9 +44,9 @@ object Dependencies {
lazy val knoraSalsah1Image = SettingKey[String]("the Knora Salsah1 Image")

val Versions = Seq(
scalaVersion := "2.12.8",
akkaVersion := "2.6.4",
akkaHttpVersion := "10.1.11",
scalaVersion := "2.12.11",
akkaVersion := "2.6.5",
akkaHttpVersion := "10.1.12",
jenaVersion := "3.4.0",
metricsVersion := "4.0.1",
sipiImage := "dhlabbasel/sipi:v2.0.1",
Expand All @@ -61,7 +61,6 @@ object Dependencies {
object Compile {
// akka
val akkaActor = Def.setting {"com.typesafe.akka" %% "akka-actor" % akkaVersion.value}
//val akkaAgent = Def.setting {"com.typesafe.akka" %% "akka-agent" % akkaVersion.value}
val akkaStream = Def.setting {"com.typesafe.akka" %% "akka-stream" % akkaVersion.value}
val akkaSlf4j = Def.setting {"com.typesafe.akka" %% "akka-slf4j" % akkaVersion.value}

Expand All @@ -76,7 +75,7 @@ object Dependencies {
// testing

//CORS support
val akkaHttpCors = "ch.megard" %% "akka-http-cors" % "0.3.4"
val akkaHttpCors = "ch.megard" %% "akka-http-cors" % "1.0.0"

// jena
val jenaLibs = Def.setting {"org.apache.jena" % "apache-jena-libs" % jenaVersion.value exclude("org.slf4j", "slf4j-log4j12") exclude("commons-codec", "commons-codec")}
Expand Down Expand Up @@ -151,7 +150,7 @@ object Dependencies {
val akkaTestkit = Def.setting {"com.typesafe.akka" %% "akka-testkit" % akkaVersion.value % "test"}
val akkaStreamTestkit = Def.setting {"com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion.value % "test"}
val akkaHttpTestkit = Def.setting {"com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion.value % "test"}
val scalaTest = "org.scalatest" %% "scalatest" % "3.0.4" % "test"
val scalaTest = "org.scalatest" %% "scalatest" % "3.1.2" % "test"

// browser tests
val selenium = "org.seleniumhq.selenium" % "selenium-java" % "3.4.0" % "test"
Expand All @@ -161,7 +160,7 @@ object Dependencies {
val akkaTestkit = Def.setting {"com.typesafe.akka" %% "akka-testkit" % akkaVersion.value % "test, it, gdbse, gdbse-it, gdbfree, gdbfree-it, tdb, fuseki, fuseki-it"}
val akkaStreamTestkit = Def.setting {"com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion.value % "test, it, gdbse, gdbse-it, gdbfree, gdbfree-it, tdb, fuseki, fuseki-it"}
val akkaHttpTestkit = Def.setting {"com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion.value % "test, it, gdbse, gdbse-it, gdbfree, gdbfree-it, tdb, fuseki, fuseki-it"}
val scalaTest = "org.scalatest" %% "scalatest" % "3.0.4" % "test, it, gdbse, gdbse-it, gdbfree, gdbfree-it, tdb, fuseki, fuseki-it"
val scalaTest = "org.scalatest" %% "scalatest" % "3.1.2" % "test, it, gdbse, gdbse-it, gdbfree, gdbfree-it, tdb, fuseki, fuseki-it"
val gatlingHighcharts = "io.gatling.highcharts" % "gatling-charts-highcharts"% "2.3.1" % "test, it, gdbse, gdbse-it, gdbfree, gdbfree-it, tdb, fuseki, fuseki-it"
val gatlingTestFramework = "io.gatling" % "gatling-test-framework" % "2.3.1" % "test, it, gdbse, gdbse-it, gdbfree, gdbfree-it, tdb, fuseki, fuseki-it"

Expand All @@ -171,7 +170,7 @@ object Dependencies {
val akkaTestkit = Def.setting {"com.typesafe.akka" %% "akka-testkit" % akkaVersion.value}
val akkaStreamTestkit = Def.setting {"com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion.value}
val akkaHttpTestkit = Def.setting {"com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion.value}
val scalaTest = "org.scalatest" %% "scalatest" % "3.0.4"
val scalaTest = "org.scalatest" %% "scalatest" % "3.1.2"
val gatlingHighcharts = "io.gatling.highcharts" % "gatling-charts-highcharts"% "2.3.1"
val gatlingTestFramework = "io.gatling" % "gatling-test-framework" % "2.3.1"
}
Expand All @@ -182,7 +181,6 @@ object Dependencies {

val salsahLibraryDependencies = l ++= Seq[sbt.ModuleID](
akkaActor.value,
//akkaAgent.value,
akkaStream.value,
akkaSlf4j.value,
akkaHttp.value,
Expand All @@ -197,7 +195,6 @@ object Dependencies {

val webapiLibraryDependencies = l ++= Seq[sbt.ModuleID](
akkaActor.value,
//akkaAgent.value,
akkaHttp.value,
akkaHttpCirce,
akkaHttpCors,
Expand Down
Expand Up @@ -27,15 +27,16 @@ import akka.stream.Materializer
import akka.util.Timeout
import com.typesafe.config.ConfigFactory
import org.knora.salsah.SettingsImpl
import org.scalatest._
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike

import scala.concurrent.duration._
import scala.concurrent.{Await, ExecutionContextExecutor}

/**
* An abstract base class for Selenium tests of the SALSAH user interface.
*/
abstract class SalsahSpec extends WordSpecLike with Matchers with RequestBuilding {
abstract class SalsahSpec extends AnyWordSpecLike with Matchers with RequestBuilding {

implicit private val system = ActorSystem()

Expand Down
72 changes: 72 additions & 0 deletions webapi/scripts/wait-for-db.sh
@@ -0,0 +1,72 @@
#!/usr/bin/env bash

#set -x

POSITIONAL=()
while [[ $# -gt 0 ]]
do
key="$1"

case $key in

-h|--host)
HOST="$2"
shift # past argument
shift # past value
;;

-t|--timeout)
TIMEOUT="$2"
shift # past argument
shift # past value
;;

-n|--name)
NAME="$3"
shift # past argument
shift # past value
;;

*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters

if [[ -z "${HOST}" ]]; then
HOST="localhost:7200"
fi

if [[ -z "${TIMEOUT}" ]]; then
TIMEOUT=360
fi

if [[ -z "${NAME}" ]]; then
NAME="knora-test-unit"
fi

poll-db() {
# STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://${HOST}/repositories/${NAME}/health?)
STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://${HOST}/rest/repositories)

if [ "${STATUS}" -eq 200 ]; then
echo "==> DB started"
return 0
else
return 1
fi
}

attempt_counter=0

until poll-db; do
if [ ${attempt_counter} -eq ${TIMEOUT} ]; then
echo "Timed out waiting for DB to start"
exit 1
fi

attempt_counter=$((attempt_counter+1))
sleep 1
done
27 changes: 15 additions & 12 deletions webapi/scripts/wait-for-knora.sh
Expand Up @@ -8,20 +8,23 @@ do
key="$1"

case $key in

-h|--host)
HOST="$2"
shift # past argument
shift # past value
;;
HOST="$2"
shift # past argument
shift # past value
;;

-t|--timeout)
TIMEOUT="$2"
shift # past argument
shift # past value
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
TIMEOUT="$2"
shift # past argument
shift # past value
;;

*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
Expand Down
17 changes: 7 additions & 10 deletions webapi/src/it/scala/org/knora/webapi/ITKnoraFakeSpec.scala
Expand Up @@ -19,24 +19,21 @@

package org.knora.webapi

import java.io.File
import java.nio.file.{Files, Paths}

import akka.actor.{ActorRef, ActorSystem, Props}
import akka.event.LoggingAdapter
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.client.RequestBuilding
import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
import akka.http.scaladsl.unmarshalling.Unmarshal
import akka.stream.Materializer
import com.typesafe.config.{Config, ConfigFactory}
import org.knora.webapi.app.{APPLICATION_MANAGER_ACTOR_NAME, ApplicationActor, LiveManagers}
import org.knora.webapi.util.StringFormatter
import org.scalatest.{BeforeAndAfterAll, Matchers, Suite, WordSpecLike}
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike
import org.scalatest.{BeforeAndAfterAll, Suite}
import spray.json.{JsObject, _}

import scala.concurrent.duration.{Duration, _}
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.concurrent.{Await, ExecutionContext}
import scala.languageFeature.postfixOps


Expand All @@ -48,7 +45,7 @@ object ITKnoraFakeSpec {
* This class can be used in End-to-End testing. It starts a Fake Knora server and
* provides access to settings and logging.
*/
class ITKnoraFakeSpec(_system: ActorSystem) extends Core with KnoraFakeCore with Suite with WordSpecLike with Matchers with BeforeAndAfterAll with RequestBuilding {
class ITKnoraFakeSpec(_system: ActorSystem) extends Core with KnoraFakeCore with Suite with AnyWordSpecLike with Matchers with BeforeAndAfterAll with RequestBuilding {

/* constructors */
def this(name: String, config: Config) = this(ActorSystem(name, config.withFallback(ITKnoraFakeSpec.defaultConfig)))
Expand All @@ -58,7 +55,7 @@ class ITKnoraFakeSpec(_system: ActorSystem) extends Core with KnoraFakeCore with

/* needed by the core trait */
implicit lazy val system: ActorSystem = _system
implicit lazy val settings: SettingsImpl = Settings(system)
implicit lazy val settings: KnoraSettingsImpl = KnoraSettings(system)
implicit val materializer: Materializer = Materializer.matFromSystem(system)
implicit val executionContext: ExecutionContext = system.dispatchers.lookup(KnoraDispatchers.KnoraActorDispatcher)

Expand Down

0 comments on commit f93677d

Please sign in to comment.