From 23e5c38910bac67f79c6c9d5516914f7c5f1dea1 Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Thu, 31 Aug 2017 14:31:47 +0200 Subject: [PATCH 01/18] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 97fe7e3a..40a217fa 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.daisy.pipeline assembly - 1.10.3 + 1.10.4-SNAPSHOT pom DAISY Pipeline 2 :: Assembly Builds the main distribution of the DAISY Pipeline 2. @@ -17,7 +17,7 @@ scm:git:git@github.com:daisy-consortium/pipeline-assembly.git scm:git:git@github.com:daisy-consortium/pipeline-assembly.git scm:git:git@github.com:daisy-consortium/pipeline-assembly.git - v1.10.3 + HEAD 2.1.0 From e3304a0f7c29d9f6d82a8d19d69846b93c30ceaa Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Tue, 29 Aug 2017 16:15:04 +0200 Subject: [PATCH 02/18] Add dtbook-to-rtf module --- pom.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 40a217fa..9996e62b 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,7 @@ org.daisy.pipeline.modules scripts-bom - 1.10.1 + 1.10.2-SNAPSHOT pom import @@ -761,6 +761,10 @@ org.daisy.pipeline.modules dtbook-to-html + + org.daisy.pipeline.modules + dtbook-to-rtf + org.daisy.pipeline.modules dtbook-to-ssml From d463250610c876813d95270b5a845505c5a5bedb Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Wed, 27 Sep 2017 17:06:52 +0200 Subject: [PATCH 03/18] Add a Dockerfile to build an image The Dockerfile uses a multistage build to first build the artifacts using maven. Then it copies the artifacts into a final image which exposes the port and starts the pipeline. --- .dockerignore | 7 +++++++ Dockerfile | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7ab9da03 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.gitignore + +.dockerignore +Dockerfile + +target \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..17fecfdc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +# Use a multistage build to first build the pipeline using maven. Then +# copy the artifacts into a final image which exposes the port and +# starts the pipeline. + +# Build the pipeline first +FROM maven:3.5-jdk-8 as builder +ADD . /usr/src/daisy-pipeline2 +WORKDIR /usr/src/daisy-pipeline2 +RUN mvn clean package + +# then use the build artifacts to create an image where the pipeline is installed +FROM openjdk:8-jre +LABEL maintainer="DAISY Consortium (http://www.daisy.org/)" +COPY --from=builder /usr/src/daisy-pipeline2/target/pipeline2-*_linux/daisy-pipeline /opt/daisy-pipeline2 +EXPOSE 8181 +ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2"] From aaa10a013747397925d123991e337b30dbeef359 Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Thu, 28 Sep 2017 14:37:11 +0200 Subject: [PATCH 04/18] Start the pipeline in remote mode --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 17fecfdc..6b5d2409 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,4 @@ FROM openjdk:8-jre LABEL maintainer="DAISY Consortium (http://www.daisy.org/)" COPY --from=builder /usr/src/daisy-pipeline2/target/pipeline2-*_linux/daisy-pipeline /opt/daisy-pipeline2 EXPOSE 8181 -ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2"] +ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2","remote"] From fbced576816a307799a9fc482a09052caa4dd2df Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Fri, 29 Sep 2017 10:06:52 +0200 Subject: [PATCH 05/18] Add a newline at the end of the file --- .dockerignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 7ab9da03..30a37e0e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,4 @@ .dockerignore Dockerfile -target \ No newline at end of file +target From 36ed95687a8e1881f8df0beec4b5f33ba847a7f9 Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Fri, 29 Sep 2017 16:22:58 +0200 Subject: [PATCH 06/18] Allow declaration of client key and secret via env variables If PIPELINE2_AUTH_CLIENTKEY and/or PIPELINE2_AUTH_CLIENTSECRET are defined in the environment, when starting the pipeline, use those values. This simplyfies dockerization of the pipeline. --- src/main/resources/bin/pipeline2 | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/main/resources/bin/pipeline2 b/src/main/resources/bin/pipeline2 index c1c07005..9a2b580c 100755 --- a/src/main/resources/bin/pipeline2 +++ b/src/main/resources/bin/pipeline2 @@ -355,23 +355,22 @@ init() { run() { if [ "x$PIPELINE2_AUTH" = "x" ]; then - PIPELINE2_AUTH="-Dorg.daisy.pipeline.ws.authentication=false" - export PIPELINE2_AUTH + OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication=false" else - PIPELINE2_AUTH="-Dorg.daisy.pipeline.ws.authentication=$PIPELINE2_AUTH" - export PIPELINE2_AUTH - + OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication=$PIPELINE2_AUTH" fi if [ "x$PIPELINE2_LOCAL" = "x" ]; then - PIPELINE2_LOCAL="-Dorg.daisy.pipeline.ws.localfs=true" - export PIPELINE2_LOCAL + OPTS="$OPTS -Dorg.daisy.pipeline.ws.localfs=true" else - PIPELINE2_LOCAL="-Dorg.daisy.pipeline.ws.localfs=$PIPELINE2_LOCAL" - export PIPELINE2_LOCAL - + OPTS="$OPTS -Dorg.daisy.pipeline.ws.localfs=$PIPELINE2_LOCAL" + fi + if [ "x$PIPELINE2_AUTH_CLIENTKEY" != "x" ]; then + OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication.key=$PIPELINE2_AUTH_CLIENTKEY" + fi + if [ "x$PIPELINE2_AUTH_CLIENTSECRET" != "x" ]; then + OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication.secret=$PIPELINE2_AUTH_CLIENTSECRET" fi MAIN=org.apache.felix.main.Main - OPTS="$PIPELINE2_LOCAL $PIPELINE2_AUTH" MODE=" -Dorg.daisy.pipeline.main.mode=webservice" case "$1" in From 91a547cffb4c1870b69871dc138f17e8fb971b41 Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Fri, 29 Sep 2017 16:25:27 +0200 Subject: [PATCH 07/18] Set up the environment non-local and authenticated for docker --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6b5d2409..bd545520 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,5 +12,9 @@ RUN mvn clean package FROM openjdk:8-jre LABEL maintainer="DAISY Consortium (http://www.daisy.org/)" COPY --from=builder /usr/src/daisy-pipeline2/target/pipeline2-*_linux/daisy-pipeline /opt/daisy-pipeline2 +ENV PIPELINE2_LOCAL=false \ + PIPELINE2_AUTH=true \ + PIPELINE2_AUTH_CLIENTKEY=clientkey \ + PIPELINE2_AUTH_CLIENTSECRET=sekret EXPOSE 8181 -ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2","remote"] +ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2"] From 387ca2c459befa986a14bbd9346769411dacdcc6 Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Wed, 4 Oct 2017 17:43:15 +0200 Subject: [PATCH 08/18] Default the clientkey to the usual default that is used everywhere else, for example in the default config of the pipeline cli --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bd545520..dc27eea9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ LABEL maintainer="DAISY Consortium (http://www.daisy.org/)" COPY --from=builder /usr/src/daisy-pipeline2/target/pipeline2-*_linux/daisy-pipeline /opt/daisy-pipeline2 ENV PIPELINE2_LOCAL=false \ PIPELINE2_AUTH=true \ - PIPELINE2_AUTH_CLIENTKEY=clientkey \ + PIPELINE2_AUTH_CLIENTKEY=clientid \ PIPELINE2_AUTH_CLIENTSECRET=sekret EXPOSE 8181 ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2"] From f23c1d64360d0b533ffbd212b81d7efd79506610 Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Wed, 4 Oct 2017 17:44:16 +0200 Subject: [PATCH 09/18] Expose the PIPELINE2_HOST as an env variable so that it can be set at run time for example when starting a Docker image and remove it from the system.properties (otherwise setting it as an option when starting the JVM seems to have no effect) --- src/main/resources/bin/pipeline2 | 3 +++ src/main/resources/etc/system.properties | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/resources/bin/pipeline2 b/src/main/resources/bin/pipeline2 index 9a2b580c..38a553a2 100755 --- a/src/main/resources/bin/pipeline2 +++ b/src/main/resources/bin/pipeline2 @@ -370,6 +370,9 @@ run() { if [ "x$PIPELINE2_AUTH_CLIENTSECRET" != "x" ]; then OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication.secret=$PIPELINE2_AUTH_CLIENTSECRET" fi + if [ "x$PIPELINE2_HOST" != "x" ]; then + OPTS="$OPTS -Dorg.daisy.pipeline.ws.host=$PIPELINE2_HOST" + fi MAIN=org.apache.felix.main.Main MODE=" -Dorg.daisy.pipeline.main.mode=webservice" diff --git a/src/main/resources/etc/system.properties b/src/main/resources/etc/system.properties index 7bf5a140..73518e06 100644 --- a/src/main/resources/etc/system.properties +++ b/src/main/resources/etc/system.properties @@ -15,7 +15,7 @@ org.daisy.pipeline.xproc.configuration=${org.daisy.pipeline.home}/etc/config-cal java.awt.headless=true #WS configuration -org.daisy.pipeline.ws.host=localhost +#org.daisy.pipeline.ws.host=localhost #allow local filesystem interaction #org.daisy.pipeline.ws.localfs=true #org.daisy.pipeline.ws.authentication=false From 35a44367cf98f327dd9b85c283386a9bbd78ed2f Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Thu, 5 Oct 2017 15:09:11 +0200 Subject: [PATCH 10/18] Add a test for the Docker image The test starts two containers based on the same image. One for the pipeline itself and a second one for the cli. It then starts a script from the cli. --- src/test/resources/test-docker-image.sh | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 src/test/resources/test-docker-image.sh diff --git a/src/test/resources/test-docker-image.sh b/src/test/resources/test-docker-image.sh new file mode 100755 index 00000000..1da24e29 --- /dev/null +++ b/src/test/resources/test-docker-image.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# This test starts the pipeline inside a Docker container and then +# starts the dtbook-to-epub script using the cli inside a second +# container + +# take any old DTBook file for testing for example from +# https://github.com/daisy/pipeline-samples +DOCUMENT=test.xml +DATA_DIR=$(pwd) +DATA=$(basename $DOCUMENT).zip +CLIENTKEY=clientid +CLIENTSECRET=sekret +MOUNT_POINT=/mnt + +zip $DATA $DOCUMENT + +# run the pipeline +docker run --name pipeline --rm --detach \ + -e PIPELINE2_HOST=0.0.0.0 \ + -e PIPELINE2_AUTH=true \ + -e PIPELINE2_AUTH_CLIENTKEY=$CLIENTKEY \ + -e PIPELINE2_AUTH_CLIENTSECRET=$CLIENTSECRET \ + -p 8181:8181 daisyorg/pipeline2 + +# wait for the pipeline to start +sleep 5 + +while ! curl localhost:8181/ws/alive >/dev/null 2>/dev/null; do + echo "Waiting for web service to be up..." >&2 + sleep 2 +done + +# run the cli +docker run --name cli --rm -it --link pipeline \ + --entrypoint /opt/daisy-pipeline2/cli/dp2 \ + --volume="$DATA_DIR:$MOUNT_POINT:rw" \ + daisyorg/pipeline2 \ + --host http://pipeline \ + --starting false \ + --client_key $CLIENTKEY \ + --client_secret $CLIENTSECRET \ + dtbook-to-epub3 --source $DOCUMENT --output $MOUNT_POINT --data $MOUNT_POINT/$DATA --persistent +# help dtbook-to-epub3 +# dtbook-validator --input-dtbook $DOCUMENT --output $MOUNT_POINT --data $MOUNT_POINT/$DATA + +docker stop pipeline + From 8851938dea1797300884d2b372e966ba5fdf9978 Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Tue, 10 Oct 2017 12:13:53 +0200 Subject: [PATCH 11/18] Make use of standardized environment variables This reverts 36ed956 and f23c1d6. --- Dockerfile | 4 ++-- pom.xml | 2 +- src/main/resources/bin/pipeline2 | 24 +++++++++++------------- src/main/resources/etc/system.properties | 2 +- src/test/resources/test-docker-image.sh | 6 +++--- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc27eea9..0edbdb19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ LABEL maintainer="DAISY Consortium (http://www.daisy.org/)" COPY --from=builder /usr/src/daisy-pipeline2/target/pipeline2-*_linux/daisy-pipeline /opt/daisy-pipeline2 ENV PIPELINE2_LOCAL=false \ PIPELINE2_AUTH=true \ - PIPELINE2_AUTH_CLIENTKEY=clientid \ - PIPELINE2_AUTH_CLIENTSECRET=sekret + PIPELINE2_WS_AUTHENTICATION_KEY=clientid \ + PIPELINE2_WS_AUTHENTICATION_SECRET=sekret EXPOSE 8181 ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2"] diff --git a/pom.xml b/pom.xml index 9996e62b..4e0834fa 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ org.daisy.pipeline framework-bom - 1.10.4 + 1.10.5-SNAPSHOT pom import diff --git a/src/main/resources/bin/pipeline2 b/src/main/resources/bin/pipeline2 index 38a553a2..c1c07005 100755 --- a/src/main/resources/bin/pipeline2 +++ b/src/main/resources/bin/pipeline2 @@ -355,25 +355,23 @@ init() { run() { if [ "x$PIPELINE2_AUTH" = "x" ]; then - OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication=false" + PIPELINE2_AUTH="-Dorg.daisy.pipeline.ws.authentication=false" + export PIPELINE2_AUTH else - OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication=$PIPELINE2_AUTH" + PIPELINE2_AUTH="-Dorg.daisy.pipeline.ws.authentication=$PIPELINE2_AUTH" + export PIPELINE2_AUTH + fi if [ "x$PIPELINE2_LOCAL" = "x" ]; then - OPTS="$OPTS -Dorg.daisy.pipeline.ws.localfs=true" + PIPELINE2_LOCAL="-Dorg.daisy.pipeline.ws.localfs=true" + export PIPELINE2_LOCAL else - OPTS="$OPTS -Dorg.daisy.pipeline.ws.localfs=$PIPELINE2_LOCAL" - fi - if [ "x$PIPELINE2_AUTH_CLIENTKEY" != "x" ]; then - OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication.key=$PIPELINE2_AUTH_CLIENTKEY" - fi - if [ "x$PIPELINE2_AUTH_CLIENTSECRET" != "x" ]; then - OPTS="$OPTS -Dorg.daisy.pipeline.ws.authentication.secret=$PIPELINE2_AUTH_CLIENTSECRET" - fi - if [ "x$PIPELINE2_HOST" != "x" ]; then - OPTS="$OPTS -Dorg.daisy.pipeline.ws.host=$PIPELINE2_HOST" + PIPELINE2_LOCAL="-Dorg.daisy.pipeline.ws.localfs=$PIPELINE2_LOCAL" + export PIPELINE2_LOCAL + fi MAIN=org.apache.felix.main.Main + OPTS="$PIPELINE2_LOCAL $PIPELINE2_AUTH" MODE=" -Dorg.daisy.pipeline.main.mode=webservice" case "$1" in diff --git a/src/main/resources/etc/system.properties b/src/main/resources/etc/system.properties index 73518e06..7bf5a140 100644 --- a/src/main/resources/etc/system.properties +++ b/src/main/resources/etc/system.properties @@ -15,7 +15,7 @@ org.daisy.pipeline.xproc.configuration=${org.daisy.pipeline.home}/etc/config-cal java.awt.headless=true #WS configuration -#org.daisy.pipeline.ws.host=localhost +org.daisy.pipeline.ws.host=localhost #allow local filesystem interaction #org.daisy.pipeline.ws.localfs=true #org.daisy.pipeline.ws.authentication=false diff --git a/src/test/resources/test-docker-image.sh b/src/test/resources/test-docker-image.sh index 1da24e29..a4425a28 100755 --- a/src/test/resources/test-docker-image.sh +++ b/src/test/resources/test-docker-image.sh @@ -17,10 +17,10 @@ zip $DATA $DOCUMENT # run the pipeline docker run --name pipeline --rm --detach \ - -e PIPELINE2_HOST=0.0.0.0 \ + -e PIPELINE2_WS_HOST=0.0.0.0 \ -e PIPELINE2_AUTH=true \ - -e PIPELINE2_AUTH_CLIENTKEY=$CLIENTKEY \ - -e PIPELINE2_AUTH_CLIENTSECRET=$CLIENTSECRET \ + -e PIPELINE2_WS_AUTHENTICATION_KEY=$CLIENTKEY \ + -e PIPELINE2_WS_AUTHENTICATION_SECRET=$CLIENTSECRET \ -p 8181:8181 daisyorg/pipeline2 # wait for the pipeline to start From 7f8f8c13bf30234a0529a0b62173391666fa1388 Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Tue, 10 Oct 2017 12:17:41 +0200 Subject: [PATCH 12/18] Add a second Dockerfile that does not build the Pipeline inside Docker Also add a Makefile --- Dockerfile.without_builder | 9 +++++++++ Makefile | 6 ++++++ 2 files changed, 15 insertions(+) create mode 100644 Dockerfile.without_builder create mode 100644 Makefile diff --git a/Dockerfile.without_builder b/Dockerfile.without_builder new file mode 100644 index 00000000..fcd6d952 --- /dev/null +++ b/Dockerfile.without_builder @@ -0,0 +1,9 @@ +FROM openjdk:8-jre +LABEL maintainer="DAISY Consortium (http://www.daisy.org/)" +ADD target/pipeline2-*_linux/daisy-pipeline /opt/daisy-pipeline2 +ENV PIPELINE2_LOCAL=false \ + PIPELINE2_AUTH=true \ + PIPELINE2_WS_AUTHENTICATION_KEY=clientid \ + PIPELINE2_WS_AUTHENTICATION_SECRET=sekret +EXPOSE 8181 +ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e2d17960 --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +docker : + gtar -cz \ + target/pipeline2-*_linux \ + Dockerfile.without_builder \ + --transform='s/Dockerfile.without_builder/Dockerfile/' \ + | docker build -t daisyorg/pipeline2 - From d8fc1e30e355bc72583cfa789fb80757b438025c Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Tue, 10 Oct 2017 12:16:48 +0200 Subject: [PATCH 13/18] Fix Docker test and add it to the Makefile --- Makefile | 9 ++++++ src/test/resources/test-docker-image.sh | 38 ++++++++++++++++--------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index e2d17960..ab7b531c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,15 @@ +help : + echo "make docker:" >&2 + echo " Build a Docker image" >&2 + echo "make check-docker:" >&2 + echo " Test the Docker image" >&2 + docker : gtar -cz \ target/pipeline2-*_linux \ Dockerfile.without_builder \ --transform='s/Dockerfile.without_builder/Dockerfile/' \ | docker build -t daisyorg/pipeline2 - + +check-docker : + bash src/test/resources/test-docker-image.sh diff --git a/src/test/resources/test-docker-image.sh b/src/test/resources/test-docker-image.sh index a4425a28..80308f66 100755 --- a/src/test/resources/test-docker-image.sh +++ b/src/test/resources/test-docker-image.sh @@ -1,22 +1,34 @@ #!/usr/bin/env bash # This test starts the pipeline inside a Docker container and then -# starts the dtbook-to-epub script using the cli inside a second +# starts the dtbook-to-epub3 script using the cli inside a second # container -# take any old DTBook file for testing for example from -# https://github.com/daisy/pipeline-samples -DOCUMENT=test.xml -DATA_DIR=$(pwd) -DATA=$(basename $DOCUMENT).zip +set -e +cd $(dirname "$0") + +# download test DTBook from https://github.com/daisy/pipeline-samples +DATA_DIR=../../../target/test-docker/data +mkdir -p $DATA_DIR +cd $DATA_DIR +DTBOOK=hauy_valid.xml +DATA=$(basename $DTBOOK).zip +rm -f $DATA +for f in \ + $DTBOOK \ + dtbook.2005.basic.css \ + valentin.jpg +do + curl https://raw.githubusercontent.com/daisy/pipeline-samples/10bbb8e/dtbook/$f 2>/dev/null >$f + zip $DATA $f +done + CLIENTKEY=clientid CLIENTSECRET=sekret MOUNT_POINT=/mnt -zip $DATA $DOCUMENT - # run the pipeline -docker run --name pipeline --rm --detach \ +docker run --name pipeline --detach \ -e PIPELINE2_WS_HOST=0.0.0.0 \ -e PIPELINE2_AUTH=true \ -e PIPELINE2_WS_AUTHENTICATION_KEY=$CLIENTKEY \ @@ -25,7 +37,6 @@ docker run --name pipeline --rm --detach \ # wait for the pipeline to start sleep 5 - while ! curl localhost:8181/ws/alive >/dev/null 2>/dev/null; do echo "Waiting for web service to be up..." >&2 sleep 2 @@ -34,15 +45,16 @@ done # run the cli docker run --name cli --rm -it --link pipeline \ --entrypoint /opt/daisy-pipeline2/cli/dp2 \ - --volume="$DATA_DIR:$MOUNT_POINT:rw" \ + --volume="$(pwd):$MOUNT_POINT:rw" \ daisyorg/pipeline2 \ --host http://pipeline \ --starting false \ --client_key $CLIENTKEY \ --client_secret $CLIENTSECRET \ - dtbook-to-epub3 --source $DOCUMENT --output $MOUNT_POINT --data $MOUNT_POINT/$DATA --persistent + dtbook-to-epub3 --source $DTBOOK --output $MOUNT_POINT --data $MOUNT_POINT/$DATA --persistent # help dtbook-to-epub3 -# dtbook-validator --input-dtbook $DOCUMENT --output $MOUNT_POINT --data $MOUNT_POINT/$DATA +# dtbook-validator --input-dtbook $DTBOOK --output $MOUNT_POINT --data $MOUNT_POINT/$DATA docker stop pipeline +docker rm pipeline From 770eec24eee407e2ed25e48e10ce10148c558df4 Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Thu, 19 Oct 2017 19:47:03 +0200 Subject: [PATCH 14/18] Simplify start script by using environment variables instead of system properties - PIPELINE2_HOME - PIPELINE2_BASE - PIPELINE2_DATA - PIPELINE2_WS_LOCALFS - PIPELINE2_WS_AUTHENTICATION *nix only. --- src/main/resources/bin/pipeline2 | 33 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/main/resources/bin/pipeline2 b/src/main/resources/bin/pipeline2 index c1c07005..73ff962b 100755 --- a/src/main/resources/bin/pipeline2 +++ b/src/main/resources/bin/pipeline2 @@ -354,24 +354,11 @@ init() { } run() { - if [ "x$PIPELINE2_AUTH" = "x" ]; then - PIPELINE2_AUTH="-Dorg.daisy.pipeline.ws.authentication=false" - export PIPELINE2_AUTH - else - PIPELINE2_AUTH="-Dorg.daisy.pipeline.ws.authentication=$PIPELINE2_AUTH" - export PIPELINE2_AUTH - - fi - if [ "x$PIPELINE2_LOCAL" = "x" ]; then - PIPELINE2_LOCAL="-Dorg.daisy.pipeline.ws.localfs=true" - export PIPELINE2_LOCAL - else - PIPELINE2_LOCAL="-Dorg.daisy.pipeline.ws.localfs=$PIPELINE2_LOCAL" - export PIPELINE2_LOCAL + # set defaults + PIPELINE2_WS_LOCALFS=true + PIPELINE2_WS_AUTHENTICATION=false - fi MAIN=org.apache.felix.main.Main - OPTS="$PIPELINE2_LOCAL $PIPELINE2_AUTH" MODE=" -Dorg.daisy.pipeline.main.mode=webservice" case "$1" in @@ -383,11 +370,13 @@ run() { # shift # ;; 'remote') - OPTS="-Dorg.daisy.pipeline.ws.localfs=false -Dorg.daisy.pipeline.ws.authentication=true" + PIPELINE2_LOCALFS=false + PIPELINE2_AUTH=true shift ;; 'local') - OPTS="-Dorg.daisy.pipeline.ws.localfs=true -Dorg.daisy.pipeline.ws.authentication=false" + PIPELINE2_LOCALFS=true + PIPELINE2_AUTH=false shift ;; 'clean') @@ -437,7 +426,13 @@ run() { fi cd "$PIPELINE2_BASE" - eval exec "\"$JAVA"\" "$JAVA_OPTS" -Djava.endorsed.dirs="\"${JAVA_ENDORSED_DIRS}"\" -Djava.ext.dirs="\"${JAVA_EXT_DIRS}"\" -Dorg.daisy.pipeline.home="\"$PIPELINE2_HOME"\" -Dorg.daisy.pipeline.base="\"$PIPELINE2_BASE"\" -Dorg.daisy.pipeline.data="\"$PIPELINE2_DATA"\" -Dfelix.config.properties="\"file:$PIPELINE2_HOME/etc/config.properties"\" -Dfelix.system.properties="\"file:$PIPELINE2_CONFIG/system.properties"\" $FELIX_OPTS $PIPELINE2_OPTS $OPTS $MODE -classpath "\"$CLASSPATH"\" $MAIN "\"$@"\" + export PIPELINE2_HOME + export PIPELINE2_BASE + export PIPELINE2_DATA + export PIPELINE2_WS_LOCALFS + export PIPELINE2_WS_AUTHENTICATION + + eval exec "\"$JAVA"\" "$JAVA_OPTS" -Djava.endorsed.dirs="\"${JAVA_ENDORSED_DIRS}"\" -Djava.ext.dirs="\"${JAVA_EXT_DIRS}"\" -Dfelix.config.properties="\"file:$PIPELINE2_HOME/etc/config.properties"\" -Dfelix.system.properties="\"file:$PIPELINE2_CONFIG/system.properties"\" $FELIX_OPTS $PIPELINE2_OPTS $MODE -classpath "\"$CLASSPATH"\" $MAIN "\"$@"\" } main() { From c3baadb97625e435ffc19cb4540fbe5f88f30d8a Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Thu, 19 Oct 2017 19:48:58 +0200 Subject: [PATCH 15/18] Remove PIPELINE2_OPTS environemnt variable because you can now directly specify the Pipeline properties through environment variables. Note that this will only work for system properties that start with "org.daisy.pipeline" though. --- src/main/resources/bin/pipeline2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/bin/pipeline2 b/src/main/resources/bin/pipeline2 index 73ff962b..cccaf878 100755 --- a/src/main/resources/bin/pipeline2 +++ b/src/main/resources/bin/pipeline2 @@ -432,7 +432,7 @@ run() { export PIPELINE2_WS_LOCALFS export PIPELINE2_WS_AUTHENTICATION - eval exec "\"$JAVA"\" "$JAVA_OPTS" -Djava.endorsed.dirs="\"${JAVA_ENDORSED_DIRS}"\" -Djava.ext.dirs="\"${JAVA_EXT_DIRS}"\" -Dfelix.config.properties="\"file:$PIPELINE2_HOME/etc/config.properties"\" -Dfelix.system.properties="\"file:$PIPELINE2_CONFIG/system.properties"\" $FELIX_OPTS $PIPELINE2_OPTS $MODE -classpath "\"$CLASSPATH"\" $MAIN "\"$@"\" + eval exec "\"$JAVA"\" "$JAVA_OPTS" -Djava.endorsed.dirs="\"${JAVA_ENDORSED_DIRS}"\" -Djava.ext.dirs="\"${JAVA_EXT_DIRS}"\" -Dfelix.config.properties="\"file:$PIPELINE2_HOME/etc/config.properties"\" -Dfelix.system.properties="\"file:$PIPELINE2_CONFIG/system.properties"\" $FELIX_OPTS $MODE -classpath "\"$CLASSPATH"\" $MAIN "\"$@"\" } main() { From c194f2e05e3eccf0e0f55976b95e691f56009492 Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Thu, 19 Oct 2017 19:51:49 +0200 Subject: [PATCH 16/18] Debian package: remove "REMOTE" variable from /etc/default Instead use the PIPELINE2_WS_LOCALFS and PIPELINE2_WS_AUTHENTICATION environment variables directly. --- src/main/deb/etc/default/daisy-pipeline2 | 5 +++-- src/main/deb/etc/init.d/daisy-pipeline2 | 5 ----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/deb/etc/default/daisy-pipeline2 b/src/main/deb/etc/default/daisy-pipeline2 index 7a37b001..b40d2fe5 100644 --- a/src/main/deb/etc/default/daisy-pipeline2 +++ b/src/main/deb/etc/default/daisy-pipeline2 @@ -1,7 +1,8 @@ # Configuration file for /etc/init.d/daisy-pipeline2 -# Execute pipeline in remote mode -#REMOTE=true +# Execute pipeline in remote mode and with authentication +#export PIPELINE2_WS_LOCALFS=true +#export PIPELINE2_WS_AUTHENTICATION=false # Java heap size #export JAVA_MIN_MEM=256M diff --git a/src/main/deb/etc/init.d/daisy-pipeline2 b/src/main/deb/etc/init.d/daisy-pipeline2 index 4f69c9b0..da3f31fd 100755 --- a/src/main/deb/etc/init.d/daisy-pipeline2 +++ b/src/main/deb/etc/init.d/daisy-pipeline2 @@ -27,11 +27,6 @@ SCRIPTNAME=/etc/init.d/$NAME # Read configuration variable file if it is present [ -r /etc/default/$NAME ] && . /etc/default/$NAME -# Set daemon argument list -if [ "$REMOTE" = "true" ]; then - DAEMON_ARGS="remote" -fi - # Load the VERBOSE setting and other rcS variables . /lib/init/vars.sh From c445d2148ba6b150e6afaaf969775f1bea7cdf14 Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Thu, 19 Oct 2017 20:04:32 +0200 Subject: [PATCH 17/18] Remove the webui profile see https://github.com/daisy/pipeline-assembly/issues/137 --- pom.xml | 96 -------------------------- src/main/assembly/components/webui.xml | 31 --------- src/main/assembly/webui-linux.xml | 16 ----- src/main/assembly/webui-mac.xml | 16 ----- src/main/assembly/webui-win.xml | 16 ----- 5 files changed, 175 deletions(-) delete mode 100644 src/main/assembly/components/webui.xml delete mode 100644 src/main/assembly/webui-linux.xml delete mode 100644 src/main/assembly/webui-mac.xml delete mode 100644 src/main/assembly/webui-win.xml diff --git a/pom.xml b/pom.xml index 4e0834fa..14f1f47d 100644 --- a/pom.xml +++ b/pom.xml @@ -2821,102 +2821,6 @@ rm -rf /opt/daisy-pipeline2/cli - - webui - - - - maven-dependency-plugin - - - unpack - package - - unpack - - - - - org.daisy.pipeline - webui-desktop - 1.8.1 - zip - true - target/webui - - - false - true - - - - - - maven-assembly-plugin - - - dist-webui-mac - package - - single - - - false - - src/main/assembly/webui-mac.xml - - - src/main/assembly/dist.properties - src/main/assembly/dist-nix.properties - - false - pipeline2-${project.version}-webui_mac - - - - dist-webui-linux - package - - single - - - false - - src/main/assembly/webui-linux.xml - - - src/main/assembly/dist.properties - src/main/assembly/dist-nix.properties - - false - pipeline2-${project.version}-webui_linux - - - - dist-webui-win - package - - single - - - false - - src/main/assembly/webui-win.xml - - - src/main/assembly/dist.properties - src/main/assembly/dist-win.properties - - false - pipeline2-${project.version}-webui_windows - - - - - - - - - - target/webui/daisy-pipeline-webui - - uninstall - uninstall.bat - start - start.bat - - /webui - - - - - target/webui/daisy-pipeline-webui - - uninstall - uninstall.bat - start - start.bat - - 0755 - /webui - - - - diff --git a/src/main/assembly/webui-linux.xml b/src/main/assembly/webui-linux.xml deleted file mode 100644 index e2be257c..00000000 --- a/src/main/assembly/webui-linux.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - webui-linux - - zip - dir - - daisy-pipeline - - src/main/assembly/components/base.xml - src/main/assembly/components/framework.xml - src/main/assembly/components/cli-linux.xml - src/main/assembly/components/modules-linux.xml - src/main/assembly/components/webui.xml - - diff --git a/src/main/assembly/webui-mac.xml b/src/main/assembly/webui-mac.xml deleted file mode 100644 index 0756f958..00000000 --- a/src/main/assembly/webui-mac.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - webui-mac - - zip - dir - - daisy-pipeline - - src/main/assembly/components/base.xml - src/main/assembly/components/framework.xml - src/main/assembly/components/cli-mac.xml - src/main/assembly/components/modules-mac.xml - src/main/assembly/components/webui.xml - - diff --git a/src/main/assembly/webui-win.xml b/src/main/assembly/webui-win.xml deleted file mode 100644 index ce91c019..00000000 --- a/src/main/assembly/webui-win.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - webui-win - - zip - dir - - daisy-pipeline - - src/main/assembly/components/base.xml - src/main/assembly/components/framework.xml - src/main/assembly/components/cli-win.xml - src/main/assembly/components/modules-win.xml - src/main/assembly/components/webui.xml - - From c946f3631207721f3c0a840cf6a40680c7f65b19 Mon Sep 17 00:00:00 2001 From: Christian Egli Date: Thu, 26 Oct 2017 14:26:42 +0200 Subject: [PATCH 18/18] Add a health check to the docker image See https://docs.docker.com/engine/reference/builder/#healthcheck --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0edbdb19..ad970576 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,4 +17,6 @@ ENV PIPELINE2_LOCAL=false \ PIPELINE2_WS_AUTHENTICATION_KEY=clientid \ PIPELINE2_WS_AUTHENTICATION_SECRET=sekret EXPOSE 8181 +# for the healthcheck use PIPELINE2_HOST if defined. Otherwise use localhost +HEALTHCHECK --interval=30s --timeout=10s --start-period=1m CMD curl --fail http://${PIPELINE2_WS_HOST-localhost}:${PIPELINE2_WS_PORT:-8181}/${PIPELINE2_WS_PATH:-ws}/alive || exit 1 ENTRYPOINT ["/opt/daisy-pipeline2/bin/pipeline2"]