diff --git a/.travis.yml b/.travis.yml index ff7e154e74..6e16c0fde0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,5 +26,5 @@ addons: - dev script: - - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar --quiet + - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent package --quiet diff --git a/README.md b/README.md index 2791d53685..3cbffc25fa 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,9 @@ [![Docker pulls](https://img.shields.io/docker/pulls/streampipes/backend.svg)](https://hub.docker.com/r/streampipes/backend/) [![Maven central](https://img.shields.io/maven-central/v/org.streampipes/streampipes-backend.svg)](https://img.shields.io/maven-central/v/org.streampipes/streampipes-backend.svg) [![License](https://img.shields.io/github/license/streampipes/streampipes-ce.svg)](https://docs.streampipes.org/license/) -[![License](https://img.shields.io/github/last-commit/streampipes/streampipes-ce.svg)]() +[![Last commit](https://img.shields.io/github/last-commit/streampipes/streampipes-ce.svg)]() +[![Twitter](https://img.shields.io/twitter/follow/StreamPipes.svg?label=Follow&style=social)](https://twitter.com/StreamPipes) +


org.streampipes streampipes-parent - 0.64.0 + 0.65.0 ../../pom.xml streampipes-archetype-pe-processors-flink diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/development/env b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/development/env index 46e54e2b7a..5f42b9e419 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/development/env +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/development/env @@ -1,5 +1,5 @@ # Those parameters are used by IntelliJ to set the default consul parameters for development SP_PORT=8005 -SP_HOST=localhost -SP_ICON_HOST=localhost +SP_HOST=host.docker.internal +SP_DEBUG=true SP_FLINK_DEBUG=true \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml index a389bcc629..2e06c9c628 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.64.0 + 0.65.0 @@ -81,7 +81,7 @@ streampipes-config ${sp.version} - + org.streampipes streampipes-dataformat-json ${sp.version} @@ -134,6 +134,13 @@ org.apache.maven.plugins maven-shade-plugin 2.3 + + + org.springframework.boot + spring-boot-maven-plugin + 2.2.0.RELEASE + + package @@ -144,11 +151,27 @@ false - ${package}.main.Init + implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> + META-INF/spring.handlers + + + META-INF/spring.factories + + + META-INF/spring.schemas + + reference.conf + + + ${package}.main.Init + diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml b/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml index 22eb743804..9e9c195349 100644 --- a/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml @@ -21,7 +21,7 @@ org.streampipes streampipes-parent - 0.64.0 + 0.65.0 ../../pom.xml streampipes-archetype-pe-processors-jvm diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/development/env b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/development/env index f620d5b591..d7d20eea2b 100644 --- a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/development/env +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/development/env @@ -1,4 +1,4 @@ # Those parameters are used by IntelliJ to set the default consul parameters for development SP_PORT=8005 -SP_HOST=localhost -SP_ICON_HOST=localhost \ No newline at end of file +SP_HOST=host.docker.internal +SP_DEBUG=true \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml index 58d06d425e..b380174605 100644 --- a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.64.0 + 0.65.0 @@ -81,39 +81,15 @@ - org.apache.maven.plugins - maven-shade-plugin - 2.3 + org.springframework.boot + spring-boot-maven-plugin - package - shade + repackage - false - - - ${package}.main.Init - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml - META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml - - - - + ${package}.main.Init diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml index 8fab5ee02a..54416d0f25 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml @@ -21,7 +21,7 @@ org.streampipes streampipes-parent - 0.64.0 + 0.65.0 ../../pom.xml streampipes-archetype-pe-sinks-flink diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/development/env b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/development/env index 46e54e2b7a..5f42b9e419 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/development/env +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/development/env @@ -1,5 +1,5 @@ # Those parameters are used by IntelliJ to set the default consul parameters for development SP_PORT=8005 -SP_HOST=localhost -SP_ICON_HOST=localhost +SP_HOST=host.docker.internal +SP_DEBUG=true SP_FLINK_DEBUG=true \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml index 3b50477a06..2e06c9c628 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.64.0 + 0.65.0 @@ -134,6 +134,13 @@ org.apache.maven.plugins maven-shade-plugin 2.3 + + + org.springframework.boot + spring-boot-maven-plugin + 2.2.0.RELEASE + + package @@ -144,11 +151,27 @@ false - ${package}.main.Init + implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> + META-INF/spring.handlers + + + META-INF/spring.factories + + + META-INF/spring.schemas + + reference.conf + + + ${package}.main.Init + diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml b/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml index f297b5962d..36a7f9633f 100644 --- a/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml @@ -21,7 +21,7 @@ org.streampipes streampipes-parent - 0.64.0 + 0.65.0 ../../pom.xml streampipes-archetype-pe-sinks-jvm diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/development/env b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/development/env index f620d5b591..d7d20eea2b 100644 --- a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/development/env +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/development/env @@ -1,4 +1,4 @@ # Those parameters are used by IntelliJ to set the default consul parameters for development SP_PORT=8005 -SP_HOST=localhost -SP_ICON_HOST=localhost \ No newline at end of file +SP_HOST=host.docker.internal +SP_DEBUG=true \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml index 58d06d425e..b380174605 100644 --- a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.64.0 + 0.65.0 @@ -81,39 +81,15 @@ - org.apache.maven.plugins - maven-shade-plugin - 2.3 + org.springframework.boot + spring-boot-maven-plugin - package - shade + repackage - false - - - ${package}.main.Init - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml - META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml - - - - + ${package}.main.Init diff --git a/archetypes/streampipes-archetype-pe-sources/pom.xml b/archetypes/streampipes-archetype-pe-sources/pom.xml index 53cccece56..5895a3150f 100644 --- a/archetypes/streampipes-archetype-pe-sources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sources/pom.xml @@ -21,7 +21,7 @@ org.streampipes streampipes-parent - 0.64.0 + 0.65.0 ../../pom.xml streampipes-archetype-pe-sources diff --git a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/development/env b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/development/env index 46e54e2b7a..5f42b9e419 100644 --- a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/development/env +++ b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/development/env @@ -1,5 +1,5 @@ # Those parameters are used by IntelliJ to set the default consul parameters for development SP_PORT=8005 -SP_HOST=localhost -SP_ICON_HOST=localhost +SP_HOST=host.docker.internal +SP_DEBUG=true SP_FLINK_DEBUG=true \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml index fbe5ede0a6..7a33de489a 100644 --- a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/pom.xml @@ -7,7 +7,7 @@ ${version} - 0.64.0 + 0.65.0 @@ -91,39 +91,15 @@ - org.apache.maven.plugins - maven-shade-plugin - 2.3 + org.springframework.boot + spring-boot-maven-plugin - package - shade + repackage - false - - - ${package}.main.Init - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml - META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml - - - - + ${package}.main.Init diff --git a/pom.xml b/pom.xml index 8aa0cb638a..7faaa01127 100644 --- a/pom.xml +++ b/pom.xml @@ -21,40 +21,81 @@ org.streampipes streampipes-parent - 0.64.0 + 0.65.0 pom UTF-8 false + 5.15.9 + 7.0 + 1.9.7 + 1.9.12 1.9.2 - 1.3.5 + 3.2.2 + 2.6 + 2.3 + 3.8.1 + 1.2 + 1.3.9 + 6.6.2 + 1.9.1 + 2.56 + 1.8 + 27.1-jre 4.5.5 - 2.9.9 - 2.1 + 4.4.9 + 2.16 + 2.10.0 + 1.3.5 + 2.0.2 + 1.6.1 + 1.3.2 + 1.0.2 + 1.0.2 4.0.1 2.0.1.Final + 2.1 3.19.0-GA - 2.27 + 3.4.0.Final + 2.29.1 9.4.19.v20190610 + 1.3.1 + 3.1.0 + 3.0.2 2.2.0 + 0.2.0 + 2.12.1 1.2.3 - 1.0.2 + 2.2 + 2.5.1 + 1.16.0 + 3.12.2 2.4.5 + 1.8 + 2.5.0 1.2.3 - 2.1.8.RELEASE + 4.5.11 + 1.7.25 + 1.1.7.2 + 2.1.2 + 5.2.0.RELEASE + 2.2.0.RELEASE 1.9.11 - 2.0.0 0.6.0 + 1.47 + 4.12 - 2.18.3 - 2.0.2 - 2.23.2 + 3.1.0 + 2.0.4 + 2.9.0 + 2.25.1 + 3.8.1 - 3.1.2 3.1.1 + 3.1.2 3.1.0 @@ -70,28 +111,38 @@ ${logback-classic.version} - javax.validation - validation-api - ${javax-validation.version} + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} - javax.ws.rs - javax.ws.rs-api - ${javax.ws.rs.version} + com.fasterxml.jackson.core + jackson-core + ${jackson.version} com.fasterxml.jackson.core jackson-databind ${jackson.version} + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + ${jackson.version} + com.fasterxml.jackson.dataformat jackson-dataformat-xml ${jackson.version} - com.fasterxml.jackson.core - jackson-core + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 ${jackson.version} @@ -100,39 +151,109 @@ ${type-parser.version} - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} + com.google.code.findbugs + jsr305 + ${jsr305.version} com.google.code.gson gson 2.8.5 + + com.google.guava + guava + ${guava.version} + com.orbitz.consul consul-client ${consul-client.version} + + com.squareup.okio + okio + ${okio.version} + + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + + com.squareup.retrofit2 + retrofit + ${retrofit.version} + commons-beanutils commons-beanutils ${commons-beanutils.version} - io.swagger.core.v3 - swagger-jaxrs2 - ${swagger.version} + commons-collections + commons-collections + ${commons-collections.version} + + + commons-io + commons-io + ${commons-io.version} + + + commons-lang + commons-lang + ${commons-lang.version} + + + commons-logging + commons-logging + ${commons-logging.version} + + + com.github.javadev + underscore + ${underscore.version} - io.swagger.core.v3 - swagger-jaxrs2-servlet-initializer - ${swagger.version} + com.squareup + javapoet + ${javapoet.version} + + + de.grundid.opendatalab + geojson-jackson + ${geojson-jackson.version} + + + de.ruedigermoeller + fst + ${fst.version} + + + io.rest-assured + json-path + ${json-path.version} + + + jakarta.annotation + jakarta.annotation-api + ${jakarta-annotation.version} + + + jakarta.validation + jakarta.validation-api + ${jakarta-validation.version} + + + javax.annotation + javax.annotation-api + ${javax-annotation.version} javax.persistence persistence-api - ${persistence-api.version} + ${javax-persistence-api.version} javax.servlet @@ -140,46 +261,44 @@ ${javax-servlet-api.version} - org.glassfish.jersey.containers - jersey-container-jetty-http - ${jersey.version} + javax.validation + validation-api + ${javax-validation.version} - org.glassfish.jersey.containers - jersey-container-servlet - ${jersey.version} + javax.ws.rs + javax.ws.rs-api + ${javax.ws.rs.version} - org.glassfish.jersey.inject - jersey-hk2 - ${jersey.version} + org.apache.httpcomponents + fluent-hc + ${httpclient.version} - org.glassfish.jersey.inject - jersey-hk2 - ${jersey.version} + org.apache.httpcomponents + httpmime + ${httpclient.version} - org.glassfish.jersey.media - jersey-media-json-jackson - ${jersey.version} + org.apache.httpcomponents + httpclient + ${httpclient.version} - org.glassfish.jersey.media - jersey-media-multipart - ${jersey.version} + org.apache.httpcomponents + httpcore + ${httpcore.version} - org.glassfish.jersey.media - jersey-media-sse - ${jersey.version} + org.apache.logging.log4j + log4j-api + ${log4j.version} - - junit - junit - ${junit.version} - test + org.apache.maven.shared + maven-invoker + ${maven-invoker.version} org.apache.kafka @@ -191,6 +310,36 @@ kafka-clients ${kafka.version} + + org.apache.kafka + kafka-streams + ${kafka.version} + + + org.apache.activemq + activemq-client + ${activemq-client.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + org.apache.flink + flink-connector-kafka_2.11 + ${flink.version} + + + org.apache.flink + flink-java + ${flink.version} + + + org.apache.flink + flink-streaming-java_2.11 + ${flink.version} + org.apache.shiro shiro-core @@ -202,14 +351,49 @@ ${shiro.version} - org.apache.httpcomponents - fluent-hc - ${httpclient.version} + org.apache.spark + spark-core_2.11 + ${spark.version} - org.apache.httpcomponents - httpmime - ${httpclient.version} + org.apache.spark + spark-streaming_2.11 + ${spark.version} + + + org.apache.spark + spark-streaming-kafka-0-10_2.11 + ${spark.version} + + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + org.eclipse.jetty + jetty-servlets + ${jetty.version} + + + org.eclipse.jetty + jetty-webapp + ${jetty.version} + + + org.eclipse.jetty + jetty-util + ${jetty.version} + + + org.eclipse.jetty + jetty-continuation + ${jetty.version} org.eclipse.rdf4j @@ -223,313 +407,303 @@ org.eclipse.rdf4j - rdf4j-rio-api + rdf4j-repository-http ${rdf4j.version} org.eclipse.rdf4j - rdf4j-model + rdf4j-repository-manager ${rdf4j.version} org.eclipse.rdf4j - rdf4j-rio-datatypes + rdf4j-sail-memory ${rdf4j.version} org.eclipse.rdf4j - rdf4j-rio-turtle + rdf4j-sail-nativerdf ${rdf4j.version} org.eclipse.rdf4j - rdf4j-rio-rdfxml + rdf4j-rio-api ${rdf4j.version} org.eclipse.rdf4j - rdf4j-rio-jsonld + rdf4j-model ${rdf4j.version} org.eclipse.rdf4j - rdf4j-repository-http + rdf4j-rio-datatypes ${rdf4j.version} org.eclipse.rdf4j - rdf4j-sail-memory + rdf4j-rio-turtle ${rdf4j.version} org.eclipse.rdf4j - rdf4j-queryrender + rdf4j-rio-rdfxml ${rdf4j.version} org.eclipse.rdf4j - rdf4j-repository-sail + rdf4j-rio-jsonld ${rdf4j.version} org.eclipse.rdf4j - rdf4j-repository-manager + rdf4j-queryrender ${rdf4j.version} org.eclipse.rdf4j - rdf4j-queryresultio-sparqljson + rdf4j-repository-sail ${rdf4j.version} - org.javassist - javassist - ${javassist.version} - - - org.reflections - reflections - 0.9.10 - - - org.springframework.boot - spring-boot-starter-jersey - ${spring.version} - - - org.springframework.boot - spring-boot-starter-jetty - ${spring.version} + org.eclipse.rdf4j + rdf4j-queryresultio-sparqljson + ${rdf4j.version} - org.springframework.boot - spring-boot-starter-web - ${spring.version} + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} - org.streampipes - streampipes-config - ${project.version} + org.jboss.logging + jboss-logging + ${jboss-logging.version} - org.streampipes - streampipes-connect - ${project.version} + org.jgrapht + jgrapht-core + ${jgrapht.version} - org.streampipes - streampipes-connect-container - ${project.version} + org.glassfish + javax.json + ${javax-json.version} - org.streampipes - streampipes-container - ${project.version} + org.glassfish.jersey.core + jersey-client + ${jersey.version} - org.streampipes - streampipes-container-embedded - ${project.version} + org.glassfish.jersey.core + jersey-common + ${jersey.version} - org.streampipes - streampipes-container-standalone - ${project.version} + org.glassfish.jersey.core + jersey-server + ${jersey.version} - org.streampipes - streampipes-code-generation - ${project.version} + org.glassfish.jersey.containers + jersey-container-jetty-http + ${jersey.version} - org.streampipes - streampipes-commons - ${project.version} + org.glassfish.jersey.containers + jersey-container-servlet + ${jersey.version} - org.streampipes - streampipes-dataformat - ${project.version} + org.glassfish.jersey.containers + jersey-container-servlet-core + ${jersey.version} - org.streampipes - streampipes-dataformat-cbor - ${project.version} + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} - org.streampipes - streampipes-dataformat-fst - ${project.version} + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} - org.streampipes - streampipes-dataformat-json - ${project.version} + org.glassfish.jersey.media + jersey-media-json-jackson + ${jersey.version} - org.streampipes - streampipes-dataformat-smile - ${project.version} + org.glassfish.jersey.media + jersey-media-multipart + ${jersey.version} - org.streampipes - streampipes-pipeline-management - ${project.version} + org.glassfish.jersey.media + jersey-media-sse + ${jersey.version} - org.streampipes - streampipes-model - ${project.version} + org.influxdb + influxdb-java + ${influxdb.version} - org.streampipes - streampipes-model-client - ${project.version} + org.javassist + javassist + ${javassist.version} - org.streampipes - streampipes-wrapper-flink - ${project.version} + org.lightcouch + lightcouch + ${lightcouch.version} - org.streampipes - streampipes-wrapper-spark - ${project.version} + org.objenesis + objenesis + ${objenesis.version} - org.streampipes - streampipes-sources - ${project.version} + org.ow2.asm + asm + ${asm.version} - org.streampipes - streampipes-performance-tests - ${project.version} + org.reflections + reflections + 0.9.10 - org.streampipes - streampipes-rest - ${project.version} + org.rendersnake + rendersnake + ${rendersnake.version} - org.streampipes - streampipes-wrapper - ${project.version} + org.slf4j + slf4j-api + ${slf4j.version} - org.streampipes - streampipes-wrapper-distributed - ${project.version} + org.springframework + spring-aop + ${spring.version} - org.streampipes - streampipes-wrapper-standalone - ${project.version} + org.springframework + spring-beans + ${spring.version} - org.streampipes - streampipes-wrapper-kafka-streams - ${project.version} + org.springframework + spring-context + ${spring.version} - org.streampipes - streampipes-backend - ${project.version} + org.springframework + spring-core + ${spring.version} - org.streampipes - streampipes-measurement-units - ${project.version} + org.springframework + spring-web + ${spring.version} - org.streampipes - streampipes-messaging - ${project.version} + org.springframework.boot + spring-boot-starter-jersey + ${spring-boot.version} - org.streampipes - streampipes-sdk - ${project.version} + org.springframework.boot + spring-boot-starter-jetty + ${spring-boot.version} - org.streampipes - streampipes-messaging-kafka - ${project.version} + org.springframework.boot + spring-boot-starter-undertow + ${spring-boot.version} - org.streampipes - streampipes-messaging-jms - ${project.version} + org.springframework.boot + spring-boot-starter-web + ${spring-boot.version} org.streampipes - streampipes-user-management - ${project.version} + streampipes-empire-annotations + ${streampipes-empire.version} org.streampipes - streampipes-app-file-export - ${project.version} + streampipes-empire-core + ${streampipes-empire.version} org.streampipes - streampipes-rest-shared - ${project.version} + streampipes-empire-rdf4j + ${streampipes-empire.version} org.streampipes - streampipes-serializers - ${project.version} + streampipes-empire-pinto + ${streampipes-empire.version} - org.streampipes - streampipes-storage-couchdb - ${project.version} + org.xerial.snappy + snappy-java + ${snappy-java.version} - org.streampipes - streampipes-storage-api - ${project.version} + uk.com.robust-it + cloning + ${cloning.version} - org.streampipes - streampipes-storage-management - ${project.version} + org.wso2.siddhi + siddhi-annotations + ${siddhi.version} - org.streampipes - streampipes-storage-rdf4j - ${project.version} + org.wso2.siddhi + siddhi-core + ${siddhi.version} - org.streampipes - streampipes-vocabulary - ${project.version} + org.wso2.siddhi + siddhi-query-api + ${siddhi.version} - org.streampipes - streampipes-empire-rdf4j - ${streampipes-empire.version} + org.wso2.siddhi + siddhi-query-compiler + ${siddhi.version} + + - org.streampipes - streampipes-empire-pinto - ${streampipes-empire.version} + com.github.tomakehurst + wiremock + ${wiremock.version} + test - org.streampipes - streampipes-logging - ${project.version} + junit + junit + ${junit.version} + test - org.streampipes - streampipes-container-base - ${project.version} + net.bytebuddy + byte-buddy + ${byte-buddy.version} + test - com.github.tomakehurst - wiremock - ${wiremock.version} + net.bytebuddy + byte-buddy-agent + ${byte-buddy.version} test @@ -553,82 +727,12 @@ com.jayway.restassured rest-assured - 2.3.2 + ${rest-assured.version} test - - org.eclipse.jetty - jetty-server - ${jetty.version} - - - org.eclipse.jetty - jetty-servlet - ${jetty.version} - - - org.eclipse.jetty - jetty-servlets - ${jetty.version} - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - - - org.eclipse.jetty - jetty-util - ${jetty.version} - - - org.eclipse.jetty - jetty-continuation - ${jetty.version} - - - - - junit - junit - ${junit.version} - test - - - com.github.tomakehurst - wiremock - 2.18.0 - test - - - org.mockito - mockito-core - ${mockito.version} - test - - - com.jayway.restassured - rest-assured - 2.3.2 - test - - - org.powermock - powermock-module-junit4 - ${powermock.version} - test - - - org.powermock - powermock-api-mockito2 - ${powermock.version} - test - - - archetypes/streampipes-archetype-pe-sources archetypes/streampipes-archetype-pe-sinks-jvm @@ -722,6 +826,8 @@ -Xdoclint:none + none + 8 @@ -779,7 +885,7 @@ org.springframework.boot spring-boot-maven-plugin - ${spring.version} + ${spring-boot.version} org.apache.rat @@ -792,6 +898,51 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M2 + + + check-at-least-java-8 + validate + + enforce + + + + + 1.8.0 + + + + + + + enforce-version-convergence + validate + + enforce + + + + + + + + + + + + org.codehaus.mojo + extra-enforcer-rules + 1.2 + + + org.apache.rat apache-rat-plugin @@ -812,6 +963,7 @@ **/.git/** **/.gitignore + **/.spignore **/UNLICENSE @@ -964,9 +1116,9 @@ https://oss.sonatype.org/service/local/staging/deploy/maven2 - deployment + ipe-nexus Internal Releases - https://laus.fzi.de/nexus/content/repositories/snapshots/ + https://ipe-nexus.fzi.de/repository/maven-snapshots/ diff --git a/streampipes-app-file-export/pom.xml b/streampipes-app-file-export/pom.xml index 9995c23a75..20c56d9fce 100644 --- a/streampipes-app-file-export/pom.xml +++ b/streampipes-app-file-export/pom.xml @@ -21,51 +21,52 @@ streampipes-parent org.streampipes - 0.64.0 + 0.65.0 + StreamPipes App File Export streampipes-app-file-export jar + - org.glassfish.jersey.containers - jersey-container-servlet + org.streampipes + streampipes-storage-couchdb + 0.65.0 + + - org.lightcouch - lightcouch - 0.1.8 + commons-logging + commons-logging - org.streampipes - streampipes-storage-couchdb + javax.ws.rs + javax.ws.rs-api + + + org.glassfish.jersey.containers + jersey-container-servlet org.apache.httpcomponents fluent-hc - com.jayway.restassured - rest-assured - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - 2.3.2 - test + org.apache.httpcomponents + httpcore org.elasticsearch.client elasticsearch-rest-high-level-client - 6.6.2 - + + org.lightcouch + lightcouch + + + + \ No newline at end of file diff --git a/streampipes-backend/development/env b/streampipes-backend/development/env index 5eeb333ff1..92d1eb76a4 100644 --- a/streampipes-backend/development/env +++ b/streampipes-backend/development/env @@ -15,15 +15,8 @@ # SP_COUCHDB_HOST=localhost -uri=http://localhost:8031/rdf4j-server -SP_KAFKA_HOST=localhost -SP_ZOOKEEPER_HOST=localhost -SP_JMS_HOST=localhost SP_KAFKA_REST_HOST=localhost SP_KAFKA_REST_PORT=8073 SP_BACKEND_HOST=localhost -SP_ELASTICSEARCH_HOST=localhost -SP_ASSETS_DIR=./assets -SP_DATALAKE_HOST=localhost -SP_DATALAKE_PORT=9200 SP_INFLUX_HOST=localhost +SP_INFLUX_PORT=8086 diff --git a/streampipes-backend/pom.xml b/streampipes-backend/pom.xml index e18d9a083b..d196ef50ad 100644 --- a/streampipes-backend/pom.xml +++ b/streampipes-backend/pom.xml @@ -20,20 +20,24 @@ org.streampipes streampipes-parent - 0.64.0 + 0.65.0 + streampipes-backend jar StreamPipes CE Backend + - org.springframework.boot - spring-boot-starter-web + org.streampipes + streampipes-app-file-export + 0.65.0 org.streampipes streampipes-container-base + 0.65.0 org.springframework.boot @@ -41,39 +45,33 @@ - - org.streampipes - streampipes-rest - org.streampipes streampipes-config + 0.65.0 org.streampipes streampipes-messaging-kafka + 0.65.0 org.streampipes - streampipes-app-file-export + streampipes-rest + 0.65.0 + + - io.swagger.core.v3 - swagger-jaxrs2 - 2.0.0 - - - javax.validation - validation-api - - + org.apache.logging.log4j + log4j-api - io.swagger.core.v3 - swagger-jaxrs2-servlet-initializer - 2.0.0 + org.springframework.boot + spring-boot-starter-web + diff --git a/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesBackendApplication.java b/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesBackendApplication.java index 370055ba51..afade215b9 100644 --- a/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesBackendApplication.java +++ b/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesBackendApplication.java @@ -15,7 +15,6 @@ */ package org.streampipes.backend; -import io.swagger.v3.jaxrs2.integration.OpenApiServlet; import org.apache.shiro.web.env.EnvironmentLoaderListener; import org.apache.shiro.web.servlet.OncePerRequestFilter; import org.apache.shiro.web.servlet.ShiroFilter; @@ -35,13 +34,10 @@ import org.streampipes.model.client.pipeline.PipelineOperationStatus; import org.streampipes.rest.notifications.NotificationListener; -import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.annotation.PreDestroy; import javax.servlet.ServletContextListener; -import javax.servlet.http.HttpServlet; @Configuration @EnableAutoConfiguration @@ -74,16 +70,6 @@ public ServletRegistrationBean appFileExportRegistrationBean() { return new ServletRegistrationBean<>(jerseyContainer, "/api/apps/*"); } - @Bean - public ServletRegistrationBean swaggerRegistrationBean() { - ServletRegistrationBean bean = new ServletRegistrationBean<>(new OpenApiServlet() - , "/api/docs/*"); - Map params = new HashMap<>(); - params.put("openApi.configuration.resourcePackages", "io.swagger.sample.resource"); - bean.setInitParameters(params); - return bean; - } - @Bean public FilterRegistrationBean shiroFilterBean() { FilterRegistrationBean bean = new FilterRegistrationBean<>(); diff --git a/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesResourceConfig.java b/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesResourceConfig.java index ced7cef16b..3b47e3c610 100644 --- a/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesResourceConfig.java +++ b/streampipes-backend/src/main/java/org/streampipes/backend/StreamPipesResourceConfig.java @@ -38,6 +38,7 @@ import org.streampipes.rest.impl.PipelineCategory; import org.streampipes.rest.impl.PipelineElementAsset; import org.streampipes.rest.impl.PipelineElementCategory; +import org.streampipes.rest.impl.PipelineElementFile; import org.streampipes.rest.impl.PipelineElementImport; import org.streampipes.rest.impl.PipelineElementRuntimeInfo; import org.streampipes.rest.impl.PipelineTemplate; @@ -53,8 +54,8 @@ import org.streampipes.rest.impl.VirtualSensor; import org.streampipes.rest.impl.Visualization; import org.streampipes.rest.impl.datalake.DataLakeNoUserResourceV3; -import org.streampipes.rest.impl.datalake.DataLakeResource; import org.streampipes.rest.impl.datalake.DataLakeResourceV3; +import org.streampipes.rest.impl.nouser.FileServingResource; import org.streampipes.rest.impl.nouser.PipelineElementImportNoUser; import org.streampipes.rest.impl.nouser.PipelineNoUserResource; import org.streampipes.rest.shared.serializer.GsonClientModelProvider; @@ -84,7 +85,6 @@ public StreamPipesResourceConfig() { register(PipelineNoUserResource.class); register(PipelineElementImportNoUser.class); register(PipelineCategory.class); - register(DataLakeResource.class); register(PipelineElementImport.class); register(SemanticEventConsumer.class); register(SemanticEventProcessingAgent.class); @@ -107,6 +107,8 @@ public StreamPipesResourceConfig() { register(PipelineElementAsset.class); register(DataLakeResourceV3.class); register(DataLakeNoUserResourceV3.class); + register(PipelineElementFile.class); + register(FileServingResource.class); // Serializers diff --git a/streampipes-backend/src/main/resources/shiro.ini b/streampipes-backend/src/main/resources/shiro.ini index 80ca512092..679932bd38 100644 --- a/streampipes-backend/src/main/resources/shiro.ini +++ b/streampipes-backend/src/main/resources/shiro.ini @@ -53,6 +53,7 @@ securityManager.rememberMeManager.cookie.maxAge = 1000000000 /api/v2/noauth/users/*/element/* = anon /api//v3/noauth/datalake/* = anon /api/v2/noauth/element/* = anon +/api/v2/noauth/files/* = anon /api/v2/kpis = anon /api/v2/kpis/* = anon /api/v2/admin/login/* = anon diff --git a/streampipes-code-generation/pom.xml b/streampipes-code-generation/pom.xml index 82cf163d3f..278688f612 100644 --- a/streampipes-code-generation/pom.xml +++ b/streampipes-code-generation/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.64.0 + 0.65.0 streampipes-code-generation @@ -34,28 +34,39 @@ + org.streampipes streampipes-container-standalone + 0.65.0 provided org.streampipes streampipes-model-client + 0.65.0 org.streampipes streampipes-sdk + 0.65.0 + + - junit - junit - test + com.google.code.gson + gson com.squareup javapoet - 1.6.1 + + + + + junit + junit + test diff --git a/streampipes-commons/pom.xml b/streampipes-commons/pom.xml index 21aa392a49..8adadf7039 100644 --- a/streampipes-commons/pom.xml +++ b/streampipes-commons/pom.xml @@ -16,37 +16,35 @@ --> - 4.0.0 - - org.streampipes - streampipes-parent - 0.64.0 - - streampipes-commons - StreamPipes Commons - - - commons-lang - commons-lang - 2.3 - - - ch.qos.logback - logback-classic - - - com.google.code.gson - gson - - - org.streampipes - streampipes-empire-rdf4j - - - tools - com.sun - - - - + 4.0.0 + + org.streampipes + streampipes-parent + 0.65.0 + + + streampipes-commons + StreamPipes Commons + + + + + + + commons-lang + commons-lang + + + ch.qos.logback + logback-classic + + + org.eclipse.rdf4j + rdf4j-rio-api + + + org.eclipse.rdf4j + rdf4j-model + + \ No newline at end of file diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/parser/BooleanParser.java b/streampipes-commons/src/main/java/org/streampipes/commons/parser/BooleanParser.java new file mode 100644 index 0000000000..067447d00a --- /dev/null +++ b/streampipes-commons/src/main/java/org/streampipes/commons/parser/BooleanParser.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.commons.parser; + +public class BooleanParser extends PrimitiveTypeParser { + + @Override + public Boolean parse(String value) { + return Boolean.parseBoolean(value); + } +} diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/parser/FloatParser.java b/streampipes-commons/src/main/java/org/streampipes/commons/parser/FloatParser.java new file mode 100644 index 0000000000..5880a29b05 --- /dev/null +++ b/streampipes-commons/src/main/java/org/streampipes/commons/parser/FloatParser.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.commons.parser; + +public class FloatParser extends PrimitiveTypeParser { + + @Override + public Float parse(String value) { + return Float.parseFloat(value); + } +} diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/parser/IntegerParser.java b/streampipes-commons/src/main/java/org/streampipes/commons/parser/IntegerParser.java new file mode 100644 index 0000000000..3614ca5386 --- /dev/null +++ b/streampipes-commons/src/main/java/org/streampipes/commons/parser/IntegerParser.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.commons.parser; + +public class IntegerParser extends PrimitiveTypeParser { + + @Override + public Integer parse(String value) { + return Integer.parseInt(value); + } +} diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/parser/PrimitiveTypeParser.java b/streampipes-commons/src/main/java/org/streampipes/commons/parser/PrimitiveTypeParser.java new file mode 100644 index 0000000000..2a70858bc6 --- /dev/null +++ b/streampipes-commons/src/main/java/org/streampipes/commons/parser/PrimitiveTypeParser.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.commons.parser; + +public abstract class PrimitiveTypeParser { + + public PrimitiveTypeParser() { + + } + + public abstract Object parse(String value); +} diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/parser/StringParser.java b/streampipes-commons/src/main/java/org/streampipes/commons/parser/StringParser.java new file mode 100644 index 0000000000..6a166c1467 --- /dev/null +++ b/streampipes-commons/src/main/java/org/streampipes/commons/parser/StringParser.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * you may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.commons.parser; + +public class StringParser extends PrimitiveTypeParser { + + @Override + public String parse(String value) { + return value; + } +} diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileExtractor.java b/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileExtractor.java index d81389899d..f00dde6345 100644 --- a/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileExtractor.java +++ b/streampipes-commons/src/main/java/org/streampipes/commons/zip/ZipFileExtractor.java @@ -33,7 +33,7 @@ public ZipFileExtractor(InputStream zipInputStream) { public void extractZipToFile(String targetFolder) throws IOException { File destDir = new File(targetFolder); if (!destDir.exists()) { - destDir.mkdir(); + destDir.mkdirs(); } byte[] buffer = new byte[1024]; ZipInputStream zis = new ZipInputStream(zipInputStream); diff --git a/streampipes-config/pom.xml b/streampipes-config/pom.xml index b017857069..a5cfc50bb0 100644 --- a/streampipes-config/pom.xml +++ b/streampipes-config/pom.xml @@ -20,25 +20,36 @@ streampipes-parent org.streampipes - 0.64.0 + 0.65.0 4.0.0 streampipes-config + org.streampipes streampipes-vocabulary + 0.65.0 + + - com.orbitz.consul - consul-client + com.google.code.findbugs + jsr305 com.google.code.gson gson + + com.google.guava + guava + + + com.orbitz.consul + consul-client + - \ No newline at end of file diff --git a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java index 9ef45bd67a..15549cf100 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java +++ b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java @@ -21,13 +21,13 @@ import org.apache.commons.lang3.RandomStringUtils; import org.streampipes.config.SpConfig; +import java.io.File; import java.security.SecureRandom; public enum BackendConfig { INSTANCE; - private final char[] possibleCharacters = (new String( - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~`!@#$%^&*()-_=+[{]}\\|;:\'\",<.>/?")).toCharArray(); + private final char[] possibleCharacters = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~`!@#$%^&*()-_=+[{]}\\|;:\'\",<.>/?").toCharArray(); private SpConfig config; BackendConfig() { @@ -36,7 +36,7 @@ public enum BackendConfig { config.register(BackendConfigKeys.SERVICE_NAME, "Backend", "Backend Configuration"); config.register(BackendConfigKeys.BACKEND_HOST, "backend", "Hostname for backend"); - config.register(BackendConfigKeys.BACKEND_PORT, 8082, "Port for backend"); + config.register(BackendConfigKeys.BACKEND_PORT, 8030, "Port for backend"); config.register(BackendConfigKeys.JMS_HOST, "activemq", "Hostname for backend service for active mq"); config.register(BackendConfigKeys.JMS_PORT, 61616, "Port for backend service for active mq"); @@ -52,8 +52,10 @@ public enum BackendConfig { config.register(BackendConfigKeys.KAFKA_REST_HOST, "kafka-rest", "The hostname of the kafka-rest module"); config.register(BackendConfigKeys.KAFKA_REST_PORT, 8082, "The port of the kafka-rest module"); config.register(BackendConfigKeys.KAFKA_REST_HOST, "kafka-rest", "The hostname of the kafka-rest module"); - config.register(BackendConfigKeys.ASSETS_DIR, "/streampipes-assets", "The directory where " + + config.register(BackendConfigKeys.ASSETS_DIR, makeAssetLocation(), "The directory where " + "pipeline element assets are stored."); + config.register(BackendConfigKeys.FILES_DIR, makeFileLocation(), "The directory where " + + "pipeline element files are stored."); config.register(BackendConfigKeys.DATA_LAKE_HOST, "elasticsearch", "The host of the data base used for the data lake"); config.register(BackendConfigKeys.DATA_LAKE_PORT, 9200, "The port of the data base used for the data lake"); @@ -66,6 +68,23 @@ public enum BackendConfig { config.register(BackendConfigKeys.ENCRYPTION_KEY, randomKey(), "A random secret key"); } + private String makeAssetLocation() { + return makeStreamPipesHomeLocation() + + "assets"; + } + + private String makeFileLocation() { + return makeStreamPipesHomeLocation() + + "files"; + } + + private String makeStreamPipesHomeLocation() { + return System.getProperty("user.home") + + File.separator + + ".streampipes" + + File.separator; + } + private String randomKey() { return RandomStringUtils.random( 10, 0, possibleCharacters.length - 1, false, false, possibleCharacters, new SecureRandom()); @@ -79,6 +98,10 @@ public int getBackendPort() { return config.getInteger(BackendConfigKeys.BACKEND_PORT); } + public String getBackendUrl() { + return "http://" + getBackendHost() + ":" + getBackendPort(); + } + public String getJmsHost() { return config.getString(BackendConfigKeys.JMS_HOST); } @@ -161,13 +184,17 @@ public Integer getKafkaRestPort() { } public String getKafkaRestUrl() { - return "http://" +getKafkaRestHost() +":" +getKafkaRestPort(); + return "http://" + getKafkaRestHost() + ":" + getKafkaRestPort(); } public String getAssetDir() { return config.getString(BackendConfigKeys.ASSETS_DIR); } + public String getFilesDir() { + return config.getString(BackendConfigKeys.FILES_DIR); + } + public String getDatalakeHost() { return config.getString(BackendConfigKeys.DATA_LAKE_HOST); } diff --git a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java index 54c7ddcadd..e417aa40c4 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java +++ b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java @@ -33,6 +33,7 @@ public class BackendConfigKeys { public static final String KAFKA_REST_HOST = "SP_KAFKA_REST_HOST"; public static final String KAFKA_REST_PORT = "SP_KAFKA_REST_PORT"; public static final String ASSETS_DIR = "SP_ASSETS_DIR"; + public static final String FILES_DIR = "SP_FILES_DIR"; public static final String DATA_LAKE_HOST = "SP_DATA_LAKE_HOST"; public static final String DATA_LAKE_PORT = "SP_DATA_LAKE_PORT"; diff --git a/streampipes-connect-container-master/pom.xml b/streampipes-connect-container-master/pom.xml index e9380d6237..b5ef758d16 100644 --- a/streampipes-connect-container-master/pom.xml +++ b/streampipes-connect-container-master/pom.xml @@ -20,21 +20,57 @@ streampipes-parent org.streampipes - 0.64.0 + 0.65.0 4.0.0 streampipes-connect-container-master + org.streampipes streampipes-connect-container + 0.65.0 + + org.streampipes + streampipes-storage-couchdb + 0.65.0 + + + org.streampipes + streampipes-user-management + 0.65.0 + + + org.apache.httpcomponents httpmime + + + + junit + junit + test + + + org.mockito + mockito-core + test + + + org.powermock + powermock-module-junit4 + test + + + org.powermock + powermock-api-mockito2 + test + diff --git a/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/management/GuessManagement.java b/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/management/GuessManagement.java index 7aa3ccc169..a0ff0fa7b9 100644 --- a/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/management/GuessManagement.java +++ b/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/management/GuessManagement.java @@ -26,7 +26,6 @@ import org.streampipes.model.client.messages.ErrorMessageLd; import org.streampipes.model.connect.adapter.AdapterDescription; import org.streampipes.model.connect.guess.GuessSchema; -import org.streampipes.model.schema.EventSchema; import org.streampipes.rest.shared.util.JsonLdUtils; import org.streampipes.vocabulary.StreamPipes; diff --git a/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/RuntimeResolvableResource.java b/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/RuntimeResolvableResource.java index e35bd7bf7d..388d076a1d 100644 --- a/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/RuntimeResolvableResource.java +++ b/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/RuntimeResolvableResource.java @@ -25,7 +25,11 @@ import org.streampipes.rest.shared.annotation.JsonLdSerialized; import org.streampipes.rest.shared.util.SpMediaType; -import javax.ws.rs.*; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.core.Response; @Path("/api/v1/{username}/master/resolvable") diff --git a/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/SourcesResource.java b/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/SourcesResource.java index 0ccfca169f..78efc32093 100644 --- a/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/SourcesResource.java +++ b/streampipes-connect-container-master/src/main/java/org/streampipes/connect/container/master/rest/SourcesResource.java @@ -19,10 +19,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.streampipes.connect.config.ConnectContainerConfig; import org.streampipes.connect.adapter.exception.AdapterException; +import org.streampipes.connect.config.ConnectContainerConfig; import org.streampipes.connect.container.master.management.SourcesManagement; -import org.streampipes.connect.container.master.management.Utils; import org.streampipes.connect.rest.AbstractContainerResource; import org.streampipes.model.SpDataSet; import org.streampipes.model.client.messages.Notifications; @@ -33,7 +32,13 @@ import org.streampipes.rest.shared.util.SpMediaType; import org.streampipes.vocabulary.StreamPipes; -import javax.ws.rs.*; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; diff --git a/streampipes-connect-container-worker/pom.xml b/streampipes-connect-container-worker/pom.xml index 9c2dfa0115..96126a7cd8 100644 --- a/streampipes-connect-container-worker/pom.xml +++ b/streampipes-connect-container-worker/pom.xml @@ -20,16 +20,32 @@ streampipes-parent org.streampipes - 0.64.0 + 0.65.0 4.0.0 streampipes-connect-container-worker + org.streampipes streampipes-connect-container + 0.65.0 + + + + + + + junit + junit + test + + + org.powermock + powermock-module-junit4 + test \ No newline at end of file diff --git a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/AdapterWorkerManagement.java b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/AdapterWorkerManagement.java index 00b0cdaaba..63f92d64c5 100644 --- a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/AdapterWorkerManagement.java +++ b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/AdapterWorkerManagement.java @@ -28,7 +28,11 @@ import org.streampipes.connect.init.AdapterDeclarerSingleton; import org.streampipes.connect.management.AdapterUtils; import org.streampipes.model.SpDataSet; -import org.streampipes.model.connect.adapter.*; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.connect.adapter.GenericAdapterSetDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; import java.util.Collection; diff --git a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/FileManagement.java b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/FileManagement.java index 8f189a00a7..37df790495 100644 --- a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/FileManagement.java +++ b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/FileManagement.java @@ -17,7 +17,6 @@ package org.streampipes.connect.container.worker.management; import org.apache.commons.io.IOUtils; -import org.streampipes.connect.config.ConnectContainerConfig; import java.io.File; import java.io.FileOutputStream; diff --git a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/GuessManagement.java b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/GuessManagement.java index 7665665cd5..bcd2935fc4 100644 --- a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/GuessManagement.java +++ b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/management/GuessManagement.java @@ -25,6 +25,7 @@ import org.streampipes.connect.management.AdapterUtils; import org.streampipes.model.connect.adapter.AdapterDescription; import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.sdk.helpers.EpProperties; import java.util.Arrays; import java.util.Optional; @@ -42,6 +43,13 @@ public GuessSchema guessSchema(AdapterDescription adapterDescription) throws Ada try { guessSchema = adapter.getSchema(adapterDescription); + // TODO remove, just for performance tests + if ("true".equals(System.getenv("SP_DEBUG_CONNECT"))) { + + guessSchema.getEventSchema().addEventProperty(EpProperties.timestampProperty("internal_t1")); + guessSchema.getEventSchema().addEventProperty(EpProperties.timestampProperty("internal_t2")); + } + for (int i = 0; i < guessSchema.getEventSchema().getEventProperties().size(); i++) { guessSchema.getEventSchema().getEventProperties().get(i).setIndex(i); } diff --git a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/rest/RuntimeResolvableResource.java b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/rest/RuntimeResolvableResource.java index 8f1a63a58f..a5d1b8cb1f 100644 --- a/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/rest/RuntimeResolvableResource.java +++ b/streampipes-connect-container-worker/src/main/java/org/streampipes/connect/container/worker/rest/RuntimeResolvableResource.java @@ -28,11 +28,15 @@ import org.streampipes.sdk.extractor.StaticPropertyExtractor; import org.streampipes.serializers.jsonld.JsonLdTransformer; -import javax.ws.rs.*; -import javax.ws.rs.core.Response; import java.io.IOException; import java.util.List; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + @Path("/api/v1/{username}/worker/resolvable") public class RuntimeResolvableResource extends AbstractContainerResource { @@ -53,7 +57,8 @@ public Response fetchConfigurations(@PathParam("id") String elementId, List