From 1f3ed5c52746a21ab6dac998884cf08f27140b74 Mon Sep 17 00:00:00 2001 From: CloudNiner Date: Tue, 24 Sep 2019 21:07:21 -0400 Subject: [PATCH] Publish GT Contrib artifacts automatically via CircleCI Using the same general strategy as geotrellis/geotrellis-server and geotrellis/maml Update sbt runner to commit 5fd24eb https://github.com/paulp/sbt-extras/commit/5fd24ebb61d882ab343ff2696bbe86909560a859 Switch tests to use a docker-compose setup for ease of use and consistency with other projects. Tests should now be run from ./scripts/test Add circleci config running scripts/cibuild We use the latest available machine image so we can run our existing docker-compose setup. A writeup considering various options: https://github.com/geotrellis/geotrellis-contrib/issues/222#issuecomment-535041064 Update to azavea-hosted openjdk-gdal image Publish build artifacts to OSS SonaType Nexus Remove old Travis CI configuration Run benchmarks with scripts/benchmark and cleanup the order of operations in the Dockerfile - Move all package installation to beginning and run in one step for better layer caching - Move all external dependency additions before sbt and other ops to improve layer caching - Reorder ops so we don't have to modify WORKDIR a bunch --- .circleci/config.yml | 104 ++++++++++++++++++ .github/ISSUE_TEMPLATE/release.md | 37 +++++++ .github/pull_request_template.md | 24 +++-- .travis.yml | 27 ----- .travis/build-and-test-docker.sh | 9 -- .travis/build-and-test.sh | 6 -- .travis/slickTestDB.sh | 10 -- CHANGELOG.md | 5 + Dockerfile.benchmark | 28 ----- Makefile.benchmark | 28 ----- README.md | 12 +-- RELEASING.md | 30 ------ benchmark/Dockerfile | 39 +++++++ benchmark/Makefile | 23 ++++ benchmark/README.md | 31 ++---- build.sbt | 82 ++++++++++---- docker-compose.yml | 48 +++++++++ gdal/version.sbt | 1 - project/plugins.sbt | 7 +- project/project/plugins.sbt | 1 + sbt | 136 ++++++++++++++++-------- scripts/benchmark | 28 +++++ scripts/cibuild | 22 ++++ scripts/cipublish | 27 +++++ scripts/test | 23 ++++ slick/src/test/resources/reference.conf | 7 +- slick/version.sbt | 1 - version.sbt | 1 - vlm/version.sbt | 1 - 29 files changed, 542 insertions(+), 256 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .github/ISSUE_TEMPLATE/release.md delete mode 100644 .travis.yml delete mode 100755 .travis/build-and-test-docker.sh delete mode 100755 .travis/build-and-test.sh delete mode 100755 .travis/slickTestDB.sh delete mode 100644 Dockerfile.benchmark delete mode 100644 Makefile.benchmark delete mode 100644 RELEASING.md create mode 100644 benchmark/Dockerfile create mode 100644 benchmark/Makefile create mode 100644 docker-compose.yml delete mode 100644 gdal/version.sbt create mode 100644 project/project/plugins.sbt create mode 100755 scripts/benchmark create mode 100755 scripts/cibuild create mode 100755 scripts/cipublish create mode 100755 scripts/test delete mode 100644 slick/version.sbt delete mode 100644 version.sbt delete mode 100644 vlm/version.sbt diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..bdca8c3f --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,104 @@ +aliases: + - &restore_sbt_cache + key: sbt-cache-{{ checksum "/tmp/scala_version" }} + + - &save_sbt_cache + key: sbt-cache-{{ checksum "/tmp/scala_version" }}-{{ epoch }} + paths: + - "~/.ivy2/cache" + - "~/.sbt" + - "~/.cache/coursier" + + - &run_cibuild + - checkout + - run: echo "${SCALA_VERSION}" > /tmp/scala_version + - restore_cache: *restore_sbt_cache + - run: + name: Executing cibuild + command: ./scripts/cibuild + - save_cache: *save_sbt_cache + + - &run_cipublish + - checkout + - run: echo "${SCALA_VERSION}" > /tmp/scala_version + - restore_cache: *restore_sbt_cache + - run: + name: "Import signing key" + command: | + echo "${GPG_KEY}" | base64 -d > signing_key.asc && \ + gpg --batch \ + --passphrase "${GPG_PASSPHRASE}" \ + --import signing_key.asc + - run: + name: Executing cipublish + command: ./scripts/cipublish + + # Build environments + - &openjdk8-scala2_11_12_environment + machine: + image: ubuntu-1604:201903-01 + environment: + SCALA_VERSION: 2.11.12 + + - &openjdk8-scala2_12_8_environment + machine: + image: ubuntu-1604:201903-01 + environment: + SCALA_VERSION: 2.12.8 + +version: 2 +workflows: + version: 2 + build: + jobs: + - "openjdk8-scala2.11.12": + filters: # required since `openjdk8-scala2.11.12_deploy` has tag filters AND requires `openjdk8-scala2.11.12` + tags: + only: + - /^(.*)$/ + - "openjdk8-scala2.12.8": + filters: # required since `openjdk8-scala2.12.8_deploy` has tag filters AND requires `openjdk8-scala2.12.8` + tags: + only: + - /^(.*)$/ + - "openjdk8-scala2.11.12_deploy": + requires: + - "openjdk8-scala2.11.12" + filters: + tags: + only: + - /^(.*)$/ + branches: + only: + - develop + - /release\/.*/ + - /hotfix\/.*/ + - "openjdk8-scala2.12.8_deploy": + requires: + - "openjdk8-scala2.12.8" + filters: + tags: + only: + - /^(.*)$/ + branches: + only: + - develop + - /release\/.*/ + - /hotfix\/.*/ + +jobs: + "openjdk8-scala2.11.12": + <<: *openjdk8-scala2_11_12_environment + steps: *run_cibuild + + "openjdk8-scala2.12.8": + <<: *openjdk8-scala2_12_8_environment + steps: *run_cibuild + + "openjdk8-scala2.11.12_deploy": + <<: *openjdk8-scala2_11_12_environment + steps: *run_cipublish + + "openjdk8-scala2.12.8_deploy": + <<: *openjdk8-scala2_12_8_environment + steps: *run_cipublish diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md new file mode 100644 index 00000000..c2ec4f7c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release.md @@ -0,0 +1,37 @@ +--- +name: Release +about: When ready to cut a release +title: Release X.Y.Z +labels: release +assignees: "" +--- + +- [ ] Start a new release branch: + +```bash +$ git flow release start X.Y.Z +``` + +- [ ] Rotate `CHANGELOG.md` (following [Keep a Changelog](https://keepachangelog.com/) principles) +- [ ] Ensure outstanding changes are committed: + +```bash +$ git status # Is the git staging area clean? +$ git add CHANGELOG.md +$ git commit -m "X.Y.Z" +``` + +- [ ] Publish the release branch: + +```bash +$ git flow release publish X.Y.Z +``` + +- [ ] Ensure that CI checks pass +- [ ] Finish and publish the release branch: + - When prompted, keep default commit messages + - Use `X.Y.Z` as the tag message + +```bash +$ git flow release finish -p X.Y.Z +``` diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 42b5f832..0f51baff 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,21 +1,25 @@ -# Overview +## Overview -Brief description of what this PR does and why it's important +Brief description of what this PR does, and why it is needed. -## Demo +### Checklist -Optional. Screenshots, etc. +- [ ] Description of PR is in an appropriate section of the CHANGELOG and grouped with similar changes if possible -## Notes +### Demo -Optional. Extra context, ancillary topics, alternative strategies that didn't work out, etc. +Optional. Screenshots, `http` examples, etc. -## Testing Instructions +### Notes -Optional. Include if there's more specifics than "CI tests should pass". +Optional. Ancillary topics, caveats, alternative strategies that didn't work out, anything else. -## Checklist +## Testing Instructions -- [ ] Add entry to CHANGELOG.md +- How to test this PR +- Prefer bulleted description +- Start after checking out this branch +- Include any setup required, such as bundling scripts, restarting services, etc. +- Include test case, and expected output Closes #XXX diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f1f27b62..00000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -sudo: false -language: scala -addons: - hostname: localhost -jdk: - - oraclejdk8 -scala: - - "2.11.12" - - "2.12.8" -before_install: - - docker pull daunnc/openjdk-gdal:2.4.0 - - docker run -d --restart=always -p 9999:5432 -e POSTGRES_DB=slick_tests quay.io/azavea/postgis:0.1.0 -services: - - docker -cache: - directories: - - $HOME/.ivy2 - - $HOME/.sbt -script: - - .travis/build-and-test-docker.sh -notifications: - email: - recipients: - - echeipesh@gmail.com - - gr.pomadchin@gmail.com -before_deploy: -- export VERSION_SUFFIX="-${TRAVIS_COMMIT:0:7}" diff --git a/.travis/build-and-test-docker.sh b/.travis/build-and-test-docker.sh deleted file mode 100755 index b7c62cde..00000000 --- a/.travis/build-and-test-docker.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -docker run -it --net=host \ - -v $HOME/.ivy2:/root/.ivy2 \ - -v $HOME/.sbt:/root/.sbt \ - -v $TRAVIS_BUILD_DIR:/geotrellis-contrib \ - -e TRAVIS_SCALA_VERSION=$TRAVIS_SCALA_VERSION \ - -e TRAVIS_COMMIT=$TRAVIS_COMMIT \ - -e TRAVIS_JDK_VERSION=$TRAVIS_JDK_VERSION daunnc/openjdk-gdal:2.4.0 /bin/bash -c "cd /geotrellis-contrib; .travis/build-and-test.sh" \ No newline at end of file diff --git a/.travis/build-and-test.sh b/.travis/build-and-test.sh deleted file mode 100755 index 4a110801..00000000 --- a/.travis/build-and-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -./sbt -J-Xmx2G "++$TRAVIS_SCALA_VERSION" \ - "project vlm" test \ - "project slick" test \ - "project gdal" test || { exit 1; } diff --git a/.travis/slickTestDB.sh b/.travis/slickTestDB.sh deleted file mode 100755 index cdee04e1..00000000 --- a/.travis/slickTestDB.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -docker pull quay.io/azavea/postgis:0.1.0 - -docker run \ - -it \ - --rm \ - -p 9999:5432 \ - -e POSTGRES_DB=slick_tests \ - quay.io/azavea/postgis:0.1.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index d587e776..c85d9a17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Artifacts are published to https://oss.sonatype.org automatically via CircleCI + - The `bintray` resolver in build.sbt is no longer necessary as of this release + - Commits to the develop branch are available as SNAPSHOT releases - Move RasterSources into GeoTrellis core, only RasterSources effects are present in this repository now. ## [3.17.1] - 2019-08-26 diff --git a/Dockerfile.benchmark b/Dockerfile.benchmark deleted file mode 100644 index 51402e38..00000000 --- a/Dockerfile.benchmark +++ /dev/null @@ -1,28 +0,0 @@ -FROM ubuntu:trusty - -RUN apt-get update -RUN apt-get install -y software-properties-common unzip -RUN add-apt-repository -y ppa:openjdk-r/ppa -RUN apt-get update -RUN apt-get install -y openjdk-8-jdk -RUN apt-get install -y wget -RUN apt-get clean - -ADD archives/geotrellis-contrib.tar / -ADD archives/imageio-ext-1.1.24-jars.zip /tmp -RUN (cd /geotrellis-contrib/lib ; unzip /tmp/imageio-ext-1.1.24-jars.zip) -ADD archives/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz /geotrellis-contrib/benchmark/gdal/native/ -ADD archives/gdal-data.zip /tmp -RUN (cd /geotrellis-contrib/benchmark/gdal ; unzip /tmp/gdal-data.zip) -ADD archives/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb /tmp/ -RUN (dpkg -x /tmp/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb /tmp/moop ; \ - cp -f /tmp/moop/usr/lib/jni/*.so /geotrellis-contrib/benchmark/gdal/native/ ; \ - cp -f /tmp/moop/usr/share/java/gdal.jar /geotrellis-contrib/lib/) - -RUN update-ca-certificates -f -RUN (cd /geotrellis-contrib ; ./sbt "project benchmark" compile) - -ADD benchmark/src/main/resources/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF /geotrellis-contrib/benchmark/src/main/resources/ - -ENV GDAL_DATA=/geotrellis-contrib/benchmark/gdal/gdal-data -ENV LD_LIBRARY_PATH=/geotrellis-contrib/benchmark/gdal/native:${LD_LIBRARY_PATH} diff --git a/Makefile.benchmark b/Makefile.benchmark deleted file mode 100644 index 10a871c3..00000000 --- a/Makefile.benchmark +++ /dev/null @@ -1,28 +0,0 @@ -.PHONY: all - - -all: archives/geotrellis-contrib.tar \ - archives/imageio-ext-1.1.24-jars.zip \ - archives/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz \ - archives/gdal-data.zip \ - archives/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb \ - benchmark/src/main/resources/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF - docker build -t benchmark -f Dockerfile.benchmark . - -archives/geotrellis-contrib.tar: - git archive --format=tar --prefix=geotrellis-contrib/ -o $@ HEAD - -archives/imageio-ext-1.1.24-jars.zip: - curl -L "https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/imageio-ext-1.1.24-jars.zip" -o $@ - -archives/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz: - curl -L "https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/native/gdal/linux/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz" -o $@ - -archives/gdal-data.zip: - curl -L "https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/native/gdal/gdal-data.zip" -o $@ - -archives/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb: - curl -L "http://mirrors.kernel.org/ubuntu/pool/universe/g/gdal/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb" -o $@ - -benchmark/src/main/resources/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF: - curl -L "http://landsat-pds.s3.amazonaws.com/c1/L8/001/003/LC08_L1GT_001003_20170921_20170921_01_RT/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF" -o $@ diff --git a/README.md b/README.md index 415c7720..33287ab6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,18 @@ # GeoTrellis Contributions repo -[![Build Status](https://travis-ci.org/geotrellis/geotrellis-contrib.svg?branch=master)](https://travis-ci.org/geotrellis/geotrellis-contrib) +[![CircleCI](https://circleci.com/gh/geotrellis/geotrellis-contrib/tree/develop.svg?style=svg)](https://circleci.com/gh/geotrellis/geotrellis-contrib/tree/develop) This is a repository is a place to put extra [GeoTrellis](https://github.com/locationtech/geotrellis) projects, that were not included into the main repo for various reasons. - - Project names should be of the form `geotrellis-contrib-{feature}` - - Project versions should start with `0.0.1` and not be tied to GeoTrellis version. - - Project publishing will happen on BinTray - +- Project names should be of the form `geotrellis-contrib-{feature}` +- Project versions should start with `0.0.1` and not be tied to GeoTrellis version. +- Project publishing will happen on SonaType ## Usage via `sbt`: ```scala -resolvers += "Azavea Public Builds" at "https://dl.bintray.com/azavea/geotrellis" -... libraryDependencies ++= Seq( "com.azavea.geotrellis" %% "geotrellis-contrib-vlm" % "X.Y.Z", "com.azavea.geotrellis" %% "geotrellis-contrib-gdal" % "X.Y.Z" @@ -23,4 +20,3 @@ libraryDependencies ++= Seq( ``` Go to [releases](https://github.com/geotrellis/geotrellis-contrib/releases) to see available versions. - diff --git a/RELEASING.md b/RELEASING.md deleted file mode 100644 index bc4a789b..00000000 --- a/RELEASING.md +++ /dev/null @@ -1,30 +0,0 @@ -# Releasing GeoTrellis Contrib - -Ensure you have bintray credentials set: https://github.com/sbt/sbt-bintray#credentials - -Ensure `master` is up to date. - -Create a new release branch `git checkout -b release/x.y.z` - -Bump each `version.sbt` to match the version you're releasing. - -Update the CHANGELOG. Ensure you've included a new link to the current release at the -bottom of the file and kept the `[Unreleased]` header at the top. - -Commit all your changes. - -Time to release! -```bash -./sbt publish -./sbt -212 publish -./sbt bintrayRelease -``` - -Tag the release with `v url("http://www.apache.org/licenses/LICENSE-2.0.html")), + // We are overriding the default behavior of sbt-git which, by default, + // only appends the `-SNAPSHOT` suffix if there are uncommitted + // changes in the workspace. + version := { + // Avoid Cyclic reference involving error + if (git.gitCurrentTags.value.isEmpty || git.gitUncommittedChanges.value) + git.gitDescribedVersion.value.get + "-SNAPSHOT" + else + git.gitDescribedVersion.value.get + }, homepage := Some(url(Info.url)), scmInfo := Some(ScmInfo( url("https://github.com/geotrellis/geotrellis-contrib"), "scm:git:git@github.com:geotrellis/geotrellis-contrib.git" @@ -23,8 +31,6 @@ lazy val commonSettings = Seq( "-language:experimental.macros", "-Ypartial-unification" // Required by Cats ), - publishMavenStyle := true, - publishArtifact in Test := false, pomIncludeRepository := { _ => false }, addCompilerPlugin("org.spire-math" % "kind-projector" % "0.9.4" cross CrossVersion.binary), addCompilerPlugin("org.scalamacros" %% "paradise" % "2.1.1" cross CrossVersion.full), @@ -35,33 +41,62 @@ lazy val commonSettings = Seq( "locationtech-snapshots" at "https://repo.locationtech.org/content/groups/snapshots", "osgeo" at "http://download.osgeo.org/webdav/geotools/", "geotrellis-staging" at "https://oss.sonatype.org/service/local/repositories/orglocationtechgeotrellis-1009/content" - ), + ) +) + +lazy val noPublishSettings = Seq( + publish := {}, + publishLocal := {}, + publishArtifact := false +) + +lazy val publishSettings = Seq( + organization := "com.azavea.geotrellis", + organizationName := "GeoTrellis", + organizationHomepage := Some(new URL("https://geotrellis.io/")), + description := "GeoTrellis Contrib is a place to put GeoTrellis projects that are not included in GeoTrellis Core for one reason or another.", headerLicense := Some(HeaderLicense.ALv2("2019", "Azavea")), - bintrayOrganization := Some("azavea"), - bintrayRepository := "geotrellis", - bintrayPackageLabels := Seq("gis", "raster", "vector"), - bintrayReleaseOnPublish := false, - publishTo := { - val bintrayPublishTo = publishTo.value - val nexus = "http://nexus.internal.azavea.com" + publishArtifact in Test := false +) ++ sonatypeSettings ++ credentialSettings - if (isSnapshot.value) { - Some("snapshots" at nexus + "/repository/azavea-snapshots") - } else { - bintrayPublishTo - } - } +lazy val sonatypeSettings = Seq( + publishMavenStyle := true, + + sonatypeProfileName := "com.azavea", + sonatypeProjectHosting := Some(GitHubHosting(user="geotrellis", repository="geotrellis-contrib", email="systems@azavea.com")), + developers := List( + Developer(id = "echeipesh", name = "Eugene Cheipesh", email = "echeipesh@azavea.com", url = url("https://github.com/echeipesh")), + Developer(id = "pomadchin", name = "Grigory Pomadchin", email = "gpomadchin@azavea.com", url = url("https://github.com/pomadchin")) + ), + licenses := Seq("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")), + publishTo := sonatypePublishTo.value +) + +lazy val credentialSettings = Seq( + credentials += Credentials( + "GnuPG Key ID", + "gpg", + System.getenv().get("GPG_KEY_ID"), + "ignored" + ), + credentials += Credentials( + "Sonatype Nexus Repository Manager", + "oss.sonatype.org", + System.getenv().get("SONATYPE_USERNAME"), + System.getenv().get("SONATYPE_PASSWORD") + ) ) lazy val root = Project("geotrellis-contrib", file(".")) .aggregate(vlm, gdal, slick) .settings(commonSettings: _*) - .settings(publish / skip := true) + .settings(noPublishSettings) lazy val IntegrationTest = config("it") extend Test lazy val vlm = project .settings(commonSettings) + .settings(publishSettings) .settings( organization := "com.azavea.geotrellis", name := "geotrellis-contrib-vlm", @@ -107,6 +142,7 @@ lazy val vlm = project lazy val gdal = project .dependsOn(vlm) .settings(commonSettings) + .settings(publishSettings) .configs(IntegrationTest) .settings(Defaults.itSettings) .settings( @@ -148,7 +184,7 @@ lazy val gdal = project lazy val slick = project .settings(commonSettings) - .settings( publish / skip := true) + .settings(noPublishSettings) .settings( organization := "com.azavea.geotrellis", name := "geotrellis-contrib-slick", @@ -163,12 +199,12 @@ lazy val slick = project lazy val benchmark = (project in file("benchmark")) .dependsOn(vlm) .settings(commonSettings: _*) + .settings(noPublishSettings) .enablePlugins(JmhPlugin) .settings( name := "benchmark", fork := true, libraryDependencies ++= Seq( sparkCore - ), - publish / skip := true + ) ) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..72d7f399 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,48 @@ +version: "2.3" +services: + database: + # When this is updated, we'll have to handle creating the postgis extension + # somewhere in the test workflow. That's handled automatically on 0.1.0 but + # not on later versions + image: quay.io/azavea/postgis:0.1.0 + environment: + - POSTGRES_USER=slick_tests + - POSTGRES_PASSWORD=slick_tests + - POSTGRES_DB=slick_tests + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 3s + timeout: 3s + retries: 3 + start_period: 5s + sbt: + image: quay.io/azavea/openjdk-gdal:2.4-jdk8-slim + depends_on: + database: + condition: service_healthy + links: + - database:database.service.internal + environment: + - COURSIER_CACHE=/root/.coursier + volumes: + - .:/opt/geotrellis-contrib + - $HOME/.coursier:/root/.coursier + - $HOME/.ivy2:/root/.ivy2 + - $HOME/.sbt:/root/.sbt + working_dir: /opt/geotrellis-contrib + entrypoint: ./sbt + benchmark: + build: + context: . + dockerfile: benchmark/Dockerfile + environment: + - AWS_PROFILE=geotrellis + - COURSIER_CACHE=/root/.coursier + volumes: + - .:/opt/geotrellis-contrib + - $HOME/.aws:/root/.aws:ro + - $HOME/.coursier:/root/.coursier + - $HOME/.ivy2:/root/.ivy2 + - $HOME/.sbt:/root/.sbt + working_dir: /opt/geotrellis-contrib + entrypoint: ./sbt diff --git a/gdal/version.sbt b/gdal/version.sbt deleted file mode 100644 index 3d1108ec..00000000 --- a/gdal/version.sbt +++ /dev/null @@ -1 +0,0 @@ -version in ThisBuild := "3.18.0-SNAPSHOT" diff --git a/project/plugins.sbt b/project/plugins.sbt index 60685763..21cec89d 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,8 +1,9 @@ -addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.4") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.4") addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.0") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.3") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.3.4") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "4.1.0") -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.9") + +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.5") +addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") +addSbtPlugin("io.crashbox" % "sbt-gpg" % "0.2.0") diff --git a/project/project/plugins.sbt b/project/project/plugins.sbt new file mode 100644 index 00000000..ef1f82c5 --- /dev/null +++ b/project/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.2") diff --git a/sbt b/sbt index bc1d9540..fc7e5225 100755 --- a/sbt +++ b/sbt @@ -6,11 +6,11 @@ set -o pipefail -declare -r sbt_release_version="0.13.16" -declare -r sbt_unreleased_version="0.13.16" +declare -r sbt_release_version="1.3.2" +declare -r sbt_unreleased_version="1.3.2" -declare -r latest_213="2.13.0-M4" -declare -r latest_212="2.12.6" +declare -r latest_213="2.13.1" +declare -r latest_212="2.12.10" declare -r latest_211="2.11.12" declare -r latest_210="2.10.7" declare -r latest_29="2.9.3" @@ -18,12 +18,12 @@ declare -r latest_28="2.8.2" declare -r buildProps="project/build.properties" -declare -r sbt_launch_ivy_release_repo="http://repo.typesafe.com/typesafe/ivy-releases" +declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases" declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots" -declare -r sbt_launch_mvn_release_repo="http://repo.scala-sbt.org/scalasbt/maven-releases" -declare -r sbt_launch_mvn_snapshot_repo="http://repo.scala-sbt.org/scalasbt/maven-snapshots" +declare -r sbt_launch_mvn_release_repo="https://repo.scala-sbt.org/scalasbt/maven-releases" +declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots" -declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m" +declare -r default_jvm_opts_common="-Xms512m -Xss2m" declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new @@ -43,11 +43,12 @@ declare -a extra_jvm_opts extra_sbt_opts echoerr () { echo >&2 "$@"; } vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } -die () { echo "Aborting: $@" ; exit 1; } +die () { echo "Aborting: $*" ; exit 1; } setTrapExit () { # save stty and trap exit, to ensure echo is re-enabled if we are interrupted. - export SBT_STTY="$(stty -g 2>/dev/null)" + SBT_STTY="$(stty -g 2>/dev/null)" + export SBT_STTY # restore stty settings (echo in particular) onSbtRunnerExit() { @@ -67,7 +68,7 @@ get_script_path () { local path="$1" [[ -L "$path" ]] || { echo "$path" ; return; } - local target="$(readlink "$path")" + local -r target="$(readlink "$path")" if [[ "${target:0:1}" == "/" ]]; then echo "$target" else @@ -75,8 +76,10 @@ get_script_path () { fi } -declare -r script_path="$(get_script_path "$BASH_SOURCE")" -declare -r script_name="${script_path##*/}" +script_path="$(get_script_path "${BASH_SOURCE[0]}")" +declare -r script_path +script_name="${script_path##*/}" +declare -r script_name init_default_option_file () { local overriding_var="${!1}" @@ -90,8 +93,8 @@ init_default_option_file () { echo "$default_file" } -declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" -declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" +sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" +jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" build_props_sbt () { [[ -r "$buildProps" ]] && \ @@ -108,7 +111,7 @@ url_base () { local version="$1" case "$version" in - 0.7.*) echo "http://simple-build-tool.googlecode.com" ;; + 0.7.*) echo "https://simple-build-tool.googlecode.com" ;; 0.10.* ) echo "$sbt_launch_ivy_release_repo" ;; 0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;; 0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" @@ -130,7 +133,7 @@ make_url () { 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; - *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;; esac } @@ -142,9 +145,9 @@ addResidual () { vlog "[residual] arg = '$1'" ; residual_args+=("$1"); } addResolver () { addSbt "set resolvers += $1"; } addDebugger () { addJava "-Xdebug" ; addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } setThisBuild () { - vlog "[addBuild] args = '$@'" + vlog "[addBuild] args = '$*'" local key="$1" && shift - addSbt "set $key in ThisBuild := $@" + addSbt "set $key in ThisBuild := $*" } setScalaVersion () { [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' @@ -159,7 +162,7 @@ setJavaHome () { } getJavaVersion() { - local str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"') + local -r str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"') # java -version on java8 says 1.8.x # but on 9 and 10 it's 9.x.y and 10.x.y. @@ -191,14 +194,14 @@ checkJava() { } java_version () { - local version=$(getJavaVersion "$java_cmd") + local -r version=$(getJavaVersion "$java_cmd") vlog "Detected Java version: $version" echo "$version" } # MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+ default_jvm_opts () { - local v="$(java_version)" + local -r v="$(java_version)" if [[ $v -ge 8 ]]; then echo "$default_jvm_opts_common" else @@ -240,11 +243,11 @@ execRunner () { jar_url () { make_url "$1"; } -is_cygwin () [[ "$(uname -a)" == "CYGWIN"* ]] +is_cygwin () { [[ "$(uname -a)" == "CYGWIN"* ]]; } jar_file () { is_cygwin \ - && echo "$(cygpath -w $sbt_launch_dir/"$1"/sbt-launch.jar)" \ + && cygpath -w "$sbt_launch_dir/$1/sbt-launch.jar" \ || echo "$sbt_launch_dir/$1/sbt-launch.jar" } @@ -252,10 +255,6 @@ download_url () { local url="$1" local jar="$2" - echoerr "Downloading sbt launcher for $sbt_version:" - echoerr " From $url" - echoerr " To $jar" - mkdir -p "${jar%/*}" && { if command -v curl > /dev/null 2>&1; then curl --fail --silent --location "$url" --output "$jar" @@ -274,10 +273,57 @@ acquire_sbt_jar () { [[ -r "$sbt_jar" ]] } || { sbt_jar="$(jar_file "$sbt_version")" - download_url "$(make_url "$sbt_version")" "$sbt_jar" + jar_url="$(make_url "$sbt_version")" + + echoerr "Downloading sbt launcher for ${sbt_version}:" + echoerr " From ${jar_url}" + echoerr " To ${sbt_jar}" + + download_url "${jar_url}" "${sbt_jar}" + + case "${sbt_version}" in + 0.*) vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check"; echo "" ;; + *) verify_sbt_jar "${sbt_jar}" ;; + esac } } +verify_sbt_jar() { + local jar="${1}" + local md5="${jar}.md5" + + download_url "$(make_url "${sbt_version}").md5" "${md5}" > /dev/null 2>&1 + + if command -v md5sum > /dev/null 2>&1; then + if echo "$(cat "${md5}") ${jar}" | md5sum -c -; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v md5 > /dev/null 2>&1; then + if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v openssl > /dev/null 2>&1; then + if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + else + echoerr "Could not find an MD5 command" + return 1 + fi +} + usage () { set_sbt_version cat < 0 )) || { @@ -484,6 +530,7 @@ EOM } # pick up completion if present; todo +# shellcheck disable=SC1091 [[ -r .sbt_completion.sh ]] && source .sbt_completion.sh # directory to store sbt launchers @@ -493,7 +540,7 @@ EOM # no jar? download it. [[ -r "$sbt_jar" ]] || acquire_sbt_jar || { # still no jar? uh-oh. - echo "Download failed. Obtain the jar manually and place it at $sbt_jar" + echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar" exit 1 } @@ -518,13 +565,13 @@ fi if [[ -r "$jvm_opts_file" ]]; then vlog "Using jvm options defined in file $jvm_opts_file" - while read opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") + while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then vlog "Using jvm options defined in \$JVM_OPTS variable" - extra_jvm_opts=( $JVM_OPTS ) + IFS=" " read -r -a extra_jvm_opts <<< "$JVM_OPTS" else vlog "Using default jvm options" - extra_jvm_opts=( $(default_jvm_opts) ) + IFS=" " read -r -a extra_jvm_opts <<< "$(default_jvm_opts)" fi # traceLevel is 0.12+ @@ -546,13 +593,12 @@ main () { # we're not going to print those lines anyway. We strip that bit of # line noise, but leave the other codes to preserve color. mainFiltered () { - local ansiOverwrite='\r\x1BM\x1B[2K' - local excludeRegex=$(egrep -v '^#|^$' ~/.sbtignore | paste -sd'|' -) + local -r excludeRegex=$(grep -E -v '^#|^$' ~/.sbtignore | paste -sd'|' -) echoLine () { - local line="$1" - local line1="$(echo "$line" | sed 's/\r\x1BM\x1B\[2K//g')" # This strips the OverwriteLine code. - local line2="$(echo "$line1" | sed 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this. + local -r line="$1" + local -r line1="${line//\r\x1BM\x1B\[2K//g}" # This strips the OverwriteLine code. + local -r line2="${line1//\x1B\[[0-9;]*[JKmsu]//g}" # This strips all codes - we test regexes against this. if [[ $line2 =~ $excludeRegex ]]; then [[ -n $debugUs ]] && echo "[X] $line1" @@ -569,7 +615,7 @@ mainFiltered () { # Obviously this is super ad hoc but I don't know how to improve on it. Testing whether # stdin is a terminal is useless because most of my use cases for this filtering are # exactly when I'm at a terminal, running sbt non-interactively. -shouldFilter () { [[ -f ~/.sbtignore ]] && ! egrep -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; } +shouldFilter () { [[ -f ~/.sbtignore ]] && ! grep -E -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; } # run sbt if shouldFilter; then mainFiltered; else main; fi diff --git a/scripts/benchmark b/scripts/benchmark new file mode 100755 index 00000000..b34d0168 --- /dev/null +++ b/scripts/benchmark @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +if [[ -n "${GEOTRELLIS_CONTRIB_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Run GeoTrellis Contrib benchmarks +" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + if [[ "${1:-}" == "--help" ]]; then + usage + else + echo "Executing GeoTrellis Contrib benchmarks" + pushd benchmark + make + popd + docker-compose \ + run --rm benchmark \ + "project benchmark" "jmh:run" + fi +fi diff --git a/scripts/cibuild b/scripts/cibuild new file mode 100755 index 00000000..f8065c7a --- /dev/null +++ b/scripts/cibuild @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +if [[ -n "${GEOTRELLIS_CONTRIB_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Execute tests. +" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + if [[ "${1:-}" == "--help" ]]; then + usage + else + SCALA_VERSION="${SCALA_VERSION:-2.11.12}" ./scripts/test + fi +fi diff --git a/scripts/cipublish b/scripts/cipublish new file mode 100755 index 00000000..90247977 --- /dev/null +++ b/scripts/cipublish @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +if [[ -n "${GEOTRELLIS_CONTRIB_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Publish artifacts to Sonatype. +" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + if [[ "${1:-}" == "--help" ]]; then + usage + else + echo "Publishing artifacts to Sonatype" + if [[ -n "${CIRCLE_TAG}" ]]; then + ./sbt ";++${SCALA_VERSION:-2.11.12};sonatypeOpen ${CIRCLE_BUILD_NUM};publish;sonatypeRelease" + else + ./sbt "++${SCALA_VERSION:-2.11.12}" publish + fi + fi +fi diff --git a/scripts/test b/scripts/test new file mode 100755 index 00000000..fc20e2da --- /dev/null +++ b/scripts/test @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +if [[ -n "${GEOTRELLIS_CONTRIB_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Update Scala dependencies and execute tests. +" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + if [[ "${1:-}" == "--help" ]]; then + usage + else + echo "Executing Scala test suite" + docker-compose run --rm sbt "++${SCALA_VERSION:-2.11.12}" test + fi +fi diff --git a/slick/src/test/resources/reference.conf b/slick/src/test/resources/reference.conf index 605428d9..3da4ada2 100644 --- a/slick/src/test/resources/reference.conf +++ b/slick/src/test/resources/reference.conf @@ -13,9 +13,8 @@ # limitations under the License. db { - user = "postgres" - password = "postgres" + user = "slick_tests" + password = "slick_tests" database = "slick_tests" - host = "localhost:9999" + host = "database.service.internal:5432" } - diff --git a/slick/version.sbt b/slick/version.sbt deleted file mode 100644 index 3d1108ec..00000000 --- a/slick/version.sbt +++ /dev/null @@ -1 +0,0 @@ -version in ThisBuild := "3.18.0-SNAPSHOT" diff --git a/version.sbt b/version.sbt deleted file mode 100644 index 3d1108ec..00000000 --- a/version.sbt +++ /dev/null @@ -1 +0,0 @@ -version in ThisBuild := "3.18.0-SNAPSHOT" diff --git a/vlm/version.sbt b/vlm/version.sbt deleted file mode 100644 index 3d1108ec..00000000 --- a/vlm/version.sbt +++ /dev/null @@ -1 +0,0 @@ -version in ThisBuild := "3.18.0-SNAPSHOT"