Skip to content

Commit

Permalink
Add benchmark to docker-compose
Browse files Browse the repository at this point in the history
Now run in one go with scripts/benchmark

And cleanup the order of operations in the Dockerfile
- Move all package installation to beginning and run
  in one step for better layer caching
- Move all external dependency additions before sbt
  and other ops to improve layer caching
- Reorder ops so we don't have to modify WORKDIR a bunch
  • Loading branch information
CloudNiner committed Sep 26, 2019
1 parent 3d1d5ea commit 3a8b908
Show file tree
Hide file tree
Showing 7 changed files with 113 additions and 77 deletions.
28 changes: 0 additions & 28 deletions Dockerfile.benchmark

This file was deleted.

28 changes: 0 additions & 28 deletions Makefile.benchmark

This file was deleted.

39 changes: 39 additions & 0 deletions benchmark/Dockerfile
@@ -0,0 +1,39 @@
FROM ubuntu:trusty

RUN set -ex \
&& apt-get update \
&& apt-get install -y software-properties-common \
&& add-apt-repository -y ppa:openjdk-r/ppa \
&& apt-get update \
&& deps=" \
unzip \
openjdk-8-jdk \
wget \
" \
&& apt-get install -y $deps --no-install-recommends \
&& update-ca-certificates -f \
&& rm -rf /var/lib/apt/lists/*

ADD archives/imageio-ext-1.1.24-jars.zip /tmp
ADD archives/gdal-data.zip /tmp
ADD archives/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb /tmp/

COPY . /opt/geotrellis-contrib

RUN set -ex \
&& unzip /tmp/imageio-ext-1.1.24-jars.zip -d /opt/geotrellis-contrib/lib \
&& unzip /tmp/gdal-data.zip -d /opt/geotrellis-contrib/benchmark/gdal

WORKDIR /opt/geotrellis-contrib

ADD archives/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz /opt/geotrellis-contrib/benchmark/gdal/native/
ADD benchmark/src/main/resources/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF /opt/geotrellis-contrib/benchmark/src/main/resources/

RUN set -ex \
&& dpkg -x /tmp/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb /tmp/moop \
&& cp -f /tmp/moop/usr/lib/jni/*.so /opt/geotrellis-contrib/benchmark/gdal/native/ \
&& cp -f /tmp/moop/usr/share/java/gdal.jar /opt/geotrellis-contrib/lib/
RUN ./sbt "project benchmark" compile

ENV GDAL_DATA=/opt/geotrellis-contrib/benchmark/gdal/gdal-data
ENV LD_LIBRARY_PATH=/opt/geotrellis-contrib/benchmark/gdal/native:${LD_LIBRARY_PATH}
23 changes: 23 additions & 0 deletions benchmark/Makefile
@@ -0,0 +1,23 @@
.PHONY: all


all: ../archives/imageio-ext-1.1.24-jars.zip \
../archives/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz \
../archives/gdal-data.zip \
../archives/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb \
src/main/resources/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF

../archives/imageio-ext-1.1.24-jars.zip:
curl -L "https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/imageio-ext-1.1.24-jars.zip" -o $@

../archives/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz:
curl -L "https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/native/gdal/linux/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz" -o $@

../archives/gdal-data.zip:
curl -L "https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/native/gdal/gdal-data.zip" -o $@

../archives/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb:
curl -L "http://mirrors.kernel.org/ubuntu/pool/universe/g/gdal/libgdal-java_1.10.1+dfsg-5ubuntu1_amd64.deb" -o $@

src/main/resources/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF:
curl -L "http://landsat-pds.s3.amazonaws.com/c1/L8/001/003/LC08_L1GT_001003_20170921_20170921_01_RT/LC08_L1GT_001003_20170921_20170921_01_RT_B1.TIF" -o $@
31 changes: 10 additions & 21 deletions benchmark/README.md
@@ -1,49 +1,38 @@
# With Docker #
# With Docker

## Build ##
In the root of the repository, run `./scripts/benchmark`

In the root of the repository, type `make -f Makefile.benchmark`.
# Without Docker

## Run ##
## Get Dependencies

With the Docker image built, type `docker run -it --rm benchmark`.
Inside of the docker container, type
```bash
cd /geotrellis-contrib
./sbt "project benchmark" "jmh:run"
```

# Without Docker #

## Get Dependencies ##

### Dependency Jars ###
### Dependency Jars

Pre-built ImageIO jars can be found [here](https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/).
The jars are in [this](https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/imageio-ext-1.1.24-jars.zip) archive.

### Native Dependencies ###
### Native Dependencies

Pre-built native dependencies can be found [here](https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/native/gdal/).
For Ubuntu 12.04 and compatible systems, [this tarball](https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/native/gdal/linux/gdal192-Ubuntu12-gcc4.6.3-x86_64.tar.gz) is known to work.
GDAL also needs certain data to operate, those files can be found [here](https://demo.geo-solutions.it/share/github/imageio-ext/releases/1.1.X/1.1.24/native/gdal/gdal-data.zip).

### GDAL Java Bindings ###
### GDAL Java Bindings

Java bindings are also needed.
For Ubuntu 12.04 and compatible systems, [this package](https://packages.ubuntu.com/trusty/amd64/libgdal-java/download) contains the needed files.

## Get Data ##
## Get Data

The test data can be found [here](https://landsatonaws.com/L8/001/003/LC08_L1GT_001003_20170921_20170921_01_RT).

## Run ##
## Run

```bash
GDAL_DATA=$(pwd)/benchmark/gdal/gdal-data LD_LIBRARY_PATH=$(pwd)/benchmark/gdal/native:$LD_LIBRARY_PATH ./sbt "project benchmark" "jmh:run"
```

# Using JMH #
# Using JMH

We use JMH to wire up all benchmarks. Understanding the semantics of the JMH
annotations takes some time. For clues, take a look at this
Expand Down
15 changes: 15 additions & 0 deletions docker-compose.yml
Expand Up @@ -30,3 +30,18 @@ services:
- $HOME/.sbt:/root/.sbt
working_dir: /opt/geotrellis-contrib
entrypoint: ./sbt
benchmark:
build:
context: .
dockerfile: benchmark/Dockerfile
environment:
- AWS_PROFILE=geotrellis
- COURSIER_CACHE=/root/.coursier
volumes:
- .:/opt/geotrellis-contrib
- $HOME/.aws:/root/.aws:ro
- $HOME/.coursier:/root/.coursier
- $HOME/.ivy2:/root/.ivy2
- $HOME/.sbt:/root/.sbt
working_dir: /opt/geotrellis-contrib
entrypoint: ./sbt
26 changes: 26 additions & 0 deletions scripts/benchmark
@@ -0,0 +1,26 @@
#!/bin/bash

set -e

if [[ -n "${GEOTRELLIS_CONTRIB_DEBUG}" ]]; then
set -x
fi

function usage() {
echo -n \
"Usage: $(basename "$0")
Run GeoTrellis Contrib benchmarks
"
}

if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
if [[ "${1:-}" == "--help" ]]; then
usage
else
echo "Executing GeoTrellis Contrib benchmarks"
pushd benchmark
make
popd
docker-compose run --rm benchmark "project benchmark" "jmh:run"
fi
fi

0 comments on commit 3a8b908

Please sign in to comment.