Merge branch 'master' into stable

.circleci/config.yml

@@ -1,6 +1,6 @@
 version: 2
 jobs:
-  x86_64:
+  postgresql:
     docker:
       - image: cossacklabs/android-build
       # postgresql image with ssl support
@@ -9,36 +9,19 @@ jobs:
           POSTGRES_PASSWORD: test
           POSTGRES_USER: test
           POSTGRES_DB: test
-      # use the same credentials for mysql db as for postgresql (which support was added first)
-      # has latest tag on 2018.03.29
-      - image: mysql:5.7.21
-        environment:
-          MYSQL_DATABASE: test
-          MYSQL_USER: test
-          MYSQL_PASSWORD: test
-          MYSQL_ROOT_PASSWORD: root
     environment:
       GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
       FILEPATH_ERROR_FLAG: /tmp/test_fail
-      VERSIONS: 1.8 1.8.7 1.9.4 1.10 1.10.3
-      TEST_DB_USER: test
-      TEST_DB_USER_PASSWORD: test
-      TEST_DB_NAME: test
+      VERSIONS: 1.9.7 1.10.5 1.11.2
+      TEST_DB_PORT: 5432
       GOPATH_FOLDER: gopath
+      TEST_RANDOM_DATA_FOLDER: /tmp/test_data
     steps:
     # prepare
-      - run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev python python-setuptools python3 python3-setuptools python3-pip git rsync psmisc
-      - run: cd $HOME && git clone https://github.com/cossacklabs/themis && cd themis && sudo make install
-      - run: cd $HOME && for version in $VERSIONS; do mkdir go_root_$version; cd go_root_$version; wget https://storage.googleapis.com/golang/go$version.linux-amd64.tar.gz; tar xf go$version.linux-amd64.tar.gz; cd -; done
-      - run: mkdir $HOME/$GOPATH_FOLDER
       - checkout
-      - run: cd $HOME && mkdir -p $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/themis; mkdir -p $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/acra; ln -s $HOME/themis/gothemis $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/themis/gothemis; cp -r $HOME/project/* $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/acra/
-      - run: cd $HOME && GOPATH=$HOME/$GOPATH_FOLDER go get -d github.com/cossacklabs/acra/...
-      - run: pip3 install -r $HOME/project/tests/requirements.txt
-      # install from sources because pip install git+https://github.com/mysql/mysql-connector-python not support recursive submodules
-      - run: git clone https://github.com/Lagovas/mysql-connector-python; cd mysql-connector-python; sudo python3 setup.py clean build_py install_lib
-      - run: cd $HOME && GOPATH=$HOME/$GOPATH_FOLDER go get -u -v github.com/golang/lint/golint
-      - run: sudo ldconfig
+      - run: .circleci/prepare.sh
+      - run: sudo apt-get install -y postgresql-client
+      - run: pg_isready -U${POSTGRES_USER} -d${POSTGRES_DB} -h127.0.0.1
     # testing
       # check that code formatted with gofmt
       - run: .circleci/check_gofmt.sh
@@ -50,11 +33,98 @@ jobs:
       - run: cd $HOME && for version in $VERSIONS; do export GOROOT=$HOME/go_root_$version/go; export PATH=$GOROOT/bin/:$PATH; export GOPATH=$HOME/$GOPATH_FOLDER; rm -rf $HOME/$GOPATH_FOLDER/bin; rm -rf $HOME/$GOPATH_FOLDER/pkg; go test -v github.com/cossacklabs/acra/...; if [ "$?" != "0" ]; then echo "$version" >> "$FILEPATH_ERROR_FLAG"; fi done
       # if file exists (exit code of stat == 0 ) then something was wrong. cat file with versions of environments where was error and return exit 1
       - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+      # generate test data for integration tests
+      - run: python3 tests/generate_random_data.py
       # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
       - run: .circleci/integration.sh
       - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+  mysql:
+    docker:
+    - image: cossacklabs/android-build
+    # use the same credentials for mysql db as for postgresql (which support was added first)
+    # has latest tag on 2018.03.29
+    - image: mysql:5.7.21
+      environment:
+        MYSQL_DATABASE: test
+        MYSQL_USER: test
+        MYSQL_PASSWORD: test
+        MYSQL_ROOT_PASSWORD: root
+    environment:
+      GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
+      FILEPATH_ERROR_FLAG: /tmp/test_fail
+      VERSIONS: 1.9.7 1.10.5 1.11.2
+      TEST_MYSQL: true
+      TEST_DB_PORT: 3306
+      GOPATH_FOLDER: gopath
+      TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+    steps:
+    # prepare
+    - checkout
+    - run: .circleci/prepare.sh
+    - run: sudo apt-get install -y mysql-client
+    - run: mysqladmin ping -h127.0.0.1
+    # testing
+    # check that code formatted with gofmt
+    - run: .circleci/check_gofmt.sh
+    # check that code doesn't have a lot of golint issues (currently removed because golint version is very different on other planforms)
+    - run: GOPATH=$HOME/$GOPATH_FOLDER .circleci/check_golint.sh
+    # delete file if exists
+    - run: if [ -f $FILEPATH_ERROR_FLAG ]; then rm "$FILEPATH_ERROR_FLAG"; fi
+    # run test in each go environment and create $FILEPATH_ERROR_FLAG file if was any error. But all tests should
+    - run: cd $HOME && for version in $VERSIONS; do export GOROOT=$HOME/go_root_$version/go; export PATH=$GOROOT/bin/:$PATH; export GOPATH=$HOME/$GOPATH_FOLDER; rm -rf $HOME/$GOPATH_FOLDER/bin; rm -rf $HOME/$GOPATH_FOLDER/pkg; go test -v github.com/cossacklabs/acra/...; if [ "$?" != "0" ]; then echo "$version" >> "$FILEPATH_ERROR_FLAG"; fi done
+    # if file exists (exit code of stat == 0 ) then something was wrong. cat file with versions of environments where was error and return exit 1
+    - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+    # generate test data for integration tests
+    - run: python3 tests/generate_random_data.py
+    # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
+    - run: .circleci/integration.sh
+    - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+  mariadb:
+    docker:
+    - image: cossacklabs/android-build
+    # use the same credentials for mysql db as for postgresql (which support was added first)
+    # has latest tag on 2018.03.29
+    - image: mariadb:latest
+      environment:
+        MYSQL_DATABASE: test
+        MYSQL_USER: test
+        MYSQL_PASSWORD: test
+        MYSQL_ROOT_PASSWORD: root
+    environment:
+      GOTHEMIS_IMPORT: github.com/cossacklabs/themis/gothemis
+      FILEPATH_ERROR_FLAG: /tmp/test_fail
+      VERSIONS: 1.9.7 1.10.5 1.11.2
+      TEST_MYSQL: true
+      TEST_DB_PORT: 3306
+      GOPATH_FOLDER: gopath
+      TEST_RANDOM_DATA_FOLDER: /tmp/test_data
+    steps:
+    # prepare
+    - checkout
+    - run: .circleci/prepare.sh
+    - run: sudo apt-get install -y mysql-client
+    - run: mysqladmin ping -h127.0.0.1
+    # testing
+    # check that code formatted with gofmt
+    - run: .circleci/check_gofmt.sh
+    # check that code doesn't have a lot of golint issues (currently removed because golint version is very different on other planforms)
+    - run: GOPATH=$HOME/$GOPATH_FOLDER .circleci/check_golint.sh
+    # delete file if exists
+    - run: if [ -f $FILEPATH_ERROR_FLAG ]; then rm "$FILEPATH_ERROR_FLAG"; fi
+    # run test in each go environment and create $FILEPATH_ERROR_FLAG file if was any error. But all tests should
+    - run: cd $HOME && for version in $VERSIONS; do export GOROOT=$HOME/go_root_$version/go; export PATH=$GOROOT/bin/:$PATH; export GOPATH=$HOME/$GOPATH_FOLDER; rm -rf $HOME/$GOPATH_FOLDER/bin; rm -rf $HOME/$GOPATH_FOLDER/pkg; go test -v github.com/cossacklabs/acra/...; if [ "$?" != "0" ]; then echo "$version" >> "$FILEPATH_ERROR_FLAG"; fi done
+    # if file exists (exit code of stat == 0 ) then something was wrong. cat file with versions of environments where was error and return exit 1
+    - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
+    # generate test data for integration tests
+    - run: python3 tests/generate_random_data.py
+    # each iteration pass to test different ports for forks to avoid problems with TCP TIME WAIT between tests
+    - run: .circleci/integration.sh
+    - run: if [ -f  $FILEPATH_ERROR_FLAG ]; then cat "$FILEPATH_ERROR_FLAG"; rm "$FILEPATH_ERROR_FLAG"; exit 1; fi
 workflows:
   version: 2
   tests:
     jobs:
-      - x86_64
+      - postgresql
+      - mysql
+      # has errors with tls connections
+      #- mariadb

.circleci/integration.sh

@@ -3,6 +3,10 @@
 export TEST_ACRASERVER_PORT=6000
 export TEST_CONNECTOR_PORT=7000
 export TEST_CONNECTOR_COMMAND_PORT=8000
+export TEST_DB_USER=test
+export TEST_DB_USER_PASSWORD=test
+export TEST_DB_NAME=test
+
 cd $HOME/project
 for version in $VERSIONS; do
     echo "-------------------- Testing Go version $version"
@@ -14,46 +18,18 @@ for version in $VERSIONS; do
     export PATH=$GOROOT/bin/:$PATH;
     export GOPATH=$HOME/$GOPATH_FOLDER;
 
-    # setup postgresql credentials
-    #export TEST_DB_USER=${POSTGRES_USER}
-    #export TEST_DB_USER_PASSWORD=${POSTGRES_PASSWORD}
-    #export TEST_DB_NAME=postgres
-    export TEST_DB_PORT=5432
-    unset TEST_MYSQL
-
     export TEST_TLS=on
 
-    echo "--------------------  Testing POSTGRES with TEST_TLS=on"
+    echo "--------------------  Testing with TEST_TLS=on"
 
     python3 tests/test.py -v;
-    if [ "$?" != "0" ]; then echo "pgsql-$version" >> "$FILEPATH_ERROR_FLAG";
+    if [ "$?" != "0" ]; then echo "golang-$version" >> "$FILEPATH_ERROR_FLAG";
     fi
 
     export TEST_TLS=off
 
-    echo "--------------------  Testing POSTGRES with TEST_TLS=off"
+    echo "--------------------  Testing with TEST_TLS=off"
     python3 tests/test.py -v;
-    if [ "$?" != "0" ]; then echo "pgsql-$version" >> "$FILEPATH_ERROR_FLAG";
+    if [ "$?" != "0" ]; then echo "golang-$version" >> "$FILEPATH_ERROR_FLAG";
     fi
-
-    # setup mysql credentials
-    #export TEST_DB_USER=${MYSQL_USER}
-    #export TEST_DB_USER_PASSWORD=${MYSQL_PASSWORD}
-    #export TEST_DB_NAME=${MYSQL_DATABASE}
-    export TEST_DB_PORT=3306
-    export TEST_MYSQL=true
-
-
-    echo "--------------------  Testing TEST_MYSQL with TEST_TLS=off"
-    export TEST_TLS=off
-    python3 tests/test.py -v;
-    if [ "$?" != "0" ]; then echo "mysql-$version" >> "$FILEPATH_ERROR_FLAG";
-    fi
-
-    echo "--------------------  Testing TEST_MYSQL with TEST_TLS=on"
-    export TEST_TLS=on
-    python3 tests/test.py -v;
-    if [ "$?" != "0" ]; then echo "mysql-$version" >> "$FILEPATH_ERROR_FLAG";
-    fi
-
 done

.circleci/prepare.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev python python-setuptools python3 python3-setuptools python3-pip git rsync psmisc
+cd $HOME && git clone https://github.com/cossacklabs/themis && cd themis && sudo make install
+cd $HOME && for version in $VERSIONS; do mkdir go_root_$version; cd go_root_$version; wget https://storage.googleapis.com/golang/go$version.linux-amd64.tar.gz; tar xf go$version.linux-amd64.tar.gz; cd -; done
+mkdir $HOME/$GOPATH_FOLDER
+cd $HOME && mkdir -p $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/themis; mkdir -p $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/acra; ln -s $HOME/themis/gothemis $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/themis/gothemis; cp -r $HOME/project/* $HOME/$GOPATH_FOLDER/src/github.com/cossacklabs/acra/
+cd $HOME && GOPATH=$HOME/$GOPATH_FOLDER go get -d github.com/cossacklabs/acra/...
+pip3 install -r $HOME/project/tests/requirements.txt
+# install from sources because pip install git+https://github.com/mysql/mysql-connector-python not support recursive submodules
+git clone https://github.com/Lagovas/mysql-connector-python; cd mysql-connector-python; sudo python3 setup.py clean build_py install_lib
+cd $HOME && GOPATH=$HOME/$GOPATH_FOLDER go get -u -v golang.org/x/lint/golint
+sudo ldconfig

.gitignore

@@ -145,3 +145,12 @@ ENV/
 .ropeproject
 
 cmd/acra-webconfig/auth.keys
+
+# cmake
+examples/cpp/CMakeCache.txt
+examples/cpp/CMakeFiles
+examples/cpp/CMakeScripts
+examples/cpp/Testing
+examples/cpp/cmake-build-debug/*
+examples/cpp/cmake_install.cmake
+examples/cpp/cmake-build-debug

CHANGELOG.md

@@ -1,5 +1,87 @@
 # Acra ChangeLog
 
+## [0.84.0](https://github.com/cossacklabs/acra/releases/tag/0.84), November 9th 2018
+
+_Core_:
+
+- **Key management**
+
+  - Improved LRU cache: fixed concurrent access to LRU cache by adding mutex. LRU cache is used for quick access to in-memory keys (private keys are stored encrypted) in AcraServer and AcraTranslator ([#272](https://github.com/cossacklabs/acra/pull/272)).
+
+  [AcraServer documentation](https://docs.cossacklabs.com/pages/documentation-acra/#getting-started-with-acraserver), [AcraTranslator documentation](https://docs.cossacklabs.com/pages/acratranslator/).
+
+  - Improved AcraRotate utility: added "dry-run" mode for testing AcraRotate before it is used for real. In the "dry-run" mode AcraRotate doesn't rotate keys: it fetches AcraStructs (from files or database), decrypts, rotates in-memory keys, encrypts the data with new public keys and prints the resulting JSON with new public keys without actually saving the rotated keys and AcraStructs. As key rotation might be tricky, we want users to make sure that AcraRotate has all the required permissions and access right before actually re-encrypting the data ([#269](https://github.com/cossacklabs/acra/pull/269)).
+
+  [AcraRotate documentation](https://docs.cossacklabs.com/pages/acrarotate/).
+
+- **AcraWriter**
+
+  - Added C++ AcraWriter library, added examples and tests. The library itself is a single header-only file `acrawriter.hpp` with dependency on Themis, placed in [wrappers/cpp](https://github.com/cossacklabs/acra/tree/master/wrappers/cpp). 
+  Read the usage guide and examples in [examples/cpp](https://github.com/cossacklabs/acra/tree/master/examples/cpp) folder 
+  ([#270](https://github.com/cossacklabs/acra/pull/270))
+
+  [AcraWriter C++ documentation](https://docs.cossacklabs.com/pages/documentation-acra/#building-acrawriter-for-c-).
+
+- **Logging**
+
+  - Improved logs of AcraConnector and AcraServer: use Debug log level for all network errors (closed connection, unavailable network, etc) and use Error log level only for cases of certainly unexpected behavior ([#275](https://github.com/cossacklabs/acra/pull/275)).
+
+  - Improved startup logs: log process PID on start of AcraServer, AcraConnector, AcraTranslator, and AcraWebConfig ([#275](https://github.com/cossacklabs/acra/pull/275)).
+
+  - Fixed timestamps: do not overwrite logs' timestamps ([#273](https://github.com/cossacklabs/acra/pull/273)).
+
+- **Tracing with OpenCensus**
+
+  - Added tracing with OpenCensus: AcraServer, AcraConnector, and AcraTranslator track every request from client application to the database and back. Each client request has a unique `traceID` that helps measure how much time it needs to perform a certain data processing functions (i.e. checking requests via AcraCensor, encrypting data, decrypting AcraStructs, etc.). Traces can be exported to Jaeger ([#279](https://github.com/cossacklabs/acra/pull/279), [#276](https://github.com/cossacklabs/acra/pull/276), [#274](https://github.com/cossacklabs/acra/pull/274)). 
+
+  You can read more about tracing in our documentation in [Tracing in Acra](https://docs.cossacklabs.com/pages/documentation-acra/#tracing-in-acra).
+
+  A blogpost about technical details, profits, and pitfalls during the implementation of traces is coming soon.
+
+- **Other**
+
+  - Improved AcraServer's connection handling: stop accepting connections after error and stop AcraServer instead of trying to accept connections after the listening socket was closed ([#275](https://github.com/cossacklabs/acra/pull/275).
+
+  - Improved AcraCensor's handling of prepared statements for PostgreSQL binary protocol ([#280](https://github.com/cossacklabs/acra/pull/280)).
+
+  - Improved handling of terminating packets (COM_QUIT for PostgreSQL and TerminatePacket for MySQL) to correctly handle the closing connections from clients ([#275](https://github.com/cossacklabs/acra/pull/275).
+
+  - Refactored inner logic of AcraCensor: improved code quality and stability, added more tests that use more patterns ([#268](https://github.com/cossacklabs/acra/pull/268)).
+
+
+_Infrastructure_:
+
+- Ceased testing and supporting Go versions below 1.9. This will only affect the users who build Acra manually from sources.
+  You can install the pre-built Acra components shipped for various Ubuntu, Debian, and CentOS distributives using [Installation guide](https://github.com/cossacklabs/acra/wiki/Quick-start-guide#installing-acra-from-the-cossack-labs-repository). Alternatively, you can check out our Docker images and Docker-compose files in [docker folder](https://github.com/cossacklabs/acra/tree/master/docker) 
+  ([#277](https://github.com/cossacklabs/acra/pull/277)).
+
+- Tested Acra suite with PostgreSQL v11 and MariaDB v10.3 and updated [docker compose examples](https://github.com/cossacklabs/acra/tree/master/docker) and [Acra Engineering Demo](https://github.com/cossacklabs/acra-engineering-demo/) to use it ([#278](https://github.com/cossacklabs/acra/pull/278)).
+
+- Published [Acra load balancing demo](https://github.com/cossacklabs/acra-balancer-demo): it discovers some of the many possible variants of building high availability and balanced infrastructure based on Acra data protection suite components, PostgreSQL, and Python web application. In these examples we used HAProxy – one of the most popular high availability balancers today.
+
+- Updated [AcraStruct Validator](https://docs.cossacklabs.com/simulator/acra/) – an online tool that can decrypt your AcraStructs. AcraStruct Validator is useful for developers who build their own AcraWriters (to validate AcraStruct binary structure).
+
+
+_Features coming soon_:
+
+- Running SQL queries over encrypted data: perform AcraServer-side lookups (search) over protected data.
+
+- Pseudonymisation: early version of pseudonymisation library/plugin for Acra for transparent data pseudonymisation.
+
+- Cryptographically protected audit log: protection for logs against tampering.
+
+
+_Documentation_:
+
+- [AcraWriter C++](https://docs.cossacklabs.com/pages/documentation-acra/#building-acrawriter-for-c-) has a short guide for installing and using AcraWriter for C++.
+
+- [AcraRotate](https://docs.cossacklabs.com/pages/acrarotate/): added description and notes about "dry-run" mode.
+
+- Updated documentation for [logging](https://docs.cossacklabs.com/pages/documentation-acra/#logging-in-acra), [collecting metrics](https://docs.cossacklabs.com/pages/documentation-acra/#metrics-in-acra), and [tracing](https://docs.cossacklabs.com/pages/documentation-acra/#tracing-in-acra) in Acra.
+
+- Many small fixes here and there to make your overall experience of using Acra's docs on a new platform distinctive and smooth ;)
+
+
 ## [0.83.0](https://github.com/cossacklabs/acra/releases/tag/0.83), September 28th 2018
 
 _Core_: