update synapse, PostgreSQL migration

- opened up 8448 to matrix clients for probable resolution of hardcoded integrations server ports in Riot element-hq/element-web#3329 (comment)
- migrated to PostgreSQL due to speed issues
- learned opening twentythousand ports in Docker creates twentythousand docker-proxy processes for this, which makes the daemon run into cgroups memory issues

docker-compose.yml

@@ -15,9 +15,13 @@ services:
       - "8080"
     volumes:
       - /srv/matrix/allmende:/data
+    logging:
+      driver: none
+    networks:
+      - frontend
 
   matrix:
-    image: allmende/synapse:v0.18.7
+    image: allmende/synapse:v0.21.1
     restart: always
     command: start
     environment:
@@ -27,13 +31,18 @@ services:
       LETSENCRYPT_EMAIL: almereyda@allmende.io
     expose:
       - "8080"
+    ports:
+      - "8448:8448"
     volumes:
       - /srv/matrix/allmende:/data
       - /srv/nginx/certs/matrix.allmende.io/fullchain.pem:/data/matrix.allmende.io.crt
       - /srv/nginx/certs/matrix.allmende.io/key.pem:/data/matrix.allmende.io.key
       - /srv/nginx/certs/dhparam.pem:/data/matrix.allmende.io.dhparam.pem
     logging:
       driver: none
+    networks:
+      - frontend
+      - backend
 
   coturn:
     image: allmende/coturn
@@ -45,11 +54,32 @@ services:
       - "5349"
       - "5350"
     ports:
+      - "3478:3478"
       - "3478:3478/udp"
+      - "3479:3479"
+      - "3479:3479/udp"
+      - "5349:5349"
+      - "5349:5349/udp"
+      - "5350:5350"
+      - "5350:5350/udp"
+#      - "49152-65535:49152-65535/udp"
     volumes:
       - /srv/matrix/allmende:/data
 
+  data:
+    image: "postgres:9.6.3-alpine"
+    restart: always
+    volumes:
+      - /srv/matrix/database:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: matrix
+      POSTGRES_PASSWORD:
+    networks:
+      - backend
+
 networks:
-  default:
+  frontend:
     external:
       name: nginx_default
+  backend:
+    driver: bridge

generate.yml

@@ -10,6 +10,19 @@ services:
     environment:
       - SERVER_NAME=matrix.allmende.io
 
+  migrate:
+    image: allmende/synapse:v0.18.7
+    command: migrate
+    volumes:
+      - /srv/matrix/allmende:/data
+      - ./jobs.sh:/start.sh
+    environment:
+      - REPORT_STATS=yes
+      - SERVER_NAME=matrix.allmende.io
+      - TERM
+    networks:
+      - live
+
   matrix:
     image: allmende/synapse
     command: generate
@@ -28,3 +41,8 @@ services:
       - /srv/matrix/example:/data
     environment:
       - SERVER_NAME=matrix.allmende.io
+
+networks:
+  live:
+    external:
+      name: docker_backend

jobs.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+OPTION="${1}"
+
+if [ ! -z "${ROOTPATH}" ]; then
+	echo ":: We have changed the semantic and doesn't need the ROOTPATH"
+	echo ":: variable anymore"
+fi
+
+case $OPTION in
+	"start")
+		echo "-=> start matrix"
+		python -m synapse.app.homeserver \
+		       --config-path /data/homeserver.yaml \
+		;;
+	"stop")
+		echo "-=> stop matrix"
+		echo "-=> via docker stop ..."
+		;;
+	"version")
+		VERSION=$(tail -n 1 /synapse.version)
+		echo "-=> Matrix Version: ${VERSION}"
+		;;
+	"migrate")
+		synapse_port_db \
+			--sqlite-database /data/homeserver.db \
+			--postgres-config /data/homeserver.yaml \
+			--curses
+		;;
+	"generate")
+		breakup="0"
+		[[ -z "${SERVER_NAME}" ]] && echo "STOP! environment variable SERVER_NAME must be set" && breakup="1"
+		[[ -z "${REPORT_STATS}" ]] && echo "STOP! environment variable REPORT_STATS must be set to 'no' or 'yes'" && breakup="1"
+		[[ "${breakup}" == "1" ]] && exit 1
+
+		[[ "${REPORT_STATS}" != "yes" ]] && [[ "${REPORT_STATS}" != "no" ]] && \
+			echo "STOP! REPORT_STATS needs to be 'no' or 'yes'" && breakup="1"
+
+		echo "-=> generate synapse config"
+		python -m synapse.app.homeserver \
+		       --config-path /data/homeserver.yaml \
+		       --generate-config \
+		       --report-stats ${REPORT_STATS} \
+		       --server-name ${SERVER_NAME}
+
+		export TURNKEY=$(</data/TURNKEY)
+		echo "-=> configure some settings in homeserver.yaml"
+		awk -v SERVER_NAME="${SERVERNAME}" \
+		    -v TURNURIES="turn_uris: [\"turn:${SERVER_NAME}:3478?transport=udp\", \"turn:${SERVER_NAME}:3478?transport=tcp\"]" \
+		    -v TURNSHAREDSECRET="turn_shared_secret: \"${TURNKEY}\"" \
+		    -v PIDFILE="pid_file: /data/homeserver.pid" \
+		    -v DATABASE="database: \"/data/homeserver.db\"" \
+		    -v LOGFILE="log_file: \"/data/homeserver.log\"" \
+		    -v MEDIASTORE="media_store_path: \"/data/media_store\"" \
+		    '{
+			sub(/turn_shared_secret: "YOUR_SHARED_SECRET"/, TURNSHAREDSECRET);
+			sub(/turn_uris: \[\]/, TURNURIES);
+			sub(/pid_file: \/homeserver.pid/, PIDFILE);
+			sub(/database: "\/homeserver.db"/, DATABASE);
+			sub(/log_file: "\/homeserver.log"/, LOGFILE);
+			sub(/media_store_path: "\/media_store"/, MEDIASTORE);
+			print;
+		    }' /data/homeserver.yaml > /data/homeserver.tmp
+		mv /data/homeserver.tmp /data/homeserver.yaml
+
+		echo "-=> you can now review the generated configuration file homeserver.yaml"
+		;;
+	*)
+		echo "-=> unknown \'$OPTION\'"
+		;;
+esac
+

readme.md

@@ -92,3 +92,10 @@ To debug the environments instead of the source images, use
     docker-compose rm
 
 instead.
+
+## Migration
+
+Create the database container first and start it, attached to the desired network.
+Then run the migration with
+
+    docker-compose -f generate.yml run --rm migrate