Heroku installation and configuration

README.md

@@ -211,6 +211,13 @@ helm install -f chart-values-k8s-pg-storage.yml pgwatch2 helm-chart-k8s-pg-stora
 
 Please have a look at the according (K8s or OpenShift) `values.yaml` files to get additional information of configurable options.
 
+# Heroku
+
+It's easy to install pgwatch2 to Heroku and run it on both Common Runtime and Private Spaces - there is a set Terraform scripts that can be used to install and configure all the required components.
+
+Refer to the [documentation](https://pgwatch2.readthedocs.io/en/latest/heroku.html) for details.
+
+
 # Contributing
 
 Feedback, suggestions, problem reports and pull requests are very much appreciated.

build-and-push-to-heroku-docker-daemon-collector.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+: "${HK_APP_NAME:=ab-cr-pgwatch2-collector-v1120}"
+: "${HK_PROC_TYPE:=collector}"
+: "${DOCKER_FILE:=docker/Dockerfile-daemon-heroku}"
+
+heroku container:login && \
+docker build -t registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} -f ${DOCKER_FILE} . && \
+docker push registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} && \
+heroku container:release ${HK_PROC_TYPE} -a ${HK_APP_NAME}

build-and-push-to-heroku-docker-db-bootstrapper.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+: "${HK_APP_NAME:=ab-cr-pgwatch2-collector-v1120}"
+: "${HK_PROC_TYPE:=db-bootstrapper}"
+: "${DOCKER_FILE:=docker/Dockerfile-db-bootstrapper-heroku}"
+
+heroku container:login && \
+docker build -t registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} -f ${DOCKER_FILE} . && \
+docker push registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} && \
+heroku container:release ${HK_PROC_TYPE} -a ${HK_APP_NAME}

build-and-push-to-heroku-docker-grafana-db-bootstrapper.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+: "${HK_APP_NAME:=ab-cr-pgwatch2-collector-v1120}"
+: "${HK_PROC_TYPE:=grafana-db-bootstrapper}"
+: "${DOCKER_FILE:=docker/Dockerfile-grafana-db-bootstrapper-heroku}"
+
+heroku container:login && \
+docker build -t registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} -f ${DOCKER_FILE} . && \
+docker push registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} && \
+heroku container:release ${HK_PROC_TYPE} -a ${HK_APP_NAME}

build-and-push-to-heroku-docker-grafanaV10.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+: "${HK_APP_NAME:=ab-cr-pgwatch2-grafana-v1120}"
+: "${HK_PROC_TYPE:=web}"
+: "${DOCKER_FILE:=docker/Dockerfile-grafanaV10-heroku}"
+
+heroku container:login && \
+docker build -t registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} -f ${DOCKER_FILE} . && \
+docker push registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} && \
+heroku container:release ${HK_PROC_TYPE} -a ${HK_APP_NAME}

build-and-push-to-heroku-docker-monitored-db-bootstrapper.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+: "${HK_APP_NAME:=ab-cr-pgwatch2-targetdb}"
+: "${HK_PROC_TYPE:=monitored-db-bootstrapper}"
+: "${DOCKER_FILE:=docker/Dockerfile-monitored-db-bootstrapper-heroku}"
+
+heroku container:login && \
+docker build -t registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} -f ${DOCKER_FILE} . && \
+docker push registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} && \
+heroku container:release ${HK_PROC_TYPE} -a ${HK_APP_NAME}

build-and-push-to-heroku-docker-pgbench.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+: "${HK_APP_NAME:=ab-cr-pgwatch2-targetdb}"
+: "${HK_PROC_TYPE:=pgbench}"
+: "${DOCKER_FILE:=docker/Dockerfile-pgbench-heroku}"
+
+heroku container:login && \
+docker build -t registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} -f ${DOCKER_FILE} . && \
+docker push registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} && \
+heroku container:release ${HK_PROC_TYPE} -a ${HK_APP_NAME}

build-and-push-to-heroku-docker-webui.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+: "${HK_APP_NAME:=ab-cr-pgwatch2-collector-v1120}"
+: "${HK_PROC_TYPE:=web}"
+: "${DOCKER_FILE:=docker/Dockerfile-webui-heroku}"
+
+heroku container:login && \
+docker build -t registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} -f ${DOCKER_FILE} . && \
+docker push registry.heroku.com/${HK_APP_NAME}/${HK_PROC_TYPE} && \
+heroku container:release ${HK_PROC_TYPE} -a ${HK_APP_NAME}

docker/Dockerfile-daemon-heroku

@@ -0,0 +1,41 @@
+FROM golang:1.21
+
+# For showing Git version via 'pgwatch2 --version'
+ARG GIT_HASH
+ARG GIT_TIME
+ENV GIT_HASH=${GIT_HASH}
+ENV GIT_TIME=${GIT_TIME}
+
+ADD pgwatch2 /pgwatch2
+RUN cd /pgwatch2 && bash build_gatherer.sh
+
+
+FROM ubuntu:22.04
+
+RUN apt-get -q update && DEBIAN_FRONTEND=noninteractive apt-get install -qy curl ca-certificates gnupg \
+  && echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+  && curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && apt-get -q update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -qy postgresql-client-15 wget unzip git && apt autoremove -y \
+  && useradd -m -s /bin/bash postgres
+
+#RUN apt-get -q update && apt-get -qy install wget git && apt autoremove -y && mkdir /pgwatch2
+
+ADD pgwatch2/metrics /pgwatch2/metrics
+ADD pgwatch2/config /pgwatch2/config
+
+COPY docker/launch-wrapper-daemon-heroku.sh /pgwatch2/launch-wrapper-daemon-heroku.sh
+
+# Copy over the compiled gatherer
+COPY --from=0 /pgwatch2/pgwatch2 /pgwatch2
+
+RUN chgrp -R 0 /pgwatch2 \
+    && chmod -R g=u /pgwatch2
+
+# pgwatch2 internal status endpoint
+EXPOSE 8081
+# Prometheus metrics scraping port
+EXPOSE 9187
+
+USER 10001
+
+CMD ["/pgwatch2/launch-wrapper-daemon-heroku.sh"]

docker/Dockerfile-db-bootstrapper-heroku

@@ -0,0 +1,26 @@
+FROM ubuntu:22.04
+
+RUN apt-get -q update && DEBIAN_FRONTEND=noninteractive apt-get install -qy curl ca-certificates gnupg \
+  && echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+  && curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && apt-get -q update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -qy postgresql-client-15 wget unzip \
+  && useradd -m -s /bin/bash postgres
+
+USER postgres
+
+COPY pgwatch2/sql/ /pgwatch2/sql
+COPY docker/launch-wrapper-db-bootstrapper-heroku.sh /pgwatch2/launch-wrapper-db-bootstrapper-heroku.sh
+
+ENV PGSSLMODE require
+ENV ON_ERROR_STOP 1
+
+# configdb | metricsdb. Required
+ENV BOOTSTRAP_TYPE configdb
+
+ENV BOOTSTRAP_ADD_TEST_MONITORING_ENTRY 1
+
+# TODO only metric-time is supported currently
+# metric-time | metric-dbname-time | timescale
+ENV BOOTSTRAP_METRICSDB_SCHEMA_TYPE metric-time
+
+CMD ["/pgwatch2/launch-wrapper-db-bootstrapper-heroku.sh"]

docker/Dockerfile-grafana-db-bootstrapper-heroku

@@ -0,0 +1,16 @@
+FROM ubuntu:22.04
+
+RUN apt-get -q update && DEBIAN_FRONTEND=noninteractive apt-get install -qy curl ca-certificates gnupg \
+  && echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+  && curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && apt-get -q update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -qy postgresql-client-15 wget unzip \
+  && useradd -m -s /bin/bash postgres
+
+USER postgres
+
+COPY docker/launch-wrapper-grafana-db-bootstrapper-heroku.sh /pgwatch2/launch-wrapper-grafana-db-bootstrapper-heroku.sh
+
+ENV PGSSLMODE require
+ENV ON_ERROR_STOP 1
+
+CMD ["/pgwatch2/launch-wrapper-grafana-db-bootstrapper-heroku.sh"]

docker/Dockerfile-grafanaV10-heroku

@@ -0,0 +1,27 @@
+FROM grafana/grafana-enterprise:10.4.0
+
+# Grafana config customizations
+RUN grafana-cli plugins install savantly-heatmap-panel \
+    && grafana-cli plugins install marcusolsson-treemap-panel \
+    && grafana-cli plugins update-all
+
+ARG GRAFANA_DASHBOARDS_PROVISIONING_PATH=/etc/grafana/provisioning/dashboards
+ARG GRAFANA_DATASOURCES_PROVISIONING_PATH=/etc/grafana/provisioning/datasources
+
+
+# Grafana dashboards provisioning
+ADD grafana_dashboards/postgres_heroku/v10 ${GRAFANA_DASHBOARDS_PROVISIONING_PATH}/pgwatch2_postgres_grafana_dashboards
+ADD grafana_dashboards/pgwatch2-V10.yaml ${GRAFANA_DASHBOARDS_PROVISIONING_PATH}/
+ADD grafana_datasources/pgwatch2-V10.yaml ${GRAFANA_DATASOURCES_PROVISIONING_PATH}/
+
+COPY docker/launch-wrapper-grafanaV10-heroku.sh /pgwatch2/launch-wrapper-grafanaV10-heroku.sh
+
+EXPOSE 3000
+
+# to reset the parent ENTRYPOINT that would otherwise be called on Heroku
+ENTRYPOINT []
+
+CMD ["/pgwatch2/launch-wrapper-grafanaV10-heroku.sh"]
+
+
+

docker/Dockerfile-monitored-db-bootstrapper-heroku

@@ -0,0 +1,17 @@
+FROM ubuntu:22.04
+
+RUN apt-get -q update && DEBIAN_FRONTEND=noninteractive apt-get install -qy curl ca-certificates gnupg \
+  && echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+  && curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && apt-get -q update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -qy postgresql-client-15 wget unzip \
+  && useradd -m -s /bin/bash postgres
+
+USER postgres
+
+COPY pgwatch2/metrics/ /pgwatch2/metrics
+COPY docker/launch-wrapper-monitored-db-bootstrapper-heroku.sh /pgwatch2/launch-wrapper-monitored-db-bootstrapper-heroku.sh
+
+ENV PGSSLMODE require
+ENV ON_ERROR_STOP 1
+
+CMD ["/pgwatch2/launch-wrapper-monitored-db-bootstrapper-heroku.sh"]

docker/Dockerfile-pgbench-heroku

@@ -0,0 +1,10 @@
+FROM postgres:15
+
+
+RUN mkdir -p /pgwatch2
+ADD docker/launch-wrapper-pgbench-heroku.sh /pgwatch2
+
+# to reset the parent ENTRYPOINT that would otherwise be called on Heroku
+ENTRYPOINT []
+
+CMD ["/pgwatch2/launch-wrapper-pgbench-heroku.sh"]

docker/Dockerfile-webui-heroku

@@ -0,0 +1,30 @@
+FROM ubuntu:22.04
+
+RUN apt-get -q update && apt-get -qy install python3-pip \
+    && mkdir /pgwatch2
+
+###
+### add webpy source and configure installed components
+###
+
+ADD webpy /pgwatch2/webpy
+ADD docker/launch-wrapper-webui-heroku.sh /pgwatch2
+
+# Get Web UI requirements
+RUN pip3 install -U "pip < 21.0" && pip3 install -r /pgwatch2/webpy/requirements_pg_metrics.txt \
+    && pip3 install -r /pgwatch2/webpy/requirements_pg_metrics.txt \
+    && chgrp -R 0 /pgwatch2 \
+    && chmod -R g=u /pgwatch2 \
+    && mkdir /pgwatch2/persistent-config \
+    && chgrp -R 0 /pgwatch2/webpy /pgwatch2/persistent-config \
+    && chmod -R g=u /pgwatch2/webpy /pgwatch2/persistent-config
+
+
+# Admin UI for configuring servers to be monitored
+EXPOSE 8080
+
+VOLUME /pgwatch2/persistent-config
+
+USER 10001
+
+CMD ["/pgwatch2/launch-wrapper-webui-heroku.sh"]

docker/launch-wrapper-daemon-heroku.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# the following is required to use alias psql within the script
+shopt -s expand_aliases
+alias psqla="psql ${DATABASE_URL} -v ON_ERROR_STOP=1"
+
+echo "pgwatch2 collector starting ..."
+
+#This regular expression is used to parse and extract information from a PostgreSQL database connection string in the format "postgres://username:password@host:port/database".
+#
+# - "^postgres://" is the starting point of the string, indicating that it begins with the protocol "postgres://".
+# - "([^:]+)" captures any characters that are not a colon ":" one or more times, representing the username in the database connection string.
+# - "([^@]+)" captures any characters that are not an "@" symbol one or more times, representing the password in the database connection string.
+# - "@([^:]+)" captures any characters that are not a colon ":" after the "@" symbol, representing the host or server address in the database connection string.
+# - "([^/]+)" captures any characters that are not a forward slash "/" one or more times, representing the port number in the database connection string.
+# - "/(.*)" captures any characters after the forward slash "/", representing the database name in the connection string.
+
+regex="^postgres://([^:]+):([^@]+)@([^:]+):([^/]+)/(.*)$"
+if [[ $PGWATCH2_URL =~ $regex ]]; then
+
+export PW2_DATASTORE="postgres"
+export PW2_PGUSER=${BASH_REMATCH[1]}
+export PW2_PGPASSWORD=${BASH_REMATCH[2]}
+export PW2_PGHOST=${BASH_REMATCH[3]}
+export PW2_PGPORT=${BASH_REMATCH[4]}
+export PW2_PGDATABASE=${BASH_REMATCH[5]}
+export PW2_PGSSL="true"
+
+#export PW2_PG_METRIC_STORE_CONN_STR=$(echo ${PGWATCH2_URL} | sed 's/postgres:\/\//postgresql:\/\//')?sslmode=require
+export PW2_PG_METRIC_STORE_CONN_STR=${PGWATCH2_URL}?sslmode=require
+
+
+# PW2_VERBOSE: vv
+
+
+# loop through all the PGWATCH2_MONITOREDDB_ attached credentials of the monitored DBs and upsert their configuration records
+# it allows to automatically insert/update those records even when credentials are updated
+
+echo "looking for PGWATCH2_MONITOREDDB_ attached credentials to be added automatically ..."
+
+for monitoreddb_env_var in $(env | grep ^PGWATCH2_MONITOREDDB_); do
+    MDB_VAR_NAME=$(echo $monitoreddb_env_var | cut -d= -f1)
+    MDB_VAR_VALUE=$(echo $monitoreddb_env_var | cut -d= -f2)   
+
+if [[ $MDB_VAR_VALUE =~ $regex ]]; then
+    MONITOREDDB_PGUSER=${BASH_REMATCH[1]}
+    MONITOREDDB_PGPASSWORD=${BASH_REMATCH[2]}
+    MONITOREDDB_PGHOST=${BASH_REMATCH[3]}
+    MONITOREDDB_PGPORT=${BASH_REMATCH[4]}
+    MONITOREDDB_PGDATABASE=${BASH_REMATCH[5]}
+
+    echo "upserting $MDB_VAR_NAME into pgwatch2.monitored_db ..."
+
+    SQL_INS=$(cat <<-EOF
+INSERT INTO pgwatch2.monitored_db (md_unique_name, md_preset_config_name, md_hostname, md_port, md_dbname, md_user, md_password, md_password_type, md_sslmode)
+  select '$MDB_VAR_NAME', 'heroku_postgres', '$MONITOREDDB_PGHOST', '$MONITOREDDB_PGPORT', '$MONITOREDDB_PGDATABASE', '$MONITOREDDB_PGUSER', '$MONITOREDDB_PGPASSWORD', 'aes-gcm-256', 'require'
+  ON CONFLICT (md_unique_name) DO UPDATE SET md_hostname = '$MONITOREDDB_PGHOST', md_port = '$MONITOREDDB_PGPORT', md_dbname = '$MONITOREDDB_PGDATABASE', md_user = '$MONITOREDDB_PGUSER', md_password = '$MONITOREDDB_PGPASSWORD' 
+  WHERE pgwatch2.monitored_db.md_unique_name = '$MDB_VAR_NAME' AND pgwatch2.monitored_db.md_hostname != '$MONITOREDDB_PGHOST' OR pgwatch2.monitored_db.md_port != '$MONITOREDDB_PGPORT' OR pgwatch2.monitored_db.md_dbname != '$MONITOREDDB_PGDATABASE' OR pgwatch2.monitored_db.md_user != '$MONITOREDDB_PGUSER' OR pgwatch2.monitored_db.md_password != '$MONITOREDDB_PGPASSWORD' ;
+EOF
+)
+    echo "$SQL_INS" | psqla -f-
+fi
+done
+
+
+/pgwatch2/pgwatch2
+
+else
+echo "PGWATCH2_URL config var is not defined, exiting..."
+
+fi