add config files to int tests (#368)

* add config files to int tests * add config files to int tests * fix names in tests * add previous pr's changes * fix crypto_test config * fix travis.yml * fix travis.yml
wal-g · Aug 12, 2019 · 0a8cb00 · 0a8cb00
1 parent ea9bddf
commit 0a8cb00
Show file tree

Hide file tree

Showing 36 changed files with 326 additions and 216 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -21,7 +21,8 @@ jobs:
     - script: make ARGS="pg_config_test" pg_integration_tests_with_args
     - script: make ARGS="pg_delete_before_name_find_full_test" pg_integration_tests_with_args
     - script: make ARGS="pg_delete_retain_full_test" pg_integration_tests_with_args
-    - script: make ARGS="pg_full_backup_test" pg_integration_tests_with_args
+    - script: make ARGS="pg_delete_before_permanent_full_test" pg_integration_tests_with_args
+    - script: make ARGS="pg_delete_before_permanent_delta_test" pg_integration_tests_with_args
     - script: make ARGS="pg_delete_before_time_find_full_test" pg_integration_tests_with_args
     - script: make ARGS="pg_delete_without_confirm_test" pg_integration_tests_with_args
     - script: make ARGS="pg_ghost_table_test" pg_integration_tests_with_args
@@ -30,7 +31,6 @@ jobs:
     - script: make ARGS="pg_several_delta_backups_test" pg_integration_tests_with_args
     - script: make ARGS="pg_crypto_test" pg_integration_tests_with_args
     - script: make ARGS="pg_delete_retain_find_full_test" pg_integration_tests_with_args
-    - script: make ARGS="pg_delta_backup_wal_delta_test" pg_integration_tests_with_args
     - script: make ARGS="pg_wale_compatibility_test" pg_integration_tests_with_args
     - script: make mysql_test
     - script: make mongo_test

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -80,13 +80,23 @@ services:
     links:
       - s3
 
+  pg_delete_before_permanent_full_test:
+    build:
+      dockerfile: docker/pg_tests/Dockerfile_delete_before_permanent_full_test
+      context: .
+    image: wal-g/delete_before_permanent_full_test
+    container_name: wal-g_pg_delete_before_permanent_full_test
+    depends_on:
+      - s3
+    links:
+      - s3
 
-  pg_delete_before_name_find_full_test:
+  pg_delete_before_permanent_delta_test:
     build:
-      dockerfile: docker/pg_tests/Dockerfile_delete_before_name_find_full_test
+      dockerfile: docker/pg_tests/Dockerfile_delete_before_permanent_delta_test
       context: .
-    image: wal-g/delete_before_name_find_full_test
-    container_name: wal-g_pg_delete_before_name_find_full_test
+    image: wal-g/delete_before_permanent_delta_test
+    container_name: wal-g_pg_delete_before_permanent_delta_test
     env_file:
       - docker/common/common_walg.env
       - docker/pg_tests/walg.env
@@ -95,187 +105,149 @@ services:
     links:
       - s3
 
+  pg_delete_before_name_find_full_test:
+    build:
+      dockerfile: docker/pg_tests/Dockerfile_delete_before_name_find_full_test
+      context: .
+    image: wal-g/delete_before_name_find_full_test
+    container_name: wal-g_pg_delete_before_name_find_full_test
+    depends_on:
+      - s3
+    links:
+      - s3
 
   pg_delete_retain_full_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_delete_retain_full_test
       context: .
     image: wal-g/delete_retain_full_test
     container_name: wal-g_pg_delete_retain_full_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_full_backup_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_full_backup_test
       context: .
     image: wal-g/full_backup_test
     container_name: wal-g_pg_full_backup_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_delete_before_time_find_full_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_delete_before_time_find_full_test
       context: .
     image: wal-g/delete_before_time_find_full_test
     container_name: wal-g_pg_delete_before_time_find_full_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_delete_without_confirm_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_delete_without_confirm_test
       context: .
     image: wal-g/delete_without_confirm_test
     container_name: wal-g_pg_delete_without_confirm_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_ghost_table_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_ghost_table_test
       context: .
     image: wal-g/ghost_table_test
     container_name: wal-g_pg_ghost_table_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_config_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_config_test
       context: .
     image: wal-g/config_test
     container_name: wal-g_pg_config_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_delete_end_to_end_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_delete_end_to_end_test
       context: .
     image: wal-g/delete_end_to_end_test
     container_name: wal-g_pg_delete_end_to_end_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_delta_backup_fullscan_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_delta_backup_fullscan_test
       context: .
     image: wal-g/delta_backup_fullscan_test
     container_name: wal-g_pg_delta_backup_fullscan_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_several_delta_backups_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_several_delta_backups_test
       context: .
     image: wal-g/several_delta_backups_test
     container_name: wal-g_pg_several_delta_backups_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_crypto_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_crypto_test
       context: .
     image: wal-g/crypto_test
     container_name: wal-g_pg_crypto_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_delete_retain_find_full_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_delete_retain_find_full_test
       context: .
     image: wal-g/delete_retain_find_full_test
     container_name: wal-g_pg_delete_retain_find_full_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_delta_backup_wal_delta_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_delta_backup_wal_delta_test
       context: .
     image: wal-g/delta_backup_wal_delta_test
     container_name: wal-g_pg_delta_backup_wal_delta_test
-    env_file:
-      - docker/common/common_walg.env
-      - docker/pg_tests/walg.env
     depends_on:
       - s3
     links:
       - s3
 
-
   pg_wale_compatibility_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_wale_compatibility_test

diff --git a/docker/pg_tests/Dockerfile_delete_before_permanent_delta_test b/docker/pg_tests/Dockerfile_delete_before_permanent_delta_test
@@ -0,0 +1,3 @@
+FROM wal-g/docker_prefix:latest
+
+CMD su postgres -c "/tmp/tests/delete_before_permanent_delta_test.sh"
diff --git a/docker/pg_tests/Dockerfile_delete_before_permanent_full_test b/docker/pg_tests/Dockerfile_delete_before_permanent_full_test
@@ -0,0 +1,3 @@
+FROM wal-g/docker_prefix:latest
+
+CMD su postgres -c "/tmp/tests/delete_before_permanent_full_test.sh"
diff --git a/docker/pg_tests/Dockerfile_prefix b/docker/pg_tests/Dockerfile_prefix
@@ -25,3 +25,4 @@ COPY --from=build /go/src/github.com/wal-g/wal-g/main/pg/wal-g /usr/bin
 
 COPY docker/pg_tests/scripts/ /tmp
 
+RUN chmod 777 tmp/configs/
diff --git a/docker/pg_tests/scripts/configs/common_config.json b/docker/pg_tests/scripts/configs/common_config.json
@@ -0,0 +1,11 @@
+"AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+"PGSSLMODE": "allow",
+"WALG_NETWORK_RATE_LIMIT": "10485760",
+"WALG_DISK_RATE_LIMIT": "41943040",
+"AWS_ACCESS_KEY_ID": "AKIAIOSFODNN7EXAMPLE",
+"WALG_UPLOAD_CONCURRENCY": "10",
+"AWS_ENDPOINT": "http://s3:9000",
+"PGDATABASE": "postgres",
+"AWS_S3_FORCE_PATH_STYLE": "true",
+"WALG_COMPRESSION_METHOD": "brotli",
+"PGHOST": "/var/run/postgresql"
diff --git a/docker/pg_tests/scripts/configs/config_test_config.json b/docker/pg_tests/scripts/configs/config_test_config.json
@@ -0,0 +1,2 @@
+"WALG_DELTA_MAX_STEPS": "6",
+"WALG_FILE_PREFIX": "/tmp/storage"
diff --git a/docker/pg_tests/scripts/configs/crypto_test_config.json b/docker/pg_tests/scripts/configs/crypto_test_config.json
@@ -0,0 +1,3 @@
+"WALE_S3_PREFIX": "s3://cryptobucket",
+"WALG_DELTA_MAX_STEPS": "6",
+"WALG_PGP_KEY_PATH": "/tmp/PGP_KEY"
diff --git a/docker/pg_tests/scripts/configs/delete_before_name_find_full_test_config.json b/docker/pg_tests/scripts/configs/delete_before_name_find_full_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "1",
+"WALE_S3_PREFIX": "s3://deletebeforenamefindfullbucket",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/delete_before_permanent_full_test_config.json b/docker/pg_tests/scripts/configs/delete_before_permanent_full_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "0",
+"WALE_S3_PREFIX": "s3://deletebeforepermanentfullbucket",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/delete_before_time_find_full_test_config.json b/docker/pg_tests/scripts/configs/delete_before_time_find_full_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "1",
+"WALG_USE_WAL_DELTA": "true",
+"WALE_S3_PREFIX": "s3://deletebeforetimefindfullbucket"
diff --git a/docker/pg_tests/scripts/configs/delete_end_to_end_test_config.json b/docker/pg_tests/scripts/configs/delete_end_to_end_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "2",
+"WALE_S3_PREFIX": "s3://deleteendtoendbucket",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/delete_retain_find_full_test_config.json b/docker/pg_tests/scripts/configs/delete_retain_find_full_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "1",
+"WALE_S3_PREFIX": "s3://deleteretainfindfullbucket",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/delete_retain_full_test_config.json b/docker/pg_tests/scripts/configs/delete_retain_full_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "1",
+"WALE_S3_PREFIX": "s3://deleteretainfullbucket",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/delete_without_confirm_test_config.json b/docker/pg_tests/scripts/configs/delete_without_confirm_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "0",
+"WALE_S3_PREFIX": "s3://deletewithoutconfirm",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/delta_backup_fullscan_test_config.json b/docker/pg_tests/scripts/configs/delta_backup_fullscan_test_config.json
@@ -0,0 +1,2 @@
+"WALE_S3_PREFIX": "s3://fullscandeltabucket",
+"WALG_DELTA_MAX_STEPS": "6"
diff --git a/docker/pg_tests/scripts/configs/delta_backup_wal_delta_test_config.json b/docker/pg_tests/scripts/configs/delta_backup_wal_delta_test_config.json
@@ -0,0 +1,3 @@
+"WALE_S3_PREFIX": "s3://waldeltabucket",
+"WALG_DELTA_MAX_STEPS": "6",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/full_backup_test_config.json b/docker/pg_tests/scripts/configs/full_backup_test_config.json
@@ -0,0 +1,3 @@
+"WALE_S3_PREFIX": "s3://fullbucket",
+"WALG_DELTA_MAX_STEPS": "6",
+"WALG_PGP_KEY_PATH": "/tmp/PGP_KEY"
diff --git a/docker/pg_tests/scripts/configs/ghost_table_test_config.json b/docker/pg_tests/scripts/configs/ghost_table_test_config.json
@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "6",
+"WALE_S3_PREFIX": "s3://ghostbucket",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/configs/several_delta_backups_test_config.json b/docker/pg_tests/scripts/configs/several_delta_backups_test_config.json
@@ -0,0 +1,3 @@
+"WALE_S3_PREFIX": "s3://compressionbucket",
+"WALG_DELTA_MAX_STEPS": "6",
+"WALG_USE_WAL_DELTA": "true"
diff --git a/docker/pg_tests/scripts/scripts/wrap_config_file.sh b/docker/pg_tests/scripts/scripts/wrap_config_file.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+TMP_CONFIG="/tmp/configs/new_config"
+
+touch ${TMP_CONFIG}
+echo "{" > ${TMP_CONFIG}
+cat $1 >> ${TMP_CONFIG}
+echo "}" >> ${TMP_CONFIG}
+cat ${TMP_CONFIG} > $1
+rm ${TMP_CONFIG}
diff --git a/docker/pg_tests/scripts/tests/config_test.sh b/docker/pg_tests/scripts/tests/config_test.sh
@@ -1,12 +1,21 @@
 #!/bin/sh
 set -e -x
-
+CONFIG_FILE="/tmp/configs/config_test_config.json"
 mkdir /tmp/storage
 
+COMMON_CONFIG="/tmp/configs/common_config.json"
+TMP_CONFIG="/tmp/configs/tmp_config.json"
+cat ${CONFIG_FILE} > ${TMP_CONFIG}
+
+echo "," >> ${TMP_CONFIG}
+cat ${COMMON_CONFIG} >> ${TMP_CONFIG}
+
+tmp/scripts/wrap_config_file.sh ${TMP_CONFIG}
+
 /usr/lib/postgresql/10/bin/initdb ${PGDATA}
 
 echo "archive_mode = on" >> /var/lib/postgresql/10/main/postgresql.conf
-echo "archive_command = '/usr/bin/timeout 600 /usr/bin/wal-g --config=/tmp/walg.json wal-push %p'" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_command = '/usr/bin/timeout 600 /usr/bin/wal-g --config=${TMP_CONFIG} wal-push %p'" >> /var/lib/postgresql/10/main/postgresql.conf
 echo "archive_timeout = 600" >> /var/lib/postgresql/10/main/postgresql.conf
 
 /usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
@@ -15,15 +24,15 @@ pgbench -i -s 10 postgres
 pg_dumpall -f /tmp/dump1
 pgbench -c 2 -T 100000000 -S &
 sleep 1
-wal-g --config=/tmp/walg.json backup-push ${PGDATA}
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
 
 pkill -9 postgres
 
 rm -rf ${PGDATA}
 
-wal-g --config=/tmp/walg.json backup-fetch ${PGDATA} LATEST
+wal-g --config=${TMP_CONFIG} backup-fetch ${PGDATA} LATEST
 
-echo "restore_command = 'echo \"WAL file restoration: %f, %p\"&& /usr/bin/wal-g --config=/tmp/walg.json wal-fetch \"%f\" \"%p\"'" > ${PGDATA}/recovery.conf
+echo "restore_command = 'echo \"WAL file restoration: %f, %p\"&& /usr/bin/wal-g --config=${TMP_CONFIG} wal-fetch \"%f\" \"%p\"'" > ${PGDATA}/recovery.conf
 
 /usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
 
@@ -33,5 +42,6 @@ diff /tmp/dump1 /tmp/dump2
 
 pkill -9 postgres
 rm -rf ${PGDATA}
+rm ${TMP_CONFIG}
 
 echo "Full backup success!!!!!!"