support backward compability with old env variables

Author: idanovinda

ENVIRONMENT.rst

@@ -15,7 +15,7 @@ Environment Configuration Settings
 - **CLONE_TARGET_TIMELINE**: timeline id of the backup for restore, 'latest' by default.
 - **CRONTAB**: anything that you want to run periodically as a cron job (empty by default)
 - **PGROOT**: a directory where we put the pgdata (by default /home/postgres/pgroot). One may adjust it to point to the mount point of the persistent volume, such as EBS.
-- **WALG_TMPDIR**: directory to store WAL-G temporary files. PGROOT/../tmp by default, make sure it has a few GBs of free space.
+- **WALE_TMPDIR** or **WALG_TMPDIR**: directory to store WAL-G temporary files. PGROOT/../tmp by default, make sure it has a few GBs of free space.
 - **PGDATA**: location of PostgreSQL data directory, by default PGROOT/pgdata.
 - **PGUSER_STANDBY**: username for the replication user, 'standby' by default.
 - **PGPASSWORD_STANDBY**: a password for the replication user, 'standby' by default.
@@ -47,19 +47,19 @@ Environment Configuration Settings
 - **SSL_RESTAPI_PRIVATE_KEY**: content of the REST Api SSL private key in the SSL_PRIVATE_KEY_FILE file (by default /run/certs/server.key).
 - **SSL_TEST_RELOAD**: whenever to test for certificate rotation and reloading (by default True if SSL_PRIVATE_KEY_FILE has been set).
 - **RESTAPI_CONNECT_ADDRESS**: when you configure Patroni RESTAPI in SSL mode some safe API (i.e. switchover) perform hostname validation. In this case could be convenient configure ````restapi.connect_address````as a hostname instead of IP. For example, you can configure it as "$(POD_NAME).<service name>".
-- **WALG_BACKUP_THRESHOLD_MEGABYTES**: maximum size of the WAL segments accumulated after the base backup to consider WAL-G restore instead of pg_basebackup.
-- **WALG_BACKUP_THRESHOLD_PERCENTAGE**: maximum ratio (in percents) of the accumulated WAL files to the base backup to consider WAL-G restore instead of pg_basebackup.
-- **WALG_ENV_DIR**: directory where to store WAL-G environment variables
+- **WALG_BACKUP_THRESHOLD_MEGABYTES** or **WALE_BACKUP_THRESHOLD_MEGABYTES**: maximum size of the WAL segments accumulated after the base backup to consider WAL-G restore instead of pg_basebackup.
+- **WALG_BACKUP_THRESHOLD_PERCENTAGE** or **WALE_BACKUP_THRESHOLD_PERCENTAGE**: maximum ratio (in percents) of the accumulated WAL files to the base backup to consider WAL-G restore instead of pg_basebackup.
+- **WALG_ENV_DIR** or **WALE_ENV_DIR**: directory where to store WAL-G environment variables
 - **WAL_RESTORE_TIMEOUT**: timeout (in seconds) for restoring a single WAL file (at most 16 MB) from the backup location, 0 by default. A duration of 0 disables the timeout.
 - **WAL_S3_BUCKET**: (optional) name of the S3 bucket used for WAL-G base backups.
 - **AWS_ACCESS_KEY_ID**: (optional) aws access key
 - **AWS_SECRET_ACCESS_KEY**: (optional) aws secret key
 - **AWS_REGION**: (optional) region of S3 bucket
 - **AWS_ENDPOINT**: (optional) in format 'https://s3.AWS_REGION.amazonaws.com:443', if not specified will be calculated from AWS_REGION
-- **WALG_S3_ENDPOINT**: (optional) in format 'https+path://s3.AWS_REGION.amazonaws.com:443', if not specified will be calculated from AWS_ENDPOINT or AWS_REGION
-- **WALG_S3_PREFIX**: (optional) the full path to the backup location on S3 in the format s3://bucket-name/very/long/path. If not specified Spilo will generate it from WAL_S3_BUCKET.
+- **WALG_S3_ENDPOINT** or **WALE_S3_ENDPOINT**: (optional) in format 'https+path://s3.AWS_REGION.amazonaws.com:443', if not specified will be calculated from AWS_ENDPOINT or AWS_REGION
+- **WALG_S3_PREFIX** or **WALE_S3_PREFIX**: (optional) the full path to the backup location on S3 in the format s3://bucket-name/very/long/path. If not specified Spilo will generate it from WAL_S3_BUCKET.
 - **WAL_GS_BUCKET**: ditto for the Google Cloud Storage (WAL-G supports both S3 and GCS).
-- **WALG_GS_PREFIX**: (optional) the full path to the backup location on the Google Cloud Storage in the format gs://bucket-name/very/long/path. If not specified Spilo will generate it from WAL_GS_BUCKET.
+- **WALG_GS_PREFIX** or **WALE_GS_PREFIX**: (optional) the full path to the backup location on the Google Cloud Storage in the format gs://bucket-name/very/long/path. If not specified Spilo will generate it from WAL_GS_BUCKET.
 - **GOOGLE_APPLICATION_CREDENTIALS**: credentials for WAL-G when running in Google Cloud.
 - **WAL_SWIFT_BUCKET**: ditto for the OpenStack Object Storage (Swift)
 - **SWIFT_AUTHURL**: see wal-g documentation https://wal-g.readthedocs.io/STORAGES/#swift
@@ -79,7 +79,7 @@ Environment Configuration Settings
 - **SWIFT_PROJECT_ID**:
 - **SWIFT_PROJECT_DOMAIN_NAME**:
 - **SWIFT_PROJECT_DOMAIN_ID**:
-- **WALG_SWIFT_PREFIX**: (optional) the full path to the backup location on the Swift Storage in the format swift://bucket-name/very/long/path. If not specified Spilo will generate it from WAL_SWIFT_BUCKET.
+- **WALG_SWIFT_PREFIX** or **WALE_SWIFT_PREFIX**: (optional) the full path to the backup location on the Swift Storage in the format swift://bucket-name/very/long/path. If not specified Spilo will generate it from WAL_SWIFT_BUCKET.
 - **SSH_USERNAME**: (optional) the username for WAL backups.
 - **SSH_PORT**: (optional) the ssh port for WAL backups.
 - **SSH_PRIVATE_KEY_PATH**: (optional) the path to the private key used for WAL backups.
@@ -109,7 +109,7 @@ Environment Configuration Settings
 - **KUBERNETES_BOOTSTRAP_LABELS**: a JSON describing names and values of labels used by Patroni as ``kubernetes.bootstrap_labels``. Default is empty.
 - **INITDB_LOCALE**: database cluster's default UTF-8 locale (en_US by default)
 - **ENABLE_WAL_PATH_COMPAT**: old Spilo images were generating wal path in the backup store using the following template ``/spilo/{WAL_BUCKET_SCOPE_PREFIX}{SCOPE}{WAL_BUCKET_SCOPE_SUFFIX}/wal/``, while new images adding one additional directory (``{PGVERSION}``) to the end. In order to avoid (unlikely) issues with restoring WALs (from S3/GC/and so on) when switching to ``spilo-13`` please set the ``ENABLE_WAL_PATH_COMPAT=true`` when deploying old cluster with ``spilo-13`` for the first time. After that the environment variable could be removed. Change of the WAL path also mean that backups stored in the old location will not be cleaned up automatically.
-- **WALG_DISABLE_S3_SSE**: by default wal-g is configured to encrypt files uploaded to S3. In order to disable it you can set this environment variable to ``true``.
+- **WALG_DISABLE_S3_SSE** or **WALE_DISABLE_S3_SSE**: by default wal-g is configured to encrypt files uploaded to S3. In order to disable it you can set this environment variable to ``true``.
 - **USE_OLD_LOCALES**: whether to use old locales from Ubuntu 18.04 in the Ubuntu 22.04-based image. Default is false.
 
 wal-g

postgres-appliance/Dockerfile

@@ -100,7 +100,7 @@ ENV LC_ALL=en_US.utf-8 \
     RW_DIR=/run \
     DEMO=$DEMO
 
-ENV WALG_ENV_DIR=$RW_DIR/etc/wal-g.d/env \
+ENV WALG_ENV_DIR=$RW_DIR/etc/wal-e.d/env \
     LOG_ENV_DIR=$RW_DIR/etc/log.d/env \
     PGROOT=$PGHOME/pgdata/pgroot
 

postgres-appliance/launch.sh

@@ -51,6 +51,7 @@ chmod -R go-w "$PGROOT"
 chmod 01777 "$RW_DIR/tmp"
 chmod 0700 "$PGDATA"
 
+WALG_ENV_DIR="${WALG_ENV_DIR:-$WALE_ENV_DIR}"
 if [ "$DEMO" = "true" ]; then
     python3 /scripts/configure_spilo.py patroni pgqd certificate pam-oauth2
 elif python3 /scripts/configure_spilo.py all; then

postgres-appliance/scripts/configure_spilo.py

@@ -496,7 +496,7 @@ def set_extended_walg_placeholders(placeholders, prefix):
         return False
     scope = placeholders.get(prefix + 'SCOPE')
     dirname = 'env-' + prefix[:-1].lower() + ('-' + scope if scope else '')
-    placeholders[prefix + 'WALG_ENV_DIR'] = os.path.join(placeholders['RW_DIR'], 'etc', 'wal-g.d', dirname)
+    placeholders[prefix + 'WALG_ENV_DIR'] = os.path.join(placeholders['RW_DIR'], 'etc', 'wal-e.d', dirname)
     placeholders[prefix + 'WITH_WALG'] = True
     return name
 
@@ -533,7 +533,7 @@ def get_placeholders(provider):
     placeholders.setdefault('BACKUP_NUM_TO_RETAIN', '5')
     placeholders.setdefault('CRONTAB', '[]')
     placeholders.setdefault('PGROOT', os.path.join(placeholders['PGHOME'], 'pgroot'))
-    placeholders.setdefault('WALG_TMPDIR', os.path.abspath(os.path.join(placeholders['PGROOT'], '../tmp')))
+    placeholders.setdefault('WALG_TMPDIR', placeholders.get('WALE_TMPDIR', os.path.abspath(os.path.join(placeholders['PGROOT'], '../tmp'))))
     placeholders.setdefault('PGDATA', os.path.join(placeholders['PGROOT'], 'pgdata'))
     placeholders.setdefault('HUMAN_ROLE', 'zalandos')
     placeholders.setdefault('PGUSER_STANDBY', 'standby')
@@ -556,8 +556,8 @@ def get_placeholders(provider):
     placeholders.setdefault('SSL_RESTAPI_CA_FILE', '')
     placeholders.setdefault('SSL_RESTAPI_CERTIFICATE_FILE', '')
     placeholders.setdefault('SSL_RESTAPI_PRIVATE_KEY_FILE', '')
-    placeholders.setdefault('WALG_BACKUP_THRESHOLD_MEGABYTES', 102400)
-    placeholders.setdefault('WALG_BACKUP_THRESHOLD_PERCENTAGE', 30)
+    placeholders.setdefault('WALG_BACKUP_THRESHOLD_MEGABYTES', placeholders.get('WALE_BACKUP_THRESHOLD_MEGABYTES', 102400))
+    placeholders.setdefault('WALG_BACKUP_THRESHOLD_PERCENTAGE', placeholders.get('WALE_BACKUP_THRESHOLD_PERCENTAGE', 30))
     placeholders.setdefault('INITDB_LOCALE', 'en_US')
     placeholders.setdefault('CLONE_TARGET_TIMELINE', 'latest')
     # if Kubernetes is defined as a DCS, derive the namespace from the POD_NAMESPACE, if not set explicitely.
@@ -570,10 +570,10 @@ def get_placeholders(provider):
                             if placeholders['NAMESPACE'] not in ('default', '') else '')
     placeholders.setdefault('WAL_BUCKET_SCOPE_SUFFIX', '')
     placeholders.setdefault('WAL_RESTORE_TIMEOUT', '0')
-    placeholders.setdefault('WALG_ENV_DIR', os.path.join(placeholders['RW_DIR'], 'etc', 'wal-g.d', 'env'))
+    placeholders.setdefault('WALG_ENV_DIR', placeholders.get('WALE_ENV_DIR', os.path.join(placeholders['RW_DIR'], 'etc', 'wal-e.d', 'env')))
     cpu_count = str(min(psutil.cpu_count(), 10))
-    placeholders.setdefault('WALG_DOWNLOAD_CONCURRENCY', cpu_count)
-    placeholders.setdefault('WALG_UPLOAD_CONCURRENCY', cpu_count)
+    placeholders.setdefault('WALG_DOWNLOAD_CONCURRENCY', placeholders.get('WALE_DOWNLOAD_CONCURRENCY', cpu_count))
+    placeholders.setdefault('WALG_UPLOAD_CONCURRENCY', placeholders.get('WALE_UPLOAD_CONCURRENCY', cpu_count))
     placeholders.setdefault('PAM_OAUTH2', '')
     placeholders.setdefault('CALLBACK_SCRIPT', '')
     placeholders.setdefault('DCS_ENABLE_KUBERNETES_API', '')
@@ -587,8 +587,11 @@ def get_placeholders(provider):
     placeholders.setdefault('KUBERNETES_BYPASS_API_SERVICE', 'true')
     placeholders.setdefault('KUBERNETES_BOOTSTRAP_LABELS', '{}')
     placeholders.setdefault('USE_PAUSE_AT_RECOVERY_TARGET', False)
-    placeholders.setdefault('CLONE_METHOD', '')
-    placeholders.setdefault('CLONE_WITH_WALG', '')
+    placeholders["CLONE_METHOD"] = (
+        "CLONE_WITH_WALG" if placeholders.setdefault("CLONE_METHOD", "") == "CLONE_WITH_WALE"
+        else placeholders["CLONE_METHOD"]
+    )
+    placeholders.setdefault('CLONE_WITH_WALG', placeholders.get('CLONE_WITH_WALE', ''))
     placeholders.setdefault('CLONE_WITH_BASEBACKUP', '')
     placeholders.setdefault('CLONE_TARGET_TIME', '')
     placeholders.setdefault('CLONE_TARGET_INCLUSIVE', True)
@@ -631,7 +634,7 @@ def get_placeholders(provider):
     else:
         set_extended_walg_placeholders(placeholders, 'STANDBY_')
 
-    placeholders.setdefault('STANDBY_WITH_WALG', '')
+    placeholders.setdefault('STANDBY_WITH_WALG', placeholders.get('STANDBY_WITH_WALE', ''))
     placeholders.setdefault('STANDBY_HOST', '')
     placeholders.setdefault('STANDBY_PORT', '')
     placeholders.setdefault('STANDBY_PRIMARY_SLOT_NAME', '')
@@ -645,7 +648,8 @@ def get_placeholders(provider):
 
     # check if we have enough parameters to enable WAL-G
     placeholders['USE_WALG'] = any(placeholders.get(n) for n in AUTO_ENABLE_WALG_RESTORE +
-                                   ('WAL_SWIFT_BUCKET', 'WAL_GCS_BUCKET', 'WAL_GS_BUCKET', 'WALG_GS_PREFIX'))
+                                   ('WAL_SWIFT_BUCKET', 'WAL_GCS_BUCKET', 'WAL_GS_BUCKET', 'WALG_GS_PREFIX') +
+                                   ('WALE_GS_PREFIX', 'WALE_S3_PREFIX', 'WALE_AZ_PREFIX', 'WALE_SSH_PREFIX'))
 
     if placeholders.get('WALG_BACKUP_FROM_REPLICA'):
         placeholders['WALG_BACKUP_FROM_REPLICA'] = str(placeholders['WALG_BACKUP_FROM_REPLICA']).lower()
@@ -851,7 +855,10 @@ def write_walg_environment(placeholders, prefix, overwrite):
                  'WAL_S3_BUCKET', 'WAL_GCS_BUCKET', 'WAL_GS_BUCKET', 'WAL_SWIFT_BUCKET', 'BACKUP_NUM_TO_RETAIN',
                  'ENABLE_WAL_PATH_COMPAT'] + s3_names + swift_names + gs_names + walg_names + azure_names + \
             azure_auth_names + ssh_names:
-        walg[name] = placeholders.get(prefix + name, '')
+        if "WALG" in name: # check if it is set as WALE_
+            walg[name] = placeholders.get(prefix + name, placeholders.get(prefix + name.replace("WALG", "WALE"), ''))
+        else:
+            walg[name] = placeholders.get(prefix + name, '')
 
     if walg.get('WAL_S3_BUCKET') or walg.get('WALG_S3_PREFIX'):
         walg_endpoint = walg.pop('WALG_S3_ENDPOINT', None)

postgres-appliance/scripts/postgres_backup.sh

@@ -23,7 +23,7 @@ else
     log "ERROR: Recovery state unknown: $IN_RECOVERY" && exit 1
 fi
 
-[[ -z $WALG_BACKUP_COMPRESSION_METHOD ]] || export WALG_COMPRESSION_METHOD=$WALG_BACKUP_COMPRESSION_METHOD
+export WALG_COMPRESSION_METHOD="${WALG_BACKUP_COMPRESSION_METHOD:-$WALE_BACKUP_COMPRESSION_METHOD}"
 export PGHOST=/var/run/postgresql
 
 # push a new base backup

postgres-appliance/tests/test_spilo.sh

@@ -54,7 +54,7 @@ function wait_backup() {
 
     # speed up backup creation
     local backup_starter_pid
-    backup_starter_pid=$(docker exec "$container" pgrep -f '/bin/bash /scripts/patroni_wait.sh -t 3600 -- envdir /run/etc/wal-g.d/env /scripts/postgres_backup.sh')
+    backup_starter_pid=$(docker exec "$container" pgrep -f '/bin/bash /scripts/patroni_wait.sh -t 3600 -- envdir /run/etc/wal-e.d/env /scripts/postgres_backup.sh')
     if [ -n "$backup_starter_pid" ]; then
         docker exec "$container" pkill -P "$backup_starter_pid" -f 'sleep 60'
     fi
@@ -66,7 +66,7 @@ function wait_backup() {
     docker_exec -i "$1" "psql -U postgres -c CHECKPOINT" > /dev/null 2>&1
 
     while true; do
-        count=$(docker_exec "$container" "envdir /run/etc/wal-g.d/env wal-g backup-list" | grep -c ^base)
+        count=$(docker_exec "$container" "envdir /run/etc/wal-e.d/env wal-e backup-list" | grep -c ^base)
         if [[ "$count" -gt 0 ]]; then
             return
         fi
@@ -136,8 +136,8 @@ function test_successful_inplace_upgrade_to_14() {
 }
 
 function test_envdir_suffix() {
-    docker_exec "$1" "cat /run/etc/wal-g.d/env/WALG_S3_PREFIX" | grep -q "$2$" \
-        && docker_exec "$1" "cat /run/etc/wal-g.d/env/WALG_S3_PREFIX" | grep -q "$2$"
+    docker_exec "$1" "cat /run/etc/wal-e.d/env/WALG_S3_PREFIX" | grep -q "$2$" \
+        && docker_exec "$1" "cat /run/etc/wal-e.d/env/WALG_S3_PREFIX" | grep -q "$2$"
 }
 
 function test_envdir_updated_to_x() {