feat(postgres-backup-s3): update AWS CLI to v2; allow support for multiple databases; allow support for defining sensitive env vars in files (eg Docker secrets)

AWS CLI updated to v2; python removed; glibc added Added support for multiple databases (note POSTGRES_DATABASE env var is deprecated and replaced by POSTGRES_DATABASES) -- this means this version of the docker image is not backward-compatible with previosu releases. Added support for _FILE env vars for sensitive values. This allows using the image with Docker secrets. For example, ``` service: postgres-backup environment: POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password ... secrets: - postgres-password ```
3 years ago · df985f2cb9
4 changed files with 121 additions and 45 deletions
--- a/postgres-backup-s3/Dockerfile
+++ b/postgres-backup-s3/Dockerfile
@ -1,25 +1,56 @@
-FROM alpine:3.9
-LABEL maintainer="Johannes Schickling <schickling.j@gmail.com>"
+FROM alpine:3.13
+LABEL maintainer="Dzango Technologies Limited <info@dzangolab.com>"

-ADD install.sh install.sh
-RUN sh install.sh && rm install.sh
+ENV GLIBC_VER=2.33-r0

-ENV POSTGRES_DATABASE **None**
+# install glibc compatibility for alpine
+RUN apk --no-cache add \
+        binutils \
+        curl \
+    && curl -sL https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub \
+    && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-${GLIBC_VER}.apk \
+    && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-bin-${GLIBC_VER}.apk \
+    && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-i18n-${GLIBC_VER}.apk \
+    && apk add --no-cache \
+        glibc-${GLIBC_VER}.apk \
+        glibc-bin-${GLIBC_VER}.apk \
+        glibc-i18n-${GLIBC_VER}.apk \
+        postgresql \
+    && /usr/glibc-compat/bin/localedef -i en_US -f UTF-8 en_US.UTF-8 \
+    && curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip \
+    && unzip awscliv2.zip \
+    && aws/install \
+    && rm -rf \
+        awscliv2.zip \
+        aws \
+        /usr/local/aws-cli/v2/*/dist/aws_completer \
+        /usr/local/aws-cli/v2/*/dist/awscli/data/ac.index \
+        /usr/local/aws-cli/v2/*/dist/awscli/examples \
+        glibc-*.apk \
+    && curl -L --insecure https://github.com/odise/go-cron/releases/download/v0.0.6/go-cron-linux.gz | zcat > /usr/local/bin/go-cron \
+    && chmod u+x /usr/local/bin/go-cron \
+    && apk --no-cache del \
+        binutils \
+        curl \
+    && rm -rf /var/cache/apk/*
+
+ADD run.sh run.sh
+ADD backup.sh backup.sh
+
+ENV POSTGRES_DATABASES **None**
+ENV POSTGRES_EXTRA_OPTS ''
 ENV POSTGRES_HOST **None**
+ENV POSTGRES_PASSWORD **None**
 ENV POSTGRES_PORT 5432
 ENV POSTGRES_USER **None**
-ENV POSTGRES_PASSWORD **None**
-ENV POSTGRES_EXTRA_OPTS ''
 ENV S3_ACCESS_KEY_ID **None**
-ENV S3_SECRET_ACCESS_KEY **None**
 ENV S3_BUCKET **None**
-ENV S3_REGION us-west-1
-ENV S3_PATH 'backup'
 ENV S3_ENDPOINT **None**
+ENV S3_FILENAME **None**
+ENV S3_PREFIX 'backup'
+ENV S3_REGION us-west-1
 ENV S3_S3V4 no
+ENV S3_SECRET_ACCESS_KEY **None**
 ENV SCHEDULE **None**

-ADD run.sh run.sh
-ADD backup.sh backup.sh
-
 CMD ["sh", "run.sh"]
--- a/postgres-backup-s3/README.md
+++ b/postgres-backup-s3/README.md
@ -28,11 +28,31 @@ pgbackups3:
    S3_SECRET_ACCESS_KEY: secret
    S3_BUCKET: my-bucket
    S3_PREFIX: backup
-    POSTGRES_DATABASE: dbname
+    POSTGRES_DATABASES: dbname
    POSTGRES_USER: user
    POSTGRES_PASSWORD: password
    POSTGRES_EXTRA_OPTS: '--schema=public --blobs'
 ```
+## Environment variables
+
+- `POSTGRES_EXTRA_OPTS` pg_dump options (default: '')
+- `POSTGRES_DATABASES` list of databases you want to backup *required*
+- `POSTGRES_HOST` the postgres host *required*
+- `POSTGRES_PORT` the postgres port (default: 5432)
+- `POSTGRES_USER` the postgres user *required*
+- `POSTGRES_PASSWORD` the mysql password *required*
+- `POSTGRES_PASSWORD_FILE` path to file containing the postgres password; alternative to `POSTGRES_PASSWORD`
+- `S3_ACCESS_KEY_ID` your AWS access key *required*
+- `S3_ACCESS_KEY_ID_FILE` path to file containing your AWS access key; alternative to `S3_ACCESS_KEY_ID`
+- `S3_SECRET_ACCESS_KEY` your AWS secret key *required*
+- `S3_SECRET_ACCESS_KEY_FILE` path to file containing  your AWS secret key; alternative to `S3_SECRET_ACCESS_KEYs`
+- `S3_BUCKET` your AWS S3 bucket path *required*
+- `S3_PREFIX` path prefix in your bucket (default: 'backup')
+- `S3_FILENAME` a consistent filename to overwrite with your backup.  If not set will use a timestamp.
+- `S3_REGION` the AWS S3 bucket region (default: us-west-1)
+- `S3_ENDPOINT` the AWS Endpoint URL, for S3 Compliant APIs such as [minio](https://minio.io) (default: none)
+- `S3_S3V4` set to `yes` to enable AWS Signature Version 4, required for [minio](https://minio.io) servers (default: no)
+- `SCHEDULE` backup schedule time, see explainatons below

 ### Automatic Periodic Backups

--- a/postgres-backup-s3/backup.sh
+++ b/postgres-backup-s3/backup.sh
@ -3,6 +3,21 @@
 set -e
 set -o pipefail

+if [ -n "${POSTGRES_PASSWORD_FILE}" ]; then
+  POSTGRES_PASSWORD=$(cat "$POSTGRES_PASSWORD_FILE")
+  export POSTGRES_PASSWORD
+fi
+
+if [ -n "${S3_ACCESS_KEY_ID_FILE}" ]; then
+  S3_ACCESS_KEY_ID=$(cat "$S3_ACCESS_KEY_ID_FILE")
+  export S3_ACCESS_KEY_ID
+fi
+
+if [ -n "${S3_SECRET_ACCESS_KEY_FILE}" ]; then
+  S3_SECRET_ACCESS_KEY=$(cat "$S3_SECRET_ACCESS_KEY_FILE")
+  export S3_SECRET_ACCESS_KEY
+fi
+
 if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then
  echo "You need to set the S3_ACCESS_KEY_ID environment variable."
  exit 1
@ -18,8 +33,8 @@ if [ "${S3_BUCKET}" = "**None**" ]; then
  exit 1
 fi

-if [ "${POSTGRES_DATABASE}" = "**None**" ]; then
-  echo "You need to set the POSTGRES_DATABASE environment variable."
+if [ "${POSTGRES_DATABASES}" = "**None**" ]; then
+  echo "You need to set the POSTGRES_DATABASES environment variable."
  exit 1
 fi

@ -55,14 +70,49 @@ export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY
 export AWS_DEFAULT_REGION=$S3_REGION

 export PGPASSWORD=$POSTGRES_PASSWORD
+
+copy_s3 () {
+  SRC_FILE=$1
+  DEST_FILE=$2
+
+  if [ "${S3_ENDPOINT}" == "**None**" ]; then
+    AWS_ARGS=""
+  else
+    AWS_ARGS="--endpoint-url ${S3_ENDPOINT}"
+  fi
+
+  echo "Uploading ${DEST_FILE} to S3..."
+
+  cat $SRC_FILE | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/$DEST_FILE
+
+  if [ $? != 0 ]; then
+    >&2 echo "Error uploading ${DEST_FILE} on S3"
+  fi
+
+  rm $SRC_FILE
+}
+
 POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTGRES_EXTRA_OPTS"
+DUMP_START_TIME=$(date +"%Y-%m-%dT%H%M%SZ")

-echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..."
+for DB in "$POSTGRES_DATABASES"; do
+  echo "Creating dump of ${DB} from ${POSTGRES_HOST}..."

-pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | gzip > dump.sql.gz
+  DUMP_FILE="/tmp/${DB}.sql.gz"
+  
+  pg_dump $POSTGRES_HOST_OPTS $DB | gzip > $DUMP_FILE

-echo "Uploading dump to $S3_BUCKET"
+  if [ $? == 0 ]; then
+    if [ "${S3_FILENAME}" == "**None**" ]; then
+      S3_FILE="${DUMP_START_TIME}.${DB}.sql.gz"
+    else
+      S3_FILE="${S3_FILENAME}.${DB}.sql.gz"
+    fi

-cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2
+    copy_s3 $DUMP_FILE $S3_FILE
+  else
+    >&2 echo "Error creating dump of ${DB}"
+  fi
+done

 echo "SQL backup uploaded successfully"
--- a/postgres-backup-s3/install.sh
+++ b/postgres-backup-s3/install.sh
@ -1,25 +0,0 @@
-#! /bin/sh
-
-# exit if a command fails
-set -e
-
-
-apk update
-
-# install pg_dump
-apk add postgresql
-
-# install s3 tools
-apk add python py2-pip
-pip install awscli
-apk del py2-pip
-
-# install go-cron
-apk add curl
-curl -L --insecure https://github.com/odise/go-cron/releases/download/v0.0.6/go-cron-linux.gz | zcat > /usr/local/bin/go-cron
-chmod u+x /usr/local/bin/go-cron
-apk del curl
-
-
-# cleanup
-rm -rf /var/cache/apk/*