diff --git a/postgres-backup-s3/Dockerfile b/postgres-backup-s3/Dockerfile index af5740d..486d845 100644 --- a/postgres-backup-s3/Dockerfile +++ b/postgres-backup-s3/Dockerfile @@ -1,25 +1,56 @@ -FROM alpine:3.9 -LABEL maintainer="Johannes Schickling " +FROM alpine:3.13 +LABEL maintainer="Dzango Technologies Limited " -ADD install.sh install.sh -RUN sh install.sh && rm install.sh +ENV GLIBC_VER=2.33-r0 -ENV POSTGRES_DATABASE **None** +# install glibc compatibility for alpine +RUN apk --no-cache add \ + binutils \ + curl \ + && curl -sL https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub \ + && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-${GLIBC_VER}.apk \ + && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-bin-${GLIBC_VER}.apk \ + && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-i18n-${GLIBC_VER}.apk \ + && apk add --no-cache \ + glibc-${GLIBC_VER}.apk \ + glibc-bin-${GLIBC_VER}.apk \ + glibc-i18n-${GLIBC_VER}.apk \ + postgresql \ + && /usr/glibc-compat/bin/localedef -i en_US -f UTF-8 en_US.UTF-8 \ + && curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip \ + && unzip awscliv2.zip \ + && aws/install \ + && rm -rf \ + awscliv2.zip \ + aws \ + /usr/local/aws-cli/v2/*/dist/aws_completer \ + /usr/local/aws-cli/v2/*/dist/awscli/data/ac.index \ + /usr/local/aws-cli/v2/*/dist/awscli/examples \ + glibc-*.apk \ + && curl -L --insecure https://github.com/odise/go-cron/releases/download/v0.0.6/go-cron-linux.gz | zcat > /usr/local/bin/go-cron \ + && chmod u+x /usr/local/bin/go-cron \ + && apk --no-cache del \ + binutils \ + curl \ + && rm -rf /var/cache/apk/* + +ADD run.sh run.sh +ADD backup.sh backup.sh + +ENV POSTGRES_DATABASES **None** +ENV POSTGRES_EXTRA_OPTS '' ENV POSTGRES_HOST **None** +ENV POSTGRES_PASSWORD **None** ENV POSTGRES_PORT 5432 ENV POSTGRES_USER **None** -ENV POSTGRES_PASSWORD **None** -ENV POSTGRES_EXTRA_OPTS '' ENV S3_ACCESS_KEY_ID **None** -ENV S3_SECRET_ACCESS_KEY **None** ENV S3_BUCKET **None** -ENV S3_REGION us-west-1 -ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** +ENV S3_FILENAME **None** +ENV S3_PREFIX 'backup' +ENV S3_REGION us-west-1 ENV S3_S3V4 no +ENV S3_SECRET_ACCESS_KEY **None** ENV SCHEDULE **None** -ADD run.sh run.sh -ADD backup.sh backup.sh - CMD ["sh", "run.sh"] diff --git a/postgres-backup-s3/README.md b/postgres-backup-s3/README.md index a5339de..d1bab0c 100644 --- a/postgres-backup-s3/README.md +++ b/postgres-backup-s3/README.md @@ -28,11 +28,31 @@ pgbackups3: S3_SECRET_ACCESS_KEY: secret S3_BUCKET: my-bucket S3_PREFIX: backup - POSTGRES_DATABASE: dbname + POSTGRES_DATABASES: dbname POSTGRES_USER: user POSTGRES_PASSWORD: password POSTGRES_EXTRA_OPTS: '--schema=public --blobs' ``` +## Environment variables + +- `POSTGRES_EXTRA_OPTS` pg_dump options (default: '') +- `POSTGRES_DATABASES` list of databases you want to backup *required* +- `POSTGRES_HOST` the postgres host *required* +- `POSTGRES_PORT` the postgres port (default: 5432) +- `POSTGRES_USER` the postgres user *required* +- `POSTGRES_PASSWORD` the mysql password *required* +- `POSTGRES_PASSWORD_FILE` path to file containing the postgres password; alternative to `POSTGRES_PASSWORD` +- `S3_ACCESS_KEY_ID` your AWS access key *required* +- `S3_ACCESS_KEY_ID_FILE` path to file containing your AWS access key; alternative to `S3_ACCESS_KEY_ID` +- `S3_SECRET_ACCESS_KEY` your AWS secret key *required* +- `S3_SECRET_ACCESS_KEY_FILE` path to file containing your AWS secret key; alternative to `S3_SECRET_ACCESS_KEYs` +- `S3_BUCKET` your AWS S3 bucket path *required* +- `S3_PREFIX` path prefix in your bucket (default: 'backup') +- `S3_FILENAME` a consistent filename to overwrite with your backup. If not set will use a timestamp. +- `S3_REGION` the AWS S3 bucket region (default: us-west-1) +- `S3_ENDPOINT` the AWS Endpoint URL, for S3 Compliant APIs such as [minio](https://minio.io) (default: none) +- `S3_S3V4` set to `yes` to enable AWS Signature Version 4, required for [minio](https://minio.io) servers (default: no) +- `SCHEDULE` backup schedule time, see explainatons below ### Automatic Periodic Backups diff --git a/postgres-backup-s3/backup.sh b/postgres-backup-s3/backup.sh index 6e5a7f0..ee5cc15 100644 --- a/postgres-backup-s3/backup.sh +++ b/postgres-backup-s3/backup.sh @@ -3,6 +3,21 @@ set -e set -o pipefail +if [ -n "${POSTGRES_PASSWORD_FILE}" ]; then + POSTGRES_PASSWORD=$(cat "$POSTGRES_PASSWORD_FILE") + export POSTGRES_PASSWORD +fi + +if [ -n "${S3_ACCESS_KEY_ID_FILE}" ]; then + S3_ACCESS_KEY_ID=$(cat "$S3_ACCESS_KEY_ID_FILE") + export S3_ACCESS_KEY_ID +fi + +if [ -n "${S3_SECRET_ACCESS_KEY_FILE}" ]; then + S3_SECRET_ACCESS_KEY=$(cat "$S3_SECRET_ACCESS_KEY_FILE") + export S3_SECRET_ACCESS_KEY +fi + if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then echo "You need to set the S3_ACCESS_KEY_ID environment variable." exit 1 @@ -18,8 +33,8 @@ if [ "${S3_BUCKET}" = "**None**" ]; then exit 1 fi -if [ "${POSTGRES_DATABASE}" = "**None**" ]; then - echo "You need to set the POSTGRES_DATABASE environment variable." +if [ "${POSTGRES_DATABASES}" = "**None**" ]; then + echo "You need to set the POSTGRES_DATABASES environment variable." exit 1 fi @@ -55,14 +70,49 @@ export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY export AWS_DEFAULT_REGION=$S3_REGION export PGPASSWORD=$POSTGRES_PASSWORD + +copy_s3 () { + SRC_FILE=$1 + DEST_FILE=$2 + + if [ "${S3_ENDPOINT}" == "**None**" ]; then + AWS_ARGS="" + else + AWS_ARGS="--endpoint-url ${S3_ENDPOINT}" + fi + + echo "Uploading ${DEST_FILE} to S3..." + + cat $SRC_FILE | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/$DEST_FILE + + if [ $? != 0 ]; then + >&2 echo "Error uploading ${DEST_FILE} on S3" + fi + + rm $SRC_FILE +} + POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTGRES_EXTRA_OPTS" +DUMP_START_TIME=$(date +"%Y-%m-%dT%H%M%SZ") -echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..." +for DB in "$POSTGRES_DATABASES"; do + echo "Creating dump of ${DB} from ${POSTGRES_HOST}..." -pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | gzip > dump.sql.gz + DUMP_FILE="/tmp/${DB}.sql.gz" + + pg_dump $POSTGRES_HOST_OPTS $DB | gzip > $DUMP_FILE -echo "Uploading dump to $S3_BUCKET" + if [ $? == 0 ]; then + if [ "${S3_FILENAME}" == "**None**" ]; then + S3_FILE="${DUMP_START_TIME}.${DB}.sql.gz" + else + S3_FILE="${S3_FILENAME}.${DB}.sql.gz" + fi -cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2 + copy_s3 $DUMP_FILE $S3_FILE + else + >&2 echo "Error creating dump of ${DB}" + fi +done echo "SQL backup uploaded successfully" diff --git a/postgres-backup-s3/install.sh b/postgres-backup-s3/install.sh deleted file mode 100644 index 16c7119..0000000 --- a/postgres-backup-s3/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#! /bin/sh - -# exit if a command fails -set -e - - -apk update - -# install pg_dump -apk add postgresql - -# install s3 tools -apk add python py2-pip -pip install awscli -apk del py2-pip - -# install go-cron -apk add curl -curl -L --insecure https://github.com/odise/go-cron/releases/download/v0.0.6/go-cron-linux.gz | zcat > /usr/local/bin/go-cron -chmod u+x /usr/local/bin/go-cron -apk del curl - - -# cleanup -rm -rf /var/cache/apk/*