backup using pg_dump firectory format with compression and parallelization

Dockerfile

@@ -1,13 +1,13 @@
-FROM restic/restic:0.12.0
+FROM restic/restic:0.12.1
 
 RUN apk update \
     && apk upgrade \
     && apk add \
         bash \
-        postgresql-client \
         tini \
     && apk add --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main \
         util-linux \
+        postgresql-client \
     && rm -rf /var/cache/apk/*
 
 ENV DOCKERIZE_VERSION=0.5.0

README.md

@@ -97,3 +97,13 @@ Then, access the latest snapshot from another terminal:
 [direnv]: https://direnv.net/
 [Homebrew]: https://brew.sh/
 [restic]: https://restic.net/
+
+# PG Restore
+
+If DB needs to be to dropped and setup again from backup file.
+
+	pg_restore --clean --if-exists --create -d postgres --format="directory" --jobs={MAX_POSSIBLE_NUM} {PG_DUMP_FOLDER}
+
+If DB is already setup with privs and access -
+
+    pg_restore -d {DB_NAME} --format="directory" --jobs={MAX_POSSIBLE_NUM} {PG_DUMP_FOLDER}

bin/backup.sh

@@ -4,6 +4,11 @@ set -e
 
 setup.sh
 
+max_pg_wait_count=${PGDUMP_BACKUP_WAIT_TIME:-120}
+work_area=${PGDUMP_BACKUP_AREA:-/pg_dump}
+
+pg_backup_jobs=${PGDUMP_BACKUP_JOBS:-1}
+
 for i in {1..5}; do
 	export HOSTNAME_VAR="HOSTNAME_$i"
 	export PGHOST_VAR="PGHOST_$i"
@@ -28,37 +33,46 @@ for i in {1..5}; do
 	echo "Dumping database cluster $i: $PGUSER@$PGHOST:$PGPORT"
 
 	# Wait for PostgreSQL to become available.
-	COUNT=0
+	count=0
 	until psql -l > /dev/null 2>&1; do
-		if [[ "$COUNT" == 0 ]]; then
+		if [[ "$count" == 0 ]]; then
 			echo "Waiting for PostgreSQL to become available..."
 		fi
-		(( COUNT += 1 ))
+		if [[ $count -ge $max_pg_wait_count ]]
+		then
+			break
+		fi
 		sleep 1
+		(( count += 1 ))
 	done
-	if (( COUNT > 0 )); then
-		echo "Waited $COUNT seconds."
+	if (( count > 0 )); then
+		echo "Waited $count seconds."
+		psql -l > /dev/null 2>&1 || {
+			echo "PostgreSQL still not available, trying next backup."
+			continue
+		}
 	fi
 
-	mkdir -p "/pg_dump"
+	mkdir "$work_area" || exit 1
 
 	# Dump individual databases directly to restic repository.
-	DBLIST=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')")
-	for dbname in $DBLIST; do
+	dblist=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')")
+	for dbname in $dblist; do
 		echo "Dumping database '$dbname'"
-		pg_dump --file="/pg_dump/$dbname.sql" --no-owner --no-privileges --dbname="$dbname" || true  # Ignore failures
+		# pg_dump working at default compression (6) 
+		pg_dump --file="$work_area/$dbname" --format="directory" --no-owner --no-privileges --dbname="$dbname" --jobs=$pg_backup_jobs || true  # Ignore failures
 	done
 
 	# echo "Dumping global objects for '$PGHOST'"
-	# pg_dumpall --file="/pg_dump/!globals.sql" --globals-only
+	# pg_dumpall --file="$work_area/!globals.sql" --globals-only
 
 	echo "Sending database dumps to S3"
-	while ! restic backup --host "$HOSTNAME" "/pg_dump"; do
+	while ! restic backup --host "$HOSTNAME" "$work_area"; do
 		echo "Sleeping for 10 seconds before retry..."
 		sleep 10
 	done
 
 	echo 'Finished sending database dumps to S3'
 
-	rm -rf "/pg_dump"
+	rm -rf "$work_area"
 done

bin/setup.sh

@@ -2,12 +2,14 @@
 
 set -e
 
+ok=1
 for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY RESTIC_PASSWORD RESTIC_REPOSITORY; do
 	eval [[ -z \${$var+1} ]] && {
 		>&2 echo "ERROR: Missing required environment variable: $var"
-		exit 1
+		ok=
 	}
 done
+[ $ok ] || exit 1
 
 if ! restic unlock; then
 	restic init

crontab.tmpl

@@ -1,5 +1,5 @@
-{{ default .Env.BACKUP_SCHEDULE "0 * * * *" }}  flock -n /opt/restic-pg-dump/backup.lockfile backup.sh
+{{ default .Env.BACKUP_SCHEDULE "0 * * * *" }}  flock -n /opt/restic-pg-dump/restic_running.lockfile backup.sh
 
 {{ if default .Env.PRUNE_SCHEDULE "0 14 * * 0" }}
-{{ default .Env.PRUNE_SCHEDULE "0 14 * * 0" }}  flock -n /opt/restic-pg-dump/prune.lockfile prune.sh
+{{ default .Env.PRUNE_SCHEDULE "0 14 * * 0" }}  flock -n /opt/restic-pg-dump/restic_running.lockfile prune.sh
 {{ end }}