diff --git a/debian/changelog b/debian/changelog index 6ea3d45d6..0c9b3e460 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +github-backup-utils (2.8.0) UNRELEASED; urgency=low + + * Adds support for GitHub Enterprise 2.8.0 + * Speedup storage restores #247 + * More portable backup-utils #260 + + -- rubiojr Wed, 09 Nov 2016 06:35:21 -0800 + github-backup-utils (2.7.1) UNRELEASED; urgency=medium * Cluster: fix offline cluster node detection #250 diff --git a/share/github-backup-utils/ghe-backup-repositories-cluster-ng b/share/github-backup-utils/ghe-backup-repositories-cluster-ng index f578c56c7..5edd7d1d4 100755 --- a/share/github-backup-utils/ghe-backup-repositories-cluster-ng +++ b/share/github-backup-utils/ghe-backup-repositories-cluster-ng @@ -74,7 +74,7 @@ user="${host%@*}" [ "$user" = "$host" ] && user="admin" # git server hostnames -hostnames=$(ghe_cluster_online_nodes "git-server") +hostnames=$(ghe-cluster-hostnames "$GHE_HOSTNAME" "git-server") for hostname in $hostnames; do config="$config Host $hostname @@ -171,7 +171,6 @@ rsync_repository_data () { "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ "$backup_dir" 1>&3 fi - } sync_data (){ @@ -356,7 +355,6 @@ RULES done bm_end "$(basename $0) - Special Data Directories Sync" - bm_start "$(basename $0) - Archived Repos" ghe-ssh "$GHE_HOSTNAME" github-env ./bin/dgit-cluster-backup-archived-repos-routes \ | while read route; do diff --git a/share/github-backup-utils/ghe-backup-store-version b/share/github-backup-utils/ghe-backup-store-version index fb033d257..6148012c5 100755 --- a/share/github-backup-utils/ghe-backup-store-version +++ b/share/github-backup-utils/ghe-backup-store-version @@ -8,14 +8,15 @@ set -e bm_start "$(basename $0)" +version_info="$BACKUP_UTILS_VERSION" if [ -d $GHE_BACKUP_ROOT/.git ]; then - version_info="$BACKUP_UTILS_VERSION" ref=$(git --git-dir=$GHE_BACKUP_ROOT/.git rev-parse HEAD || true) if [ -n "$ref" ]; then version_info="$version_info:$ref" fi - echo "$version_info" | - ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1" fi +echo "$version_info" | + ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1" + bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-cluster-hostnames b/share/github-backup-utils/ghe-cluster-hostnames new file mode 100755 index 000000000..32643d976 --- /dev/null +++ b/share/github-backup-utils/ghe-cluster-hostnames @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +#/ Usage: ghe-cluster-hostnames +#/ +#/ Finds all nodes of the cluster using the config on . +#/ If it is a 2.8 cluster the results are returned as prefix-uuid, +#/ otherwise the configured hostnames are returned. +#/ +#/ Note: This script typically isn't called directly. It's invoked by the +#/ ghe-restore-* commands when restoring into a cluster. +set -e + +# Bring in the backup configuration +. $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +GHE_HOSTNAME="$1" +prefix="$2" + +if ghe-ssh "$GHE_HOSTNAME" test -f /data/user/common/uuid; then + node_uuids=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.uuid | cut -d ' ' -f 2) + hostnames='' + for uuid in $node_uuids; do + hostnames+="$prefix-$uuid " + done +else + hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2) +fi + +echo "$hostnames" diff --git a/share/github-backup-utils/ghe-restore-alambic-cluster-ng b/share/github-backup-utils/ghe-restore-alambic-cluster-ng index 675167cb9..2794737d0 100755 --- a/share/github-backup-utils/ghe-restore-alambic-cluster-ng +++ b/share/github-backup-utils/ghe-restore-alambic-cluster-ng @@ -44,8 +44,7 @@ user="${host%@*}" # Generate SSH config for forwarding config="" -hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2) -for hostname in $hostnames; do +for hostname in $(ghe-cluster-hostnames "$GHE_HOSTNAME" "storage-server"); do config="$config Host $hostname ServerAliveInterval 60 diff --git a/share/github-backup-utils/ghe-restore-pages-dpages b/share/github-backup-utils/ghe-restore-pages-dpages index 581c7f570..4014e80df 100755 --- a/share/github-backup-utils/ghe-restore-pages-dpages +++ b/share/github-backup-utils/ghe-restore-pages-dpages @@ -45,8 +45,7 @@ host=$(ssh_host_part "$GHE_HOSTNAME") user="${host%@*}" [ "$user" = "$host" ] && user="admin" -hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2) -for hostname in $hostnames; do +for hostname in $(ghe-cluster-hostnames "$GHE_HOSTNAME" "pages-server"); do config="$config Host $hostname ServerAliveInterval 60 diff --git a/share/github-backup-utils/ghe-restore-repositories-dgit-ng b/share/github-backup-utils/ghe-restore-repositories-dgit-ng index 651f3e63a..488351277 100755 --- a/share/github-backup-utils/ghe-restore-repositories-dgit-ng +++ b/share/github-backup-utils/ghe-restore-repositories-dgit-ng @@ -6,6 +6,32 @@ #/ ghe-restore command when restoring into a cluster. set -e +sync_repo_info() { + # node names changed in 2.8 and `ghe-cluster-each -u` isn't available + # on GHE <= 2.7.X. We need to be backwards compatible. + if ghe-ssh "$GHE_HOSTNAME" test -f /data/user/common/uuid; then + for uuid in `ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -u`; do + if ! ghe-rsync -av --delete \ + -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \ + --rsync-path="sudo -u git rsync" \ + "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info/" \ + "git-server-$uuid:$GHE_REMOTE_DATA_USER_DIR/repositories/info" 1>&3; then + echo "Error restoring /data/repositories/info to git-server-$uuid" 1>&2 + fi + done + else + for route in `ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -p`; do + if ! ghe-rsync -av --delete \ + -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \ + --rsync-path="sudo -u git rsync" \ + "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info/" \ + "$route:$GHE_REMOTE_DATA_USER_DIR/repositories/info" 1>&3; then + echo "Error restoring /data/repositories/info to $route" 1>&2 + fi + done + fi +} + # Bring in the backup configuration . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config @@ -46,7 +72,7 @@ opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecki tmp_list=$tempdir/tmp_list to_restore=$tempdir/to_restore -hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2) +hostnames=$(ghe-cluster-hostnames "$GHE_HOSTNAME" "git-server") for hostname in $hostnames; do echo " Host $hostname @@ -129,15 +155,7 @@ cat $to_restore | ghe-ssh "$GHE_HOSTNAME" github-env ./bin/dgit-cluster-restore- if [ -d $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info ]; then ghe_verbose "* Transferring repository info data" - for route in `ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -p`; do - if ! ghe-rsync -av --delete \ - -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \ - --rsync-path="sudo -u git rsync" \ - "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info/" \ - "$route:$GHE_REMOTE_DATA_USER_DIR/repositories/info" 1>&3; then - echo "Error restoring /data/repositories/info to $route" - fi - done + sync_repo_info else ghe_verbose "* Removing repository info data" ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -- rm -f /data/repositories/info/* diff --git a/share/github-backup-utils/ghe-restore-repositories-gist-ng b/share/github-backup-utils/ghe-restore-repositories-gist-ng index 05c6f0b64..d68d3a2ac 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist-ng +++ b/share/github-backup-utils/ghe-restore-repositories-gist-ng @@ -46,8 +46,7 @@ opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecki tmp_list=$tempdir/tmp_list to_restore=$tempdir/to_restore -hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2) -for hostname in $hostnames; do +for hostname in $(ghe-cluster-hostnames "$GHE_HOSTNAME" "git-server"); do echo " Host $hostname ServerAliveInterval 60 diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 860487ca1..834f26295 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -2.7.1 +2.8.0 diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 22b23eda1..7269d3848 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -441,10 +441,26 @@ begin_test "ghe-backup fsck" ) end_test -begin_test "ghe-backup with leaked SSH host key detection for current backup" +begin_test "ghe-backup stores version when not run from a clone" ( set -e + # Make sure this doesn't exist + rm -f "$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version" + + mv "$ROOTDIR/.git" "$ROOTDIR/.gittmp" + ghe-backup + mv "$ROOTDIR/.gittmp" "$ROOTDIR/.git" + + # verify that ghe-backup wrote its version information to the host + [ -f "$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version" ] +) +end_test + +begin_test "ghe-backup with leaked SSH host key detection for current backup" +( + set -e + SHARED_UTILS_PATH=$(dirname $(which ghe-detect-leaked-ssh-keys)) # Inject the fingerprint into the blacklist echo 98:d8:99:d3:be:c0:55:05:db:b0:53:2f:1f:ad:b3:60 >> "$SHARED_UTILS_PATH/ghe-ssh-leaked-host-keys-list.txt"