Merge pull request #261 from github/release-2.8.0

Bump version: 2.8.0

Author: rubiojr

debian/changelog

@@ -1,3 +1,11 @@
+github-backup-utils (2.8.0) UNRELEASED; urgency=low
+
+  * Adds support for GitHub Enterprise 2.8.0
+  * Speedup storage restores #247
+  * More portable backup-utils #260
+
+ -- rubiojr <rubiojr@github.com>  Wed, 09 Nov 2016 06:35:21 -0800
+
 github-backup-utils (2.7.1) UNRELEASED; urgency=medium
 
   * Cluster: fix offline cluster node detection #250

share/github-backup-utils/ghe-backup-repositories-cluster-ng

@@ -74,7 +74,7 @@ user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
 # git server hostnames
-hostnames=$(ghe_cluster_online_nodes "git-server")
+hostnames=$(ghe-cluster-hostnames "$GHE_HOSTNAME" "git-server")
 for hostname in $hostnames; do
   config="$config
 Host $hostname
@@ -171,7 +171,6 @@ rsync_repository_data () {
       "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \
       "$backup_dir" 1>&3
   fi
-
 }
 
 sync_data (){
@@ -356,7 +355,6 @@ RULES
 done
 bm_end "$(basename $0) - Special Data Directories Sync"
 
-
 bm_start "$(basename $0) - Archived Repos"
 ghe-ssh "$GHE_HOSTNAME" github-env ./bin/dgit-cluster-backup-archived-repos-routes \
   | while read route; do

share/github-backup-utils/ghe-backup-store-version

@@ -8,14 +8,15 @@ set -e
 
 bm_start "$(basename $0)"
 
+version_info="$BACKUP_UTILS_VERSION"
 if [ -d $GHE_BACKUP_ROOT/.git ]; then
-  version_info="$BACKUP_UTILS_VERSION"
   ref=$(git --git-dir=$GHE_BACKUP_ROOT/.git rev-parse HEAD || true)
   if [ -n "$ref" ]; then
     version_info="$version_info:$ref"
   fi
-  echo "$version_info" |
-  ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1"
 fi
 
+echo "$version_info" |
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1"
+
 bm_end "$(basename $0)"

share/github-backup-utils/ghe-cluster-hostnames

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-cluster-hostnames <host> <prefix>
+#/
+#/ Finds all nodes of the cluster using the config on <host>.
+#/ If it is a 2.8 cluster the results are returned as prefix-uuid,
+#/ otherwise the configured hostnames are returned.
+#/
+#/ Note: This script typically isn't called directly. It's invoked by the
+#/ ghe-restore-* commands when restoring into a cluster.
+set -e
+
+# Bring in the backup configuration
+. $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+GHE_HOSTNAME="$1"
+prefix="$2"
+
+if ghe-ssh "$GHE_HOSTNAME" test -f /data/user/common/uuid; then
+  node_uuids=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.uuid | cut -d ' ' -f 2)
+  hostnames=''
+  for uuid in $node_uuids; do
+    hostnames+="$prefix-$uuid "
+  done
+else
+  hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2)
+fi
+
+echo "$hostnames"

share/github-backup-utils/ghe-restore-alambic-cluster-ng

@@ -44,8 +44,7 @@ user="${host%@*}"
 
 # Generate SSH config for forwarding
 config=""
-hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2)
-for hostname in $hostnames; do
+for hostname in $(ghe-cluster-hostnames "$GHE_HOSTNAME" "storage-server"); do
   config="$config
 Host $hostname
   ServerAliveInterval 60

share/github-backup-utils/ghe-restore-pages-dpages

@@ -45,8 +45,7 @@ host=$(ssh_host_part "$GHE_HOSTNAME")
 user="${host%@*}"
 [ "$user" = "$host" ] && user="admin"
 
-hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2)
-for hostname in $hostnames; do
+for hostname in $(ghe-cluster-hostnames "$GHE_HOSTNAME" "pages-server"); do
   config="$config
 Host $hostname
   ServerAliveInterval 60

share/github-backup-utils/ghe-restore-repositories-dgit-ng

@@ -6,6 +6,32 @@
 #/ ghe-restore command when restoring into a cluster.
 set -e
 
+sync_repo_info() {
+  # node names changed in 2.8 and `ghe-cluster-each -u` isn't available
+  # on GHE <= 2.7.X. We need to be backwards compatible.
+  if ghe-ssh "$GHE_HOSTNAME" test -f /data/user/common/uuid; then
+    for uuid in `ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -u`; do
+      if ! ghe-rsync -av --delete \
+          -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \
+          --rsync-path="sudo -u git rsync" \
+          "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info/" \
+          "git-server-$uuid:$GHE_REMOTE_DATA_USER_DIR/repositories/info" 1>&3; then
+        echo "Error restoring /data/repositories/info to git-server-$uuid" 1>&2
+      fi
+    done
+  else
+    for route in `ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -p`; do
+      if ! ghe-rsync -av --delete \
+          -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \
+          --rsync-path="sudo -u git rsync" \
+          "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info/" \
+          "$route:$GHE_REMOTE_DATA_USER_DIR/repositories/info" 1>&3; then
+        echo "Error restoring /data/repositories/info to $route" 1>&2
+      fi
+    done
+  fi
+}
+
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
@@ -46,7 +72,7 @@ opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecki
 tmp_list=$tempdir/tmp_list
 to_restore=$tempdir/to_restore
 
-hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2)
+hostnames=$(ghe-cluster-hostnames "$GHE_HOSTNAME" "git-server")
 for hostname in $hostnames; do
   echo "
 Host $hostname
@@ -129,15 +155,7 @@ cat $to_restore | ghe-ssh "$GHE_HOSTNAME" github-env ./bin/dgit-cluster-restore-
 
 if [ -d $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info ]; then
   ghe_verbose "* Transferring repository info data"
-  for route in `ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -p`; do
-    if ! ghe-rsync -av --delete \
-        -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \
-        --rsync-path="sudo -u git rsync" \
-        "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/info/" \
-        "$route:$GHE_REMOTE_DATA_USER_DIR/repositories/info" 1>&3; then
-      echo "Error restoring /data/repositories/info to $route"
-    fi
-  done
+  sync_repo_info
 else
   ghe_verbose "* Removing repository info data"
   ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r git -- rm -f /data/repositories/info/*

share/github-backup-utils/ghe-restore-repositories-gist-ng

@@ -46,8 +46,7 @@ opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecki
 tmp_list=$tempdir/tmp_list
 to_restore=$tempdir/to_restore
 
-hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2)
-for hostname in $hostnames; do
+for hostname in $(ghe-cluster-hostnames "$GHE_HOSTNAME" "git-server"); do
   echo "
 Host $hostname
   ServerAliveInterval 60

share/github-backup-utils/version

@@ -1 +1 @@
-2.7.1
+2.8.0

test/test-ghe-backup.sh

@@ -441,10 +441,26 @@ begin_test "ghe-backup fsck"
 )
 end_test
 
-begin_test "ghe-backup with leaked SSH host key detection for current backup"
+begin_test "ghe-backup stores version when not run from a clone"
 (
   set -e
 
+  # Make sure this doesn't exist
+  rm -f "$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version"
+
+  mv "$ROOTDIR/.git" "$ROOTDIR/.gittmp"
+  ghe-backup
+  mv "$ROOTDIR/.gittmp" "$ROOTDIR/.git"
+
+  # verify that ghe-backup wrote its version information to the host
+  [ -f "$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version" ]
+)
+end_test
+
+begin_test "ghe-backup with leaked SSH host key detection for current backup"
+(
+  set -e 
+  
   SHARED_UTILS_PATH=$(dirname $(which ghe-detect-leaked-ssh-keys))
   # Inject the fingerprint into the blacklist
   echo 98:d8:99:d3:be:c0:55:05:db:b0:53:2f:1f:ad:b3:60 >> "$SHARED_UTILS_PATH/ghe-ssh-leaked-host-keys-list.txt"