pullsnapshot

#!/bin/bash

# ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#
# USAGE:
#
#	pullsnapshot user@source.tld
#
#		OR
#
#	pullsnapshot target.tld
#
#		OR in cron:
#
#	0 */4 * * * /bin/pullsnapshot --ssh-identity /root/.ssh/id_rsa --backup "bin,Documents,work" ga@reetstreet.com --archive /home/ga/archives/reetstreet-vps >>/var/log/pullsnapshot.log 2>&1
#
#	Runs every four hours; change the "4" to a "1" to run every hour.
#
#
# For remote server:
#
#	Logged in as user 'ga':
#
#	pullsnapshot --backup 'local,mail' reetstreet.com
#	pullsnapshot --backup mail barbara@reetstreet.com
#
# FLAGS:
#
#	--backup BACKUP_ITEM_LIST
#
#	Specify the list of local folders to back up.  There are two
#	formatting options; PICK ONE:
#
#		folder1,folder2,folder3		List of folders in /home/$localUser
#
#		/home/user/folder /etc/profile.d /srv/www
#
#	In other words, you can have either a comma-separated list of
#	folders in the user's home directory, or a space-separated list
#	of full paths.  Leading slashes are IGNORED in comma-separated
#	lists; you'll get /home/$USER//path, which will still be relative
#	to the user's home directory.
#
#	--archive PATH
#
#	The path to the archives, or just the simple name of the archive
#	folder if the archives are stored in $HOME/archives.  This is
#	where the backup files will be stored.  The default is $HOME/archives/sourceMachine.
#
#	--owner USER
#
#	If --owner is used, the archive folder will be chown'ed to the specified user
#	after the backup is complete.  This option is usually only useful
#	when running this script as root.
#
#	--user USERNAME
#
#	Use this flag to specify that the local user should  be something
#	other than $USER.  Useful when running as root from cron.
#
#	Note that the user is only relevant if there is no "user@"
#	in the backup source.  If the remote user is explicitly specified,
#	then the local user is irrelevant.
#
#	--ssh-identity IDENTITY_FILE
#
#	Specifies where to find the identity file.  You must specify your
#	identity file if its name does not begin with "id" or if you have
#	more than one private key or if your identity files are not stored
#	in /home/$USER/.ssh (e.g. when running as root).
#
#	--ssh-key-ok
#
#	This flag indicates that it is okay to run this script without first
#	invoking ssh-agent.  In general, it is not necessary to use this flag
#	directly, as this script will detect if you have already used ssh-add
#	on your key, and this flag is not necessary in other situations.
#
#	--rsync-flags flag1 flag2 flag3 ;
#
#	Specifies additional options to pass to rsync, terminated by a ";".  
#	These can have various different effects.  For example, if you have
#	one machine backing up to greg@ftp.adc.rsv.ricoh.com:/home2/greg/archives/ga-laptop,
#	and another backing up to greg@ftp.adc.rsv.ricoh.com:/home2/greg/archives/ga-desktop,
#	and the same files exist on both machines, then you might want to 
#	add --rsync-flags --link-dest=/home2/greg/archives/ga-desktop ; to the 
#	laptop, and --rsync-flags --link-dest=/home2/greg/archives/ga-laptop ;
#	to the desktop so that the files common to both machines are stored
#	only once in the backup.  This parameter can also be used to
#	override the default 
#
# SSH KEY SETUP:
#
#	Create id_dsa and id_dsa.pub on the client
#	machine, and copy id_dsa.pub into authorized_keys
#	on the target.
#
#	Follow the instructions at:
#
#		http://kimmo.suominen.com/docs/ssh/
#
#	For example:
#
#		sudo bash
#		cd /root
#		ssh-keygen -t rsa
#		ls -al .ssh
#		scp .ssh/id_rsa.pub USER@domain.com:/home/USER/.ssh/my_id_rsa.pub
#		ssh USER@domain.com
#		cd .ssh
#		cat my_id_rsa.pub
#		cat my_id_rsa.pub >> authorized_keys
#		chmod 600 authorized_keys
#
#	If this program will run in cron as root, then 
#	your id_dsa and id_dsa.pub should be in /root/.ssh,
#	and the key should not be protected with a password.
#	In general, identity files should always be password-protected;
#	however, if the identity file is only readable by root,
#	and if it is only used to access a remote user account
#	that is only used to store backup files, then a passwordless
#	identity file is reasonable.
#
# CRON SETUP:
#
#	It is presumed that you will want to run this script
#	as root; running as root insures that the existance of
#	files that you cannot read without su permissions
#	in your home directory should not interfere with your
#	backup process.  If you never become root (e.g if 
#	you are unable to log in as root), then you may run
#	this script without su priveledges.
#
#	Cron is configured with the crontab command, which is
#	stateless.  To change the settings of the cron table,
#	you must replace the whole table.  So, the first step
#	is to determine what is currently in the crontab
#	like so:
#
#		$ sudo crontab -l > tmp.crontab
#		$ cat tmp.crontab
#
#		0 */1 * * * /bin/cron-snapshot2 --ssh-identity /root/.ssh/id_rsa --require-network 10.100.1.1 eth0 --user ga --backup "bin,Documents,work" reetstreet.com >>/var/log/snapshot.log 2>&1
#		57 */1 * * * /bin/cron-snapshot2 --ssh-identity /root/.ssh/id_rsa --require-network 172.30.10.1 eth0 --user ga --backup "bin,Documents,work" greg@ftp.ricohsv.com:/home2/greg/archives/ga-laptop >>/var/log/snapshot.log 2>&1
#
#	Next, edit tmp.crontab, and then load it:
#
#		$ sudo crontab tmp.crontab
#
#	DO NOT FORGET to copy snapshot2 and migratehistories to
#	/bin/cron-snapshot2 and /bin/cron-migratehistories.
#
#	That's it.
#
#
# ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

unset PATH

CP=/bin/cp
MV=/bin/mv
RM=/bin/rm
LS=/bin/ls
CHMOD=/bin/chmod
ECHO=/bin/echo
MKDIR=/bin/mkdir
DATE=/bin/date
UNAME=/bin/uname
GREP=/bin/grep
SED=/bin/sed
TOUCH=/bin/touch
MKTEMP=/bin/mktemp
RSYNC=/usr/bin/rsync
SSH=/usr/bin/ssh
SCP=/usr/bin/scp
SSHADD=/usr/bin/ssh-add
SSHAGENT=/usr/bin/ssh-agent
TEE=/usr/bin/tee
STAT=/usr/bin/stat
NICE="/usr/bin/env nice"
SORT="/usr/bin/env sort"
TAIL=/usr/bin/tail
HEAD=/usr/bin/head
ROUTE=/sbin/route



#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#
# S E T   D E F A U L T S   A N D   P A R S E   O P T I O N S
#
#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

sshKeyOk=0

backupSource=""
requiredGateway=""
requiredInterface=""
sshIdentityFile=""
sshIdentity=""
otherOptions=""
foldersToBackup=bin,Documents,work
archiveFolder=
archiveOwner=

#
# Use the name of the local machine in the target path
#
localMachine=`$UNAME -n`
localUser=$USER

#
# Remember the original script paramters, as 'shift' destroys $*.
#
originalScriptParameters="$@"

#
# Parse options
#
while [ $# -gt 0 ] ; do

	option=$1
	shift
	
	case "$option" in
		--ssh-key-ok )
			sshKeyOk=1
			;;
		--ssh-identity )
			sshIdentityFile=$1
			shift
			;;
		--user )
			localUser=$1
			shift
			;;
		--backup )
			foldersToBackup=$1
			shift
			;;
		--archive )
			archiveFolder=$1
			shift
			;;
		--owner )
			archiveOwner=$1
			shift
			;;
		--rsync-flags )
			while [ $# -gt 0 ] && [ "X$1" != "X;" ] ; do
				otherOptions="$otherOptions $1"
				shift
			done
			shift
			;;
		-* )
			$ECHO "Unknown option $option" 1>&2
			exit 1
			;;
		* )
			backupSource=$option
			;;
	esac
done

#
# Force exit if user did not specify a backup source
#
if [ "X$backupSource" = "X" ] ; then
	$ECHO "Backup target not specified" 1>&2
	$ECHO "Flags for $0 were:  $originalScriptParameters"
	exit 1
fi

#
# Check to see if this script is already being
# run under ssh-agent, with our id specified via
# ssh-add.
#
sshIdentityPattern="/home/$USER/.ssh/id"
if [ "X$sshIdentityFile" != "X" ] ; then
	sshIdentityPattern=$sshIdentityFile
fi
if [ "X$sshKeyOk" != "X1" ] ; then
	hasID=`$SSHADD -l | $GREP $sshIdentityPattern`
fi

#
# Check to see if it's time to make the snapshot, or if
# we still need to invoke ssh-agent.  We will go
# ahead and snapshot if we have been called recursively
# ("sync" in the second parameter), or if ssh-add -l
# reported that we've already cached a key for this
# system in memory.
#
if [ "X$sshKeyOk" = "X1" ] || [ "X$hasID" != "X" ] ; then

	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	#
	# P R E P A R E   S S H   K E Y
	#
	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	
	#
	# If we do not have a key yet, then add
	# our identity file.  If the identity file
	# is password-protected, this statement will
	# prompt for the password.
	#	
	if [ "X$hasID" = "X" ] ; then
		$SSHADD $sshIdentityFile
	fi
	
	#
	# Check to see if we have a username in the target
	# or not.  If we do not, then we will use $localUser
	#
	if [ X`$ECHO "$backupSource" | $GREP "@"` \!= "X" ] ; then
		remoteuser=${backupSource/@*/}
		sourceMachine=$backupSource
	else
		remoteuser=$localUser
		sourceMachine=$localUser"@"$backupSource
	fi
	
	
	#
	# If there is no archive folder, then set it to a location 
	# named after the source machine
	#
	if [ "X$archiveFolder" = "X" ] ; then
		archiveFolder=$HOME/archives/$sourceMachine
	else
		#
		# If the archive folder was given a name, but not a
		# full path, then add the usual path in front of
		# the specified name.
		#
		if [ X`$ECHO "$archiveFolder" | $GREP "/"` = "X" ] ; then
			archiveFolder=$HOME/archives/$archiveFolder
		fi
	fi

	#
	# If the --backup parameter is a simple comma-separated list, "a,b,c"
	# then we will expand it to "/home/$remoteuser/a /home/$remoteuser/b /home/$remoteuser/c".
	# We do this by using "echo" to invoke bash brace-expansion. 
	#
	if [ "X"`$ECHO $foldersToBackup | $GREP "^[a-zA-Z0-9][a-zA-Z0-9]*,[a-zA-Z0-9,]*[a-zA-Z0-9]$"` != "X" ] ; then
		foldersToBackup=`eval $ECHO "/home/$remoteuser/{$foldersToBackup}"`
	else
		foldersToBackup="${foldersToBackup//##/ }"
	fi
	
	#
	# Print out our welcome message
	#
	target="Back up $sourceMachine to $archiveFolder"
	$ECHO ".............................................................................."
	$ECHO
	$ECHO "BEGIN ${0/*\//}                                " `$DATE`
	$ECHO "Target is $target"
	$ECHO "Running as user $USER, local user is $localUser, remote user is $remoteuser"
	$ECHO "Folders to back up are:  $foldersToBackup"
		
	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	#
	# S E T    U P   D I R E C T O R I E S   F O R   S Y N C
	#
	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	
	#
	# First step:  make sure that the archive folder exists
	#
	$MKDIR -p $archiveFolder/history
	
	#
	# Next, use cp -al to copy "current" to "previous"
	#
	# TODO:  Don't rm the in-progress when there's no change,
	# and skip the cp here if it already exists.
	#
	$CP -alf $archiveFolder/current $archiveFolder/previous.in-progress

	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	#
	# B A C K   U P   W I T H   R S Y N C
	#
	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	
	#
	# Push the documents in the folders to sync
	# from this machine to the target machine.
	#
	# -a == -rlptgoD
	#
	# Flags we've selected:
	#
	#	-r	recursive
	#	-l	copy symlinks as symlinks
	#	-t	preserve times
	#	-C	skip files the same way that CVS does
	#	-u	skip files that are newer on the receiver
	#	-v	verbose
	#	-z	compress data during transfer
	#
	#	--out-format="::: %M ::: %f"	Print modification date and filename
	#
	#	For a while, we did -a with --no-o, --no-g and --no-p:
	#
	#	-a	archive, implies:
	#
	#		-r	recursive
	#		-l	copy symlinks as symlinks
	#		-p	preserve permissions
	#		-t	preserve times
	#		-g	preserve groups
	#		-o	preserve owner
	#		-D	preserve device files
	#
	#	--no-o	make all files owned by user on receiving system
	#	--no-g	similarly, do not preserve group ownership.
	#	--no-p	do not preserve permissions (all backups will be writable)
	#
	# That was a bit overdone, though; removing -a and specifying the flags
	# we want directly was more straightforward.
	#
	# Normally, we might prefer to make the backups non-writable,
	# and perhaps we should do that (-chmod=-w); however, we would then
	# also have to explicitly chmod -R +w before removing backups during
	# history migration (maybe a good idea....)
	#
	$ECHO "Backing up $foldersToBackup"
	rsyncLog=`$MKTEMP /tmp/rsynclog.XXXXXXXX`
	$ECHO "BEGIN ${0/*\//} at" `$DATE` > $rsyncLog
	echo $RSYNC -rltCuvz \
		--exclude='~*' \
		--delete --delete-excluded \
		--out-format="::: %M ::: %f" \
		-e ssh $otherOptions \
		"$backupSource:'"$foldersToBackup"'" \
		$archiveFolder/current		| $TEE -a $rsyncLog
	
	for f in $foldersToBackup ; do
	$RSYNC -rltCuvz \
		--chmod=+w \
		--exclude='~*' \
		--delete --delete-excluded \
		--out-format="::: %M ::: %f" \
		-e ssh $otherOptions \
		"$backupSource:$f" \
		$archiveFolder/current		| $TEE -a $rsyncLog
	done
	
	#
	# Paw through the rsync log and pull out the modification
	# date of the most-recently-modified item from the transfer.
	# We take only the lines in the rsync lines that begin with
	# ":::", as those are the lines that we've specifically formatted
	# to include modification dates.  We then reverse-sort the result
	# and take the first item to find the most recent.
	#
	mostRecentModDate=`$GREP '^:::' $rsyncLog | grep -v '^:::.*::: \.$' | $SORT -r | $HEAD -n 1 | $SED -e 's/^::: //' -e 's/:[0-9][0-9] :::.*//' -e 's|-| |'`
	mostRecentModDateForTouch=`$ECHO $mostRecentModDate | $SED -e 's|[:/ ]*||g'`
	if [ "X$mostRecentModDate" != "X" ] ; then
		#
		# Transfer the rsync log to the remote machine.
		#
		$ECHO "The most recently modified file is $mostRecentModDate"
		$RM -f $archiveFolder/current/RSYNC_LOG
		$CP $rsyncLog $archiveFolder/current/RSYNC_LOG
		$RM -rf $rsyncLog
	else
		#
		# If nothing changed, then delete the 'previous in progress'
		# folder and the rsync log and quit.  (Note: it wouldn't
		# be necessary to delete "previous in progress" if we
		# also skipped the "cp -al" when "previous in progress" already
		# exists.)
		#
		$ECHO "Nothing has changed; exit."
		$RM -rf $archiveFolder/previous.in-progress
		$RM -rf $rsyncLog
		exit 0
	fi
	
	#
	# We have to use 'touch' to update the timestamp on
	# the sync'ed folder; otherwise, it won't change its
	# modification date from the time it is first created.
	# We set the modification date of the folder to the 
	# date of the newest item anywhere inside it rather than
	# using the backup time because it is a better representation
	# of the folder contents.
	#
	$TOUCH -t "$mostRecentModDateForTouch" $archiveFolder/current
	currentTimestampSinceEpoc=`$DATE --date="$mostRecentModDate" "+%s"`
	
	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	#
	# M A N A G E   S N A P S H O T   H I S T O R I E S
	#
	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	
	/bin/migratehistories --migrating $archiveFolder/previous --history-root $archiveFolder/history
	
	#
	# Move previous.in-progress to previous.  At this point, previous either
	# did not exist at all, or it was moved to one of the history folders.
	# We will go ahead and remove it, though, just in case migratehistories
	# failed for some reason.  (Otherwise we might end up with previous.in-progress
	# inside of the old previous folder.)
	#
	$RM -rf $archiveFolder/previous
	$MV $archiveFolder/previous.in-progress $archiveFolder/previous
	
	#
	# If an alternate owner was specified (e.g. if running as root),
	# then chown the archive folder as specified once we are done.
	#
	if [ "X$archiveOwner" != "X" ] ; then
		chown -R $archiveOwner $archiveFolder
	fi
	
	$ECHO "Done; exiting at" `$DATE`
	
else

	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	#
	# S E T   U P   S S H - A G E N T
	#
	#:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
	
	$ECHO "Recursive call with $originalScriptParameters"
	#
	# Make a recursive call to re-invoke this
	# script using ssh-agent.  Ssh-agent will
	# cache our keys in memory (keys are cached
	# via calls to ssh-add), so we do not have
	# to enter a password every time we call
	# rsync or ssh or scp.
	#
	$SSHAGENT $0 --ssh-key-ok $originalScriptParameters
fi