ddbr.sh

#!/usr/bin/env bash

set -euo pipefail

# Bare script name
MYNAME=`basename "$0"`
_PRINT_HELP=no

#!/bin/bash
#
# ARG_OPTIONAL_SINGLE([profile],[],[AWS profile],[default])
# ARG_OPTIONAL_SINGLE([source-region],[],[From region],[])
# ARG_OPTIONAL_SINGLE([destination-region],[],[To region],[])
# ARG_OPTIONAL_SINGLE([pipeline-definition],[],[Data-pipeline definition],[])
# ARG_OPTIONAL_SINGLE([s3-temp],[],[S3 path to store temporary data],[])
# ARG_OPTIONAL_SINGLE([s3-log],[],[S3 path to store log files],[])
# ARG_OPTIONAL_REPEATED([table],[],[table to replicate],[])
# ARG_VERBOSE([v])
# ARG_HELP([Cross region replicate DynamoDB Tables])
# ARGBASH_SET_INDENT([  ])
# ARGBASH_GO()
# needed because of Argbash --> m4_ignore([
### START OF CODE GENERATED BY Argbash v2.6.1 one line above ###
# Argbash is a bash code generator used to get arguments parsing right.
# Argbash is FREE SOFTWARE, see https://argbash.io for more info
# Generated online by https://argbash.io/generate

# When called, the process ends.
# Args:
#   $1: The exit message (print to stderr)
#   $2: The exit code (default is 1)
# if env var _PRINT_HELP is set to 'yes', the usage is print to stderr (prior to )
# Example:
#   test -f "$_arg_infile" || _PRINT_HELP=yes die "Can't continue, have to supply file as an argument, got '$_arg_infile'" 4
die()
{
  local _ret=1
  if [ "$#" -gt 1 ]; then
    local _ret=$2
  fi
  test -n "$_ret" || _ret=1
  test "$_PRINT_HELP" = yes && print_help >&2
  echo "$1" >&2
  exit ${_ret}
}

# Function that evaluates whether a value passed to it begins by a character
# that is a short option of an argument the script knows about.
# This is required in order to support getopts-like short options grouping.
begins_with_short_option()
{
  local first_option all_short_options
  all_short_options='vh'
  first_option="${1:0:1}"
  test "$all_short_options" = "${all_short_options/$first_option/}" && return 1 || return 0
}



# THE DEFAULTS INITIALIZATION - OPTIONALS
_arg_profile="default"
_arg_source_region=
_arg_destination_region=
_arg_pipeline_definition=
_arg_s3_temp=
_arg_s3_log=
_arg_table=()
_arg_verbose=0

# Function that prints general usage of the script.
# This is useful if users asks for it, or if there is an argument parsing error (unexpected / spurious arguments)
# and it makes sense to remind the user how the script is supposed to be called.
print_help ()
{
  printf '%s\n' "Cross region replicate DynamoDB Tables"
  printf 'Usage: %s [--profile <arg>] [--source-region <arg>] [--destination-region <arg>] [--pipeline-definition <arg>] [--s3-temp <arg>] [--s3-log <arg>] [--table <arg>] [-v|--verbose] [-h|--help]\n' "$0"
  printf '\t%s\n' "--profile: AWS profile (default: 'default')"
  printf '\t%s\n' "--source-region: From region (no default)"
  printf '\t%s\n' "--destination-region: To region (no default)"
  printf '\t%s\n' "--pipeline-definition: Data-pipeline definition (no default)"
  printf '\t%s\n' "--s3-temp: S3 path to store temporary data (no default)"
  printf '\t%s\n' "--s3-log: S3 path to store log files (no default)"
  printf '\t%s\n' "--table: table to replicate (empty by default)"
  printf '\t%s\n' "-v,--verbose: Set verbose output (can be specified multiple times to increase the effect)"
  printf '\t%s\n' "-h,--help: Prints help"
}

# The parsing of the command-line
parse_commandline ()
{
  while test $# -gt 0
  do
    _key="$1"
    case "$_key" in
      # We support whitespace as a delimiter between option argument and its value.
      # Therefore, we expect the --profile value, so we watch for --profile.
      # Since we know that we got the long option,
      # we just reach out for the next argument to get the value.
      --profile)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_profile="$2"
        shift
        ;;
      # We support the = as a delimiter between option argument and its value.
      # Therefore, we expect --profile=value, so we watch for --profile=*
      # For whatever we get, we strip '--profile=' using the ${var##--profile=} notation
      # to get the argument value
      --profile=*)
        _arg_profile="${_key##--profile=}"
        ;;
      # See the comment of option '--profile' to see what's going on here - principle is the same.
      --source-region)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_source_region="$2"
        shift
        ;;
      # See the comment of option '--profile=' to see what's going on here - principle is the same.
      --source-region=*)
        _arg_source_region="${_key##--source-region=}"
        ;;
      # See the comment of option '--profile' to see what's going on here - principle is the same.
      --destination-region)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_destination_region="$2"
        shift
        ;;
      # See the comment of option '--profile=' to see what's going on here - principle is the same.
      --destination-region=*)
        _arg_destination_region="${_key##--destination-region=}"
        ;;
      # See the comment of option '--profile' to see what's going on here - principle is the same.
      --pipeline-definition)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_pipeline_definition="$2"
        shift
        ;;
      # See the comment of option '--profile=' to see what's going on here - principle is the same.
      --pipeline-definition=*)
        _arg_pipeline_definition="${_key##--pipeline-definition=}"
        ;;
      # See the comment of option '--profile' to see what's going on here - principle is the same.
      --s3-temp)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_s3_temp="$2"
        shift
        ;;
      # See the comment of option '--profile=' to see what's going on here - principle is the same.
      --s3-temp=*)
        _arg_s3_temp="${_key##--s3-temp=}"
        ;;
      # See the comment of option '--profile' to see what's going on here - principle is the same.
      --s3-log)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_s3_log="$2"
        shift
        ;;
      # See the comment of option '--profile=' to see what's going on here - principle is the same.
      --s3-log=*)
        _arg_s3_log="${_key##--s3-log=}"
        ;;
      # See the comment of option '--profile' to see what's going on here - principle is the same.
      --table)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_table+=("$2")
        shift
        ;;
      # See the comment of option '--profile=' to see what's going on here - principle is the same.
      --table=*)
        _arg_table+=("${_key##--table=}")
        ;;
      # The verbose argurment doesn't accept a value,
      # we expect the --verbose or -v, so we watch for them.
      -v|--verbose)
        _arg_verbose=$((_arg_verbose + 1))
        ;;
      # We support getopts-style short arguments clustering,
      # so as -v doesn't accept value, other short options may be appended to it, so we watch for -v*.
      # After stripping the leading -v from the argument, we have to make sure
      # that the first character that follows coresponds to a short option.
      -v*)
        _arg_verbose=$((_arg_verbose + 1))
        _next="${_key##-v}"
        if test -n "$_next" -a "$_next" != "$_key"
        then
          begins_with_short_option "$_next" && shift && set -- "-v" "-${_next}" "$@" || die "The short option '$_key' can't be decomposed to ${_key:0:2} and -${_key:2}, because ${_key:0:2} doesn't accept value and '-${_key:2:1}' doesn't correspond to a short option."
        fi
        ;;
      # See the comment of option '--verbose' to see what's going on here - principle is the same.
      -h|--help)
        print_help
        exit 0
        ;;
      # See the comment of option '-v' to see what's going on here - principle is the same.
      -h*)
        print_help
        exit 0
        ;;
      *)
        _PRINT_HELP=yes die "FATAL ERROR: Got an unexpected argument '$1'" 1
        ;;
    esac
    shift
  done
}

# Logs information to output with following style [<DATE> <TIME>] [<LEVEL>] [<FILENAME>] <MESSAGE>
# @param $1 Log level, can be {DEBUG, INFO, WARN, ERROR}
# @param $2 Log message
# Note: DEBUG logs are only shown if {-v|--verbose} option is active
# Note: ERROR logs are sent to stderr, all other logs are sent to stdout
function log() {
  if [ "$1" == "DEBUG" ] && [ ${_arg_verbose} -eq 0 ]; then return; fi;
  local LEVEL="$1"
  if [ ${#LEVEL} -eq 4 ]; then LEVEL="${LEVEL} "; fi
  local time_and_date=`date '+%Y-%m-%d %H:%M:%S'`
  if [ "$LEVEL" == "ERROR" ]; then
    cat <<< "[$time_and_date] [ERROR] [$MYNAME] $2" 1>&2
  else
    echo "[$time_and_date] [$LEVEL] [$MYNAME] $2"
  fi
}

validate_tables() {
  for table in "${_arg_table[@]}"
  do
    log DEBUG "Validating table ${table}"
    aws --region "$_arg_source_region" dynamodb describe-table --table-name "$table" --profile "$_arg_profile" > /dev/null || die "Nonexistent tables cannot be processed, aborting process"
    aws --region "$_arg_destination_region" dynamodb describe-table --table-name "$table" --profile "$_arg_profile" > /dev/null || die "Table is not defined in destination region, aborting process"
    log DEBUG "Table ${table} was found"
  done
}

validate_required_args() {
  test ! -z "$_arg_profile" || log INFO "No profile was provided, default profile will be used"
  test ! -z "$_arg_pipeline_definition" || die "Pipeline definition must be provided, aborting"
  test -r "$_arg_pipeline_definition" || die "Cannot find or read pipeline definition file $_arg_pipeline_definition"
  test ! -z "$_arg_source_region" || die "Source region must be provided, aborting"
  test ! -z "$_arg_destination_region" || die "Destination region must be provided, aborting"
  test ! -z "$_arg_s3_temp" || die "S3 path for temporary files must be provided, aborting"
  aws s3 ls "$_arg_s3_temp" --profile "$_arg_profile" > /dev/null || die "S3 path $_arg_s3_temp does not exist"
  test ! -z "$_arg_s3_log" || die "S3 path for log files must be provided, aborting"
  aws s3 ls "$_arg_s3_log" --profile "$_arg_profile" > /dev/null || die "S3 path $_arg_s3_log does not exist"
  test ${#_arg_table[@]} -ne 0 || die "Minimum one table must be provided, aborting"
  validate_tables
}

create_pipeline() {
  log DEBUG "Creating pipeline with name $2"
  local PIPELINE_ID=$(aws datapipeline create-pipeline --name "$2" --unique-id "$2" --profile "$_arg_profile" | jq '.pipelineId')
  PIPELINE_ID=$(sed -e 's/^"//' -e 's/"$//' <<<"$PIPELINE_ID")
  log INFO "Created pipeline with id $PIPELINE_ID and name $2"
  eval "$1=${PIPELINE_ID}"
}

update_pipeline_definition() {
  log DEBUG "Updating pipeline definition for table $1 and pipeline id $2"
  aws datapipeline put-pipeline-definition --pipeline-definition "file://${_arg_pipeline_definition}" \
  --parameter-values myTempS3Folder="$_arg_s3_temp" myDDBSourceTableName="$1" \
  myDDBDestinationTableName="$1" myDDBSourceRegion="$_arg_source_region" \
  myDDBDestinationRegion="$_arg_destination_region" myS3LogsPath="$_arg_s3_log" --pipeline-id "$2" --profile "$_arg_profile"\
   > /dev/null || die "Failed to update pipeline definition for table $1 and pipeline id $2, aborting process"
}

activate_pipeline() {
  log DEBUG "Trying to activate pipeline $1"
  aws datapipeline activate-pipeline --pipeline-id "$1" --profile "$_arg_profile" > /dev/null || die "Failed to activate pipeline id $1, aborting process"
}

perform_replication() {
  for table in "${_arg_table[@]}"
  do
    log INFO "Preparing replication for table ${table} with name dynamodb_copy_${table}"
    create_pipeline "pipeline_id" "dynamodb_copy_${table}"
    update_pipeline_definition "$table" "$pipeline_id"
    log INFO "Finished updating pipeline definition ${pipeline_id}"
    activate_pipeline "$pipeline_id"
  done
}

# Now call all the functions defined above that are needed to get the job done
log DEBUG "Parsing arguments"
parse_commandline "$@"
validate_required_args
log DEBUG "Finished parsing arguments"
log DEBUG "Performing replication with aws profile $_arg_profile on tables ${_arg_table[@]} to replicate from source
region $_arg_source_region to destination region $_arg_destination_region."
log INFO "Validating given tables"
perform_replication