Add script to check AWS regions missing lambda layers (#96)

* Script to add layers to new region

* Script to check regions missing layers

* Call check regions after publishing to prod

* Bugfixes + get regions from aws

* Logic to make script idempotent

* Show regions missing layers

* Run list layers before publishing to prod

* Fix condition logic

* Get regions from AWS

* Set default batch size to 1

* Return 0 if no version found for region

Author: hghotra

scripts/add_new_region.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+OLD_REGION='us-east-1'
+
+LAYER_NAMES=("Datadog-Node10-x" "Datadog-Node12-x")
+NODE_VERSIONS_FOR_AWS_CLI=("nodejs10.x" "nodejs12.x")
+NEW_REGION=$1
+
+publish_layer() {
+    region=$1
+    layer_name=$2
+    aws_version_key=$3
+    layer_path=$4
+
+    version_nbr=$(aws lambda publish-layer-version --layer-name $layer_name \
+        --description "Datadog Lambda Layer for Node" \
+        --zip-file "fileb://$layer_path" \
+        --region $region \
+        --compatible-runtimes $aws_version_key \
+                        | jq -r '.Version')
+
+    aws lambda add-layer-version-permission --layer-name $layer_name \
+        --version-number $version_nbr \
+        --statement-id "release-$version_nbr" \
+        --action lambda:GetLayerVersion --principal "*" \
+        --region $region
+
+    echo "Published layer for region $region, version $aws_version_key, layer_name $layer_name, layer_version $version_nbr"
+}
+
+get_max_version() {
+    layer_name=$1
+    region=$2
+    last_layer_version=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] |  .Version")
+    if [ "$last_layer_version" == "null" ]; then
+        echo 0
+    else
+        echo $last_layer_version
+    fi
+}
+
+if [ -z "$1" ]; then
+    echo "Region parameter not specified, exiting"
+    exit 1
+fi
+
+j=0
+for layer_name in "${LAYER_NAMES[@]}"; do
+    # get latest version
+    last_layer_version=$(get_max_version $layer_name $OLD_REGION)
+    starting_version=$(get_max_version $layer_name $NEW_REGION)
+    starting_version=$(expr $starting_version + 1)
+
+    # exit if region is already all caught up
+    if [ $starting_version -gt $last_layer_version ]; then
+        echo "INFO: $NEW_REGION is already up to date for $layer_name"
+        continue
+    fi
+
+    # run for each version of layer
+    for i in $(seq 1 $last_layer_version); do 
+        layer_path=$layer_name"_"$i.zip
+        aws_version_key="${NODE_VERSIONS_FOR_AWS_CLI[$j]}"
+
+        # download layer versions 
+        URL=$(AWS_REGION=$OLD_REGION aws lambda get-layer-version --layer-name $layer_name --version-number $i --query Content.Location --output text)
+        curl $URL -o $layer_path
+
+        # publish layer to new region
+        publish_layer $NEW_REGION $layer_name $aws_version_key $layer_path
+        rm $layer_path
+    done
+
+    j=$(expr $j + 1)
+done

scripts/list_layers.sh

@@ -11,19 +11,21 @@
 set -e
 
 LAYER_NAMES=("Datadog-Node10-x" "Datadog-Node12-x")
-AVAILABLE_REGIONS=(us-east-2 us-east-1 us-west-1 us-west-2 ap-east-1 ap-south-1 ap-northeast-2 ap-southeast-1 ap-southeast-2 ap-northeast-1 ca-central-1 eu-north-1 eu-central-1 eu-west-1 eu-west-2 eu-west-3 sa-east-1)
+AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
+LAYERS_MISSING_REGIONS=()
 
 # Check region arg
 if [ -z "$2" ]; then
     >&2 echo "Region parameter not specified, running for all available regions."
-    REGIONS=("${AVAILABLE_REGIONS[@]}")
+    REGIONS=$AVAILABLE_REGIONS
 else
+
     >&2 echo "Region parameter specified: $2"
-    if [[ ! " ${AVAILABLE_REGIONS[@]} " =~ " ${2} " ]]; then
-        >&2 echo "Could not find $2 in available regions: ${AVAILABLE_REGIONS[@]}"
+    if [[ ! "$AVAILABLE_REGIONS" == *"$2"* ]]; then
+        >&2 echo "Could not find $2 in available regions:" $AVAILABLE_REGIONS
         >&2 echo ""
         >&2 echo "EXITING SCRIPT."
-        return 1
+        exit 1
     fi
     REGIONS=($2)
 fi
@@ -43,15 +45,24 @@ else
     LAYERS=($1)
 fi
 
-for region in "${REGIONS[@]}"
+for region in $REGIONS
 do
     for layer_name in "${LAYERS[@]}"
     do
         last_layer_arn=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] |  .LayerVersionArn")
-        if [ -z $last_layer_arn ]; then
-             >&2 echo "No layer found for $region, $layer_name"
+        if [ "$last_layer_arn" == "null" ]; then
+            >&2 echo "No layer found for $region, $layer_name"
+            if [[ ! " ${LAYERS_MISSING_REGIONS[@]} " =~ " ${region} " ]]; then
+                LAYERS_MISSING_REGIONS+=( $region )
+            fi
         else
             echo $last_layer_arn
         fi
     done
-done
+done
+
+if [ ${#LAYERS_MISSING_REGIONS[@]} -gt 0 ]; then
+    echo "WARNING: Following regions missing layers: ${LAYERS_MISSING_REGIONS[@]}"
+    echo "Please run ./add_new_region.sh <new_region> to add layers to the missing regions" 
+    exit 1
+fi

scripts/publish_layers.sh

@@ -16,7 +16,7 @@ trap "pkill -P $$; exit 1;" INT
 NODE_VERSIONS_FOR_AWS_CLI=("nodejs10.x" "nodejs12.x")
 LAYER_PATHS=(".layers/datadog_lambda_node10.15.zip" ".layers/datadog_lambda_node12.13.zip")
 LAYER_NAMES=("Datadog-Node10-x" "Datadog-Node12-x")
-AVAILABLE_REGIONS=(us-east-2 us-east-1 us-west-1 us-west-2 ap-east-1 ap-south-1 ap-northeast-2 ap-southeast-1 ap-southeast-2 ap-northeast-1 ca-central-1 eu-north-1 eu-central-1 eu-west-1 eu-west-2 eu-west-3 sa-east-1)
+AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
 
 # Check that the layer files exist
 for layer_file in "${LAYER_PATHS[@]}"
@@ -30,19 +30,19 @@ done
 # Check region arg
 if [ -z "$1" ]; then
     echo "Region parameter not specified, running for all available regions."
-    REGIONS=("${AVAILABLE_REGIONS[@]}")
+    REGIONS=$AVAILABLE_REGIONS
 else
     echo "Region parameter specified: $1"
-    if [[ ! " ${AVAILABLE_REGIONS[@]} " =~ " ${1} " ]]; then
-        echo "Could not find $1 in available regions: ${AVAILABLE_REGIONS[@]}"
+    if [[ ! "$AVAILABLE_REGIONS" == *"$1"* ]]; then
+        echo "Could not find $1 in available regions: $AVAILABLE_REGIONS"
         echo ""
         echo "EXITING SCRIPT."
         exit 1
     fi
     REGIONS=($1)
 fi
 
-echo "Starting publishing layers for regions: ${REGIONS[*]}"
+echo "Starting publishing layers for regions: $REGIONS"
 
 publish_layer() {
     region=$1
@@ -66,7 +66,7 @@ publish_layer() {
     echo "Published layer for region $region, node version $aws_version_key, layer_name $layer_name, layer_version $version_nbr"
 }
 
-BATCH_SIZE=60
+BATCH_SIZE=1
 PIDS=()
 
 wait_for_processes() {
@@ -76,7 +76,7 @@ wait_for_processes() {
     PIDS=()
 }
 
-for region in "${REGIONS[@]}"
+for region in $REGIONS
 do
     echo "Starting publishing layer for region $region..."
 

scripts/publish_prod.sh

@@ -22,6 +22,9 @@ if [ -z "$AWS_SESSION_TOKEN" ]; then
     exit 1
 fi
 
+echo 'Checking Regions'
+./scripts/list_layers.sh
+
 yarn login
 
 PACKAGE_VERSION=$(node -pe "require('./package.json').version")
@@ -36,4 +39,4 @@ git push origin "refs/tags/v$PACKAGE_VERSION"
 
 echo 'Publishing Lambda Layer'
 ./scripts/build_layers.sh
-./scripts/publish_layers.sh
+./scripts/publish_layers.sh