From 0bd7d2a2fd4db221ad4b7e4efc6636e248b64464 Mon Sep 17 00:00:00 2001 From: Brad Woodward Date: Tue, 7 Jan 2020 16:09:14 -0700 Subject: [PATCH] Moving configs to envvars, making regions not hardcoded --- .gitignore | 1 + ..._iam_roles.tf => cognito_iam_roles.tf.bak} | 0 terraform/deploy.sh | 12 +- terraform/dictionaries.auto.tfvars | 16 +- terraform/jsonnet/cognito_iam_roles.libsonnet | 186 +++++++++--- terraform/jsonnet/lambda_functions.libsonnet | 85 +++++- terraform/jsonnet/lambda_iam_roles.libsonnet | 37 ++- terraform/jsonnet/null_resources.libsonnet | 6 +- terraform/jsonnet/templates.libsonnet | 49 +--- terraform/jsonnet/variables.libsonnet | 18 +- .../proxy_api_handler/main.js | 79 +++--- .../proxy_api_handler/package-lock.json | 266 ++++++++++++++++-- .../proxy_api_handler/package.json | 3 +- .../lambda_functions/spot_monitor/main.js | 10 +- .../spot_monitor/package-lock.json | 265 ++++++++++++++++- .../spot_monitor/package.json | 3 +- .../lambda_functions/status_reporter/main.js | 4 +- .../status_reporter/package-lock.json | 20 +- .../status_reporter/package.json | 2 +- terraform/npk-settings.json.sample | 3 +- .../templates/api_handler_variables-fresh.tpl | 13 - terraform/templates/api_handler_variables.tpl | 38 --- terraform/templates/npk_settings-fresh.tpl | 6 - terraform/templates/npk_settings.tpl | 31 -- terraform/templates/userdata.tpl | 22 +- terraform/terraform.jsonnet | 22 +- 26 files changed, 877 insertions(+), 320 deletions(-) rename terraform/{cognito_iam_roles.tf => cognito_iam_roles.tf.bak} (100%) delete mode 100644 terraform/templates/api_handler_variables-fresh.tpl delete mode 100644 terraform/templates/api_handler_variables.tpl delete mode 100644 terraform/templates/npk_settings-fresh.tpl delete mode 100644 terraform/templates/npk_settings.tpl diff --git a/.gitignore b/.gitignore index c993cb1..91003c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.auto.tfvars +*.pem !terraform/dictionaries.auto.tfvars */.terraform* *.tfstate diff --git a/terraform/cognito_iam_roles.tf b/terraform/cognito_iam_roles.tf.bak similarity index 100% rename from terraform/cognito_iam_roles.tf rename to terraform/cognito_iam_roles.tf.bak diff --git a/terraform/deploy.sh b/terraform/deploy.sh index 888b2c1..6479b92 100755 --- a/terraform/deploy.sh +++ b/terraform/deploy.sh @@ -83,16 +83,6 @@ echo "[*] Generating Terraform configurations" # Generate terraform configs jsonnet -m . terraform.jsonnet -echo "[*] Creating dynamic templates" -# Inject the dynamic template content into the template -cat templates/api_handler_variables-fresh.tpl > templates/api_handler_variables.tpl -cat template-inject_api_handler.json >> templates/api_handler_variables.tpl -cat templates/npk_settings-fresh.tpl > templates/npk_settings.tpl -cat template-inject_api_handler.json | jq -r 'to_entries | map( {(.key) : (.value | keys)}) | add' >> templates/npk_settings.tpl - -echo -n "}" >> templates/api_handler_variables.tpl -echo -n "}" >> templates/npk_settings.tpl - terraform init terraform apply -auto-approve -terraform apply -auto-approve # Yes, userdata.sh is an unresolvable cyclical dependency. I am ashamed. \ No newline at end of file +terraform apply -auto-approve \ No newline at end of file diff --git a/terraform/dictionaries.auto.tfvars b/terraform/dictionaries.auto.tfvars index 747be74..0d5f768 100644 --- a/terraform/dictionaries.auto.tfvars +++ b/terraform/dictionaries.auto.tfvars @@ -6,12 +6,12 @@ /* Dictionaries Variables */ -dictionary-east-1 = "arn:aws:s3:::npk-dictionary-east-1-20181029005812833000000004" -dictionary-east-2 = "arn:aws:s3:::npk-dictionary-east-2-20181029005812776500000003" -dictionary-west-1 = "arn:aws:s3:::npk-dictionary-west-1-20181029005812746900000001" -dictionary-west-2 = "arn:aws:s3:::npk-dictionary-west-2-20181029005812750900000002" +dictionary-us-east-1 = "arn:aws:s3:::npk-dictionary-east-1-20181029005812833000000004" +dictionary-us-east-2 = "arn:aws:s3:::npk-dictionary-east-2-20181029005812776500000003" +dictionary-us-west-1 = "arn:aws:s3:::npk-dictionary-west-1-20181029005812746900000001" +dictionary-us-west-2 = "arn:aws:s3:::npk-dictionary-west-2-20181029005812750900000002" -dictionary-east-1-id = "npk-dictionary-east-1-20181029005812833000000004" -dictionary-east-2-id = "npk-dictionary-east-2-20181029005812776500000003" -dictionary-west-1-id = "npk-dictionary-west-1-20181029005812746900000001" -dictionary-west-2-id = "npk-dictionary-west-2-20181029005812750900000002" \ No newline at end of file +dictionary-us-east-1-id = "npk-dictionary-east-1-20181029005812833000000004" +dictionary-us-east-2-id = "npk-dictionary-east-2-20181029005812776500000003" +dictionary-us-west-1-id = "npk-dictionary-west-1-20181029005812746900000001" +dictionary-us-west-2-id = "npk-dictionary-west-2-20181029005812750900000002" \ No newline at end of file diff --git a/terraform/jsonnet/cognito_iam_roles.libsonnet b/terraform/jsonnet/cognito_iam_roles.libsonnet index 7d0d4b8..9a68041 100644 --- a/terraform/jsonnet/cognito_iam_roles.libsonnet +++ b/terraform/jsonnet/cognito_iam_roles.libsonnet @@ -1,42 +1,160 @@ { - "aws_iam_role": { - "cognito_authenticated": { - "name_prefix": "cognito_authenticated_role_", - "assume_role_policy": '{"Version": "2012-10-17","Statement": [{ - "Effect": "Allow", - "Principal": {"Federated": "cognito-identity.amazonaws.com"}, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": {"cognito-identity.amazonaws.com:aud": "${aws_cognito_identity_pool.main.id}"}, - "ForAnyValue:StringLike": {"cognito-identity.amazonaws.com:amr": "authenticated"} - }}]}' + "resource": { + "aws_iam_role": { + "cognito_authenticated": { + "name_prefix": "cognito_authenticated_role_", + "assume_role_policy": '{"Version": "2012-10-17","Statement": [{ + "Effect": "Allow", + "Principal": {"Federated": "cognito-identity.amazonaws.com"}, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": {"cognito-identity.amazonaws.com:aud": "${aws_cognito_identity_pool.main.id}"}, + "ForAnyValue:StringLike": {"cognito-identity.amazonaws.com:amr": "authenticated"} + }}]}' + }, + "cognito_unauthenticated": { + "name_prefix": "cognito_unauthenticated_role_", + "assume_role_policy": '{"Version": "2012-10-17","Statement": [{ + "Effect": "Allow","Principal": {"Federated": "cognito-identity.amazonaws.com"}, + "Action": "sts:AssumeRoleWithWebIdentity"} + ]}' + }, }, - "cognito_unauthenticated": { - "name_prefix": "cognito_unauthenticated_role_", - "assume_role_policy": '{"Version": "2012-10-17","Statement": [{ - "Effect": "Allow","Principal": {"Federated": "cognito-identity.amazonaws.com"}, - "Action": "sts:AssumeRoleWithWebIdentity"} - ]}' - }, - }, - "aws_iam_role_policy": { - "cognito_authenticated": { - "name_prefix": "cognito_authenticated_policy_", - "role": "${aws_iam_role.cognito_authenticated.id}", - "policy": "${data.aws_iam_policy_document.cognito_authenticated.json}" + "aws_iam_role_policy": { + "cognito_authenticated": { + "name_prefix": "cognito_authenticated_policy_", + "role": "${aws_iam_role.cognito_authenticated.id}", + "policy": "${data.aws_iam_policy_document.cognito_authenticated.json}" + }, + "cognito_unauthenticated": { + "name_prefix": "cognito_authenticated_policy_", + "role": "${aws_iam_role.cognito_unauthenticated.id}", + "policy": "${data.aws_iam_policy_document.cognito_unauthenticated.json}" + } }, - "cognito_unauthenticated": { - "name_prefix": "cognito_authenticated_policy_", - "role": "${aws_iam_role.cognito_unauthenticated.id}", - "policy": "${data.aws_iam_policy_document.cognito_unauthenticated.json}" + "aws_cognito_identity_pool_roles_attachment": { + "default": { + "identity_pool_id": "${aws_cognito_identity_pool.main.id}", + "roles": { + "authenticated": "${aws_iam_role.cognito_authenticated.arn}", + "unauthenticated": "${aws_iam_role.cognito_unauthenticated.arn}" + } + } } }, - "aws_cognito_identity_pool_roles_attachment": { - "default": { - "identity_pool_id": "${aws_cognito_identity_pool.main.id}", - "roles": { - "authenticated": "${aws_iam_role.cognito_authenticated.arn}", - "unauthenticated": "${aws_iam_role.cognito_unauthenticated.arn}" + data(settings):: + local regionKeys = std.objectFields(settings.regions); + { + "aws_iam_policy_document": { + "cognito_authenticated": { + "statement": [{ + "sid": "1", + "actions": [ + "cognito-identity:*", + "mobileanalytics:PutEvents", + "cognito-sync:*", + "ec2:describeSpotPriceHistory", + "pricing:*" + ], + "resources": [ + "*" + ] + },{ + "sid": "2", + "actions": [ + "s3:PutObject" + ], + "resources": [ + "${aws_s3_bucket.user_data.arn}/&{cognito-identity.amazonaws.com:sub}/uploads/*" + ] + },{ + "sid": "3", + "actions": [ + "s3:GetObject", + "s3:ListObjectVersions", + "s3:DeleteObject" + ], + "resources": [ + "${aws_s3_bucket.user_data.arn}/&{cognito-identity.amazonaws.com:sub}", + "${aws_s3_bucket.user_data.arn}/&{cognito-identity.amazonaws.com:sub}/*" + ] + },{ + "sid": "4", + "actions": [ + "s3:ListBucket" + ], + "resources": [ + "${aws_s3_bucket.user_data.arn}", + ], + "condition": [{ + "test": "StringLike", + "variable": "s3:prefix", + + "values": [ + "&{cognito-identity.amazonaws.com:sub}/", + "&{cognito-identity.amazonaws.com:sub}/*" + ] + }] + },{ + "sid": "5", + "actions": [ + "dynamodb:GetItem", + "dynamodb:BatchGetItem", + "dynamodb:Query" + ], + "resources": [ + "${aws_dynamodb_table.campaigns.arn}", + "${aws_dynamodb_table.settings.arn}" + ], + "condition": [{ + "test": "ForAllValues:StringEquals", + "variable": "dynamodb:LeadingKeys", + + "values": [ + "&{cognito-identity.amazonaws.com:sub}", + "admin" + ] + }] + },{ + "sid": "6", + "actions": [ + "s3:ListBucket" + ], + "resources": [ + "${var.dictionary-" + regionKeys[i] + "}" + for i in std.range(0, std.length(regionKeys) - 1) + ] + },{ + "sid": "7", + "actions": [ + "s3:GetObject" + ], + "resources": [ + "${var.dictionary-" + regionKeys[i] + "}/*" + for i in std.range(0, std.length(regionKeys) - 1) + ] + },{ + "sid": "8", + "actions": [ + "execute-api:Invoke" + ], + "resources": [ + "${aws_api_gateway_deployment.npk.execution_arn}/*/userproxy/*" + ] + }] + }, + "cognito_unauthenticated": { + "statement": [{ + "sid": "logs", + "actions": [ + "cognito-identity:*", + "mobileanalytics:PutEvents", + "cognito-sync:*" + ], + "resources": [ + "*" + ] + }] } } } diff --git a/terraform/jsonnet/lambda_functions.libsonnet b/terraform/jsonnet/lambda_functions.libsonnet index 981f423..1a6f257 100644 --- a/terraform/jsonnet/lambda_functions.libsonnet +++ b/terraform/jsonnet/lambda_functions.libsonnet @@ -1,5 +1,7 @@ { - "resource": { + resources(settings):: + local regionKeys = std.objectFields(settings.regions); + { "aws_lambda_function": { "proxy_api_handler": { "depends_on": ["data.archive_file.proxy_api_handler", "aws_iam_role_policy.lambda_proxy_api_handler"], @@ -11,10 +13,35 @@ "runtime": "nodejs8.10", "timeout": 60, + "environment": { + "variables": { + "www_dns_names": std.toString(settings.dnsNames.www), + "campaign_max_price": "${var.campaign_max_price}", + "userdata_bucket": "${aws_s3_bucket.user_data.id}", + "instanceProfile": "${aws_iam_instance_profile.npk_node.arn}", + "iamFleetRole": "${aws_iam_role.npk_fleet_role.arn}", + "availabilityZones": std.manifestJsonEx({ + [regionKeys[i]]: { + [settings.regions[regionKeys[i]][azi]]: "${aws_subnet." + settings.regions[regionKeys[i]][azi] + ".id}" + for azi in std.range(0, std.length(settings.regions[regionKeys[i]]) - 1) + } + for i in std.range(0, std.length(regionKeys) - 1) + }, ""), + "dictionaryBuckets": std.manifestJsonEx({ + [regionKeys[i]]: "${var.dictionary-" + regionKeys[i] + "-id}" + for i in std.range(0, std.length(regionKeys) - 1) + }, "") + } + }, + "dead_letter_config": { "target_arn": "${aws_sqs_queue.api_handler_dlq.arn}" - }, - }, + } + } + if std.objectHas(settings, "debug_lambda") && settings.debug_lambda == true then { + "tracing_config": { + "mode": "Active" + } + } else {}, "status_reporter": { "depends_on": ["data.archive_file.status_reporter", "aws_iam_role_policy.lambda_status_reporter"], "filename": "./lambda_functions/zip_files/status_reporter.zip", @@ -22,13 +49,33 @@ "role": "${aws_iam_role.lambda_status_reporter.arn}", "handler": "main.main", "source_code_hash": "${data.archive_file.status_reporter.output_base64sha256}", - "runtime": "nodejs8.10", + "runtime": "nodejs12.x", "timeout": 60, + "environment": { + "variables": { + "www_dns_name": std.toString(settings.dnsNames.www), + "region": "${var.region}", + "campaign_max_price": "${var.campaign_max_price}", + "critical_events_sns_topic": "${aws_sns_topic.critical_events.id}", + "availabilityZones": std.manifestJsonEx({ + [regionKeys[i]]: { + [settings.regions[regionKeys[i]][azi]]: "${aws_subnet." + settings.regions[regionKeys[i]][azi] + ".id}" + for azi in std.range(0, std.length(settings.regions[regionKeys[i]]) - 1) + } + for i in std.range(0, std.length(regionKeys) - 1) + }, "") + } + }, + "dead_letter_config": { "target_arn": "${aws_sqs_queue.status_reporter_dlq.arn}" }, - }, + } + if std.objectHas(settings, "debug_lambda") && settings.debug_lambda == true then { + "tracing_config": { + "mode": "Active" + } + } else {}, "spot_monitor": { "depends_on": ["data.archive_file.spot_monitor", "aws_iam_role_policy.lambda_spot_monitor"], "filename": "./lambda_functions/zip_files/spot_monitor.zip", @@ -36,14 +83,34 @@ "role": "${aws_iam_role.lambda_spot_monitor.arn}", "handler": "main.main", "source_code_hash": "${data.archive_file.spot_monitor.output_base64sha256}", - "runtime": "nodejs8.10", + "runtime": "nodejs12.x", "memory_size": 512, "timeout": 10, + "environment": { + "variables": { + "www_dns_name": std.toString(settings.dnsNames.www), + "region": "${var.region}", + "campaign_max_price": "${var.campaign_max_price}", + "critical_events_sns_topic": "${aws_sns_topic.critical_events.id}", + "availabilityZones": std.manifestJsonEx({ + [regionKeys[i]]: { + [settings.regions[regionKeys[i]][azi]]: "${aws_subnet." + settings.regions[regionKeys[i]][azi] + ".id}" + for azi in std.range(0, std.length(settings.regions[regionKeys[i]]) - 1) + } + for i in std.range(0, std.length(regionKeys) - 1) + }, "") + } + }, + "dead_letter_config": { "target_arn": "${aws_sns_topic.critical_events.arn}" } - } + } + if std.objectHas(settings, "debug_lambda") && settings.debug_lambda == true then { + "tracing_config": { + "mode": "Active" + } + } else {} }, "aws_lambda_permission": { "spot_monitor": { @@ -82,8 +149,8 @@ "archive_file": { "proxy_api_handler": { "depends_on": [ - "local_file.api_handler_variables", "null_resource.npm_install_proxy_api_handler" + // "data.template_file.userdata_template" //TODO: Fix cyclical dependency ], "type": "zip", "source_dir": "${path.module}/lambda_functions/proxy_api_handler/", @@ -91,7 +158,6 @@ }, "status_reporter": { "depends_on": [ - "local_file.lambda_functions_settings-status_reporter", "null_resource.npm_install_status_reporter" ], "type": "zip", @@ -100,7 +166,6 @@ }, "spot_monitor": { "depends_on": [ - "local_file.lambda_functions_settings-spot_monitor", "null_resource.npm_install_spot_monitor" ], "type": "zip", diff --git a/terraform/jsonnet/lambda_iam_roles.libsonnet b/terraform/jsonnet/lambda_iam_roles.libsonnet index e84dd0e..c73b725 100644 --- a/terraform/jsonnet/lambda_iam_roles.libsonnet +++ b/terraform/jsonnet/lambda_iam_roles.libsonnet @@ -1,5 +1,7 @@ + + { - "resource": { + resource(settings):: { "aws_iam_role": { "lambda_proxy_api_handler": { "name_prefix": "npk_lambda_api_handler_role_", @@ -46,8 +48,31 @@ "policy": "${data.aws_iam_policy_document.lambda_spot_monitor.json}", } } - }, - "data": { + } + if std.objectHas(settings, "debug_lambda") && settings.debug_lambda == true then { + "aws_iam_policy_attachment": { + "lambda_proxy_api_handler-xray": { + "name": "lambda_proxy_api_handler-xray", + "roles": ["${aws_iam_role.lambda_proxy_api_handler.id}"], + + "policy_arn": "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess", + }, + "lambda_status_reporter-xray": { + "name": "lambda_status_reporter-xray", + "roles": ["${aws_iam_role.lambda_status_reporter.id}"], + + "policy_arn": "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess", + }, + "lambda_spot_monitor-xray": { + "name": "lambda_spot_monitor-xray", + "roles": ["${aws_iam_role.lambda_spot_monitor.id}"], + + "policy_arn": "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess", + } + } + } else {}, + data(settings):: + local regionKeys = std.objectFields(settings.regions); + { "aws_iam_policy_document": { "lambda_proxy_api_handler": { "statement": [{ @@ -84,10 +109,8 @@ "s3:GetObject" ], "resources": [ - "${var.dictionary-east-1}/*", - "${var.dictionary-east-2}/*", - "${var.dictionary-west-1}/*", - "${var.dictionary-west-2}/*" + "${var.dictionary-" + regionKeys[i] + "}/*" + for i in std.range(0, std.length(regionKeys) - 1) ] },{ "sid": "4", diff --git a/terraform/jsonnet/null_resources.libsonnet b/terraform/jsonnet/null_resources.libsonnet index bd988f0..0b9bb40 100644 --- a/terraform/jsonnet/null_resources.libsonnet +++ b/terraform/jsonnet/null_resources.libsonnet @@ -9,7 +9,11 @@ } }], - "depends_on": ["local_file.npk_config", "aws_s3_bucket.static_site"] + "depends_on": ["local_file.npk_config", "aws_s3_bucket.static_site"], + "triggers": { + "always-trigger": "${timestamp()}" + } + } } } diff --git a/terraform/jsonnet/templates.libsonnet b/terraform/jsonnet/templates.libsonnet index d6f7f79..a144a36 100644 --- a/terraform/jsonnet/templates.libsonnet +++ b/terraform/jsonnet/templates.libsonnet @@ -23,43 +23,14 @@ local az(region) = { "${element(split(\"/\", aws_api_gateway_deployment.npk.invoke_url), 2)}" } }, - "api_handler_variables": { - "template": "${file(\"${path.module}/templates/api_handler_variables.tpl\")}", - - "vars": { - "www_dns_name": std.toString(settings.dnsNames.www), - "use1": "${var.dictionary-east-1-id}", - "use2": "${var.dictionary-east-2-id}", - "usw1": "${var.dictionary-west-1-id}", - "usw2": "${var.dictionary-west-2-id}", - "campaign_max_price": "${var.campaign_max_price}", - "userdata": "${aws_s3_bucket.user_data.id}", - "instanceProfile": "${aws_iam_instance_profile.npk_node.arn}", - "iamFleetRole": "${aws_iam_role.npk_fleet_role.arn}" - } + { - [settings.regions[regionKeys[i]][azi]]: "${aws_subnet." + settings.regions[regionKeys[i]][azi] + ".id}" - for i in std.range(0, std.length(regionKeys) - 1) - for azi in std.range(0, std.length(settings.regions[regionKeys[i]]) - 1) - } - }, - "lambda_functions_settings": { - "template": "${file(\"${path.module}/templates/npk_settings.tpl\")}", - - "vars": { - "www_dns_name": std.toString(settings.dnsNames.www), - "region": "${var.region}", - "campaign_max_price": "${var.campaign_max_price}", - "critical_events_sns_topic": "${aws_sns_topic.critical_events.id}" - } - }, "userdata_template": { "template": "${file(\"${path.module}/templates/userdata.tpl\")}", "vars": { - "use1": "${var.dictionary-east-1-id}", - "use2": "${var.dictionary-east-2-id}", - "usw1": "${var.dictionary-west-1-id}", - "usw2": "${var.dictionary-west-2-id}", + "dictionaryBuckets": std.strReplace(std.manifestJsonEx({ + [regionKeys[i]]: "${var.dictionary-" + regionKeys[i] + "-id}" + for i in std.range(0, std.length(regionKeys) - 1) + }, ""), "\n", ""), "userdata": "${aws_s3_bucket.user_data.id}", "apigateway": if settings.useCustomDNS then settings.dnsNames.api[0] @@ -75,18 +46,6 @@ local az(region) = { "content": "${data.template_file.npk_config.rendered}", "filename": "${path.module}/../site-content/angular/npk_config.js", }, - "api_handler_variables": { - "content": "${data.template_file.api_handler_variables.rendered}", - "filename": "${path.module}/lambda_functions/proxy_api_handler/api_handler_variables.js", - }, - "lambda_functions_settings-spot_monitor": { - "content": "${data.template_file.lambda_functions_settings.rendered}", - "filename": "${path.module}/lambda_functions/spot_monitor/npk_settings.js", - }, - "lambda_functions_settings-status_reporter": { - "content": "${data.template_file.lambda_functions_settings.rendered}", - "filename": "${path.module}/lambda_functions/status_reporter/npk_settings.js", - }, "userdata_template": { "content": "${data.template_file.userdata_template.rendered}", "filename": "${path.module}/lambda_functions/proxy_api_handler/userdata.sh", diff --git a/terraform/jsonnet/variables.libsonnet b/terraform/jsonnet/variables.libsonnet index d30bf93..5652aaa 100644 --- a/terraform/jsonnet/variables.libsonnet +++ b/terraform/jsonnet/variables.libsonnet @@ -1,17 +1,17 @@ { - "variables": { + variables(settings):: + local regionKeys = std.objectFields(settings.regions); + { "campaign_max_price": {}, "campaign_data_ttl": {}, - "dictionary-east-1": {}, - "dictionary-east-2": {}, - "dictionary-west-1": {}, - "dictionary-west-2": {}, - "dictionary-east-1-id": { }, - "dictionary-east-2-id": { }, - "dictionary-west-1-id": { }, - "dictionary-west-2-id": { }, "cognito_user_mfa": { "default": "OFF" } + } + { + ["dictionary-" + regionKeys[i]]: {} + for i in std.range(0, std.length(regionKeys) - 1) + } + { + ["dictionary-" + regionKeys[i] + "-id"]: {} + for i in std.range(0, std.length(regionKeys) - 1) } } \ No newline at end of file diff --git a/terraform/lambda_functions/proxy_api_handler/main.js b/terraform/lambda_functions/proxy_api_handler/main.js index 058bb18..0b11e30 100644 --- a/terraform/lambda_functions/proxy_api_handler/main.js +++ b/terraform/lambda_functions/proxy_api_handler/main.js @@ -4,10 +4,14 @@ "use strict"; var fs = require('fs'); -var aws = require('aws-sdk'); +var AWSXRay = require('aws-xray-sdk'); +var aws = AWSXRay.captureAWS(require('aws-sdk')); var uuid = require('uuid/v4'); var ddbTypes = require('dynamodb-data-types').AttributeValue; -var variables = require('./api_handler_variables'); +var variables = JSON.parse(JSON.stringify(process.env)); +variables.availabilityZones = JSON.parse(variables.availabilityZones); +variables.dictionaryBuckets = JSON.parse(variables.dictionaryBuckets); + var cb = ""; var origin = ""; @@ -30,7 +34,7 @@ var allowed_regions = [ "us-west-1", "us-west-2", "us-east-1", - "us-east-2", + "us-east-2" ]; var allowed_intances = [ @@ -48,6 +52,10 @@ var allowed_intances = [ "p3.16xlarge" ]; +process.on('unhandledRejection', error => { + console.log('unhandledRejection', error.message); +}); + Object.prototype.require = function (elements) { var self = this; @@ -120,7 +128,7 @@ function getCampaign(entity, campaign) { TableName: "Campaigns" }, function (err, data) { if (err) { - return failure(respond(500, "Unable to retrieve active campaigns.", false)); + return failure(respond(500, "Unable to retrieve active campaigns. " + err, false)); } return success(data); @@ -148,8 +156,6 @@ function editCampaign(entity, campaign, values) { AttributeUpdates: values }; - // console.log(JSON.stringify(ddbParams)); - db.updateItem(ddbParams, function (err, data) { if (err) { return failure(respond(500, "Error updating table: " + err, false)); @@ -180,8 +186,6 @@ function deleteCampaign(entity, campaign) { data.Items.forEach(function(c) { var campaign = ddbTypes.unwrap(c); - console.log(campaign); - promises.push( new Promise((success, failure) => { db.deleteItem({ @@ -238,8 +242,6 @@ function getObjectAverage(what) { count++; }); - console.log(sum, count); - return sum / count; } @@ -463,7 +465,7 @@ function createCampaign(entity, campaign) { }); s3dict = new aws.S3({region: campaign.region}); - var bucket = variables.dictionary_buckets[campaign.region]; + var bucket = variables.dictionaryBuckets[campaign.region]; // Verify dictionary promises.push(new Promise((success, failure) => { @@ -806,26 +808,24 @@ function executeCampaign(entity, campaignId) { // return respond(200, "Campaign " + campaignId + " started.", true); } -function stopCampaign(entity, campaignId) { - - getCampaign(entity, campaignId).then(function(data) { - if (data.Items.length < 1) { - return respond(404, "Campaign " + campaignId + " not found", false); - } +function stopCampaign(entity, campaign) { - var campaign = ddbTypes.unwrap(data.Items[0]); + return new Promise((success, failure) => { + var campaignId = campaign.keyid.split(':')[1]; var ec2 = new aws.EC2({region: campaign.region}); + console.log("Starting EC2 call"); ec2.describeSpotFleetRequests({ SpotFleetRequestIds: [campaign.spotFleetRequestId] }, function(err, fleet) { + if (err) { - return respond(500, "Error retrieving spot fleet data: " + err, false); + return failure(respond(500, "Error retrieving spot fleet data: " + err, false)); } if (fleet.SpotFleetRequestConfigs.length < 1) { // TODO: Set the campaign to inactive if this result is reliable enough. - return respond(404, "Error retrieving spot fleet data: not found.", false); + return failure(respond(404, "Error retrieving spot fleet data: not found.", false)); } var request = fleet.SpotFleetRequestConfigs[0]; @@ -835,25 +835,25 @@ function stopCampaign(entity, campaignId) { TerminateInstances: true }, function(err, response) { if (err) { - return respond(500, "Error cancelling spot fleet: " + err, false); + return failure(respond(500, "Error cancelling spot fleet: " + err, false)); } console.log(response); if (response.SuccessfulFleetRequests.length < 1) { - return respond(500, "Error cancelling spot fleet: " + err, false); + return failure(respond(500, "Error cancelling spot fleet: " + err, false)); } if (response.SuccessfulFleetRequests[0].CurrentSpotFleetRequestState.indexOf('cancelled') < 0) { - return respond(400, "Error cancelling spot fleet. Current state: " + response.SuccessfulFleetRequests[0].CurrentSpotFleetRequestState, false); + return failure(respond(400, "Error cancelling spot fleet. Current state: " + response.SuccessfulFleetRequests[0].CurrentSpotFleetRequestState, false)); } editCampaign(entity, campaignId, { active: false }).then(function(data) { - return respond(200, "Campaign stoppped.", true); + return success(respond(200, "Campaign stoppped.", true)); }, function (err) { - return respond(500, "Unable to deactivate campaign: " + err, false); + return failure(respond(500, "Unable to deactivate campaign: " + err, false)); }); }); } else { @@ -861,15 +861,12 @@ function stopCampaign(entity, campaignId) { active: false, status: "CANCELLED" }).then(function(data) { - return respond(200, "Campaign stoppped.", true); + return success(respond(200, "Campaign stoppped.", true)); }, function (err) { - return respond(500, "Unable to deactivate campaign: " + err, false); + return failure(respond(500, "Unable to deactivate campaign: " + err, false)); }); } - }); - - }, function (err) { - return respond(500, "Error retrieving campaign: " + err, false); + }) }); } @@ -901,17 +898,17 @@ function processHttpRequest(path, method, entity, body) { case "DELETE": return getCampaign(entity, params[1]).then((data) => { - console.log(data); - if (data.Items.length < 1) { return respond(404, "Invalid campaign", false); } data = ddbTypes.unwrap(data.Items[0]); - if (data.status == "RUNNING") { - return stopCampaign(entity, params[1]); + if (data.status == "RUNNING" || data.status == "STARTING") { + console.log("Stopping campaign " + params[1]); + return stopCampaign(entity, data); } else { + console.log("Deleting campaign " + params[1]); return deleteCampaign(entity, params[1]); } }); @@ -932,7 +929,7 @@ function respond(statusCode, body, success) { // Include terraform dns names as allowed origins, as well as localhost. - var allowed_origins = variables.www_dns_names; + var allowed_origins = JSON.parse(variables.www_dns_names); allowed_origins.push("https://localhost"); var headers = {'Content-Type': 'text/plain'}; @@ -951,11 +948,15 @@ function respond(statusCode, body, success) { break; } - cb(null, { + var response = { statusCode: statusCode, headers: headers, body: JSON.stringify(body), - }); + } + + console.log(JSON.stringify(response)); + + cb(null, response); if (success == true) { return Promise.resolve(body.msg); @@ -1021,7 +1022,7 @@ exports.main = function(event, context, callback) { return processHttpRequest(event.pathParameters.proxy, event.httpMethod, event.requestContext.identity.cognitoIdentityId, body); } catch (e) { - console.log(e); + console.log("Fail", e); respond(500, "Unknown Error", false); } }; \ No newline at end of file diff --git a/terraform/lambda_functions/proxy_api_handler/package-lock.json b/terraform/lambda_functions/proxy_api_handler/package-lock.json index 8218833..4bf89a7 100644 --- a/terraform/lambda_functions/proxy_api_handler/package-lock.json +++ b/terraform/lambda_functions/proxy_api_handler/package-lock.json @@ -4,27 +4,180 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/body-parser": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.17.1.tgz", + "integrity": "sha512-RoX2EZjMiFMjZh9lmYrwgoP9RTpAjSHiJxdp4oidAQVO02T7HER3xj9UKue5534ULWeqVEkujhWcyvUce+d68w==", + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/connect": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.33.tgz", + "integrity": "sha512-2+FrkXY4zllzTNfJth7jOqEHC+enpLeGslEhpnTAkg21GkRrWV4SsAtqchtT4YS9/nODBU2/ZfsBY2X4J/dX7A==", + "requires": { + "@types/node": "*" + } + }, + "@types/continuation-local-storage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/continuation-local-storage/-/continuation-local-storage-3.2.2.tgz", + "integrity": "sha512-aItm+aYPJ4rT1cHmAxO+OdWjSviQ9iB5UKb5f0Uvgln0N4hS2mcDodHtPiqicYBXViUYhqyBjhA5uyOcT+S34Q==", + "requires": { + "@types/node": "*" + } + }, + "@types/express": { + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.2.tgz", + "integrity": "sha512-5mHFNyavtLoJmnusB8OKJ5bshSzw+qkMIBAobLrIM48HJvunFva9mOa6aBwh64lBFyNwBbs0xiEFuj4eU/NjCA==", + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.1.tgz", + "integrity": "sha512-9e7jj549ZI+RxY21Cl0t8uBnWyb22HzILupyHZjYEVK//5TT/1bZodU+yUbLnPdoYViBBnNWbxp4zYjGV0zUGw==", + "requires": { + "@types/node": "*", + "@types/range-parser": "*" + } + }, + "@types/mime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", + "integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==" + }, + "@types/mysql": { + "version": "2.15.8", + "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.8.tgz", + "integrity": "sha512-l0TUdg6KDEaLO75/yjdjksobJDRWv8iZlpRfv/WW1lQZCQDKdTDnKCkeH10oapzP/JTuKiTy6Cvq/sm/0GgcUw==", + "requires": { + "@types/node": "*" + } + }, + "@types/node": { + "version": "13.1.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.4.tgz", + "integrity": "sha512-Lue/mlp2egZJoHXZr4LndxDAd7i/7SQYhV0EjWfb/a4/OZ6tuVwMCVPiwkU5nsEipxEf7hmkSU7Em5VQ8P5NGA==" + }, + "@types/pg": { + "version": "7.14.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.14.1.tgz", + "integrity": "sha512-gQgg4bLuykokypx4O1fwEzl5e6UjjyaBtN3znn5zhm0YB9BnKyHDw+e4cQY9rAPzpdM2qpJbn9TNzUazbmTsdw==", + "requires": { + "@types/node": "*", + "@types/pg-types": "*" + } + }, + "@types/pg-types": { + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/@types/pg-types/-/pg-types-1.11.5.tgz", + "integrity": "sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ==" + }, + "@types/range-parser": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.3.tgz", + "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" + }, + "@types/serve-static": { + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.3.tgz", + "integrity": "sha512-oprSwp094zOglVrXdlo/4bAHtKTAxX6VT8FOZlBKrmyLbNvE1zxZyJ6yikMVtHIvwP45+ZQGJn+FdXGKTozq0g==", + "requires": { + "@types/express-serve-static-core": "*", + "@types/mime": "*" + } + }, + "async": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=" + }, + "async-listener": { + "version": "0.6.10", + "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==", + "requires": { + "semver": "^5.3.0", + "shimmer": "^1.1.0" + } + }, + "atomic-batcher": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/atomic-batcher/-/atomic-batcher-1.0.2.tgz", + "integrity": "sha1-0WkB0QzOxZUWwZe5zNiTBom4E7Q=" + }, "aws-sdk": { - "version": "2.384.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.384.0.tgz", - "integrity": "sha1-yP3OfHLwaPbYsZi1PPM7bzXz6BE=", + "version": "2.599.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.599.0.tgz", + "integrity": "sha512-7yTXnV5SC9W6m+STbziPd1ZNVh9fTtEZ7Mm0rMqEDm/B2fJBa5xd45TwWG8JvS40X5+9jUBykiWdCuVBBx82rg==", "requires": { "buffer": "4.9.1", "events": "1.1.1", - "ieee754": "1.1.8", + "ieee754": "1.1.13", "jmespath": "0.15.0", "querystring": "0.2.0", "sax": "1.2.1", "url": "0.10.3", - "uuid": "3.1.0", + "uuid": "3.3.2", "xml2js": "0.4.19" - }, - "dependencies": { - "uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha1-PdPT55Crwk17DToDT/q6vijrvAQ=" - } + } + }, + "aws-xray-sdk": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk/-/aws-xray-sdk-2.5.0.tgz", + "integrity": "sha512-KBTF9QUq0zu24t+gFh3VATYDNx295VzZUUjpZ1OmIEbcs3Zqmox5L6VAqjCgGfJeLaCUAWPLkduxau3LuDpqcw==", + "requires": { + "aws-xray-sdk-core": "^2.5.0", + "aws-xray-sdk-express": "^2.5.0", + "aws-xray-sdk-mysql": "^2.5.0", + "aws-xray-sdk-postgres": "^2.5.0", + "pkginfo": "^0.4.0" + } + }, + "aws-xray-sdk-core": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-core/-/aws-xray-sdk-core-2.5.0.tgz", + "integrity": "sha512-qe60bv0kn5KY6gAIF88TPCOIdu/A3dTmcKISj+kE4OH02eF6kMm1ctL7OgoBAasnsDNSn0VMLhIaA1izgoWuxA==", + "requires": { + "@types/continuation-local-storage": "*", + "atomic-batcher": "^1.0.2", + "aws-sdk": "^2.304.0", + "continuation-local-storage": "^3.2.0", + "date-fns": "^1.29.0", + "pkginfo": "^0.4.0", + "semver": "^5.3.0", + "winston": "^2.4.4" + } + }, + "aws-xray-sdk-express": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-express/-/aws-xray-sdk-express-2.5.0.tgz", + "integrity": "sha512-VfUXlxFlI+gr6ImMF+3000fqPRTBS9MbAywdtC8dt/U57WQbLFDjPW+V6oJowTwrkgKwpOKSXncQDfBs8QHvIw==", + "requires": { + "@types/express": "*" + } + }, + "aws-xray-sdk-mysql": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-mysql/-/aws-xray-sdk-mysql-2.5.0.tgz", + "integrity": "sha512-OI3cDkVcKTiWytwoOE1Oj0D2UbchOVHkiQNXWktDk9QYzqbOz+GNEwD5QneqyxFu0z0bQieClBM+0okUprFBXQ==", + "requires": { + "@types/mysql": "*" + } + }, + "aws-xray-sdk-postgres": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-postgres/-/aws-xray-sdk-postgres-2.5.0.tgz", + "integrity": "sha512-8LcXTjr272gKMSg9a+QSGQPjAabQb87SA0c4ZlVcPqzoG7nAd3pULR6/HDtujeukbRAb6rEoxzKvPX0s18NyFw==", + "requires": { + "@types/pg": "*" } }, "aws4": { @@ -33,9 +186,9 @@ "integrity": "sha1-8OAD2cqef1nHpQiUXXsu+aBKVC8=" }, "base64-js": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", - "integrity": "sha1-yrHmEY8FEJXli1KBrqjBzSK/wOM=" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "buffer": { "version": "4.9.1", @@ -47,31 +200,78 @@ "isarray": "^1.0.0" } }, + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + }, + "continuation-local-storage": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", + "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==", + "requires": { + "async-listener": "^0.6.0", + "emitter-listener": "^1.1.1" + } + }, + "cycle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" + }, + "date-fns": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.30.1.tgz", + "integrity": "sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==" + }, "dynamodb-data-types": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dynamodb-data-types/-/dynamodb-data-types-3.0.1.tgz", "integrity": "sha1-VWhTFXTOLhkLwnzDpYlfvzeu/KU=" }, + "emitter-listener": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", + "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==", + "requires": { + "shimmer": "^1.2.0" + } + }, "events": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" }, + "eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" + }, "ieee754": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", - "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, + "pkginfo": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.1.tgz", + "integrity": "sha1-tUGO8EOd5UJfxJlQQtztFPsqhP8=" + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -87,6 +287,21 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", @@ -101,10 +316,23 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" }, + "winston": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz", + "integrity": "sha512-NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q==", + "requires": { + "async": "~1.0.0", + "colors": "1.0.x", + "cycle": "1.0.x", + "eyes": "0.1.x", + "isstream": "0.1.x", + "stack-trace": "0.0.x" + } + }, "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { "sax": ">=0.6.0", "xmlbuilder": "~9.0.1" diff --git a/terraform/lambda_functions/proxy_api_handler/package.json b/terraform/lambda_functions/proxy_api_handler/package.json index d38f7ff..12d01d4 100644 --- a/terraform/lambda_functions/proxy_api_handler/package.json +++ b/terraform/lambda_functions/proxy_api_handler/package.json @@ -4,7 +4,8 @@ "description": "NPK API Gateway Proxy Handler", "main": "main.js", "dependencies": { - "aws-sdk": "^2.361.0", + "aws-sdk": "^2.599.0", + "aws-xray-sdk": "^2.5.0", "aws4": "^1.8.0", "dynamodb-data-types": "^3.0.1", "uuid": "^3.3.2" diff --git a/terraform/lambda_functions/spot_monitor/main.js b/terraform/lambda_functions/spot_monitor/main.js index 88d4488..dad5319 100644 --- a/terraform/lambda_functions/spot_monitor/main.js +++ b/terraform/lambda_functions/spot_monitor/main.js @@ -4,8 +4,11 @@ "use strict"; var cb = ""; -var aws = require('aws-sdk'); -var settings = require('./npk_settings'); +var AWSXRay = require('aws-xray-sdk'); +var aws = AWSXRay.captureAWS(require('aws-sdk')); +var settings = JSON.parse(JSON.stringify(process.env)); +settings.availabilityZones = JSON.parse(settings.availabilityZones); + var ddbTypes = require('dynamodb-data-types').AttributeValue; aws.config.apiVersions = { @@ -116,7 +119,7 @@ var listSpotFleetRequests = function(region) { return new Promise((success, failure) => { ec2.describeSpotFleetRequests({}, function(err, data) { if (err) { - return failure(cb("Failed retrieving spot fleet information.")); + return failure(cb("Failed retrieving spot fleet information." + err)); } data.SpotFleetRequestConfigs.forEach(function(e) { @@ -302,6 +305,7 @@ var evaluateAllSpotInstances = function() { var promises = []; Object.keys(availabilityZones).forEach(function(region) { + console.log(region); promises.push(listSpotFleetRequests(region)); promises.push(listSpotInstanceRequests(region)); }); diff --git a/terraform/lambda_functions/spot_monitor/package-lock.json b/terraform/lambda_functions/spot_monitor/package-lock.json index 05b3e62..48ca577 100644 --- a/terraform/lambda_functions/spot_monitor/package-lock.json +++ b/terraform/lambda_functions/spot_monitor/package-lock.json @@ -4,26 +4,186 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/body-parser": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.17.1.tgz", + "integrity": "sha512-RoX2EZjMiFMjZh9lmYrwgoP9RTpAjSHiJxdp4oidAQVO02T7HER3xj9UKue5534ULWeqVEkujhWcyvUce+d68w==", + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/connect": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.33.tgz", + "integrity": "sha512-2+FrkXY4zllzTNfJth7jOqEHC+enpLeGslEhpnTAkg21GkRrWV4SsAtqchtT4YS9/nODBU2/ZfsBY2X4J/dX7A==", + "requires": { + "@types/node": "*" + } + }, + "@types/continuation-local-storage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/continuation-local-storage/-/continuation-local-storage-3.2.2.tgz", + "integrity": "sha512-aItm+aYPJ4rT1cHmAxO+OdWjSviQ9iB5UKb5f0Uvgln0N4hS2mcDodHtPiqicYBXViUYhqyBjhA5uyOcT+S34Q==", + "requires": { + "@types/node": "*" + } + }, + "@types/express": { + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.2.tgz", + "integrity": "sha512-5mHFNyavtLoJmnusB8OKJ5bshSzw+qkMIBAobLrIM48HJvunFva9mOa6aBwh64lBFyNwBbs0xiEFuj4eU/NjCA==", + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.1.tgz", + "integrity": "sha512-9e7jj549ZI+RxY21Cl0t8uBnWyb22HzILupyHZjYEVK//5TT/1bZodU+yUbLnPdoYViBBnNWbxp4zYjGV0zUGw==", + "requires": { + "@types/node": "*", + "@types/range-parser": "*" + } + }, + "@types/mime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", + "integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==" + }, + "@types/mysql": { + "version": "2.15.8", + "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.8.tgz", + "integrity": "sha512-l0TUdg6KDEaLO75/yjdjksobJDRWv8iZlpRfv/WW1lQZCQDKdTDnKCkeH10oapzP/JTuKiTy6Cvq/sm/0GgcUw==", + "requires": { + "@types/node": "*" + } + }, + "@types/node": { + "version": "13.1.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.4.tgz", + "integrity": "sha512-Lue/mlp2egZJoHXZr4LndxDAd7i/7SQYhV0EjWfb/a4/OZ6tuVwMCVPiwkU5nsEipxEf7hmkSU7Em5VQ8P5NGA==" + }, + "@types/pg": { + "version": "7.14.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.14.1.tgz", + "integrity": "sha512-gQgg4bLuykokypx4O1fwEzl5e6UjjyaBtN3znn5zhm0YB9BnKyHDw+e4cQY9rAPzpdM2qpJbn9TNzUazbmTsdw==", + "requires": { + "@types/node": "*", + "@types/pg-types": "*" + } + }, + "@types/pg-types": { + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/@types/pg-types/-/pg-types-1.11.5.tgz", + "integrity": "sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ==" + }, + "@types/range-parser": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.3.tgz", + "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" + }, + "@types/serve-static": { + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.3.tgz", + "integrity": "sha512-oprSwp094zOglVrXdlo/4bAHtKTAxX6VT8FOZlBKrmyLbNvE1zxZyJ6yikMVtHIvwP45+ZQGJn+FdXGKTozq0g==", + "requires": { + "@types/express-serve-static-core": "*", + "@types/mime": "*" + } + }, + "async": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=" + }, + "async-listener": { + "version": "0.6.10", + "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==", + "requires": { + "semver": "^5.3.0", + "shimmer": "^1.1.0" + } + }, + "atomic-batcher": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/atomic-batcher/-/atomic-batcher-1.0.2.tgz", + "integrity": "sha1-0WkB0QzOxZUWwZe5zNiTBom4E7Q=" + }, "aws-sdk": { - "version": "2.384.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.384.0.tgz", - "integrity": "sha1-yP3OfHLwaPbYsZi1PPM7bzXz6BE=", + "version": "2.599.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.599.0.tgz", + "integrity": "sha512-7yTXnV5SC9W6m+STbziPd1ZNVh9fTtEZ7Mm0rMqEDm/B2fJBa5xd45TwWG8JvS40X5+9jUBykiWdCuVBBx82rg==", "requires": { "buffer": "4.9.1", "events": "1.1.1", - "ieee754": "1.1.8", + "ieee754": "1.1.13", "jmespath": "0.15.0", "querystring": "0.2.0", "sax": "1.2.1", "url": "0.10.3", - "uuid": "3.1.0", + "uuid": "3.3.2", "xml2js": "0.4.19" } }, + "aws-xray-sdk": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk/-/aws-xray-sdk-2.5.0.tgz", + "integrity": "sha512-KBTF9QUq0zu24t+gFh3VATYDNx295VzZUUjpZ1OmIEbcs3Zqmox5L6VAqjCgGfJeLaCUAWPLkduxau3LuDpqcw==", + "requires": { + "aws-xray-sdk-core": "^2.5.0", + "aws-xray-sdk-express": "^2.5.0", + "aws-xray-sdk-mysql": "^2.5.0", + "aws-xray-sdk-postgres": "^2.5.0", + "pkginfo": "^0.4.0" + } + }, + "aws-xray-sdk-core": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-core/-/aws-xray-sdk-core-2.5.0.tgz", + "integrity": "sha512-qe60bv0kn5KY6gAIF88TPCOIdu/A3dTmcKISj+kE4OH02eF6kMm1ctL7OgoBAasnsDNSn0VMLhIaA1izgoWuxA==", + "requires": { + "@types/continuation-local-storage": "*", + "atomic-batcher": "^1.0.2", + "aws-sdk": "^2.304.0", + "continuation-local-storage": "^3.2.0", + "date-fns": "^1.29.0", + "pkginfo": "^0.4.0", + "semver": "^5.3.0", + "winston": "^2.4.4" + } + }, + "aws-xray-sdk-express": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-express/-/aws-xray-sdk-express-2.5.0.tgz", + "integrity": "sha512-VfUXlxFlI+gr6ImMF+3000fqPRTBS9MbAywdtC8dt/U57WQbLFDjPW+V6oJowTwrkgKwpOKSXncQDfBs8QHvIw==", + "requires": { + "@types/express": "*" + } + }, + "aws-xray-sdk-mysql": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-mysql/-/aws-xray-sdk-mysql-2.5.0.tgz", + "integrity": "sha512-OI3cDkVcKTiWytwoOE1Oj0D2UbchOVHkiQNXWktDk9QYzqbOz+GNEwD5QneqyxFu0z0bQieClBM+0okUprFBXQ==", + "requires": { + "@types/mysql": "*" + } + }, + "aws-xray-sdk-postgres": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-postgres/-/aws-xray-sdk-postgres-2.5.0.tgz", + "integrity": "sha512-8LcXTjr272gKMSg9a+QSGQPjAabQb87SA0c4ZlVcPqzoG7nAd3pULR6/HDtujeukbRAb6rEoxzKvPX0s18NyFw==", + "requires": { + "@types/pg": "*" + } + }, "base64-js": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", - "integrity": "sha1-yrHmEY8FEJXli1KBrqjBzSK/wOM=" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "buffer": { "version": "4.9.1", @@ -35,31 +195,78 @@ "isarray": "^1.0.0" } }, + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + }, + "continuation-local-storage": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", + "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==", + "requires": { + "async-listener": "^0.6.0", + "emitter-listener": "^1.1.1" + } + }, + "cycle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" + }, + "date-fns": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.30.1.tgz", + "integrity": "sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==" + }, "dynamodb-data-types": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dynamodb-data-types/-/dynamodb-data-types-3.0.1.tgz", "integrity": "sha1-VWhTFXTOLhkLwnzDpYlfvzeu/KU=" }, + "emitter-listener": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", + "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==", + "requires": { + "shimmer": "^1.2.0" + } + }, "events": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" }, + "eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" + }, "ieee754": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", - "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, + "pkginfo": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.1.tgz", + "integrity": "sha1-tUGO8EOd5UJfxJlQQtztFPsqhP8=" + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -75,6 +282,21 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", @@ -85,14 +307,27 @@ } }, "uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha1-PdPT55Crwk17DToDT/q6vijrvAQ=" + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "winston": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz", + "integrity": "sha512-NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q==", + "requires": { + "async": "~1.0.0", + "colors": "1.0.x", + "cycle": "1.0.x", + "eyes": "0.1.x", + "isstream": "0.1.x", + "stack-trace": "0.0.x" + } }, "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { "sax": ">=0.6.0", "xmlbuilder": "~9.0.1" diff --git a/terraform/lambda_functions/spot_monitor/package.json b/terraform/lambda_functions/spot_monitor/package.json index 7469164..39b0ad5 100644 --- a/terraform/lambda_functions/spot_monitor/package.json +++ b/terraform/lambda_functions/spot_monitor/package.json @@ -9,7 +9,8 @@ "author": "Brad Woodward (brad@bradwoodward.io)", "license": "MIT", "dependencies": { - "aws-sdk": "^2.363.0", + "aws-sdk": "^2.599.0", + "aws-xray-sdk": "^2.5.0", "dynamodb-data-types": "^3.0.1" } } diff --git a/terraform/lambda_functions/status_reporter/main.js b/terraform/lambda_functions/status_reporter/main.js index 0adf3ef..6faf4d7 100644 --- a/terraform/lambda_functions/status_reporter/main.js +++ b/terraform/lambda_functions/status_reporter/main.js @@ -7,7 +7,9 @@ var fs = require('fs'); var aws = require('aws-sdk'); var uuid = require('uuid/v4'); var ddbTypes = require('dynamodb-data-types').AttributeValue; -var settings = require('./npk_settings'); +var settings = JSON.parse(JSON.stringify(process.env)); +settings.availabilityZones = JSON.parse(settings.availabilityZones); + var cb = ""; var lambdaEvent = {}; diff --git a/terraform/lambda_functions/status_reporter/package-lock.json b/terraform/lambda_functions/status_reporter/package-lock.json index a0493d2..bbae12d 100644 --- a/terraform/lambda_functions/status_reporter/package-lock.json +++ b/terraform/lambda_functions/status_reporter/package-lock.json @@ -5,13 +5,13 @@ "requires": true, "dependencies": { "aws-sdk": { - "version": "2.418.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.418.0.tgz", - "integrity": "sha512-15aCtqqCsiyMW+CDwo6Fq3V5jDzpgb5//aPMosL+5FQnQu65t2GiLidcIPx4fWvsYpRiE/i4enz3a0Kqtt2acQ==", + "version": "2.599.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.599.0.tgz", + "integrity": "sha512-7yTXnV5SC9W6m+STbziPd1ZNVh9fTtEZ7Mm0rMqEDm/B2fJBa5xd45TwWG8JvS40X5+9jUBykiWdCuVBBx82rg==", "requires": { "buffer": "4.9.1", "events": "1.1.1", - "ieee754": "1.1.8", + "ieee754": "1.1.13", "jmespath": "0.15.0", "querystring": "0.2.0", "sax": "1.2.1", @@ -21,9 +21,9 @@ } }, "base64-js": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", - "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "buffer": { "version": "4.9.1", @@ -46,9 +46,9 @@ "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" }, "ieee754": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", - "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "isarray": { "version": "1.0.0", diff --git a/terraform/lambda_functions/status_reporter/package.json b/terraform/lambda_functions/status_reporter/package.json index 5f0e27d..82c3038 100644 --- a/terraform/lambda_functions/status_reporter/package.json +++ b/terraform/lambda_functions/status_reporter/package.json @@ -9,7 +9,7 @@ "author": "Brad Woodward (brad@bradwoodward.io)", "license": "MIT", "dependencies": { - "aws-sdk": "^2.418.0", + "aws-sdk": "^2.599.0", "dynamodb-data-types": "^3.0.1", "uuid": "^3.3.2" } diff --git a/terraform/npk-settings.json.sample b/terraform/npk-settings.json.sample index c4319b2..a32f374 100644 --- a/terraform/npk-settings.json.sample +++ b/terraform/npk-settings.json.sample @@ -14,5 +14,6 @@ }, "awsProfile": "npk", "criticalEventsSMS": "+13035551234", - "adminEmail": "demo@npkproject.io" + "adminEmail": "demo@npkproject.io", + "debug_lambda": false } diff --git a/terraform/templates/api_handler_variables-fresh.tpl b/terraform/templates/api_handler_variables-fresh.tpl deleted file mode 100644 index fd09b83..0000000 --- a/terraform/templates/api_handler_variables-fresh.tpl +++ /dev/null @@ -1,13 +0,0 @@ -module.exports = { - www_dns_names: ['${www_dns_name}'], - campaign_max_price: ${campaign_max_price}, - dictionary_buckets: { - "us-east-1": "${use1}", - "us-east-2": "${use2}", - "us-west-1": "${usw1}", - "us-west-2": "${usw2}", - }, - userdata_bucket: "${userdata}", - instanceProfile: "${instanceProfile}", - iamFleetRole: "${iamFleetRole}", - availabilityZones: \ No newline at end of file diff --git a/terraform/templates/api_handler_variables.tpl b/terraform/templates/api_handler_variables.tpl deleted file mode 100644 index fba7a27..0000000 --- a/terraform/templates/api_handler_variables.tpl +++ /dev/null @@ -1,38 +0,0 @@ -module.exports = { - www_dns_names: ['${www_dns_name}'], - campaign_max_price: ${campaign_max_price}, - dictionary_buckets: { - "us-east-1": "${use1}", - "us-east-2": "${use2}", - "us-west-1": "${usw1}", - "us-west-2": "${usw2}", - }, - userdata_bucket: "${userdata}", - instanceProfile: "${instanceProfile}", - iamFleetRole: "${iamFleetRole}", - availabilityZones: { - "us-east-1": { - "us-east-1a": "${us-east-1a}", - "us-east-1b": "${us-east-1b}", - "us-east-1c": "${us-east-1c}", - "us-east-1d": "${us-east-1d}", - "us-east-1e": "${us-east-1e}", - "us-east-1f": "${us-east-1f}" - }, - "us-east-2": { - "us-east-2a": "${us-east-2a}", - "us-east-2b": "${us-east-2b}", - "us-east-2c": "${us-east-2c}" - }, - "us-west-1": { - "us-west-1b": "${us-west-1b}", - "us-west-1c": "${us-west-1c}" - }, - "us-west-2": { - "us-west-2a": "${us-west-2a}", - "us-west-2b": "${us-west-2b}", - "us-west-2c": "${us-west-2c}", - "us-west-2d": "${us-west-2d}" - } -} -} \ No newline at end of file diff --git a/terraform/templates/npk_settings-fresh.tpl b/terraform/templates/npk_settings-fresh.tpl deleted file mode 100644 index f241f28..0000000 --- a/terraform/templates/npk_settings-fresh.tpl +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - "www_dns_names": ['${www_dns_name}'], - "region": "${region}", - "campaign_max_price": ${campaign_max_price}, - "critical_events_sns_topic": "${critical_events_sns_topic}", - "availabilityZones": \ No newline at end of file diff --git a/terraform/templates/npk_settings.tpl b/terraform/templates/npk_settings.tpl deleted file mode 100644 index 77e1fb2..0000000 --- a/terraform/templates/npk_settings.tpl +++ /dev/null @@ -1,31 +0,0 @@ -module.exports = { - "www_dns_names": ['${www_dns_name}'], - "region": "${region}", - "campaign_max_price": ${campaign_max_price}, - "critical_events_sns_topic": "${critical_events_sns_topic}", - "availabilityZones": { - "us-east-1": [ - "us-east-1a", - "us-east-1b", - "us-east-1c", - "us-east-1d", - "us-east-1e", - "us-east-1f" - ], - "us-east-2": [ - "us-east-2a", - "us-east-2b", - "us-east-2c" - ], - "us-west-1": [ - "us-west-1b", - "us-west-1c" - ], - "us-west-2": [ - "us-west-2a", - "us-west-2b", - "us-west-2c", - "us-west-2d" - ] -} -} \ No newline at end of file diff --git a/terraform/templates/userdata.tpl b/terraform/templates/userdata.tpl index 9d2b212..4e4901b 100644 --- a/terraform/templates/userdata.tpl +++ b/terraform/templates/userdata.tpl @@ -1,26 +1,24 @@ #! /bin/bash cd /root/ -APIGATEWAY=${apigateway} +export APIGATEWAY=${apigateway} echo $APIGATEWAY > /root/apigateway -declare -A BUCKETS -BUCKETS[us-east-1]=${use1} -BUCKETS[us-east-2]=${use2} -BUCKETS[us-west-1]=${usw1} -BUCKETS[us-west-2]=${usw2} +export USERDATA=${userdata} -USERDATA=${userdata} - -INSTANCE_ID=`wget -qO- http://169.254.169.254/latest/meta-data/instance-id` -REGION=`wget -qO- http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/.$//'` +export INSTANCE_ID=`wget -qO- http://169.254.169.254/latest/meta-data/instance-id` +export REGION=`wget -qO- http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/.$//'` aws ec2 describe-tags --region $REGION --filter "Name=resource-id,Values=$INSTANCE_ID" --output=text | sed -r 's/TAGS\t(.*)\t.*\t.*\t(.*)/\1="\2"/' | sed -r 's/aws:ec2spot:fleet-request-id/SpotFleet/' > ec2-tags . ec2-tags echo $ManifestPath > /root/manifestpath -BUCKET=$${BUCKETS[$REGION]} +yum install -y jq + +export BUCKET=`echo '${dictionaryBuckets}' | jq -r --arg REGION $REGION '.[$REGION]'` + +echo "Using dictionary bucket $BUCKET"; mkdir /potfiles @@ -37,7 +35,7 @@ aws s3 cp s3://$BUCKET/components/maskprocessor.7z . aws s3 cp s3://$BUCKET/components/compute-node.zip . aws s3 cp s3://$USERDATA/$ManifestPath/manifest.json . rpm -Uvh epel.rpm -yum install -y p7zip p7zip-plugins jq +yum install -y p7zip p7zip-plugins # Install nvm curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | /bin/bash diff --git a/terraform/terraform.jsonnet b/terraform/terraform.jsonnet index e170148..eafd2d5 100644 --- a/terraform/terraform.jsonnet +++ b/terraform/terraform.jsonnet @@ -108,7 +108,8 @@ local regionKeys = std.objectFields(settings.regions); }, 'cloudwatch.tf.json': cloudwatch, 'cognito_iam_roles.tf.json': { - "resource": cognito_iam_roles + "resource": cognito_iam_roles.resource, + "data": cognito_iam_roles.data(settings) }, 'cognito.tf.json': { "resource": cognito.resource(settings), @@ -157,11 +158,24 @@ local regionKeys = std.objectFields(settings.regions); "key_name": "npk-key", "public_key": "${tls_private_key.ssh.public_key_openssh}" } for region in regionKeys + }, + "local_file": { + "ssh_key": { + "sensitive_content": "${tls_private_key.ssh.private_key_pem}", + "filename": "${path.module}/npk.pem", + "file_permission": "0600" + } } } }, - 'lambda_functions.tf.json': lambda_functions, - 'lambda_iam_roles.tf.json': lambda_iam_roles, + 'lambda_functions.tf.json': { + "resource": lambda_functions.resources(settings), + "data": lambda_functions.data + }, + 'lambda_iam_roles.tf.json': { + "resource": lambda_iam_roles.resource(settings), + "data": lambda_iam_roles.data(settings) + }, 'null_resources.tf.json': null_resources.resource(settings), 'provider-aws.tf.json': { "provider": [ @@ -321,7 +335,7 @@ local regionKeys = std.objectFields(settings.regions); for i in std.range(0, std.length(regionKeys) - 1) }, 'variables.tf.json': { - "variable": variables.variables + { + "variable": variables.variables(settings) + { "access_key": { "default": settings.access_key }, "secret_key": { "default": settings.secret_key }, "region": { "default": settings.defaultRegion },