From 1bd4d089186fd3cb4b709b65f8c58a1ebbf5613a Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Wed, 29 Jan 2025 21:53:46 +0000 Subject: [PATCH 01/14] test server-test --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ff130ec1cc..04a1186ab6b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,16 +28,16 @@ variables: GOLANGCI_LINT_VERBOSE: "-v" # Specify the environment: loadtest, demo, exp - DP3_ENV: &dp3_env placeholder_env + DP3_ENV: &dp3_env demo # Specify the branch to deploy TODO: this might be not needed. So far useless - DP3_BRANCH: &dp3_branch placeholder_branch_name + DP3_BRANCH: &dp3_branch B-22106-ServerTest # Ignore branches for integration tests - INTEGRATION_IGNORE_BRANCH: &integration_ignore_branch placeholder_branch_name - INTEGRATION_MTLS_IGNORE_BRANCH: &integration_mtls_ignore_branch placeholder_branch_name - CLIENT_IGNORE_BRANCH: &client_ignore_branch placeholder_branch_name - SERVER_IGNORE_BRANCH: &server_ignore_branch placeholder_branch_name + INTEGRATION_IGNORE_BRANCH: &integration_ignore_branch B-22106-ServerTest + INTEGRATION_MTLS_IGNORE_BRANCH: &integration_mtls_ignore_branch B-22106-ServerTest + CLIENT_IGNORE_BRANCH: &client_ignore_branch B-22106-ServerTest + SERVER_IGNORE_BRANCH: &server_ignore_branch B-22106-ServerTest RUNNER_TAG: &runner_tag milmove DOCKER_RUNNER_TAG: &docker_runner_tag eks_cluster_runner From ec75eb535b434460fc1aef30586c1fdbe797eb3c Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Wed, 29 Jan 2025 21:54:51 +0000 Subject: [PATCH 02/14] self hosted runner --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04a1186ab6b..52ac962bd49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ variables: CLIENT_IGNORE_BRANCH: &client_ignore_branch B-22106-ServerTest SERVER_IGNORE_BRANCH: &server_ignore_branch B-22106-ServerTest - RUNNER_TAG: &runner_tag milmove + RUNNER_TAG: &runner_tag eks_cluster_runner DOCKER_RUNNER_TAG: &docker_runner_tag eks_cluster_runner postgres: &postgres postgres:16.4 From acc54e625851885486edf33646251eb6839fe5f5 Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Wed, 29 Jan 2025 23:02:24 -0600 Subject: [PATCH 03/14] test setting host file doubt it works --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52ac962bd49..49255a3d267 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -241,13 +241,12 @@ stages: export OKTA_OFFICE_GROUP_ID=notrealgroupId export OKTA_CUSTOMER_GROUP_ID=notrealcustomergroupId -# -# .setup_host_intergration_tests: &setup_host_intergration_tests -# - echo "Setting up /etc/hosts for local domain simulation" -# - echo "127.0.0.1 milmovelocal" | sudo tee -a /etc/hosts -# - echo "127.0.0.1 officelocal" | sudo tee -a /etc/hosts -# - echo "127.0.0.1 adminlocal" | sudo tee -a /etc/hosts -# - echo "127.0.0.1 primelocal" | sudo tee -a /etc/hosts +.setup_host_intergration_tests: &setup_host_intergration_tests + - echo "Setting up /etc/hosts for local domain simulation" + - echo "127.0.0.1 milmovelocal" | sudo tee -a /etc/hosts + - echo "127.0.0.1 officelocal" | sudo tee -a /etc/hosts + - echo "127.0.0.1 adminlocal" | sudo tee -a /etc/hosts + - echo "127.0.0.1 primelocal" | sudo tee -a /etc/hosts .setup_env_intergration_mtls: &setup_env_intergration_mtls - | @@ -935,6 +934,7 @@ integration_test_mtls: - compile_app_server before_script: - *setup_milmove_env + - *setup_host_intergration_tests - *setup_env_intergration_mtls script: - echo "TODO Add steps" From dec1dfc78a3d425dfbccd65f84d97af55bbb7030 Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Wed, 29 Jan 2025 23:21:45 -0600 Subject: [PATCH 04/14] host failed lack of sudo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49255a3d267..e25b00840fb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -934,7 +934,7 @@ integration_test_mtls: - compile_app_server before_script: - *setup_milmove_env - - *setup_host_intergration_tests + # - *setup_host_intergration_tests - *setup_env_intergration_mtls script: - echo "TODO Add steps" From 3c3e7184f9071e0603d1133173cc3b7fcceed0b1 Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Thu, 30 Jan 2025 17:40:00 +0000 Subject: [PATCH 05/14] add some env variables --- .gitlab-ci.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e25b00840fb..b7f730a7d7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -265,10 +265,29 @@ stages: export HERE_MAPS_ROUTING_ENDPOINT=https://route.api.here.com/routing/7.2/calculateroute.json export LOGIN_GOV_CALLBACK_PORT=4000 export LOGIN_GOV_CALLBACK_PROTOCOL=http + + export SERVE_API_SUPPORT=true + export SERVE_PRIME_SIMULATOR=true + export DEVLOCAL_CA=$PWD/config/tls/devlocal-ca.pem + export DOD_CA_PACKAGE=$PWD/config/tls/milmove-cert-bundle.p7b + export HTTP_ADMIN_SERVER_NAME=adminlocal + export HTTP_MY_SERVER_NAME=milmovelocal + export HTTP_OFFICE_SERVER_NAME=officelocal + export HTTP_ORDERS_SERVER_NAME=orderslocal + export HTTP_PRIME_SERVER_NAME=primelocal + export ENVIRONMENT=test + export MUTUAL_TLS_ENABLED=true + export MUTUAL_TLS_PORT=9443 + export SERVE_API_PRIME=true + # disable for speed, tests can fail otherwise + export DB_DEBUG=false + make db_dev_create bin/milmove migrate + # mtls tests do not need client mkdir -p build touch build/index.html + # mtls tests DO NOT NEED SEED DATA bin/milmove serve 2>&1 | tee server.log & .e2e_tests_playwright: &e2e_tests_playwright @@ -934,7 +953,6 @@ integration_test_mtls: - compile_app_server before_script: - *setup_milmove_env - # - *setup_host_intergration_tests - *setup_env_intergration_mtls script: - echo "TODO Add steps" From 2f36f772b4a1e575af147a9314197512e9dbeeed Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Thu, 30 Jan 2025 18:56:26 +0000 Subject: [PATCH 06/14] add some env variables --- .gitlab-ci.yml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7f730a7d7e..87f7bbe11f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -268,27 +268,30 @@ stages: export SERVE_API_SUPPORT=true export SERVE_PRIME_SIMULATOR=true - export DEVLOCAL_CA=$PWD/config/tls/devlocal-ca.pem - export DOD_CA_PACKAGE=$PWD/config/tls/milmove-cert-bundle.p7b - export HTTP_ADMIN_SERVER_NAME=adminlocal - export HTTP_MY_SERVER_NAME=milmovelocal - export HTTP_OFFICE_SERVER_NAME=officelocal - export HTTP_ORDERS_SERVER_NAME=orderslocal - export HTTP_PRIME_SERVER_NAME=primelocal + export DEVLOCAL_CA=$(cat config/tls/devlocal-ca.pem) + export DOD_CA_PACKAGE=$(cat config/tls/milmove-cert-bundle.p7b) + + # export HTTP_ADMIN_SERVER_NAME=adminlocal + # export HTTP_MY_SERVER_NAME=milmovelocal + # export HTTP_OFFICE_SERVER_NAME=officelocal + # export HTTP_ORDERS_SERVER_NAME=orderslocal + # export HTTP_PRIME_SERVER_NAME=primelocal + - | export ENVIRONMENT=test export MUTUAL_TLS_ENABLED=true export MUTUAL_TLS_PORT=9443 export SERVE_API_PRIME=true - # disable for speed, tests can fail otherwise + # disable for speed, tests can fail otherwise + - | export DB_DEBUG=false make db_dev_create bin/milmove migrate - # mtls tests do not need client - mkdir -p build - touch build/index.html - # mtls tests DO NOT NEED SEED DATA - bin/milmove serve 2>&1 | tee server.log & + # mtls tests do not need client + - mkdir -p build + - touch build/index.html + # mtls tests DO NOT NEED SEED DATA + - bin/milmove serve 2>&1 | tee server.log & .e2e_tests_playwright: &e2e_tests_playwright - | From 4d5ed25220adc2da6d1705f92e0e97f710898232 Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Fri, 31 Jan 2025 14:50:34 +0000 Subject: [PATCH 07/14] test backup timeout --- .gitlab-ci.yml | 4 ++++ scripts/rds-snapshot-app-db | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87f7bbe11f3..7190260029c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -87,6 +87,8 @@ stages: export AWS_ACCOUNT_ID=$(eval echo \$${ENV}_ACCOUNT_ID); export AWS_ACCESS_KEY_ID=$(eval echo \$${ENV}_ACCESS_KEY_ID); export AWS_SECRET_ACCESS_KEY=$(eval echo \$${ENV}_SECRET_ACCESS_KEY); + export SERVICE_RESERVATION_CPU=2048 + export SERVICE_RESERVATION_MEM=3072 fi .setup_release_dp3: &setup_release_dp3 @@ -119,6 +121,8 @@ stages: - export AWS_ACCOUNT_ID=$PRD_ACCOUNT_ID - export AWS_ACCESS_KEY_ID=$PRD_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$PRD_SECRET_ACCESS_KEY + - export SERVICE_RESERVATION_CPU=2048 + - export SERVICE_RESERVATION_MEM=8192 .setup_tls_vars_prd: &setup_tls_vars_prd - export TLS_CERT=$PRD_MOVE_MIL_DOD_TLS_CERT diff --git a/scripts/rds-snapshot-app-db b/scripts/rds-snapshot-app-db index d27326e25a3..13cf1b27084 100755 --- a/scripts/rds-snapshot-app-db +++ b/scripts/rds-snapshot-app-db @@ -25,7 +25,7 @@ time aws rds wait db-snapshot-completed --db-instance-identifier "$db_instance_i echo echo "Create database snapshot for ${db_instance_identifier} with identifier ${db_snapshot_identifier}" -aws rds create-db-snapshot --db-instance-identifier "$db_instance_identifier" --db-snapshot-identifier "$db_snapshot_identifier" --tags "${tags[@]}" +aws rds create-db-snapshot --cli-read-timeout 600 --db-instance-identifier "$db_instance_identifier" --db-snapshot-identifier "$db_snapshot_identifier" --tags "${tags[@]}" echo echo "Wait for current database snapshot ${db_snapshot_identifier} to complete before continuing ..." From 5cae28498c9686cf0f876de06adf2874d1dbfc1c Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Fri, 31 Jan 2025 17:43:50 +0000 Subject: [PATCH 08/14] test gitlab updates --- .gitlab-ci.yml | 128 +++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 68 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7190260029c..8dd06f085ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ variables: CLIENT_IGNORE_BRANCH: &client_ignore_branch B-22106-ServerTest SERVER_IGNORE_BRANCH: &server_ignore_branch B-22106-ServerTest - RUNNER_TAG: &runner_tag eks_cluster_runner + RUNNER_TAG: &runner_tag milmove DOCKER_RUNNER_TAG: &docker_runner_tag eks_cluster_runner postgres: &postgres postgres:16.4 @@ -87,8 +87,6 @@ stages: export AWS_ACCOUNT_ID=$(eval echo \$${ENV}_ACCOUNT_ID); export AWS_ACCESS_KEY_ID=$(eval echo \$${ENV}_ACCESS_KEY_ID); export AWS_SECRET_ACCESS_KEY=$(eval echo \$${ENV}_SECRET_ACCESS_KEY); - export SERVICE_RESERVATION_CPU=2048 - export SERVICE_RESERVATION_MEM=3072 fi .setup_release_dp3: &setup_release_dp3 @@ -124,6 +122,7 @@ stages: - export SERVICE_RESERVATION_CPU=2048 - export SERVICE_RESERVATION_MEM=8192 + .setup_tls_vars_prd: &setup_tls_vars_prd - export TLS_CERT=$PRD_MOVE_MIL_DOD_TLS_CERT - export TLS_KEY=$PRD_MOVE_MIL_DOD_TLS_KEY @@ -245,13 +244,6 @@ stages: export OKTA_OFFICE_GROUP_ID=notrealgroupId export OKTA_CUSTOMER_GROUP_ID=notrealcustomergroupId -.setup_host_intergration_tests: &setup_host_intergration_tests - - echo "Setting up /etc/hosts for local domain simulation" - - echo "127.0.0.1 milmovelocal" | sudo tee -a /etc/hosts - - echo "127.0.0.1 officelocal" | sudo tee -a /etc/hosts - - echo "127.0.0.1 adminlocal" | sudo tee -a /etc/hosts - - echo "127.0.0.1 primelocal" | sudo tee -a /etc/hosts - .setup_env_intergration_mtls: &setup_env_intergration_mtls - | echo "Setting up environment variables" @@ -269,33 +261,11 @@ stages: export HERE_MAPS_ROUTING_ENDPOINT=https://route.api.here.com/routing/7.2/calculateroute.json export LOGIN_GOV_CALLBACK_PORT=4000 export LOGIN_GOV_CALLBACK_PROTOCOL=http - - export SERVE_API_SUPPORT=true - export SERVE_PRIME_SIMULATOR=true - export DEVLOCAL_CA=$(cat config/tls/devlocal-ca.pem) - export DOD_CA_PACKAGE=$(cat config/tls/milmove-cert-bundle.p7b) - - # export HTTP_ADMIN_SERVER_NAME=adminlocal - # export HTTP_MY_SERVER_NAME=milmovelocal - # export HTTP_OFFICE_SERVER_NAME=officelocal - # export HTTP_ORDERS_SERVER_NAME=orderslocal - # export HTTP_PRIME_SERVER_NAME=primelocal - - | - export ENVIRONMENT=test - export MUTUAL_TLS_ENABLED=true - export MUTUAL_TLS_PORT=9443 - export SERVE_API_PRIME=true - # disable for speed, tests can fail otherwise - - | - export DB_DEBUG=false - make db_dev_create bin/milmove migrate - # mtls tests do not need client - - mkdir -p build - - touch build/index.html - # mtls tests DO NOT NEED SEED DATA - - bin/milmove serve 2>&1 | tee server.log & + mkdir -p build + touch build/index.html + bin/milmove serve 2>&1 | tee server.log & .e2e_tests_playwright: &e2e_tests_playwright - | @@ -535,16 +505,18 @@ deploy_storybook_dp3: - build_storybook before_script: - *setup_milmove_env + - *setup_aws_vars_dp3 + - ls -l $CI_PROJECT_DIR/storybook-static || echo "storybook-static directory not found!" script: - - echo "TODO Add steps" - - echo "deploy_storybook_dp3" + - echo "Deploying Storybook to S3..." + - ./scripts/push-storybook-assets "storybook.$APP_ENVIRONMENT.move.mil" after_script: - *announce_failure artifacts: paths: - /builds/milmove/mymove/storybook-static rules: - - *check_main + - *check_main compile_app_client: stage: build @@ -1244,17 +1216,23 @@ push_otel_collector_image_dp3: - compile_app_server - compile_app_client script: - - echo "Logging in to Amazon ECR with Crane..." - - aws ecr get-login-password --region ${AWS_REGION} | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin + - echo "Logging in to DockerHub with Crane..." + - echo "${DOCKERHUB_PASSWORD}" | crane auth login docker.io -u "${DOCKERHUB_USERNAME}" --password-stdin + + - echo "Pulling the AWS OTel Collector image from DockerHub..." + - crane pull ${ECR_REPOSITORY_OTEL}:${OTEL_VERSION} image.tar + + - echo "Extracting the OTel image tag..." + - export OTEL_IMAGE_TAG="git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA}" - - echo "Pulling the AWS OTel Collector image from the public registry with Crane..." - - crane pull --insecure public.ecr.aws/aws-observability/aws-otel-collector:v0.31.0 image.tar + - echo "Logging into AWS ECR with Crane..." + - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com -u AWS --password-stdin - - echo "Pushing the image to our private ECR using Crane..." - - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA} + - echo "Pushing the image to our private ECR using Crane..." + - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${OTEL_IMAGE_TAG} - - echo "Cleaning up the temporary image file..." - - rm image.tar + - echo "Cleaning up temporary image file..." + - rm image.tar allow_failure: false after_script: - *announce_failure @@ -1349,7 +1327,7 @@ deploy_app_client_tls_dp3: - echo "Getting Digest from AWS" - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector Digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app-client-tls service" - ./scripts/ecs-deploy-service-container app-client-tls "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1400,7 +1378,7 @@ deploy_app_dp3: - echo "Getting Digest from AWS" - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app service" - ./scripts/ecs-deploy-service-container app "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1507,17 +1485,24 @@ push_otel_collector_image_stg: - *setup_aws_vars_stg - *setup_release_stg script: - - echo "Logging in to Amazon ECR with Crane..." - - aws ecr get-login-password --region ${AWS_REGION} | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin + - echo "Logging in to DockerHub with Crane..." + - echo "${DOCKERHUB_PASSWORD}" | crane auth login docker.io -u "${DOCKERHUB_USERNAME}" --password-stdin - - echo "Pulling the AWS OTel Collector image from the public registry with Crane..." - - crane pull --insecure public.ecr.aws/aws-observability/aws-otel-collector:v0.31.0 image.tar + - echo "Pulling the AWS OTel Collector image from DockerHub..." + - crane pull ${ECR_REPOSITORY_OTEL}:${OTEL_VERSION} image.tar - - echo "Pushing the image to our private ECR using Crane..." - - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA} + - echo "Extracting the OTel image tag..." + - export OTEL_IMAGE_TAG="git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA}" + - echo "${OTEL_IMAGE_TAG}" - - echo "Cleaning up the temporary image file..." - - rm image.tar + - echo "Logging into AWS ECR with Crane..." + - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com -u AWS --password-stdin + + - echo "Pushing the image to our private ECR using Crane..." + - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${OTEL_IMAGE_TAG} + + - echo "Cleaning up temporary image file..." + - rm image.tar allow_failure: false after_script: - *announce_failure @@ -1610,7 +1595,7 @@ deploy_app_client_tls_stg: - echo "Getting Digest from AWS" - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector Digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app-client-tls service" - ./scripts/ecs-deploy-service-container app-client-tls "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1662,7 +1647,7 @@ deploy_app_stg: - echo "Getting Digest from AWS" - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app service" - ./scripts/ecs-deploy-service-container app "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1789,17 +1774,24 @@ push_otel_collector_image_prd: - *setup_aws_vars_prd - *setup_release_prd script: - - echo "Logging in to Amazon ECR with Crane..." - - aws ecr get-login-password --region ${AWS_REGION} | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin + - echo "Logging in to DockerHub with Crane..." + - echo "${DOCKERHUB_PASSWORD}" | crane auth login docker.io -u "${DOCKERHUB_USERNAME}" --password-stdin + + - echo "Pulling the AWS OTel Collector image from DockerHub..." + - crane pull ${ECR_REPOSITORY_OTEL}:${OTEL_VERSION} image.tar + + - echo "Extracting the OTel image tag..." + - export OTEL_IMAGE_TAG="git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA}" + - echo "${OTEL_IMAGE_TAG}" - - echo "Pulling the AWS OTel Collector image from the public registry with Crane..." - - crane pull --insecure public.ecr.aws/aws-observability/aws-otel-collector:v0.31.0 image.tar + - echo "Logging into AWS ECR with Crane..." + - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com -u AWS --password-stdin - - echo "Pushing the image to our private ECR using Crane..." - - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA} + - echo "Pushing the image to our private ECR using Crane..." + - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${OTEL_IMAGE_TAG} - - echo "Cleaning up the temporary image file..." - - rm image.tar + - echo "Cleaning up temporary image file..." + - rm image.tar allow_failure: false after_script: - *announce_failure @@ -1894,7 +1886,7 @@ deploy_app_client_tls_prd: - echo "Getting Digest from AWS" - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector Digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app-client-tls service" - ./scripts/ecs-deploy-service-container app-client-tls "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1946,7 +1938,7 @@ deploy_app_prd: - echo "Getting Digest from AWS" - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app service" - ./scripts/ecs-deploy-service-container app "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" From cccfac09509741c339924b0a718950c4c937f0cf Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Fri, 31 Jan 2025 18:24:28 +0000 Subject: [PATCH 09/14] test gitlab updates --- .gitlab-ci.yml | 76 +++++++++++++++++------------------ scripts/push-storybook-assets | 2 +- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8dd06f085ee..8be1d3d86af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,6 +39,8 @@ variables: CLIENT_IGNORE_BRANCH: &client_ignore_branch B-22106-ServerTest SERVER_IGNORE_BRANCH: &server_ignore_branch B-22106-ServerTest + OTEL_IMAGE_TAG: &otel_image_tag "git-$OTEL_VERSION-$CI_COMMIT_SHORT_SHA" + RUNNER_TAG: &runner_tag milmove DOCKER_RUNNER_TAG: &docker_runner_tag eks_cluster_runner @@ -101,6 +103,12 @@ stages: export APP_ENVIRONMENT=$DP3_ENV fi +.setup_aws_vars_com_dev: &setup_aws_vars_com_dev + - export AWS_DEFAULT_REGION=$COM_REGION + - export AWS_ACCOUNT_ID=$DEV_ACCOUNT_ID + - export AWS_ACCESS_KEY_ID=$DEV_ACCESS_KEY_ID + - export AWS_SECRET_ACCESS_KEY=$DEV_SECRET_KEY + .setup_aws_vars_stg: &setup_aws_vars_stg - export AWS_DEFAULT_REGION=$STG_REGION - export AWS_ACCOUNT_ID=$STG_ACCOUNT_ID @@ -505,13 +513,14 @@ deploy_storybook_dp3: - build_storybook before_script: - *setup_milmove_env - - *setup_aws_vars_dp3 + - *setup_aws_vars_com_dev - ls -l $CI_PROJECT_DIR/storybook-static || echo "storybook-static directory not found!" script: - echo "Deploying Storybook to S3..." - - ./scripts/push-storybook-assets "storybook.$APP_ENVIRONMENT.move.mil" + - ./scripts/push-storybook-assets "storybook.dp3.move.mil" after_script: - *announce_failure + allow_failure: true artifacts: paths: - /builds/milmove/mymove/storybook-static @@ -1146,7 +1155,7 @@ build_push_app_dp3: - *kaniko_before_setup script: - echo "Building and Pushing app Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${APP_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${APP_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1170,7 +1179,7 @@ build_push_migrations_dp3: - *kaniko_before_setup script: - echo "Building and Pushing migrations Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/Dockerfile.migrations" --destination "${ECR_REPOSITORY_URI}/app-migrations:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/Dockerfile.migrations" --destination "${ECR_REPOSITORY_URI}/app-migrations:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1194,7 +1203,7 @@ build_push_tasks_dp3: - *kaniko_before_setup script: - echo "Building tasks Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${TASK_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app-tasks:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${TASK_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app-tasks:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1222,9 +1231,6 @@ push_otel_collector_image_dp3: - echo "Pulling the AWS OTel Collector image from DockerHub..." - crane pull ${ECR_REPOSITORY_OTEL}:${OTEL_VERSION} image.tar - - echo "Extracting the OTel image tag..." - - export OTEL_IMAGE_TAG="git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA}" - - echo "Logging into AWS ECR with Crane..." - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com -u AWS --password-stdin @@ -1259,7 +1265,7 @@ deploy_migrations_dp3: script: # Step 1: Get the Digest - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-migrations --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-migrations --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) # Step 2: Ensure exclusive execution and Snapshot - echo "Snapshotting database" - ./scripts/rds-snapshot-app-db "$APP_ENVIRONMENT" @@ -1290,7 +1296,7 @@ deploy_tasks_dp3: - *setup_release_dp3 script: - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-tasks --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-tasks --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Deploying GHC fuel price data task service" - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" - echo "Deploying payment reminder email task service" @@ -1325,9 +1331,9 @@ deploy_app_client_tls_dp3: # - echo "Comparing against deployed commit" # - ./scripts/compare-deployed-commit "" $CI_COMMIT_SHA ${TLS_KEY} ${TLS_CERT} ${TLS_CA} - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector Digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA} --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=${OTEL_IMAGE_TAG} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app-client-tls service" - ./scripts/ecs-deploy-service-container app-client-tls "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1376,7 +1382,7 @@ deploy_app_dp3: echo $GO_VERSION > .go-version fi - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector digest from AWS" - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" @@ -1415,7 +1421,7 @@ build_push_app_stg: - *kaniko_before_setup script: - echo "Building and Pushing app Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${APP_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${APP_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1439,7 +1445,7 @@ build_push_migrations_stg: - *kaniko_before_setup script: - echo "Building and Pushing migrations Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/Dockerfile.migrations" --destination "${ECR_REPOSITORY_URI}/app-migrations:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/Dockerfile.migrations" --destination "${ECR_REPOSITORY_URI}/app-migrations:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1463,7 +1469,7 @@ build_push_tasks_stg: - *kaniko_before_setup script: - echo "Building tasks Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${TASK_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app-tasks:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${TASK_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app-tasks:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1491,10 +1497,6 @@ push_otel_collector_image_stg: - echo "Pulling the AWS OTel Collector image from DockerHub..." - crane pull ${ECR_REPOSITORY_OTEL}:${OTEL_VERSION} image.tar - - echo "Extracting the OTel image tag..." - - export OTEL_IMAGE_TAG="git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA}" - - echo "${OTEL_IMAGE_TAG}" - - echo "Logging into AWS ECR with Crane..." - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com -u AWS --password-stdin @@ -1529,7 +1531,7 @@ deploy_migrations_stg: script: # Step 1: Get the Digest - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-migrations --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-migrations --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) # Step 2: Ensure exclusive execution and Snapshot - echo "Snapshotting database" - ./scripts/rds-snapshot-app-db "$APP_ENVIRONMENT" @@ -1560,7 +1562,7 @@ deploy_tasks_stg: - *setup_release_stg script: - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-tasks --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-tasks --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Deploying GHC fuel price data task service" - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" - echo "Deploying payment reminder email task service" @@ -1593,9 +1595,9 @@ deploy_app_client_tls_stg: # - echo "Comparing against deployed commit" # - ./scripts/compare-deployed-commit "" $CI_COMMIT_SHA ${TLS_KEY} ${TLS_CERT} ${TLS_CA} - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector Digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA} --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=${OTEL_IMAGE_TAG} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app-client-tls service" - ./scripts/ecs-deploy-service-container app-client-tls "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1645,9 +1647,9 @@ deploy_app_stg: echo $GO_VERSION > .go-version fi - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=${OTEL_IMAGE_TAG} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app service" - ./scripts/ecs-deploy-service-container app "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1701,7 +1703,7 @@ build_push_app_prd: - *kaniko_before_setup script: - echo "Building and Pushing app Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${APP_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${APP_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1726,7 +1728,7 @@ build_push_migrations_prd: - *kaniko_before_setup script: - echo "Building and Pushing migrations Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/Dockerfile.migrations" --destination "${ECR_REPOSITORY_URI}/app-migrations:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/Dockerfile.migrations" --destination "${ECR_REPOSITORY_URI}/app-migrations:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1751,7 +1753,7 @@ build_push_tasks_prd: - *kaniko_before_setup script: - echo "Building tasks Docker image..." - - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${TASK_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app-tasks:$CI_COMMIT_SHORT_SHA" + - /kaniko/executor --context "${CI_PROJECT_DIR}/" --dockerfile "${CI_PROJECT_DIR}/${TASK_DOCKER_FILE}" --destination "${ECR_REPOSITORY_URI}/app-tasks:git-$CI_COMMIT_SHORT_SHA" after_script: - *announce_failure rules: @@ -1780,10 +1782,6 @@ push_otel_collector_image_prd: - echo "Pulling the AWS OTel Collector image from DockerHub..." - crane pull ${ECR_REPOSITORY_OTEL}:${OTEL_VERSION} image.tar - - echo "Extracting the OTel image tag..." - - export OTEL_IMAGE_TAG="git-${OTEL_VERSION}-${CI_COMMIT_SHORT_SHA}" - - echo "${OTEL_IMAGE_TAG}" - - echo "Logging into AWS ECR with Crane..." - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com -u AWS --password-stdin @@ -1818,7 +1816,7 @@ deploy_migrations_prd: script: # Step 1: Get the Digest - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-migrations --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-migrations --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) # Step 2: Ensure exclusive execution and Snapshot - echo "Snapshotting database" - ./scripts/rds-snapshot-app-db "$APP_ENVIRONMENT" @@ -1849,7 +1847,7 @@ deploy_tasks_prd: - *setup_release_prd script: - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-tasks --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app-tasks --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Deploying GHC fuel price data task service" - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" - echo "Deploying payment reminder email task service" @@ -1884,9 +1882,9 @@ deploy_app_client_tls_prd: # - echo "Comparing against deployed commit" # - ./scripts/compare-deployed-commit "" $CI_COMMIT_SHA ${TLS_KEY} ${TLS_CERT} ${TLS_CA} - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector Digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=${OTEL_IMAGE_TAG} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app-client-tls service" - ./scripts/ecs-deploy-service-container app-client-tls "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" @@ -1936,9 +1934,9 @@ deploy_app_prd: echo $GO_VERSION > .go-version fi - echo "Getting Digest from AWS" - - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export ECR_DIGEST=$(aws ecr describe-images --repository-name app --image-ids imageTag=git-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) - echo "Getting otel collector digest from AWS" - - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=git-${OTEL_VERSION}-$CI_COMMIT_SHORT_SHA --query 'imageDetails[0].imageDigest' --output text) + - export OTEL_ECR_DIGEST=$(aws ecr describe-images --repository-name otel-collector --image-ids imageTag=${OTEL_IMAGE_TAG} --query 'imageDetails[0].imageDigest' --output text) - export OTEL_COLLECTOR_IMAGE="${ECR_REPOSITORY_URI}/otel-collector@${OTEL_ECR_DIGEST}" - echo "Deploying app service" - ./scripts/ecs-deploy-service-container app "${ECR_REPOSITORY_URI}/app@${ECR_DIGEST}" "${APP_ENVIRONMENT}" "/bin/milmove serve" diff --git a/scripts/push-storybook-assets b/scripts/push-storybook-assets index 15523762fa7..9674ed27fc3 100755 --- a/scripts/push-storybook-assets +++ b/scripts/push-storybook-assets @@ -16,4 +16,4 @@ readonly bucket=${1:-} # Sync files from source to destination # encrypt files on upload # delete files at destination that are not on source (bucket versioning is enabled so it only adds a delete marker) -aws s3 sync --delete --sse AES256 /tmp/storybook/storybook-static/ s3://"${bucket}"/ +aws s3 sync --delete --sse AES256 /builds/milmove/mymove/storybook-static s3://"${bucket}"/ From 8455368fc663e46080e8fbaae2e3e1bdacffa975 Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Fri, 31 Jan 2025 18:32:04 +0000 Subject: [PATCH 10/14] test story book --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8be1d3d86af..bac13f500e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -500,7 +500,7 @@ build_storybook: paths: - /builds/milmove/mymove/storybook-static rules: - - *check_main + - *check_client_ignore_branch deploy_storybook_dp3: stage: deploy @@ -525,7 +525,7 @@ deploy_storybook_dp3: paths: - /builds/milmove/mymove/storybook-static rules: - - *check_main + - *check_client_ignore_branch compile_app_client: stage: build From 4d864af83ecacdc5ef25e223557239004927f6ae Mon Sep 17 00:00:00 2001 From: Jacinta Callahan Date: Fri, 31 Jan 2025 18:51:23 +0000 Subject: [PATCH 11/14] update otel --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bac13f500e8..639ad9302a4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,16 +28,16 @@ variables: GOLANGCI_LINT_VERBOSE: "-v" # Specify the environment: loadtest, demo, exp - DP3_ENV: &dp3_env demo + DP3_ENV: &dp3_env placeholder_env # Specify the branch to deploy TODO: this might be not needed. So far useless - DP3_BRANCH: &dp3_branch B-22106-ServerTest + DP3_BRANCH: &dp3_branch placeholder_branch_name # Ignore branches for integration tests - INTEGRATION_IGNORE_BRANCH: &integration_ignore_branch B-22106-ServerTest - INTEGRATION_MTLS_IGNORE_BRANCH: &integration_mtls_ignore_branch B-22106-ServerTest - CLIENT_IGNORE_BRANCH: &client_ignore_branch B-22106-ServerTest - SERVER_IGNORE_BRANCH: &server_ignore_branch B-22106-ServerTest + INTEGRATION_IGNORE_BRANCH: &integration_ignore_branch placeholder_branch_name + INTEGRATION_MTLS_IGNORE_BRANCH: &integration_mtls_ignore_branch placeholder_branch_name + CLIENT_IGNORE_BRANCH: &client_ignore_branch placeholder_branch_name + SERVER_IGNORE_BRANCH: &server_ignore_branch placeholder_branch_name OTEL_IMAGE_TAG: &otel_image_tag "git-$OTEL_VERSION-$CI_COMMIT_SHORT_SHA" @@ -500,7 +500,7 @@ build_storybook: paths: - /builds/milmove/mymove/storybook-static rules: - - *check_client_ignore_branch + - *check_main deploy_storybook_dp3: stage: deploy @@ -525,7 +525,7 @@ deploy_storybook_dp3: paths: - /builds/milmove/mymove/storybook-static rules: - - *check_client_ignore_branch + - *check_main compile_app_client: stage: build From edeed60180d6ec1c61807322b96565c4977b4a8d Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Fri, 31 Jan 2025 18:56:22 +0000 Subject: [PATCH 12/14] white space --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 639ad9302a4..94a69f35706 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ variables: DOCKER_TOKEN: DOCKER_TOKEN registry: https://registry.hub.docker.com/ + # #Circle CI need to replace #CIRCLE_PROJECT_USERNAME: "my-username" # Replace with the actual namespace CIRCLE_PROJECT_REPONAME: "mymove" # Replace with your GitLab project name From e89d5eb33ae1ea7f9e03adf774a4ea7a7f9fd154 Mon Sep 17 00:00:00 2001 From: Jacinta Callahan Date: Fri, 31 Jan 2025 19:32:24 +0000 Subject: [PATCH 13/14] added cpu in dp3 anchor --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 639ad9302a4..e31d0210969 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,6 +82,8 @@ stages: fi .setup_aws_vars_dp3: &setup_aws_vars_dp3 + - export SERVICE_RESERVATION_CPU=2048 + - export SERVICE_RESERVATION_MEM=4096 - | if [[ "$DP3_ENV" == "exp" || "$DP3_ENV" == "loadtest" || "$DP3_ENV" == "demo" ]]; then export ENV=$(echo ${DP3_ENV} | tr '[:lower:]' '[:upper:]'); @@ -514,10 +516,9 @@ deploy_storybook_dp3: before_script: - *setup_milmove_env - *setup_aws_vars_com_dev - - ls -l $CI_PROJECT_DIR/storybook-static || echo "storybook-static directory not found!" script: - echo "Deploying Storybook to S3..." - - ./scripts/push-storybook-assets "storybook.dp3.move.mil" + - ./scripts/push-storybook-assets "storybook.dp3.us" after_script: - *announce_failure allow_failure: true From 7d010eae5fccb3f090e6cef6e302f47eacfd50dd Mon Sep 17 00:00:00 2001 From: josiahzimmerman-caci Date: Fri, 31 Jan 2025 19:38:21 +0000 Subject: [PATCH 14/14] update timeout to 20 min, remove storybook from circle --- .circleci/config.yml | 5 ++++- scripts/rds-snapshot-app-db | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d154602cf7..8744b04c2ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2704,12 +2704,15 @@ workflows: - build_storybook filters: branches: - only: main + only: nolonger_main - build_storybook: requires: - anti_virus - pre_deps_yarn + filters: + branches: + only: nolonger_main - push_otel_collector_image_prd: requires: diff --git a/scripts/rds-snapshot-app-db b/scripts/rds-snapshot-app-db index 13cf1b27084..493be887936 100755 --- a/scripts/rds-snapshot-app-db +++ b/scripts/rds-snapshot-app-db @@ -25,7 +25,7 @@ time aws rds wait db-snapshot-completed --db-instance-identifier "$db_instance_i echo echo "Create database snapshot for ${db_instance_identifier} with identifier ${db_snapshot_identifier}" -aws rds create-db-snapshot --cli-read-timeout 600 --db-instance-identifier "$db_instance_identifier" --db-snapshot-identifier "$db_snapshot_identifier" --tags "${tags[@]}" +aws rds create-db-snapshot --cli-read-timeout 1200 --db-instance-identifier "$db_instance_identifier" --db-snapshot-identifier "$db_snapshot_identifier" --tags "${tags[@]}" echo echo "Wait for current database snapshot ${db_snapshot_identifier} to complete before continuing ..."