Remove sensitive info (#544)

* use gh secrets

* use gh secrets also for prod

* infra bucket dynamic lookup and documentation update

* temporary trigger deploy from remove-sensitive-info branch to test actions

* adding additional required policy

* allow correct resource for description

* fix cf stack update params handling

* Revert "temporary trigger deploy from remove-sensitive-info branch to test actions"

This reverts commit d97a32e.

* typo fix

Author: psacc

.github/workflows/demo_deploy.yml

@@ -74,7 +74,7 @@ jobs:
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-region: eu-west-1
-          role-to-assume: arn:aws:iam::***REMOVED***:role/pcluster-manager-github-PrivateInfrastructureUpdat-1HXHHPQ292PNI
+          role-to-assume: ${{ secrets.ACTION_DEMO_DEPLOY_JOB_UPDATE_INFRASTRUCTURE_ROLE }}
 
       - name: Update staging infrastructure
         run: ./infrastructure/update-environment-infra.sh demo
@@ -92,7 +92,7 @@ jobs:
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-region: eu-west-1
-          role-to-assume: arn:aws:iam::***REMOVED***:role/pcluster-manager-github-PrivateDeployRole-8D5K5C4RM02U
+          role-to-assume: ${{ secrets.ACTION_DEMO_DEPLOY_JOB_BUILD_AND_DEPLOY_ROLE }}
 
       - name: Update staging environment
         run: ./scripts/build_and_update_lambda.sh --stack-name "parallelcluster-ui-demo" --region eu-west-1

.github/workflows/e2e_tests.yml

@@ -24,7 +24,7 @@ jobs:
       uses: aws-actions/configure-aws-credentials@v1
       with:
         aws-region: eu-west-1
-        role-to-assume: arn:aws:iam::***REMOVED***:role/pcluster-manager-github-E2ETestExecution-Z7IMRUIJAZAO
+        role-to-assume: ${{ secrets.ACTION_E2E_TESTS_ROLE }}
 
     - name: Retrieve test user email and password
       uses: aws-actions/aws-secretsmanager-get-secrets@v1

.github/workflows/production_release.yml

@@ -71,7 +71,7 @@ jobs:
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-region: us-east-1
-          role-to-assume: arn:aws:iam::***REMOVED***:role/pcluster-manager-github-prod-ProductionDeploy-1I1HYDA5DH7T3
+          role-to-assume: ${{ secrets.ACTION_PRODUCTION_RELEASE_ROLE }}
 
       - name: Build and upload Docker image
         run: ./scripts/build_and_release_image.sh

infrastructure/readme.md renamed to infrastructure/README.md

@@ -4,14 +4,16 @@ To access AWS resources inside a Github workflow you need to create new IAM role
 To create the resources needed by the workflow action you can deploy the `./github-env-setup.yml` to [CloudFormation](https://aws.amazon.com/cloudformation/).
 - Go under `CloudFormation > Stacks > Create stack`
 - Upload a template file using `github-env-setup.yml`
-- Give the stack a name (it doesn't matter which one)
+- Give the stack a name (it should match the `INFRA_BUCKET_STACK_NAME` for env deploy, i.e. `INFRA_BUCKET_STACK_NAME=pcluster-manager-github` for the demo env)
 - Create the stack
-- Go to the IAM console, find the role name `*PrivateDeploy*`, copy the ARN and use it with the [AWS credentials action](https://github.com/marketplace/actions/configure-aws-credentials-action-for-github-actions) to authenticate
-- Same needs to be done for the `*PrivateInfrastructureUpdateRole*` role
+- Go to the IAM console, find the roles created (see list below), copy the ARN and use it with the [AWS credentials action](https://github.com/marketplace/actions/configure-aws-credentials-action-for-github-actions) to authenticate using those in the matching GitHub Secrets
 
-The stack will create two new roles:
-- the `PrivateDeployRole` with the minimum set of policies needed to build and deploy an instance of PCluster Manager
-- the `PrivateInfrastructureUpdateRole` with the minimum set of policies needed to update the infrastructure of an environment running PCluster Manager
+The stack will create three new roles:
+- the `PrivateDeployRole` with the minimum set of policies needed to build and deploy an instance of PCluster Manager, its arn should be put in the secret named `ACTION_DEMO_DEPLOY_JOB_BUILD_AND_DEPLOY_ROLE`
+- the `PrivateInfrastructureUpdateRole` with the minimum set of policies needed to update the infrastructure of an environment running PCluster Manager, its arn should be put in the secret named `ACTION_DEMO_DEPLOY_JOB_UPDATE_INFRASTRUCTURE_ROLE`
+- the `E2ETestExecutionRole` with the minimum set of policies needed to in order to run E2E tests workflow, its arn should be put in the secret named `ACTION_E2E_TESTS_ROLE`
+
+The same steps are required for the production release workflow, using the `./github-env-setup-prod.yml` stack, to create the role `ProductionDeploy` that should be put in the secret named `ACTION_PRODUCTION_RELEASE_ROLE`.
 
 **This procedure must be done only once per AWS account since IAM it's a global service.**
 

infrastructure/environments/demo-cfn.yaml renamed to infrastructure/environments/demo-cfn-update-args.yaml

@@ -1,9 +1,9 @@
-TemplateURL: https://pcluster-manager-github-infrastructurebucket-7y5h202iem8l.s3.eu-west-1.amazonaws.com/parallelcluster-ui.yaml
+TemplateURL: BUCKET_URL_PLACEHOLDER/parallelcluster-ui.yaml
 Parameters:
   - ParameterKey: Version
     ParameterValue: 3.4.0
   - ParameterKey: InfrastructureBucket
-    ParameterValue: https://pcluster-manager-github-infrastructurebucket-7y5h202iem8l.s3.eu-west-1.amazonaws.com
+    ParameterValue: BUCKET_URL_PLACEHOLDER
   - ParameterKey: UserPoolId
     UsePreviousValue: true
   - ParameterKey: UserPoolAuthDomain

infrastructure/environments/demo-variables.sh

@@ -1,3 +1,3 @@
 REGION=eu-west-1
-BUCKET=pcluster-manager-github-infrastructurebucket-7y5h202iem8l
-STACK_NAME=parallelcluster-ui-demo
+STACK_NAME=parallelcluster-ui-demo
+INFRA_BUCKET_STACK_NAME=pcluster-manager-github

infrastructure/github-env-setup-prod.yml

@@ -1,3 +1,5 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
 Parameters:
   GitHubOrg:
     Type: String

infrastructure/github-env-setup.yml

@@ -1,3 +1,5 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
 Parameters:
   GitHubOrg:
     Type: String
@@ -107,10 +109,12 @@ Resources:
               - Effect: Allow
                 Action:
                   - cloudformation:DescribeStacks
+                  - cloudformation:DescribeStackResources
                   - cloudformation:UpdateStack
                   - cloudformation:CreateChangeSet
                 Resource:
                   - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/parallelcluster-ui-demo*
+                  - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}*
                   - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:aws:transform/*
               - Effect: Allow
                 Action:
@@ -123,8 +127,8 @@ Resources:
                   - s3:GetObjectAcl
                   - s3:PutObjectAcl
                 Resource:
-                  - !Sub arn:${AWS::Partition}:s3:::pcluster-manager-github-infrastructurebucket-7y5h202iem8l
-                  - !Sub arn:${AWS::Partition}:s3:::pcluster-manager-github-infrastructurebucket-7y5h202iem8l/*
+                  - !GetAtt 'InfrastructureBucket.Arn'
+                  - !Join ['', [!GetAtt 'InfrastructureBucket.Arn', '/*']]
                   - !Sub arn:${AWS::Partition}:s3:::${AWS::Region}-aws-parallelcluster
                   - !Sub arn:${AWS::Partition}:s3:::${AWS::Region}-aws-parallelcluster/*
               - Effect: Allow
@@ -151,7 +155,7 @@ Resources:
                   - cognito-idp:UpdateGroup
                   - cognito-idp:DeleteGroup
                 Resource:
-                  - arn:aws:cognito-idp:eu-west-1:***REMOVED***:userpool/*
+                  - !Sub arn:aws:cognito-idp:${AWS::Region}:${AWS::AccountId}:userpool/*
         - PolicyName: ImageBuilderPolicy
           PolicyDocument:
             Version: 2012-10-17
@@ -198,7 +202,7 @@ Resources:
                 Resource: "*"
               - Effect: Allow
                 Action: secretsmanager:GetSecretValue
-                Resource: arn:aws:secretsmanager:eu-west-1:***REMOVED***:secret:e2e/test1-AUsDgk
+                Resource: !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:e2e/test1-AUsDgk
 
 Outputs:
   PrivateDeployRole:

infrastructure/update-environment-infra.sh

@@ -21,18 +21,34 @@
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 FILES=(SSMSessionProfile-cfn.yaml parallelcluster-ui-cognito.yaml parallelcluster-ui.yaml)
 ENVIRONMENT=$1
-. ${SCRIPT_DIR}/environments/${ENVIRONMENT}-variables.sh
+. "${SCRIPT_DIR}/environments/${ENVIRONMENT}-variables.sh"
+
+BUCKET=$(aws cloudformation describe-stack-resources \
+  --stack-name "${INFRA_BUCKET_STACK_NAME}" \
+  --logical-resource-id InfrastructureBucket \
+  --output json \
+  --query 'StackResources[0].PhysicalResourceId'\
+  | tr -d '"' )
 
 # The yaml files describing the infrastructure are uploaded to a private S3 bucket
 # and then used to update the CloudFormation stack, where the same bucket is passed as parameters.
 # This is done to make sure that we deploy all the changes to the infrastructure, and not only the changes
 # made to parallelcluster-ui.yaml (the parent stack)
-echo Uploading to: ${BUCKET}
+echo Uploading to: "${BUCKET}"
 for FILE in "${FILES[@]}"
 do
-  aws s3 cp ${SCRIPT_DIR}/${FILE} s3://${BUCKET}/${FILE}
+  aws s3 cp "${SCRIPT_DIR}/${FILE}" "s3://${BUCKET}/${FILE}"
 done
 
+BUCKET_URL="https://${BUCKET}.s3.${REGION}.amazonaws.com"
+CLI_INPUT_YAML=$(sed "s#BUCKET_URL_PLACEHOLDER#${BUCKET_URL}#g" "${SCRIPT_DIR}/environments/${ENVIRONMENT}-cfn-update-args.yaml")
+
 # Launches a new CFN update: the script hangs until the stack is updated
-AWS_PAGER="cat" aws cloudformation update-stack --cli-input-yaml file://${SCRIPT_DIR}/environments/${ENVIRONMENT}-cfn.yaml --stack-name ${STACK_NAME} --region ${REGION}
-aws cloudformation wait stack-update-complete --stack-name ${STACK_NAME} --region ${REGION}
+AWS_PAGER="cat" aws cloudformation update-stack \
+  --cli-input-yaml "${CLI_INPUT_YAML}" \
+  --stack-name "${STACK_NAME}" \
+  --region "${REGION}"
+
+aws cloudformation wait stack-update-complete \
+  --stack-name "${STACK_NAME}" \
+  --region "${REGION}"