From 49264da0e12095841321d025b212f3213da0ac9f Mon Sep 17 00:00:00 2001 From: msaligane Date: Wed, 15 Sep 2021 01:57:35 +0000 Subject: [PATCH 01/11] README edits --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 6de857424c4..f5f2889a5fe 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,35 @@ cd buildroot && make BR2_EXTERNAL=../overlay/ scalenode_gcp_defconfig && make # Prepare a disk for GCP ./make_gcp_image.sh +1. Save the bucket's IAM policy to a temporary (arbitrary) JSON file +``` +gsutil iam get gs://$BUCKET > /arbitrary/path/file.json +``` +2. Get the project name and default service account email address. Adjust filter accordingly if a different service account is used +``` +export PROJECT=$(gcloud config get-value project) +export SA=$(gcloud iam service-accounts list --filter=default | grep -E -o '[a-z0-9._%+-]+@[a-z0-9.-]+(\.[a-z0-9._%+-]+)?[a-z]{2,4}') +``` +3. Get the absolute path of the Bucket config file +``` +export BUCKET_FILE=/arbitrary/path/file.json +``` +4. Using the `sed` utility to insert required permissions associated with the bucket +``` +sed -i 's/"bindings": \[/"bindings": \[\ + {\ + "members": \[\ + "projectEditor:'"$PROJECT"'",\ + "projectOwner:'"$PROJECT"'",\ + "serviceAccount:'"$SA"'"\ + \],\ + "role": "roles\/storage.legacyBucketOwner"\ + \},/' $BUCKET_FILE +``` +5. Upload the modified bucket file back to GCloud +``` +gsutil iam set $BUCKET_FILE gs://$BUCKET +``` # Upload the resulting tar archive ./upload_gcp_image.sh $PROJECT $BUCKET ``` From a4bff6d4e2211a5ab2ae91257d0ac5580a327755 Mon Sep 17 00:00:00 2001 From: Mehdi Saligane <48068702+msaligane@users.noreply.github.com> Date: Tue, 14 Sep 2021 22:01:44 -0400 Subject: [PATCH 02/11] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f5f2889a5fe..14c7c7a7bc8 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ cd buildroot && make BR2_EXTERNAL=../overlay/ scalenode_gcp_defconfig && make # Prepare a disk for GCP ./make_gcp_image.sh +``` 1. Save the bucket's IAM policy to a temporary (arbitrary) JSON file ``` From 9b75daea991586f746a8943cd36335171d6be6f0 Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Mon, 27 Sep 2021 01:25:23 -0400 Subject: [PATCH 03/11] Update README.md --- README.md | 66 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 14c7c7a7bc8..1e448f853a8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,25 @@ The following packages must be installed: * `build-essential` * [Terraform](https://www.terraform.io/docs/cli/install/apt.html) +```bash +sudo apt-get install software-properties-common + +curl -fsSL https://apt.releases.hashicorp.com/gpg | + sudo apt-key add - + +sudo apt-add-repository "deb [arch=$(dpkg --print-architecture)] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +``` * [Google Cloud SDK](https://cloud.google.com/sdk/docs/install#deb) +```bash +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | + sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + +sudo apt-get install apt-transport-https ca-certificates gnupg +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | + sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + +sudo apt-get update && sudo apt-get install google-cloud-sdk +``` ### Installation steps @@ -46,22 +64,24 @@ export SERVICE_ACCOUNT_ID=runner-manager gcloud iam service-accounts create $SERVICE_ACCOUNT_ID gcloud projects add-iam-policy-binding $PROJECT \ - --member="serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT \ + --member="serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT.iam.gserviceaccount.com" \ --role="roles/compute.admin" gcloud projects add-iam-policy-binding $PROJECT \ - --member="serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT \ + --member="serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT.iam.gserviceaccount.com" \ --role="roles/iam.serviceAccountCreator" gcloud projects add-iam-policy-binding $PROJECT \ - --member="serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT \ + --member="serviceAccount:$SERVICE_ACCOUNT_ID@$PROJECT.iam.gserviceaccount.com" \ --role="roles/iam.serviceAccountUser" # Create and download SA key. # WARNING: the export below will be used by Terraform later. -export GOOGLE_APPLICATION_CREDENTIALS +# This command is for the OUTPUT_FILE option for "keys create" +export GOOGLE_APPLICATION_CREDENTIALS=~/key.out + gcloud iam service-accounts keys create $GOOGLE_APPLICATION_CREDENTIALS \ - --iam-account=$SERVICE_ACCOUNT_ID@$PROJECT + --iam-account=$SERVICE_ACCOUNT_ID@$PROJECT.iam.gserviceaccount.com # Create a GCP bucket for worker image. export BUCKET=$PROJECT-worker-bucket @@ -72,7 +92,7 @@ Build and upload the worker image: ```bash # Clone the repository -git clone https://github.com/antmicro/github-actions-runner-scalerunner.git +git clone --recursive https://github.com/antmicro/github-actions-runner-scalerunner.git cd github-actions-runner-scalerunner # Compile bzImage @@ -82,21 +102,24 @@ cd buildroot && make BR2_EXTERNAL=../overlay/ scalenode_gcp_defconfig && make ./make_gcp_image.sh ``` -1. Save the bucket's IAM policy to a temporary (arbitrary) JSON file -``` +### Adjust Service Account priviliges before uploading built disk to GCP + +Save the bucket's IAM policy to a temporary (arbitrary) JSON file +```bash gsutil iam get gs://$BUCKET > /arbitrary/path/file.json ``` -2. Get the project name and default service account email address. Adjust filter accordingly if a different service account is used -``` +Get the project name and default service account email address. Adjust filter accordingly if a different service account is used +```bash export PROJECT=$(gcloud config get-value project) -export SA=$(gcloud iam service-accounts list --filter=default | grep -E -o '[a-z0-9._%+-]+@[a-z0-9.-]+(\.[a-z0-9._%+-]+)?[a-z]{2,4}') -``` -3. Get the absolute path of the Bucket config file +export SA=$(gcloud iam service-accounts list --filter=default | + grep -E -o '[a-z0-9._%+-]+@[a-z0-9.-]+(\.[a-z0-9._%+-]+)?[a-z]{2,4}') ``` +Get the absolute path of the Bucket config file +```bash export BUCKET_FILE=/arbitrary/path/file.json ``` -4. Using the `sed` utility to insert required permissions associated with the bucket -``` +Using the `sed` utility to insert required permissions associated with the bucket +```bash sed -i 's/"bindings": \[/"bindings": \[\ {\ "members": \[\ @@ -107,16 +130,17 @@ sed -i 's/"bindings": \[/"bindings": \[\ "role": "roles\/storage.legacyBucketOwner"\ \},/' $BUCKET_FILE ``` -5. Upload the modified bucket file back to GCloud -``` +Upload the modified bucket file back to GCloud +```bash gsutil iam set $BUCKET_FILE gs://$BUCKET ``` -# Upload the resulting tar archive +Upload the resulting tar archive + +```bash ./upload_gcp_image.sh $PROJECT $BUCKET ``` -Setup virtual infrastructure using Terraform: - +### Setup virtual infrastructure using Terraform: ```bash git clone https://github.com/antmicro/github-actions-runner-terraform.git terraform init && terraform apply @@ -125,7 +149,7 @@ terraform init && terraform apply Connect to the coordinator instance created in the previous step: ```bash -gcloud compute --zone ssh +gcloud compute ssh gha-runner-coordinator --zone=us-west1-a ``` Install and configure the runner on the coordinator instance: From 5dd7c58b5463ceed99f66efd23922e6089a6790d Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Fri, 1 Oct 2021 00:06:52 -0400 Subject: [PATCH 04/11] Create refresh.sh --- refresh.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 refresh.sh diff --git a/refresh.sh b/refresh.sh new file mode 100644 index 00000000000..de5f9555c31 --- /dev/null +++ b/refresh.sh @@ -0,0 +1,19 @@ +# Stop instance before deleting +gcloud compute instances stop instance-2 + +# Delete then create instance +gcloud compute instances delete instance-2 + +gcloud compute instances create instance-2 --project=catx-ext-umich \ +--zone=us-central1-a \ +--machine-type=e2-standard-32 \ +--network-interface=network-tier=PREMIUM,subnet=default \ +--maintenance-policy=MIGRATE \ +--service-account=terraform-runner@catx-ext-umich.iam.gserviceaccount.com \ +--scopes=https://www.googleapis.com/auth/cloud-platform \ +--create-disk=auto-delete=yes,boot=yes,device-name=instance-2, \ +image=projects/debian-cloud/global/images/debian-10-buster-v20210916, \ +mode=rw,size=256, \ +type=projects/catx-ext-umich/zones/us-central1-a/diskTypes/pd-balanced \ +--no-shielded-secure-boot --shielded-vtpm \ +--shielded-integrity-monitoring --reservation-affinity=any From b1025b3a7f2baf5830b24849738ee890709616d1 Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Fri, 1 Oct 2021 00:16:38 -0400 Subject: [PATCH 05/11] Update and rename refresh.sh to setup.sh --- refresh.sh | 19 ------------------- setup.sh | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 19 deletions(-) delete mode 100644 refresh.sh create mode 100644 setup.sh diff --git a/refresh.sh b/refresh.sh deleted file mode 100644 index de5f9555c31..00000000000 --- a/refresh.sh +++ /dev/null @@ -1,19 +0,0 @@ -# Stop instance before deleting -gcloud compute instances stop instance-2 - -# Delete then create instance -gcloud compute instances delete instance-2 - -gcloud compute instances create instance-2 --project=catx-ext-umich \ ---zone=us-central1-a \ ---machine-type=e2-standard-32 \ ---network-interface=network-tier=PREMIUM,subnet=default \ ---maintenance-policy=MIGRATE \ ---service-account=terraform-runner@catx-ext-umich.iam.gserviceaccount.com \ ---scopes=https://www.googleapis.com/auth/cloud-platform \ ---create-disk=auto-delete=yes,boot=yes,device-name=instance-2, \ -image=projects/debian-cloud/global/images/debian-10-buster-v20210916, \ -mode=rw,size=256, \ -type=projects/catx-ext-umich/zones/us-central1-a/diskTypes/pd-balanced \ ---no-shielded-secure-boot --shielded-vtpm \ ---shielded-integrity-monitoring --reservation-affinity=any diff --git a/setup.sh b/setup.sh new file mode 100644 index 00000000000..39c38316edb --- /dev/null +++ b/setup.sh @@ -0,0 +1,49 @@ +# Setup prerequisites +sudo apt install build-essential -y +sudo apt install git wget unzip rsync bc \ + libelf-dev autotools-dev automake \ + gcc-multilib texinfo dosfstools mtools -y + +# Install Go for compiling BzImage +cd ~ +wget https://golang.org/dl/go1.16.5.linux-amd64.tar.gz && \ + sudo tar -xzf go1.16.5.linux-amd64.tar.gz -C /usr/local/ && \ + export PATH=$PATH:/usr/local/go/bin + +# Install Terraform +sudo apt-get install software-properties-common -y +curl -fsSL https://apt.releases.hashicorp.com/gpg | + sudo apt-key add - +sudo apt-add-repository "deb [arch=$(dpkg --print-architecture)] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +sudo apt update && sudo apt install terraform -y + +# Install Google Cloud SDK +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | + sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + +sudo apt-get install apt-transport-https ca-certificates gnupg -y +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | + sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + +sudo apt-get update && sudo apt-get install google-cloud-sdk -y + +# Rebuild image +git clone --recursive \ + https://github.com/antmicro/github-actions-runner-scalerunner.git && \ + cd github-actions-runner-scalerunner/buildroot && \ + make BR2_EXTERNAL=../overlay/ scalenode_gcp_defconfig && \ + make + +export PROJECT=catx-ext-umich && \ + export BUCKET=$PROJECT-worker-bucket + +# Make and upload image +cd ../ && \ + ./make_gcp_image.sh && \ + ./upload_gcp_image.sh $PROJECT $BUCKET + + +export name=$(gcloud compute instances list | grep gha | awk '{print $1}') && \ + export zone=$(gcloud compute instances list | grep gha | awk '{print $2}') && \ + cat coor.sh | gcloud compute ssh $name --zone=$zone + From 200f65b4837a2a0ade575aaa05962e8feda68df9 Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Fri, 1 Oct 2021 00:29:13 -0400 Subject: [PATCH 06/11] Create coor.sh --- coor.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 coor.sh diff --git a/coor.sh b/coor.sh new file mode 100644 index 00000000000..98eb9729d0c --- /dev/null +++ b/coor.sh @@ -0,0 +1,12 @@ +sudo -i -u runner bash +cd ~/github-actions-runner + +export REPOSITORY_ORG=idea-fasoc \ + export REPOSITORY_NAME=OpenFASOC \ + export TOKEN= \ + export SLOTS=1 \ + export SCALE=1 + +./config.sh --url https://github.com/$REPOSITORY_ORG/$REPOSITORY_NAME \ + --token $TOKEN \ + --num $SLOTS From 02426e9f46086d7e362909b64408a9e8031e9eea Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Fri, 1 Oct 2021 00:47:35 -0400 Subject: [PATCH 07/11] Update README.md --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e448f853a8..1272a97641d 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,49 @@ The repositories listed below contain the definitions of the required components For convenience, an [installation script](https://raw.githubusercontent.com/antmicro/runner/vm-runners/scripts/install.sh) is available that installs dependencies, configures the system, clones the repository and builds the runner. + ## Installation and configuration -The manual below assumes that Debian Buster is used to deploy the runner. +### catx-ext-umich specific procedures + +Start from scratch by following the procedure below: + +``` bash +gcloud compute instances stop instance-2 + +gcloud compute instances delete instance-2 + +gcloud compute instances create instance-2 --project=catx-ext-umich \ +--zone=us-central1-a \ +--machine-type=e2-standard-32 \ +--network-interface=network-tier=PREMIUM,subnet=default \ +--maintenance-policy=MIGRATE \ +--service-account=terraform-runner@catx-ext-umich.iam.gserviceaccount.com \ +--scopes=https://www.googleapis.com/auth/cloud-platform \ +--create-disk=auto-delete=yes,boot=yes,device-name=instance-2, \ +image=projects/debian-cloud/global/images/debian-10-buster-v20210916, \ +mode=rw,size=256, \ +type=projects/catx-ext-umich/zones/us-central1-a/diskTypes/pd-balanced \ +--no-shielded-secure-boot --shielded-vtpm \ +--shielded-integrity-monitoring --reservation-affinity=any + +# Delete previous image archive in bucket +# TODO: How to delete uploaded image (Not just in bucket) +gsutil rm gs://catx-ext-umich-worker-bucket/scalenode-9e1d63d.tar.gz + +# Connect to refreshed instance +gcloud compute ssh instance-2 --ssh-flag="-ServerAliveInterval=30" +``` + +Before running setup.sh make sure that the following is done: +- The archive in the bucket and corresponding image is deleted +- Fill in the TOKEN in `coor.sh` + ### Host prerequisites +The manual below assumes that Debian Buster is used to deploy the runner. + The following packages must be installed: * `build-essential` From 48e864e00e5118e2fbd084c0cd1f2e8bd270aea2 Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Fri, 1 Oct 2021 00:59:15 -0400 Subject: [PATCH 08/11] Update README.md --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1272a97641d..422dabae150 100644 --- a/README.md +++ b/README.md @@ -26,19 +26,21 @@ gcloud compute instances stop instance-2 gcloud compute instances delete instance-2 -gcloud compute instances create instance-2 --project=catx-ext-umich \ +gcloud compute instances create instance-2 \ +--project=catx-ext-umich \ --zone=us-central1-a \ --machine-type=e2-standard-32 \ --network-interface=network-tier=PREMIUM,subnet=default \ --maintenance-policy=MIGRATE \ --service-account=terraform-runner@catx-ext-umich.iam.gserviceaccount.com \ --scopes=https://www.googleapis.com/auth/cloud-platform \ ---create-disk=auto-delete=yes,boot=yes,device-name=instance-2, \ -image=projects/debian-cloud/global/images/debian-10-buster-v20210916, \ -mode=rw,size=256, \ -type=projects/catx-ext-umich/zones/us-central1-a/diskTypes/pd-balanced \ ---no-shielded-secure-boot --shielded-vtpm \ ---shielded-integrity-monitoring --reservation-affinity=any +--create-disk=auto-delete=yes,boot=yes,device-name=instance-2,\ +image=projects/debian-cloud/global/images/debian-10-buster-v20210916,\ +mode=rw,size=256,type=projects/catx-ext-umich/zones/us-central1-a/diskTypes/pd-balanced \ +--no-shielded-secure-boot \ +--shielded-vtpm \ +--shielded-integrity-monitoring \ +--reservation-affinity=any # Delete previous image archive in bucket # TODO: How to delete uploaded image (Not just in bucket) From 9c3f9e4235dc086aed7aae445bbe1fd10ebd1750 Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Fri, 1 Oct 2021 01:13:55 -0400 Subject: [PATCH 09/11] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 422dabae150..96677ef8dac 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,10 @@ gsutil rm gs://catx-ext-umich-worker-bucket/scalenode-9e1d63d.tar.gz gcloud compute ssh instance-2 --ssh-flag="-ServerAliveInterval=30" ``` -Before running setup.sh make sure that the following is done: +Before running `setup.sh` make sure that the following is done: +- Install `git` upon ssh to the created instance - The archive in the bucket and corresponding image is deleted -- Fill in the TOKEN in `coor.sh` +- Fill in the `TOKEN` in `coor.sh` ### Host prerequisites From d17b4bfc1250fd6f857973def39e830d17811928 Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Fri, 1 Oct 2021 02:13:02 -0400 Subject: [PATCH 10/11] Update setup.sh --- setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 39c38316edb..2ed6fd14ebb 100644 --- a/setup.sh +++ b/setup.sh @@ -42,7 +42,8 @@ cd ../ && \ ./make_gcp_image.sh && \ ./upload_gcp_image.sh $PROJECT $BUCKET - +# ssh into coordinator instance to setup runner and run conifg +cd ~/runner export name=$(gcloud compute instances list | grep gha | awk '{print $1}') && \ export zone=$(gcloud compute instances list | grep gha | awk '{print $2}') && \ cat coor.sh | gcloud compute ssh $name --zone=$zone From 4d675d1758f60d513d2ff52c29d57e707ed96166 Mon Sep 17 00:00:00 2001 From: Ke-Haur Taur Date: Mon, 4 Oct 2021 12:55:07 -0400 Subject: [PATCH 11/11] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 96677ef8dac..e9f1720b51b 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,11 @@ gcloud compute ssh instance-2 --ssh-flag="-ServerAliveInterval=30" Before running `setup.sh` make sure that the following is done: - Install `git` upon ssh to the created instance - The archive in the bucket and corresponding image is deleted + - Screenshot to-be included - Fill in the `TOKEN` in `coor.sh` + - The `TOKEN` can be found here: https://github.com/idea-fasoc/OpenFASOC/settings/actions/runners/new?arch=x64&os=linux under `configure`. The `TOKEN` expires in a hour. +After everything is setup, run `setup.sh` to build everything from scratch. (This script assumes that all virtual infrastructure created by Terraform does not need to be re-configured.) ### Host prerequisites