Skip to content

Commit 2289520

Browse files
TwistedTwiglegTwistedTwigleg
authored
CI Adjustment (#290)
Adjusts how CI runs so it uses OpenID and the following: * Adjusts the PubSub JS sample to use the same logic as the Typescript PubSub JS. The old code was not stable in CI, sometimes randomly getting uncaught promise exceptions. * Adds a new Jobs sample! It works exactly the same as the Java V2 Jobs sample, but for Javascript. * Adds a new command line argument to detect if a sample is running in CI. Adjusted Fleet provisioning not to spam quite as much data when in CI and the Shadow sample to automatically update the Shadow document when in CI.
1 parent 4342ce6 commit 2289520

File tree

19 files changed

+1329
-117
lines changed

19 files changed

+1329
-117
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 174 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,38 +13,207 @@ env:
1313
PACKAGE_NAME: aws-iot-device-sdk-js-v2
1414
LINUX_BASE_IMAGE: ubuntu-16-x64
1515
RUN: ${{ github.run_id }}-${{ github.run_number }}
16-
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
17-
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
18-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }}
19-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }}
2016
AWS_DEFAULT_REGION: us-east-1
17+
DA_TOPIC: test/da
18+
DA_SHADOW_PROPERTY: datest
19+
DA_SHADOW_VALUE_SET: ON
20+
DA_SHADOW_VALUE_DEFAULT: OFF
21+
CI_IOT_CONTAINERS: ${{ secrets.AWS_CI_IOT_CONTAINERS }}
22+
CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }}
23+
CI_CUSTOM_AUTHORIZER_ROLE: ${{ secrets.AWS_CI_CUSTOM_AUTHORIZER_ROLE }}
24+
CI_SHADOW_ROLE: ${{ secrets.AWS_CI_SHADOW_ROLE }}
25+
CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }}
26+
CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }}
27+
CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }}
2128

2229
jobs:
2330

2431
al2:
2532
runs-on: ubuntu-latest
33+
permissions:
34+
id-token: write # This is required for requesting the JWT
2635
steps:
27-
# We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
36+
- name: configure AWS credentials (containers)
37+
uses: aws-actions/configure-aws-credentials@v1
38+
with:
39+
role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
40+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
41+
# We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
2842
- name: Build ${{ env.PACKAGE_NAME }}
2943
run: |
3044
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
3145
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-al2-x64 build -p ${{ env.PACKAGE_NAME }}
46+
3247
windows:
3348
runs-on: windows-latest
49+
permissions:
50+
id-token: write # This is required for requesting the JWT
3451
steps:
3552
- name: Build ${{ env.PACKAGE_NAME }}
3653
run: |
3754
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
3855
python builder.pyz build -p ${{ env.PACKAGE_NAME }}
56+
- name: Running samples in CI setup
57+
run: |
58+
python -m pip install boto3
59+
- name: configure AWS credentials (PubSub)
60+
uses: aws-actions/configure-aws-credentials@v1
61+
with:
62+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
63+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
64+
- name: run PubSub sample
65+
run: |
66+
python ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file "./aws-iot-device-sdk-js-v2/samples/node/pub_sub" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
67+
- name: run Windows Certificate Connect sample
68+
run: |
69+
python ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file "./aws-iot-device-sdk-js-v2/samples/node/windows_cert_connect" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true
70+
- name: configure AWS credentials (Device Advisor)
71+
uses: aws-actions/configure-aws-credentials@v1
72+
with:
73+
role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
74+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
75+
- name: run DeviceAdvisor
76+
run: |
77+
cd ./aws-iot-device-sdk-js-v2
78+
python ./deviceadvisor/script/DATestRun.py
3979
4080
osx:
4181
runs-on: macos-latest
82+
permissions:
83+
id-token: write # This is required for requesting the JWT
4284
steps:
4385
- name: Build ${{ env.PACKAGE_NAME }}
4486
run: |
4587
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
4688
chmod a+x builder
4789
./builder build -p ${{ env.PACKAGE_NAME }}
90+
- name: Running samples in CI setup
91+
run: |
92+
python3 -m pip install boto3
93+
- name: configure AWS credentials (PubSub)
94+
uses: aws-actions/configure-aws-credentials@v1
95+
with:
96+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
97+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
98+
- name: run PubSub sample
99+
run: |
100+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/pub_sub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
101+
- name: configure AWS credentials (Device Advisor)
102+
uses: aws-actions/configure-aws-credentials@v1
103+
with:
104+
role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
105+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
106+
- name: run DeviceAdvisor
107+
run: |
108+
cd ./aws-iot-device-sdk-js-v2
109+
python3 ./deviceadvisor/script/DATestRun.py
110+
111+
linux:
112+
runs-on: ubuntu-20.04 # latest
113+
permissions:
114+
id-token: write # This is required for requesting the JWT
115+
steps:
116+
- name: Build ${{ env.PACKAGE_NAME }}
117+
run: |
118+
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
119+
chmod a+x builder
120+
./builder build -p ${{ env.PACKAGE_NAME }}
121+
- name: Running samples in CI setup
122+
run: |
123+
python3 -m pip install boto3
124+
- name: configure AWS credentials (PubSub)
125+
uses: aws-actions/configure-aws-credentials@v1
126+
with:
127+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
128+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
129+
- name: run PubSub sample
130+
run: |
131+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/pub_sub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
132+
- name: configure AWS credentials (Device Advisor)
133+
uses: aws-actions/configure-aws-credentials@v1
134+
with:
135+
role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
136+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
137+
- name: run DeviceAdvisor
138+
run: |
139+
cd ./aws-iot-device-sdk-js-v2
140+
python3 ./deviceadvisor/script/DATestRun.py
141+
142+
# Runs the samples and ensures that everything is working
143+
linux-smoke-tests:
144+
runs-on: ubuntu-latest
145+
permissions:
146+
id-token: write # This is required for requesting the JWT
147+
steps:
148+
- name: Build ${{ env.PACKAGE_NAME }}
149+
run: |
150+
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
151+
chmod a+x builder
152+
./builder build -p ${{ env.PACKAGE_NAME }}
153+
- name: Running samples in CI setup
154+
run: |
155+
python3 -m pip install boto3
156+
sudo apt-get update -y
157+
sudo apt-get install softhsm -y
158+
softhsm2-util --version
159+
- name: configure AWS credentials (Connect and PubSub)
160+
uses: aws-actions/configure-aws-credentials@v1
161+
with:
162+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
163+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
164+
- name: run Basic Connect sample
165+
run: |
166+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/basic_connect' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
167+
- name: run Websocket Connect sample
168+
run: |
169+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/websocket_connect' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_arguments '--signing_region us-east-1'
170+
- name: run PubSub sample
171+
run: |
172+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/pub_sub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
173+
- name: run PubSub JS sample
174+
run: |
175+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/pub_sub_js' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --node_cmd "node index.js"
176+
- name: run PKCS11 Connect sample
177+
run: |
178+
mkdir -p /tmp/tokens
179+
export SOFTHSM2_CONF=/tmp/softhsm2.conf
180+
echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf
181+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/pkcs11_connect' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key'
182+
- name: configure AWS credentials (Custom Authorizer)
183+
uses: aws-actions/configure-aws-credentials@v1
184+
with:
185+
role-to-assume: ${{ env.CI_CUSTOM_AUTHORIZER_ROLE }}
186+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
187+
- name: run CustomAuthorizerConnect sample
188+
run: |
189+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/custom_authorizer_connect' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_custom_authorizer_name 'ci/CustomAuthorizer/name' --sample_secret_custom_authorizer_password 'ci/CustomAuthorizer/password'
190+
- name: configure AWS credentials (Shadow)
191+
uses: aws-actions/configure-aws-credentials@v1
192+
with:
193+
role-to-assume: ${{ env.CI_SHADOW_ROLE }}
194+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
195+
- name: run Shadow sample
196+
run: |
197+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/shadow' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Shadow/cert' --sample_secret_private_key 'ci/Shadow/key' --sample_arguments '--thing_name CI_Shadow_Thing'
198+
- name: configure AWS credentials (Jobs)
199+
uses: aws-actions/configure-aws-credentials@v1
200+
with:
201+
role-to-assume: ${{ env.CI_JOBS_ROLE }}
202+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
203+
- name: run Jobs sample
204+
run: |
205+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/jobs' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Jobs/cert' --sample_secret_private_key 'ci/Jobs/key' --sample_arguments '--thing_name CI_Jobs_Thing'
206+
- name: configure AWS credentials (Fleet provisioning)
207+
uses: aws-actions/configure-aws-credentials@v1
208+
with:
209+
role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }}
210+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
211+
- name: run Fleet Provisioning sample
212+
run: |
213+
echo "Generating UUID for IoT thing"
214+
Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())")
215+
python3 ./aws-iot-device-sdk-js-v2/utils/run_sample_ci.py --language Javascript --sample_file './aws-iot-device-sdk-js-v2/samples/node/fleet_provisioning' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters {\"SerialNumber\":\"${Sample_UUID}\"}"
216+
python3 ./aws-iot-device-sdk-js-v2/utils/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
48217
49218
# check that docs can still build
50219
check-docs:

‎builder.json‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,5 @@
88
"build_steps": [
99
"npm install"
1010
],
11-
"test_steps": [
12-
"python3 -m pip install boto3",
13-
"python3 deviceadvisor/script/DATestRun.py"],
14-
"env": {
15-
"DA_TOPIC": "test/da",
16-
"DA_SHADOW_PROPERTY": "datest",
17-
"DA_SHADOW_VALUE_SET": "ON",
18-
"DA_SHADOW_VALUE_DEFAULT": "OFF",
19-
"DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs"
20-
}
11+
"env": {}
2112
}

‎codebuild/samples/connect-custom-authorizer-linux.sh‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎codebuild/samples/linux-smoke-tests.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ phases:
99
- echo Build started on `date`
1010
- $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh
1111
- $CODEBUILD_SRC_DIR/codebuild/samples/connect-linux.sh
12-
- $CODEBUILD_SRC_DIR/codebuild/samples/connect-custom-authorizer-linux.sh
1312
post_build:
1413
commands:
1514
- echo Build completed on `date`

‎deviceadvisor/script/DATestConfig.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
"tests" :["MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"],
33
"test_suite_ids" :
44
{
5-
"MQTT Connect" : "ejbdzmo3hf3v",
6-
"MQTT Publish" : "euw7favf6an4",
7-
"MQTT Subscribe" : "01o8vo6no7sd",
8-
"Shadow Publish" : "elztm2jebc1q",
9-
"Shadow Update" : "vuydgrbbbfce"
5+
"MQTT Connect" : "mxn32qkm8npn",
6+
"MQTT Publish" : "gcjhujhhz50p",
7+
"MQTT Subscribe" : "nyiuiwx5yxtj",
8+
"Shadow Publish" : "fttdr8ufljnf",
9+
"Shadow Update" : "ng9t8am2jnry"
1010
},
1111
"test_exe_path" :
1212
{

0 commit comments

Comments
 (0)