Skip to content

Commit 0e93d21

Browse files
carlcsaposs-canonicalcarlcsaposs-canonical
authored
Use charmcraft test & concierge (#609)
Ported from canonical/mysql-router-k8s-operator#379
1 parent 1072683 commit 0e93d21

File tree

77 files changed

+2135
-1263
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+2135
-1263
lines changed

‎.github/workflows/ci.yaml

+4-35
Original file line numberDiff line numberDiff line change
@@ -63,45 +63,14 @@ jobs:
6363
uses: canonical/data-platform-workflows/.github/workflows/[email protected]
6464

6565
integration-test:
66-
strategy:
67-
fail-fast: false
68-
matrix:
69-
juju:
70-
- agent: 2.9.51 # renovate: juju-agent-pin-minor
71-
libjuju: ^2
72-
allure_on_amd64: false
73-
- agent: 3.6.2 # renovate: juju-agent-pin-minor
74-
allure_on_amd64: true
75-
architecture:
76-
- amd64
77-
include:
78-
- juju:
79-
agent: 3.6.2 # renovate: juju-agent-pin-minor
80-
allure_on_amd64: true
81-
architecture: arm64
82-
name: Integration | ${{ matrix.juju.agent }} | ${{ matrix.architecture }}
66+
name: Integration test charm
8367
needs:
8468
- lint
8569
- unit-test
8670
- build
87-
uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v29.0.0
71+
uses: ./.github/workflows/integration_test.yaml
8872
with:
8973
artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
90-
architecture: ${{ matrix.architecture }}
91-
cloud: lxd
92-
juju-agent-version: ${{ matrix.juju.agent }}
93-
libjuju-version-constraint: ${{ matrix.juju.libjuju }}
94-
_beta_allure_report: ${{ matrix.juju.allure_on_amd64 && matrix.architecture == 'amd64' }}
95-
secrets:
96-
# GitHub appears to redact each line of a multi-line secret
97-
# Avoid putting `{` or `}` on a line by itself so that it doesn't get redacted in logs
98-
integration-test: |
99-
{ "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
100-
"AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
101-
"GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
102-
"GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
103-
"UBUNTU_PRO_TOKEN" : "${{ secrets.UBUNTU_PRO_TOKEN }}",
104-
"LANDSCAPE_ACCOUNT_NAME": "${{ secrets.LANDSCAPE_ACCOUNT_NAME }}",
105-
"LANDSCAPE_REGISTRATION_KEY": "${{ secrets.LANDSCAPE_REGISTRATION_KEY }}", }
74+
secrets: inherit
10675
permissions:
107-
contents: write # Needed for Allure Report beta
76+
contents: write # Needed for Allure Report

‎.github/workflows/integration_test.yaml

+316
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
on:
2+
workflow_call:
3+
inputs:
4+
artifact-prefix:
5+
description: |
6+
Prefix for charm package GitHub artifact(s)
7+
8+
Use canonical/data-platform-workflows build_charm.yaml to build the charm(s)
9+
required: true
10+
type: string
11+
12+
jobs:
13+
collect-integration-tests:
14+
name: Collect integration test spread jobs
15+
runs-on: ubuntu-latest
16+
timeout-minutes: 5
17+
steps:
18+
- name: Checkout
19+
uses: actions/checkout@v4
20+
- name: Set up environment
21+
run: |
22+
sudo snap install charmcraft --classic
23+
pipx install tox poetry
24+
- name: Collect spread jobs
25+
id: collect-jobs
26+
shell: python
27+
run: |
28+
import json
29+
import os
30+
import subprocess
31+
32+
spread_jobs = (
33+
subprocess.run(
34+
["charmcraft", "test", "--list", "github-ci"], capture_output=True, check=True, text=True
35+
)
36+
.stdout.strip()
37+
.split("\n")
38+
)
39+
jobs = []
40+
for job in spread_jobs:
41+
# Example `job`: "github-ci:ubuntu-24.04:tests/spread/test_charm.py:juju36"
42+
_, runner, task, variant = job.split(":")
43+
# Example: "test_charm.py"
44+
task = task.removeprefix("tests/spread/")
45+
if runner.endswith("-arm"):
46+
architecture = "arm64"
47+
else:
48+
architecture = "amd64"
49+
# Example: "test_charm.py:juju36 | amd64"
50+
name = f"{task}:{variant} | {architecture}"
51+
# ":" character not valid in GitHub Actions artifact
52+
name_in_artifact = f"{task}-{variant}-{architecture}"
53+
jobs.append({
54+
"spread_job": job,
55+
"name": name,
56+
"name_in_artifact": name_in_artifact,
57+
"runner": runner,
58+
})
59+
output = f"jobs={json.dumps(jobs)}"
60+
print(output)
61+
with open(os.environ["GITHUB_OUTPUT"], "a") as file:
62+
file.write(output)
63+
- name: Generate Allure default test results
64+
if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }}
65+
run: tox run -e integration -- tests/integration --allure-default-dir=allure-default-results
66+
- name: Upload Allure default results
67+
# Default test results in case the integration tests time out or runner set up fails
68+
# (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test)
69+
if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }}
70+
uses: actions/upload-artifact@v4
71+
with:
72+
name: allure-default-results-integration-test
73+
path: allure-default-results/
74+
if-no-files-found: error
75+
outputs:
76+
jobs: ${{ steps.collect-jobs.outputs.jobs }}
77+
78+
integration-test:
79+
strategy:
80+
fail-fast: false
81+
matrix:
82+
job: ${{ fromJSON(needs.collect-integration-tests.outputs.jobs) }}
83+
name: ${{ matrix.job.name }}
84+
needs:
85+
- collect-integration-tests
86+
runs-on: ${{ matrix.job.runner }}
87+
timeout-minutes: 217 # Sum of steps `timeout-minutes` + 5
88+
steps:
89+
- name: Free up disk space
90+
timeout-minutes: 1
91+
run: |
92+
printf '\nDisk usage before cleanup\n'
93+
df --human-readable
94+
# Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
95+
rm -r /opt/hostedtoolcache/
96+
printf '\nDisk usage after cleanup\n'
97+
df --human-readable
98+
- name: Checkout
99+
timeout-minutes: 3
100+
uses: actions/checkout@v4
101+
- name: Set up environment
102+
timeout-minutes: 5
103+
run: sudo snap install charmcraft --classic
104+
# TODO: remove when https://github.com/canonical/charmcraft/issues/2105 and
105+
# https://github.com/canonical/charmcraft/issues/2130 fixed
106+
- run: |
107+
sudo snap install go --classic
108+
go install github.com/snapcore/spread/cmd/spread@latest
109+
- name: Download packed charm(s)
110+
timeout-minutes: 5
111+
uses: actions/download-artifact@v4
112+
with:
113+
pattern: ${{ inputs.artifact-prefix }}-*
114+
merge-multiple: true
115+
- name: Run spread job
116+
timeout-minutes: 180
117+
id: spread
118+
# TODO: replace with `charmcraft test` when
119+
# https://github.com/canonical/charmcraft/issues/2105 and
120+
# https://github.com/canonical/charmcraft/issues/2130 fixed
121+
run: ~/go/bin/spread -vv -artifacts=artifacts '${{ matrix.job.spread_job }}'
122+
env:
123+
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
124+
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
125+
GCP_ACCESS_KEY: ${{ secrets.GCP_ACCESS_KEY }}
126+
GCP_SECRET_KEY: ${{ secrets.GCP_SECRET_KEY }}
127+
UBUNTU_PRO_TOKEN: ${{ secrets.UBUNTU_PRO_TOKEN }}
128+
LANDSCAPE_ACCOUNT_NAME: ${{ secrets.LANDSCAPE_ACCOUNT_NAME }}
129+
LANDSCAPE_REGISTRATION_KEY: ${{ secrets.LANDSCAPE_REGISTRATION_KEY }}
130+
- name: Upload Allure results
131+
timeout-minutes: 3
132+
# Only upload results from one spread system & one spread variant
133+
# Allure can only process one result per pytest test ID. If parameterization is done via
134+
# spread instead of pytest, there will be overlapping pytest test IDs.
135+
if: ${{ (success() || (failure() && steps.spread.outcome == 'failure')) && startsWith(matrix.job.spread_job, 'github-ci:ubuntu-24.04:') && endsWith(matrix.job.spread_job, ':juju36') && github.event_name == 'schedule' && github.run_attempt == '1' }}
136+
uses: actions/upload-artifact@v4
137+
with:
138+
name: allure-results-integration-test-${{ matrix.job.name_in_artifact }}
139+
path: artifacts/${{ matrix.job.spread_job }}/allure-results/
140+
if-no-files-found: error
141+
- timeout-minutes: 1
142+
if: ${{ success() || (failure() && steps.spread.outcome == 'failure') }}
143+
run: snap list
144+
- name: Select model
145+
timeout-minutes: 1
146+
# `!contains(matrix.job.spread_job, 'juju29')` workaround for juju 2 error:
147+
# "ERROR cannot acquire lock file to read controller concierge-microk8s: unable to open
148+
# /tmp/juju-store-lock-3635383939333230: permission denied"
149+
# Unable to workaround error with `sudo rm /tmp/juju-*`
150+
if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }}
151+
id: juju-switch
152+
run: |
153+
# sudo needed since spread runs scripts as root
154+
# "testing" is default model created by concierge
155+
sudo juju switch testing
156+
mkdir ~/logs/
157+
- name: juju status
158+
timeout-minutes: 1
159+
if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }}
160+
run: sudo juju status --color --relations | tee ~/logs/juju-status.txt
161+
- name: juju debug-log
162+
timeout-minutes: 3
163+
if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }}
164+
run: sudo juju debug-log --color --replay --no-tail | tee ~/logs/juju-debug-log.txt
165+
- name: jhack tail
166+
timeout-minutes: 3
167+
if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }}
168+
run: sudo jhack tail --printer raw --replay --no-watch | tee ~/logs/jhack-tail.txt
169+
- name: Upload logs
170+
timeout-minutes: 5
171+
if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }}
172+
uses: actions/upload-artifact@v4
173+
with:
174+
name: logs-integration-test-${{ matrix.job.name_in_artifact }}
175+
path: ~/logs/
176+
if-no-files-found: error
177+
- name: Disk usage
178+
timeout-minutes: 1
179+
if: ${{ success() || (failure() && steps.spread.outcome == 'failure') }}
180+
run: df --human-readable
181+
182+
allure-report:
183+
# TODO future improvement: use concurrency group for job
184+
name: Publish Allure report
185+
if: ${{ !cancelled() && github.event_name == 'schedule' && github.run_attempt == '1' }}
186+
needs:
187+
- integration-test
188+
runs-on: ubuntu-latest
189+
timeout-minutes: 5
190+
steps:
191+
- name: Download Allure
192+
# Following instructions from https://allurereport.org/docs/install-for-linux/#install-from-a-deb-package
193+
run: gh release download --repo allure-framework/allure2 --pattern 'allure_*.deb'
194+
env:
195+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
196+
- name: Install Allure
197+
run: |
198+
sudo apt-get update
199+
sudo apt-get install ./allure_*.deb -y
200+
# For first run, manually create branch with no history
201+
# (e.g.
202+
# git checkout --orphan gh-pages-beta
203+
# git rm -rf .
204+
# touch .nojekyll
205+
# git add .nojekyll
206+
# git commit -m "Initial commit"
207+
# git push origin gh-pages-beta
208+
# )
209+
- name: Checkout GitHub pages branch
210+
uses: actions/checkout@v4
211+
with:
212+
ref: gh-pages-beta
213+
path: repo/
214+
- name: Download default test results
215+
# Default test results in case the integration tests time out or runner set up fails
216+
# (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test)
217+
uses: actions/download-artifact@v4
218+
with:
219+
path: allure-default-results/
220+
name: allure-default-results-integration-test
221+
- name: Download test results
222+
uses: actions/download-artifact@v4
223+
with:
224+
path: allure-results/
225+
pattern: allure-results-integration-test-*
226+
merge-multiple: true
227+
- name: Combine Allure default results & actual results
228+
# For every test: if actual result available, use that. Otherwise, use default result
229+
# So that, if actual result not available, Allure report will show "unknown"/"failed" test result
230+
# instead of omitting the test
231+
shell: python
232+
run: |
233+
import dataclasses
234+
import json
235+
import pathlib
236+
237+
238+
@dataclasses.dataclass(frozen=True)
239+
class Result:
240+
test_case_id: str
241+
path: pathlib.Path
242+
243+
def __eq__(self, other):
244+
if not isinstance(other, type(self)):
245+
return False
246+
return self.test_case_id == other.test_case_id
247+
248+
249+
actual_results = pathlib.Path("allure-results")
250+
default_results = pathlib.Path("allure-default-results")
251+
252+
results: dict[pathlib.Path, set[Result]] = {
253+
actual_results: set(),
254+
default_results: set(),
255+
}
256+
for directory, results_ in results.items():
257+
for path in directory.glob("*-result.json"):
258+
with path.open("r") as file:
259+
id_ = json.load(file)["testCaseId"]
260+
results_.add(Result(id_, path))
261+
262+
actual_results.mkdir(exist_ok=True)
263+
264+
missing_results = results[default_results] - results[actual_results]
265+
for default_result in missing_results:
266+
# Move to `actual_results` directory
267+
default_result.path.rename(actual_results / default_result.path.name)
268+
- name: Load test report history
269+
run: |
270+
if [[ -d repo/_latest/history/ ]]
271+
then
272+
echo 'Loading history'
273+
cp -r repo/_latest/history/ allure-results/
274+
fi
275+
- name: Create executor.json
276+
shell: python
277+
run: |
278+
# Reverse engineered from https://github.com/simple-elf/allure-report-action/blob/eca283b643d577c69b8e4f048dd6cd8eb8457cfd/entrypoint.sh
279+
import json
280+
281+
DATA = {
282+
"name": "GitHub Actions",
283+
"type": "github",
284+
"buildOrder": ${{ github.run_number }}, # TODO future improvement: use run ID
285+
"buildName": "Run ${{ github.run_id }}",
286+
"buildUrl": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",
287+
"reportUrl": "../${{ github.run_number }}/",
288+
}
289+
with open("allure-results/executor.json", "w") as file:
290+
json.dump(DATA, file)
291+
- name: Generate Allure report
292+
run: allure generate
293+
- name: Create index.html
294+
shell: python
295+
run: |
296+
DATA = f"""<!DOCTYPE html>
297+
<meta charset="utf-8">
298+
<meta http-equiv="cache-control" content="no-cache">
299+
<meta http-equiv="refresh" content="0; url=${{ github.run_number }}">
300+
"""
301+
with open("repo/index.html", "w") as file:
302+
file.write(DATA)
303+
- name: Update GitHub pages branch
304+
working-directory: repo/
305+
# TODO future improvement: commit message
306+
run: |
307+
mkdir '${{ github.run_number }}'
308+
rm -f _latest
309+
ln -s '${{ github.run_number }}' _latest
310+
cp -r ../allure-report/. _latest/
311+
git add .
312+
git config user.name "GitHub Actions"
313+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
314+
git commit -m "Allure report ${{ github.run_number }}"
315+
# Uses token set in checkout step
316+
git push origin gh-pages-beta

‎.github/workflows/release.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
uses: ./.github/workflows/ci.yaml
1414
secrets: inherit
1515
permissions:
16-
contents: write # Needed for Allure Report beta
16+
contents: write # Needed for Allure Report
1717

1818
release-libraries:
1919
name: Release libraries

‎CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ poetry install
4141
tox run -e format # update your code according to linting rules
4242
tox run -e lint # code style
4343
tox run -e unit # unit tests
44-
tox run -e integration # integration tests
44+
charmcraft test lxd-vm: # integration tests
4545
tox # runs 'lint' and 'unit' environments
4646
```
4747

‎concierge.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
juju:
2+
model-defaults:
3+
logging-config: <root>=INFO; unit=DEBUG
4+
providers:
5+
lxd:
6+
enable: true
7+
bootstrap: true
8+
host:
9+
snaps:
10+
jhack:
11+
channel: latest/edge
12+
connections:
13+
- jhack:dot-local-share-juju snapd

0 commit comments

Comments
 (0)