Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add informative pipeline step labels #43

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add informative pipeline step labels #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c8c71df
Remove two unused functions
gilles-peskine-arm Apr 21, 2022
68ddf82
Remove all.sh --list-components detection
gilles-peskine-arm Apr 21, 2022
055fabb
Put informative labels on shell steps
gilles-peskine-arm Apr 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 10 additions & 18 deletions vars/common.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,30 +158,22 @@ def get_branch_information() {

for (platform in linux_platforms) {
get_docker_image(platform)
def all_sh_help = sh(
available_all_sh_components[platform] = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--help"
platform, "./tests/scripts/all.sh", "--list-components"
),
label: "./tests/scripts/all.sh --list-components #$platform",
returnStdout: true
)
if (all_sh_help.contains('list-components')) {
available_all_sh_components[platform] = sh(
).trim().split('\n')
if (all_all_sh_components == []) {
all_all_sh_components = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--list-components"
platform, "./tests/scripts/all.sh",
"--list-all-components"
),
label: "./tests/scripts/all.sh --list-all-components",
returnStdout: true
).trim().split('\n')
if (all_all_sh_components == []) {
all_all_sh_components = sh(
script: docker_script(
platform, "./tests/scripts/all.sh",
"--list-all-components"
),
returnStdout: true
).trim().split('\n')
}
} else {
error('Pre Test Checks failed: Base branch out of date. Please rebase')
}
}
}
Expand Down Expand Up @@ -245,7 +237,7 @@ def maybe_notify_github(context, state, description) {
}

def archive_zipped_log_files(job_name) {
sh """\
sh label: "rename+compress *.log", script: """\
for i in *.log; do
[ -f "\$i" ] || break
mv "\$i" "$job_name-\$i"
Expand Down
122 changes: 23 additions & 99 deletions vars/gen_jobs.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def gen_simple_windows_jobs(label, script) {
checkout_repo.checkout_repo()
timeout(time: common.perJobTimeout.time,
unit: common.perJobTimeout.unit) {
bat script
bat script: script, label: label
}
}
} catch (err) {
Expand All @@ -50,86 +50,6 @@ def gen_simple_windows_jobs(label, script) {
return jobs
}

def gen_docker_jobs_foreach(label, platforms, compilers, script) {
def jobs = [:]

for (platform in platforms) {
for (compiler in compilers) {
def job_name = "${label}-${compiler}-${platform}"
def shell_script = sprintf(script, common.compiler_paths[compiler])
jobs[job_name] = {
node('container-host') {
try {
deleteDir()
common.get_docker_image(platform)
dir('src') {
checkout_repo.checkout_repo()
writeFile file: 'steps.sh', text: """\
#!/bin/sh
set -eux
ulimit -f 20971520
${shell_script}
"""
sh 'chmod +x steps.sh'
}
timeout(time: common.perJobTimeout.time,
unit: common.perJobTimeout.unit) {
try {
sh common.docker_script(
platform, "/var/lib/build/steps.sh"
)
} finally {
dir('src/tests/') {
common.archive_zipped_log_files(job_name)
}
}
}
} catch (err) {
failed_builds[job_name] = true
throw (err)
} finally {
deleteDir()
}
}
}
}
}
return jobs
}

def gen_node_jobs_foreach(label, platforms, compilers, script) {
def jobs = [:]

for (platform in platforms) {
for (compiler in compilers) {
def job_name = "${label}-${compiler}-${platform}"
def shell_script = sprintf(script, common.compiler_paths[compiler])
jobs[job_name] = {
node(platform) {
try {
deleteDir()
checkout_repo.checkout_repo()
shell_script = """\
ulimit -f 20971520
export PYTHON=/usr/local/bin/python2.7
""" + shell_script
timeout(time: common.perJobTimeout.time,
unit: common.perJobTimeout.unit) {
sh shell_script
}
} catch (err) {
failed_builds[job_name] = true
throw (err)
} finally {
deleteDir()
}
}
}
}
}
return jobs
}

def node_label_for_platform(platform) {
switch (platform) {
case ~/^(debian|ubuntu)(-.*)?/: return 'container-host';
Expand Down Expand Up @@ -234,12 +154,13 @@ ${extra_setup_code}
unit: common.perJobTimeout.unit) {
try {
if (use_docker) {
sh common.docker_script(
sh script: common.docker_script(
platform, "/var/lib/build/steps.sh"
)
), label: "docker run steps.sh #${job_name}"
} else {
dir('src') {
sh './steps.sh'
sh script: './steps.sh',
label: "docker run steps.sh #${job_name}"
}
}
} finally {
Expand Down Expand Up @@ -345,7 +266,8 @@ def gen_abi_api_checking_job(platform) {
/* The credentials here are the SSH credentials for accessing the repositories.
They are defined at {JENKINS_URL}/credentials */
withCredentials([sshUserPrivateKey(credentialsId: credentials_id, keyFileVariable: 'keyfile')]) {
sh "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}"
sh script: "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}",
label: "git fetch origin ${CHANGE_TARGET}"
}
writeFile file: 'steps.sh', text: """\
#!/bin/sh
Expand All @@ -363,9 +285,9 @@ scripts/abi_check.py -o FETCH_HEAD -n HEAD -s identifiers --brief
}
timeout(time: common.perJobTimeout.time,
unit: common.perJobTimeout.unit) {
sh common.docker_script(
sh script: common.docker_script(
platform, "/var/lib/build/steps.sh"
)
), label: "steps.sh #${job_name}"
}
} catch (err) {
failed_builds[job_name] = true
Expand Down Expand Up @@ -414,9 +336,9 @@ fi
timeout(time: common.perJobTimeout.time,
unit: common.perJobTimeout.unit) {
try {
sh common.docker_script(
sh script: common.docker_script(
platform, "/var/lib/build/steps.sh"
)
), label: "steps.sh #${job_name}"
dir('src') {
String coverage_log = readFile('coverage-summary.txt')
coverage_details['coverage'] = coverage_log.substring(
Expand Down Expand Up @@ -472,7 +394,7 @@ def gen_mbed_os_example_job(repo, branch, example, compiler, platform, raas) {
try {
deleteDir()
/* Create python virtual environment and install mbed tools */
sh """\
sh label: "virtualenv #${job_name}", script: """\
ulimit -f 20971520
virtualenv $WORKSPACE/mbed-venv
. $WORKSPACE/mbed-venv/bin/activate
Expand All @@ -486,7 +408,7 @@ pip install mbed-host-tests
/* If the job is targeting an example repo, then we wish to use the versions
* of Mbed OS, TLS and Crypto specified by the mbed-os.lib file. */
if (env.TARGET_REPO == 'example') {
sh """\
sh label: "mbed deploy #${job_name}", script: """\
ulimit -f 20971520
. $WORKSPACE/mbed-venv/bin/activate
mbed config root .
Expand All @@ -498,7 +420,7 @@ mbed deploy -vv
* checking it out twice. Mbed deploy is still run in case other libraries
* are required to be deployed. We then check out Mbed OS, TLS and Crypto
* according to the job parameters. */
sh """\
sh label: "rm mbed-os.lib; mbed deploy #${job_name}", script: """\
ulimit -f 20971520
. $WORKSPACE/mbed-venv/bin/activate
rm -f mbed-os.lib
Expand All @@ -509,7 +431,7 @@ mbed deploy -vv
deleteDir()
checkout_repo.checkout_mbed_os()
/* Check that python requirements are up to date */
sh """\
sh label: "pip install #${job_name}", script: """\
ulimit -f 20971520
. $WORKSPACE/mbed-venv/bin/activate
pip install -r requirements.txt
Expand All @@ -523,14 +445,14 @@ pip install -r requirements.txt
if (example == 'atecc608a') {
tag_filter = "--tag-filters HAS_CRYPTOKIT"
}
sh """\
sh label: "mbed compile #${job_name}", script: """\
ulimit -f 20971520
. $WORKSPACE/mbed-venv/bin/activate
mbed compile -m ${platform} -t ${compiler}
"""
for (int attempt = 1; attempt <= 3; attempt++) {
try {
sh """\
sh label: "mbedhtrun #${job_name}#${attempt}", script: """\
ulimit -f 20971520
if [ -e BUILD/${platform}/${compiler}/${example}.bin ]
then
Expand Down Expand Up @@ -653,7 +575,9 @@ def gen_dockerfile_builder_job(platform, overwrite=false) {
node('dockerfile-builder') {
def image_exists = false
if (!overwrite) {
image_exists = sh(script: check_docker_image, returnStatus: true) == 0
image_exists = sh(script: check_docker_image,
label: "check_docker_image ${tag}",
returnStatus: true) == 0
}
if (overwrite || !image_exists) {
dir('docker') {
Expand All @@ -665,7 +589,7 @@ def gen_dockerfile_builder_job(platform, overwrite=false) {
extra_build_args = '--build-arg [email protected]'

withCredentials([string(credentialsId: 'DOCKER_AUTH', variable: 'TOKEN')]) {
sh """\
sh label: ">.docker/config.json #${platform}", script: """\
mkdir -p ${env.HOME}/.docker
cat > ${env.HOME}/.docker/config.json << EOF
{
Expand All @@ -680,12 +604,12 @@ chmod 0600 ${env.HOME}/.docker/config.json
"""
}
} else {
sh """\
sh label: "docker login #${platform}", script: """\
aws ecr get-login-password | docker login --username AWS --password-stdin $common.docker_ecr
"""
}

sh """\
sh label: "docker push #${tag}", script: """\
# Use BuildKit and a remote build cache to pull only the reuseable layers
# from the last successful build for this platform
DOCKER_BUILDKIT=1 docker build \
Expand Down