From c8c71df4a092e8544b096b5cdd2c664b0daa3c59 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 21 Apr 2022 17:58:31 +0200 Subject: [PATCH 1/3] Remove two unused functions gen_docker_jobs_foreach and gen_node_jobs_foreach were last used for mbed-crypto only. They became unused when we removed code that was specific to mbed-crypto, but we didn't notice at the time. Signed-off-by: Gilles Peskine --- vars/gen_jobs.groovy | 80 -------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index 9b71c084b..16f48acf4 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -50,86 +50,6 @@ def gen_simple_windows_jobs(label, script) { return jobs } -def gen_docker_jobs_foreach(label, platforms, compilers, script) { - def jobs = [:] - - for (platform in platforms) { - for (compiler in compilers) { - def job_name = "${label}-${compiler}-${platform}" - def shell_script = sprintf(script, common.compiler_paths[compiler]) - jobs[job_name] = { - node('container-host') { - try { - deleteDir() - common.get_docker_image(platform) - dir('src') { - checkout_repo.checkout_repo() - writeFile file: 'steps.sh', text: """\ -#!/bin/sh -set -eux -ulimit -f 20971520 -${shell_script} -""" - sh 'chmod +x steps.sh' - } - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { - try { - sh common.docker_script( - platform, "/var/lib/build/steps.sh" - ) - } finally { - dir('src/tests/') { - common.archive_zipped_log_files(job_name) - } - } - } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() - } - } - } - } - } - return jobs -} - -def gen_node_jobs_foreach(label, platforms, compilers, script) { - def jobs = [:] - - for (platform in platforms) { - for (compiler in compilers) { - def job_name = "${label}-${compiler}-${platform}" - def shell_script = sprintf(script, common.compiler_paths[compiler]) - jobs[job_name] = { - node(platform) { - try { - deleteDir() - checkout_repo.checkout_repo() - shell_script = """\ -ulimit -f 20971520 -export PYTHON=/usr/local/bin/python2.7 -""" + shell_script - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { - sh shell_script - } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() - } - } - } - } - } - return jobs -} - def node_label_for_platform(platform) { switch (platform) { case ~/^(debian|ubuntu)(-.*)?/: return 'container-host'; From 68ddf827db5cf6f9926c2b22e319eee561006774 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 21 Apr 2022 18:34:04 +0200 Subject: [PATCH 2/3] Remove all.sh --list-components detection all.sh has supported --list-components since 2.17.0 and 2.16.10. We don't need a fancy message for antique branches anymore: the exception from the all.sh call returning a nonzero status will do. Signed-off-by: Gilles Peskine --- vars/common.groovy | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/vars/common.groovy b/vars/common.groovy index c36b0d519..75b4b47ed 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -158,30 +158,20 @@ def get_branch_information() { for (platform in linux_platforms) { get_docker_image(platform) - def all_sh_help = sh( + available_all_sh_components[platform] = sh( script: docker_script( - platform, "./tests/scripts/all.sh", "--help" + platform, "./tests/scripts/all.sh", "--list-components" ), returnStdout: true - ) - if (all_sh_help.contains('list-components')) { - available_all_sh_components[platform] = sh( + ).trim().split('\n') + if (all_all_sh_components == []) { + all_all_sh_components = sh( script: docker_script( - platform, "./tests/scripts/all.sh", "--list-components" + platform, "./tests/scripts/all.sh", + "--list-all-components" ), returnStdout: true ).trim().split('\n') - if (all_all_sh_components == []) { - all_all_sh_components = sh( - script: docker_script( - platform, "./tests/scripts/all.sh", - "--list-all-components" - ), - returnStdout: true - ).trim().split('\n') - } - } else { - error('Pre Test Checks failed: Base branch out of date. Please rebase') } } } From 055fabb94551bf3cf5caf2772ba1cc24b446e02f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 21 Apr 2022 19:07:33 +0200 Subject: [PATCH 3/3] Put informative labels on shell steps Many sh or bat pipeline steps have an uninformative label. Groovy uses the script as a label by default, but this is uninformative when the script is something like "./steps.sh", and it's unreadable or truncated when the script is very long. In such cases, use an informative label, containing the job name. In particular, this makes it possible to quickly search for the execution of a given all.sh component in the Jenkins pipeline view: search for "steps.sh #${job_name}", e.g. "steps.sh #all_u16-test_clang_opt". Signed-off-by: Gilles Peskine --- vars/common.groovy | 4 +++- vars/gen_jobs.groovy | 42 +++++++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/vars/common.groovy b/vars/common.groovy index 75b4b47ed..0312d9490 100644 --- a/vars/common.groovy +++ b/vars/common.groovy @@ -162,6 +162,7 @@ def get_branch_information() { script: docker_script( platform, "./tests/scripts/all.sh", "--list-components" ), + label: "./tests/scripts/all.sh --list-components #$platform", returnStdout: true ).trim().split('\n') if (all_all_sh_components == []) { @@ -170,6 +171,7 @@ def get_branch_information() { platform, "./tests/scripts/all.sh", "--list-all-components" ), + label: "./tests/scripts/all.sh --list-all-components", returnStdout: true ).trim().split('\n') } @@ -235,7 +237,7 @@ def maybe_notify_github(context, state, description) { } def archive_zipped_log_files(job_name) { - sh """\ + sh label: "rename+compress *.log", script: """\ for i in *.log; do [ -f "\$i" ] || break mv "\$i" "$job_name-\$i" diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index 16f48acf4..1f6c1ca3f 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -36,7 +36,7 @@ def gen_simple_windows_jobs(label, script) { checkout_repo.checkout_repo() timeout(time: common.perJobTimeout.time, unit: common.perJobTimeout.unit) { - bat script + bat script: script, label: label } } } catch (err) { @@ -154,12 +154,13 @@ ${extra_setup_code} unit: common.perJobTimeout.unit) { try { if (use_docker) { - sh common.docker_script( + sh script: common.docker_script( platform, "/var/lib/build/steps.sh" - ) + ), label: "docker run steps.sh #${job_name}" } else { dir('src') { - sh './steps.sh' + sh script: './steps.sh', + label: "docker run steps.sh #${job_name}" } } } finally { @@ -265,7 +266,8 @@ def gen_abi_api_checking_job(platform) { /* The credentials here are the SSH credentials for accessing the repositories. They are defined at {JENKINS_URL}/credentials */ withCredentials([sshUserPrivateKey(credentialsId: credentials_id, keyFileVariable: 'keyfile')]) { - sh "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}" + sh script: "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}", + label: "git fetch origin ${CHANGE_TARGET}" } writeFile file: 'steps.sh', text: """\ #!/bin/sh @@ -283,9 +285,9 @@ scripts/abi_check.py -o FETCH_HEAD -n HEAD -s identifiers --brief } timeout(time: common.perJobTimeout.time, unit: common.perJobTimeout.unit) { - sh common.docker_script( + sh script: common.docker_script( platform, "/var/lib/build/steps.sh" - ) + ), label: "steps.sh #${job_name}" } } catch (err) { failed_builds[job_name] = true @@ -334,9 +336,9 @@ fi timeout(time: common.perJobTimeout.time, unit: common.perJobTimeout.unit) { try { - sh common.docker_script( + sh script: common.docker_script( platform, "/var/lib/build/steps.sh" - ) + ), label: "steps.sh #${job_name}" dir('src') { String coverage_log = readFile('coverage-summary.txt') coverage_details['coverage'] = coverage_log.substring( @@ -392,7 +394,7 @@ def gen_mbed_os_example_job(repo, branch, example, compiler, platform, raas) { try { deleteDir() /* Create python virtual environment and install mbed tools */ - sh """\ + sh label: "virtualenv #${job_name}", script: """\ ulimit -f 20971520 virtualenv $WORKSPACE/mbed-venv . $WORKSPACE/mbed-venv/bin/activate @@ -406,7 +408,7 @@ pip install mbed-host-tests /* If the job is targeting an example repo, then we wish to use the versions * of Mbed OS, TLS and Crypto specified by the mbed-os.lib file. */ if (env.TARGET_REPO == 'example') { - sh """\ + sh label: "mbed deploy #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate mbed config root . @@ -418,7 +420,7 @@ mbed deploy -vv * checking it out twice. Mbed deploy is still run in case other libraries * are required to be deployed. We then check out Mbed OS, TLS and Crypto * according to the job parameters. */ - sh """\ + sh label: "rm mbed-os.lib; mbed deploy #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate rm -f mbed-os.lib @@ -429,7 +431,7 @@ mbed deploy -vv deleteDir() checkout_repo.checkout_mbed_os() /* Check that python requirements are up to date */ - sh """\ + sh label: "pip install #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate pip install -r requirements.txt @@ -443,14 +445,14 @@ pip install -r requirements.txt if (example == 'atecc608a') { tag_filter = "--tag-filters HAS_CRYPTOKIT" } - sh """\ + sh label: "mbed compile #${job_name}", script: """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate mbed compile -m ${platform} -t ${compiler} """ for (int attempt = 1; attempt <= 3; attempt++) { try { - sh """\ + sh label: "mbedhtrun #${job_name}#${attempt}", script: """\ ulimit -f 20971520 if [ -e BUILD/${platform}/${compiler}/${example}.bin ] then @@ -573,7 +575,9 @@ def gen_dockerfile_builder_job(platform, overwrite=false) { node('dockerfile-builder') { def image_exists = false if (!overwrite) { - image_exists = sh(script: check_docker_image, returnStatus: true) == 0 + image_exists = sh(script: check_docker_image, + label: "check_docker_image ${tag}", + returnStatus: true) == 0 } if (overwrite || !image_exists) { dir('docker') { @@ -585,7 +589,7 @@ def gen_dockerfile_builder_job(platform, overwrite=false) { extra_build_args = '--build-arg ARMLMD_LICENSE_FILE=27000@flexnet.trustedfirmware.org' withCredentials([string(credentialsId: 'DOCKER_AUTH', variable: 'TOKEN')]) { - sh """\ + sh label: ">.docker/config.json #${platform}", script: """\ mkdir -p ${env.HOME}/.docker cat > ${env.HOME}/.docker/config.json << EOF { @@ -600,12 +604,12 @@ chmod 0600 ${env.HOME}/.docker/config.json """ } } else { - sh """\ + sh label: "docker login #${platform}", script: """\ aws ecr get-login-password | docker login --username AWS --password-stdin $common.docker_ecr """ } - sh """\ + sh label: "docker push #${tag}", script: """\ # Use BuildKit and a remote build cache to pull only the reuseable layers # from the last successful build for this platform DOCKER_BUILDKIT=1 docker build \