From 29906d583d36090d422f64b003031e88e3989e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Sat, 28 Jan 2023 00:26:54 +0100 Subject: [PATCH] Add instrumentation to the PR jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measure how much time we spend in the groovy scripts versus how much time is spent executing the test scripts for each component. Signed-off-by: Bence Szépkúti --- src/org/mbed/tls/jenkins/JobTimestamps.groovy | 83 ++++ vars/analysis.groovy | 91 +++- vars/gen_jobs.groovy | 450 +++++++++--------- vars/mbedtls.groovy | 220 ++++----- 4 files changed, 504 insertions(+), 340 deletions(-) create mode 100755 src/org/mbed/tls/jenkins/JobTimestamps.groovy diff --git a/src/org/mbed/tls/jenkins/JobTimestamps.groovy b/src/org/mbed/tls/jenkins/JobTimestamps.groovy new file mode 100755 index 000000000..5108245ba --- /dev/null +++ b/src/org/mbed/tls/jenkins/JobTimestamps.groovy @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023, Arm Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of Mbed TLS (https://www.trustedfirmware.org/projects/mbed-tls/) + */ + +package org.mbed.tls.jenkins + +import java.util.concurrent.atomic.AtomicLong + +import com.cloudbees.groovy.cps.NonCPS + +class JobTimestamps { + private final AtomicLong start, end, innerStart, innerEnd + + JobTimestamps() { + this.start = new AtomicLong(-1) + this.end = new AtomicLong(-1) + this.innerStart = new AtomicLong(-1) + this.innerEnd = new AtomicLong(-1) + } + + @NonCPS + long getStart() { + return this.@start.get() + } + + @NonCPS + long getEnd() { + return this.@end.get() + } + + @NonCPS + long getInnerStart() { + return this.@innerStart.get() + } + + @NonCPS + long getInnerEnd() { + return this.@innerEnd.get() + } + + private static void set(String name, AtomicLong var, long val) { + if (!var.compareAndSet(-1, val)) { + throw new IllegalAccessError("$name set twice") + } + } + + void setStart(long val) { + set('start', start, val) + } + + void setEnd(long val) { + set('end', end, val) + } + + void setInnerStart(long val) { + set('innerStart', innerStart, val) + } + + void setInnerEnd(long val) { + set('innerEnd', innerEnd, val) + } + + @NonCPS + @Override + String toString() { + return "JobTimestamps(start:$start, end:$end, innerStart:$innerStart, innerEnd:$innerEnd)" + } +} diff --git a/vars/analysis.groovy b/vars/analysis.groovy index 472f58cab..f9d7781f5 100644 --- a/vars/analysis.groovy +++ b/vars/analysis.groovy @@ -17,11 +17,83 @@ * This file is part of Mbed TLS (https://www.trustedfirmware.org/projects/mbed-tls/) */ +import java.util.concurrent.Callable +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap +import java.util.function.Function + import groovy.transform.Field +import com.cloudbees.groovy.cps.NonCPS +import net.sf.json.JSONObject + +import org.mbed.tls.jenkins.JobTimestamps + // A static field has its content preserved across stages. @Field static outcome_stashes = [] +@Field private static ConcurrentMap> timestamps = + new ConcurrentHashMap>(); + +void record_timestamps(String group, String job_name, Callable body, String node_label = null) { + def ts = new JobTimestamps() + def group_map = timestamps.computeIfAbsent(group, new Function>() { + @Override + @NonCPS + ConcurrentMap apply(String key) { + return new ConcurrentHashMap() + } + }) + + if (group_map.putIfAbsent(job_name, ts) != null) { + throw new IllegalArgumentException("Group and job name pair '$group:$job_name' used multiple times.") + } + + try { + def stamped_body = { + ts.start = System.currentTimeMillis() + body() + } + if (node_label != null) { + node(node_label, stamped_body) + } else { + stamped_body() + } + } finally { + ts.end = System.currentTimeMillis() + } +} + +void node_record_timestamps(String node_label, String job_name, Callable body) { + record_timestamps(node_label, job_name, body, node_label) +} + +void record_inner_timestamps(String group, String job_name, Callable body) { + def ts = timestamps[group][job_name] + if (ts == null) { + throw new NoSuchElementException(job_name) + } + ts.innerStart = System.currentTimeMillis() + try { + body() + } finally { + ts.innerEnd = System.currentTimeMillis() + } +} + +void print_timestamps() { + writeFile( + file: 'timestamps.json', + text: JSONObject.fromObject([ + job: currentBuild.fullProjectName, + build: currentBuild.number, + main: timestamps.remove('main'), + subtasks: timestamps + ]).toString() + ) + archiveArtifacts(artifacts: 'timestamps.json') +} + def stash_outcomes(job_name) { def stash_name = job_name + '-outcome' if (findFiles(glob: '*-outcome.csv')) { @@ -77,21 +149,22 @@ def gather_outcomes() { if (outcome_stashes.isEmpty()) { return } - node('helper-container-host') { - dir('outcomes') { + dir('outcomes') { + deleteDir() + try { + checkout_repo.checkout_repo() + process_outcomes() + } finally { deleteDir() - try { - checkout_repo.checkout_repo() - process_outcomes() - } finally { - deleteDir() - } } } } def analyze_results() { - gather_outcomes() + node('helper-container-host') { + gather_outcomes() + print_timestamps() + } } def analyze_results_and_notify_github() { diff --git a/vars/gen_jobs.groovy b/vars/gen_jobs.groovy index f54423686..c7800f5d0 100644 --- a/vars/gen_jobs.groovy +++ b/vars/gen_jobs.groovy @@ -17,6 +17,8 @@ * This file is part of Mbed TLS (https://www.trustedfirmware.org/projects/mbed-tls/) */ +import java.util.concurrent.Callable + import groovy.transform.Field import hudson.AbortException @@ -28,29 +30,36 @@ import hudson.AbortException //Record coverage details for reporting @Field coverage_details = ['coverage': 'Code coverage job did not run'] -def gen_simple_windows_jobs(label, script) { - def jobs = [:] +private Map> job(String label, Callable body) { + return Collections.singletonMap(label, body) +} - jobs[label] = { - node("windows") { - try { - dir("src") { - deleteDir() - checkout_repo.checkout_repo() - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { +private Map> instrumented_node_job(String node_label, String job_name, Callable body) { + return job(job_name) { + analysis.node_record_timestamps(node_label, job_name, body) + } +} + +Map> gen_simple_windows_jobs(String label, String script) { + return instrumented_node_job('windows', label) { + try { + dir('src') { + deleteDir() + checkout_repo.checkout_repo() + timeout(time: common.perJobTimeout.time, + unit: common.perJobTimeout.unit) { + analysis.record_inner_timestamps('windows', label) { bat script } } - } catch (err) { - failed_builds[label] = true - throw (err) - } finally { - deleteDir() } + } catch (err) { + failed_builds[label] = true + throw (err) + } finally { + deleteDir() } } - return jobs } def node_label_for_platform(platform) { @@ -73,7 +82,6 @@ def platform_lacks_tls_tools(platform) { } def gen_all_sh_jobs(platform, component, label_prefix='') { - def jobs = [:] def shorthands = [ "ubuntu-16.04": "u16", "ubuntu-18.04": "u18", @@ -134,16 +142,15 @@ scripts/min_requirements.py --user ''' } - jobs[job_name] = { - node(node_label) { - try { - deleteDir() - if (use_docker) { - common.get_docker_image(platform) - } - dir('src') { - checkout_repo.checkout_repo() - writeFile file: 'steps.sh', text: """\ + return instrumented_node_job(node_label, job_name) { + try { + deleteDir() + if (use_docker) { + common.get_docker_image(platform) + } + dir('src') { + checkout_repo.checkout_repo() + writeFile file: 'steps.sh', text: """\ #!/bin/sh set -eux ulimit -f 20971520 @@ -151,93 +158,93 @@ export MBEDTLS_TEST_OUTCOME_FILE='${job_name}-outcome.csv' ${extra_setup_code} ./tests/scripts/all.sh --seed 4 --keep-going $component """ - sh 'chmod +x steps.sh' - } - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { - try { - if (use_docker) { + sh 'chmod +x steps.sh' + } + timeout(time: common.perJobTimeout.time, + unit: common.perJobTimeout.unit) { + try { + if (use_docker) { + analysis.record_inner_timestamps(node_label, job_name) { sh common.docker_script( platform, "/var/lib/build/steps.sh" ) - } else { - dir('src') { - sh './steps.sh' - } } - } finally { + } else { dir('src') { - analysis.stash_outcomes(job_name) - } - dir('src/tests/') { - common.archive_zipped_log_files(job_name) + analysis.record_inner_timestamps(node_label, job_name) { + sh './steps.sh' + } } } + } finally { + dir('src') { + analysis.stash_outcomes(job_name) + } + dir('src/tests/') { + common.archive_zipped_log_files(job_name) + } } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() } + } catch (err) { + failed_builds[job_name] = true + throw (err) + } finally { + deleteDir() } } - return jobs } def gen_windows_testing_job(build, label_prefix='') { - def jobs = [:] def job_name = "${label_prefix}Windows-${build}" - jobs[job_name] = { - node("windows") { - try { - dir("src") { - deleteDir() - checkout_repo.checkout_repo() - } - /* The empty files are created to re-create the directory after it - * and its contents have been removed by deleteDir. */ - dir("logs") { - deleteDir() - writeFile file:'_do_not_delete_this_directory.txt', text:'' - } + return instrumented_node_job('windows', job_name) { + try { + dir("src") { + deleteDir() + checkout_repo.checkout_repo() + } + /* The empty files are created to re-create the directory after it + * and its contents have been removed by deleteDir. */ + dir("logs") { + deleteDir() + writeFile file:'_do_not_delete_this_directory.txt', text:'' + } - dir("worktrees") { - deleteDir() - writeFile file:'_do_not_delete_this_directory.txt', text:'' - } + dir("worktrees") { + deleteDir() + writeFile file:'_do_not_delete_this_directory.txt', text:'' + } - if (common.has_min_requirements) { - dir("src") { - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { - bat "python scripts\\min_requirements.py" - } + if (common.has_min_requirements) { + dir("src") { + timeout(time: common.perJobTimeout.time, + unit: common.perJobTimeout.unit) { + bat "python scripts\\min_requirements.py" } } + } - /* libraryResource loads the file as a string. This is then - * written to a file so that it can be run on a node. */ - def windows_testing = libraryResource 'windows/windows_testing.py' - writeFile file: 'windows_testing.py', text: windows_testing - timeout(time: common.perJobTimeout.time + - common.perJobTimeout.windowsTestingOffset, - unit: common.perJobTimeout.unit) { + /* libraryResource loads the file as a string. This is then + * written to a file so that it can be run on a node. */ + def windows_testing = libraryResource 'windows/windows_testing.py' + writeFile file: 'windows_testing.py', text: windows_testing + timeout(time: common.perJobTimeout.time + + common.perJobTimeout.windowsTestingOffset, + unit: common.perJobTimeout.unit) { + analysis.record_inner_timestamps('windows', job_name) { bat "python windows_testing.py src logs -b $build" } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() } + } catch (err) { + failed_builds[job_name] = true + throw (err) + } finally { + deleteDir() } } - return jobs } -def gen_windows_jobs(label_prefix='') { +def gen_windows_jobs(String label_prefix='') { def jobs = [:] jobs = jobs + gen_simple_windows_jobs( label_prefix + 'win32-mingw', scripts.win32_mingw_test_bat @@ -255,113 +262,108 @@ def gen_windows_jobs(label_prefix='') { } def gen_abi_api_checking_job(platform) { - def jobs = [:] - def job_name = "ABI-API-checking" + def job_name = 'ABI-API-checking' def credentials_id = common.is_open_ci_env ? "mbedtls-github-ssh" : "742b7080-e1cc-41c6-bf55-efb72013bc28" - jobs[job_name] = { - node('container-host') { - try { - deleteDir() - common.get_docker_image(platform) - dir('src') { - checkout_repo.checkout_repo() - /* The credentials here are the SSH credentials for accessing the repositories. - They are defined at {JENKINS_URL}/credentials */ - withCredentials([sshUserPrivateKey(credentialsId: credentials_id, keyFileVariable: 'keyfile')]) { - sh "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}" - } - writeFile file: 'steps.sh', text: """\ + return instrumented_node_job('container-host', job_name) { + try { + deleteDir() + common.get_docker_image(platform) + dir('src') { + checkout_repo.checkout_repo() + /* The credentials here are the SSH credentials for accessing the repositories. + They are defined at {JENKINS_URL}/credentials */ + withCredentials([sshUserPrivateKey(credentialsId: credentials_id, keyFileVariable: 'keyfile')]) { + sh "GIT_SSH_COMMAND=\"ssh -i ${keyfile}\" git fetch origin ${CHANGE_TARGET}" + } + writeFile file: 'steps.sh', text: """\ #!/bin/sh set -eux ulimit -f 20971520 if [ -e scripts/min_requirements.py ]; then - scripts/min_requirements.py --user +scripts/min_requirements.py --user fi tests/scripts/list-identifiers.sh --internal scripts/abi_check.py -o FETCH_HEAD -n HEAD -s identifiers --brief """ - sh 'chmod +x steps.sh' - } - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { + sh 'chmod +x steps.sh' + } + timeout(time: common.perJobTimeout.time, + unit: common.perJobTimeout.unit) { + analysis.record_inner_timestamps('container-host', job_name) { sh common.docker_script( - platform, "/var/lib/build/steps.sh" + platform, "/var/lib/build/steps.sh" ) } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() } + } catch (err) { + failed_builds[job_name] = true + throw (err) + } finally { + deleteDir() } } - return jobs } def gen_code_coverage_job(platform) { - def jobs = [:] def job_name = 'code-coverage' - - jobs[job_name] = { - node('container-host') { - try { - deleteDir() - common.get_docker_image(platform) - dir('src') { - checkout_repo.checkout_repo() - writeFile file: 'steps.sh', text: '''#!/bin/sh + return instrumented_node_job('container-host', job_name) { + try { + deleteDir() + common.get_docker_image(platform) + dir('src') { + checkout_repo.checkout_repo() + writeFile file: 'steps.sh', text: '''#!/bin/sh set -eux ulimit -f 20971520 if [ -e scripts/min_requirements.py ]; then - scripts/min_requirements.py --user +scripts/min_requirements.py --user fi if grep -q -F coverage-summary.txt tests/scripts/basic-build-test.sh; then - # New basic-build-test, generates coverage-summary.txt - ./tests/scripts/basic-build-test.sh +# New basic-build-test, generates coverage-summary.txt +./tests/scripts/basic-build-test.sh else - # Old basic-build-test, only prints the coverage summary to stdout - { stdbuf -oL ./tests/scripts/basic-build-test.sh 2>&1; echo $?; } | - tee basic-build-test.log - [ "$(tail -n1 basic-build-test.log)" -eq 0 ] - sed -n '/^Test Report Summary/,$p' basic-build-test.log >coverage-summary.txt - rm basic-build-test.log +# Old basic-build-test, only prints the coverage summary to stdout +{ stdbuf -oL ./tests/scripts/basic-build-test.sh 2>&1; echo $?; } | + tee basic-build-test.log +[ "$(tail -n1 basic-build-test.log)" -eq 0 ] +sed -n '/^Test Report Summary/,$p' basic-build-test.log >coverage-summary.txt +rm basic-build-test.log fi ''' - sh 'chmod +x steps.sh' - } - timeout(time: common.perJobTimeout.time, - unit: common.perJobTimeout.unit) { - try { + sh 'chmod +x steps.sh' + } + timeout(time: common.perJobTimeout.time, + unit: common.perJobTimeout.unit) { + try { + analysis.record_inner_timestamps('container-host', job_name) { sh common.docker_script( platform, "/var/lib/build/steps.sh" ) - dir('src') { - String coverage_log = readFile('coverage-summary.txt') - coverage_details['coverage'] = coverage_log.substring( - coverage_log.indexOf('\nCoverage\n') + 1 - ) - } - } finally { - dir('src/tests/') { - common.archive_zipped_log_files(job_name) - } + } + dir('src') { + String coverage_log = readFile('coverage-summary.txt') + coverage_details['coverage'] = coverage_log.substring( + coverage_log.indexOf('\nCoverage\n') + 1 + ) + } + } finally { + dir('src/tests/') { + common.archive_zipped_log_files(job_name) } } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() } + } catch (err) { + failed_builds[job_name] = true + throw (err) + } finally { + deleteDir() } } - return jobs } /* Mbed OS Example job generation */ @@ -391,127 +393,124 @@ def gen_mbed_os_example_job(repo, branch, example, compiler, platform, raas) { def jobs = [:] def job_name = "mbed-os-${example}-${platform}-${compiler}" - jobs[job_name] = { - node(compiler) { - try { - deleteDir() + return instrumented_node_job(compiler, job_name) { + try { + deleteDir() /* Create python virtual environment and install mbed tools */ - sh """\ + sh """\ ulimit -f 20971520 virtualenv $WORKSPACE/mbed-venv . $WORKSPACE/mbed-venv/bin/activate pip install mbed-cli pip install mbed-host-tests """ - dir('mbed-os-example') { - deleteDir() - checkout_repo.checkout_mbed_os_example_repo(repo, branch) - dir(example) { + dir('mbed-os-example') { + deleteDir() + checkout_repo.checkout_mbed_os_example_repo(repo, branch) + dir(example) { /* If the job is targeting an example repo, then we wish to use the versions - * of Mbed OS, TLS and Crypto specified by the mbed-os.lib file. */ - if (env.TARGET_REPO == 'example') { - sh """\ +* of Mbed OS, TLS and Crypto specified by the mbed-os.lib file. */ + if (env.TARGET_REPO == 'example') { + sh """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate mbed config root . mbed deploy -vv """ - } else { + } else { /* If the job isn't targeting an example repo, the versions of Mbed OS, TLS and - * Crypto will be specified by the job. We remove mbed-os.lib so we aren't - * checking it out twice. Mbed deploy is still run in case other libraries - * are required to be deployed. We then check out Mbed OS, TLS and Crypto - * according to the job parameters. */ - sh """\ +* Crypto will be specified by the job. We remove mbed-os.lib so we aren't +* checking it out twice. Mbed deploy is still run in case other libraries +* are required to be deployed. We then check out Mbed OS, TLS and Crypto +* according to the job parameters. */ + sh """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate rm -f mbed-os.lib mbed config root . mbed deploy -vv """ - dir('mbed-os') { - deleteDir() - checkout_repo.checkout_mbed_os() + dir('mbed-os') { + deleteDir() + checkout_repo.checkout_mbed_os() /* Check that python requirements are up to date */ - sh """\ + sh """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate pip install -r requirements.txt """ - } } - timeout(time: common.perJobTimeout.time + - common.perJobTimeout.raasOffset, - unit: common.perJobTimeout.unit) { - def tag_filter = "" - if (example == 'atecc608a') { - tag_filter = "--tag-filters HAS_CRYPTOKIT" - } - sh """\ + } + timeout(time: common.perJobTimeout.time + + common.perJobTimeout.raasOffset, + unit: common.perJobTimeout.unit) { + def tag_filter = "" + if (example == 'atecc608a') { + tag_filter = "--tag-filters HAS_CRYPTOKIT" + } + sh """\ ulimit -f 20971520 . $WORKSPACE/mbed-venv/bin/activate mbed compile -m ${platform} -t ${compiler} """ - for (int attempt = 1; attempt <= 3; attempt++) { - try { - sh """\ + for (int attempt = 1; attempt <= 3; attempt++) { + try { + sh """\ ulimit -f 20971520 if [ -e BUILD/${platform}/${compiler}/${example}.bin ] then - BINARY=BUILD/${platform}/${compiler}/${example}.bin +BINARY=BUILD/${platform}/${compiler}/${example}.bin else - if [ -e BUILD/${platform}/${compiler}/${example}.hex ] - then - BINARY=BUILD/${platform}/${compiler}/${example}.hex - fi +if [ -e BUILD/${platform}/${compiler}/${example}.hex ] +then + BINARY=BUILD/${platform}/${compiler}/${example}.hex +fi fi export RAAS_PYCLIENT_FORCE_REMOTE_ALLOCATION=1 export RAAS_PYCLIENT_ALLOCATION_QUEUE_TIMEOUT=3600 mbedhtrun -m ${platform} ${tag_filter} \ -g raas_client:https://${raas}.mbedcloudtesting.com:443 -P 1000 --sync=0 -v \ - --compare-log ../tests/${example}.log -f \$BINARY +--compare-log ../tests/${example}.log -f \$BINARY """ - break - } catch (AbortException err) { - if (attempt == 3) throw (err) - } + break + } catch (AbortException err) { + if (attempt == 3) throw (err) } } } } - } catch (err) { - failed_builds[job_name] = true - throw (err) - } finally { - deleteDir() } + } catch (err) { + failed_builds[job_name] = true + throw (err) + } finally { + deleteDir() } } - return jobs } def gen_coverity_push_jobs() { def jobs = [:] - def job_name = "coverity-push" + def job_name = 'coverity-push' if (env.MBED_TLS_BRANCH == "development") { - jobs[] = { - node('container-host') { - try { - dir("src") { - deleteDir() - checkout_repo.checkout_repo() - sshagent([env.GIT_CREDENTIALS_ID]) { + jobs << instrumented_node_job('container-host', job_name) { + try { + dir("src") { + deleteDir() + checkout_repo.checkout_repo() + sshagent([env.GIT_CREDENTIALS_ID]) { + analysis.record_inner_timestamps('container-host', job_name) { sh 'git push origin HEAD:coverity_scan' } } - } catch (err) { - failed_builds[job_name]= true - throw (err) - } finally { - deleteDir() } + } catch (err) { + failed_builds[job_name]= true + throw (err) + } finally { + deleteDir() } } } @@ -558,19 +557,23 @@ def gen_release_jobs(label_prefix='', run_examples=true) { return jobs } -def gen_dockerfile_builder_job(platform, overwrite=false) { - def jobs = [:] +def gen_dockerfile_builder_job(String platform, boolean overwrite=false) { def dockerfile = libraryResource "docker_files/$platform/Dockerfile" def tag = "$platform-${common.git_hash_object(dockerfile)}" - def check_docker_image = common.is_open_ci_env ? "docker manifest inspect $common.docker_repo:$tag > /dev/null 2>&1" : "aws ecr describe-images --repository-name $common.docker_repo_name --image-ids imageTag=$tag" + def check_docker_image + if (common.is_open_ci_env) { + check_docker_image = "docker manifest inspect $common.docker_repo:$tag > /dev/null 2>&1" + } else { + check_docker_image = "aws ecr describe-images --repository-name $common.docker_repo_name --image-ids imageTag=$tag" + } common.docker_tags[platform] = tag - jobs[platform] = { + return job(platform) { /* Take the lock on the master node, so we don't tie up an executor while waiting */ lock(tag) { - node('dockerfile-builder') { + analysis.node_record_timestamps('dockerfile-builder', platform) { def image_exists = false if (!overwrite) { image_exists = sh(script: check_docker_image, returnStatus: true) == 0 @@ -605,7 +608,8 @@ aws ecr get-login-password | docker login --username AWS --password-stdin $commo """ } - sh """\ + analysis.record_inner_timestamps('helper-container-host', platform) { + sh """\ # Use BuildKit and a remote build cache to pull only the reuseable layers # from the last successful build for this platform DOCKER_BUILDKIT=1 docker build \ @@ -619,10 +623,10 @@ DOCKER_BUILDKIT=1 docker build \ docker push $common.docker_repo:$tag docker push $common.docker_repo:$platform-cache """ + } } } } } } - return jobs } diff --git a/vars/mbedtls.groovy b/vars/mbedtls.groovy index 998aba426..e18b4640a 100644 --- a/vars/mbedtls.groovy +++ b/vars/mbedtls.groovy @@ -36,7 +36,9 @@ def run_tls_tests(label_prefix='') { jobs = common.wrap_report_errors(jobs) jobs.failFast = false - parallel jobs + analysis.record_inner_timestamps('main', 'run_pr_job') { + parallel jobs + } common.maybe_notify_github "TLS Testing", 'SUCCESS', 'All tests passed' } catch (err) { @@ -53,122 +55,124 @@ def run_tls_tests(label_prefix='') { def run_pr_job(is_production=true) { timestamps { try { - if (is_production) { - // Cancel in-flight jobs for the same PR when a new job is launched - def buildNumber = env.BUILD_NUMBER as int - if (buildNumber > 1) - milestone(buildNumber - 1) - /* If buildNumber > 1, the following statement aborts all builds - * whose most-recently passed milestone was the previous milestone - * passed by this job (buildNumber - 1). - * After this, it checks to see if a later build has already passed - * milestone(buildNumber), and if so aborts the current build as well. - * - * Because of the order of operations, each build is only responsible - * for aborting the one directly before it, and itself if necessary. - * Thus we don't have to iterate over all milestones 1 to buildNumber. - */ - milestone(buildNumber) - - /* Discarding old builds has to be individually configured for each - * branch in a multibranch pipeline, so do it from groovy. - */ - properties([ - buildDiscarder( - logRotator( - numToKeepStr: '5' - ) - ) - ]) - } - - /* During the nightly branch indexing, if a target branch has been - * updated, new merge jobs are triggered for each PR to that branch. - * If a PR hasn't been updated recently enough, don't run the merge - * job for that PR. - */ - if (env.BRANCH_NAME ==~ /PR-\d+-merge/ ) { - long upd_timestamp_ms = 0L - long now_timestamp_ms = currentBuild.startTimeInMillis + analysis.record_timestamps('main', 'run_pr_job') { try { - if (currentBuild.rawBuild.causes[0] instanceof BranchIndexingCause) { - /* Try to retrieve the update timestamp from the previous run. */ - upd_timestamp_ms = (currentBuild.previousBuild?.buildVariables?.UPD_TIMESTAMP_MS ?: 0L) as long - echo "Previous update timestamp: ${new Date(upd_timestamp_ms)}" - - /* current threshold is 2 days */ - long threshold_ms = 2L * 24L * 60L * 60L * 1000L - - if (now_timestamp_ms - upd_timestamp_ms > threshold_ms) { - /* Check the time of the latest review */ - def src = (GitHubSCMSource) SCMSource.SourceByItem.findSource(currentBuild.rawBuild.parent) - def cred = Connector.lookupScanCredentials((Item) src.owner, src.apiUri, src.credentialsId) - - def gh = Connector.connect(src.apiUri, cred) - def pr = gh.getRepository("$src.repoOwner/$src.repository").getPullRequest(env.CHANGE_ID as int) - - try { - long review_timestamp_ms = pr.listReviews().last().submittedAt.time - echo "Latest review timestamp: ${new Date(review_timestamp_ms)}" - upd_timestamp_ms = Math.max(review_timestamp_ms, upd_timestamp_ms) - } catch (NoSuchElementException err) { - /* No reviews */ - } + if (is_production) { + // Cancel in-flight jobs for the same PR when a new job is launched + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) + milestone(buildNumber - 1) + /* If buildNumber > 1, the following statement aborts all builds + * whose most-recently passed milestone was the previous milestone + * passed by this job (buildNumber - 1). + * After this, it checks to see if a later build has already passed + * milestone(buildNumber), and if so aborts the current build as well. + * + * Because of the order of operations, each build is only responsible + * for aborting the one directly before it, and itself if necessary. + * Thus we don't have to iterate over all milestones 1 to buildNumber. + */ + milestone(buildNumber) + + /* Discarding old builds has to be individually configured for each + * branch in a multibranch pipeline, so do it from groovy. + */ + properties([ + buildDiscarder( + logRotator( + numToKeepStr: '5' + ) + ) + ]) + } - if (upd_timestamp_ms == 0L) { - /* Fall back to updatedAt */ - upd_timestamp_ms = pr.updatedAt.time - echo "PR updatedAt timestamp: ${new Date(upd_timestamp_ms)}" + /* During the nightly branch indexing, if a target branch has been + * updated, new merge jobs are triggered for each PR to that branch. + * If a PR hasn't been updated recently enough, don't run the merge + * job for that PR. + */ + if (env.BRANCH_NAME ==~ /PR-\d+-merge/) { + long upd_timestamp_ms = 0L + long now_timestamp_ms = currentBuild.startTimeInMillis + try { + if (currentBuild.rawBuild.causes[0] instanceof BranchIndexingCause) { + /* Try to retrieve the update timestamp from the previous run. */ + upd_timestamp_ms = (currentBuild.previousBuild?.buildVariables?.UPD_TIMESTAMP_MS ?: 0L) as long + echo "Previous update timestamp: ${new Date(upd_timestamp_ms)}" + + /* current threshold is 2 days */ + long threshold_ms = 2L * 24L * 60L * 60L * 1000L + + if (now_timestamp_ms - upd_timestamp_ms > threshold_ms) { + /* Check the time of the latest review */ + def src = (GitHubSCMSource) SCMSource.SourceByItem.findSource(currentBuild.rawBuild.parent) + def cred = Connector.lookupScanCredentials((Item) src.owner, src.apiUri, src.credentialsId) + + def gh = Connector.connect(src.apiUri, cred) + def pr = gh.getRepository("$src.repoOwner/$src.repository").getPullRequest(env.CHANGE_ID as int) + + try { + long review_timestamp_ms = pr.listReviews().last().submittedAt.time + echo "Latest review timestamp: ${new Date(review_timestamp_ms)}" + upd_timestamp_ms = Math.max(review_timestamp_ms, upd_timestamp_ms) + } catch (NoSuchElementException err) { + /* No reviews */ + } + + if (upd_timestamp_ms == 0L) { + /* Fall back to updatedAt */ + upd_timestamp_ms = pr.updatedAt.time + echo "PR updatedAt timestamp: ${new Date(upd_timestamp_ms)}" + } + } + + if (now_timestamp_ms - upd_timestamp_ms > threshold_ms) { + currentBuild.result = 'NOT_BUILT' + error("Pre Test Checks did not run: PR was last updated on ${new Date(upd_timestamp_ms)}.") + } + } else { + /* Job triggered manually, or by pushing the branch. Update the timestamp */ + upd_timestamp_ms = now_timestamp_ms } + } finally { + /* Record the update timestamp in the environment, so it can be retrieved by the next run */ + env.UPD_TIMESTAMP_MS = upd_timestamp_ms + echo "UPD_TIMESTAMP_MS=$env.UPD_TIMESTAMP_MS (${new Date(upd_timestamp_ms)})" } - - if (now_timestamp_ms - upd_timestamp_ms > threshold_ms) { - currentBuild.result = 'NOT_BUILT' - error("Pre Test Checks did not run: PR was last updated on ${new Date(upd_timestamp_ms)}.") - } - } else { - /* Job triggered manually, or by pushing the branch. Update the timestamp */ - upd_timestamp_ms = now_timestamp_ms } - } finally { - /* Record the update timestamp in the environment, so it can be retrieved by the next run */ - env.UPD_TIMESTAMP_MS = upd_timestamp_ms - echo "UPD_TIMESTAMP_MS=$env.UPD_TIMESTAMP_MS (${new Date(upd_timestamp_ms)})" - } - } - common.maybe_notify_github "Pre Test Checks", 'PENDING', - 'Checking if all PR tests can be run' - common.maybe_notify_github "TLS Testing", 'PENDING', - 'In progress' - common.maybe_notify_github "Result analysis", 'PENDING', - 'In progress' + common.maybe_notify_github "Pre Test Checks", 'PENDING', + 'Checking if all PR tests can be run' + common.maybe_notify_github "TLS Testing", 'PENDING', + 'In progress' + common.maybe_notify_github "Result analysis", 'PENDING', + 'In progress' - common.init_docker_images() + common.init_docker_images() - stage('pre-test-checks') { - environ.set_tls_pr_environment(is_production) - common.get_branch_information() - common.check_every_all_sh_component_will_be_run() - common.maybe_notify_github "Pre Test Checks", 'SUCCESS', 'OK' - } - } catch (err) { - def description = 'Pre Test Checks failed.' - if (err.message?.startsWith('Pre Test Checks')) { - description = err.message - } - common.maybe_notify_github "Pre Test Checks", 'FAILURE', - description - common.maybe_notify_github 'TLS Testing', 'FAILURE', - 'Did not run' - common.maybe_notify_github 'Result analysis', 'FAILURE', - 'Did not run' - throw (err) - } + stage('pre-test-checks') { + environ.set_tls_pr_environment(is_production) + common.get_branch_information() + common.check_every_all_sh_component_will_be_run() + common.maybe_notify_github "Pre Test Checks", 'SUCCESS', 'OK' + } + } catch (err) { + def description = 'Pre Test Checks failed.' + if (err.message?.startsWith('Pre Test Checks')) { + description = err.message + } + common.maybe_notify_github "Pre Test Checks", 'FAILURE', + description + common.maybe_notify_github 'TLS Testing', 'FAILURE', + 'Did not run' + common.maybe_notify_github 'Result analysis', 'FAILURE', + 'Did not run' + throw (err) + } - try { - stage('tls-testing') { - run_tls_tests() + stage('tls-testing') { + run_tls_tests() + } } } finally { stage('result-analysis') {