Skip to content

Commit 2dbd1bf

Browse files
rnroLukasa
rnro
and
Lukasa
authored
A script for updating Integration Tests thresholds from CI (#3443)
### Motivation: We have a way to update Benchmarks thresholds from CI logs but not for the older Integration Tests. ### Modifications: A new script which pulls integration test logs and uses our existing parsing script to pull out updated values. ### Result: Easier threshold updates. ``` ❯ ./dev/update-integration-test-thresholds.sh Usage: ./dev/update-integration-test-thresholds.sh <url> or: URL=<url> ./dev/update-integration-test-thresholds.sh Example: ./dev/update-integration-test-thresholds.sh #3442 ./dev/update-integration-test-thresholds.sh https://github.com/apple/swift-nio/actions/runs/19234929677 URL=#3442 ./dev/update-integration-test-thresholds.sh ``` ``` ❯ ./dev/update-integration-test-thresholds.sh 'https://github.com/apple/swift-nio/actions/runs/19234929677/job/54982236271?pr=3442' ** Processing run in apple/swift-nio ** Fetching integration test checks from workflow run 19234929677 ** Pulling logs for Swift nightly-main (job 54982236243) ** Updated IntegrationTests/tests_04_performance/Thresholds/nightly-main.json ** Pulling logs for Swift 6.2 (job 54982236259) ** Updated IntegrationTests/tests_04_performance/Thresholds/6.2.json ** Pulling logs for Swift nightly-next (job 54982236267) ** Updated IntegrationTests/tests_04_performance/Thresholds/nightly-next.json ** Pulling logs for Swift 6.0 (job 54982236271) ** Updated IntegrationTests/tests_04_performance/Thresholds/6.0.json ** Pulling logs for Swift 6.1 (job 54982236345) ** Updated IntegrationTests/tests_04_performance/Thresholds/6.1.json ** Done! Updated 5 threshold file(s) ``` Co-authored-by: Cory Benfield <[email protected]>
1 parent 45b463c commit 2dbd1bf

File tree

1 file changed

+285
-0
lines changed

1 file changed

+285
-0
lines changed

‎dev/update-integration-test-thresholds.sh‎

Lines changed: 285 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,285 @@
1+
#!/bin/bash
2+
##===----------------------------------------------------------------------===##
3+
##
4+
## This source file is part of the SwiftNIO open source project
5+
##
6+
## Copyright (c) 2025 Apple Inc. and the SwiftNIO project authors
7+
## Licensed under Apache License v2.0
8+
##
9+
## See LICENSE.txt for license information
10+
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
11+
##
12+
## SPDX-License-Identifier: Apache-2.0
13+
##
14+
##===----------------------------------------------------------------------===##
15+
16+
# This script allows you to consume GitHub Actions integration test logs and
17+
# update allocation threshold JSON files automatically
18+
19+
set -uo pipefail
20+
21+
log() { printf -- "** %s\n" "$*" >&2; }
22+
error() { printf -- "** ERROR: %s\n" "$*" >&2; }
23+
fatal() { error "$@"; exit 1; }
24+
25+
usage() {
26+
echo >&2 "Usage: $0 <url>"
27+
echo >&2 " or: URL=<url> $0"
28+
echo >&2
29+
echo >&2 "Example:"
30+
echo >&2 " $0 https://github.com/apple/swift-nio/pull/3442"
31+
echo >&2 " $0 https://github.com/apple/swift-nio/actions/runs/19234929677"
32+
echo >&2 " URL=https://github.com/apple/swift-nio/pull/3442 $0"
33+
}
34+
35+
url="${1:-${URL:-}}"
36+
37+
if [ -z "$url" ]; then
38+
usage
39+
fatal "Pull request or workflow run URL must be specified."
40+
fi
41+
42+
# Check for required tools
43+
GH_BIN="${GH_BIN:-$(which gh)}" || fatal "GH_BIN unset and no gh on PATH"
44+
JQ_BIN="${JQ_BIN:-$(which jq)}" || fatal "JQ_BIN unset and no jq on PATH"
45+
46+
# Check repository and script paths
47+
REPO_ROOT="$(git rev-parse --show-toplevel)" || fatal "Must be run from within git repository"
48+
ALLOC_LIMITS_SCRIPT="$REPO_ROOT/dev/alloc-limits-from-test-output"
49+
50+
if [ ! -x "$ALLOC_LIMITS_SCRIPT" ]; then
51+
fatal "alloc-limits-from-test-output script not found at: $ALLOC_LIMITS_SCRIPT"
52+
fi
53+
54+
THRESHOLDS_DIR="${THRESHOLDS_DIR:-$REPO_ROOT/IntegrationTests/tests_04_performance/Thresholds}"
55+
56+
if [ ! -d "$THRESHOLDS_DIR" ]; then
57+
fatal "Thresholds directory not found at: $THRESHOLDS_DIR"
58+
fi
59+
60+
# Map job name pattern to threshold file name
61+
map_version() {
62+
check_line=$1
63+
64+
case "$check_line" in
65+
*"(5.8)"*)
66+
echo "5.8"
67+
;;
68+
*"(5.10)"*)
69+
echo "5.10"
70+
;;
71+
*"(6.0)"*)
72+
echo "6.0"
73+
;;
74+
*"(6.1)"*)
75+
echo "6.1"
76+
;;
77+
*"(6.2)"*)
78+
echo "6.2"
79+
;;
80+
*"(nightly-6.1)"*)
81+
echo "nightly-6.1"
82+
;;
83+
*"(nightly-next)"*)
84+
echo "nightly-next"
85+
;;
86+
*"(nightly-main)"*)
87+
echo "nightly-main"
88+
;;
89+
*)
90+
return 1
91+
;;
92+
esac
93+
}
94+
95+
# Track whether any files were updated
96+
updated_count=0
97+
98+
parse_url() {
99+
workflow_url=$1
100+
# https://github.com/apple/swift-nio/actions/runs/15269806473
101+
# https://github.com/apple/swift-nio/actions/runs/19234929677/job/54982236271?pr=3442
102+
# https://github.com/apple/swift-nio/pull/3257
103+
if [[ "$url" =~ pull ]]; then
104+
type="PR"
105+
elif [[ "$url" =~ actions/runs ]]; then
106+
type="run"
107+
else
108+
fatal "Cannot parse URL: $url"
109+
fi
110+
echo "$url" | awk -v type="$type" -F '/' '{print $4, $5, type}'
111+
}
112+
113+
parse_workflow_url() {
114+
workflow_url=$1
115+
# https://github.com/apple/swift-nio/actions/runs/15269806473
116+
# https://github.com/apple/swift-nio/actions/runs/19234929677/job/54982236271?pr=3442
117+
echo "$workflow_url" | sed -E 's|.*/actions/runs/([0-9]+).*|\1|'
118+
}
119+
120+
fetch_checks_for_pr() {
121+
pr_url=$1
122+
123+
"$GH_BIN" pr checks "$pr_url" | grep "Integration tests" | grep -v Construct
124+
}
125+
126+
fetch_checks_for_workflow() {
127+
repo=$1
128+
run=$2
129+
130+
"$GH_BIN" --repo "$repo" run view "$run" | grep "Integration tests" | grep ID | grep -v Construct
131+
}
132+
133+
parse_check() {
134+
type=$1
135+
check_line=$2
136+
137+
case "$type" in
138+
"PR")
139+
parse_check_for_pr "$check_line"
140+
;;
141+
142+
"run")
143+
parse_check_for_workflow "$check_line"
144+
;;
145+
146+
*)
147+
fatal "Unknown type '$type'"
148+
;;
149+
esac
150+
}
151+
152+
parse_check_for_workflow() {
153+
check_line=$1
154+
155+
# Something like:
156+
# ✓ Integration tests / Integration tests / Linux (5.10) in 5m10s (ID 42942543009)
157+
echo "$check_line" | sed -En 's/.*ID ([0-9][0-9]*).*/\1/p'
158+
}
159+
160+
parse_check_for_pr() {
161+
check_line=$1
162+
163+
# Something like:
164+
# Integration tests / Integration tests / Linux (5.10) pass 4m21s https://github.com/apple/swift-nio/actions/runs/13793783082/job/38580234681
165+
echo "$check_line" | sed -E 's/.*\(([^\)]+)\).*github\.com\/(.*)\/actions\/runs\/[0-9]+\/job\/([0-9]+)/\3/g'
166+
}
167+
168+
extract_version_from_check_line() {
169+
check_line=$1
170+
171+
# Extract version from job name like "Integration tests / Linux (6.0)" or "X Integration tests / Linux (6.0) in 22m31s (ID 123)"
172+
if version=$(map_version "$check_line"); then
173+
echo "$version"
174+
return 0
175+
fi
176+
177+
error "Could not extract version from check line: $check_line"
178+
return 1
179+
}
180+
181+
fetch_check_logs() {
182+
version=$1
183+
job=$2
184+
185+
log "Pulling logs for Swift $version (job $job)"
186+
# We use `gh api` rather than `gh run view --log` because of https://github.com/cli/cli/issues/5011.
187+
"$GH_BIN" api "/repos/${repo}/actions/jobs/${job}/logs"
188+
}
189+
190+
update_threshold_file() {
191+
version=$1
192+
logs=$2
193+
194+
threshold_file="$THRESHOLDS_DIR/$version.json"
195+
196+
# Compute relative path from repo root
197+
threshold_file_relative="${threshold_file#"$REPO_ROOT"/}"
198+
199+
# Parse logs and update threshold file
200+
new_thresholds=$(echo "$logs" | "$ALLOC_LIMITS_SCRIPT" --json)
201+
202+
if [ -z "$new_thresholds" ]; then
203+
error "No thresholds extracted from logs for version $version"
204+
return 1
205+
fi
206+
207+
# Compare old and new files (both sorted with jq) and only overwrite if there are substantive differences
208+
if [ -f "$threshold_file" ]; then
209+
old_normalized=$("$JQ_BIN" -S . "$threshold_file" 2>/dev/null || echo "")
210+
new_normalized=$(echo "$new_thresholds" | "$JQ_BIN" -S . 2>/dev/null || echo "")
211+
212+
if [ "$old_normalized" = "$new_normalized" ]; then
213+
log "No changes for $threshold_file_relative (skipping)"
214+
return 0
215+
fi
216+
fi
217+
218+
echo "$new_thresholds" > "$threshold_file"
219+
log "Updated $threshold_file_relative"
220+
updated_count=$((updated_count + 1))
221+
}
222+
223+
####
224+
225+
read -r repo_org repo_name type <<< "$(parse_url "$url")"
226+
repo="$repo_org/$repo_name"
227+
log "Processing $type in $repo"
228+
229+
case "$type" in
230+
"PR")
231+
log "Fetching integration test checks from PR"
232+
check_lines="$(fetch_checks_for_pr "$url")"
233+
234+
if [ -z "$check_lines" ]; then
235+
fatal "Could not locate integration test checks on PR: $url"
236+
fi
237+
;;
238+
239+
"run")
240+
run="$(parse_workflow_url "$url")"
241+
242+
log "Fetching integration test checks from workflow run $run"
243+
check_lines="$(fetch_checks_for_workflow "$repo" "$run")"
244+
245+
if [ -z "$check_lines" ]; then
246+
fatal "Could not locate integration test checks on workflow run: $url"
247+
fi
248+
;;
249+
250+
*)
251+
fatal "Unknown type '$type'"
252+
;;
253+
esac
254+
255+
while read -r check_line; do
256+
version="$(extract_version_from_check_line "$check_line")"
257+
258+
if [ -z "$version" ]; then
259+
log "Skipping check line (couldn't extract version): $check_line"
260+
continue
261+
fi
262+
263+
job="$(parse_check "$type" "$check_line")"
264+
265+
if [ -z "$job" ]; then
266+
error "Could not extract job ID from check line: $check_line"
267+
continue
268+
fi
269+
270+
logs=$(fetch_check_logs "$version" "$job")
271+
272+
if [ -z "$logs" ]; then
273+
log "No logs found for Swift $version (job $job)"
274+
continue
275+
fi
276+
277+
update_threshold_file "$version" "$logs"
278+
279+
done <<< "$check_lines"
280+
281+
if [ "$updated_count" -eq 0 ]; then
282+
log "Done! No updates necessary"
283+
else
284+
log "Done! Updated $updated_count threshold file(s)"
285+
fi

0 commit comments

Comments
 (0)