Skip to content

Commit 17befc9

Browse files
georgeeegeorgeee
committed
Merge remote-tracking branch 'origin/compatible' into georgeee/fix-16397-mini
2 parents 1b95a6e + 3ebfd0a commit 17befc9

File tree

595 files changed

+19786
-15834
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

595 files changed

+19786
-15834
lines changed

‎buildkite/scripts/bench/install.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
5+
# Don't prompt for answers during apt-get install
6+
export DEBIAN_FRONTEND=noninteractive
7+
8+
sudo apt-get update
9+
sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3
10+
11+
TESTNET_NAME="berkeley"
12+
13+
git config --global --add safe.directory /workdir
14+
source buildkite/scripts/export-git-env-vars.sh
15+
16+
source buildkite/scripts/debian/install.sh "mina-test-suite,mina-$TESTNET_NAME" 1
17+
18+
pip3 install -r scripts/benchmarks/requirements.txt

‎buildkite/scripts/bench/snark_transaction_profiler.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
5+
source buildkite/scripts/bench/install.sh
6+
7+
K=1
8+
MAX_NUM_UPDATES=4
9+
MIN_NUM_UPDATES=2
10+
11+
echo "-- Run Snark Transaction Profiler with parameters: --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES}"
12+
python3 ./scripts/benchmarks run --benchmark snark --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES} --outfile snark.out

‎buildkite/scripts/bench/zkapp_metrics.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
5+
source buildkite/scripts/bench/install.sh
6+
7+
python3 ./scripts/benchmarks run --benchmark zkapp --outfile zakpp-out
8+
9+
python3 ./scripts/benchmarks run --benchmark heap-usage --outfile heap-usage.out

‎buildkite/scripts/build-artifact.sh

+9-2
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ make -C src/app/libp2p_helper
2323
MAINNET_TARGETS=""
2424
[[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe"
2525

26-
echo "--- Build all major tagets required for packaging"
26+
echo "--- Build all major targets required for packaging"
2727
echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
2828
echo "Rust Version: $(rustc --version)"
2929
dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
@@ -44,4 +44,11 @@ dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
4444
src/app/rosetta/indexer_test/indexer_test.exe \
4545
src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
4646
src/app/test_executive/test_executive.exe \
47-
src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log
47+
src/app/benchmarks/benchmarks.exe \
48+
src/app/ledger_export_bench/ledger_export_benchmark.exe \
49+
src/app/disk_caching_stats/disk_caching_stats.exe \
50+
src/app/heap_usage/heap_usage.exe \
51+
src/app/zkapp_limits/zkapp_limits.exe \
52+
src/lib/snark_worker/standalone/run_snark_worker.exe \
53+
src/test/command_line_tests/command_line_tests.exe \
54+
src/test/archive/patch_archive_test/patch_archive_test.exe

‎buildkite/scripts/debian/install.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ else
3434
debs=(${DEBS//,/ })
3535
for i in "${debs[@]}"; do
3636
case $i in
37-
mina-berkeley|mina-devnet|mina-mainnet|mina-berkeley-lightnet)
37+
mina-berkeley*|mina-devnet|mina-mainnet)
3838
# Downaload mina-logproc too
3939
source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER
4040
;;

‎buildkite/scripts/debian/promote.sh

+13-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ while [[ "$#" -gt 0 ]]; do case $1 in
1212
-s|--from-component) FROM_COMPONENT="$2"; shift;;
1313
-t|--to-component) TO_COMPONENT="$2"; shift;;
1414
--new-name) NEW_NAME="$2"; shift;;
15+
--new-repo) NEW_REPO="$2"; shift;;
16+
-r|--repo) REPO="$2"; shift;;
17+
--repo-key) REPO_KEY="$2"; shift;;
1518
*) echo "Unknown parameter passed: $1"; exit 1;;
1619
esac; shift; done
1720

@@ -28,6 +31,9 @@ function usage() {
2831
echo " -s, --from-component The source channel in which package currently resides"
2932
echo " -t, --to-component The target channel for package (unstable, alpha, beta etc.)"
3033
echo " -c, --codename The Debian codename (bullseye, focal etc.)"
34+
echo " -r, --repo The Debian source repo"
35+
echo " --new-repo The Debian target repo. By default equal to repo"
36+
echo " --repo-key The Debian target repo key"
3137
echo ""
3238
echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly"
3339
exit 1
@@ -40,6 +46,9 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi;
4046
if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi;
4147
if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi;
4248
if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi;
49+
if [[ -z "$REPO" ]]; then usage "Repository is not set!"; fi;
50+
if [[ -z "$NEW_REPO" ]]; then NEW_REPO=$REPO; fi;
51+
if [[ -z "$REPO_KEY" ]]; then usage "Target repository key is not set!"; fi;
4352

4453
# check for AWS Creds
4554
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
@@ -65,5 +74,8 @@ else
6574
--new-version $NEW_VERSION \
6675
--suite $FROM_COMPONENT \
6776
--new-suite $TO_COMPONENT \
68-
--new-name $NEW_NAME
77+
--new-name $NEW_NAME \
78+
--repo $REPO \
79+
--new-repo $NEW_REPO
80+
--sign $REPO_KEY
6981
fi

‎buildkite/scripts/debian/publish.sh

+11-1
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,18 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then
1515
exit 0
1616
fi
1717

18+
if [ -z "${SIGN:-}" ]; then
19+
SIGN_ARG=""
20+
else
21+
sudo chown -R opam ~/.gnupg/
22+
gpg --batch --yes --import /var/secrets/debian/key.gpg
23+
SIGN_ARG="--sign $SIGN"
24+
fi
25+
1826
source scripts/debian/publish.sh \
1927
--names "${DOWNLOAD_FOLDER}/mina-*.deb" \
2028
--release $MINA_DEB_RELEASE \
2129
--version $MINA_DEB_VERSION \
22-
--codename $MINA_DEB_CODENAME
30+
--codename $MINA_DEB_CODENAME \
31+
--bucket $BUCKET \
32+
$SIGN_ARG

‎buildkite/scripts/rosetta-integration-tests.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ cat <<EOF >"$MINA_CONFIG_FILE"
9696
"daemon": { "network_id": "${MINA_NETWORK}" },
9797
"ledger": {
9898
"accounts": [
99-
{ "pk": "${BLOCK_PRODUCER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },
99+
{ "pk": "${BLOCK_PRODUCER_PUB_KEY}", "balance": "600000000", "delegate": null, "sk": null },
100100
{ "pk": "${SNARK_PRODUCER_PK}", "balance": "2000000", "delegate": "${BLOCK_PRODUCER_PUB_KEY}", "sk": null },
101101
{ "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },
102102
{ "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },

‎buildkite/scripts/run-snark-transaction-profiler.sh

-23
This file was deleted.

‎buildkite/scripts/tests/ledger_test_apply.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
5+
# Don't prompt for answers during apt-get install
6+
export DEBIAN_FRONTEND=noninteractive
7+
8+
sudo apt-get update
9+
sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 python3-pip wget
10+
11+
git config --global --add safe.directory /workdir
12+
13+
source buildkite/scripts/export-git-env-vars.sh
14+
15+
source buildkite/scripts/debian/install.sh "mina-berkeley-instrumented" 1
16+
17+
echo "removing magic config files"
18+
sudo rm -f /var/lib/coda/config_*
19+
20+
./scripts/tests/ledger_test_apply.sh \
21+
--mina-app mina \
22+
--runtime-ledger-app mina-create-genesis

‎buildkite/scripts/zkapp_metrics.sh

-5
This file was deleted.

‎buildkite/src/Command/Base.dhall

+1
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ let targetToAgent =
131131
, Integration = toMap { size = "integration" }
132132
, QA = toMap { size = "qa" }
133133
, Hardfork = toMap { size = "hardfork" }
134+
, Perf = toMap { size = "perf" }
134135
, Multi = toMap { size = "generic-multi" }
135136
}
136137
target

‎buildkite/src/Command/Bench/Base.dhall

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
let PipelineMode = ../../Pipeline/Mode.dhall
2+
3+
let PipelineTag = ../../Pipeline/Tag.dhall
4+
5+
let Pipeline = ../../Pipeline/Dsl.dhall
6+
7+
let JobSpec = ../../Pipeline/JobSpec.dhall
8+
9+
let DebianVersions = ../../Constants/DebianVersions.dhall
10+
11+
let RunInToolchain = ../../Command/RunInToolchain.dhall
12+
13+
let Profiles = ../../Constants/Profiles.dhall
14+
15+
let Command = ../../Command/Base.dhall
16+
17+
let Docker = ../../Command/Docker/Type.dhall
18+
19+
let Size = ../Size.dhall
20+
21+
let Benchmarks = ../../Constants/Benchmarks.dhall
22+
23+
let SelectFiles = ../../Lib/SelectFiles.dhall
24+
25+
let Spec =
26+
{ Type =
27+
{ key : Text
28+
, bench : Text
29+
, label : Text
30+
, size : Size
31+
, name : Text
32+
, path : Text
33+
, mode : PipelineMode.Type
34+
, dependsOn : List Command.TaggedKey.Type
35+
, additionalDirtyWhen : List SelectFiles.Type
36+
, yellowThreshold : Double
37+
, redThreshold : Double
38+
}
39+
, default =
40+
{ mode = PipelineMode.Type.PullRequest
41+
, size = Size.Medium
42+
, dependsOn =
43+
DebianVersions.dependsOn
44+
DebianVersions.DebVersion.Bullseye
45+
Profiles.Type.Standard
46+
, additionalDirtyWhen = [] : List SelectFiles.Type
47+
, yellowThreshold = 0.1
48+
, redThreshold = 0.2
49+
}
50+
}
51+
52+
let command
53+
: Spec.Type -> Command.Type
54+
= \(spec : Spec.Type)
55+
-> Command.build
56+
Command.Config::{
57+
, commands =
58+
RunInToolchain.runInToolchain
59+
(Benchmarks.toEnvList Benchmarks.Type::{=})
60+
"./buildkite/scripts/benchmarks.sh ${spec.bench} --red-threshold ${Double/show
61+
spec.redThreshold} --yellow-threshold ${Double/show
62+
spec.yellowThreshold}"
63+
, label = "Perf: ${spec.label}"
64+
, key = spec.key
65+
, target = spec.size
66+
, docker = None Docker.Type
67+
, depends_on = spec.dependsOn
68+
}
69+
70+
let pipeline
71+
: Spec.Type -> Pipeline.Config.Type
72+
= \(spec : Spec.Type)
73+
-> Pipeline.Config::{
74+
, spec = JobSpec::{
75+
, dirtyWhen =
76+
[ SelectFiles.strictlyStart (SelectFiles.contains "src")
77+
, SelectFiles.exactly
78+
"buildkite/src/Command/Bench/Base"
79+
"dhall"
80+
, SelectFiles.contains "scripts/benchmark"
81+
, SelectFiles.contains "buildkite/scripts/benchmark"
82+
]
83+
# spec.additionalDirtyWhen
84+
, path = spec.path
85+
, name = spec.name
86+
, mode = spec.mode
87+
, tags =
88+
[ PipelineTag.Type.Long
89+
, PipelineTag.Type.Test
90+
, PipelineTag.Type.Stable
91+
]
92+
}
93+
, steps = [ command spec ]
94+
}
95+
96+
in { command = command, pipeline = pipeline, Spec = Spec }

‎buildkite/src/Command/DockerImage.dhall

+15-10
Original file line numberDiff line numberDiff line change
@@ -90,18 +90,23 @@ let generateStep =
9090
++ " --deb-build-flags ${BuildFlags.lowerName
9191
spec.build_flags}"
9292

93+
let remoteRepoCmds =
94+
[ Cmd.run
95+
( exportMinaDebCmd
96+
++ " && source ./buildkite/scripts/export-git-env-vars.sh "
97+
++ " && "
98+
++ buildDockerCmd
99+
++ " && "
100+
++ releaseDockerCmd
101+
)
102+
]
103+
93104
let commands =
94105
merge
95-
{ PackagesO1Test =
96-
[ Cmd.run
97-
( exportMinaDebCmd
98-
++ " && source ./buildkite/scripts/export-git-env-vars.sh "
99-
++ " && "
100-
++ buildDockerCmd
101-
++ " && "
102-
++ releaseDockerCmd
103-
)
104-
]
106+
{ PackagesO1Test = remoteRepoCmds
107+
, Unstable = remoteRepoCmds
108+
, Nightly = remoteRepoCmds
109+
, Stable = remoteRepoCmds
105110
, Local =
106111
[ Cmd.run
107112
( exportMinaDebCmd

‎buildkite/src/Command/MinaArtifact.dhall

+16-10
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ let MinaBuildSpec =
4646
, mode : PipelineMode.Type
4747
, tags : List PipelineTag.Type
4848
, channel : DebianChannel.Type
49+
, debianRepo : DebianRepo.Type
4950
}
5051
, default =
5152
{ prefix = "MinaArtifact"
@@ -58,6 +59,7 @@ let MinaBuildSpec =
5859
, mode = PipelineMode.Type.PullRequest
5960
, tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
6061
, channel = DebianChannel.Type.Unstable
62+
, debianRepo = DebianRepo.Type.PackagesO1Test
6163
}
6264
}
6365

@@ -112,12 +114,16 @@ let publish_to_debian_repo =
112114
Toolchain.select
113115
spec.toolchainSelectMode
114116
spec.debVersion
115-
[ "AWS_ACCESS_KEY_ID"
116-
, "AWS_SECRET_ACCESS_KEY"
117-
, "MINA_DEB_CODENAME=${DebianVersions.lowerName
118-
spec.debVersion}"
119-
, "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}"
120-
]
117+
( [ "AWS_ACCESS_KEY_ID"
118+
, "AWS_SECRET_ACCESS_KEY"
119+
, "MINA_DEB_CODENAME=${DebianVersions.lowerName
120+
spec.debVersion}"
121+
, "MINA_DEB_RELEASE=${DebianChannel.lowerName
122+
spec.channel}"
123+
]
124+
# DebianRepo.keyIdEnvList spec.debianRepo
125+
# DebianRepo.bucketEnvList spec.debianRepo
126+
)
121127
"./buildkite/scripts/debian/publish.sh"
122128
, label =
123129
"Publish Mina for ${DebianVersions.capitalName
@@ -251,10 +257,10 @@ let docker_step
251257
, deb_repo = DebianRepo.Type.Local
252258
, deb_profile = spec.profile
253259
, step_key =
254-
"test-suite-${DebianVersions.lowerName
255-
spec.debVersion}${Profiles.toLabelSegment
256-
spec.profile}${BuildFlags.toLabelSegment
257-
spec.buildFlags}--docker-image"
260+
"functional_test_suite-${DebianVersions.lowerName
261+
spec.debVersion}${Profiles.toLabelSegment
262+
spec.profile}${BuildFlags.toLabelSegment
263+
spec.buildFlags}-docker-image"
258264
, network = "berkeley"
259265
}
260266
]

0 commit comments

Comments
 (0)