diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 00000000..53cc4d31 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,93 @@ +name: Integration Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +env: + SCYLLADB_IMAGE_REGISTRY: "scylladb/scylla" + SCYLLADB_IMAGE_TAG: "2026.1" + CARGO_TERM_COLOR: always + +jobs: + integration-test: + name: Integration Test with ScyllaDB + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: true + + - name: Start 3-node ScyllaDB cluster + run: | + sudo sysctl -w fs.aio-max-nr=1048576 + + log_resources() { + echo "=== Resources: $1 ===" + free -h + docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" 2>/dev/null || true + echo "===" + } + + wait_for_cql() { + local container="$1" + echo "Waiting for ${container} to be ready..." + for i in $(seq 1 60); do + if ! docker ps --format '{{.Names}}' | grep -q "^${container}$"; then + echo "${container} is not running!" + docker logs "${container}" + return 1 + fi + if docker exec "${container}" cqlsh -e "SELECT now() FROM system.local;" 2>/dev/null; then + echo "${container} ready after ${i} attempt(s)" + return 0 + fi + if [ "$((i % 12))" -eq 0 ]; then + echo "${container} not ready after ${i} attempts, last 20 log lines:" + docker logs --tail 20 "${container}" + fi + if [ "$i" -eq 60 ]; then + echo "${container} failed to become ready" + docker logs "${container}" + return 1 + fi + sleep 5 + done + } + + log_resources "before any containers" + + docker run -d --name scylladb-node1 \ + -p 9042:9042 \ + ${{ env.SCYLLADB_IMAGE_REGISTRY }}:${{ env.SCYLLADB_IMAGE_TAG }} \ + --smp 1 --memory 512M --overprovisioned 1 --skip-wait-for-gossip-to-settle 0 + wait_for_cql scylladb-node1 + log_resources "after scylladb-node1" + + docker run -d --name scylladb-node2 \ + ${{ env.SCYLLADB_IMAGE_REGISTRY }}:${{ env.SCYLLADB_IMAGE_TAG }} \ + --smp 1 --memory 512M --overprovisioned 1 --skip-wait-for-gossip-to-settle 0 \ + --seeds="$(docker inspect --format='{{ .NetworkSettings.Networks.bridge.IPAddress }}' scylladb-node1)" + wait_for_cql scylladb-node2 + log_resources "after scylladb-node2" + + docker run -d --name scylladb-node3 \ + ${{ env.SCYLLADB_IMAGE_REGISTRY }}:${{ env.SCYLLADB_IMAGE_TAG }} \ + --smp 1 --memory 512M --overprovisioned 1 --skip-wait-for-gossip-to-settle 0 \ + --seeds="$(docker inspect --format='{{ .NetworkSettings.Networks.bridge.IPAddress }}' scylladb-node1)" + wait_for_cql scylladb-node3 + log_resources "after scylladb-node3" + + - name: Run integration test(s) + env: + SCYLLA_TEST_HOST: "127.0.0.1" + SCYLLA_TEST_PORT: "9042" + run: cargo test --test integration_test -- --ignored --nocapture diff --git a/Cargo.lock b/Cargo.lock index dbd5da61..26a67f76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -642,6 +642,56 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-named-pipe", + "hyper-rustls 0.27.7", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls 0.23.35", + "rustls-native-certs 0.8.2", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.17", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "borsh" version = "1.5.7" @@ -959,8 +1009,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -977,13 +1037,38 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.110", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", "quote", "syn 2.0.110", ] @@ -1009,6 +1094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -1033,12 +1119,29 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "docker_credential" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -1076,12 +1179,34 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.4" @@ -1313,7 +1438,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -1332,7 +1457,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -1421,6 +1546,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "0.2.12" @@ -1526,6 +1660,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -1534,6 +1669,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.8.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1604,6 +1754,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hytra" version = "0.1.2" @@ -1749,6 +1914,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.12.0" @@ -1757,6 +1933,8 @@ checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", "hashbrown 0.16.0", + "serde", + "serde_core", ] [[package]] @@ -1882,6 +2060,7 @@ dependencies = [ "statrs", "status-line", "strum", + "testcontainers", "thiserror 2.0.17", "time", "tokio", @@ -1926,6 +2105,17 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags 2.10.0", + "libc", + "redox_syscall 0.7.0", +] + [[package]] name = "libssh2-sys" version = "0.3.1" @@ -2392,11 +2582,36 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.110", +] + [[package]] name = "parse_duration" version = "2.1.1" @@ -2689,6 +2904,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2698,6 +2922,35 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +dependencies = [ + "bitflags 2.10.0", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "regex" version = "1.12.2" @@ -2772,7 +3025,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -3053,6 +3306,7 @@ checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.103.8", "subtle", @@ -3066,7 +3320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework 2.11.1", ] @@ -3092,6 +3346,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.13.1" @@ -3162,6 +3425,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -3231,7 +3518,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16c9e9660119726312cd6c7bd3e286ffc80fb06a9b6d0e214a16df00600829e9" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.110", @@ -3338,6 +3625,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "serde_spanned" version = "1.0.3" @@ -3359,6 +3657,37 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.12.0", + "schemars 0.9.0", + "schemars 1.2.0", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3487,6 +3816,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.110", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "strum" version = "0.26.3" @@ -3603,6 +3955,35 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "testcontainers" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a4f01f39bb10fc2a5ab23eb0d888b1e2bb168c157f61a1b98e6c501c639c74" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.17", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "url", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -3789,6 +4170,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.17" @@ -3808,7 +4204,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ - "indexmap", + "indexmap 2.12.0", "serde_core", "serde_spanned", "toml_datetime", @@ -3832,7 +4228,7 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap", + "indexmap 2.12.0", "toml_datetime", "toml_parser", "winnow", @@ -4538,6 +4934,16 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index f424c228..1ff1dcd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ rstest = "0.22" # NOTE: pin the tokio version to be 1.44.2 to avoid newer versions with performance regression # See: https://github.com/tokio-rs/tokio/issues/7744 tokio = { version = "=1.44.2", features = ["rt", "test-util", "macros"] } +testcontainers = "0.23" [features] default = ["cql"] diff --git a/docs/plans/integration-test-with-testcontainers.md b/docs/plans/integration-test-with-testcontainers.md new file mode 100644 index 00000000..e7482f25 --- /dev/null +++ b/docs/plans/integration-test-with-testcontainers.md @@ -0,0 +1,59 @@ +# Integration Test with Testcontainers-rs + +## Objective + +Create CLI-level integration tests using testcontainers-rs with a real ScyllaDB instance. +Tests invoke the `latte` binary as a subprocess (like a user would), validating the full stack end-to-end. + +## Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Testcontainers API | Async (`tokio` runner) | Project is tokio-based; avoids `blocking` feature overhead | +| Wait strategy | TCP connect loop on CQL port (9042) with backoff | Reliable (~5-15s), no log-message fragility, no wasteful fixed sleep | +| ScyllaDB image registry | configurable via 'SCYLLADB_IMAGE_REGISTRY', default is `scylladb/scylla` | ScyllaDB image registry | +| ScyllaDB image tag | configurable via 'SCYLLADB_IMAGE_TAG', default is `2026.1` | ScyllaDB image tag | +| Test level | CLI (subprocess invocation of `latte` binary) | End-to-end validation, tests what users actually run | +| Assertions | Exit code 0 + stdout contains throughput metrics | Meaningful validation beyond "something printed" | +| Test attribute | `#[ignore]` | Don't slow down regular `cargo test` | +| CI trigger | push to main + PRs + manual `workflow_dispatch` | Flexibility for on-demand runs | + +## Implementation + +### Files Modified + +- `Cargo.toml` – add `testcontainers = "0.23"` to dev-dependencies (no `blocking` feature) + +### Files Created + +- `tests/integration_test.rs` – async integration test +- `.github/workflows/integration-test.yml` – dedicated CI workflow + +### Test Strategy + +1. Start `scylladb/scylla-enterprise:{version}` container via testcontainers async API +2. Wait for CQL port readiness using TCP connect loop (max 60s, 2s interval) +3. Build latte in release mode (within the test, leveraging cargo caching) +4. Invoke `latte run workloads/basic/write.rn --hosts 127.0.0.1:{port} --duration 10s --warmup 0s` +5. Assert: exit code 0, stdout contains throughput indicator + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `SCYLLADB_IMAGE_REGISTRY` | `scylladb/scylla` | ScyllaDB image registry | +| `SCYLLADB_IMAGE_TAG` | `2026.1` | ScyllaDB image tag | + +### Running Locally + +```bash +cargo test --test integration_test -- --ignored --nocapture +``` + +### Success Criteria + +- ScyllaDB container starts and becomes ready +- `latte schema` command runs successfully +- `latte run` commands run successfully +- Test passes in GitHub Actions CI +- No changes to existing functionality diff --git a/tests/integration_test.rs b/tests/integration_test.rs new file mode 100644 index 00000000..6c965c23 --- /dev/null +++ b/tests/integration_test.rs @@ -0,0 +1,248 @@ +use std::net::TcpStream; +use std::os::unix::process::CommandExt; +use std::process::{Command, ExitStatus, Stdio}; +use std::sync::OnceLock; +use std::time::Duration; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; + +const CQL_PORT: u16 = 9042; + +struct ScyllaDb { + _container: Option>, + host: String, + port: u16, +} + +type StartResult = Result; + +static SCYLLA: OnceLock = OnceLock::new(); +static LATTE_BUILT: OnceLock = OnceLock::new(); + +fn scylla() -> &'static ScyllaDb { + SCYLLA + .get_or_init(|| { + tokio::runtime::Runtime::new() + .unwrap() + .block_on(start_scylla()) + }) + .as_ref() + .expect("ScyllaDB container is not available") +} + +async fn start_scylla() -> StartResult { + if let Ok(host) = std::env::var("SCYLLA_TEST_HOST") { + let port = std::env::var("SCYLLA_TEST_PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(CQL_PORT); + eprintln!("Using pre-existing ScyllaDB at {}:{}", host, port); + return Ok(ScyllaDb { + _container: None, + host, + port, + }); + } + + let image_registry = + std::env::var("SCYLLADB_IMAGE_REGISTRY").unwrap_or_else(|_| "scylladb/scylla".into()); + let image_tag = std::env::var("SCYLLADB_IMAGE_TAG").unwrap_or_else(|_| "latest".into()); + eprintln!("Starting {}:{}", image_registry, image_tag); + + let container = GenericImage::new(&image_registry, &image_tag) + .with_exposed_port(CQL_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr("serving")) + .with_cmd(vec![ + "--smp".to_string(), + "1".to_string(), + "--memory".to_string(), + "512M".to_string(), + "--overprovisioned".to_string(), + "1".to_string(), + "--skip-wait-for-gossip-to-settle".to_string(), + "0".to_string(), + ]) + .with_startup_timeout(Duration::from_secs(120)) + .start() + .await + .map_err(|e| format!("failed to start ScyllaDB container: {e}"))?; + + let port = container + .get_host_port_ipv4(CQL_PORT.tcp()) + .await + .map_err(|e| format!("failed to get mapped port: {e}"))?; + + let host = String::from("127.0.0.1"); + + wait_for_cql(&host, port).await; + + Ok(ScyllaDb { + _container: Some(container), + host, + port, + }) +} + +async fn wait_for_cql(host: &str, port: u16) { + let addr = format!("{}:{}", host, port); + for attempt in 0..60 { + if TcpStream::connect(&addr).is_ok() { + eprintln!("ScyllaDB ready on {} (attempt {})", addr, attempt + 1); + return; + } + tokio::time::sleep(Duration::from_secs(2)).await; + if attempt > 0 && attempt % 10 == 0 { + eprintln!("Still waiting for ScyllaDB on {}...", addr); + } + } + panic!("ScyllaDB did not become ready on {} within 120s", addr); +} + +fn ensure_latte_built() { + LATTE_BUILT.get_or_init(|| { + let status = Command::new("cargo") + .args(["build", "--release"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .status() + .expect("Failed to invoke cargo build"); + assert!(status.success(), "cargo build --release failed"); + true + }); +} + +fn latte_binary() -> String { + format!("{}/target/release/latte", env!("CARGO_MANIFEST_DIR")) +} + +fn hosts_arg(db: &ScyllaDb) -> String { + format!("{}:{}", db.host, db.port) +} + +fn workload_path(name: &str) -> String { + format!("{}/workloads/{}", env!("CARGO_MANIFEST_DIR"), name) +} + +struct CommandResult { + status: ExitStatus, + output: String, +} + +fn run_command(mut cmd: Command) -> CommandResult { + unsafe { + cmd.pre_exec(|| { + // Redirect stderr (fd 2) to stdout (fd 1) so both streams share one pipe + extern "C" { + fn dup2(oldfd: i32, newfd: i32) -> i32; + } + if dup2(1, 2) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + + let output = cmd + .stdout(Stdio::piped()) + .output() + .expect("Failed to run command"); + + CommandResult { + status: output.status, + output: String::from_utf8_lossy(&output.stdout).into_owned(), + } +} + +fn latte_schema(db: &ScyllaDb, workload: &str) -> CommandResult { + println!( + "{}", + format_args!( + "Running the 'latte schema' command for the '{}' rune script", + workload + ) + ); + let mut cmd = Command::new(latte_binary()); + cmd.args(["schema", &workload_path(workload), &hosts_arg(db)]) + .current_dir(env!("CARGO_MANIFEST_DIR")); + let result = run_command(cmd); + assert!( + result.status.success(), + "latte schema failed:\n{}", + result.output + ); + result +} + +fn latte_run(db: &ScyllaDb, workload: &str, duration: &str, extra_args: &[&str]) -> CommandResult { + let workload = workload_path(workload); + let hosts = hosts_arg(db); + let mut args: Vec<&str> = vec![ + "run", + &workload, + &hosts, + "--duration", + &duration, + "--warmup", + "0s", + "-q", + ]; + args.extend_from_slice(extra_args); + + println!( + "{}", + format_args!( + "Running the 'latte run' command with the following params: {:?}", + &args + ) + ); + + let mut cmd = Command::new(latte_binary()); + cmd.args(&args).current_dir(env!("CARGO_MANIFEST_DIR")); + let result = run_command(cmd); + + eprintln!("latte output:\n{}", result.output); + + result +} + +fn assert_latte_success(result: &CommandResult) { + assert!( + result.status.success(), + "latte failed (exit {:?}):\n{}", + result.status.code(), + result.output + ); +} + +fn assert_has_throughput_metrics(result: &CommandResult) { + assert!( + result.output.contains("thrpt") + || result.output.contains("op/s") + || result.output.contains("req/s"), + "Expected throughput metrics in latte output:\n{}", + result.output + ); +} + +#[test] +#[ignore] +fn test_latte_data_validation_workload() { + let db = scylla(); + ensure_latte_built(); + + let rune_path = "data_validation.rn"; + let duration = "50000"; + + println!("\n[TEST-INFO] Phase 1: Create the schema"); + latte_schema(db, rune_path); + + println!("\n[TEST-INFO] Phase 2: Data population"); + let populate_result = latte_run(db, rune_path, duration, &["-f=insert"]); + assert_latte_success(&populate_result); + assert_has_throughput_metrics(&populate_result); + + println!("\n[TEST-INFO] Phase 3: Data validation"); + let data_validation_result = latte_run(db, rune_path, duration, &["-f=get_by_ck"]); + assert_latte_success(&data_validation_result); + assert_has_throughput_metrics(&data_validation_result); +}