DNM: Azurite testing

ci/nightly/pipeline.template.yml

@@ -226,18 +226,6 @@ steps:
               composition: testdrive
               args: [--redpanda]
 
-      - id: redpanda-testdrive-aarch64
-        label: ":panda_face: :racing_car: testdrive aarch64"
-        depends_on: build-aarch64
-        timeout_in_minutes: 180
-        agents:
-          queue: hetzner-aarch64-8cpu-16gb
-        plugins:
-          - ./ci/plugins/mzcompose:
-              composition: testdrive
-              args: [--redpanda]
-        skip: "Disabled due to taking too long for the value provided"
-
       # TODO(def-) Remove this when old upsert implementation is removed
       - id: testdrive-old-upsert
         label: ":racing_car: testdrive with old Upsert"
@@ -309,6 +297,51 @@ steps:
                 --system-param=persist_encoding_enable_dictionary=true,
               ]
 
+      - id: persistence-testdrive
+        label: ":racing_car: testdrive with --persistent-user-tables"
+        depends_on: build-aarch64
+        timeout_in_minutes: 30
+        agents:
+          queue: hetzner-aarch64-4cpu-8gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: testdrive
+              args: [--persistent-user-tables]
+        skip: "Persistence tests disabled"
+
+      - id: azurite-testdrive
+        label: "testdrive with :azure: blob store"
+        depends_on: build-aarch64
+        timeout_in_minutes: 180
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: testdrive
+              args: [--azurite]
+
+      - id: azurite-testdrive-replicas-4
+        label: ":racing_car: testdrive 4 replicas with :azure: blob store"
+        depends_on: build-aarch64
+        timeout_in_minutes: 180
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: testdrive
+              args: [--replicas=4, --azurite]
+
+      - id: azurite-testdrive-size-8
+        label: ":racing_car: testdrive with SIZE 8 and :azure: blob store"
+        depends_on: build-aarch64
+        timeout_in_minutes: 180
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: testdrive
+              args: [--default-size=8, --azurite]
+
       - id: testdrive-in-cloudtest
         label: "Full Testdrive in Cloudtest (K8s)"
         depends_on: build-aarch64
@@ -326,17 +359,6 @@ steps:
               args: [-m=long, test/cloudtest/test_full_testdrive.py, --no-test-parallelism]
         sanitizer: skip
 
-      - id: persistence-testdrive
-        label: ":racing_car: testdrive with --persistent-user-tables"
-        depends_on: build-aarch64
-        timeout_in_minutes: 30
-        agents:
-          queue: hetzner-aarch64-4cpu-8gb
-        plugins:
-          - ./ci/plugins/mzcompose:
-              composition: testdrive
-              args: [--persistent-user-tables]
-        skip: "Persistence tests disabled"
 
   - group: Limits
     key: limits-group
@@ -456,6 +478,17 @@ steps:
               composition: zippy
               args: [--scenario=KafkaSources, --actions=10000, --max-execution-time=30m]
 
+      - id: zippy-kafka-sources-azurite
+        label: "Zippy Kafka Sources with :azure: blob store"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: zippy
+              args: [--scenario=KafkaSources, --actions=10000, --max-execution-time=30m, --azurite]
+
       # TODO(def-) Remove this when old upsert implementation is removed
       - id: zippy-kafka-sources-old-upsert
         label: "Zippy Kafka Sources with old Upsert"
@@ -468,6 +501,18 @@ steps:
               composition: zippy
               args: [--scenario=KafkaSources, --actions=10000, --max-execution-time=30m, --system-param=storage_use_continual_feedback_upsert=false]
 
+      # TODO(def-) Remove this when old upsert implementation is removed
+      - id: zippy-kafka-sources-old-upsert-azurite
+        label: "Zippy Kafka Sources with old Upsert and :azure: blob store"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: zippy
+              args: [--scenario=KafkaSources, --actions=10000, --max-execution-time=30m, --system-param=storage_use_continual_feedback_upsert=false, --azurite]
+
       - id: zippy-kafka-parallel-insert
         label: "Zippy Kafka Parallel Insert"
         depends_on: build-aarch64
@@ -479,6 +524,17 @@ steps:
               composition: zippy
               args: [--scenario=KafkaParallelInsert, --transaction-isolation=serializable, --actions=10000, --max-execution-time=30m]
 
+      - id: zippy-kafka-parallel-insert-azurite
+        label: "Zippy Kafka Parallel Insert with :azurite: blob store"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
+        agents:
+          queue: hetzner-aarch64-8cpu-16gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: zippy
+              args: [--scenario=KafkaParallelInsert, --transaction-isolation=serializable, --actions=10000, --max-execution-time=30m, --azurite]
+
       - id: zippy-user-tables
         label: "Zippy User Tables"
         depends_on: build-aarch64
@@ -488,7 +544,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: zippy
-              args: [--scenario=UserTables, --actions=10000, --max-execution-time=30m]
+              args: [--scenario=UserTables, --actions=10000, --max-execution-time=30m, --azurite]
 
       - id: zippy-postgres-cdc
         label: "Zippy Postgres CDC"
@@ -499,7 +555,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: zippy
-              args: [--scenario=PostgresCdc, --actions=10000, --max-execution-time=30m]
+              args: [--scenario=PostgresCdc, --actions=10000, --max-execution-time=30m, --azurite]
 
       - id: zippy-mysql-cdc
         label: "Zippy MySQL CDC"
@@ -510,7 +566,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: zippy
-              args: [--scenario=MySqlCdc, --actions=10000, --max-execution-time=30m]
+              args: [--scenario=MySqlCdc, --actions=10000, --max-execution-time=30m, --azurite]
 
       - id: zippy-debezium-postgres
         label: "Zippy Debezium Postgres"
@@ -629,6 +685,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: testdrive-old-kafka-src-syntax
+              args: [--azurite]
         agents:
           queue: hetzner-aarch64-8cpu-16gb
 
@@ -675,6 +732,19 @@ steps:
   - group: "Platform checks"
     key: platform-checks
     steps:
+      - id: checks-no-restart-no-upgrade-azurite
+        label: "Checks without restart or upgrade with :azure: blob store %N"
+        depends_on: build-aarch64
+        inputs: [misc/python/materialize/checks]
+        timeout_in_minutes: 45
+        parallelism: 2
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: platform-checks
+              args: [--scenario=NoRestartNoUpgrade, "--seed=$BUILDKITE_JOB_ID", --azurite]
+
       - id: checks-restart-entire-mz
         label: "Checks + restart of the entire Mz"
         depends_on: build-aarch64
@@ -687,7 +757,25 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: platform-checks
-              args: [--scenario=RestartEntireMz, "--seed=$BUILDKITE_JOB_ID"]
+              args: [--scenario=RestartEntireMz, "--seed=$BUILDKITE_JOB_ID", --azurite]
+
+      - id: checks-restart-environmentd-clusterd-storage-azurite
+        label: "Checks + restart of environmentd & storage clusterd with :azure: blob store %N"
+        depends_on: build-aarch64
+        inputs: [misc/python/materialize/checks]
+        timeout_in_minutes: 45
+        parallelism: 2
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: platform-checks
+              args:
+                [
+                  --scenario=RestartEnvironmentdClusterdStorage,
+                  "--seed=$BUILDKITE_JOB_ID",
+                  --azurite,
+                ]
 
       # TODO(def-) Remove this when old upsert implementation is removed
       - id: checks-restart-entire-mz-old-upsert
@@ -752,6 +840,18 @@ steps:
               composition: platform-checks
               args: [--scenario=RestartEntireMz, --execution-mode=parallel, "--seed=$BUILDKITE_JOB_ID"]
 
+      - id: checks-parallel-restart-entire-mz-azurite
+        label: "Checks parallel + restart of the entire Mz with :azure: blob store %N"
+        depends_on: build-aarch64
+        timeout_in_minutes: 180
+        parallelism: 2
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: platform-checks
+              args: [--scenario=RestartEntireMz, --execution-mode=parallel, "--seed=$BUILDKITE_JOB_ID", --azurite]
+
       - id: checks-parallel-restart-environmentd-clusterd-storage
         label: "Checks parallel + restart of environmentd & storage clusterd"
         depends_on: build-aarch64
@@ -824,8 +924,8 @@ steps:
               composition: platform-checks
               args: [--scenario=UpgradeEntireMzFourVersions, "--seed=$BUILDKITE_JOB_ID"]
 
-      - id: checks-0dt-restart-entire-mz-forced-migrations
-        label: "Checks 0dt restart of the entire Mz with forced migrations"
+      - id: checks-0dt-restart-entire-mz-forced-migrations-azurite
+        label: "Checks 0dt restart of the entire Mz with forced migrations with :azure: blob store"
         depends_on: build-aarch64
         timeout_in_minutes: 60
         parallelism: 2
@@ -834,7 +934,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: platform-checks
-              args: [--scenario=ZeroDowntimeRestartEntireMzForcedMigrations, "--seed=$BUILDKITE_JOB_ID"]
+              args: [--scenario=ZeroDowntimeRestartEntireMzForcedMigrations, "--seed=$BUILDKITE_JOB_ID", --azurite]
 
       - id: checks-0dt-upgrade-entire-mz
         label: "Checks 0dt upgrade, whole-Mz restart"
@@ -1414,6 +1514,7 @@ steps:
     plugins:
       - ./ci/plugins/mzcompose:
           composition: data-ingest
+          args: [--azurite]
 
   - group: "Parallel Workload"
     key: parallel-workload
@@ -1431,7 +1532,7 @@ steps:
               args: [--runtime=1500, --complexity=dml, --threads=16]
 
       - id: parallel-workload-ddl
-        label: "Parallel Workload (DDL)"
+        label: "Parallel Workload (DDL) with :azure: blob store"
         depends_on: build-aarch64
         artifact_paths: [parallel-workload-queries.log.zst]
         timeout_in_minutes: 90
@@ -1440,7 +1541,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: parallel-workload
-              args: [--runtime=1500, --threads=8]
+              args: [--runtime=1500, --threads=8, --azurite]
 
       - id: parallel-workload-ddl-only
         label: "Parallel Workload (DDL Only)"
@@ -1481,7 +1582,7 @@ steps:
               args: [--runtime=1500, --scenario=rename, --naughty-identifiers, --threads=16]
 
       - id: parallel-workload-rename
-        label: "Parallel Workload (rename)"
+        label: "Parallel Workload (rename) with :azure: blob store"
         depends_on: build-aarch64
         artifact_paths: [parallel-workload-queries.log.zst]
         timeout_in_minutes: 90
@@ -1490,7 +1591,7 @@ steps:
         plugins:
           - ./ci/plugins/mzcompose:
               composition: parallel-workload
-              args: [--runtime=1500, --scenario=rename, --threads=16]
+              args: [--runtime=1500, --scenario=rename, --threads=16, --azurite]
 
       - id: parallel-workload-cancel
         label: "Parallel Workload (cancel)"
@@ -1628,6 +1729,7 @@ steps:
     plugins:
       - ./ci/plugins/mzcompose:
           composition: txn-wal-fencing
+          args: [--azurite]
     agents:
       queue: hetzner-aarch64-8cpu-16gb
 
@@ -1720,6 +1822,18 @@ steps:
           composition: sqllogictest
           run: slow-tests
 
+  - id: cluster-tests-azurite
+    label: "Cluster tests with :azure: blob store"
+    depends_on: build-aarch64
+    timeout_in_minutes: 120
+    inputs: [test/cluster]
+    plugins:
+      - ./ci/plugins/mzcompose:
+          composition: cluster
+          args: ["--azurite"]
+    agents:
+      queue: hetzner-aarch64-8cpu-16gb
+
   - group: "Language tests"
     key: language-tests
     steps:

deny.toml

@@ -48,10 +48,8 @@ skip = [
     { name = "windows-sys", version = "0.52.0" },
     # Newer versions of crates like `tempfile` are held back by crates like `atty`.
     # This is very Unfortunate as we don't actually use these platforms.
-    { name = "hermit-abi", version = "0.1.6" },
     { name = "hermit-abi", version = "0.2.6" },
     { name = "redox_syscall", version = "0.2.10" },
-    { name = "linux-raw-sys", version = "0.3.4" },
     { name = "rustix", version = "0.38.21" },
 
     # Will require updating many crates
@@ -118,6 +116,9 @@ skip = [
     { name = "sync_wrapper", version = "0.1.2" },
 
     { name = "memmap2", version = "0.5.4" },
+
+    { name = "wasi", version = "0.9.0+wasi-snapshot-preview1" },
+    { name = "wasi", version = "0.11.0+wasi-snapshot-preview1" },
 ]
 
 # Use `tracing` instead.
@@ -187,7 +188,6 @@ wrappers = [
   "bindgen",
   "bstr",
   "console",
-  "criterion",
   "dynfmt",
   "findshlibs",
   "insta",
@@ -227,14 +227,10 @@ version = 2
 ignore = [
     # Consider switching `yaml-rust` to the actively maintained `yaml-rust2` fork of the original project
     "RUSTSEC-2024-0320",
-    # Consider `std::io::IsTerminal` or `is-terminal` instead of `atty` (unmaintained)
-    "RUSTSEC-2021-0145",
     # Consider `encoding_rs` instead of `encoding` (unmaintained)
     "RUSTSEC-2021-0153",
     # proc-macro-error is unmaintained, possible alternative: proc-macro-error2
     "RUSTSEC-2024-0370",
-    # Use standard library's IsTerminal trait instead of `atty` (unmaintained)
-    "RUSTSEC-2024-0375",
     # `derivative` is unmaintained; consider using an alternative (unmaintained)
     "RUSTSEC-2024-0388",
     # `instant` is unmaintained, and the author recommends using the maintained [`web-time`] crate instead.

misc/python/materialize/checks/mzcompose_actions.py

@@ -61,6 +61,7 @@ def __init__(
         self.restart = restart
         self.force_migrations = force_migrations
         self.publish = publish
+        self.scenario = scenario
 
     def execute(self, e: Executor) -> None:
         c = e.mzcompose_composition()
@@ -72,7 +73,8 @@ def execute(self, e: Executor) -> None:
             name=self.mz_service,
             image=image,
             external_metadata_store=True,
-            external_minio=True,
+            external_blob_store=True,
+            blob_store_is_azure=self.scenario.azurite,
             environment_extra=self.environment_extra,
             system_parameter_defaults=self.system_parameter_defaults,
             additional_system_parameter_defaults=self.additional_system_parameter_defaults,