diff --git a/.gitignore b/.gitignore index 08cc6b4312c56..bc02ffc5b3150 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ flake.lock /known-docker-images.txt /test/sqllogictest/sqlite my-local-mz/ +/test/orchestratord/cluster.yaml +uv.lock diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index 0d299f27db68d..1e2fcee23d2d3 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -2367,6 +2367,7 @@ steps: steps: - id: orchestratord-defaults label: "Orchestratord test (defaults from documentation)" + artifact_paths: ["mz_debug_*.zip"] depends_on: devel-docker-tags timeout_in_minutes: 120 plugins: @@ -2379,6 +2380,7 @@ steps: - id: orchestratord-default-properties label: "Orchestratord test (defaults for properties)" + artifact_paths: ["mz_debug_*.zip"] depends_on: devel-docker-tags timeout_in_minutes: 120 plugins: @@ -2391,6 +2393,7 @@ steps: - id: orchestratord-individual label: "Orchestratord test (individual properties)" + artifact_paths: ["mz_debug_*.zip"] depends_on: devel-docker-tags timeout_in_minutes: 120 plugins: @@ -2403,76 +2406,65 @@ steps: - id: orchestratord-combine label: "Orchestratord test (combine properties)" + artifact_paths: ["mz_debug_*.zip"] depends_on: build-aarch64 - timeout_in_minutes: 120 + timeout_in_minutes: 60 plugins: - ./ci/plugins/mzcompose: composition: orchestratord - args: [--action=noop, --properties=combine, --runtime=3600, --recreate-cluster] + args: [--action=noop, --properties=combine, --runtime=1800, --recreate-cluster] ci-builder: stable agents: queue: hetzner-aarch64-16cpu-32gb - id: orchestratord-upgrade-individual label: "Orchestratord test (upgrade, individual props)" + artifact_paths: ["mz_debug_*.zip"] depends_on: devel-docker-tags timeout_in_minutes: 120 plugins: - ./ci/plugins/mzcompose: composition: orchestratord - args: [--action=upgrade, --properties=individual, --runtime=3600, --recreate-cluster] + args: [--action=upgrade, --properties=individual, --runtime=1800, --recreate-cluster] ci-builder: stable - env: - # Old versions are not on GHCR yet - MZ_GHCR: 0 agents: - queue: hetzner-aarch64-8cpu-16gb - skip: "https://github.com/MaterializeInc/materialize/pull/34214" + queue: hetzner-aarch64-16cpu-32gb - id: orchestratord-upgrade-combine label: "Orchestratord test (upgrade, combine props)" + artifact_paths: ["mz_debug_*.zip"] depends_on: devel-docker-tags timeout_in_minutes: 120 plugins: - ./ci/plugins/mzcompose: composition: orchestratord - args: [--action=upgrade, --properties=combine, --runtime=3600, --recreate-cluster] + args: [--action=upgrade, --properties=combine, --runtime=1800, --recreate-cluster] ci-builder: stable - env: - # Old versions are not on GHCR yet - MZ_GHCR: 0 agents: - queue: hetzner-aarch64-8cpu-16gb - skip: "https://github.com/MaterializeInc/materialize/pull/34214" + queue: hetzner-aarch64-16cpu-32gb - id: orchestratord-upgrade-chain-individual label: "Orchestratord test (upgrade chain, individual props)" + artifact_paths: ["mz_debug_*.zip"] depends_on: devel-docker-tags timeout_in_minutes: 120 plugins: - ./ci/plugins/mzcompose: composition: orchestratord - args: [--action=upgrade-chain, --properties=individual, --runtime=3600, --recreate-cluster] + args: [--action=upgrade-chain, --properties=individual, --runtime=1800, --recreate-cluster] ci-builder: stable - env: - # Old versions are not on GHCR yet - MZ_GHCR: 0 agents: - queue: hetzner-aarch64-8cpu-16gb - skip: "https://github.com/MaterializeInc/materialize/pull/34214" + queue: hetzner-aarch64-16cpu-32gb - id: orchestratord-upgrade-chain-combine label: "Orchestratord test (upgrade chain, combine props)" + artifact_paths: ["mz_debug_*.zip"] depends_on: devel-docker-tags timeout_in_minutes: 120 plugins: - ./ci/plugins/mzcompose: composition: orchestratord - args: [--action=upgrade-chain, --properties=combine, --runtime=3600, --recreate-cluster] + args: [--action=upgrade-chain, --properties=combine, --runtime=1800, --recreate-cluster] ci-builder: stable - env: - # Old versions are not on GHCR yet - MZ_GHCR: 0 agents: queue: hetzner-aarch64-16cpu-32gb - skip: "https://github.com/MaterializeInc/materialize/pull/34214" diff --git a/misc/python/materialize/version_list.py b/misc/python/materialize/version_list.py index d9ce031b04f6d..8cb625f811221 100644 --- a/misc/python/materialize/version_list.py +++ b/misc/python/materialize/version_list.py @@ -94,6 +94,9 @@ def get_supported_self_managed_versions() -> list[MzVersion]: MzVersion.parse_mz("v0.130.2"), MzVersion.parse_mz("v0.130.3"), MzVersion.parse_mz("v0.130.4"), + # orchestratord test is failing with v0.130.7/8 versions, not sure yet why + MzVersion.parse_mz("v0.130.7"), + MzVersion.parse_mz("v0.130.8"), MzVersion.parse_mz( "v0.147.7" ), # Incompatible for upgrades because it clears login attribute for roles due to catalog migration diff --git a/test/orchestratord/cluster.yaml b/test/orchestratord/cluster.yaml.tmpl similarity index 96% rename from test/orchestratord/cluster.yaml rename to test/orchestratord/cluster.yaml.tmpl index bd7f1aa35cedc..0d0a9a461da95 100644 --- a/test/orchestratord/cluster.yaml +++ b/test/orchestratord/cluster.yaml.tmpl @@ -23,6 +23,9 @@ kubeadmConfigPatches: nodes: - role: control-plane image: kindest/node:v1.32.5 + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: "$HOME/.docker/config.json" extraPortMappings: - containerPort: 32000 hostPort: 32000 @@ -164,6 +167,7 @@ nodes: image: kindest/node:v1.32.5 labels: materialize.cloud/scratch-fs: "true" + materialize.cloud/disk: "true" materialize.cloud/availability-zone: "2" topology.kubernetes.io/zone: "2" workload: "materialize-instance" diff --git a/test/orchestratord/mzcompose.py b/test/orchestratord/mzcompose.py index d49cbd2c1f26b..2edfe6bd6c161 100644 --- a/test/orchestratord/mzcompose.py +++ b/test/orchestratord/mzcompose.py @@ -30,8 +30,7 @@ import yaml from semver.version import Version -from materialize import MZ_ROOT, ci_util, git, spawn, ui -from materialize.docker import MZ_GHCR_DEFAULT +from materialize import MZ_ROOT, ci_util, git, spawn from materialize.mz_version import MzVersion from materialize.mzcompose.composition import ( Composition, @@ -41,6 +40,7 @@ from materialize.mzcompose.services.balancerd import Balancerd from materialize.mzcompose.services.clusterd import Clusterd from materialize.mzcompose.services.environmentd import Environmentd +from materialize.mzcompose.services.mz_debug import MzDebug from materialize.mzcompose.services.orchestratord import Orchestratord from materialize.mzcompose.services.testdrive import Testdrive from materialize.util import all_subclasses @@ -55,9 +55,26 @@ Environmentd(), Clusterd(), Balancerd(), + MzDebug(), ] +def run_mz_debug() -> None: + # TODO: Hangs a lot in CI + # Only using capture because it's too noisy + # spawn.capture( + # [ + # "./mz-debug", + # "self-managed", + # "--k8s-namespace", + # "materialize-environment", + # "--mz-instance-name", + # "12345678-1234-1234-1234-123456789012", + # ] + # ) + pass + + def get_tag(tag: str | None = None) -> str: # We can't use the mzbuild tag because it has a different fingerprint for # environmentd/clusterd/balancerd and the orchestratord depends on them @@ -242,7 +259,9 @@ def all_modifications() -> list[type[Modification]]: class LicenseKey(Modification): @classmethod def values(cls, version: MzVersion) -> list[Any]: - return ["valid", "invalid", "del"] + # TODO: Reenable "del" when database-issues#9928 is fixed + # return ["valid", "invalid", "del"] + return ["valid", "invalid"] @classmethod def failed_reconciliation_values(cls) -> list[Any]: @@ -498,14 +517,9 @@ def validate(self, mods: dict[type[Modification], Any]) -> None: def check() -> None: environmentd = get_environmentd_data() image = environmentd["items"][0]["spec"]["containers"][0]["image"] - image_registry = ( - "ghcr.io/materializeinc/materialize" - if ui.env_is_truthy("MZ_GHCR", MZ_GHCR_DEFAULT) - else "materialize" - ) - expected = f"{image_registry}/environmentd:{self.value}" + expected = f"materialize/environmentd:{self.value}" assert ( - image == expected + image == expected or f"ghcr.io/materializeinc/{image}" == expected ), f"Expected environmentd image {expected}, but found {image}" retry(check, 240) @@ -1070,11 +1084,11 @@ def check_pods() -> None: class AuthenticatorKind(Modification): @classmethod def values(cls, version: MzVersion) -> list[Any]: - # Test None, Password (v0.147.7+), and Sasl (v0.147.16+) + # Test None, Password (v0.147.7+), and Sasl result = ["None"] if version >= MzVersion.parse_mz("v0.147.7"): result.append("Password") - if version >= MzVersion.parse_mz("v0.147.16"): + if version >= MzVersion.parse_mz("v26.0.0"): result.append("Sasl") return result @@ -1100,13 +1114,13 @@ def validate(self, mods: dict[type[Modification], Any]) -> None: if self.value == "Password" and version <= MzVersion.parse_mz("v0.147.6"): return - if self.value == "Sasl" and version < MzVersion.parse_mz("v0.147.16"): + if self.value == "Sasl" and version < MzVersion.parse_mz("v26.0.0"): return port = ( 6875 if (version >= MzVersion.parse_mz("v0.147.0") and self.value == "Password") - or (version >= MzVersion.parse_mz("v0.147.16") and self.value == "Sasl") + or (version >= MzVersion.parse_mz("v26.0.0") and self.value == "Sasl") else 6877 ) for i in range(120): @@ -1252,10 +1266,23 @@ def workflow_defaults(c: Composition, parser: WorkflowArgumentParser) -> None: ) args = parser.parse_args() - current_version = get_tag(args.tag) + c.up(Service("mz-debug", idle=True)) + c.invoke("cp", "mz-debug:/usr/local/bin/mz-debug", ".") + + current_version = get_version(args.tag) # Following https://materialize.com/docs/installation/install-on-local-kind/ - for version in reversed(get_self_managed_versions() + [get_version(args.tag)]): + # orchestratord test can't run against future versions, so ignore those + versions = reversed( + [ + version + for version in get_self_managed_versions() + if version < current_version + ] + + [current_version] + ) + for version in versions: + print(f"--- Running with defaults against {version}") dir = "my-local-mz" if os.path.exists(dir): shutil.rmtree(dir) @@ -1392,9 +1419,6 @@ def workflow_defaults(c: Composition, parser: WorkflowArgumentParser) -> None: materialize_setup = list(yaml.load_all(f, Loader=yaml.Loader)) assert len(materialize_setup) == 3 - print(version) - print(current_version) - print(version == current_version) if version == current_version: materialize_setup[2]["spec"][ "environmentdImageRef" @@ -1493,6 +1517,7 @@ def workflow_defaults(c: Composition, parser: WorkflowArgumentParser) -> None: ] ) raise ValueError("Never completed") + run_mz_debug() def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: @@ -1534,7 +1559,8 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: "0.29.0" ), f"kind >= v0.29.0 required, while you are on {kind_version}" - c.up(Service("testdrive", idle=True)) + c.up(Service("testdrive", idle=True), Service("mz-debug", idle=True)) + c.invoke("cp", "mz-debug:/usr/local/bin/mz-debug", ".") cluster = "kind" clusters = spawn.capture(["kind", "get", "clusters"]).strip().split("\n") @@ -1658,9 +1684,6 @@ def get_mods() -> Iterator[list[Modification]]: mods.append(EnvironmentdImageRef(str(args.tag))) run_scenario([mods], definition) elif action == Action.Upgrade: - assert not ui.env_is_truthy( - "MZ_GHCR", MZ_GHCR_DEFAULT - ), "Manually set MZ_GHCR=0 as an environment variable for upgrade testing" assert args.runtime end_time = ( datetime.datetime.now() + datetime.timedelta(seconds=args.runtime) @@ -1683,9 +1706,6 @@ def get_mods() -> Iterator[list[Modification]]: ] run_scenario(scenario, definition) elif action == Action.UpgradeChain: - assert not ui.env_is_truthy( - "MZ_GHCR", MZ_GHCR_DEFAULT - ), "Manually set MZ_GHCR=0 as an environment variable for upgrade testing" assert args.runtime end_time = ( datetime.datetime.now() + datetime.timedelta(seconds=args.runtime) @@ -1722,6 +1742,13 @@ def get_mods() -> Iterator[list[Modification]]: def setup(cluster: str): spawn.runv(["kind", "delete", "cluster", "--name", cluster]) + with ( + open(MZ_ROOT / "test" / "orchestratord" / "cluster.yaml.tmpl") as in_file, + open(MZ_ROOT / "test" / "orchestratord" / "cluster.yaml", "w") as out_file, + ): + text = in_file.read() + out_file.write(text.replace("$HOME", os.environ["HOME"])) + spawn.runv( [ "kind", @@ -1806,18 +1833,14 @@ def run_scenario( mod.modify(definition) if mod.value in mod.failed_reconciliation_values(): expect_fail = True - if not initialize: - definition["materialize"]["spec"][ - "rolloutStrategy" - ] = "ImmediatelyPromoteCausingDowntime" - definition["materialize"]["spec"]["requestRollout"] = str(uuid.uuid4()) - run(definition, expect_fail) if initialize: init(definition) run(definition, expect_fail) initialize = False # only initialize once else: - upgrade(definition, expect_fail) + upgrade_operator_helm_chart(definition, expect_fail) + definition["materialize"]["spec"]["requestRollout"] = str(uuid.uuid4()) + run(definition, expect_fail) mod_dict = {mod.__class__: mod.value for mod in mods} for subclass in all_subclasses(Modification): if subclass not in mod_dict: @@ -1831,6 +1854,9 @@ def run_scenario( f"Reproduce with bin/mzcompose --find orchestratord run default --recreate-cluster --scenario='{scenario_json}'" ) raise + finally: + if not expect_fail: + run_mz_debug() def init(definition: dict[str, Any]) -> None: @@ -1866,7 +1892,7 @@ def init(definition: dict[str, Any]) -> None: stderr=subprocess.DEVNULL, ) - for i in range(120): + for i in range(240): try: spawn.capture( [ @@ -1890,7 +1916,7 @@ def init(definition: dict[str, Any]) -> None: raise ValueError("Never completed") -def upgrade(definition: dict[str, Any], expect_fail: bool) -> None: +def upgrade_operator_helm_chart(definition: dict[str, Any], expect_fail: bool) -> None: spawn.runv( [ "helm", @@ -1907,7 +1933,6 @@ def upgrade(definition: dict[str, Any], expect_fail: bool) -> None: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - post_run_check(definition, expect_fail) def run(definition: dict[str, Any], expect_fail: bool) -> None: @@ -1930,23 +1955,54 @@ def run(definition: dict[str, Any], expect_fail: bool) -> None: def post_run_check(definition: dict[str, Any], expect_fail: bool) -> None: - for i in range(60): + for i in range(900): + time.sleep(1) try: - spawn.capture( - [ - "kubectl", - "get", - "materializes", - "-n", - "materialize-environment", - ], - stderr=subprocess.DEVNULL, + data = json.loads( + spawn.capture( + [ + "kubectl", + "get", + "materializes", + "-n", + "materialize-environment", + "-o", + "json", + ], + stderr=subprocess.DEVNULL, + ) ) - break + status = data["items"][0].get("status") + if not status: + continue + if expect_fail: + break + if ( + not status["conditions"] + or status["conditions"][0]["type"] != "UpToDate" + ): + continue + if status["conditions"][0]["status"] != "True": + continue + if ( + status["lastCompletedRolloutRequest"] + == data["items"][0]["spec"]["requestRollout"] + ): + break except subprocess.CalledProcessError: pass - time.sleep(1) else: + spawn.runv( + [ + "kubectl", + "get", + "materializes", + "-n", + "materialize-environment", + "-o", + "yaml", + ], + ) raise ValueError("Never completed") for i in range(480): @@ -2005,5 +2061,3 @@ def post_run_check(definition: dict[str, Any], expect_fail: bool) -> None: ] ) raise ValueError("Never completed") - # Wait a bit for the status to stabilize - time.sleep(60)