From a0e9753ba05ab306e97a5708175eac2979f38822 Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:53:12 -0700 Subject: [PATCH 01/10] prep for updated github action --- pyproject.toml | 1 + setup.py | 1 + titan/cli.py | 19 +++++------ titan/gitops.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 92d364ee..03b896ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ ignore-words-list = [ "priv", "sproc", "snowpark", + "pathspec", ] skip = [ "./build/", diff --git a/setup.py b/setup.py index d99870c7..62882e1e 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "snowflake-connector-python>=3.7.0", "snowflake-snowpark-python>=1.14.0", "jinja2", + "pathspec", ], extras_require={ "dev": [ diff --git a/titan/cli.py b/titan/cli.py index 300b56e1..c77b1ea6 100644 --- a/titan/cli.py +++ b/titan/cli.py @@ -6,11 +6,10 @@ from titan.blueprint import dump_plan from titan.enums import RunMode +from titan.gitops import collect_vars_from_environment, merge_vars, parse_resources from titan.operations.blueprint import blueprint_apply, blueprint_apply_plan, blueprint_plan -from titan.operations.export import export_resources from titan.operations.connector import connect, get_env_vars - -from .identifiers import resource_type_for_label +from titan.operations.export import export_resources class JsonParamType(click.ParamType): @@ -30,12 +29,6 @@ def convert(self, value, param, ctx): return parse_resources(value) -def parse_resources(resource_labels_str): - if resource_labels_str is None: - return None - return [resource_type_for_label(resource_label) for resource_label in resource_labels_str.split(",")] - - def load_config(config_file): with open(config_file, "r") as f: config = yaml.safe_load(f) @@ -102,6 +95,10 @@ def plan(config_file, json_output, output_file, vars: dict, allowlist, run_mode, if schema: cli_config["schema"] = schema + env_vars = collect_vars_from_environment() + if env_vars: + cli_config["vars"] = merge_vars(cli_config.get("vars", {}), env_vars) + plan_obj = blueprint_plan(yaml_config, cli_config) if output_file: with open(output_file, "w") as f: @@ -150,6 +147,10 @@ def apply(config_file, plan_file, vars, allowlist, run_mode, dry_run): if allowlist: cli_config["allowlist"] = allowlist + env_vars = collect_vars_from_environment() + if env_vars: + cli_config["vars"] = merge_vars(cli_config.get("vars", {}), env_vars) + if config_file: yaml_config = load_config(config_file) if yaml_config is None: diff --git a/titan/gitops.py b/titan/gitops.py index 28ff2a47..4911d0c1 100644 --- a/titan/gitops.py +++ b/titan/gitops.py @@ -1,11 +1,15 @@ +import os +import yaml import logging from typing import Any, Optional from inflection import pluralize +from pathspec import PathSpec +from pathspec.patterns.gitwildmatch import GitWildMatchPattern from .blueprint_config import BlueprintConfig, set_vars_defaults from .enums import BlueprintScope, ResourceType, RunMode -from .identifiers import resource_label_for_type +from .identifiers import resource_label_for_type, resource_type_for_label from .resources import ( Database, Resource, @@ -243,3 +247,83 @@ def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, A raise ValueError(f"Unknown keys in config: {yaml_config_.keys()}") return BlueprintConfig(**blueprint_args) + + +def crawl(path: str): + # Load .titanignore patterns if the file exists + gitignore_path = os.path.join(path, ".titanignore") + if os.path.exists(gitignore_path): + with open(gitignore_path) as f: + spec = PathSpec.from_lines(GitWildMatchPattern, f.readlines()) + else: + spec = PathSpec([]) + + for root, _, files in os.walk(path): + for file in files: + if file.endswith(".yaml") or file.endswith(".yml"): + full_path = os.path.join(root, file) + # Get path relative to the base path for titanignore matching + rel_path = os.path.relpath(full_path, path) + if not spec.match_file(rel_path): + yield full_path + + +def read_config(file) -> dict: + config_path = os.path.join(os.path.dirname(__file__), file) + with open(config_path, "r") as f: + config = yaml.safe_load(f) + return config + + +def merge_configs(config1: dict, config2: dict) -> dict: + merged = config1.copy() + for key, value in config2.items(): + if key in merged: + if isinstance(merged[key], list): + merged[key] = merged[key] + value + elif merged[key] is None: + merged[key] = value + else: + raise ValueError(f"Found a conflict for key {key} with {value} and {merged[key]}") + else: + merged[key] = value + return merged + + +def collect_configs_from_path(path: str) -> list[tuple[str, dict]]: + configs = [] + + if not os.path.exists(path): + raise ValueError(f"Invalid path: {path}. Must be a file or directory.") + + for file in crawl(path): + config = read_config(file) + configs.append((file, config)) + + if len(configs) == 0: + raise ValueError(f"No valid YAML files were read from the given path: {path}") + + return configs + + +def parse_resources(resource_labels_str): + if resource_labels_str is None or resource_labels_str == "all": + return None + return [resource_type_for_label(resource_label) for resource_label in resource_labels_str.split(",")] + + +def collect_vars_from_environment() -> dict: + vars = {} + for key, value in os.environ.items(): + if key.startswith("TITAN_VAR_"): + vars[key[10:].lower()] = value + return vars + + +def merge_vars(vars: dict, other_vars: dict) -> dict: + for key in other_vars.keys(): + if key in vars: + raise ValueError(f"Conflicting var found: '{key}'") + merged = vars.copy() + merged.update(other_vars) + return merged From 6feac72beaacc00e684aba02e5e8e73131ffd3ca Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:38:29 -0700 Subject: [PATCH 02/10] share filename when yaml parsing fails --- titan/gitops.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/titan/gitops.py b/titan/gitops.py index 4911d0c1..5ca42cd4 100644 --- a/titan/gitops.py +++ b/titan/gitops.py @@ -271,7 +271,10 @@ def crawl(path: str): def read_config(file) -> dict: config_path = os.path.join(os.path.dirname(__file__), file) with open(config_path, "r") as f: - config = yaml.safe_load(f) + try: + config = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ValueError(f"Error parsing YAML file: {file}") from e return config From 062e841e429f61b43c1ebbda9ad66d59d00c9668 Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:54:22 -0700 Subject: [PATCH 03/10] vars input as None bug --- titan/gitops.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/titan/gitops.py b/titan/gitops.py index 5ca42cd4..427c59d8 100644 --- a/titan/gitops.py +++ b/titan/gitops.py @@ -184,12 +184,8 @@ def _resources_for_config(config: dict, vars: dict): def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, Any]] = None) -> BlueprintConfig: - - if cli_config is None: - cli_config = {} - yaml_config_ = yaml_config.copy() - cli_config_ = cli_config.copy() + cli_config_ = cli_config.copy() if cli_config else {} blueprint_args: dict[str, Any] = {} for key in ["allowlist", "dry_run", "name", "run_mode"]: @@ -203,7 +199,7 @@ def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, A run_mode = yaml_config_.pop("run_mode", None) or cli_config_.pop("run_mode", None) scope = yaml_config_.pop("scope", None) or cli_config_.pop("scope", None) schema = yaml_config_.pop("schema", None) or cli_config_.pop("schema", None) - vars = cli_config_.pop("vars", {}) + input_vars = cli_config_.pop("vars", {}) or {} vars_spec = yaml_config_.pop("vars", []) if allowlist: @@ -227,16 +223,15 @@ def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, A if schema: blueprint_args["schema"] = schema - if vars: - blueprint_args["vars"] = vars + blueprint_args["vars"] = input_vars if vars_spec: if not isinstance(vars_spec, list): raise ValueError("vars config entry must be a list of dicts") blueprint_args["vars_spec"] = vars_spec + blueprint_args["vars"] = set_vars_defaults(vars_spec, blueprint_args["vars"]) - vars = set_vars_defaults(vars_spec, vars) - resources = _resources_for_config(yaml_config_, vars) + resources = _resources_for_config(yaml_config_, blueprint_args["vars"]) if len(resources) == 0: raise ValueError("No resources found in config") From e02b8f8a4bd1861824f136e0483b39f2b12d4194 Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:04:28 -0700 Subject: [PATCH 04/10] better error --- titan/blueprint_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/titan/blueprint_config.py b/titan/blueprint_config.py index 0fb93af0..ef851e94 100644 --- a/titan/blueprint_config.py +++ b/titan/blueprint_config.py @@ -96,7 +96,9 @@ def __post_init__(self): if self.scope == BlueprintScope.DATABASE and self.schema is not None: raise ValueError("Cannot specify a schema when using DATABASE scope") elif self.scope == BlueprintScope.ACCOUNT and (self.database is not None or self.schema is not None): - raise ValueError("Cannot specify a database or schema when using ACCOUNT scope") + raise ValueError( + f"Cannot specify a database or schema when using ACCOUNT scope (database={repr(self.database)}, schema={repr(self.schema)})" + ) def set_vars_defaults(vars_spec: list[dict], vars: dict) -> dict: From 3703264d35037e55149b31be359206ec69ce988e Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:12:36 -0700 Subject: [PATCH 05/10] types --- titan/gitops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titan/gitops.py b/titan/gitops.py index 427c59d8..6e930922 100644 --- a/titan/gitops.py +++ b/titan/gitops.py @@ -304,7 +304,7 @@ def collect_configs_from_path(path: str) -> list[tuple[str, dict]]: return configs -def parse_resources(resource_labels_str): +def parse_resources(resource_labels_str: Optional[str]) -> Optional[list[ResourceType]]: if resource_labels_str is None or resource_labels_str == "all": return None return [resource_type_for_label(resource_label) for resource_label in resource_labels_str.split(",")] From 3fd8bd74462df97e9976030ed6d8191586c11782 Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:32:23 -0700 Subject: [PATCH 06/10] allow path in cli --- titan/cli.py | 50 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/titan/cli.py b/titan/cli.py index c77b1ea6..946b19d9 100644 --- a/titan/cli.py +++ b/titan/cli.py @@ -6,7 +6,13 @@ from titan.blueprint import dump_plan from titan.enums import RunMode -from titan.gitops import collect_vars_from_environment, merge_vars, parse_resources +from titan.gitops import ( + collect_configs_from_path, + collect_vars_from_environment, + merge_configs, + merge_vars, + parse_resources, +) from titan.operations.blueprint import blueprint_apply, blueprint_apply_plan, blueprint_plan from titan.operations.connector import connect, get_env_vars from titan.operations.export import export_resources @@ -29,12 +35,6 @@ def convert(self, value, param, ctx): return parse_resources(value) -def load_config(config_file): - with open(config_file, "r") as f: - config = yaml.safe_load(f) - return config - - def load_plan(plan_file): with open(plan_file, "r") as f: plan = json.load(f) @@ -48,7 +48,9 @@ def titan_cli(): @titan_cli.command("plan", no_args_is_help=True) -@click.option("--config", "config_file", type=str, help="Path to configuration YAML file", metavar="") +@click.option( + "--config", "config_path", type=str, help="Path to configuration YAML file or directory", metavar="" +) @click.option("--json", "json_output", is_flag=True, help="Output plan in machine-readable JSON format") @click.option("--out", "output_file", type=str, help="Write plan to a file", metavar="") @click.option("--vars", type=JsonParamType(), help="Vars to pass to the blueprint") @@ -74,12 +76,16 @@ def titan_cli(): ) @click.option("--database", type=str, help="Database to limit the scope to", metavar="") @click.option("--schema", type=str, help="Schema to limit the scope to", metavar="") -def plan(config_file, json_output, output_file, vars: dict, allowlist, run_mode, scope, database, schema): +def plan(config_path, json_output, output_file, vars: dict, allowlist, run_mode, scope, database, schema): """Generate an execution plan based on your configuration""" - yaml_config = load_config(config_file) - if yaml_config is None: - raise click.UsageError(f"Config file {config_file} is empty") + if not config_path: + raise click.UsageError("--config is required") + + yaml_config = {} + configs = collect_configs_from_path(config_path) + for config in configs: + yaml_config = merge_configs(yaml_config, config[1]) cli_config: dict[str, Any] = {} if vars: @@ -113,7 +119,9 @@ def plan(config_file, json_output, output_file, vars: dict, allowlist, run_mode, @titan_cli.command("apply", no_args_is_help=True) -@click.option("--config", "config_file", type=str, help="Path to configuration YAML file", metavar="") +@click.option( + "--config", "config_path", type=str, help="Path to configuration YAML file or directory", metavar="" +) @click.option("--plan", "plan_file", type=str, help="Path to plan JSON file", metavar="") @click.option("--vars", type=JsonParamType(), help="Vars to pass to the blueprint") @click.option( @@ -130,11 +138,12 @@ def plan(config_file, json_output, output_file, vars: dict, allowlist, run_mode, help="Run mode", ) @click.option("--dry-run", is_flag=True, help="Perform a dry run without applying changes") -def apply(config_file, plan_file, vars, allowlist, run_mode, dry_run): +def apply(config_path, plan_file, vars, allowlist, run_mode, dry_run): """Apply an execution plan to a Snowflake account""" - if config_file and plan_file: + + if config_path and plan_file: raise click.UsageError("Cannot specify both --config and --plan.") - if not config_file and not plan_file: + if not config_path and not plan_file: raise click.UsageError("Either --config or --plan must be specified.") cli_config = {} @@ -151,10 +160,11 @@ def apply(config_file, plan_file, vars, allowlist, run_mode, dry_run): if env_vars: cli_config["vars"] = merge_vars(cli_config.get("vars", {}), env_vars) - if config_file: - yaml_config = load_config(config_file) - if yaml_config is None: - raise click.UsageError(f"Config file {config_file} is empty") + if config_path: + yaml_config = {} + configs = collect_configs_from_path(config_path) + for config in configs: + yaml_config = merge_configs(yaml_config, config[1]) blueprint_apply(yaml_config, cli_config) elif plan_file: plan_obj = load_plan(plan_file) From b8a234056eb9e545bbd404bfb4027895c909e3d2 Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:44:57 -0700 Subject: [PATCH 07/10] cli fix --- titan/gitops.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/titan/gitops.py b/titan/gitops.py index 6e930922..4abff7f0 100644 --- a/titan/gitops.py +++ b/titan/gitops.py @@ -253,6 +253,10 @@ def crawl(path: str): else: spec = PathSpec([]) + if os.path.isfile(path): + yield path + return + for root, _, files in os.walk(path): for file in files: if file.endswith(".yaml") or file.endswith(".yml"): @@ -263,13 +267,12 @@ def crawl(path: str): yield full_path -def read_config(file) -> dict: - config_path = os.path.join(os.path.dirname(__file__), file) +def read_config(config_path) -> dict: with open(config_path, "r") as f: try: config = yaml.safe_load(f) except yaml.YAMLError as e: - raise ValueError(f"Error parsing YAML file: {file}") from e + raise ValueError(f"Error parsing YAML file: {config_path}") from e return config @@ -282,7 +285,7 @@ def merge_configs(config1: dict, config2: dict) -> dict: elif merged[key] is None: merged[key] = value else: - raise ValueError(f"Found a conflict for key {key} with {value} and {merged[key]}") + raise ValueError(f"Found a conflict for key `{key}` with {value} and {merged[key]}") else: merged[key] = value return merged From 253bd75d5983b090429dd64897b089a4abbd8ae8 Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:03:41 -0700 Subject: [PATCH 08/10] update CLI --- titan/cli.py | 152 +++++++++++++++++++++++++++++++++--------------- titan/gitops.py | 2 +- 2 files changed, 107 insertions(+), 47 deletions(-) diff --git a/titan/cli.py b/titan/cli.py index 946b19d9..0b47a510 100644 --- a/titan/cli.py +++ b/titan/cli.py @@ -5,7 +5,7 @@ import yaml from titan.blueprint import dump_plan -from titan.enums import RunMode +from titan.enums import RunMode, BlueprintScope from titan.gitops import ( collect_configs_from_path, collect_vars_from_environment, @@ -18,6 +18,20 @@ from titan.operations.export import export_resources +class RunModeParamType(click.ParamType): + name = "run_mode" + + def convert(self, value, param, ctx): + return RunMode(value) + + +class ScopeParamType(click.ParamType): + name = "scope" + + def convert(self, value, param, ctx): + return BlueprintScope(value) + + class JsonParamType(click.ParamType): name = "json" @@ -47,35 +61,85 @@ def titan_cli(): pass +# Shared click options + + +def config_path_option(): + return click.option( + "--config", + "config_path", + type=str, + help="Path to configuration YAML file or directory", + metavar="", + ) + + +def vars_option(): + return click.option( + "--vars", + type=JsonParamType(), + help="Dynamic values, specified as a JSON dictionary", + metavar="", + ) + + +def allowlist_option(): + return click.option( + "--allowlist", + type=CommaSeparatedListParamType(), + help="List of resources types allowed in the plan. If not specified, all resources are allowed.", + metavar="", + ) + + +def run_mode_option(): + return click.option( + "--mode", + "run_mode", + type=RunModeParamType(), + metavar="", + show_default=True, + help="Run mode", + ) + + +def scope_option(): + return click.option( + "--scope", + type=ScopeParamType(), + help="Limit the scope of resources to a specific database or schema", + metavar="", + ) + + +def database_option(): + return click.option( + "--database", + type=str, + help="Database to limit the scope to", + metavar="", + ) + + +def schema_option(): + return click.option( + "--schema", + type=str, + help="Schema to limit the scope to", + metavar="", + ) + + @titan_cli.command("plan", no_args_is_help=True) -@click.option( - "--config", "config_path", type=str, help="Path to configuration YAML file or directory", metavar="" -) +@config_path_option() @click.option("--json", "json_output", is_flag=True, help="Output plan in machine-readable JSON format") @click.option("--out", "output_file", type=str, help="Write plan to a file", metavar="") -@click.option("--vars", type=JsonParamType(), help="Vars to pass to the blueprint") -@click.option( - "--allowlist", - type=CommaSeparatedListParamType(), - help="List of resources types allowed in the plan. If not specified, all resources are allowed.", - metavar="", -) -@click.option( - "--mode", - "run_mode", - type=click.Choice(["CREATE-OR-UPDATE", "SYNC"]), - metavar="", - show_default=True, - help="Run mode", -) -@click.option( - "--scope", - type=click.Choice(["ACCOUNT", "DATABASE", "SCHEMA"]), - help="Limit the scope of resources to a specific database or schema", - metavar="", -) -@click.option("--database", type=str, help="Database to limit the scope to", metavar="") -@click.option("--schema", type=str, help="Schema to limit the scope to", metavar="") +@vars_option() +@allowlist_option() +@run_mode_option() +@scope_option() +@database_option() +@schema_option() def plan(config_path, json_output, output_file, vars: dict, allowlist, run_mode, scope, database, schema): """Generate an execution plan based on your configuration""" @@ -119,26 +183,16 @@ def plan(config_path, json_output, output_file, vars: dict, allowlist, run_mode, @titan_cli.command("apply", no_args_is_help=True) -@click.option( - "--config", "config_path", type=str, help="Path to configuration YAML file or directory", metavar="" -) +@config_path_option() @click.option("--plan", "plan_file", type=str, help="Path to plan JSON file", metavar="") -@click.option("--vars", type=JsonParamType(), help="Vars to pass to the blueprint") -@click.option( - "--allowlist", - type=CommaSeparatedListParamType(), - help="List of resources types allowed in the plan. If not specified, all resources are allowed.", -) -@click.option( - "--mode", - "run_mode", - type=click.Choice(["CREATE-OR-UPDATE", "SYNC"]), - metavar="", - show_default=True, - help="Run mode", -) -@click.option("--dry-run", is_flag=True, help="Perform a dry run without applying changes") -def apply(config_path, plan_file, vars, allowlist, run_mode, dry_run): +@vars_option() +@allowlist_option() +@run_mode_option() +@scope_option() +@database_option() +@schema_option() +@click.option("--dry-run", is_flag=True, help="When dry run is true, Titan will not make any changes to Snowflake") +def apply(config_path, plan_file, vars, allowlist, run_mode, scope, database, schema, dry_run): """Apply an execution plan to a Snowflake account""" if config_path and plan_file: @@ -155,6 +209,12 @@ def apply(config_path, plan_file, vars, allowlist, run_mode, dry_run): cli_config["dry_run"] = dry_run if allowlist: cli_config["allowlist"] = allowlist + if scope: + cli_config["scope"] = scope + if database: + cli_config["database"] = database + if schema: + cli_config["schema"] = schema env_vars = collect_vars_from_environment() if env_vars: diff --git a/titan/gitops.py b/titan/gitops.py index 4abff7f0..7f4e9c52 100644 --- a/titan/gitops.py +++ b/titan/gitops.py @@ -295,7 +295,7 @@ def collect_configs_from_path(path: str) -> list[tuple[str, dict]]: configs = [] if not os.path.exists(path): - raise ValueError(f"Invalid path: {path}. Must be a file or directory.") + raise ValueError(f"Invalid path: `{path}`. Must be a file or directory.") for file in crawl(path): config = read_config(file) From 948490854a25fad6ef0eec7649befe6542e04a3d Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:10:03 -0700 Subject: [PATCH 09/10] typing --- titan/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/titan/cli.py b/titan/cli.py index 0b47a510..4a7ecc9a 100644 --- a/titan/cli.py +++ b/titan/cli.py @@ -146,7 +146,7 @@ def plan(config_path, json_output, output_file, vars: dict, allowlist, run_mode, if not config_path: raise click.UsageError("--config is required") - yaml_config = {} + yaml_config: dict[str, Any] = {} configs = collect_configs_from_path(config_path) for config in configs: yaml_config = merge_configs(yaml_config, config[1]) @@ -200,7 +200,7 @@ def apply(config_path, plan_file, vars, allowlist, run_mode, scope, database, sc if not config_path and not plan_file: raise click.UsageError("Either --config or --plan must be specified.") - cli_config = {} + cli_config: dict[str, Any] = {} if vars: cli_config["vars"] = vars if run_mode: @@ -221,7 +221,7 @@ def apply(config_path, plan_file, vars, allowlist, run_mode, scope, database, sc cli_config["vars"] = merge_vars(cli_config.get("vars", {}), env_vars) if config_path: - yaml_config = {} + yaml_config: dict[str, Any] = {} configs = collect_configs_from_path(config_path) for config in configs: yaml_config = merge_configs(yaml_config, config[1]) @@ -277,7 +277,7 @@ def export(resources, export_all, exclude_resources, out, format): if resources and export_all: raise click.UsageError("You can't specify both --resource and --all options at the same time.") - resource_config = {} + resource_config: dict[str, Any] = {} if resources: resource_config = export_resources(include=resources) elif export_all: From 8c044a098b380e065b943665072adf74b8c7baa6 Mon Sep 17 00:00:00 2001 From: TJ Murphy <1796+teej@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:25:37 -0700 Subject: [PATCH 10/10] flaky tests --- tests/integration/data_provider/test_list_resource.py | 8 +++++--- tests/integration/test_lifecycle.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/data_provider/test_list_resource.py b/tests/integration/data_provider/test_list_resource.py index 7320be45..5070e90e 100644 --- a/tests/integration/data_provider/test_list_resource.py +++ b/tests/integration/data_provider/test_list_resource.py @@ -66,14 +66,16 @@ def test_list_resource(cursor, list_resources_database, resource, marked_for_cle f"Skipping {resource.__class__.__name__}, not supported by account edition {session_ctx['account_edition']}" ) + if not hasattr(data_provider, f"list_{pluralize(resource_label_for_type(resource.resource_type))}"): + pytest.skip(f"{resource.resource_type} is not supported") + + if resource.__class__ == res.ScannerPackage: + pytest.skip("Flaky test, skipping for now") if isinstance(resource.scope, DatabaseScope): list_resources_database.add(resource) elif isinstance(resource.scope, SchemaScope): list_resources_database.public_schema.add(resource) - if not hasattr(data_provider, f"list_{pluralize(resource_label_for_type(resource.resource_type))}"): - pytest.skip(f"{resource.resource_type} is not supported") - try: create(cursor, resource) marked_for_cleanup.append(resource) diff --git a/tests/integration/test_lifecycle.py b/tests/integration/test_lifecycle.py index dfe624ed..03aca132 100644 --- a/tests/integration/test_lifecycle.py +++ b/tests/integration/test_lifecycle.py @@ -37,6 +37,7 @@ def test_create_drop_from_json(resource, cursor, suffix): res.Grant, res.RoleGrant, res.FutureGrant, + res.ScannerPackage, ): pytest.skip("Skipping")