diff --git a/pyproject.toml b/pyproject.toml index 92d364e..03b896a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ ignore-words-list = [ "priv", "sproc", "snowpark", + "pathspec", ] skip = [ "./build/", diff --git a/setup.py b/setup.py index d99870c..62882e1 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "snowflake-connector-python>=3.7.0", "snowflake-snowpark-python>=1.14.0", "jinja2", + "pathspec", ], extras_require={ "dev": [ diff --git a/tests/integration/data_provider/test_list_resource.py b/tests/integration/data_provider/test_list_resource.py index 7320be4..5070e90 100644 --- a/tests/integration/data_provider/test_list_resource.py +++ b/tests/integration/data_provider/test_list_resource.py @@ -66,14 +66,16 @@ def test_list_resource(cursor, list_resources_database, resource, marked_for_cle f"Skipping {resource.__class__.__name__}, not supported by account edition {session_ctx['account_edition']}" ) + if not hasattr(data_provider, f"list_{pluralize(resource_label_for_type(resource.resource_type))}"): + pytest.skip(f"{resource.resource_type} is not supported") + + if resource.__class__ == res.ScannerPackage: + pytest.skip("Flaky test, skipping for now") if isinstance(resource.scope, DatabaseScope): list_resources_database.add(resource) elif isinstance(resource.scope, SchemaScope): list_resources_database.public_schema.add(resource) - if not hasattr(data_provider, f"list_{pluralize(resource_label_for_type(resource.resource_type))}"): - pytest.skip(f"{resource.resource_type} is not supported") - try: create(cursor, resource) marked_for_cleanup.append(resource) diff --git a/tests/integration/test_lifecycle.py b/tests/integration/test_lifecycle.py index dfe624e..03aca13 100644 --- a/tests/integration/test_lifecycle.py +++ b/tests/integration/test_lifecycle.py @@ -37,6 +37,7 @@ def test_create_drop_from_json(resource, cursor, suffix): res.Grant, res.RoleGrant, res.FutureGrant, + res.ScannerPackage, ): pytest.skip("Skipping") diff --git a/titan/blueprint_config.py b/titan/blueprint_config.py index 0fb93af..ef851e9 100644 --- a/titan/blueprint_config.py +++ b/titan/blueprint_config.py @@ -96,7 +96,9 @@ def __post_init__(self): if self.scope == BlueprintScope.DATABASE and self.schema is not None: raise ValueError("Cannot specify a schema when using DATABASE scope") elif self.scope == BlueprintScope.ACCOUNT and (self.database is not None or self.schema is not None): - raise ValueError("Cannot specify a database or schema when using ACCOUNT scope") + raise ValueError( + f"Cannot specify a database or schema when using ACCOUNT scope (database={repr(self.database)}, schema={repr(self.schema)})" + ) def set_vars_defaults(vars_spec: list[dict], vars: dict) -> dict: diff --git a/titan/cli.py b/titan/cli.py index 300b56e..4a7ecc9 100644 --- a/titan/cli.py +++ b/titan/cli.py @@ -5,12 +5,31 @@ import yaml from titan.blueprint import dump_plan -from titan.enums import RunMode +from titan.enums import RunMode, BlueprintScope +from titan.gitops import ( + collect_configs_from_path, + collect_vars_from_environment, + merge_configs, + merge_vars, + parse_resources, +) from titan.operations.blueprint import blueprint_apply, blueprint_apply_plan, blueprint_plan -from titan.operations.export import export_resources from titan.operations.connector import connect, get_env_vars +from titan.operations.export import export_resources + + +class RunModeParamType(click.ParamType): + name = "run_mode" + + def convert(self, value, param, ctx): + return RunMode(value) -from .identifiers import resource_type_for_label + +class ScopeParamType(click.ParamType): + name = "scope" + + def convert(self, value, param, ctx): + return BlueprintScope(value) class JsonParamType(click.ParamType): @@ -30,18 +49,6 @@ def convert(self, value, param, ctx): return parse_resources(value) -def parse_resources(resource_labels_str): - if resource_labels_str is None: - return None - return [resource_type_for_label(resource_label) for resource_label in resource_labels_str.split(",")] - - -def load_config(config_file): - with open(config_file, "r") as f: - config = yaml.safe_load(f) - return config - - def load_plan(plan_file): with open(plan_file, "r") as f: plan = json.load(f) @@ -54,39 +61,95 @@ def titan_cli(): pass +# Shared click options + + +def config_path_option(): + return click.option( + "--config", + "config_path", + type=str, + help="Path to configuration YAML file or directory", + metavar="", + ) + + +def vars_option(): + return click.option( + "--vars", + type=JsonParamType(), + help="Dynamic values, specified as a JSON dictionary", + metavar="", + ) + + +def allowlist_option(): + return click.option( + "--allowlist", + type=CommaSeparatedListParamType(), + help="List of resources types allowed in the plan. If not specified, all resources are allowed.", + metavar="", + ) + + +def run_mode_option(): + return click.option( + "--mode", + "run_mode", + type=RunModeParamType(), + metavar="", + show_default=True, + help="Run mode", + ) + + +def scope_option(): + return click.option( + "--scope", + type=ScopeParamType(), + help="Limit the scope of resources to a specific database or schema", + metavar="", + ) + + +def database_option(): + return click.option( + "--database", + type=str, + help="Database to limit the scope to", + metavar="", + ) + + +def schema_option(): + return click.option( + "--schema", + type=str, + help="Schema to limit the scope to", + metavar="", + ) + + @titan_cli.command("plan", no_args_is_help=True) -@click.option("--config", "config_file", type=str, help="Path to configuration YAML file", metavar="") +@config_path_option() @click.option("--json", "json_output", is_flag=True, help="Output plan in machine-readable JSON format") @click.option("--out", "output_file", type=str, help="Write plan to a file", metavar="") -@click.option("--vars", type=JsonParamType(), help="Vars to pass to the blueprint") -@click.option( - "--allowlist", - type=CommaSeparatedListParamType(), - help="List of resources types allowed in the plan. If not specified, all resources are allowed.", - metavar="", -) -@click.option( - "--mode", - "run_mode", - type=click.Choice(["CREATE-OR-UPDATE", "SYNC"]), - metavar="", - show_default=True, - help="Run mode", -) -@click.option( - "--scope", - type=click.Choice(["ACCOUNT", "DATABASE", "SCHEMA"]), - help="Limit the scope of resources to a specific database or schema", - metavar="", -) -@click.option("--database", type=str, help="Database to limit the scope to", metavar="") -@click.option("--schema", type=str, help="Schema to limit the scope to", metavar="") -def plan(config_file, json_output, output_file, vars: dict, allowlist, run_mode, scope, database, schema): +@vars_option() +@allowlist_option() +@run_mode_option() +@scope_option() +@database_option() +@schema_option() +def plan(config_path, json_output, output_file, vars: dict, allowlist, run_mode, scope, database, schema): """Generate an execution plan based on your configuration""" - yaml_config = load_config(config_file) - if yaml_config is None: - raise click.UsageError(f"Config file {config_file} is empty") + if not config_path: + raise click.UsageError("--config is required") + + yaml_config: dict[str, Any] = {} + configs = collect_configs_from_path(config_path) + for config in configs: + yaml_config = merge_configs(yaml_config, config[1]) cli_config: dict[str, Any] = {} if vars: @@ -102,6 +165,10 @@ def plan(config_file, json_output, output_file, vars: dict, allowlist, run_mode, if schema: cli_config["schema"] = schema + env_vars = collect_vars_from_environment() + if env_vars: + cli_config["vars"] = merge_vars(cli_config.get("vars", {}), env_vars) + plan_obj = blueprint_plan(yaml_config, cli_config) if output_file: with open(output_file, "w") as f: @@ -116,31 +183,24 @@ def plan(config_file, json_output, output_file, vars: dict, allowlist, run_mode, @titan_cli.command("apply", no_args_is_help=True) -@click.option("--config", "config_file", type=str, help="Path to configuration YAML file", metavar="") +@config_path_option() @click.option("--plan", "plan_file", type=str, help="Path to plan JSON file", metavar="") -@click.option("--vars", type=JsonParamType(), help="Vars to pass to the blueprint") -@click.option( - "--allowlist", - type=CommaSeparatedListParamType(), - help="List of resources types allowed in the plan. If not specified, all resources are allowed.", -) -@click.option( - "--mode", - "run_mode", - type=click.Choice(["CREATE-OR-UPDATE", "SYNC"]), - metavar="", - show_default=True, - help="Run mode", -) -@click.option("--dry-run", is_flag=True, help="Perform a dry run without applying changes") -def apply(config_file, plan_file, vars, allowlist, run_mode, dry_run): +@vars_option() +@allowlist_option() +@run_mode_option() +@scope_option() +@database_option() +@schema_option() +@click.option("--dry-run", is_flag=True, help="When dry run is true, Titan will not make any changes to Snowflake") +def apply(config_path, plan_file, vars, allowlist, run_mode, scope, database, schema, dry_run): """Apply an execution plan to a Snowflake account""" - if config_file and plan_file: + + if config_path and plan_file: raise click.UsageError("Cannot specify both --config and --plan.") - if not config_file and not plan_file: + if not config_path and not plan_file: raise click.UsageError("Either --config or --plan must be specified.") - cli_config = {} + cli_config: dict[str, Any] = {} if vars: cli_config["vars"] = vars if run_mode: @@ -149,11 +209,22 @@ def apply(config_file, plan_file, vars, allowlist, run_mode, dry_run): cli_config["dry_run"] = dry_run if allowlist: cli_config["allowlist"] = allowlist + if scope: + cli_config["scope"] = scope + if database: + cli_config["database"] = database + if schema: + cli_config["schema"] = schema + + env_vars = collect_vars_from_environment() + if env_vars: + cli_config["vars"] = merge_vars(cli_config.get("vars", {}), env_vars) - if config_file: - yaml_config = load_config(config_file) - if yaml_config is None: - raise click.UsageError(f"Config file {config_file} is empty") + if config_path: + yaml_config: dict[str, Any] = {} + configs = collect_configs_from_path(config_path) + for config in configs: + yaml_config = merge_configs(yaml_config, config[1]) blueprint_apply(yaml_config, cli_config) elif plan_file: plan_obj = load_plan(plan_file) @@ -206,7 +277,7 @@ def export(resources, export_all, exclude_resources, out, format): if resources and export_all: raise click.UsageError("You can't specify both --resource and --all options at the same time.") - resource_config = {} + resource_config: dict[str, Any] = {} if resources: resource_config = export_resources(include=resources) elif export_all: diff --git a/titan/gitops.py b/titan/gitops.py index 28ff2a4..7f4e9c5 100644 --- a/titan/gitops.py +++ b/titan/gitops.py @@ -1,11 +1,15 @@ +import os +import yaml import logging from typing import Any, Optional from inflection import pluralize +from pathspec import PathSpec +from pathspec.patterns.gitwildmatch import GitWildMatchPattern from .blueprint_config import BlueprintConfig, set_vars_defaults from .enums import BlueprintScope, ResourceType, RunMode -from .identifiers import resource_label_for_type +from .identifiers import resource_label_for_type, resource_type_for_label from .resources import ( Database, Resource, @@ -180,12 +184,8 @@ def _resources_for_config(config: dict, vars: dict): def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, Any]] = None) -> BlueprintConfig: - - if cli_config is None: - cli_config = {} - yaml_config_ = yaml_config.copy() - cli_config_ = cli_config.copy() + cli_config_ = cli_config.copy() if cli_config else {} blueprint_args: dict[str, Any] = {} for key in ["allowlist", "dry_run", "name", "run_mode"]: @@ -199,7 +199,7 @@ def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, A run_mode = yaml_config_.pop("run_mode", None) or cli_config_.pop("run_mode", None) scope = yaml_config_.pop("scope", None) or cli_config_.pop("scope", None) schema = yaml_config_.pop("schema", None) or cli_config_.pop("schema", None) - vars = cli_config_.pop("vars", {}) + input_vars = cli_config_.pop("vars", {}) or {} vars_spec = yaml_config_.pop("vars", []) if allowlist: @@ -223,16 +223,15 @@ def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, A if schema: blueprint_args["schema"] = schema - if vars: - blueprint_args["vars"] = vars + blueprint_args["vars"] = input_vars if vars_spec: if not isinstance(vars_spec, list): raise ValueError("vars config entry must be a list of dicts") blueprint_args["vars_spec"] = vars_spec + blueprint_args["vars"] = set_vars_defaults(vars_spec, blueprint_args["vars"]) - vars = set_vars_defaults(vars_spec, vars) - resources = _resources_for_config(yaml_config_, vars) + resources = _resources_for_config(yaml_config_, blueprint_args["vars"]) if len(resources) == 0: raise ValueError("No resources found in config") @@ -243,3 +242,89 @@ def collect_blueprint_config(yaml_config: dict, cli_config: Optional[dict[str, A raise ValueError(f"Unknown keys in config: {yaml_config_.keys()}") return BlueprintConfig(**blueprint_args) + + +def crawl(path: str): + # Load .titanignore patterns if the file exists + gitignore_path = os.path.join(path, ".titanignore") + if os.path.exists(gitignore_path): + with open(gitignore_path) as f: + spec = PathSpec.from_lines(GitWildMatchPattern, f.readlines()) + else: + spec = PathSpec([]) + + if os.path.isfile(path): + yield path + return + + for root, _, files in os.walk(path): + for file in files: + if file.endswith(".yaml") or file.endswith(".yml"): + full_path = os.path.join(root, file) + # Get path relative to the base path for titanignore matching + rel_path = os.path.relpath(full_path, path) + if not spec.match_file(rel_path): + yield full_path + + +def read_config(config_path) -> dict: + with open(config_path, "r") as f: + try: + config = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ValueError(f"Error parsing YAML file: {config_path}") from e + return config + + +def merge_configs(config1: dict, config2: dict) -> dict: + merged = config1.copy() + for key, value in config2.items(): + if key in merged: + if isinstance(merged[key], list): + merged[key] = merged[key] + value + elif merged[key] is None: + merged[key] = value + else: + raise ValueError(f"Found a conflict for key `{key}` with {value} and {merged[key]}") + else: + merged[key] = value + return merged + + +def collect_configs_from_path(path: str) -> list[tuple[str, dict]]: + configs = [] + + if not os.path.exists(path): + raise ValueError(f"Invalid path: `{path}`. Must be a file or directory.") + + for file in crawl(path): + config = read_config(file) + configs.append((file, config)) + + if len(configs) == 0: + raise ValueError(f"No valid YAML files were read from the given path: {path}") + + return configs + + +def parse_resources(resource_labels_str: Optional[str]) -> Optional[list[ResourceType]]: + if resource_labels_str is None or resource_labels_str == "all": + return None + return [resource_type_for_label(resource_label) for resource_label in resource_labels_str.split(",")] + + +def collect_vars_from_environment() -> dict: + vars = {} + for key, value in os.environ.items(): + if key.startswith("TITAN_VAR_"): + vars[key[10:].lower()] = value + return vars + + +def merge_vars(vars: dict, other_vars: dict) -> dict: + for key in other_vars.keys(): + if key in vars: + raise ValueError(f"Conflicting var found: '{key}'") + merged = vars.copy() + merged.update(other_vars) + return merged