From 6b7385c61fc183307560d6899cbdde3eed0c3c87 Mon Sep 17 00:00:00 2001 From: TJ Murphy Date: Fri, 8 Nov 2024 09:52:51 -0800 Subject: [PATCH] [CHORE] tests and docs (#150) * tests and docs and bugfixes --------- Co-authored-by: TJ Murphy <1796+teej@users.noreply.github.com> --- README.md | 140 +++++++----- docs/README.md | 11 +- docs/blueprint.md | 209 +++++++++++++++++- docs/getting-started.md | 98 ++++---- docs/titan-core-github-action.md | 175 +++++++++------ docs/vars.md | 33 --- ...orials-create-your-first-iceberg-table.yml | 2 +- tests/fixtures/json/network_policy.json | 6 +- .../data_provider/test_list_resource.py | 8 + titan/cli.py | 8 +- titan/data_provider.py | 62 +++--- tools/test_account_configs/base.yml | 12 + .../business_critical.yml | 6 + tools/test_account_configs/enterprise.yml | 6 + 14 files changed, 521 insertions(+), 255 deletions(-) delete mode 100644 docs/vars.md diff --git a/README.md b/README.md index 0126881b..e49b7bff 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,5 @@ # `titan core` - Snowflake infrastructure as code - - Titan Core helps you provision, deploy, and secure resources in Snowflake. It replaces tools like Terraform, Schemachange, or Permifrost. Deploy any Snowflake resource, including users, roles, schemas, databases, integrations, pipes, stages, functions, stored procedures, and more. Convert adhoc, bug-prone SQL management scripts into simple, repeatable configuration. @@ -12,7 +8,7 @@ Titan Core is for: * DevOps engineers looking to automate and manage Snowflake infrastructure. * Analytics engineers working with dbt who want to manage Snowflake resources without macros. -* Data engineers needing a reliable tool for deploying and managing Snowflake resources. +* Data platform teams who need to reliably manage Snowflake with CI/CD. * Organizations that prefer a git-based workflow for infrastructure management. * Teams seeking to replace Terraform for Snowflake-related tasks. @@ -20,19 +16,19 @@ Titan Core is for: ╔══════════╗ ╔═══════════╗ ║ CONFIG ║ ║ SNOWFLAKE ║ ╚══════════╝ ╚═══════════╝ - ┏━━━━━━━━━━━┓ ┏━━━━━━━━━━━┓ -┌┫ WAREHOUSE ┣──────┐ ┌┫ WAREHOUSE ┣───────────┐ -│┗━━━━━━━━━━━┛ │ ALTER │┗━━━━━━━━━━━┛ │ -│ name: ETL │─────┐ ┌─ WAREHOUSE ─▶│ name: ETL │ -│ auto_suspend: 60 │ │ │ │ auto_suspend: 300 -> 60│ -└───────────────────┘ ╔══▼═══════════╩═╗ └────────────────────────┘ + ┏━━━━━━━━━━━┓ ┏━━━━━━━━━━━┓ +┌─┫ WAREHOUSE ┣─────┐ ┌─┫ WAREHOUSE ┣───────────┐ +│ ┗━━━━━━━━━━━┛ │ ALTER │ ┗━━━━━━━━━━━┛ │ +│ name: ETL │─────┐ ┌─ WAREHOUSE ─▶│ name: ETL │ +│ auto_suspend: 60 │ │ │ │ auto_suspend: 300 -> 60 │ +└───────────────────┘ ╔══▼═══════════╩═╗ └─────────────────────────┘ ║ ║ ║ TITAN CORE ║ ┏━━━━━━┓ ║ ║ ┏━━━━━━┓ -┌─┫ ROLE ┣──────────┐ ╚══▲═══════════╦═╝ ┌─┫ ROLE ┣───────────────┐ -│ ┗━━━━━━┛ │ │ │ │ ┗━━━━━━┛ │ -│ name: TRANSFORMER │─────┘ └─ CREATE ────▶│ name: TRANSFORMER │ -└───────────────────┘ ROLE └────────────────────────┘ +┌─┫ ROLE ┣──────────┐ ╚══▲═══════════╦═╝ ┌─┫ ROLE ┣────────────────┐ +│ ┗━━━━━━┛ │ │ │ │ ┗━━━━━━┛ │ +│ name: TRANSFORMER │─────┘ └─ CREATE ────▶│ name: TRANSFORMER │ +└───────────────────┘ ROLE └─────────────────────────┘ ``` @@ -42,11 +38,11 @@ Titan Core is for: * **Comprehensive** » Nearly every Snowflake resource is supported - * **Pythonic** » Written in Python so you can use it with your existing Python workflow + * **Flexible** » Write resource configuration in YAML or Python - * **Fast** » Titan Core runs in seconds, even with complex environments + * **Fast** » Titan Core runs 50-90% faster than Terraform and Permifrost - * **SQL** » The only tool that allows you to write Python, YAML, or SQL + * **Migration-friendly** » Generate config automatically with the export CLI ## Open Source @@ -60,20 +56,23 @@ You can find comprehensive [Titan Core documentation on GitBook](https://titan-c If you're new, the best place to start is with the Python package. -### Python + CLI Installation - -### Install from PyPi +### Install from PyPi (MacOS, Linux) ```sh python -m venv .venv -# linux / mac: source .venv/bin/activate -# windows: +python -m pip install titan-core +``` + +### Install from PyPi (Windows) + +```bat +python -m venv .venv .\.venv\Scripts\activate python -m pip install titan-core ``` -### Using the Python package +### Python example ```Python import os @@ -168,7 +167,7 @@ The CLI allows you to `plan` and `apply` a Titan Core YAML config. You can speci In addition to `plan` and `apply`, the CLI also allows you to `export` resources. This makes it easy to generate a config for an existing Snowflake environment. -To connect with Snowflake, the CLI uses environment variables. These environment variables are supported: +To connect with Snowflake, the CLI uses environment variables. The following `are supported: * `SNOWFLAKE_ACCOUNT` * `SNOWFLAKE_USER` @@ -182,11 +181,12 @@ To connect with Snowflake, the CLI uses environment variables. These environment ### CLI Example +Show the help message + ```sh -# Show the help message -python -m titan --help +titan --help -# Usage: python -m titan [OPTIONS] COMMAND [ARGS]... +# Usage: titan [OPTIONS] COMMAND [ARGS]... # # titan core helps you manage your Snowflake environment. # @@ -194,12 +194,16 @@ python -m titan --help # --help Show this message and exit. # # Commands: -# apply Apply a plan to Titan resources -# export Export Titan resources -# plan Generate an execution plan based on your configuration +# apply Apply a resource config to a Snowflake account +# connect Test the connection to Snowflake +# export Generate a resource config for existing Snowflake resources +# plan Compare a resource config to the current state of Snowflake +``` -# The CLI uses YAML config. This command creates a sample config file. +Apply a resource config to Snowflake +```sh +# Create a resource config file cat < titan.yml roles: - name: transformer @@ -221,16 +225,24 @@ export SNOWFLAKE_USER="my-user" export SNOWFLAKE_PASSWORD="my-password" # Generate a plan -python -m titan plan --config titan.yml +titan plan --config titan.yml # Apply the config -python -m titan apply --config titan.yml +titan apply --config titan.yml ``` -The CLI can be used to export your current resource config to a file. +Export existing Snowflake resources to YAML. ```sh -python -m titan export --resource=warehouse,grant,role --out=titan.yml +titan export \ + --resource=warehouse,grant,role \ + --out=titan.yml +``` + +The Titan Core Python package installs the CLI script `titan`. You can alternatively use Python CLI module syntax if you need fine-grained control over the Python environment. + +```sh +python -m titan plan --config titan.yml ``` ### Using the GitHub Action @@ -238,35 +250,32 @@ The Titan Core GitHub Action allows you to automate the deployment of Snowflake ### GitHub Action Example -```yaml -# .github/workflows/titan.yml -name: Titan Snowflake +```YAML +-- .github/workflows/titan.yml +name: Deploy to Snowflake with Titan on: push: - branches: ["main"] - # The directory in your repo where titan configs live. + branches: [ main ] paths: - - 'envs/prod/**' - + - 'titan/**' jobs: deploy: runs-on: ubuntu-latest - name: Deploy to Snowflake with Titan - - # The Github environment to use - environment: prod steps: - - uses: actions/checkout@v4 - - name: Deploy with Titan - id: titan-core-action + - name: Checkout code + uses: actions/checkout@v4 + + - name: Deploy to Snowflake uses: Titan-Systems/titan-core-action@main with: - resource-path: envs/prod - valid-resource-types: database,user,warehouse,role + run-mode: 'create-or-update' + resource-path: './titan' + allowlist: 'warehouse,role,grant' + dry-run: 'false' env: SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} @@ -281,14 +290,15 @@ jobs: ## `titan core` vs other tools -| Feature | Titan Core | Terraform | Schemachange | Permifrost | -|-----------------------------------------|------------|-----------|--------------| ------------| -| Plan and Execute Changes | ✅ | ✅ | ❌ | ✅ | -| Declarative Config | ✅ | ✅ | ❌ | ✅ | -| Python-Based Definitions | ✅ | w/ CDKTF | ❌ | ❌ | -| SQL Support | ✅ | ❌ | ✅ | ❌ | -| Dynamic Role Switching | ✅ | ❌ | N/A | ❌ | -| No State File Dependency | ✅ | ❌ | ✅ | ✅ | +| Feature | Titan Core | Terraform | Schemachange | Permifrost | SnowDDL | +|-----------------------------------------|------------|-----------|--------------| -----------| -------- | +| Plan and Execute Changes | ✅ | ✅ | ❌ | ✅ | ✅ | +| Declarative Config | ✅ | ✅ | ❌ | ✅ | ✅ | +| No State File Dependency | ✅ | ❌ | ✅ | ✅ | ✅ | +| Python-Based Definitions | ✅ | w/ CDKTF | ❌ | ❌ | ✅ | +| SQL Support | ✅ | ❌ | ✅ | ❌ | ❌ | +| Dynamic Role Switching | ✅ | ❌ | N/A | ❌ | ❌ | +| Export Snowflake resources | ✅ | ❌ | ❌ | ❌ | ❌ | ### `titan core` vs Terraform @@ -315,6 +325,11 @@ Declarative config is less error-prone and more scalable, especially in dynamic Permifrost can be very slow. Running simple Permifrost configs can take minutes to run. Titan Core is designed to run in seconds, even with complex environments. +### `titan core` vs SnowDDL +[SnowDDL](https://github.com/littleK0i/SnowDDL) is a declarative object management tool for Snowflake, similar to Titan Core. It uses a streamlined [permissions model](https://docs.snowddl.com/guides/permission-model) that simplifies granting read and write access to databases and schemas. + +SnowDDL takes a strongly opinionated stance on roles in Snowflake. If you don't need a [3-tier role heirarchy](https://docs.snowddl.com/guides/role-hierarchy), SnowDDL may not be a good fit. + ## Resource support ### Legend @@ -438,6 +453,9 @@ Permifrost can be very slow. Running simple Permifrost configs can take minutes | ↳ SQL | ❌ | | View | ✅ | +### What if I need a type of resource isn't supported? + +Please [create a GitHub issue](https://github.com/Titan-Systems/titan/issues) if there's a resource you need that isn't currently supported. ## Contributing diff --git a/docs/README.md b/docs/README.md index acf98dde..69e36261 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,11 +4,11 @@ Titan Core helps you provision, deploy, and secure resources in Snowflake. It re Deploy any Snowflake resource, including users, roles, schemas, databases, integrations, pipes, stages, functions, stored procedures, and more. Convert adhoc, bug-prone SQL management scripts into simple, repeatable configuration. -Titan Core is for: +## Titan Core is for * DevOps engineers looking to automate and manage Snowflake infrastructure. * Analytics engineers working with dbt who want to manage Snowflake resources without macros. -* Data engineers needing a reliable tool for deploying and managing Snowflake resources. +* Data platform teams who need to reliably manage Snowflake with CI/CD. * Organizations that prefer a git-based workflow for infrastructure management. * Teams seeking to replace Terraform for Snowflake-related tasks. @@ -19,12 +19,11 @@ Titan Core is for: * **Comprehensive** » Nearly every Snowflake resource is supported - * **Pythonic** » Written in Python so you can use it with your existing Python workflow + * **Flexible** » Write resource configuration in YAML or Python - * **Fast** » Titan Core runs in seconds, even with complex environments + * **Fast** » Titan Core runs 50-90% faster than Terraform and Permifrost - * **SQL** » The only tool that allows you to write Python, YAML, or SQL - + * **Migration-friendly** » Generate config automatically with the export CLI ## Contents diff --git a/docs/blueprint.md b/docs/blueprint.md index 6c6a4dcd..0f4a49f0 100644 --- a/docs/blueprint.md +++ b/docs/blueprint.md @@ -17,19 +17,45 @@ bp = Blueprint( resources=[ Database('my_database'), Schema('my_schema', database='my_database'), - ] + ], + allowlist=["database", "schema"], + dry_run=False, ) plan = bp.plan(session) bp.apply(session, plan) ``` ## Blueprint parameters -- **run_mode** (`str`): Defines how the blueprint interacts with the Snowflake account +**run_mode** `str` +- Defines how the blueprint interacts with the Snowflake account - **create-or-update** (*default*): Resources are either created or updated, no resources are destroyed - - **sync**: Modifies your Snowflake account to match the blueprint exactly. When in use, `allowlist` must be specified. ⚠️`WARNING`⚠️: Resources not defined in the blueprint but present in your account will be dropped. -- **resources** (`list[Resource]`): List of resources initialized in the blueprint. -- **allowlist** (`list[str]`): Specifies the allowed resource types in the blueprint. -- **dry_run** (`bool`): `apply()` will return a list of SQL commands that would be executed without applying them. + - **sync**: + - `⚠️ WARNING` Sync mode will drop resources. + - Titan will update Snowflake to match the blueprint exactly. Must be used with `allowlist`. + +**resources** `list[Resource]` +- List of resources initialized in the blueprint. + +**allowlist** `list[str]` +- Specifies the allowed resource types in the blueprint. + +**dry_run** `bool` +- `apply()` will return a list of SQL commands that would be executed without applying them. + +**vars** `dict` +- A key-value dictionary that specifies the names and values of vars. + +**vars_spec** `list[dict]` +- A list of dictionaries defining the `name`, `type` and `default` (optional) of all expected vars. + +**scope** `str` +- Limit Titan's scope to a single database or schema. Must be one of "DATABASE" or "SCHEMA". If not specified, Titan will manage any resource. + +**database** `str` +- The name of a database to limit Titan's scope to. Must be used with `scope`. + +**schema** `str` +- The name of a schema to limit Titan's scope to. Must be used with `scope` and `database`. ## Methods @@ -65,3 +91,174 @@ Alternate uses: - `add([resource_1, resource_2, ...])` The add method allows you to add a resource to the blueprint. + +## Using vars + +### Vars in YAML +In YAML, vars are specified with double curly braces. +```YAML +-- titan.yml +databases: + - name: "db_{{ var.fruit }}" +``` + +In the CLI, use the `--vars` flag to pass values to Titan +```sh +# Specify values as a key-value JSON string +titan plan --config titan.yml \ + --vars '{"fruit": "banana"}' +``` + +Alternatively, use environment variables to pass values to Titan. Vars environment variables must start with `TITAN_VAR_` and must be in uppercase. + +```sh +export TITAN_VAR_FRUIT="peach" +titan plan --config titan.yml +``` + +### Vars defaults in YAML + +Use the top-level `vars:` key to define a list of expected vars. You must specify a `type`, you can optionally specify a `default`. + +```YAML +vars: + - name: color + type: string + + - name: fruit + type: string + default: apple + +databases: + - name: "db_{{ var.color }}_{{ var.fruit }}" +``` + + +### Vars in Python + +```Python +from titan.blueprint import Blueprint +from titan.resources import Database + +# In Python, a var can be specifed using Titan's var module +from titan import var +db1 = Database(name=var.db1_name) + +# Alternatively, a var can be specified inside a string with double curly braces. This is Jinja-style template syntax, not an f-string. +db2 = Database(name="db_{{ var.db2_name }}") + +# Use the vars parameter to pass values to Titan +Blueprint( + resources=[db1, db2], + vars={ + "db1_name": "pineapple", + "db2_name": "durian", + }, +) +``` + +### Vars defaults in Python + +```Python +from titan.blueprint import Blueprint +from titan.resources import Database + +# Use the vars_spec parameter to define a list of expected vars. You must specify a `type`, you can optionally specify a `default`. +Blueprint( + resources=[Database(name="db_{{ var.color }}_{{ var.fruit }}")], + vars={"color": "blue"}, + vars_spec=[ + { + "name": "color", + "type": "string", + }, + { + "name": "fruit", + "type": "string", + "default": "apple", + } + + ] +) +``` + +## Using scope + +`🔬 EXPERIMENTAL` + +When the `scope` parameter is used, Titan will limit which resources are allowed and limit where those resources are located within Snowflake. + +### Using database scope + +```YAML +-- raw.yml +scope: DATABASE +database: RAW + +schemas: + - name: FINANCE + - name: LEGAL + - name: MARKETING + +tables: + - name: products + schema: FINANCE + columns: + - name: product + data_type: string +``` + + +### Using schema scope +```YAML +-- salesforce.yml +scope: SCHEMA +database: DEV +schema: SALESFORCE + +tables: + - name: products + columns: + - name: product + data_type: string + +tags: + - name: cost_center + allowed_values: ["finance", "engineering"] +``` + +### Scope example: re-use the same schema setup for multiple engineers + +```YAML +-- dev_schema.yml +scope: SCHEMA +database: DEV + +tables: ... +views: ... +procedures: ... +``` + +```sh +titan apply --config dev_schema.yml --schema=SCH_TEEJ +titan apply --config dev_schema.yml --schema=SCH_ALLY +titan apply --config dev_schema.yml --schema=SCH_DAVE +``` + +### Scope example: combine scope with vars + +```YAML +-- finance.yml +scope: SCHEMA +database: "ANALYTICS_{{ vars.env }}" +schema: FINANCE + +tables: ... +views: ... +procedures: ... +``` + +```sh +titan apply --config finance.yml --vars='{"env": "stage"}' +titan apply --config finance.yml --vars='{"env": "prod"}' +``` \ No newline at end of file diff --git a/docs/getting-started.md b/docs/getting-started.md index d60a5ed8..f82d0aac 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,12 +2,7 @@ If you're new, the best place to start is with the Python package. -### Python + CLI Installation - -### Install from PyPi -coming soon - -### Install from source +### Install from PyPi (MacOS, Linux) ```sh python -m venv .venv @@ -15,7 +10,15 @@ source .venv/bin/activate python -m pip install titan-core ``` -### Using the Python package +### Install from PyPi (Windows) + +```bat +python -m venv .venv +.\.venv\Scripts\activate +python -m pip install titan-core +``` + +## Using the Python package ```Python import os @@ -102,9 +105,11 @@ bp.apply(session, plan) # => """ ``` -### Using the CLI +For more advanced usage, see [Blueprint](blueprint.md). -You can use the CLI to generate a plan, apply a plan, or export resources. To use the CLI, install the Python package and call `python -m titan` from the command line. +## Using the CLI + +You can use the CLI to generate a plan, apply a plan, or export resources. To use the CLI, install the Python package and call `titan` from the command line. The CLI allows you to `plan` and `apply` a Titan Core YAML config. You can specify a single input file or a directory of configs. @@ -122,13 +127,15 @@ To connect with Snowflake, the CLI uses environment variables. These environment * `SNOWFLAKE_MFA_PASSCODE` * `SNOWFLAKE_AUTHENTICATOR` + ### CLI Example +Show the help message + ```sh -# Show the help message -python -m titan --help +titan --help -# Usage: python -m titan [OPTIONS] COMMAND [ARGS]... +# Usage: titan [OPTIONS] COMMAND [ARGS]... # # titan core helps you manage your Snowflake environment. # @@ -136,12 +143,16 @@ python -m titan --help # --help Show this message and exit. # # Commands: -# apply Apply a plan to Titan resources -# export Export Titan resources -# plan Generate an execution plan based on your configuration +# apply Apply a resource config to a Snowflake account +# connect Test the connection to Snowflake +# export Generate a resource config for existing Snowflake resources +# plan Compare a resource config to the current state of Snowflake +``` -# The CLI uses YAML config. This command creates a sample config file. +Apply a resource config to Snowflake +```sh +# Create a resource config file cat < titan.yml roles: - name: transformer @@ -163,47 +174,60 @@ export SNOWFLAKE_USER="my-user" export SNOWFLAKE_PASSWORD="my-password" # Generate a plan -python -m titan plan --config titan.yml +titan plan --config titan.yml # Apply the config -python -m titan apply --config titan.yml +titan apply --config titan.yml ``` -### Using the GitHub Action +Export existing Snowflake resources to YAML. + +```sh +titan export \ + --resource=warehouse,grant,role \ + --out=titan.yml +``` + +The Titan Core Python package installs the CLI script `titan`. You can alternatively use Python CLI module syntax if you need fine-grained control over the Python environment. + +```sh +python -m titan plan --config titan.yml +``` + +## Using the GitHub Action The Titan Core GitHub Action allows you to automate the deployment of Snowflake resources using a git-based workflow. ### GitHub Action Example -```yaml -# .github/workflows/titan.yml -name: Titan Snowflake +```YAML +-- .github/workflows/titan.yml +name: Deploy to Snowflake with Titan on: push: - branches: ["main"] - # The directory in your repo where titan configs live. + branches: [ main ] paths: - - 'envs/prod/**' - + - 'titan/**' jobs: deploy: runs-on: ubuntu-latest - name: Deploy to Snowflake with Titan - - # The Github environment to use - environment: prod steps: - - uses: actions/checkout@v4 - - name: Deploy with Titan - id: titan-core-action + - name: Checkout code + uses: actions/checkout@v4 + + - name: Deploy to Snowflake uses: Titan-Systems/titan-core-action@main with: - resource-path: envs/prod - valid-resource-types: database,user,warehouse,role + run-mode: 'create-or-update' + resource-path: './titan' + allowlist: 'warehouse,role,grant' + dry-run: 'false' env: SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} -``` \ No newline at end of file +``` + +For in-depth documentation, see [Titan Core GitHub Action](titan-core-github-action.md). \ No newline at end of file diff --git a/docs/titan-core-github-action.md b/docs/titan-core-github-action.md index da0c2c4c..4d538473 100644 --- a/docs/titan-core-github-action.md +++ b/docs/titan-core-github-action.md @@ -1,57 +1,56 @@ # `titan core` GitHub Action -## Installation - -To add the Titan Core GitHub Action to your repository, follow these steps: - -1. **Create or modify the workflow file**: Add the following code to your `.github/workflows/titan.yml` file. - - ```yaml - name: Titan Core Workflow - on: [push] - - jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Deploy to Snowflake with Titan - uses: Titan-Systems/titan-core-action@main - with: - run-mode: 'create-or-update' - resource-path: './resources' - allowlist: 'warehouse,role,grant' - dry-run: 'false' - env: - SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} - SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} - ``` - -2. **Specify Snowflake secrets**: Go to your GitHub repository settings, navigate to `Secrets`, and add the following secrets: - - `SNOWFLAKE_ACCOUNT` - - `SNOWFLAKE_USER` - - `SNOWFLAKE_PASSWORD` - - `SNOWFLAKE_ROLE` - - `SNOWFLAKE_WAREHOUSE` - -3. Create a `resources` directory in your repository with your Snowflake resources. +## Using the GitHub action + +To add the Titan Core GitHub action to your repository, follow these steps: + +### Create a Titan workflow file + +Create a file in the GitHub workflows directory of your repo (`.github/workflows/titan.yml`) + +```YAML +-- .github/workflows/titan.yml +name: Deploy to Snowflake with Titan +on: + push: + branches: [ main ] + paths: + - 'titan/**' + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Deploy to Snowflake + uses: Titan-Systems/titan-core-action@main + with: + run-mode: 'create-or-update' + resource-path: './titan' + allowlist: 'warehouse,role,grant' + dry-run: 'false' + env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} +``` + +### Configure your Snowflake connection + +Go to your GitHub repository settings, navigate to `Secrets`. There, add a secret for `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, and whatever other connection settings you need. + + +### Create a `titan` directory in your repository + +Add YAML resource configs to the `titan` directory. ```YAML -# resources/warehouses.yml +# titan/warehouses.yml warehouses: - - name: loading - warehouse_size: XSMALL - auto_suspend: 60 - auto_resume: true - - name: transforming - warehouse_size: XSMALL - auto_suspend: 60 - auto_resume: true - name: reporting warehouse_size: XSMALL auto_suspend: 60 @@ -59,24 +58,13 @@ warehouses: ``` ```YAML -# resources/rbac.yml -# Specify roles +# titan/rbac.yml + roles: - - name: loader - comment: "Owns the tables in your raw database, and connects to the loading warehouse." - - name: transformer - comment: "Has query permissions on tables in raw database and owns tables in the analytics database. This is for dbt developers and scheduled jobs." - name: reporter - comment: "Has permissions on the analytics database only. This role is for data consumers, such as analysts and BI tools. These users will not have permissions to read data from the raw database." + comment: "Has permissions on the analytics database..." -# Permissions grants: - - to_role: loader - priv: usage - on_warehouse: loading - - to_role: transformer - priv: usage - on_warehouse: transforming - to_role: reporter priv: usage on_warehouse: reporting @@ -84,17 +72,62 @@ grants: priv: usage on_database: analytics -# Grant all roles to SYSADMIN role_grants: - - role: loader - roles: - - SYSADMIN - - role: transformer - roles: - - SYSADMIN - role: reporter roles: - SYSADMIN ``` -4. Commit and push your changes. \ No newline at end of file +### Commit and push your changes + +When you push to `main` changes to files in the `titan/` directory, the Github Action will deploy them to Snowflake. + +## Configuration options + +**run-mode** `string` + +Defines how the blueprint interacts with the Snowflake account + +- Default: `"create-or-update"` +- **create-or-update** + - Resources are either created or updated, no resources are destroyed +- **sync**: + - `⚠️ WARNING` Sync mode will drop resources. + - Titan will update Snowflake to match the blueprint exactly. Must be used with `allowlist`. + +**resource-path** `string` + +Defines the file or directory where Titan will look for the resource configs + +- Default: `"."` + +**allowlist** `list[string] or "all"` + +Defines which resource types are allowed + + - Default: `"all"` + +**dry_run** `bool` + +**vars** `dict` + +**vars_spec** `list[dict]` + +**scope** `str` + +**database** `str` + +**schema** `str` + +## Ignore files with `.titanignore` + +If you specify a directory as the `resource-path`, Titan will recursively look for all files with a `.yaml` or `.yml` file extension. You can tell Titan to exclude files or directories with a `.titanignore` file. This file uses [gitignore syntax](https://git-scm.com/docs/gitignore). + +### `.titanignore` example + +``` +# .titanignore + +# Ignore dbt config +dbt_project.yml +``` \ No newline at end of file diff --git a/docs/vars.md b/docs/vars.md deleted file mode 100644 index 9ee5e743..00000000 --- a/docs/vars.md +++ /dev/null @@ -1,33 +0,0 @@ -# Vars - -First pass proposal at vars behavior - -## YAML/CLI - -titan.yml -```yaml -vars: - - name: foobar - type: string - default: some_default_value - sensitive: true -databases: - - name: "db_{{ var.foobar }}" -``` - -```sh -titan plan --config titan.yml --vars '{"foobar": "blimblam"}' -titan plan --config titan.yml --vars 'foobar: blimblam' -``` - -## Python -```Python -from titan.blueprint import Blueprint -from titan import var - -# Deferred style -db = Database(name=var.foobar) -# Interpolation style -db = Database(name="db_{{var.foobar}}") -Blueprint(resources=[db], vars={"foobar": "blimblam"}) -``` \ No newline at end of file diff --git a/examples/snowflake-tutorials-create-your-first-iceberg-table.yml b/examples/snowflake-tutorials-create-your-first-iceberg-table.yml index d877a502..4af5ef08 100644 --- a/examples/snowflake-tutorials-create-your-first-iceberg-table.yml +++ b/examples/snowflake-tutorials-create-your-first-iceberg-table.yml @@ -41,7 +41,7 @@ grants: external_volumes: - name: iceberg_external_volume owner: iceberg_tutorial_role - allow_writes: false + allow_writes: true storage_locations: - name: my-s3-us-west-2 storage_provider: S3 diff --git a/tests/fixtures/json/network_policy.json b/tests/fixtures/json/network_policy.json index cf7f7728..421c2fc4 100644 --- a/tests/fixtures/json/network_policy.json +++ b/tests/fixtures/json/network_policy.json @@ -1,10 +1,10 @@ { "name": "some_network_policy", "allowed_network_rule_list": [ - "static_database.public.static_network_rule" + "static_database.public.static_network_rule_ingress_allow_all" ], "blocked_network_rule_list": [ - "static_database.public.static_network_rule" + "static_database.public.static_network_rule_ingress_suspicious_ip" ], "allowed_ip_list": [ "192.168.1.0/24" @@ -12,6 +12,6 @@ "blocked_ip_list": [ "192.168.1.99" ], - "comment": "Commend for a network policy", + "comment": "Comment for a network policy", "owner": "SECURITYADMIN" } \ No newline at end of file diff --git a/tests/integration/data_provider/test_list_resource.py b/tests/integration/data_provider/test_list_resource.py index 5070e90e..714d1b63 100644 --- a/tests/integration/data_provider/test_list_resource.py +++ b/tests/integration/data_provider/test_list_resource.py @@ -88,3 +88,11 @@ def test_list_resource(cursor, list_resources_database, resource, marked_for_cle list_resources = data_provider.list_resource(cursor, resource_label_for_type(resource.resource_type)) assert len(list_resources) > 0 assert resource.fqn in list_resources + + +# @pytest.mark.enterprise +# def test_list_tag_references(cursor): +# data_provider.fetch_session.cache_clear() +# reset_cache() +# tag_references = data_provider.list_tag_references(cursor) +# assert len(tag_references) > 0 diff --git a/titan/cli.py b/titan/cli.py index 4a7ecc9a..c711f13d 100644 --- a/titan/cli.py +++ b/titan/cli.py @@ -141,7 +141,7 @@ def schema_option(): @database_option() @schema_option() def plan(config_path, json_output, output_file, vars: dict, allowlist, run_mode, scope, database, schema): - """Generate an execution plan based on your configuration""" + """Compare a resource config to the current state of Snowflake""" if not config_path: raise click.UsageError("--config is required") @@ -193,7 +193,7 @@ def plan(config_path, json_output, output_file, vars: dict, allowlist, run_mode, @schema_option() @click.option("--dry-run", is_flag=True, help="When dry run is true, Titan will not make any changes to Snowflake") def apply(config_path, plan_file, vars, allowlist, run_mode, scope, database, schema, dry_run): - """Apply an execution plan to a Snowflake account""" + """Apply a resource config to a Snowflake account""" if config_path and plan_file: raise click.UsageError("Cannot specify both --config and --plan.") @@ -253,7 +253,9 @@ def apply(config_path, plan_file, vars, allowlist, run_mode, scope, database, sc @click.option("--format", type=click.Choice(["json", "yml"]), default="yml", help="Output format") def export(resources, export_all, exclude_resources, out, format): """ - This command allows you to export resources from Titan in either JSON or YAML format. + Generate a resource config for existing Snowflake resources + + This command allows you to export resources from Snowflake in either JSON or YAML format. You can specify the type of resource to export and the output filename for the exported data. Resource types are specified with snake case (eg. Warehouse => warehouse, NetworkRule => network_rule, etc.). diff --git a/titan/data_provider.py b/titan/data_provider.py index 17939cf5..f74796b2 100644 --- a/titan/data_provider.py +++ b/titan/data_provider.py @@ -777,7 +777,6 @@ def fetch_columns(session: SnowflakeConnection, resource_type: str, fqn: FQN): if col["kind"] != "COLUMN": raise Exception(f"Unexpected kind {col['kind']} in desc result") columns.append( - # remove_none_values( { "name": col["name"], "data_type": col["type"], @@ -787,7 +786,6 @@ def fetch_columns(session: SnowflakeConnection, resource_type: str, fqn: FQN): "constraint": None, "collate": None, } - # ) ) return columns @@ -2616,50 +2614,46 @@ def list_tables(session: SnowflakeConnection) -> list[FQN]: def list_tag_references(session: SnowflakeConnection) -> list[FQN]: + # FIXME + # This function previously relied on a system table function with 2 hours of latency. + try: - show_result = execute(session, "SHOW TAGS IN ACCOUNT") + # show_result = execute(session, "SHOW TAGS IN ACCOUNT") tag_references: list[FQN] = [] - for tag in show_result: - if tag["database_name"] in SYSTEM_DATABASES or tag["schema_name"] == "INFORMATION_SCHEMA": - continue + # for tag in show_result: + # if tag["database_name"] in SYSTEM_DATABASES or tag["schema_name"] == "INFORMATION_SCHEMA": + # continue + + # tag_refs = execute( + # session, + # f""" + # SELECT * + # FROM table(snowflake.account_usage.tag_references_with_lineage( + # '{tag['database_name']}.{tag['schema_name']}.{tag['name']}' + # )) + # """, + # ) - tag_refs = execute( - session, - f""" - SELECT * - FROM table(snowflake.account_usage.tag_references_with_lineage( - '{tag['database_name']}.{tag['schema_name']}.{tag['name']}' - )) - """, - ) + # for ref in tag_refs: + # if ref["OBJECT_DELETED"] is not None: + # continue + + # tag_references.append( + # FQN( + # database=resource_name_from_snowflake_metadata(ref["TAG_DATABASE"]), + # schema=resource_name_from_snowflake_metadata(ref["TAG_SCHEMA"]), + # name=resource_name_from_snowflake_metadata(ref["TAG_NAME"]), + # ) + # ) - for ref in tag_refs: - if ref["OBJECT_DELETED"] is not None: - continue - print(ref) - # raise return tag_references - # tags.append( - # FQN( - # database=resource_name_from_snowflake_metadata(row["database_name"]), - # schema=resource_name_from_snowflake_metadata(tag["schema_name"]), - # name=resource_name_from_snowflake_metadata(tag["name"]), - # ) - # ) - # return tags except ProgrammingError as err: if err.errno == UNSUPPORTED_FEATURE: return [] else: raise - tag_map = {} - for tag_ref in tag_refs: - tag_name = f"{tag_ref['TAG_DATABASE']}.{tag_ref['TAG_SCHEMA']}.{tag_ref['TAG_NAME']}" - tag_map[tag_name] = tag_ref["TAG_VALUE"] - return tag_map - def list_tags(session: SnowflakeConnection) -> list[FQN]: try: diff --git a/tools/test_account_configs/base.yml b/tools/test_account_configs/base.yml index 5bd6908d..6677ca4b 100644 --- a/tools/test_account_configs/base.yml +++ b/tools/test_account_configs/base.yml @@ -116,6 +116,18 @@ network_rules: mode: EGRESS database: static_database schema: public + - name: static_network_rule_ingress_allow_all + type: IPV4 + value_list: ["0.0.0.0/0"] + mode: INGRESS + database: static_database + schema: public + - name: static_network_rule_ingress_suspicious_ip + type: IPV4 + value_list: ["104.223.91.28"] + mode: INGRESS + database: static_database + schema: public grants: diff --git a/tools/test_account_configs/business_critical.yml b/tools/test_account_configs/business_critical.yml index ffe1f5f2..28fce0eb 100644 --- a/tools/test_account_configs/business_critical.yml +++ b/tools/test_account_configs/business_critical.yml @@ -8,6 +8,12 @@ grants: - GRANT APPLY ROW ACCESS POLICY ON ACCOUNT TO ROLE EVERY_PRIVILEGE - GRANT APPLY TAG ON ACCOUNT TO ROLE EVERY_PRIVILEGE +schemas: + - name: tagged_schema + database: static_database + tags: + static_database.public.static_tag: STATIC_TAG_VALUE + tags: - name: static_tag database: static_database diff --git a/tools/test_account_configs/enterprise.yml b/tools/test_account_configs/enterprise.yml index ffe1f5f2..28fce0eb 100644 --- a/tools/test_account_configs/enterprise.yml +++ b/tools/test_account_configs/enterprise.yml @@ -8,6 +8,12 @@ grants: - GRANT APPLY ROW ACCESS POLICY ON ACCOUNT TO ROLE EVERY_PRIVILEGE - GRANT APPLY TAG ON ACCOUNT TO ROLE EVERY_PRIVILEGE +schemas: + - name: tagged_schema + database: static_database + tags: + static_database.public.static_tag: STATIC_TAG_VALUE + tags: - name: static_tag database: static_database