Merge pull request #28 from godatadriven/parse-manifest

Adding logic to parse manifest.json

.gitignore

@@ -162,6 +162,7 @@ cython_debug/
 *.ipynb
 pytest-coverage.txt
 
+dbt_project/dbt_packages
 dbt_project/logs
 !dbt_project/target
 dbt_project/target/compiled

dbt_bouncer/main.py

@@ -1,6 +1,8 @@
+import json
 from pathlib import Path
 
 import click
+from dbt_artifacts_parser.parser import parse_manifest
 
 from dbt_bouncer.logger import logger
 from dbt_bouncer.version import version
@@ -16,8 +18,32 @@
 @click.version_option()
 def cli(dbt_project_dir):
     logger.info(f"Running dbt_bouncer ({version()})...")
-    logger.debug(f"{dbt_project_dir=}")
 
+    # Load manifest
     manifest_json_path = Path(dbt_project_dir) / "target/manifest.json"
+    logger.info(f"Loading manifest.json from {manifest_json_path}...")
     if not manifest_json_path.exists():
-        raise RuntimeError(f"No manifest.json found at {manifest_json_path}.")
+        raise FileNotFoundError(f"No manifest.json found at {manifest_json_path}.")
+
+    with Path.open(manifest_json_path, "r") as fp:
+        manifest_obj = parse_manifest(manifest=json.load(fp))
+
+    logger.debug(f"{manifest_obj.metadata.project_name=}")
+
+    project_models = []
+    project_tests = []
+    for _, v in manifest_obj.nodes.items():
+        if v.package_name == manifest_obj.metadata.project_name:
+            if v.resource_type == "model":
+                project_models.append(v)
+            elif v.resource_type == "test":
+                project_tests.append(v)
+
+    project_sources = []
+    for _, v in manifest_obj.sources.items():
+        if v.package_name == manifest_obj.metadata.project_name:
+            project_sources.append(v)
+
+    logger.info(
+        f"Parsed `{manifest_obj.metadata.project_name}` project, found {len(project_models)} nodes, {len(project_sources)} sources and {len(project_tests)} tests."
+    )

dbt_project/dbt_project.yml

@@ -1,4 +1,4 @@
-name: 'dbt_bouncer'
+name: 'dbt_bouncer_test_project'
 version: '1.0.0'
 config-version: 2
 

dbt_project/models/_schema.yml

@@ -0,0 +1,8 @@
+version: 2
+
+models:
+  - name: model_1
+    columns:
+      - name: id
+        tests:
+          - not_null

dbt_project/models/_source.yml

@@ -0,0 +1,7 @@
+version: 2
+
+sources:
+  - name: "source"
+    tables:
+      - name: source_1
+      - name: source_2

dbt_project/package-lock.yml

@@ -0,0 +1,6 @@
+packages:
+  - package: dbt-labs/dbt_project_evaluator
+    version: 0.12.1
+  - package: dbt-labs/dbt_utils
+    version: 1.2.0
+sha1_hash: 316722b8f7c25ad5739be5095cac1154196f98d4

dbt_project/packages.yml

@@ -0,0 +1,3 @@
+packages:
+  - package: dbt-labs/dbt_project_evaluator
+    version: 0.12.1

pyproject.toml

@@ -28,6 +28,10 @@ disallow_any_generics = true
 check_untyped_defs = true
 no_implicit_reexport = true
 
+[[tool.mypy.overrides]]
+ignore_missing_imports = true
+module = "dbt_artifacts_parser.*"
+
 [tool.poetry]
 authors = ["Padraic Slattery <[email protected]>"]
 description = "Configure and enforce conventions for your dbt project."
@@ -44,6 +48,7 @@ version = "0.0.0"
 [tool.poetry.dependencies]
 python = ">=3.8,<3.13"
 click = "*"
+dbt-artifacts-parser = "^0"
 
 [tool.poetry.group.dev.dependencies]
 dbt-core="*"

tests/test_logger.py

@@ -20,7 +20,7 @@ def test_logger_debug() -> None:
                 [
                     record
                     for record in records_to_tuples(handler.records)
-                    if record[2].startswith("dbt_project_dir=")
+                    if record[2].startswith("Loading manifest.json from ")
                 ]
             )
             == 1

tests/test_main.py

@@ -17,10 +17,10 @@ def test_cli_happy_path(caplog):
         ],
     )
     assert "Running dbt_bouncer (0.0.0)..." in caplog.text
+    assert "oading manifest.json from dbt_project/target/manifest.json..." in caplog.text
 
 
 def test_cli_dbt_dir_doesnt_exist():
-
     runner = CliRunner()
     result = runner.invoke(
         cli,
@@ -33,7 +33,6 @@ def test_cli_dbt_dir_doesnt_exist():
 
 
 def test_cli_manifest_doesnt_exist(tmp_path):
-
     runner = CliRunner()
     result = runner.invoke(
         cli,
@@ -42,7 +41,7 @@ def test_cli_manifest_doesnt_exist(tmp_path):
             tmp_path,
         ],
     )
-    assert type(result.exception) in [RuntimeError]
+    assert type(result.exception) in [FileNotFoundError]
     assert (
         result.exception.args[0]
         == f"No manifest.json found at {tmp_path / 'target/manifest.json'}."