feat(query): Add bqetl query render support for Jinja imports/includes from outside the ETL directory (DENG-9950) (#8485)

* feat(query): Add `bqetl query render` support for Jinja imports/includes from outside the ETL directory (DENG-9950).

* feat(util): Add a `get_bqetl_project_root()` function to make that logic reusable and cache the return value.

Author: sean-rose

bigquery_etl/util/common.py

@@ -8,6 +8,7 @@
 import string
 import tempfile
 import warnings
+from functools import cache
 from pathlib import Path
 from typing import List, Optional, Set, Tuple
 from uuid import uuid4
@@ -17,7 +18,7 @@
 from google.cloud import bigquery
 from jinja2 import Environment, FileSystemLoader
 
-from bigquery_etl.config import ConfigLoader
+from bigquery_etl.config import BQETL_PROJECT_CONFIG, ConfigLoader
 from bigquery_etl.format_sql.formatter import reformat
 from bigquery_etl.metrics import MetricHub
 
@@ -70,6 +71,18 @@ def random_str(length: int = 12) -> str:
     return "".join(random.choice(string.ascii_lowercase) for i in range(length))
 
 
+@cache
+def get_bqetl_project_root() -> Path | None:
+    """Return the root path of the bqetl project the user is currently in."""
+    cwd = Path.cwd()
+    search_paths = [cwd]
+    search_paths.extend(cwd.parents)
+    for possible_project_root in search_paths:
+        if (possible_project_root / BQETL_PROJECT_CONFIG).exists():
+            return possible_project_root
+    return None
+
+
 def render(
     sql_filename,
     template_folder=".",
@@ -78,7 +91,8 @@ def render(
     **kwargs,
 ) -> str:
     """Render a given template query using Jinja."""
-    path = Path(template_folder) / sql_filename
+    template_folder_path = Path(template_folder)
+    path = template_folder_path / sql_filename
     skip = {
         file
         for skip in ConfigLoader.get("render", "skip", fallback=[])
@@ -122,14 +136,20 @@ def render(
                 checks_template.write_text(
                     macro_imports
                     + "\n"
-                    + (Path(template_folder) / sql_filename).read_text()
+                    + (template_folder_path / sql_filename).read_text()
                 )
 
                 file_loader = FileSystemLoader(f"{str(checks_template.parent)}")
                 env = Environment(loader=file_loader)
                 main_sql = env.get_template(checks_template.name)
         else:
-            file_loader = FileSystemLoader(f"{template_folder}")
+            # Add the bigquery-etl project root to the search path to support Jinja imports/includes.
+            file_loader_search_paths = [template_folder_path, ROOT]
+            # Also dynamically detect the project root so imports/includes in the private-bigquery-etl repo work.
+            if bqetl_project_root := get_bqetl_project_root():
+                if bqetl_project_root not in file_loader_search_paths:
+                    file_loader_search_paths.append(bqetl_project_root)
+            file_loader = FileSystemLoader(file_loader_search_paths)
             env = Environment(loader=file_loader)
             main_sql = env.get_template(sql_filename)