[DENG-3526] Support for publishing/cleaning authorized views only (#8406)

Author: scholtzan

bigquery_etl/cli/view.py

@@ -165,6 +165,12 @@ def _view_is_valid(v: View) -> bool:
     is_flag=True,
     help="Don't publish views with labels: {authorized: true} in metadata.yaml",
 )
+@click.option(
+    "--authorized-only",
+    "--authorized_only",
+    is_flag=True,
+    help="Only publish views with labels: {authorized: true} in metadata.yaml",
+)
 @click.option(
     "--force",
     is_flag=True,
@@ -190,6 +196,7 @@ def publish(
     dry_run,
     user_facing_only,
     skip_authorized,
+    authorized_only,
     force,
     add_managed_label,
     respect_dryrun_skip,
@@ -200,9 +207,17 @@ def publish(
         logging.basicConfig(level=log_level, format="%(levelname)s %(message)s")
     except ValueError as e:
         raise click.ClickException(f"argument --log-level: {e}")
+
+    if skip_authorized and authorized_only:
+        raise click.ClickException(
+            "Cannot use both --skip-authorized and --authorized-only"
+        )
+
     credentials = get_credentials()
 
-    views = _collect_views(name, sql_dir, project_id, user_facing_only, skip_authorized)
+    views = _collect_views(
+        name, sql_dir, project_id, user_facing_only, skip_authorized, authorized_only
+    )
     if respect_dryrun_skip:
         views = [view for view in views if view.path not in DryRun.skipped_files()]
     if add_managed_label:
@@ -247,7 +262,9 @@ def _view_has_changes(target_project, credentials, view):
     return view.has_changes(target_project, credentials)
 
 
-def _collect_views(name, sql_dir, project_id, user_facing_only, skip_authorized):
+def _collect_views(
+    name, sql_dir, project_id, user_facing_only, skip_authorized, authorized_only=False
+):
     view_files = paths_matching_name_pattern(
         name, sql_dir, project_id, files=("view.sql",)
     )
@@ -267,6 +284,17 @@ def _collect_views(name, sql_dir, project_id, user_facing_only, skip_authorized)
                 and v.metadata.labels.get("authorized") == ""
             )
         ]
+    if authorized_only:
+        views = [
+            v
+            for v in views
+            if (
+                v.metadata
+                and v.metadata.labels
+                # labels with boolean true are translated to ""
+                and v.metadata.labels.get("authorized") == ""
+            )
+        ]
     return views
 
 
@@ -314,7 +342,13 @@ def _collect_views(name, sql_dir, project_id, user_facing_only, skip_authorized)
     "--skip-authorized",
     "--skip_authorized",
     is_flag=True,
-    help="Don't publish views with labels: {authorized: true} in metadata.yaml",
+    help="Don't clean views with labels: {authorized: true} in metadata.yaml",
+)
+@click.option(
+    "--authorized-only",
+    "--authorized_only",
+    is_flag=True,
+    help="Only clean views with labels: {authorized: true} in metadata.yaml",
 )
 def clean(
     name,
@@ -326,6 +360,7 @@ def clean(
     dry_run,
     user_facing_only,
     skip_authorized,
+    authorized_only,
 ):
     """Clean managed views."""
     # set log level
@@ -334,13 +369,23 @@ def clean(
     except ValueError as e:
         raise click.ClickException(f"argument --log-level: {e}")
 
+    if skip_authorized and authorized_only:
+        raise click.ClickException(
+            "Cannot use both --skip-authorized and --authorized-only"
+        )
+
     if project_id is None and target_project is None:
         raise click.ClickException("command requires --project-id or --target-project")
 
     expected_view_ids = {
         view.target_view_identifier(target_project)
         for view in _collect_views(
-            name, sql_dir, project_id, user_facing_only, skip_authorized
+            name,
+            sql_dir,
+            project_id,
+            user_facing_only,
+            skip_authorized,
+            authorized_only,
         )
     }
 
@@ -365,7 +410,13 @@ def clean(
             for views in p.starmap(
                 client_q.with_client,
                 (
-                    (_list_managed_views, dataset, name, skip_authorized)
+                    (
+                        _list_managed_views,
+                        dataset,
+                        name,
+                        skip_authorized,
+                        authorized_only,
+                    )
                     for dataset in datasets
                 ),
                 chunksize=1,
@@ -381,7 +432,9 @@ def clean(
         )
 
 
-def _list_managed_views(client, dataset, pattern, skip_authorized):
+def _list_managed_views(
+    client, dataset, pattern, skip_authorized, authorized_only=False
+):
     query = f"""
       SELECT
         table_catalog || "." || table_schema || "." || table_name AS table_id,
@@ -407,6 +460,7 @@ def _list_managed_views(client, dataset, pattern, skip_authorized):
         for row in result
         if (pattern is None or fnmatchcase(sql_table_id(row.table_id), f"*{pattern}"))
         and (not skip_authorized or not row.is_authorized)
+        and (not authorized_only or row.is_authorized)
     ]
 
 

tests/view/test_view.py

@@ -6,6 +6,8 @@
 from google.api_core.exceptions import NotFound
 from google.cloud.bigquery import SchemaField
 
+from bigquery_etl.cli.view import _collect_views
+from bigquery_etl.metadata.parse_metadata import Metadata
 from bigquery_etl.view import CREATE_VIEW_PATTERN, View
 
 TEST_DIR = Path(__file__).parent.parent
@@ -212,3 +214,117 @@ def test_view_has_changes_changed_schema(
 
         assert simple_view.has_changes()
         assert "schema" in capsys.readouterr().out
+
+    @patch("bigquery_etl.cli.view.get_id_token")
+    def test_collect_views_authorized_only(self, mock_get_id_token, runner):
+        """Test that authorized_only flag filters views correctly."""
+        mock_get_id_token.return_value = None
+
+        with runner.isolated_filesystem():
+            # Create test directory structure
+            Path("sql/moz-fx-data-shared-prod/test").mkdir(parents=True)
+
+            # Create an authorized view
+            authorized_view_dir = Path(
+                "sql/moz-fx-data-shared-prod/test/authorized_view"
+            )
+            authorized_view_dir.mkdir()
+            (authorized_view_dir / "view.sql").write_text(
+                "CREATE OR REPLACE VIEW `moz-fx-data-shared-prod.test.authorized_view` AS SELECT 1"
+            )
+            authorized_metadata = Metadata(
+                friendly_name="Authorized View",
+                description="Test authorized view",
+                owners=["[email protected]"],
+                labels={"authorized": True},
+            )
+            authorized_metadata.write(authorized_view_dir / "metadata.yaml")
+
+            # Create a non-authorized view
+            regular_view_dir = Path("sql/moz-fx-data-shared-prod/test/regular_view")
+            regular_view_dir.mkdir()
+            (regular_view_dir / "view.sql").write_text(
+                "CREATE OR REPLACE VIEW `moz-fx-data-shared-prod.test.regular_view` AS SELECT 1"
+            )
+            regular_metadata = Metadata(
+                friendly_name="Regular View",
+                description="Test regular view",
+                owners=["[email protected]"],
+            )
+            regular_metadata.write(regular_view_dir / "metadata.yaml")
+
+            # Test authorized_only=True
+            views = _collect_views(
+                name=None,
+                sql_dir="sql",
+                project_id="moz-fx-data-shared-prod",
+                user_facing_only=False,
+                skip_authorized=False,
+                authorized_only=True,
+            )
+            assert len(views) == 1
+            assert views[0].name == "authorized_view"
+
+            # Test skip_authorized=True
+            views = _collect_views(
+                name=None,
+                sql_dir="sql",
+                project_id="moz-fx-data-shared-prod",
+                user_facing_only=False,
+                skip_authorized=True,
+                authorized_only=False,
+            )
+            assert len(views) == 1
+            assert views[0].name == "regular_view"
+
+            # Test both flags False (get all views)
+            views = _collect_views(
+                name=None,
+                sql_dir="sql",
+                project_id="moz-fx-data-shared-prod",
+                user_facing_only=False,
+                skip_authorized=False,
+                authorized_only=False,
+            )
+            assert len(views) == 2
+
+    def test_publish_authorized_only_mutually_exclusive(self, runner):
+        """Test that --authorized-only and --skip-authorized are mutually exclusive."""
+        from bigquery_etl.cli.view import publish
+
+        with runner.isolated_filesystem():
+            Path("sql/moz-fx-data-shared-prod/test").mkdir(parents=True)
+
+            result = runner.invoke(
+                publish,
+                [
+                    "--authorized-only",
+                    "--skip-authorized",
+                    "--project-id=moz-fx-data-shared-prod",
+                ],
+            )
+            assert result.exit_code != 0
+            assert (
+                "Cannot use both --skip-authorized and --authorized-only"
+                in result.output
+            )
+
+    def test_clean_authorized_only_mutually_exclusive(self, runner):
+        """Test that --authorized-only and --skip-authorized are mutually exclusive in clean."""
+        from bigquery_etl.cli.view import clean
+
+        with runner.isolated_filesystem():
+            with pytest.raises(AssertionError):
+                result = runner.invoke(
+                    clean,
+                    [
+                        "--authorized-only",
+                        "--skip-authorized",
+                        "--target-project=moz-fx-data-shared-prod",
+                    ],
+                )
+                assert result.exit_code != 0
+                assert (
+                    "Cannot use both --skip-authorized and --authorized-only"
+                    in result.output
+                )