From c022c931e0b005a5629085aa30b826de521a8c70 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Thu, 6 Nov 2025 10:42:12 -0800 Subject: [PATCH 1/5] chore(shredder): propagate fxa and subplat deletes to cirrus --- bigquery_etl/shredder/config.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bigquery_etl/shredder/config.py b/bigquery_etl/shredder/config.py index 9c0049a9214..50049d6f04a 100755 --- a/bigquery_etl/shredder/config.py +++ b/bigquery_etl/shredder/config.py @@ -703,6 +703,22 @@ def fields(self) -> tuple[str, ...]: table="experimenter_cirrus_stable.enrollment_status_v1", field=CIRRUS_EVENTS_NIMBUS_USER_ID, ): EXPERIMENTER_BACKEND_SRC, + DeleteTarget( + table="accounts_cirrus_stable.enrollment_v1", + field=CIRRUS_EVENTS_NIMBUS_USER_ID, + ): FXA_UNHASHED_SRC, + DeleteTarget( + table="accounts_cirrus_stable.enrollment_status_v1", + field=CIRRUS_EVENTS_NIMBUS_USER_ID, + ): FXA_UNHASHED_SRC, + DeleteTarget( + table="subplat_cirrus_stable.enrollment_v1", + field=CIRRUS_EVENTS_NIMBUS_USER_ID, + ): FXA_UNHASHED_SRC, + DeleteTarget( + table="subplat_cirrus_stable.enrollment_status_v1", + field=CIRRUS_EVENTS_NIMBUS_USER_ID, + ): FXA_UNHASHED_SRC, } SEARCH_IGNORE_TABLES = {source.table for source in SOURCES} From c789800e57c5bd4bd052088bc5a44f65adaade38 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Thu, 6 Nov 2025 12:14:36 -0800 Subject: [PATCH 2/5] chore(experiment_monitoring): add config for experimenter and subplat cirrus --- .../experiment_monitoring/templates/templating.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sql_generators/experiment_monitoring/templates/templating.yaml b/sql_generators/experiment_monitoring/templates/templating.yaml index 60fc9d1e587..20be4ef5f86 100644 --- a/sql_generators/experiment_monitoring/templates/templating.yaml +++ b/sql_generators/experiment_monitoring/templates/templating.yaml @@ -128,6 +128,14 @@ search_metrics: ad_clicks_count: null search_with_ads_count: null search_count: null + experimenter_cirrus: + ad_clicks_count: null + search_with_ads_count: null + search_count: null + subplat_cirrus: + ad_clicks_count: null + search_with_ads_count: null + search_count: null applications: - org_mozilla_firefox_beta - org_mozilla_fenix @@ -145,3 +153,5 @@ applications: - org_mozilla_ios_focus - monitor_cirrus - accounts_cirrus + - experimenter_cirrus + - subplat_cirrus From b95650863256a759e8ea6c65cdfca68341a493c5 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Mon, 10 Nov 2025 10:20:56 -0800 Subject: [PATCH 3/5] Rename subplat_cirrus to subscription_platform_backend_cirrus --- .../experiment_monitoring/templates/templating.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql_generators/experiment_monitoring/templates/templating.yaml b/sql_generators/experiment_monitoring/templates/templating.yaml index 20be4ef5f86..c7090c29ebd 100644 --- a/sql_generators/experiment_monitoring/templates/templating.yaml +++ b/sql_generators/experiment_monitoring/templates/templating.yaml @@ -132,7 +132,7 @@ search_metrics: ad_clicks_count: null search_with_ads_count: null search_count: null - subplat_cirrus: + subscription_platform_backend_cirrus: ad_clicks_count: null search_with_ads_count: null search_count: null @@ -154,4 +154,4 @@ applications: - monitor_cirrus - accounts_cirrus - experimenter_cirrus - - subplat_cirrus + - subscription_platform_backend_cirrus From 96f5d71ea2f170f6faadcf1f592919e74c41b72f Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Mon, 10 Nov 2025 10:21:31 -0800 Subject: [PATCH 4/5] Update table names in DeleteTarget configurations --- bigquery_etl/shredder/config.py | 19 +++---- .../delete_events/view.sql | 7 +++ .../delete_events_v1/metadata.yaml | 17 ++++++ .../delete_events_v1/query.sql | 53 +++++++++++++++++++ .../delete_events_v1/schema.yaml | 7 +++ .../encrypted_keys_v1/metadata.yaml | 2 + 6 files changed, 93 insertions(+), 12 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus/delete_events/view.sql create mode 100644 sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/subscription_platform_derived/encrypted_keys_v1/metadata.yaml diff --git a/bigquery_etl/shredder/config.py b/bigquery_etl/shredder/config.py index 50049d6f04a..839987fc0a0 100755 --- a/bigquery_etl/shredder/config.py +++ b/bigquery_etl/shredder/config.py @@ -151,6 +151,9 @@ def fields(self) -> tuple[str, ...]: FXA_FRONTEND_GLEAN_SRC = DeleteSource( table="accounts_frontend_stable.deletion_request_v1", field=GLEAN_CLIENT_ID ) +SUBPLAT_CIRRUS_SRC = DeleteSource( + table="subscription_platform_backend_cirrus.delete_events", field="nimbus_user_id" +) EXPERIMENTER_BACKEND_SRC = DeleteSource( table="experimenter_backend_stable.data_collection_opt_out_v1", field=NIMBUS_USER_ID, @@ -704,21 +707,13 @@ def fields(self) -> tuple[str, ...]: field=CIRRUS_EVENTS_NIMBUS_USER_ID, ): EXPERIMENTER_BACKEND_SRC, DeleteTarget( - table="accounts_cirrus_stable.enrollment_v1", - field=CIRRUS_EVENTS_NIMBUS_USER_ID, - ): FXA_UNHASHED_SRC, - DeleteTarget( - table="accounts_cirrus_stable.enrollment_status_v1", - field=CIRRUS_EVENTS_NIMBUS_USER_ID, - ): FXA_UNHASHED_SRC, - DeleteTarget( - table="subplat_cirrus_stable.enrollment_v1", + table="subscription_platform_backend_cirrus_stable.enrollment_v1", field=CIRRUS_EVENTS_NIMBUS_USER_ID, - ): FXA_UNHASHED_SRC, + ): SUBPLAT_CIRRUS_SRC, DeleteTarget( - table="subplat_cirrus_stable.enrollment_status_v1", + table="subscription_platform_backend_cirrus_stable.enrollment_status_v1", field=CIRRUS_EVENTS_NIMBUS_USER_ID, - ): FXA_UNHASHED_SRC, + ): SUBPLAT_CIRRUS_SRC, } SEARCH_IGNORE_TABLES = {source.table for source in SOURCES} diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus/delete_events/view.sql b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus/delete_events/view.sql new file mode 100644 index 00000000000..8d0f4906c9f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus/delete_events/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.subscription_platform_backend_cirrus.delete_events` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.subscription_platform_backend_cirrus_derived.delete_events_v1` diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/metadata.yaml new file mode 100644 index 00000000000..9ddc354874f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/metadata.yaml @@ -0,0 +1,17 @@ +--- +friendly_name: Subscription Platform Backend Cirrus Delete Events +description: Deletion events for subscription platform backend cirrus, derived + from FxA's delete events and subplat's namespace secrets +owners: + - mwilliams@mozilla.com +labels: + incremental: true + schedule: daily + table_type: client_level +scheduling: + dag_name: bqetl_fxa_events +bigquery: + time_partitioning: + type: day + field: submission_timestamp + require_partition_filter: false diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql new file mode 100644 index 00000000000..a5670489163 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql @@ -0,0 +1,53 @@ +CREATE TEMP FUNCTION format_uuid(value STRING) +RETURNS STRING AS ( + CONCAT( + SUBSTR(value, 1, 8), + "-", + SUBSTR(value, 9, 4), + "-", + SUBSTR(value, 13, 4), + "-", + SUBSTR(value, 17, 4), + "-", + SUBSTR(value, 21, 12) + ) +); + +CREATE TEMP FUNCTION uuid_v5(value STRING, namespace STRING) +RETURNS STRING AS ( + FORMAT_UUID( + TO_HEX( + LEFT( + SHA1(CONCAT(FROM_HEX(REPLACE(namespace, "-", "")), CAST(value AS BYTES FORMAT 'UTF8'))), + 16 + ) -- + & FROM_HEX('ffffffffffff0fff3fffffffffffffff') -- + | FROM_HEX('00000000000050008000000000000000') + ) + ) +); + +WITH subplat_namespaces AS ( + SELECT + CAST( + AEAD.DECRYPT_BYTES( + (SELECT keyset FROM `moz-fx-dataops-secrets.airflow_query_keys.subplat_namespaces_prod`), + ciphertext, + CAST(key_id AS BYTES) + ) AS STRING + ) AS namespace + FROM + `moz-fx-data-shared-prod.subscription_platform_derived.encrypted_keys_v1` + WHERE + key_id = 'subplat_namespaces_prod' +) +SELECT + delete_events.submission_timestamp, + uuid_v5(delete_events.user_id_unhashed, subplat_namespaces.namespace) AS nimbus_user_id, +FROM + `moz-fx-data-shared-prod.firefox_accounts.delete_events` AS delete_events +CROSS JOIN + subplat_namespaces +WHERE + DATE(delete_events.submission_timestamp) = @submission_date + AND delete_events.user_id_unhashed IS NOT NULL diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/schema.yaml new file mode 100644 index 00000000000..a00fd4eb636 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/schema.yaml @@ -0,0 +1,7 @@ +fields: +- name: submission_timestamp + type: TIMESTAMP + mode: NULLABLE +- name: nimbus_user_id + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_derived/encrypted_keys_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/subscription_platform_derived/encrypted_keys_v1/metadata.yaml new file mode 100644 index 00000000000..8de39c757dc --- /dev/null +++ b/sql/moz-fx-data-shared-prod/subscription_platform_derived/encrypted_keys_v1/metadata.yaml @@ -0,0 +1,2 @@ +friendly_name: Encrypted Keys V1 +description: Encrypted keys From d9f2de739a1bf463e46fbcd6dc81a4eaf887a7d5 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Tue, 2 Dec 2025 13:40:07 -0800 Subject: [PATCH 5/5] skip dryrun for restricted query --- bqetl_project.yaml | 1 + .../delete_events_v1/query.sql | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bqetl_project.yaml b/bqetl_project.yaml index 37e693d38bf..f067728b1da 100644 --- a/bqetl_project.yaml +++ b/bqetl_project.yaml @@ -54,6 +54,7 @@ dry_run: - sql/moz-fx-data-shared-prod/monitoring_derived/table_storage_v1/query.sql - sql/moz-fx-data-shared-prod/activity_stream/impression_stats_by_experiment/view.sql - sql/moz-fx-data-shared-prod/activity_stream/impression_stats_flat/view.sql + - sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql # uses time travel, will error on dates prior to the time travel window - sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql - sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_recovery_phones_counts_v1/query.sql diff --git a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql index a5670489163..21ea9f40d7a 100644 --- a/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/subscription_platform_backend_cirrus_derived/delete_events_v1/query.sql @@ -42,12 +42,12 @@ WITH subplat_namespaces AS ( key_id = 'subplat_namespaces_prod' ) SELECT - delete_events.submission_timestamp, - uuid_v5(delete_events.user_id_unhashed, subplat_namespaces.namespace) AS nimbus_user_id, + fxa_delete_events.submission_timestamp, + uuid_v5(fxa_delete_events.user_id_unhashed, subplat_namespaces.namespace) AS nimbus_user_id, FROM - `moz-fx-data-shared-prod.firefox_accounts.delete_events` AS delete_events + `moz-fx-data-shared-prod.firefox_accounts.fxa_delete_events` AS fxa_delete_events CROSS JOIN subplat_namespaces WHERE - DATE(delete_events.submission_timestamp) = @submission_date - AND delete_events.user_id_unhashed IS NOT NULL + DATE(fxa_delete_events.submission_timestamp) = @submission_date + AND fxa_delete_events.user_id_unhashed IS NOT NULL