From aa4c9525aab44630fe132acb12bbb801f0d962d4 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Tue, 19 Aug 2025 14:29:51 +0200 Subject: [PATCH 1/9] feat: update firefox_app_store_territory_source_type_report view to include new connector data --- .../view.sql | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql index 9bfc014b401..1b6df03ea4a 100644 --- a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql +++ b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql @@ -1,11 +1,42 @@ CREATE OR REPLACE VIEW `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` AS +-- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports +-- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT. SELECT - -- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports - -- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT." - -- Date conversion in the query is required to unify the dates to UTC timezone which is what we use. - * REPLACE (TIMESTAMP(DATETIME(`date`, "America/Los_Angeles")) AS `date`), - `date` AS date_pst, + -- However, this timestamp appear to always show midnight meaning if we do timezone conversion + -- we will end up moving all results 1 day back if we attempt conversion to UTC. + DATE(`date`) AS `date`, + app_id, + CAST(NULL AS STRING) AS app_name, + source_type, + territory, + impressions, + impressions_unique_device, + page_views, + page_views_unique_device, + COALESCE(first_time_downloads, 0) AS first_time_downloads, + COALESCE(redownloads, 0) AS redownloads, + COALESCE(total_downloads, 0) AS total_downloads, FROM `moz-fx-data-shared-prod.app_store_syndicate.app_store_territory_source_type_report` +WHERE + `date` < "2024-01-01" +UNION ALL +SELECT + date_day AS `date`, + app_id, + app_name, + source_type, + territory_long AS territory, + impressions, + impressions_unique_device, + page_views, + page_views_unique_device, + first_time_downloads, + redownloads, + total_downloads, +FROM + `moz-fx-data-bq-fivetran.firefox_app_store_v2_apple_store.apple_store__territory_report` +WHERE + date_day >= "2024-01-01" From 11cb97823fcdd5f83910d89ef304f79ef8c7da43 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Tue, 19 Aug 2025 17:09:35 +0200 Subject: [PATCH 2/9] feat: enable monitoring and remove clustering settings --- .../firefox_ios_derived/app_store_funnel_v1/metadata.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml index 658d1966081..3e80aa9bcb1 100644 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml @@ -18,6 +18,5 @@ bigquery: type: day field: submission_date require_partition_filter: false - clustering: - fields: - - country +monitoring: + enabled: true From bb2e02a4ebb68629b790384cb03bcfd15537be80 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Tue, 19 Aug 2025 17:10:11 +0200 Subject: [PATCH 3/9] feat: remove bqetl checks and add bugconfig --- .../app_store_funnel_v1/bigconfig.yml | 39 +++++++++++ .../app_store_funnel_v1/checks.sql | 66 ------------------- 2 files changed, 39 insertions(+), 66 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/bigconfig.yml delete mode 100644 sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/bigconfig.yml new file mode 100644 index 00000000000..2c2608c2aa1 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/bigconfig.yml @@ -0,0 +1,39 @@ +type: BIGCONFIG_FILE +tag_deployments: +- collection: + name: Operational Checks + notification_channels: + - slack: '#de-bigeye-triage' + + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.session_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.country + metrics: + - saved_metric_id: is_not_null + rct_overrides: + - date + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.country + metrics: + - saved_metric_id: composite_key_uniqueness_2_column + rct_overrides: + - date + parameters: + - key: col_1 + column_name: submission_date + - key: col_1_string + string_value: submission_date + - key: col_2 + column_name: country + - key: col_2_string + string_value: country + - key: table + string_value: moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1 + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.* + metrics: + - saved_metric_id: volume + - saved_metric_id: freshness diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql deleted file mode 100644 index 28092cc9ef2..00000000000 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql +++ /dev/null @@ -1,66 +0,0 @@ -#warn -{{ is_unique(["submission_date", "country"], "country IS NOT NULL") }} --- min_row_count helps us detect if we're seeing delays in the data arriving --- could also be an indicator of an upstream issue. - -#fail -{{ min_row_count(1, "submission_date = @submission_date") }} - -#warn -WITH fx_ios_count AS ( - SELECT - COUNT(*) - FROM - `moz-fx-data-shared-prod.firefox_ios.firefox_ios_clients` - WHERE - first_seen_date = DATE_SUB(@submission_date, INTERVAL 7 DAY) - AND channel = "release" -), -new_profiles_count AS ( - SELECT - SUM(new_profiles) - FROM - `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` - WHERE - submission_date = @submission_date -) -SELECT - IF( - (SELECT * FROM fx_ios_count) - (SELECT * FROM new_profiles_count) <> 0, - ERROR("There's a 'new_profiles' mismatch between firefox_ios_clients and this funnel table"), - NULL - ); - -#warn -WITH base AS ( - SELECT - SUM(total_downloads) AS total_downloads, - SUM(first_time_downloads) AS first_time_downloads, - SUM(redownloads) AS redownloads, - FROM - `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` - WHERE - submission_date = @submission_date -) -SELECT - IF( - total_downloads <> (first_time_downloads + redownloads), - ERROR("The sum of first time downloads and redownloads does not match that of total_downloads"), - NULL - ) -FROM - base; - -#warn -SELECT - IF( - DATE_DIFF(submission_date, first_seen_date, DAY) <> 7, - ERROR("Day difference between submission_date and `date` is not equal to 7 as expected"), - NULL - ) -FROM - `moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1` -WHERE - submission_date = @submission_date; - --- TODO: for this query it'd be useful to compare sum variance between each day to improve our confidence the data was complete at the execution time. From d588a5e08c25e0b47d94a9c96b860bc6b5f9f62f Mon Sep 17 00:00:00 2001 From: kik-kik Date: Tue, 19 Aug 2025 17:51:15 +0200 Subject: [PATCH 4/9] feat: update app_store_funnel_v1 to use new app store data --- .../app_store_funnel_v1/query.sql | 124 +++++++++++++----- 1 file changed, 89 insertions(+), 35 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql index 5abc6522c25..818544c3428 100755 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql @@ -1,75 +1,129 @@ -- TODO: should we run this job with 7 day delay to make sure all data landed (a wider window to be on the safe side). -WITH views_data AS ( +WITH historical_store_data AS ( + WITH views_data AS ( + SELECT + DATE(`date`) AS `date`, + territory AS country_name, + SUM(impressions_unique_device) AS views, + FROM + `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` + WHERE + DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) + AND source_type <> 'Institutional Purchase' + AND app_id = 989804926 -- Filter to only include the Firefox app + GROUP BY + `date`, + country_name + ), + downloads_data AS ( + SELECT + DATE(`date`) AS `date`, + territory AS country_name, + SUM(total_downloads) AS total_downloads, + SUM(first_time_downloads) AS first_time_downloads, + SUM(redownloads) AS redownloads, + FROM + `moz-fx-data-shared-prod.app_store.firefox_downloads_territory_source_type_report` + WHERE + DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) + AND source_type <> 'Institutional Purchase' + AND app_id = 989804926 -- Filter to only include the Firefox app + GROUP BY + ALL + ) SELECT DATE(`date`) AS `date`, - territory AS country_name, - SUM(impressions_unique_device) AS views, + country_name, + COALESCE(views, 0) AS views, + COALESCE(total_downloads, 0) AS total_downloads, + COALESCE(first_time_downloads, 0) AS first_time_downloads, + COALESCE(redownloads, 0) AS redownloads, FROM - `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` - WHERE - DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) - GROUP BY - `date`, - country_name + views_data + FULL OUTER JOIN + downloads_data + USING (`date`, country_name) ), -downloads_data AS ( +app_store_data AS ( SELECT - DATE(`date`) AS `date`, - territory AS country_name, + date_day AS `date`, + territory_long AS country_name, + SUM(impressions_unique_device) AS views, SUM(total_downloads) AS total_downloads, SUM(first_time_downloads) AS first_time_downloads, SUM(redownloads) AS redownloads, FROM - `moz-fx-data-shared-prod.app_store.firefox_downloads_territory_source_type_report` + `moz-fx-data-bq-fivetran.firefox_app_store_v2_apple_store.apple_store__territory_report` WHERE - DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) + DATE(date_day) = DATE_SUB(@submission_date, INTERVAL 7 DAY) AND source_type <> 'Institutional Purchase' + AND app_id = 989804926 -- Filter to only include the Firefox app GROUP BY + ALL +), +combine_app_store_data AS ( + SELECT `date`, - country_name + country_name, + views, + total_downloads, + first_time_downloads, + redownloads, + FROM + historical_store_data + WHERE + `date` < "2024-01-01" + UNION ALL + SELECT + `date`, + country_name, + views, + total_downloads, + first_time_downloads, + redownloads, + FROM + app_store_data + WHERE + `date` >= "2024-01-01" ), -store_stats AS ( +normalize_country AS ( SELECT - DATE(`date`) AS `date`, + `date`, country_names.code AS country, views, total_downloads, first_time_downloads, redownloads, FROM - views_data - FULL OUTER JOIN - downloads_data - USING (`date`, country_name) + combine_app_store_data LEFT JOIN `moz-fx-data-shared-prod.static.country_names_v1` AS country_names - ON country_names.name = views_data.country_name + ON combine_app_store_data.country_name = country_names.name ), _new_profiles AS ( SELECT first_seen_date AS `date`, - first_reported_country AS country, - COUNT(*) AS new_profiles, + country, + COALESCE(SUM(new_profiles), 0) AS new_profiles, FROM - `moz-fx-data-shared-prod.firefox_ios.firefox_ios_clients` + `moz-fx-data-shared-prod.firefox_ios.new_profiles` WHERE first_seen_date = DATE_SUB(@submission_date, INTERVAL 7 DAY) - AND channel = "release" + AND normalized_channel = "release" GROUP BY - `date`, - country + ALL ) SELECT @submission_date AS submission_date, `date` AS first_seen_date, country, - COALESCE(views, 0) AS impressions, - COALESCE(total_downloads, 0) AS total_downloads, - COALESCE(first_time_downloads, 0) AS first_time_downloads, - COALESCE(redownloads, 0) AS redownloads, - COALESCE(new_profiles, 0) AS new_profiles, + views, + total_downloads, + first_time_downloads, + redownloads, + new_profiles, FROM - store_stats -FULL OUTER JOIN + normalize_country +LEFT JOIN _new_profiles USING (`date`, country) From a55bd5b30d0ac5c0cc6032157f3fffe0dfad083c Mon Sep 17 00:00:00 2001 From: kik-kik Date: Tue, 19 Aug 2025 18:29:51 +0200 Subject: [PATCH 5/9] fix: invalid view error and comments --- .../view.sql | 9 ++++++--- .../firefox_ios_derived/app_store_funnel_v1/query.sql | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql index 1b6df03ea4a..ee7ff1118d0 100644 --- a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql +++ b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql @@ -15,14 +15,17 @@ SELECT impressions_unique_device, page_views, page_views_unique_device, - COALESCE(first_time_downloads, 0) AS first_time_downloads, - COALESCE(redownloads, 0) AS redownloads, - COALESCE(total_downloads, 0) AS total_downloads, + -- These fields did not exist in the previous version. + CAST(NULL AS INTEGER) AS first_time_downloads, + CAST(NULL AS INTEGER) AS redownloads, + CAST(NULL AS INTEGER) AS total_downloads, FROM `moz-fx-data-shared-prod.app_store_syndicate.app_store_territory_source_type_report` WHERE `date` < "2024-01-01" UNION ALL +-- New connector's data. The schema is a bit different and the new one includes additional metrics: +-- first_time_downloads, redownloads, and total_downloads. SELECT date_day AS `date`, app_id, diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql index 818544c3428..9aa8111dc39 100755 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql @@ -1,4 +1,3 @@ --- TODO: should we run this job with 7 day delay to make sure all data landed (a wider window to be on the safe side). WITH historical_store_data AS ( WITH views_data AS ( SELECT From 39b72ea769f1f0ca33f5f135f0d2f391414be63a Mon Sep 17 00:00:00 2001 From: kik-kik Date: Tue, 19 Aug 2025 19:28:32 +0200 Subject: [PATCH 6/9] fix: use correct field name impressions instead of views --- .../app_store_funnel_v1/query.sql | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql index 9aa8111dc39..030a6ffe590 100755 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql @@ -1,9 +1,9 @@ WITH historical_store_data AS ( - WITH views_data AS ( + WITH impression_data AS ( SELECT DATE(`date`) AS `date`, territory AS country_name, - SUM(impressions_unique_device) AS views, + SUM(impressions_unique_device) AS impressions, FROM `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` WHERE @@ -33,12 +33,12 @@ WITH historical_store_data AS ( SELECT DATE(`date`) AS `date`, country_name, - COALESCE(views, 0) AS views, + COALESCE(impressions, 0) AS impressions, COALESCE(total_downloads, 0) AS total_downloads, COALESCE(first_time_downloads, 0) AS first_time_downloads, COALESCE(redownloads, 0) AS redownloads, FROM - views_data + impression_data FULL OUTER JOIN downloads_data USING (`date`, country_name) @@ -47,7 +47,7 @@ app_store_data AS ( SELECT date_day AS `date`, territory_long AS country_name, - SUM(impressions_unique_device) AS views, + SUM(impressions_unique_device) AS impressions, SUM(total_downloads) AS total_downloads, SUM(first_time_downloads) AS first_time_downloads, SUM(redownloads) AS redownloads, @@ -64,7 +64,7 @@ combine_app_store_data AS ( SELECT `date`, country_name, - views, + impressions, total_downloads, first_time_downloads, redownloads, @@ -76,7 +76,7 @@ combine_app_store_data AS ( SELECT `date`, country_name, - views, + impressions, total_downloads, first_time_downloads, redownloads, @@ -89,7 +89,7 @@ normalize_country AS ( SELECT `date`, country_names.code AS country, - views, + impressions, total_downloads, first_time_downloads, redownloads, @@ -103,7 +103,7 @@ _new_profiles AS ( SELECT first_seen_date AS `date`, country, - COALESCE(SUM(new_profiles), 0) AS new_profiles, + SUM(new_profiles) AS new_profiles, FROM `moz-fx-data-shared-prod.firefox_ios.new_profiles` WHERE @@ -116,11 +116,11 @@ SELECT @submission_date AS submission_date, `date` AS first_seen_date, country, - views, + impressions, total_downloads, first_time_downloads, redownloads, - new_profiles, + COALESCE(new_profiles, 0) AS new_profiles, FROM normalize_country LEFT JOIN From d97186b61d4558f788567c15055d1ed918db8061 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 20 Aug 2025 11:44:10 +0200 Subject: [PATCH 7/9] feat: revert firefox_app_store_territory_source_type_report --- .../view.sql | 44 +++---------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql index ee7ff1118d0..9bfc014b401 100644 --- a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql +++ b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql @@ -1,45 +1,11 @@ CREATE OR REPLACE VIEW `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` AS --- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports --- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT. SELECT - -- However, this timestamp appear to always show midnight meaning if we do timezone conversion - -- we will end up moving all results 1 day back if we attempt conversion to UTC. - DATE(`date`) AS `date`, - app_id, - CAST(NULL AS STRING) AS app_name, - source_type, - territory, - impressions, - impressions_unique_device, - page_views, - page_views_unique_device, - -- These fields did not exist in the previous version. - CAST(NULL AS INTEGER) AS first_time_downloads, - CAST(NULL AS INTEGER) AS redownloads, - CAST(NULL AS INTEGER) AS total_downloads, + -- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports + -- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT." + -- Date conversion in the query is required to unify the dates to UTC timezone which is what we use. + * REPLACE (TIMESTAMP(DATETIME(`date`, "America/Los_Angeles")) AS `date`), + `date` AS date_pst, FROM `moz-fx-data-shared-prod.app_store_syndicate.app_store_territory_source_type_report` -WHERE - `date` < "2024-01-01" -UNION ALL --- New connector's data. The schema is a bit different and the new one includes additional metrics: --- first_time_downloads, redownloads, and total_downloads. -SELECT - date_day AS `date`, - app_id, - app_name, - source_type, - territory_long AS territory, - impressions, - impressions_unique_device, - page_views, - page_views_unique_device, - first_time_downloads, - redownloads, - total_downloads, -FROM - `moz-fx-data-bq-fivetran.firefox_app_store_v2_apple_store.apple_store__territory_report` -WHERE - date_day >= "2024-01-01" From fd996377cd1d8ffe8b480a20e9bc83502fe07963 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 20 Aug 2025 13:00:38 +0200 Subject: [PATCH 8/9] feat: add comments to the views and remove timezone converversion --- .../firefox_app_store_territory_source_type_report/view.sql | 6 ++++-- .../firefox_ios_derived/app_store_funnel_v1/query.sql | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql index 9bfc014b401..9fd79197c69 100644 --- a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql +++ b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql @@ -5,7 +5,9 @@ SELECT -- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports -- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT." -- Date conversion in the query is required to unify the dates to UTC timezone which is what we use. - * REPLACE (TIMESTAMP(DATETIME(`date`, "America/Los_Angeles")) AS `date`), - `date` AS date_pst, + -- However, the `date` timestamp field appear to always show midnight meaning if we do timezone conversion + -- we will end up moving all results 1 day back if we attempt conversion to UTC. + -- This is why we are not doing timezone converstions here. + *, FROM `moz-fx-data-shared-prod.app_store_syndicate.app_store_territory_source_type_report` diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql index 030a6ffe590..832ffbdaa0a 100755 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql @@ -1,3 +1,8 @@ +-- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports +-- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT. +-- However, the `date` timestamp field appear to always show midnight meaning if we do timezone conversion +-- we will end up moving all results 1 day back if we attempt conversion to UTC. +-- This is why we are not doing timezone converstions here. WITH historical_store_data AS ( WITH impression_data AS ( SELECT From 45bb1cfba446f1d1624e3cd1a2ca9f0a037fab1c Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 10 Sep 2025 14:17:42 +0200 Subject: [PATCH 9/9] feat: limit which source types count toward impressions in the old version of the data to match the new report data --- .../firefox_ios_derived/app_store_funnel_v1/query.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql index 832ffbdaa0a..3f99f328e56 100755 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql @@ -8,7 +8,15 @@ WITH historical_store_data AS ( SELECT DATE(`date`) AS `date`, territory AS country_name, - SUM(impressions_unique_device) AS impressions, + -- Apple used to count pageviews from these sources as impressions as well but no longer do in the new data they're sending back + -- for compatibility with the new report we only want to count impressions from specific source types? + SUM( + CASE + WHEN source_type IN ('App Referrer', 'Unavailable', 'Web Referrer') + THEN 0 + ELSE impressions_unique_device + END + ) AS impressions, FROM `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` WHERE