diff --git a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql index 9bfc014b401..9fd79197c69 100644 --- a/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql +++ b/sql/moz-fx-data-shared-prod/app_store/firefox_app_store_territory_source_type_report/view.sql @@ -5,7 +5,9 @@ SELECT -- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports -- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT." -- Date conversion in the query is required to unify the dates to UTC timezone which is what we use. - * REPLACE (TIMESTAMP(DATETIME(`date`, "America/Los_Angeles")) AS `date`), - `date` AS date_pst, + -- However, the `date` timestamp field appear to always show midnight meaning if we do timezone conversion + -- we will end up moving all results 1 day back if we attempt conversion to UTC. + -- This is why we are not doing timezone converstions here. + *, FROM `moz-fx-data-shared-prod.app_store_syndicate.app_store_territory_source_type_report` diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/bigconfig.yml new file mode 100644 index 00000000000..2c2608c2aa1 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/bigconfig.yml @@ -0,0 +1,39 @@ +type: BIGCONFIG_FILE +tag_deployments: +- collection: + name: Operational Checks + notification_channels: + - slack: '#de-bigeye-triage' + + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.session_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.country + metrics: + - saved_metric_id: is_not_null + rct_overrides: + - date + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.country + metrics: + - saved_metric_id: composite_key_uniqueness_2_column + rct_overrides: + - date + parameters: + - key: col_1 + column_name: submission_date + - key: col_1_string + string_value: submission_date + - key: col_2 + column_name: country + - key: col_2_string + string_value: country + - key: table + string_value: moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1 + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1.* + metrics: + - saved_metric_id: volume + - saved_metric_id: freshness diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql deleted file mode 100644 index 28092cc9ef2..00000000000 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/checks.sql +++ /dev/null @@ -1,66 +0,0 @@ -#warn -{{ is_unique(["submission_date", "country"], "country IS NOT NULL") }} --- min_row_count helps us detect if we're seeing delays in the data arriving --- could also be an indicator of an upstream issue. - -#fail -{{ min_row_count(1, "submission_date = @submission_date") }} - -#warn -WITH fx_ios_count AS ( - SELECT - COUNT(*) - FROM - `moz-fx-data-shared-prod.firefox_ios.firefox_ios_clients` - WHERE - first_seen_date = DATE_SUB(@submission_date, INTERVAL 7 DAY) - AND channel = "release" -), -new_profiles_count AS ( - SELECT - SUM(new_profiles) - FROM - `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` - WHERE - submission_date = @submission_date -) -SELECT - IF( - (SELECT * FROM fx_ios_count) - (SELECT * FROM new_profiles_count) <> 0, - ERROR("There's a 'new_profiles' mismatch between firefox_ios_clients and this funnel table"), - NULL - ); - -#warn -WITH base AS ( - SELECT - SUM(total_downloads) AS total_downloads, - SUM(first_time_downloads) AS first_time_downloads, - SUM(redownloads) AS redownloads, - FROM - `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` - WHERE - submission_date = @submission_date -) -SELECT - IF( - total_downloads <> (first_time_downloads + redownloads), - ERROR("The sum of first time downloads and redownloads does not match that of total_downloads"), - NULL - ) -FROM - base; - -#warn -SELECT - IF( - DATE_DIFF(submission_date, first_seen_date, DAY) <> 7, - ERROR("Day difference between submission_date and `date` is not equal to 7 as expected"), - NULL - ) -FROM - `moz-fx-data-shared-prod.firefox_ios_derived.app_store_funnel_v1` -WHERE - submission_date = @submission_date; - --- TODO: for this query it'd be useful to compare sum variance between each day to improve our confidence the data was complete at the execution time. diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml index 658d1966081..3e80aa9bcb1 100644 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/metadata.yaml @@ -18,6 +18,5 @@ bigquery: type: day field: submission_date require_partition_filter: false - clustering: - fields: - - country +monitoring: + enabled: true diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql index 5abc6522c25..3f99f328e56 100755 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_funnel_v1/query.sql @@ -1,75 +1,141 @@ --- TODO: should we run this job with 7 day delay to make sure all data landed (a wider window to be on the safe side). -WITH views_data AS ( +-- https://developer.apple.com/help/app-store-connect/view-sales-and-trends/download-and-view-reports +-- "Time zone: Reports are based on Pacific Time (PT). A day includes transactions that happened from 12:00 a.m. to 11:59 p.m. PT. +-- However, the `date` timestamp field appear to always show midnight meaning if we do timezone conversion +-- we will end up moving all results 1 day back if we attempt conversion to UTC. +-- This is why we are not doing timezone converstions here. +WITH historical_store_data AS ( + WITH impression_data AS ( + SELECT + DATE(`date`) AS `date`, + territory AS country_name, + -- Apple used to count pageviews from these sources as impressions as well but no longer do in the new data they're sending back + -- for compatibility with the new report we only want to count impressions from specific source types? + SUM( + CASE + WHEN source_type IN ('App Referrer', 'Unavailable', 'Web Referrer') + THEN 0 + ELSE impressions_unique_device + END + ) AS impressions, + FROM + `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` + WHERE + DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) + AND source_type <> 'Institutional Purchase' + AND app_id = 989804926 -- Filter to only include the Firefox app + GROUP BY + `date`, + country_name + ), + downloads_data AS ( + SELECT + DATE(`date`) AS `date`, + territory AS country_name, + SUM(total_downloads) AS total_downloads, + SUM(first_time_downloads) AS first_time_downloads, + SUM(redownloads) AS redownloads, + FROM + `moz-fx-data-shared-prod.app_store.firefox_downloads_territory_source_type_report` + WHERE + DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) + AND source_type <> 'Institutional Purchase' + AND app_id = 989804926 -- Filter to only include the Firefox app + GROUP BY + ALL + ) SELECT DATE(`date`) AS `date`, - territory AS country_name, - SUM(impressions_unique_device) AS views, + country_name, + COALESCE(impressions, 0) AS impressions, + COALESCE(total_downloads, 0) AS total_downloads, + COALESCE(first_time_downloads, 0) AS first_time_downloads, + COALESCE(redownloads, 0) AS redownloads, FROM - `moz-fx-data-shared-prod.app_store.firefox_app_store_territory_source_type_report` - WHERE - DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) - GROUP BY - `date`, - country_name + impression_data + FULL OUTER JOIN + downloads_data + USING (`date`, country_name) ), -downloads_data AS ( +app_store_data AS ( SELECT - DATE(`date`) AS `date`, - territory AS country_name, + date_day AS `date`, + territory_long AS country_name, + SUM(impressions_unique_device) AS impressions, SUM(total_downloads) AS total_downloads, SUM(first_time_downloads) AS first_time_downloads, SUM(redownloads) AS redownloads, FROM - `moz-fx-data-shared-prod.app_store.firefox_downloads_territory_source_type_report` + `moz-fx-data-bq-fivetran.firefox_app_store_v2_apple_store.apple_store__territory_report` WHERE - DATE(`date`) = DATE_SUB(@submission_date, INTERVAL 7 DAY) + DATE(date_day) = DATE_SUB(@submission_date, INTERVAL 7 DAY) AND source_type <> 'Institutional Purchase' + AND app_id = 989804926 -- Filter to only include the Firefox app GROUP BY + ALL +), +combine_app_store_data AS ( + SELECT `date`, - country_name + country_name, + impressions, + total_downloads, + first_time_downloads, + redownloads, + FROM + historical_store_data + WHERE + `date` < "2024-01-01" + UNION ALL + SELECT + `date`, + country_name, + impressions, + total_downloads, + first_time_downloads, + redownloads, + FROM + app_store_data + WHERE + `date` >= "2024-01-01" ), -store_stats AS ( +normalize_country AS ( SELECT - DATE(`date`) AS `date`, + `date`, country_names.code AS country, - views, + impressions, total_downloads, first_time_downloads, redownloads, FROM - views_data - FULL OUTER JOIN - downloads_data - USING (`date`, country_name) + combine_app_store_data LEFT JOIN `moz-fx-data-shared-prod.static.country_names_v1` AS country_names - ON country_names.name = views_data.country_name + ON combine_app_store_data.country_name = country_names.name ), _new_profiles AS ( SELECT first_seen_date AS `date`, - first_reported_country AS country, - COUNT(*) AS new_profiles, + country, + SUM(new_profiles) AS new_profiles, FROM - `moz-fx-data-shared-prod.firefox_ios.firefox_ios_clients` + `moz-fx-data-shared-prod.firefox_ios.new_profiles` WHERE first_seen_date = DATE_SUB(@submission_date, INTERVAL 7 DAY) - AND channel = "release" + AND normalized_channel = "release" GROUP BY - `date`, - country + ALL ) SELECT @submission_date AS submission_date, `date` AS first_seen_date, country, - COALESCE(views, 0) AS impressions, - COALESCE(total_downloads, 0) AS total_downloads, - COALESCE(first_time_downloads, 0) AS first_time_downloads, - COALESCE(redownloads, 0) AS redownloads, + impressions, + total_downloads, + first_time_downloads, + redownloads, COALESCE(new_profiles, 0) AS new_profiles, FROM - store_stats -FULL OUTER JOIN + normalize_country +LEFT JOIN _new_profiles USING (`date`, country)