Added mode last retain nulls

Author: wwyc

sql_generators/baseline_clients_city_seen_v1/templates/metadata.yaml

@@ -32,7 +32,7 @@ scheduling:
 bigquery:
   time_partitioning:
     type: day
-    field: last_seen_geo_date
+    field: first_seen_city_date
     require_partition_filter: false
     expiration_days: null
   clustering:

sql_generators/baseline_clients_city_seen_v1/templates/query.sql

@@ -14,8 +14,9 @@ WITH
     mozfun.glean.parse_datetime(ping_info.end_time) AS parsed_end_time,
     `moz-fx-data-shared-prod.udf.glean_timespan_seconds`( metrics.timespan.glean_baseline_duration ) AS duration,
     metadata.geo.city,
-    metadata.geo.subdivision1 AS geo_subdivision1,
-    metadata.geo.subdivision2 AS geo_subdivision2,
+    metadata.geo.subdivision1 AS subdivision1,
+    metadata.geo.subdivision2 AS subdivision2,
+    metadata.geo.country AS country,
   FROM
     `{{ project_id }}.{{ app_id }}_stable.baseline_v1`
   WHERE
@@ -56,9 +57,15 @@ WITH
     client_id,
     sample_id,
     ROW_NUMBER() OVER w1_unframed AS _n,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(city) OVER w1) AS city,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(geo_subdivision1) OVER w1) AS geo_subdivision1,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(geo_subdivision2) OVER w1) AS geo_subdivision2,
+    `moz-fx-data-shared-prod.udf.mode_last_retain_nulls`(
+      ARRAY_AGG(STRUCT(
+        city AS city,
+        subdivision1 AS subdivision1,
+        subdivision2 AS subdivision2,
+        country AS country
+        )
+      ) OVER w1
+    ) AS geo,
   FROM
     with_date_offsets_{{ app_id }}
   LEFT JOIN
@@ -91,15 +98,16 @@ WITH
   FROM
     windowed_{{ app_id }} AS cd
   WHERE
-    _n = 1 ),
+    _n = 1 AND geo.city IS NOT NULL),
 clients_city_first_seen_{{ app_id }} AS (
   SELECT
     client_id,
     sample_id,
-    submission_date AS first_seen_geo_date,
-    city AS first_seen_geo_city,
-    geo_subdivision1 AS first_seen_geo_subdivision1,
-    geo_subdivision2 AS first_seen_geo_subdivision2,
+    submission_date AS first_seen_city_date,
+    geo.city AS first_seen_city,
+    geo.subdivision1 AS first_seen_subdivision1,
+    geo.subdivision2 AS first_seen_subdivision2,
+    geo.country AS first_seen_country,
   FROM
     clients_daily_{{ app_id }}
   QUALIFY
@@ -108,10 +116,11 @@ clients_city_first_seen_{{ app_id }} AS (
   SELECT
     client_id,
     sample_id,
-    submission_date AS last_seen_geo_date,
-    city AS last_seen_geo_city,
-    geo_subdivision1 AS last_seen_geo_subdivision1,
-    geo_subdivision2 AS last_seen_geo_subdivision2,
+    submission_date AS last_seen_city_date,
+    geo.city AS last_seen_city,
+    geo.subdivision1 AS last_seen_subdivision1,
+    geo.subdivision2 AS last_seen_subdivision2,
+    geo.country AS last_seen_country,
   FROM
     clients_daily_{{ app_id }}
   QUALIFY
@@ -122,20 +131,21 @@ SELECT
   "{{ app_id }}" AS app_id,
   COALESCE(cfs.client_id, cls.client_id) AS client_id,
   COALESCE(cfs.sample_id, cls.sample_id) AS sample_id,
-  first_seen_geo_date,
-  first_seen_geo_city,
-  first_seen_geo_subdivision1,
-  first_seen_geo_subdivision2,
-  last_seen_geo_date,
-  last_seen_geo_city,
-  last_seen_geo_subdivision1,
-  last_seen_geo_subdivision2,
+  first_seen_city_date,
+  first_seen_city,
+  first_seen_subdivision1,
+  first_seen_subdivision2,
+  first_seen_country,
+  last_seen_city_date,
+  last_seen_city,
+  last_seen_subdivision1,
+  last_seen_subdivision2,
+  last_seen_country,
 FROM
   clients_city_first_seen_{{ app_id }} cfs
 FULL OUTER JOIN
   clients_city_last_seen_{{ app_id }} cls
-ON
-  cfs.client_id = cls.client_id
+  ON cfs.client_id = cls.client_id
   AND cfs.sample_id = cls.sample_id
 {{ "UNION ALL" if not loop.last }}
 {% endfor -%}
@@ -157,14 +167,16 @@ _current_windowed_{{ app_id }} AS (
     client_info.client_id AS client_id,
     sample_id,
     ROW_NUMBER() OVER w1_unframed AS _n,
-    @submission_date AS first_seen_geo_date,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(metadata.geo.city) OVER w1) AS first_seen_geo_city,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(metadata.geo.subdivision1) OVER w1) AS first_seen_geo_subdivision1,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(metadata.geo.subdivision2) OVER w1) AS first_seen_geo_subdivision2,
-    @submission_date AS last_seen_geo_date,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(metadata.geo.city) OVER w1) AS last_seen_geo_city,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(metadata.geo.subdivision1) OVER w1) AS last_seen_geo_subdivision1,
-    `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(metadata.geo.subdivision2) OVER w1) AS last_seen_geo_subdivision2,
+    submission_date,
+    `moz-fx-data-shared-prod.udf.mode_last_retain_nulls`(
+      ARRAY_AGG(STRUCT(
+        city AS city,
+        subdivision1 AS subdivision1,
+        subdivision2 AS subdivision2,
+        country AS country
+        )
+      ) OVER w1
+    ) AS geo,
   FROM
     `moz-fx-data-shared-prod.{{ app_id }}_live.baseline_v1`
   WHERE
@@ -189,40 +201,73 @@ _current_windowed_{{ app_id }} AS (
   _current_{{ app_id }} AS (
   SELECT
     cw.* EXCEPT (_n),
+    submission_date AS first_seen_city_date,
+    geo.city AS first_seen_city,
+    geo.subdivision1 AS first_seen_subdivision1,
+    geo.subdivision2 AS first_seen_subdivision2,
+    geo.country AS first_seen_country,
+    submission_date AS last_seen_city_date,
+    geo.city AS last_seen_city,
+    geo.subdivision1 AS last_seen_subdivision1,
+    geo.subdivision2 AS last_seen_subdivision2,
+    geo.country AS last_seen_country
   FROM
     _current_windowed_{{ app_id }} AS cw
   WHERE
-    _n = 1){{ "," if not loop.last }}
+    _n = 1
+    AND geo.city IS NOT NULL){{ "," if not loop.last }}
   {% endfor -%}
 {% for app_id in app_id_list -%}
 SELECT
   app_id,
   client_id,
   sample_id,
 IF
-  (_p.client_id IS NULL, _c.first_seen_geo_date, _p.first_seen_geo_date) AS first_seen_geo_date,
+  (_p.client_id IS NULL,
+  _c.first_seen_city_date,
+  _p.first_seen_city_date
+  ) AS first_seen_city_date,
 IF
-  (_p.client_id IS NULL, _c.first_seen_geo_city, _p.first_seen_geo_city) AS first_seen_geo_city,
+  (_p.client_id IS NULL,
+  _c.first_seen_city,
+  _p.first_seen_city
+  ) AS first_seen_city,
 IF
-  (_p.client_id IS NULL, _c.first_seen_geo_subdivision1, _p.first_seen_geo_subdivision1) AS first_seen_geo_subdivision1,
+  (_p.client_id IS NULL,
+  _c.first_seen_subdivision1,
+  _p.first_seen_subdivision1
+  ) AS first_seen_subdivision1,
 IF
-  (_p.client_id IS NULL, _c.first_seen_geo_subdivision2, _p.first_seen_geo_subdivision2) AS first_seen_geo_subdivision2,
+  (_p.client_id IS NULL,
+  _c.first_seen_subdivision2,
+  _p.first_seen_subdivision2
+  ) AS first_seen_subdivision2,
 IF
-  (_p.last_seen_geo_date < _c.last_seen_geo_date, _c.last_seen_geo_date, _p.last_seen_geo_date) AS last_seen_geo_date,
+  (_p.last_seen_city_date < _c.last_seen_city_date,
+  _c.last_seen_city_date,
+  _p.last_seen_city_date
+  ) AS last_seen_city_date,
 IF
-  (_p.last_seen_geo_date < _c.last_seen_geo_date, _c.last_seen_geo_city, _p.last_seen_geo_city) AS last_seen_geo_city,
+  (_p.last_seen_city_date < _c.last_seen_city_date,
+  _c.last_seen_city,
+  _p.last_seen_city
+  ) AS last_seen_city,
 IF
-  (_p.last_seen_geo_date < _c.last_seen_geo_date, _c.last_seen_geo_subdivision1, _p.last_seen_geo_subdivision1) AS last_seen_geo_subdivision1,
+  (_p.last_seen_city_date < _c.last_seen_city_date,
+  _c.last_seen_subdivision1,
+  _p.last_seen_subdivision1
+  ) AS last_seen_subdivision1,
 IF
-  (_p.last_seen_geo_date < _c.last_seen_geo_date, _c.last_seen_geo_subdivision2, _p.last_seen_geo_subdivision2) AS last_seen_geo_subdivision2,
+  (_p.last_seen_city_date < _c.last_seen_city_date,
+  _c.last_seen_subdivision2,
+  _p.last_seen_subdivision2
+  ) AS last_seen_subdivision2,
 FROM
   _current_{{ app_id }} AS _c
 FULL JOIN
   _previous_{{ app_id }} AS _p
 USING
-  (client_id,
-    sample_id,
-    app_id)
+  (client_id, sample_id, app_id)
 {{ "UNION ALL" if not loop.last }}
 {% endfor -%}
 {% raw %}

sql_generators/baseline_clients_city_seen_v1/templates/schema.yaml

@@ -11,35 +11,43 @@ fields:
   type: INTEGER
   mode: NULLABLE
   description: A number, 0-99, that samples by client_id and allows filtering data for analysis.
-- name: first_seen_geo_date
+- name: first_seen_city_date
   type: DATE
   mode: NULLABLE
-  description: Date when the first seen geo fields were captured.
-- name: first_seen_geo_city
+  description: Date when the first seen city was captured.
+- name: first_seen_city
   type: STRING
   mode: NULLABLE
-  description: City captured on first_seen_geo_date.
-- name: first_seen_geo_subdivision1
+  description: City captured on first_seen_city_date.
+- name: first_seen_subdivision1
   type: STRING
   mode: NULLABLE
-  description: Major country subdivision, typically a state, province, or county captured on first_seen_geo_date.
-- name: first_seen_geo_subdivision2
+  description: Major country subdivision, typically a state, province, or county captured on first_seen_city_date.
+- name: first_seen_subdivision2
   type: STRING
   mode: NULLABLE
-  description: Second major country subdivision; not applicable for most countries captured on first_seen_geo_date.
-- name: last_seen_geo_date
+  description: Second major country subdivision; not applicable for most countries captured on first_seen_city_date.
+- name: first_seen_country
+  type: STRING
+  mode: NULLABLE
+  description: An ISO 3166-1 alpha-2 country code captured on first_seen_city_date.
+- name: last_seen_city_date
   type: DATE
   mode: NULLABLE
-  description: Date when the last seen geo fields were captured.
-- name: last_seen_geo_city
+  description: Date when the last seen city was captured.
+- name: last_seen_city
+  type: STRING
+  mode: NULLABLE
+  description: City captured on last_seen_city_date.
+- name: last_seen_subdivision1
   type: STRING
   mode: NULLABLE
-  description: City captured on last_seen_geo_city.
-- name: last_seen_geo_subdivision1
+  description: Major country subdivision, typically a state, province, or county captured on last_seen_city_date.
+- name: last_seen_subdivision2
   type: STRING
   mode: NULLABLE
-  description: Major country subdivision, typically a state, province, or county captured on last_seen_geo_date.
-- name: last_seen_geo_subdivision2
+  description: Second major country subdivision; not applicable for most countries captured on last_seen_city_date.
+- name: last_seen_country
   type: STRING
   mode: NULLABLE
-  description: Second major country subdivision; not applicable for most countries captured on last_seen_geo_date.
+  description: An ISO 3166-1 alpha-2 country code captured on last_seen_city_date.