Create staging GTFS download config view: stg_gtfs_schedule__download_configs

Author: erikamov

warehouse/models/staging/gtfs/_src_gtfs_schedule_external_tables.yml

@@ -14,6 +14,7 @@ sources:
     database: "{{ env_var('GOOGLE_CLOUD_PROJECT', var('GOOGLE_CLOUD_PROJECT')) }}"
     schema: external_gtfs_schedule
     tables:
+      - name: download_configs
       - name: download_outcomes
       - name: unzip_outcomes
       - name: agency

warehouse/models/staging/gtfs/_stg_gtfs.yml

@@ -1,6 +1,28 @@
 version: 2
 
 models:
+  - name: stg_gtfs_schedule__download_configs
+    description: Displays all existing download configs with related schedule validation url
+    columns:
+      - name: name
+      - name: url
+      - name: base64_url
+      - name: feed_type
+      - name: auth_headers
+      - name: auth_query_params
+      - name: first_extracted_date
+        description: First date this configuration was extracted
+      - name: last_extracted_date
+        description: Last date this configuration was extracted
+      - name: schedule_url
+      - name: schedule_base64_url
+      - name: schedule_name
+      - name: schedule_auth_headers
+      - name: schedule_auth_query_params
+      - name: schedule_first_extracted_date
+        description: First date this configuration schedule was extracted
+      - name: schedule_last_extracted_date
+        description: Last date this configuration schedule was extracted
   - name: stg_gtfs_schedule__download_outcomes
     description: Outcomes from download attempts of GTFS schedule data.
     tests:

warehouse/models/staging/gtfs/stg_gtfs_schedule__download_configs.sql

@@ -0,0 +1,34 @@
+WITH stg_gtfs_schedule__download_configs AS (
+    SELECT configs.name,
+           configs.url,
+           {{ to_url_safe_base64('configs.url') }} AS base64_url,
+           configs.feed_type,
+           TO_JSON_STRING(configs.auth_headers) AS auth_headers,
+           TO_JSON_STRING(configs.auth_query_params) AS auth_query_params,
+           MIN(configs.dt) AS first_extracted_date,
+           MAX(configs.dt) AS last_extracted_date,
+           configs.schedule_url_for_validation AS schedule_url,
+           {{ to_url_safe_base64('schedules.url') }} AS schedule_base64_url,
+           schedules.name AS schedule_name,
+           TO_JSON_STRING(schedules.auth_headers) AS schedule_auth_headers,
+           TO_JSON_STRING(schedules.auth_query_params) AS schedule_auth_query_params,
+           MIN(schedules.dt) AS schedule_first_extracted_date,
+           MAX(schedules.dt) AS schedule_last_extracted_date
+      FROM {{ source('external_gtfs_schedule', 'download_configs') }} AS configs
+      LEFT JOIN {{ source('external_gtfs_schedule', 'download_configs') }} AS schedules
+             ON configs.schedule_url_for_validation = schedules.url
+            AND schedules.feed_type = 'schedule'
+     GROUP BY
+           configs.name,
+           configs.url,
+           configs.feed_type,
+           auth_headers,
+           auth_query_params,
+           configs.schedule_url_for_validation,
+           schedules.url,
+           schedules.name,
+           schedule_auth_query_params,
+           schedule_auth_headers
+)
+
+SELECT * FROM stg_gtfs_schedule__download_configs