Replace NTD agency information source and deprecate manual process

[#3735]
cal-itp · Feb 28, 2025 · e28010b · e28010b
1 parent 10bb076
commit e28010b
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 2,417 deletions.
diff --git a/script/poetry.lock b/script/poetry.lock
diff --git a/script/pyproject.toml b/script/pyproject.toml
diff --git a/script/scrape_ntd.py b/script/scrape_ntd.py
diff --git a/warehouse/models/mart/ntd/dim_annual_agency_information.sql b/warehouse/models/mart/ntd/dim_annual_agency_information.sql
@@ -1,15 +1,23 @@
-WITH stg_ntd__annual_database_agency_information AS (
+WITH stg_ntd__2022_agency_information AS (
     SELECT
         *,
         -- TODO: this does not handle deletes
-        LEAD(ts) OVER (PARTITION BY year, ntd_id, state_parent_ntd_id ORDER BY ts ASC) AS next_ts,
-    FROM {{ ref('stg_ntd__annual_database_agency_information') }}
+        LEAD(execution_ts) OVER (PARTITION BY ntd_id, state_parent_ntd_id ORDER BY execution_ts ASC) AS next_ts,
+    FROM {{ ref('stg_ntd__2022_agency_information') }}
+),
+
+stg_ntd__2023_agency_information AS (
+    SELECT
+        *,
+        -- TODO: this does not handle deletes
+        LEAD(execution_ts) OVER (PARTITION BY ntd_id, state_parent_ntd_id ORDER BY execution_ts ASC) AS next_ts,
+    FROM {{ ref('stg_ntd__2023_agency_information') }}
 ),
 
 dim_annual_agency_information AS (
     SELECT
-       {{ dbt_utils.generate_surrogate_key(['year', 'ntd_id', 'state_parent_ntd_id', 'ts']) }} AS key,
-        year,
+       {{ dbt_utils.generate_surrogate_key([2023, 'ntd_id', 'state_parent_ntd_id', 'execution_ts']) }} AS key,
+        2023 AS year,
         ntd_id,
         state_parent_ntd_id,
         agency_name,
@@ -38,8 +46,61 @@ dim_annual_agency_information AS (
         ueid,
         service_area_sq_miles,
         service_area_pop,
-        primary_uza_code,
-        primary_uza_name,
+        primary_uza_uace_code AS primary_uza_code,
+        uza_name AS primary_uza_name,
+        tribal_area_name,
+        population,
+        density,
+        sq_miles,
+        voms_do,
+        voms_pt,
+        total_voms,
+        volunteer_drivers,
+        personal_vehicles,
+        tam_tier,
+        number_of_state_counties,
+        number_of_counties_with_service,
+        state_admin_funds_expended,
+        execution_ts AS _valid_from,
+        {{ make_end_of_valid_range('COALESCE(next_ts, CAST("2099-01-01" AS TIMESTAMP))') }} AS _valid_to,
+        next_ts IS NULL AS _is_current,
+    FROM stg_ntd__2023_agency_information
+
+    UNION ALL
+
+    SELECT
+       {{ dbt_utils.generate_surrogate_key([2022, 'ntd_id', 'state_parent_ntd_id', 'execution_ts']) }} AS key,
+        2022 AS year,
+        ntd_id,
+        state_parent_ntd_id,
+        agency_name,
+        reporter_acronym,
+        doing_business_as,
+        NULL AS division_department,
+        legacy_ntd_id,
+        reported_by_ntd_id,
+        reported_by_name,
+        reporter_type,
+        reporting_module,
+        organization_type,
+        subrecipient_type,
+        fy_end_date,
+        original_due_date,
+        address_line_1,
+        address_line_2,
+        p_o__box,
+        city,
+        state,
+        zip_code,
+        zip_code_ext,
+        region,
+        url,
+        fta_recipient_id,
+        ueid,
+        service_area_sq_miles,
+        service_area_pop,
+        primary_uza_uace_code AS primary_uza_code,
+        uza_name AS primary_uza_name,
         tribal_area_name,
         population,
         density,
@@ -53,10 +114,10 @@ dim_annual_agency_information AS (
         number_of_state_counties,
         number_of_counties_with_service,
         state_admin_funds_expended,
-        ts AS _valid_from,
+        execution_ts AS _valid_from,
         {{ make_end_of_valid_range('COALESCE(next_ts, CAST("2099-01-01" AS TIMESTAMP))') }} AS _valid_to,
         next_ts IS NULL AS _is_current,
-    FROM stg_ntd__annual_database_agency_information
+    FROM stg_ntd__2022_agency_information
 )
 
 SELECT * FROM dim_annual_agency_information