From a664b7ca7ab2f97fa8cc4718b542513eb6bd529c Mon Sep 17 00:00:00 2001 From: V Date: Fri, 23 Aug 2024 16:01:59 -0700 Subject: [PATCH 1/2] feat(mart_gtfs_quality): Adds a query that produces gtfs rt and vp completeness for use in reports --- .../mart/gtfs_quality/_mart_gtfs_quality.yml | 4 ++- ...updates_vehicle_positions_completeness.sql | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 warehouse/models/mart/gtfs_quality/fct_daily_trip_updates_vehicle_positions_completeness.sql diff --git a/warehouse/models/mart/gtfs_quality/_mart_gtfs_quality.yml b/warehouse/models/mart/gtfs_quality/_mart_gtfs_quality.yml index 082fe8d53f..33cb0acf31 100644 --- a/warehouse/models/mart/gtfs_quality/_mart_gtfs_quality.yml +++ b/warehouse/models/mart/gtfs_quality/_mart_gtfs_quality.yml @@ -1,7 +1,9 @@ version: 2 models: - + - name: fct_dailytrip_updates_vehicle_positions_completeness + description: | + A daily model of how many trips had either a single trip update or a single vehicle position message - name: fct_daily_rt_feed_validation_notices description: | A daily model of validation notices found per feed. Data diff --git a/warehouse/models/mart/gtfs_quality/fct_daily_trip_updates_vehicle_positions_completeness.sql b/warehouse/models/mart/gtfs_quality/fct_daily_trip_updates_vehicle_positions_completeness.sql new file mode 100644 index 0000000000..6c340b23ad --- /dev/null +++ b/warehouse/models/mart/gtfs_quality/fct_daily_trip_updates_vehicle_positions_completeness.sql @@ -0,0 +1,34 @@ +{{ config(materialized='table') }} + +WITH map AS ( + SELECT * + FROM {{ ref('int_gtfs_quality__organization_dataset_map') }} + WHERE public_customer_facing_or_regional_subfeed_fixed_route +), +st as (SELECT * from {{ ref('fct_scheduled_trips') }}), +ot as (SELECT * from {{ ref('fct_observed_trips') }}), +fct_daily_trip_updates_vehicle_positions_completeness AS ( + SELECT + organization_name, + organization_itp_id, + organization_source_record_id, + DATE_TRUNC(st.service_date, DAY) AS service_date, + + -- Percentage of trips with TU messages + (CAST(SUM(IF(ot.tu_num_distinct_message_ids > 0, 1, 0)) AS FLOAT64) / NULLIF(COUNT(*), 0)) * 100 AS percent_of_trips_with_TU_messages, + + -- Percentage of trips with VP messages + (CAST(SUM(IF(ot.vp_num_distinct_message_ids > 0, 1, 0)) AS FLOAT64) / NULLIF(COUNT(*), 0)) * 100 AS percent_of_trips_with_VP_messages, + NULLIF(COUNT(*), 0) as scheduled_trips, + FROM st + LEFT JOIN ot + ON st.trip_instance_key = ot.trip_instance_key + LEFT JOIN map + ON map.schedule_feed_key = st.feed_key + WHERE st.service_date < DATE_TRUNC(CURRENT_DATE('America/Los_Angeles'), DAY) + AND organization_itp_id IS NOT NULL + GROUP BY 1, 2, 3, 4 +) + +SELECT * +FROM fct_daily_trip_updates_vehicle_positions_completeness From a7d9d1932f821cd3d22efe5eeff76ba3827a7b84 Mon Sep 17 00:00:00 2001 From: V Date: Tue, 1 Oct 2024 10:17:49 -0700 Subject: [PATCH 2/2] feat(data-infra-github): Quarterly maintenance tickets and a workflow that makes them --- .../quarterly_infrastructure_maintenance.md | 26 +++++ .github/workflows/quarterly-maintenance.yml | 99 +++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md create mode 100644 .github/workflows/quarterly-maintenance.yml diff --git a/.github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md b/.github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md new file mode 100644 index 0000000000..34b50371e1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md @@ -0,0 +1,26 @@ +## Q{{ quarter }} Quarterly Maintenance: {{ software }} + +_Please Check for updated stable versions of the above software and relevant security patches._ + +### Prior Work + +_Please Link to previously completed relevant tickets._ + +### Dependencies + +- List any systems or components that might be affected by this Metabase maintenance. + +### Acceptance Criteria + +- [ ] Describe the upgrade methodology and any specific steps involved (unless already documented in prior work). +- [ ] Document the upgrade process and any relevant changes. +- [ ] Create issues around any roadblocks encountered. +- [ ] Conduct thorough testing to verify functionality after the upgrade and patching. + +### Rollback Plan + +- Briefly outline the steps to revert to the previous version in case of issues. + +### Notes + +_Please enter any additional information that will facilitate the completion of this maintenance task. For example: Are there any constraints not mentioned above? Are there any alternatives you have considered?_ diff --git a/.github/workflows/quarterly-maintenance.yml b/.github/workflows/quarterly-maintenance.yml new file mode 100644 index 0000000000..0206d0f236 --- /dev/null +++ b/.github/workflows/quarterly-maintenance.yml @@ -0,0 +1,99 @@ +name: Quarterly Maintenance + +on: + schedule: + - cron: '0 0 1 1,4,7,10 *' # Run at midnight on the 1st of January, April, July, and October + +jobs: + create_issues: + runs-on: ubuntu-latest + steps: + - name: Set quarter variable + run: | + const now = new Date(); + const quarter = Math.floor((now.getMonth() / 3) + 1); + echo "quarter=${quarter}" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Create Metabase issue + uses: peter-evans/create-issue-from-file@v4 + with: + token: ${{ secrets.GH_PROJECTS_TOKEN }} + title: Quarterly Maintenance - Metabase Q${{ quarter }} + content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + labels: quarterly, maintenance, metabase + # template-variables: {"software": "Metabase", "quarter": "${{ quarter }}"} + + # - name: Create Jupyterhub issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - Jupyterhub Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, jupyterhub + # template-variables: '{"software": "Jupyterhub", "quarter": "'${{ env.quarter }}'"}' + + # - name: Create Kubernetes issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - Kubernetes Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, kubernetes + # template-variables: '{"software": "Kubernetes", "quarter": "'${{ env.quarter }}'"}' + + # - name: Create Sentry issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - Sentry Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, sentry + # template-variables: '{"software": "Sentry", "quarter": "'${{ env.quarter }}'"}' + + # - name: Create Airflow issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - Airflow Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, airflow + # template-variables: '{"software": "Airflow", "quarter": "'${{ env.quarter }}'"}' + + # - name: Create Composer issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - Composer Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, composer + # template-variables: '{"software": "Composer", "quarter": "'${{ env.quarter }}'"}' + + # - name: Create RT Archiver base image issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - RT Archiver base image Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, rt-archiver + # template-variables: '{"software": "RT Archiver base image", "quarter": "'${{ env.quarter }}'"}' + + # - name: Create dependabot PRs issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - Review and Merge dependabot PRs Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, dependabot + # template-variables: '{"software": "dependabot pull requests", "quarter": "'${{ env.quarter }}'"}' + + # - name: Create AWS key rotation issue + # uses: peter-evans/create-issue-from-file@v4 + # with: + # token: ${{ secrets.GH_PROJECTS_TOKEN }} + # title: Quarterly Maintenance - AWS key rotation Q${{ env.quarter }} + # content-filepath: .github/ISSUE_TEMPLATE/quarterly_infrastructure_maintenance.md + # labels: quarterly, maintenance, aws + # template-variables: '{"software": "AWS key rotation", "quarter": "'${{ env.quarter }}'"}'