From 6d63d80023211631c337ca6af45767209ccb9045 Mon Sep 17 00:00:00 2001 From: sspaeti Date: Thu, 17 Nov 2022 10:37:40 +0100 Subject: [PATCH] sqlfluff fix --- .../1-staging/stg_airbyte__connection.sql | 2 +- .../intermediate/join_cloud_attempts_jobs.sql | 16 +++++++--- .../models/3-mart/mart_sync_success.sql | 27 ++++++++--------- .../models/3-mart/mart_sync_success_day.sql | 29 ++++++++++--------- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/transformation_dbt/models/1-staging/stg_airbyte__connection.sql b/transformation_dbt/models/1-staging/stg_airbyte__connection.sql index a90f2b3..220b18a 100644 --- a/transformation_dbt/models/1-staging/stg_airbyte__connection.sql +++ b/transformation_dbt/models/1-staging/stg_airbyte__connection.sql @@ -20,7 +20,7 @@ renamed as ( resource_requirements, {{ json_extract_text_custom('schedule', 'timeUnit') }} as time_unit, - {{ json_extract_text_custom('schedule', 'units', 'integer') }} as units +{{ json_extract_text_custom('schedule', 'units', 'integer') }} as units from source ) diff --git a/transformation_dbt/models/2-core/intermediate/join_cloud_attempts_jobs.sql b/transformation_dbt/models/2-core/intermediate/join_cloud_attempts_jobs.sql index d3e8b63..ff308d1 100644 --- a/transformation_dbt/models/2-core/intermediate/join_cloud_attempts_jobs.sql +++ b/transformation_dbt/models/2-core/intermediate/join_cloud_attempts_jobs.sql @@ -37,15 +37,21 @@ attempts_data as (select distinct updated_at, ended_at from attempts_raw_data -), attempts_incl_row_num as ( select +), + +attempts_incl_row_num as ( select sync_unique_key, job_id, attempt_id, failure_reasons, dt, - row_number() over (partition by sync_unique_key, job_id, attempt_id, temporal_workflow_id order by dt desc, updated_at desc, ended_at desc, loaded_at desc) as seq + row_number() over ( + partition by sync_unique_key, job_id, attempt_id, temporal_workflow_id order by dt desc, updated_at desc, ended_at desc, loaded_at desc + ) as seq from attempts_raw_data -), attempts_failure_reasons_data as ( +), + +attempts_failure_reasons_data as ( select sync_unique_key, job_id, @@ -54,7 +60,9 @@ attempts_data as (select distinct dt from attempts_incl_row_num where seq = 1 -), jobs_data as ( +), + +jobs_data as ( select distinct --except(loaded_at, operation_sequence, cpu_request, memory_request) job_id, diff --git a/transformation_dbt/models/3-mart/mart_sync_success.sql b/transformation_dbt/models/3-mart/mart_sync_success.sql index 346ef83..746624e 100644 --- a/transformation_dbt/models/3-mart/mart_sync_success.sql +++ b/transformation_dbt/models/3-mart/mart_sync_success.sql @@ -1,17 +1,18 @@ with sync_status as (select * from {{ ref('core_airbyte_sync_status') }} ) -SELECT + +select source2destination_name, - count(*) AS total_attempts, - coalesce(sum(CASE WHEN attempt_succeeded THEN 1 ELSE 0 END), 0) AS attempt_succeeded, - coalesce(sum(CASE WHEN attempt_failed THEN 1 ELSE 0 END), 0) AS attempt_failed, - coalesce(sum(CASE WHEN attempt_succeeded_first_attempt THEN 1 ELSE 0 END), 0) AS attempt_succeeded_first_attempt, - coalesce(sum(CASE WHEN attempt_succeeded THEN 1 ELSE 0 END), 0) * 100.0 / count(*) AS success_rate, - coalesce(sum(CASE WHEN attempt_succeeded_first_attempt THEN 1 ELSE 0 END), 0) * 100.0 / count(*) AS success_rate_first_attempt, - coalesce(sum(CASE WHEN attempt_running THEN 1 ELSE 0 END), 0) AS attempt_running, - sum(attempt_duration) AS sum_attempt_duration, - sum(volume_rows) AS sum_volume_rows, - sum(volume_mb) AS sum_volume_mb + count(*) as total_attempts, + coalesce(sum(case when attempt_succeeded then 1 else 0 end), 0) as attempt_succeeded, + coalesce(sum(case when attempt_failed then 1 else 0 end), 0) as attempt_failed, + coalesce(sum(case when attempt_succeeded_first_attempt then 1 else 0 end), 0) as attempt_succeeded_first_attempt, + coalesce(sum(case when attempt_succeeded then 1 else 0 end), 0) * 100.0 / count(*) as success_rate, + coalesce(sum(case when attempt_succeeded_first_attempt then 1 else 0 end), 0) * 100.0 / count(*) as success_rate_first_attempt, + coalesce(sum(case when attempt_running then 1 else 0 end), 0) as attempt_running, + sum(attempt_duration) as sum_attempt_duration, + sum(volume_rows) as sum_volume_rows, + sum(volume_mb) as sum_volume_mb -FROM sync_status +from sync_status -GROUP BY source2destination_name +group by source2destination_name diff --git a/transformation_dbt/models/3-mart/mart_sync_success_day.sql b/transformation_dbt/models/3-mart/mart_sync_success_day.sql index 293ed40..1e7cc4c 100644 --- a/transformation_dbt/models/3-mart/mart_sync_success_day.sql +++ b/transformation_dbt/models/3-mart/mart_sync_success_day.sql @@ -1,18 +1,19 @@ with sync_status as (select * from {{ ref('core_airbyte_sync_status') }} ) -SELECT + +select source2destination_name, - date(sync_updated) AS sync_updated, - count(*) AS total_attempts, - coalesce(sum(CASE WHEN attempt_succeeded THEN 1 ELSE 0 END), 0) AS attempt_succeeded, - coalesce(sum(CASE WHEN attempt_failed THEN 1 ELSE 0 END), 0) AS attempt_failed, - coalesce(sum(CASE WHEN attempt_succeeded_first_attempt THEN 1 ELSE 0 END), 0) AS attempt_succeeded_first_attempt, - coalesce(sum(CASE WHEN attempt_succeeded THEN 1 ELSE 0 END), 0) * 100.0 / count(*) AS success_rate, - coalesce(sum(CASE WHEN attempt_succeeded_first_attempt THEN 1 ELSE 0 END), 0) * 100.0 / count(*) AS success_rate_first_attempt, - coalesce(sum(CASE WHEN attempt_running THEN 1 ELSE 0 END), 0) AS attempt_running, - sum(attempt_duration) AS sum_attempt_duration, - sum(volume_rows) AS sum_volume_rows, - sum(volume_mb) AS sum_volume_mb + date(sync_updated) as sync_updated, + count(*) as total_attempts, + coalesce(sum(case when attempt_succeeded then 1 else 0 end), 0) as attempt_succeeded, + coalesce(sum(case when attempt_failed then 1 else 0 end), 0) as attempt_failed, + coalesce(sum(case when attempt_succeeded_first_attempt then 1 else 0 end), 0) as attempt_succeeded_first_attempt, + coalesce(sum(case when attempt_succeeded then 1 else 0 end), 0) * 100.0 / count(*) as success_rate, + coalesce(sum(case when attempt_succeeded_first_attempt then 1 else 0 end), 0) * 100.0 / count(*) as success_rate_first_attempt, + coalesce(sum(case when attempt_running then 1 else 0 end), 0) as attempt_running, + sum(attempt_duration) as sum_attempt_duration, + sum(volume_rows) as sum_volume_rows, + sum(volume_mb) as sum_volume_mb -FROM sync_status +from sync_status -GROUP BY source2destination_name, date(sync_updated) +group by source2destination_name, date(sync_updated)