feat: convert document_metadata to have microbatching

medic · Oct 14, 2024 · 40e0bfa · 40e0bfa
1 parent af1cf8d
commit 40e0bfa
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/models/root/document_metadata.sql b/models/root/document_metadata.sql
@@ -1,6 +1,10 @@
 {{
   config(
     materialized = 'incremental',
+    incremental_strategy='microbatch',
+    event_time='saved_timestamp',
+    begin='2020-01-01',
+    batch_size='day'
     unique_key='uuid',
     on_schema_change='append_new_columns',
     post_hook='delete from {{this}} where _deleted=true',
@@ -19,6 +23,3 @@ SELECT
   saved_timestamp,
   doc->>'type' as doc_type
 from {{ source('couchdb', env_var('POSTGRES_TABLE')) }} source_table
-{% if is_incremental() %}
-WHERE source_table.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }}
-{% endif %}