Remove dbt-postgres dependency

.changes/unreleased/Breaking Changes-20240605-171746.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Remove `dbt-postgres` dependency
+time: 2024-06-05T17:17:46.387535-04:00
+custom:
+  Author: mikealfare
+  Issue: "811"

.github/scripts/update_dbt_core_branch.sh

@@ -4,16 +4,13 @@ set -e
 git_branch=$1
 target_req_file="dev-requirements.txt"
 core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
-postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g"
 tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
 if [[ "$OSTYPE" == darwin* ]]; then
  # mac ships with a different version of sed that requires a delimiter arg
  sed -i "" "$core_req_sed_pattern" $target_req_file
- sed -i "" "$postgres_req_sed_pattern" $target_req_file
  sed -i "" "$tests_req_sed_pattern" $target_req_file
 else
  sed -i "$core_req_sed_pattern" $target_req_file
- sed -i "$postgres_req_sed_pattern" $target_req_file
  sed -i "$tests_req_sed_pattern" $target_req_file
 fi
 core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2)

.github/scripts/update_dependencies.sh

@@ -4,15 +4,12 @@ set -e
 git_branch=$1
 target_req_file="dev-requirements.txt"
 core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
-postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g"
 tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
 if [[ "$OSTYPE" == darwin* ]]; then
  # mac ships with a different version of sed that requires a delimiter arg
  sed -i "" "$core_req_sed_pattern" $target_req_file
- sed -i "" "$postgres_req_sed_pattern" $target_req_file
  sed -i "" "$tests_req_sed_pattern" $target_req_file
 else
  sed -i "$core_req_sed_pattern" $target_req_file
- sed -i "$postgres_req_sed_pattern" $target_req_file
  sed -i "$tests_req_sed_pattern" $target_req_file
 fi

dbt/include/redshift/macros/adapters.sql

@@ -96,12 +96,22 @@
 
 
 {% macro redshift__create_schema(relation) -%}
-  {{ postgres__create_schema(relation) }}
+  {% if relation.database -%}
+    {{ adapter.verify_database(relation.database) }}
+  {%- endif -%}
+  {%- call statement('create_schema') -%}
+    create schema if not exists {{ relation.without_identifier().include(database=False) }}
+  {%- endcall -%}
 {% endmacro %}
 
 
 {% macro redshift__drop_schema(relation) -%}
-  {{ postgres__drop_schema(relation) }}
+  {% if relation.database -%}
+    {{ adapter.verify_database(relation.database) }}
+  {%- endif -%}
+  {%- call statement('drop_schema') -%}
+    drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade
+  {%- endcall -%}
 {% endmacro %}
 
 
@@ -251,16 +261,31 @@
 {% endmacro %}
 
 {% macro redshift__information_schema_name(database) -%}
-  {{ return(postgres__information_schema_name(database)) }}
+  {% if database_name -%}
+    {{ adapter.verify_database(database_name) }}
+  {%- endif -%}
+  information_schema
 {%- endmacro %}
 
 
 {% macro redshift__list_schemas(database) -%}
-  {{ return(postgres__list_schemas(database)) }}
+  {% if database -%}
+    {{ adapter.verify_database(database) }}
+  {%- endif -%}
+  {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
+    select distinct nspname from pg_namespace
+  {% endcall %}
+  {{ return(load_result('list_schemas').table) }}
 {%- endmacro %}
 
 {% macro redshift__check_schema_exists(information_schema, schema) -%}
-  {{ return(postgres__check_schema_exists(information_schema, schema)) }}
+  {% if information_schema.database -%}
+    {{ adapter.verify_database(information_schema.database) }}
+  {%- endif -%}
+  {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}
+    select count(*) from pg_namespace where nspname = '{{ schema }}'
+  {% endcall %}
+  {{ return(load_result('check_schema_exists').table) }}
 {%- endmacro %}
 
 
@@ -277,13 +302,80 @@
 {% endmacro %}
 
 
+{#
+  Redshift tables have a maximum length of 127 characters, anything longer is silently truncated.
+  Temp and backup relations add a lot of extra characters to the end of table names to ensure uniqueness.
+  To prevent this going over the character limit, the base_relation name is truncated to ensure
+  that name + suffix + uniquestring is < 128 characters.
+#}
+
+{% macro redshift__make_relation_with_suffix(base_relation, suffix, dstring) %}
+    {% if dstring %}
+      {% set dt = modules.datetime.datetime.now() %}
+      {% set dtstring = dt.strftime("%H%M%S%f") %}
+      {% set suffix = suffix ~ dtstring %}
+    {% endif %}
+    {% set suffix_length = suffix|length %}
+    {% set relation_max_name_length = base_relation.relation_max_name_length() %}
+    {% if suffix_length > relation_max_name_length %}
+        {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}
+    {% endif %}
+    {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}
+
+    {{ return(base_relation.incorporate(path={"identifier": identifier })) }}
+
+  {% endmacro %}
+
+{% macro redshift__make_intermediate_relation(base_relation, suffix) %}
+    {{ return(redshift__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}
+{% endmacro %}
+
+{% macro redshift__make_temp_relation(base_relation, suffix) %}
+    {% set temp_relation = redshift__make_relation_with_suffix(base_relation, suffix, dstring=True) %}
+    {{ return(temp_relation.incorporate(path={"schema": none,
+                                              "database": none})) }}
+{% endmacro %}
+
+{% macro redshift__make_backup_relation(base_relation, backup_relation_type, suffix) %}
+    {% set backup_relation = redshift__make_relation_with_suffix(base_relation, suffix, dstring=False) %}
+    {{ return(backup_relation.incorporate(type=backup_relation_type)) }}
+{% endmacro %}
+
+
+{#
+  By using dollar-quoting like this, users can embed anything they want into their comments
+  (including nested dollar-quoting), as long as they do not use this exact dollar-quoting
+  label. It would be nice to just pick a new one but eventually you do have to give up.
+#}
+{% macro dbt_escape_comment(comment) -%}
+  {% if comment is not string %}
+    {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}
+  {% endif %}
+  {%- set magic = '$dbt_comment_literal_block$' -%}
+  {%- if magic in comment -%}
+    {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}
+  {%- endif -%}
+  {{ magic }}{{ comment }}{{ magic }}
+{%- endmacro %}
+
+{# This is not a namespace macro, keep the name for backwards compatibility post dbt-postgres decoupling #}
+{% macro postgres_escape_comment(comment) %}
+  {{ escape_comment(comment) }}
+{% endmacro %}
+
 {% macro redshift__alter_relation_comment(relation, comment) %}
-  {% do return(postgres__alter_relation_comment(relation, comment)) %}
+  {% set escaped_comment = dbt_escape_comment(comment) %}
+  comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};
 {% endmacro %}
 
 
 {% macro redshift__alter_column_comment(relation, column_dict) %}
-  {% do return(postgres__alter_column_comment(relation, column_dict)) %}
+  {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %}
+  {% for column_name in column_dict if (column_name in existing_columns) %}
+    {% set comment = column_dict[column_name]['description'] %}
+    {% set escaped_comment = dbt_escape_comment(comment) %}
+    comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};
+  {% endfor %}
 {% endmacro %}
 
 

dbt/include/redshift/macros/materializations/incremental_strategies.sql

@@ -0,0 +1,9 @@
+{% macro redshift__get_incremental_default_sql(arg_dict) %}
+
+  {% if arg_dict["unique_key"] %}
+    {% do return(get_incremental_delete_insert_sql(arg_dict)) %}
+  {% else %}
+    {% do return(get_incremental_append_sql(arg_dict)) %}
+  {% endif %}
+
+{% endmacro %}

dbt/include/redshift/macros/materializations/snapshot_merge.sql

@@ -1,4 +1,18 @@
 
 {% macro redshift__snapshot_merge_sql(target, source, insert_cols) -%}
-    {{ postgres__snapshot_merge_sql(target, source, insert_cols) }}
+    {%- set insert_cols_csv = insert_cols | join(', ') -%}
+
+    update {{ target }}
+    set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to
+    from {{ source }} as DBT_INTERNAL_SOURCE
+    where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text
+      and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)
+      and {{ target }}.dbt_valid_to is null;
+
+    insert into {{ target }} ({{ insert_cols_csv }})
+    select {% for column in insert_cols -%}
+        DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}
+    {%- endfor %}
+    from {{ source }} as DBT_INTERNAL_SOURCE
+    where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;
 {% endmacro %}

dbt/include/redshift/macros/relations.sql

@@ -43,3 +43,8 @@ join relation dep
 {{ return(load_result('relations').table) }}
 
 {% endmacro %}
+
+{# This is not a namespace macro, keep the name for backwards compatibility post dbt-postgres decoupling #}
+{% macro postgres_get_relations() %}
+  {{ return(redshift__get_relations()) }}
+{% endmacro %}

dbt/include/redshift/macros/utils/any_value.sql

@@ -0,0 +1,10 @@
+{#- /*
+While Redshift supports any_value, Postgres does not support any_value.
+This mimics the previous behavior before decoupling dbt-redshift from dbt-postgres.
+*/ -#}
+
+{% macro redshift__any_value(expression) -%}
+
+    min({{ expression }})
+
+{%- endmacro %}

dev-requirements.txt

@@ -1,9 +1,8 @@
-# install latest changes in dbt-core + dbt-postgres
+# install latest changes in dbt-core
 git+https://github.com/dbt-labs/dbt-adapters.git
 git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
 git+https://github.com/dbt-labs/dbt-common.git
 git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core
-git+https://github.com/dbt-labs/dbt-postgres.git
 
 # dev
 ipdb~=0.13.13

setup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
-import re
 import sys
 
+
 if sys.version_info < (3, 8):
     print("Error: dbt does not support this version of Python.")
     print("Please upgrade to Python 3.8 or higher.")
@@ -37,16 +37,6 @@ def _plugin_version() -> str:
     return attributes["version"]
 
 
-def _plugin_version_trim() -> str:
-    """
-    Pull the package version from the main package version file
-    """
-    attributes = {}
-    exec(VERSION.read_text(), attributes)
-    pattern = r"\+build\d+$"
-    return re.sub(pattern, "", attributes["version"])
-
-
 setup(
     name="dbt-redshift",
     version=_plugin_version(),
@@ -61,7 +51,6 @@ def _plugin_version_trim() -> str:
     install_requires=[
         "dbt-common>=0.1.0a1,<2.0",
         "dbt-adapters>=0.1.0a1,<2.0",
-        f"dbt-postgres~={_plugin_version_trim()}",
         # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases
         # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift.
         "redshift-connector<2.0.918,>=2.0.913,!=2.0.914",

tests/functional/adapter/test_macros.py

@@ -31,7 +31,7 @@
     {{ exceptions.raise_compiler_error(msg) }}
 {% endmacro %}
 
-{% macro postgres__dispatch_to_parent() %}
+{% macro redshift__dispatch_to_parent() %}
     {{ return('') }}
 {% endmacro %}
 """

tests/functional/adapter/test_query_comment.py

@@ -20,7 +20,7 @@ class TestMacroQueryCommentsRedshift(BaseMacroQueryComments):
 class TestMacroArgsQueryCommentsRedshift(BaseMacroArgsQueryComments):
     @pytest.mark.skip(
         "This test is incorrectly comparing the version of `dbt-core`"
-        "to the version of `dbt-postgres`, which is not always the same."
+        "to the version of `dbt-redshift`, which is not always the same."
     )
     def test_matches_comment(self, project, get_package_version):
         pass

tests/unit/utils.py

@@ -123,7 +123,6 @@ def inject_plugin(plugin):
 
 
 def inject_plugin_for(config):
-    # from dbt.adapters.postgres import Plugin, PostgresAdapter
     from dbt.adapters.factory import FACTORY
 
     FACTORY.load_plugin(config.credentials.type)