From c681338bc2ed036fff0516ff2ca0dad6024533d3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 17:08:21 -0400 Subject: [PATCH 01/12] remove dbt-postgres dependency --- .github/scripts/update_dbt_core_branch.sh | 3 - .github/scripts/update_dependencies.sh | 3 - dbt/adapters/redshift/__init__.py | 1 - dbt/include/redshift/macros/adapters.sql | 62 ++++++++++++++++--- .../materializations/snapshot_merge.sql | 16 ++++- dev-requirements.txt | 3 +- setup.py | 13 +--- tests/functional/adapter/test_macros.py | 2 +- .../functional/adapter/test_query_comment.py | 2 +- tests/unit/utils.py | 1 - 10 files changed, 74 insertions(+), 32 deletions(-) diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh index d28a40c35..1a5a5c2d7 100755 --- a/.github/scripts/update_dbt_core_branch.sh +++ b/.github/scripts/update_dbt_core_branch.sh @@ -4,16 +4,13 @@ set -e git_branch=$1 target_req_file="dev-requirements.txt" core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file sed -i "" "$tests_req_sed_pattern" $target_req_file else sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file sed -i "$tests_req_sed_pattern" $target_req_file fi core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh index 6000b5006..c3df48e52 100644 --- a/.github/scripts/update_dependencies.sh +++ b/.github/scripts/update_dependencies.sh @@ -4,15 +4,12 @@ set -e git_branch=$1 target_req_file="dev-requirements.txt" core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file sed -i "" "$tests_req_sed_pattern" $target_req_file else sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file sed -i "$tests_req_sed_pattern" $target_req_file fi diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index 70e77ef5e..3bdb5fb62 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -14,5 +14,4 @@ adapter=RedshiftAdapter, # type: ignore credentials=RedshiftCredentials, include_path=redshift.PACKAGE_PATH, - dependencies=["postgres"], ) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 5da047f58..da849bc57 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -96,12 +96,22 @@ {% macro redshift__create_schema(relation) -%} - {{ postgres__create_schema(relation) }} + {% if relation.database -%} + {{ adapter.verify_database(relation.database) }} + {%- endif -%} + {%- call statement('create_schema') -%} + create schema if not exists {{ relation.without_identifier().include(database=False) }} + {%- endcall -%} {% endmacro %} {% macro redshift__drop_schema(relation) -%} - {{ postgres__drop_schema(relation) }} + {% if relation.database -%} + {{ adapter.verify_database(relation.database) }} + {%- endif -%} + {%- call statement('drop_schema') -%} + drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade + {%- endcall -%} {% endmacro %} @@ -251,16 +261,31 @@ {% endmacro %} {% macro redshift__information_schema_name(database) -%} - {{ return(postgres__information_schema_name(database)) }} + {% if database_name -%} + {{ adapter.verify_database(database_name) }} + {%- endif -%} + information_schema {%- endmacro %} {% macro redshift__list_schemas(database) -%} - {{ return(postgres__list_schemas(database)) }} + {% if database -%} + {{ adapter.verify_database(database) }} + {%- endif -%} + {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} + select distinct nspname from pg_namespace + {% endcall %} + {{ return(load_result('list_schemas').table) }} {%- endmacro %} {% macro redshift__check_schema_exists(information_schema, schema) -%} - {{ return(postgres__check_schema_exists(information_schema, schema)) }} + {% if information_schema.database -%} + {{ adapter.verify_database(information_schema.database) }} + {%- endif -%} + {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %} + select count(*) from pg_namespace where nspname = '{{ schema }}' + {% endcall %} + {{ return(load_result('check_schema_exists').table) }} {%- endmacro %} @@ -277,13 +302,36 @@ {% endmacro %} +{# + By using dollar-quoting like this, users can embed anything they want into their comments + (including nested dollar-quoting), as long as they do not use this exact dollar-quoting + label. It would be nice to just pick a new one but eventually you do have to give up. +#} +{% macro _escape_comment(comment) -%} + {% if comment is not string %} + {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %} + {% endif %} + {%- set magic = '$dbt_comment_literal_block$' -%} + {%- if magic in comment -%} + {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%} + {%- endif -%} + {{ magic }}{{ comment }}{{ magic }} +{%- endmacro %} + + {% macro redshift__alter_relation_comment(relation, comment) %} - {% do return(postgres__alter_relation_comment(relation, comment)) %} + {% set escaped_comment = _escape_comment(comment) %} + comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }}; {% endmacro %} {% macro redshift__alter_column_comment(relation, column_dict) %} - {% do return(postgres__alter_column_comment(relation, column_dict)) %} + {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} + {% for column_name in column_dict if (column_name in existing_columns) %} + {% set comment = column_dict[column_name]['description'] %} + {% set escaped_comment = _escape_comment(comment) %} + comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }}; + {% endfor %} {% endmacro %} diff --git a/dbt/include/redshift/macros/materializations/snapshot_merge.sql b/dbt/include/redshift/macros/materializations/snapshot_merge.sql index eda314727..8e2f647c8 100644 --- a/dbt/include/redshift/macros/materializations/snapshot_merge.sql +++ b/dbt/include/redshift/macros/materializations/snapshot_merge.sql @@ -1,4 +1,18 @@ {% macro redshift__snapshot_merge_sql(target, source, insert_cols) -%} - {{ postgres__snapshot_merge_sql(target, source, insert_cols) }} + {%- set insert_cols_csv = insert_cols | join(', ') -%} + + update {{ target }} + set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to + from {{ source }} as DBT_INTERNAL_SOURCE + where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text + and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text) + and {{ target }}.dbt_valid_to is null; + + insert into {{ target }} ({{ insert_cols_csv }}) + select {% for column in insert_cols -%} + DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %} + {%- endfor %} + from {{ source }} as DBT_INTERNAL_SOURCE + where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text; {% endmacro %} diff --git a/dev-requirements.txt b/dev-requirements.txt index d02863ae0..0025c11e2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,8 @@ -# install latest changes in dbt-core + dbt-postgres +# install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core -git+https://github.com/dbt-labs/dbt-postgres.git # dev ipdb~=0.13.13 diff --git a/setup.py b/setup.py index dbb3913b9..66a52e54f 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import re import sys + if sys.version_info < (3, 8): print("Error: dbt does not support this version of Python.") print("Please upgrade to Python 3.8 or higher.") @@ -37,16 +37,6 @@ def _plugin_version() -> str: return attributes["version"] -def _plugin_version_trim() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - pattern = r"\+build\d+$" - return re.sub(pattern, "", attributes["version"]) - - setup( name="dbt-redshift", version=_plugin_version(), @@ -61,7 +51,6 @@ def _plugin_version_trim() -> str: install_requires=[ "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", - f"dbt-postgres~={_plugin_version_trim()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", diff --git a/tests/functional/adapter/test_macros.py b/tests/functional/adapter/test_macros.py index 0596ab549..706fe3125 100644 --- a/tests/functional/adapter/test_macros.py +++ b/tests/functional/adapter/test_macros.py @@ -31,7 +31,7 @@ {{ exceptions.raise_compiler_error(msg) }} {% endmacro %} -{% macro postgres__dispatch_to_parent() %} +{% macro redshift__dispatch_to_parent() %} {{ return('') }} {% endmacro %} """ diff --git a/tests/functional/adapter/test_query_comment.py b/tests/functional/adapter/test_query_comment.py index 75c87ee38..f1287fb76 100644 --- a/tests/functional/adapter/test_query_comment.py +++ b/tests/functional/adapter/test_query_comment.py @@ -20,7 +20,7 @@ class TestMacroQueryCommentsRedshift(BaseMacroQueryComments): class TestMacroArgsQueryCommentsRedshift(BaseMacroArgsQueryComments): @pytest.mark.skip( "This test is incorrectly comparing the version of `dbt-core`" - "to the version of `dbt-postgres`, which is not always the same." + "to the version of `dbt-redshift`, which is not always the same." ) def test_matches_comment(self, project, get_package_version): pass diff --git a/tests/unit/utils.py b/tests/unit/utils.py index ee580eb9a..49d05850b 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -123,7 +123,6 @@ def inject_plugin(plugin): def inject_plugin_for(config): - # from dbt.adapters.postgres import Plugin, PostgresAdapter from dbt.adapters.factory import FACTORY FACTORY.load_plugin(config.credentials.type) From dec8167e68664ccbcc3340170582efae8ecd8b87 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 17:17:55 -0400 Subject: [PATCH 02/12] changelog --- .changes/unreleased/Breaking Changes-20240605-171746.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Breaking Changes-20240605-171746.yaml diff --git a/.changes/unreleased/Breaking Changes-20240605-171746.yaml b/.changes/unreleased/Breaking Changes-20240605-171746.yaml new file mode 100644 index 000000000..f4a66c314 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20240605-171746.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Remove `dbt-postgres` dependency +time: 2024-06-05T17:17:46.387535-04:00 +custom: + Author: mikealfare + Issue: "811" From d6133886ae63fb14512968e9bdbe1d87c19aaa2a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 18:48:10 -0400 Subject: [PATCH 03/12] add dbt-postgres back as dependency --- dbt/adapters/redshift/__init__.py | 1 + dev-requirements.txt | 3 ++- setup.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index 3bdb5fb62..70e77ef5e 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -14,4 +14,5 @@ adapter=RedshiftAdapter, # type: ignore credentials=RedshiftCredentials, include_path=redshift.PACKAGE_PATH, + dependencies=["postgres"], ) diff --git a/dev-requirements.txt b/dev-requirements.txt index 0025c11e2..d02863ae0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,9 @@ -# install latest changes in dbt-core +# install latest changes in dbt-core + dbt-postgres git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core +git+https://github.com/dbt-labs/dbt-postgres.git # dev ipdb~=0.13.13 diff --git a/setup.py b/setup.py index 66a52e54f..cb36d43a3 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import re import sys @@ -37,6 +38,16 @@ def _plugin_version() -> str: return attributes["version"] +def _plugin_version_trim() -> str: + """ + Pull the package version from the main package version file + """ + attributes = {} + exec(VERSION.read_text(), attributes) + pattern = r"\+build\d+$" + return re.sub(pattern, "", attributes["version"]) + + setup( name="dbt-redshift", version=_plugin_version(), @@ -51,6 +62,7 @@ def _plugin_version() -> str: install_requires=[ "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", + f"dbt-postgres~={_plugin_version_trim()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", From e5b9fc4bb622bedd36309a360918a9145109be2e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 19:16:12 -0400 Subject: [PATCH 04/12] add postgres backup and temp relation macros --- dbt/include/redshift/macros/adapters.sql | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index da849bc57..193ee28cb 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -302,6 +302,46 @@ {% endmacro %} +{# + Redshift tables have a maximum length of 127 characters, anything longer is silently truncated. + Temp and backup relations add a lot of extra characters to the end of table names to ensure uniqueness. + To prevent this going over the character limit, the base_relation name is truncated to ensure + that name + suffix + uniquestring is < 128 characters. +#} + +{% macro redshift__make_relation_with_suffix(base_relation, suffix, dstring) %} + {% if dstring %} + {% set dt = modules.datetime.datetime.now() %} + {% set dtstring = dt.strftime("%H%M%S%f") %} + {% set suffix = suffix ~ dtstring %} + {% endif %} + {% set suffix_length = suffix|length %} + {% set relation_max_name_length = base_relation.relation_max_name_length() %} + {% if suffix_length > relation_max_name_length %} + {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %} + {% endif %} + {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %} + + {{ return(base_relation.incorporate(path={"identifier": identifier })) }} + + {% endmacro %} + +{% macro redshift__make_intermediate_relation(base_relation, suffix) %} + {{ return(redshift__make_relation_with_suffix(base_relation, suffix, dstring=False)) }} +{% endmacro %} + +{% macro redshift__make_temp_relation(base_relation, suffix) %} + {% set temp_relation = redshift__make_relation_with_suffix(base_relation, suffix, dstring=True) %} + {{ return(temp_relation.incorporate(path={"schema": none, + "database": none})) }} +{% endmacro %} + +{% macro redshift__make_backup_relation(base_relation, backup_relation_type, suffix) %} + {% set backup_relation = redshift__make_relation_with_suffix(base_relation, suffix, dstring=False) %} + {{ return(backup_relation.incorporate(type=backup_relation_type)) }} +{% endmacro %} + + {# By using dollar-quoting like this, users can embed anything they want into their comments (including nested dollar-quoting), as long as they do not use this exact dollar-quoting From 57d159d5cd0e46b22fdaf746ed904834622ab064 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 19:16:23 -0400 Subject: [PATCH 05/12] remove postgres macros as a dependency --- dbt/adapters/redshift/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index 70e77ef5e..3bdb5fb62 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -14,5 +14,4 @@ adapter=RedshiftAdapter, # type: ignore credentials=RedshiftCredentials, include_path=redshift.PACKAGE_PATH, - dependencies=["postgres"], ) From 975458e1e84da68e007ca07c0402774a89de22e9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 00:16:10 -0400 Subject: [PATCH 06/12] add default incremental strategy --- .../macros/materializations/incremental_strategies.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 dbt/include/redshift/macros/materializations/incremental_strategies.sql diff --git a/dbt/include/redshift/macros/materializations/incremental_strategies.sql b/dbt/include/redshift/macros/materializations/incremental_strategies.sql new file mode 100644 index 000000000..f2fbf41e0 --- /dev/null +++ b/dbt/include/redshift/macros/materializations/incremental_strategies.sql @@ -0,0 +1,9 @@ +{% macro postgres__get_incremental_default_sql(arg_dict) %} + + {% if arg_dict["unique_key"] %} + {% do return(get_incremental_delete_insert_sql(arg_dict)) %} + {% else %} + {% do return(get_incremental_append_sql(arg_dict)) %} + {% endif %} + +{% endmacro %} From 1d3433f1068d35bd5f596c447772d4cb894212e8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 14:02:21 -0400 Subject: [PATCH 07/12] ensure local macro names match previous names --- dbt/include/redshift/macros/adapters.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 193ee28cb..e0dac1b22 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -347,7 +347,7 @@ (including nested dollar-quoting), as long as they do not use this exact dollar-quoting label. It would be nice to just pick a new one but eventually you do have to give up. #} -{% macro _escape_comment(comment) -%} +{% macro postgres_escape_comment(comment) -%} {% if comment is not string %} {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %} {% endif %} @@ -360,7 +360,7 @@ {% macro redshift__alter_relation_comment(relation, comment) %} - {% set escaped_comment = _escape_comment(comment) %} + {% set escaped_comment = postgres_escape_comment(comment) %} comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }}; {% endmacro %} @@ -369,7 +369,7 @@ {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} {% for column_name in column_dict if (column_name in existing_columns) %} {% set comment = column_dict[column_name]['description'] %} - {% set escaped_comment = _escape_comment(comment) %} + {% set escaped_comment = postgres_escape_comment(comment) %} comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }}; {% endfor %} {% endmacro %} From 76d7639841c7adc496d3cccb63abfcdcf6a10499 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 14:02:48 -0400 Subject: [PATCH 08/12] add two more dbt-postgres macros that were missing --- dbt/include/redshift/macros/relations.sql | 4 ++++ dbt/include/redshift/macros/utils/any_value.sql | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 dbt/include/redshift/macros/utils/any_value.sql diff --git a/dbt/include/redshift/macros/relations.sql b/dbt/include/redshift/macros/relations.sql index 6d83c36b9..000a2f169 100644 --- a/dbt/include/redshift/macros/relations.sql +++ b/dbt/include/redshift/macros/relations.sql @@ -43,3 +43,7 @@ join relation dep {{ return(load_result('relations').table) }} {% endmacro %} + +{% macro postgres_get_relations() %} + {{ return(redshift__get_relations()) }} +{% endmacro %} diff --git a/dbt/include/redshift/macros/utils/any_value.sql b/dbt/include/redshift/macros/utils/any_value.sql new file mode 100644 index 000000000..c4bfdf8d4 --- /dev/null +++ b/dbt/include/redshift/macros/utils/any_value.sql @@ -0,0 +1,10 @@ +{#- /* +While Redshift supports any_value, Postgres does not support any_value. +This mimics the previous behavior before decoupling dbt-redshift from dbt-postgres. +*/ -#} + +{% macro redshift__any_value(expression) -%} + + min({{ expression }}) + +{%- endmacro %} From e464012dbba559225093e89e86378806f1dfa09f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 14:03:01 -0400 Subject: [PATCH 09/12] fix adapter prefix for macro name --- .../redshift/macros/materializations/incremental_strategies.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/redshift/macros/materializations/incremental_strategies.sql b/dbt/include/redshift/macros/materializations/incremental_strategies.sql index f2fbf41e0..a4cb6474d 100644 --- a/dbt/include/redshift/macros/materializations/incremental_strategies.sql +++ b/dbt/include/redshift/macros/materializations/incremental_strategies.sql @@ -1,4 +1,4 @@ -{% macro postgres__get_incremental_default_sql(arg_dict) %} +{% macro redshift__get_incremental_default_sql(arg_dict) %} {% if arg_dict["unique_key"] %} {% do return(get_incremental_delete_insert_sql(arg_dict)) %} From 61c4b2a77bbde6ff550c631435c48504c81c2eaf Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 15:14:53 -0400 Subject: [PATCH 10/12] add clarifying comments on why there is still a reference to postgres --- dbt/include/redshift/macros/adapters.sql | 10 +++++++--- dbt/include/redshift/macros/relations.sql | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index e0dac1b22..5c5fa1834 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -347,7 +347,7 @@ (including nested dollar-quoting), as long as they do not use this exact dollar-quoting label. It would be nice to just pick a new one but eventually you do have to give up. #} -{% macro postgres_escape_comment(comment) -%} +{% macro dbt_escape_comment(comment) -%} {% if comment is not string %} {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %} {% endif %} @@ -358,9 +358,13 @@ {{ magic }}{{ comment }}{{ magic }} {%- endmacro %} +{# This is not a namespace macro, keep the name for backwards compatibility post dbt-postgres decoupling #} +{% macro postgres_escape_comment(comment) %} + {{ escape_comment(comment) }} +{% endmacro %} {% macro redshift__alter_relation_comment(relation, comment) %} - {% set escaped_comment = postgres_escape_comment(comment) %} + {% set escaped_comment = dbt_escape_comment(comment) %} comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }}; {% endmacro %} @@ -369,7 +373,7 @@ {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} {% for column_name in column_dict if (column_name in existing_columns) %} {% set comment = column_dict[column_name]['description'] %} - {% set escaped_comment = postgres_escape_comment(comment) %} + {% set escaped_comment = dbt_escape_comment(comment) %} comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }}; {% endfor %} {% endmacro %} diff --git a/dbt/include/redshift/macros/relations.sql b/dbt/include/redshift/macros/relations.sql index 000a2f169..58a7f2531 100644 --- a/dbt/include/redshift/macros/relations.sql +++ b/dbt/include/redshift/macros/relations.sql @@ -44,6 +44,7 @@ join relation dep {% endmacro %} +{# This is not a namespace macro, keep the name for backwards compatibility post dbt-postgres decoupling #} {% macro postgres_get_relations() %} {{ return(redshift__get_relations()) }} {% endmacro %} From 1ecb96a9d4f7aa2efa0cc0b456ce66bd875c96cf Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 15:15:17 -0400 Subject: [PATCH 11/12] remove the dbt-postgres dependency --- dev-requirements.txt | 3 +-- setup.py | 12 ------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index d02863ae0..0025c11e2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,8 @@ -# install latest changes in dbt-core + dbt-postgres +# install latest changes in dbt-core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core -git+https://github.com/dbt-labs/dbt-postgres.git # dev ipdb~=0.13.13 diff --git a/setup.py b/setup.py index cb36d43a3..66a52e54f 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import re import sys @@ -38,16 +37,6 @@ def _plugin_version() -> str: return attributes["version"] -def _plugin_version_trim() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - pattern = r"\+build\d+$" - return re.sub(pattern, "", attributes["version"]) - - setup( name="dbt-redshift", version=_plugin_version(), @@ -62,7 +51,6 @@ def _plugin_version_trim() -> str: install_requires=[ "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", - f"dbt-postgres~={_plugin_version_trim()}", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", From 7add399c0ac985a4fe895fafd3ac204dad3187c6 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 17:35:52 -0400 Subject: [PATCH 12/12] add postgres as a dependency in the adapter plugin in an attempt to pick up custom dispatched macros from postgres --- dbt/adapters/redshift/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index 3bdb5fb62..70e77ef5e 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -14,4 +14,5 @@ adapter=RedshiftAdapter, # type: ignore credentials=RedshiftCredentials, include_path=redshift.PACKAGE_PATH, + dependencies=["postgres"], )