diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4e33fa2..4823226 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,6 +6,6 @@ version: 2 updates: - package-ecosystem: "uv" - directory: "/" # Location of package manifests + directory: "/" # Location of package manifests schedule: interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 109e6fb..3b67c15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ concurrency: jobs: test: name: Tests - Python ${{ matrix.python-version }} - Resolution Strat ${{ matrix.uv-resolution-strategy }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: true matrix: @@ -51,13 +51,16 @@ jobs: - name: Install BCP Utility run: | + set -x # https://docs.microsoft.com/en-us/sql/linux/sql-server-linux-setup-tools?view=sql-server-ver15#ubuntu curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + curl https://packages.microsoft.com/config/ubuntu/24.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list sudo apt-get update - sudo apt-get install mssql-tools unixodbc-dev - echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc - source ~/.bashrc + sudo apt-get install mssql-tools18 unixodbc-dev odbcinst + ls -l /opt/mssql-tools18/bin + echo /opt/mssql-tools18/bin >> $GITHUB_PATH + ls -l /opt/ + cat /etc/odbcinst.ini - name: Test BCP run: bcp -v diff --git a/README.md b/README.md index d212717..991ac9e 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ The user has 2 options when constructing it. In [2]: creds = SqlCreds('my_server', 'my_db', 'my_username', 'my_password') In [3]: creds.engine - Out[3]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 17 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password) + Out[3]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 18 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password) ``` @@ -210,10 +210,10 @@ The user has 2 options when constructing it. In [4]: creds2 = SqlCreds.from_engine(creds.engine) In [5]: creds2.engine - Out[5]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 17 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password) + Out[5]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 18 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password) In [6]: creds2 - Out[6]: SqlCreds(server='my_server', database='my_db', username='my_username', with_krb_auth=False, engine=Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 17 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password), password=[REDACTED]) + Out[6]: SqlCreds(server='my_server', database='my_db', username='my_username', with_krb_auth=False, engine=Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 18 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password), password=[REDACTED]) ``` ### Recommended Usage @@ -293,7 +293,7 @@ my_df['some_text_column'].str.contains('\|').sum() If you get this error message when writing to the database: ``` -Error = [Microsoft][ODBC Driver 17 for SQL Server]Incorrect host-column number found in BCP format-file +Error = [Microsoft][ODBC Driver 18 for SQL Server]Incorrect host-column number found in BCP format-file ``` Try replacing any space characters in your column names, with a command like `my_df.columns = my_df.columns.str.replace(' ','_')` ([source](https://github.com/yehoshuadimarsky/bcpandas/issues/30)). diff --git a/bcpandas/main.py b/bcpandas/main.py index 4b94801..af1b6c3 100644 --- a/bcpandas/main.py +++ b/bcpandas/main.py @@ -441,7 +441,18 @@ def to_sql( # save to temp path csv_file_path = get_temp_file(work_directory) # replace bools with 1 or 0, this is what pandas native does when writing to SQL Server - df.replace({True: 1, False: 0}).to_csv( + # attention: the `(lambda col: lambda...)(copy(col))` part looks odd but is + # needed to ensure loop iterations create lambdas working on different columns. + df_out = df.assign( + **{ + col: lambda df, col=col: df[col].map({True: 1, False: 0}).astype(pd.Int8Dtype()) + for col, dtype in df.dtypes[ + (df.dtypes == "bool[pyarrow]") | (df.dtypes == "bool") | (df.dtypes == "boolean") + ].items() + } + ) + # write to CSV + df_out.to_csv( path_or_buf=csv_file_path, sep=delim, header=False, diff --git a/benchmarks/read_sql/conftest.py b/benchmarks/read_sql/conftest.py index 6ca5025..aae297e 100644 --- a/benchmarks/read_sql/conftest.py +++ b/benchmarks/read_sql/conftest.py @@ -61,7 +61,7 @@ def sql_creds(): @pytest.fixture(scope="session") def pyodbc_creds(database): db_url = ( - "Driver={ODBC Driver 17 for SQL Server};" + "Driver={ODBC Driver 18 for SQL Server};" + f"Server={docker_db_obj.address};" + f"Database={_db_name};UID=sa;PWD={docker_db_obj.sa_sql_password};" ) diff --git a/tests/conftest.py b/tests/conftest.py index 59f0230..f29743c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,6 +57,7 @@ def sql_creds(): database=_db_name, username="sa", password=docker_db_obj.sa_sql_password, + odbc_kwargs=dict(encrypt="no"), ) return creds @@ -64,9 +65,9 @@ def sql_creds(): @pytest.fixture(scope="session") def pyodbc_creds(database): db_url = ( - "Driver={ODBC Driver 17 for SQL Server};" + "Driver={ODBC Driver 18 for SQL Server};" + f"Server={docker_db_obj.address};" - + f"Database={_db_name};UID=sa;PWD={docker_db_obj.sa_sql_password};" + + f"Database={_db_name};UID=sa;PWD={docker_db_obj.sa_sql_password};encrypt=no" ) engine = sa.engine.create_engine( f"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(db_url)}" diff --git a/tests/test_sqlcreds.py b/tests/test_sqlcreds.py index 6cbaf03..a5875ed 100644 --- a/tests/test_sqlcreds.py +++ b/tests/test_sqlcreds.py @@ -438,6 +438,7 @@ def test_sqlcreds_connection_from_sqlalchemy(sql_creds): f"Database={sql_creds.database};" f"UID={sql_creds.username};" f"PWD={sql_creds.password};" + f"encrypt={sql_creds.odbc_kwargs.get('encrypt', 'no')};" ) params = quote_plus(conn_str) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1ef63a6..e784a2e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -202,6 +202,7 @@ def test_bcp_login_failure(sql_creds: SqlCreds): database=sql_creds.database, username=sql_creds.username, password="mywrongpassword", + odbc_kwargs=dict(encrypt="no"), ) df = pd.DataFrame([{"col1": "value"}]) with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/utils.py b/tests/utils.py index df1c38d..31db9bf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -161,8 +161,8 @@ def remove(self): def create_engine(self, db_name="master") -> sa.engine.Engine: """Creates SQLAlchemy pyodbc engine for connecting to specified database (default master) as SA user""" db_url = ( - "Driver={ODBC Driver 17 for SQL Server};" - + f"Server={self.address};Database={db_name};UID=sa;PWD={self.sa_sql_password};" + "Driver={ODBC Driver 18 for SQL Server};" + + f"Server={self.address};Database={db_name};UID=sa;PWD={self.sa_sql_password};encrypt=no" ) return sa.engine.create_engine( f"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(db_url)}"