support "bool[pyarrow]" columns

.github/dependabot.yml

@@ -6,6 +6,6 @@
 version: 2
 updates:
   - package-ecosystem: "uv"
-    directory: "/" # Location of package manifests
+    directory: "/"  # Location of package manifests
     schedule:
       interval: "weekly"

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ concurrency:
 jobs:
   test:
     name: Tests - Python ${{ matrix.python-version }} - Resolution Strat ${{ matrix.uv-resolution-strategy }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: true
       matrix:
@@ -51,13 +51,16 @@ jobs:
 
       - name: Install BCP Utility
         run: |
+          set -x
           # https://docs.microsoft.com/en-us/sql/linux/sql-server-linux-setup-tools?view=sql-server-ver15#ubuntu
           curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-          curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
+          curl https://packages.microsoft.com/config/ubuntu/24.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
           sudo apt-get update
-          sudo apt-get install mssql-tools unixodbc-dev
-          echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc
-          source ~/.bashrc
+          sudo apt-get install mssql-tools18 unixodbc-dev odbcinst
+          ls -l /opt/mssql-tools18/bin
+          echo /opt/mssql-tools18/bin >> $GITHUB_PATH
+          ls -l /opt/
+          cat /etc/odbcinst.ini
 
       - name: Test BCP
         run: bcp -v

README.md

@@ -196,7 +196,7 @@ The user has 2 options when constructing it.
    In [2]: creds = SqlCreds('my_server', 'my_db', 'my_username', 'my_password')
 
    In [3]: creds.engine
-   Out[3]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 17 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password)
+   Out[3]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 18 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password)
 
    ```
 
@@ -210,10 +210,10 @@ The user has 2 options when constructing it.
    In [4]: creds2 = SqlCreds.from_engine(creds.engine)
 
    In [5]: creds2.engine
-   Out[5]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 17 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password)
+   Out[5]: Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 18 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password)
 
    In [6]: creds2
-   Out[6]: SqlCreds(server='my_server', database='my_db', username='my_username', with_krb_auth=False, engine=Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 17 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password), password=[REDACTED])
+   Out[6]: SqlCreds(server='my_server', database='my_db', username='my_username', with_krb_auth=False, engine=Engine(mssql+pyodbc:///?odbc_connect=Driver={ODBC Driver 18 for SQL Server};Server=tcp:my_server,1433;Database=my_db;UID=my_username;PWD=my_password), password=[REDACTED])
    ```
 
 ### Recommended Usage
@@ -293,7 +293,7 @@ my_df['some_text_column'].str.contains('\|').sum()
 
 If you get this error message when writing to the database:
 ```
-Error = [Microsoft][ODBC Driver 17 for SQL Server]Incorrect host-column number found in BCP format-file
+Error = [Microsoft][ODBC Driver 18 for SQL Server]Incorrect host-column number found in BCP format-file
 ```
 Try replacing any space characters in your column names, with a command like `my_df.columns = my_df.columns.str.replace(' ','_')` ([source](https://github.com/yehoshuadimarsky/bcpandas/issues/30)).
 

bcpandas/main.py

@@ -7,6 +7,7 @@
 import csv
 import logging
 import os
+from copy import copy
 from pathlib import Path
 from textwrap import dedent
 from typing import Dict, List, Optional, Union
@@ -441,7 +442,20 @@ def to_sql(
     # save to temp path
     csv_file_path = get_temp_file(work_directory)
     # replace bools with 1 or 0, this is what pandas native does when writing to SQL Server
-    df.replace({True: 1, False: 0}).to_csv(
+    # attention: the `(lambda col: lambda...)(copy(col))` part looks odd but is
+    # needed to ensure loop iterations create lambdas working on different columns.
+    df_out = df.assign(
+        **{
+            col: (lambda col: lambda df: df[col].map({True: 1, False: 0}).astype(pd.Int8Dtype()))(
+                copy(col)
+            )
+            for col, dtype in df.dtypes[
+                (df.dtypes == "bool[pyarrow]") | (df.dtypes == "bool") | (df.dtypes == "boolean")
+            ].items()
+        }
+    )
+    # write to CSV
+    df_out.replace({True: 1, False: 0}).to_csv(
         path_or_buf=csv_file_path,
         sep=delim,
         header=False,

benchmarks/read_sql/conftest.py

@@ -61,7 +61,7 @@ def sql_creds():
 @pytest.fixture(scope="session")
 def pyodbc_creds(database):
     db_url = (
-        "Driver={ODBC Driver 17 for SQL Server};"
+        "Driver={ODBC Driver 18 for SQL Server};"
         + f"Server={docker_db_obj.address};"
         + f"Database={_db_name};UID=sa;PWD={docker_db_obj.sa_sql_password};"
     )

tests/conftest.py

@@ -57,16 +57,17 @@ def sql_creds():
         database=_db_name,
         username="sa",
         password=docker_db_obj.sa_sql_password,
+        odbc_kwargs=dict(encrypt="no"),
     )
     return creds
 
 
 @pytest.fixture(scope="session")
 def pyodbc_creds(database):
     db_url = (
-        "Driver={ODBC Driver 17 for SQL Server};"
+        "Driver={ODBC Driver 18 for SQL Server};"
         + f"Server={docker_db_obj.address};"
-        + f"Database={_db_name};UID=sa;PWD={docker_db_obj.sa_sql_password};"
+        + f"Database={_db_name};UID=sa;PWD={docker_db_obj.sa_sql_password};encrypt=no"
     )
     engine = sa.engine.create_engine(
         f"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(db_url)}"

tests/test_sqlcreds.py

@@ -438,6 +438,7 @@ def test_sqlcreds_connection_from_sqlalchemy(sql_creds):
         f"Database={sql_creds.database};"
         f"UID={sql_creds.username};"
         f"PWD={sql_creds.password};"
+        f"encrypt={sql_creds.odbc_kwargs.get('encrypt', 'no')};"
     )
     params = quote_plus(conn_str)
 

tests/test_utils.py

@@ -202,6 +202,7 @@ def test_bcp_login_failure(sql_creds: SqlCreds):
         database=sql_creds.database,
         username=sql_creds.username,
         password="mywrongpassword",
+        odbc_kwargs=dict(encrypt="no"),
     )
     df = pd.DataFrame([{"col1": "value"}])
     with tempfile.TemporaryDirectory() as tmpdir:

tests/utils.py

@@ -161,8 +161,8 @@ def remove(self):
     def create_engine(self, db_name="master") -> sa.engine.Engine:
         """Creates SQLAlchemy pyodbc engine for connecting to specified database (default master) as SA user"""
         db_url = (
-            "Driver={ODBC Driver 17 for SQL Server};"
-            + f"Server={self.address};Database={db_name};UID=sa;PWD={self.sa_sql_password};"
+            "Driver={ODBC Driver 18 for SQL Server};"
+            + f"Server={self.address};Database={db_name};UID=sa;PWD={self.sa_sql_password};encrypt=no"
         )
         return sa.engine.create_engine(
             f"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(db_url)}"