Support for SQL Server

.github/workflows/test-suite.yml

@@ -11,10 +11,9 @@ jobs:
   tests:
     name: "Python ${{ matrix.python-version }}"
     runs-on: "ubuntu-latest"
-
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     services:
       mysql:
@@ -38,11 +37,31 @@ jobs:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
+      mssql:
+        image: mcr.microsoft.com/mssql/server:2019-GA-ubuntu-16.04
+        env:
+          MSSQL_SA_PASSWORD: Mssql123mssql-
+          ACCEPT_EULA: "Y"
+          MSSQL_PID: Developer
+        ports:
+          - 1433:1433
+        options: >-
+          --health-cmd "/opt/mssql-tools/bin/sqlcmd -U sa -P Mssql123mssql- -Q 'select 1' -b -o /dev/null"
+          --health-interval 60s
+          --health-timeout 30s
+          --health-start-period 20s
+          --health-retries 3
+
     steps:
       - uses: "actions/checkout@v3"
       - uses: "actions/setup-python@v4"
         with:
           python-version: "${{ matrix.python-version }}"
+      - name: "Install drivers"
+        run: |
+          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17
+          sudo apt-get install -y unixodbc-dev
+
       - name: "Install dependencies"
         run: "scripts/install"
       - name: "Run linting checks"
@@ -59,5 +78,8 @@ jobs:
             mysql+asyncmy://username:password@localhost:3306/testsuite,
             postgresql://username:password@localhost:5432/testsuite,
             postgresql+aiopg://username:[email protected]:5432/testsuite,
-            postgresql+asyncpg://username:password@localhost:5432/testsuite
+            postgresql+asyncpg://username:password@localhost:5432/testsuite,
+            mssql://sa:Mssql123mssql-@localhost:1433/master?driver=ODBC+Driver+17+for+SQL+Server,
+            mssql+pyodbc://sa:Mssql123mssql-@localhost:1433/master?driver=ODBC+Driver+17+for+SQL+Server,
+            mssql+aioodbc://sa:Mssql123mssql-@localhost:1433/master?driver=ODBC+Driver+17+for+SQL+Server
         run: "scripts/test"

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+### Added
+
+- Support for SQLAlchemy 2.0+
+- Added internal support for the new psycopg dialect.
+
 ## 0.7.0 (Dec 18th, 2022)
 
 ### Fixed

README.md

@@ -36,6 +36,7 @@ Database drivers supported are:
 * [aiomysql][aiomysql]
 * [asyncmy][asyncmy]
 * [aiosqlite][aiosqlite]
+* [aioodbc][aioodbc]
 
 You can install the required database drivers with:
 
@@ -45,9 +46,10 @@ $ pip install databases[aiopg]
 $ pip install databases[aiomysql]
 $ pip install databases[asyncmy]
 $ pip install databases[aiosqlite]
+$ pip install databases[aioodbc]
 ```
 
-Note that if you are using any synchronous SQLAlchemy functions such as `engine.create_all()` or [alembic][alembic] migrations then you still have to install a synchronous DB driver: [psycopg2][psycopg2] for PostgreSQL and [pymysql][pymysql] for MySQL.
+Note that if you are using any synchronous SQLAlchemy functions such as `engine.create_all()` or [alembic][alembic] migrations then you still have to install a synchronous DB driver: [psycopg2][psycopg2] for PostgreSQL, [pymysql][pymysql] for MySQL and [pyodbc][pyodbc] for SQL Server.
 
 ---
 
@@ -85,7 +87,7 @@ values = [
 ]
 await database.execute_many(query=query, values=values)
 
-# Run a database query.
+# Run a database query.
 query = "SELECT * FROM HighScores"
 rows = await database.fetch_all(query=query)
 print('High Scores:', rows)
@@ -103,16 +105,18 @@ for examples of how to start using databases together with SQLAlchemy core expre
 [alembic]: https://alembic.sqlalchemy.org/en/latest/
 [psycopg2]: https://www.psycopg.org/
 [pymysql]: https://github.com/PyMySQL/PyMySQL
+[pyodbc]: https://github.com/mkleehammer/pyodbc
 [asyncpg]: https://github.com/MagicStack/asyncpg
 [aiopg]: https://github.com/aio-libs/aiopg
 [aiomysql]: https://github.com/aio-libs/aiomysql
 [asyncmy]: https://github.com/long2ice/asyncmy
 [aiosqlite]: https://github.com/omnilib/aiosqlite
+[aioodbc]: https://aioodbc.readthedocs.io/en/latest/
 
 [starlette]: https://github.com/encode/starlette
 [sanic]: https://github.com/huge-success/sanic
 [responder]: https://github.com/kennethreitz/responder
 [quart]: https://gitlab.com/pgjones/quart
 [aiohttp]: https://github.com/aio-libs/aiohttp
 [tornado]: https://github.com/tornadoweb/tornado
-[fastapi]: https://github.com/tiangolo/fastapi
+[fastapi]: https://github.com/tiangolo/fastapi

databases/backends/aiopg.py

@@ -5,19 +5,20 @@
 import uuid
 
 import aiopg
-from aiopg.sa.engine import APGCompiler_psycopg2
-from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
 from sqlalchemy.engine.cursor import CursorResultMetaData
 from sqlalchemy.engine.interfaces import Dialect, ExecutionContext
 from sqlalchemy.engine.row import Row
 from sqlalchemy.sql import ClauseElement
 from sqlalchemy.sql.ddl import DDLElement
 
-from databases.core import DatabaseURL
+from databases.backends.common.records import Record, Row, create_column_maps
+from databases.backends.compilers.psycopg import PGCompiler_psycopg
+from databases.backends.dialects.psycopg import PGDialect_psycopg
+from databases.core import LOG_EXTRA, DatabaseURL
 from databases.interfaces import (
     ConnectionBackend,
     DatabaseBackend,
-    Record,
+    Record as RecordInterface,
     TransactionBackend,
 )
 
@@ -34,10 +35,10 @@ def __init__(
         self._pool: typing.Union[aiopg.Pool, None] = None
 
     def _get_dialect(self) -> Dialect:
-        dialect = PGDialect_psycopg2(
+        dialect = PGDialect_psycopg(
             json_serializer=json.dumps, json_deserializer=lambda x: x
         )
-        dialect.statement_compiler = APGCompiler_psycopg2
+        dialect.statement_compiler = PGCompiler_psycopg
         dialect.implicit_returning = True
         dialect.supports_native_enum = True
         dialect.supports_smallserial = True  # 9.2+
@@ -117,15 +118,18 @@ async def release(self) -> None:
         await self._database._pool.release(self._connection)
         self._connection = None
 
-    async def fetch_all(self, query: ClauseElement) -> typing.List[Record]:
+    async def fetch_all(self, query: ClauseElement) -> typing.List[RecordInterface]:
         assert self._connection is not None, "Connection is not acquired"
-        query_str, args, context = self._compile(query)
+        query_str, args, result_columns, context = self._compile(query)
+        column_maps = create_column_maps(result_columns)
+        dialect = self._dialect
+
         cursor = await self._connection.cursor()
         try:
             await cursor.execute(query_str, args)
             rows = await cursor.fetchall()
             metadata = CursorResultMetaData(context, cursor.description)
-            return [
+            rows = [
                 Row(
                     metadata,
                     metadata._processors,
@@ -135,32 +139,36 @@ async def fetch_all(self, query: ClauseElement) -> typing.List[Record]:
                 )
                 for row in rows
             ]
+            return [Record(row, result_columns, dialect, column_maps) for row in rows]
         finally:
             cursor.close()
 
-    async def fetch_one(self, query: ClauseElement) -> typing.Optional[Record]:
+    async def fetch_one(self, query: ClauseElement) -> typing.Optional[RecordInterface]:
         assert self._connection is not None, "Connection is not acquired"
-        query_str, args, context = self._compile(query)
+        query_str, args, result_columns, context = self._compile(query)
+        column_maps = create_column_maps(result_columns)
+        dialect = self._dialect
         cursor = await self._connection.cursor()
         try:
             await cursor.execute(query_str, args)
             row = await cursor.fetchone()
             if row is None:
                 return None
             metadata = CursorResultMetaData(context, cursor.description)
-            return Row(
+            row = Row(
                 metadata,
                 metadata._processors,
                 metadata._keymap,
                 Row._default_key_style,
                 row,
             )
+            return Record(row, result_columns, dialect, column_maps)
         finally:
             cursor.close()
 
     async def execute(self, query: ClauseElement) -> typing.Any:
         assert self._connection is not None, "Connection is not acquired"
-        query_str, args, context = self._compile(query)
+        query_str, args, _, _ = self._compile(query)
         cursor = await self._connection.cursor()
         try:
             await cursor.execute(query_str, args)
@@ -173,7 +181,7 @@ async def execute_many(self, queries: typing.List[ClauseElement]) -> None:
         cursor = await self._connection.cursor()
         try:
             for single_query in queries:
-                single_query, args, context = self._compile(single_query)
+                single_query, args, _, _ = self._compile(single_query)
                 await cursor.execute(single_query, args)
         finally:
             cursor.close()
@@ -182,36 +190,38 @@ async def iterate(
         self, query: ClauseElement
     ) -> typing.AsyncGenerator[typing.Any, None]:
         assert self._connection is not None, "Connection is not acquired"
-        query_str, args, context = self._compile(query)
+        query_str, args, result_columns, context = self._compile(query)
+        column_maps = create_column_maps(result_columns)
+        dialect = self._dialect
         cursor = await self._connection.cursor()
         try:
             await cursor.execute(query_str, args)
             metadata = CursorResultMetaData(context, cursor.description)
             async for row in cursor:
-                yield Row(
+                record = Row(
                     metadata,
                     metadata._processors,
                     metadata._keymap,
                     Row._default_key_style,
                     row,
                 )
+                yield Record(record, result_columns, dialect, column_maps)
         finally:
             cursor.close()
 
     def transaction(self) -> TransactionBackend:
         return AiopgTransaction(self)
 
-    def _compile(
-        self, query: ClauseElement
-    ) -> typing.Tuple[str, dict, CompilationContext]:
+    def _compile(self, query: ClauseElement) -> typing.Tuple[str, list, tuple]:
         compiled = query.compile(
             dialect=self._dialect, compile_kwargs={"render_postcompile": True}
         )
-
         execution_context = self._dialect.execution_ctx_cls()
         execution_context.dialect = self._dialect
 
         if not isinstance(query, DDLElement):
+            compiled_params = sorted(compiled.params.items())
+
             args = compiled.construct_params()
             for key, val in args.items():
                 if key in compiled._bind_processors:
@@ -224,11 +234,23 @@ def _compile(
                 compiled._ad_hoc_textual,
                 compiled._loose_column_name_matching,
             )
+
+            mapping = {
+                key: "$" + str(i) for i, (key, _) in enumerate(compiled_params, start=1)
+            }
+            compiled_query = compiled.string % mapping
+            result_map = compiled._result_columns
+
         else:
             args = {}
+            result_map = None
+            compiled_query = compiled.string
 
-        logger.debug("Query: %s\nArgs: %s", compiled.string, args)
-        return compiled.string, args, CompilationContext(execution_context)
+        query_message = compiled_query.replace(" \n", " ").replace("\n", " ")
+        logger.debug(
+            "Query: %s Args: %s", query_message, repr(tuple(args)), extra=LOG_EXTRA
+        )
+        return compiled.string, args, result_map, CompilationContext(execution_context)
 
     @property
     def raw_connection(self) -> aiopg.connection.Connection: