Support SQL DATETIME2 and SQL TIME when supported by FreeTDS

CHANGELOG.md

@@ -6,6 +6,10 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Add FreeTDS 1.1 to test matrix.
 
+### Fixed
+- Preserve microsecond precision of _TIME_ and _DATETIME2_ SQL types when
+converting to Python (and supported by FreeTDS).
+
 ## [1.9.0] - 2018-11-05
 ### Added
 - Add support for passing sequences of `dict` values to

Makefile

@@ -23,9 +23,8 @@ CHECKED_FREETDS_VERSIONS := \
     0.91.112 \
     0.92.405 \
     0.95.95 \
-    1.00.55 \
     1.00.80 \
-	1.1
+	1.1.4
 
 # Valgrind FreeTDS versions are limited to one without sp_executesql support
 # and one with.

appveyor.yml

@@ -9,7 +9,7 @@ environment:
 
   BUILD_INSTALL_PREFIX: "C:\\usr\\local"
 
-  FREETDS_VERSION: "1.00.94"
+  FREETDS_VERSION: "1.1.4"
 
   matrix:
   - PYTHON: "C:\\Python27-x64"
@@ -38,13 +38,17 @@ environment:
   - PYTHON: "C:\\Python36-x64"
     SQLSERVER_INSTANCENAME: "SQL2016"
 
+  - PYTHON: "C:\\Python37-x64"
+    SQLSERVER_INSTANCENAME: "SQL2016"
+
 # Note: Database services should *not* be specified in the services section.
 # The database service will be explicitly started by the script which enables
 # TCP/IP connections for the database.
 services:
 
 cache:
   -  '%BUILD_INSTALL_PREFIX% -> appveyor\\install.ps1'
+  -  '%BUILD_INSTALL_PREFIX% -> appveyor.yml'
 
 install:
   - choco install opencppcoverage

appveyor/test_script.ps1

@@ -21,6 +21,8 @@ $env:PATH += ";$env:BUILD_INSTALL_PREFIX\lib"
 # The computer's hostname is returned in messages from SQL Server.
 $env:HOSTNAME = "$env:COMPUTERNAME"
 
+& "$env:PYTHON\python.exe" -c 'import ctds; print(ctds.freetds_version)'
+
 & "$env:ProgramFiles\OpenCppCoverage\OpenCppCoverage.exe" `
     --export_type=cobertura:cobertura.xml --optimized_build `
     --sources "$env:APPVEYOR_BUILD_FOLDER\src" `

scripts/ensure-sqlserver.sh

@@ -16,6 +16,13 @@ fi
 CONTAINER_ID=`docker ps -a -f name="^/$CONTAINER$" -q`
 if [ -z "$CONTAINER_ID" ]; then
     echo "MS SQL Server docker container not running; starting ..."
+
+    # Remove the container if it is stopped.
+    CONTAINER_ID=`docker ps -a -f name="^/$CONTAINER$" -q`
+    if [ -n "$CONTAINER_ID" ]; then
+        docker rm $CONTAINER_ID
+    fi
+
     CONTAINER_ID=`docker run -d \
            -e 'ACCEPT_EULA=Y' \
            -e 'SA_PASSWORD=cTDS-unitest123' \

src/ctds/include/tds.h

@@ -56,6 +56,10 @@ enum ParamStyle {
 #  define CTDS_HAVE_NTLMV2 1
 #endif
 
+#if defined(SYBMSTIME)
+#  define CTDS_HAVE_TDSTIME 1
+#endif
+
 /*
     Use `sp_executesql` when possible for the execute*() methods. This method
     won't work on older versions of FreeTDS which don't properly support passing

src/ctds/include/type.h

@@ -212,13 +212,15 @@ PyObject* encode_for_dblib(PyObject* unicode, const char** utf8bytes, size_t* nu
     Convert a Python datetime, date or time to a DBDATETIME.
 
     @note: This will silently discard the microsecond precision for Python's
-    `time` and `datetime` objects.
+    `time` and `datetime` objects if not supported by FreeTDS.
 
     @param o [in] The date, datetime or time object.
-    @param dbdatetime [out] The converted value.
+    @param tdstype [out] The TDS type of the convert value.
+    @param converted [out] The converted value.
+    @param cbconverted [in] The size of buffer, in bytes.
 
-    @retval 0 on success, -1 on error.
+    @retval bytes written to `converted` on success, -1 on error.
 */
-int datetime_to_sql(PyObject* o, DBDATETIME* dbdatetime);
+int datetime_to_sql(PyObject* o, enum TdsType* tdstype, void* converted, size_t cbconverted);
 
 #endif /* ifndef __TYPE_H__ */

src/ctds/parameter.c

@@ -66,7 +66,11 @@ struct Parameter
         DBINT dbint;
         DBBIGINT dbbigint;
         DBDECIMAL dbdecimal;
+#if defined(CTDS_HAVE_TDSTIME)
+        DBDATETIMEALL dbdatetime;
+#else /* if defined(CTDS_HAVE_TDSTIME) */
         DBDATETIME dbdatetime;
+#endif /* else if defined(CTDS_HAVE_TDSTIME) */
         DBFLT8 dbflt8;
     } buffer;
 
@@ -257,23 +261,14 @@ static int Parameter_bind(struct Parameter* parameter, PyObject* value)
         struct SqlType* sqltype = (struct SqlType*)value;
         parameter->input = sqltype->data;
         parameter->ninput = sqltype->ndata;
+        parameter->tdstypesize = (DBINT)sqltype->size;
+        parameter->tdstype = sqltype->tdstype;
 
-        switch (sqltype->tdstype)
+        if (TDSDATE == sqltype->tdstype)
         {
-            case TDSDATE:
-            {
-                /* FreeTDS 0.9.15 doesn't support TDSDATE properly. Fallback to DATETIME. */
-                parameter->tdstype = TDSDATETIME;
-                break;
-            }
-            default:
-            {
-                parameter->tdstype = sqltype->tdstype;
-                break;
-            }
+            /* FreeTDS doesn't support passing TDSDATE properly. Fallback to DATETIME. */
+            parameter->tdstype = TDSDATETIME;
         }
-
-        parameter->tdstypesize = (DBINT)sqltype->size;
     }
     else
     {
@@ -512,14 +507,17 @@ static int Parameter_bind(struct Parameter* parameter, PyObject* value)
             }
             else if (PyDate_Check_(value) || PyTime_Check_(value))
             {
-                if (-1 == datetime_to_sql(value, &parameter->buffer.dbdatetime))
+                int ninput = datetime_to_sql(value,
+                                             &parameter->tdstype,
+                                             &parameter->buffer.dbdatetime,
+                                             sizeof(parameter->buffer.dbdatetime));
+                if (-1 == ninput)
                 {
                     PyErr_Format(PyExc_tds_InterfaceError, "failed to convert datetime");
                     break;
                 }
 
-                parameter->ninput = sizeof(parameter->buffer.dbdatetime);
-                parameter->tdstype = TDSDATETIME;
+                parameter->ninput = (size_t)ninput;
                 parameter->input = (const void*)&parameter->buffer;
             }
             else if (PyUuid_Check(value))
@@ -637,7 +635,11 @@ struct Parameter* Parameter_create(PyObject* value, bool output)
                         case TDSTIME:
                         case TDSDATETIME2:
                         {
+#if defined(CTDS_HAVE_TDSTIME)
+                            parameter->noutput = sizeof(DBDATETIMEALL);
+#else /* if defined(CTDS_HAVE_TDSTIME) */
                             parameter->noutput = sizeof(DBDATETIME);
+#endif /* else if defined(CTDS_HAVE_TDSTIME) */
                             break;
                         }
                         case TDSSMALLMONEY:
@@ -838,6 +840,7 @@ char* Parameter_sqltype(struct Parameter* rpcparam, bool maximum_width)
 
         case TDSDATETIMEN:
         CONST_CASE(DATETIME)
+        CONST_CASE(DATETIME2)
         CONST_CASE(SMALLDATETIME)
         CONST_CASE(DATE)
         CONST_CASE(TIME)

src/ctds/type.c

@@ -699,7 +699,11 @@ SQL_TYPE_DEF(NVarChar, s_SqlNVarChar_doc);
 struct SqlDate
 {
     SqlType_HEAD;
+#if defined(CTDS_HAVE_TDSTIME)
+    DBDATETIMEALL dbdatetime;
+#else /* if defined(CTDS_HAVE_TDSTIME) */
     DBDATETIME dbdatetime;
+#endif /* else if defined(CTDS_HAVE_TDSTIME) */
 };
 
 static const char s_SqlDate_doc[] =
@@ -727,7 +731,11 @@ static int SqlDate_init(PyObject* self, PyObject* args, PyObject* kwargs)
 
     if (PyDate_Check_(value))
     {
-        DBINT result = datetime_to_sql(value, &date->dbdatetime);
+        enum TdsType unused = (enum TdsType)-1;
+        DBINT result = datetime_to_sql(value,
+                                       &unused,
+                                       &date->dbdatetime,
+                                       sizeof(date->dbdatetime));
         if (-1 == result)
         {
             PyErr_SetObject(PyExc_ValueError, value);
@@ -1024,7 +1032,7 @@ static PyObject* DATETIME_topython(enum TdsType tdstype, const void* data, size_
             DBINT size = dbconvert(NULL,
                                    tdstype,
                                    data,
-                                   -1,
+                                   (DBINT)ndata,
                                    SYBDATETIME,
                                    (BYTE*)&dbdatetime, -1);
             if (-1 == size)
@@ -1038,11 +1046,26 @@ static PyObject* DATETIME_topython(enum TdsType tdstype, const void* data, size_
 
             /* Intentional fall-through. */
         }
+#if defined(CTDS_HAVE_TDSTIME)
+        case TDSDATE:
+        case TDSTIME:
+        case TDSDATETIME2:
+        case TDSSMALLDATETIME:
+#endif /* if defined(CTDS_HAVE_TDSTIME) */
         case TDSDATETIME:
         case TDSDATETIMEN:
         {
+            int usecond;
+#if defined(CTDS_HAVE_TDSTIME)
+            DBDATEREC2 dbdaterec;
+            (void)dbanydatecrack(NULL, &dbdaterec, tdstype, data);
+
+            usecond = dbdaterec.nanosecond / 1000;
+#else /* if defined(CTDS_HAVE_TDSTIME) */
             DBDATEREC dbdaterec;
             (void)dbdatecrack(NULL, &dbdaterec, (DBDATETIME*)data);
+            usecond = dbdaterec.millisecond * 1000;
+#endif /* else if defined(CTDS_HAVE_TDSTIME) */
 
             /*
                 If freetds was not compiled with MSDBLIB defined, the month,
@@ -1067,7 +1090,7 @@ static PyObject* DATETIME_topython(enum TdsType tdstype, const void* data, size_
                     return PyTime_FromTime_(dbdaterec.hour,
                                             dbdaterec.minute,
                                             dbdaterec.second,
-                                            dbdaterec.millisecond * 1000);
+                                            usecond);
                 }
                 default:
                 {
@@ -1077,7 +1100,7 @@ static PyObject* DATETIME_topython(enum TdsType tdstype, const void* data, size_
                                                        dbdaterec.hour,
                                                        dbdaterec.minute,
                                                        dbdaterec.second,
-                                                       dbdaterec.millisecond * 1000);
+                                                       usecond);
                 }
             }
             break;
@@ -1346,11 +1369,17 @@ PyObject* encode_for_dblib(PyObject* unicode, const char** utf8bytes, size_t* nu
     return encoded;
 }
 
-int datetime_to_sql(PyObject* o, DBDATETIME* dbdatetime)
+int datetime_to_sql(PyObject* o, enum TdsType* tdstype, void* converted, size_t cbconverted)
 {
-    DBINT size;
     int written = 0;
-    char buffer[ARRAYSIZE("YYYY-MM-DD HH:MM:SS.nnn")];
+    /* Python only supports microsecond precision. */
+    char buffer[ARRAYSIZE("YYYY-MM-DD HH:MM:SS.nnnnnn")];
+
+    /*
+        The best _supported_ TDS type. Default to DATETIME which is widely
+        supported across TDS and FreeTDS versions.
+    */
+    *tdstype = TDSDATETIME;
 
     if (PyDate_Check_(o))
     {
@@ -1383,20 +1412,31 @@ int datetime_to_sql(PyObject* o, DBDATETIME* dbdatetime)
 
         if (useconds)
         {
+#if defined(CTDS_HAVE_TDSTIME)
+            written += sprintf(&buffer[written], ".%06d", useconds);
+            /* Always use DATETIME2 to preserve fractional second precision. */
+            *tdstype = (PyDateTime_Check_(o)) ? TDSDATETIME2 : TDSTIME;
+#else /* if defined(CTDS_HAVE_TDSTIME) */
             /*
                 For compatibility with the MS SQL DATETIME type, only include
                 microsecond granularity.
             */
             written += sprintf(&buffer[written], ".%03d", useconds / 1000);
+            *tdstype = TDSDATETIME;
+#endif /* else if defined(CTDS_HAVE_TDSTIME) */
         }
+#if defined(CTDS_HAVE_TDSTIME)
+        else
+        {
+            *tdstype = (PyDateTime_Check_(o)) ? TDSDATETIME : TDSTIME;
+        }
+#endif /* if defined(CTDS_HAVE_TDSTIME) */
     }
-    size = dbconvert(NULL,
-                     TDSCHAR,
-                     (const BYTE*)buffer,
-                     (DBINT)written,
-                     TDSDATETIME,
-                     (BYTE*)dbdatetime,
-                     -1);
-
-    return (-1 == size) ? -1 : 0;
+    return (int)dbconvert(NULL,
+                          TDSCHAR,
+                          (const BYTE*)buffer,
+                          (DBINT)written,
+                          *tdstype,
+                          (BYTE*)converted,
+                          (DBINT)cbconverted);
 }

tests/base.py

@@ -158,6 +158,14 @@ def have_valid_rowcount(self):
         # FreeTDS 1.1+ properly returns rowcount, even when calling sp_executesql.
         return self.freetds_version >= (1, 1, 0) or not self.use_sp_executesql
 
+    @property
+    def tdstime_supported(self):
+        return self.freetds_version >= (0, 95, 0)
+
+    @property
+    def tdsdatetime2_supported(self):
+        return self.freetds_version >= (0, 95, 0)
+
     # Older versions of FreeTDS improperly round the money to the nearest hundredth.
     def round_money(self, money):
         if self.freetds_version > (0, 92, 405): # pragma: nobranch