Merge pull request #92 from static-frame/91/dt64-seg-fault

segmentation fault on bad datetime64 input

Author: flexatone

src/_arraykit.c

@@ -1924,25 +1924,35 @@ AK_CPL_to_array_bytes(AK_CodePointLine* cpl, PyArray_Descr* dtype)
     return array;
 }
 
-// If we cannot direclty convert bytes to values, create a bytes array and then use PyArray_CastToType to use numpy to interpet it as a new a array.
+// If we cannot directly convert bytes to values in a pre-loadsed array, we can create a bytes or unicode array and then use PyArray_CastToType to use numpy to interpret it as a new a array and handle conversions. Note that we can use bytes for a smaller memory load if we are confident that the values are not unicode. This is a safe assumption for complext. Fore datetim64, we have to use Unicode to get errors on malformed inputs: using bytes causes a seg fault with these interfaces (the same is not observed with astyping a byte array in Python).
 static inline PyObject*
-AK_CPL_to_array_via_cast(AK_CodePointLine* cpl, PyArray_Descr* dtype)
+AK_CPL_to_array_via_cast(AK_CodePointLine* cpl,
+        PyArray_Descr* dtype,
+        int type_inter)
 {
-    PyArray_Descr *dtype_bytes = PyArray_DescrNewFromType(NPY_STRING);
-    if (dtype_bytes == NULL) {
+    PyArray_Descr* dtype_inter; // interchange array
+    PyObject* array_inter = NULL;
+
+    dtype_inter = PyArray_DescrNewFromType(type_inter);
+    if (dtype_inter == NULL) {
         Py_DECREF(dtype);
         return NULL;
     }
-    PyObject* array_bytes = AK_CPL_to_array_bytes(cpl, dtype_bytes);
-    if (array_bytes == NULL) {
-        Py_DECREF(dtype);
-        // dtype_bytes stolen even if array creation failed
+    if (type_inter == NPY_STRING) {
+        array_inter = AK_CPL_to_array_bytes(cpl, dtype_inter);
+    }
+    else if (type_inter == NPY_UNICODE) {
+        array_inter = AK_CPL_to_array_unicode(cpl, dtype_inter);
+    }
+
+    if (array_inter == NULL) {
+        Py_DECREF(dtype); // dtype_inter ref already stolen
         return NULL;
     }
-    PyObject *array = PyArray_CastToType((PyArrayObject*)array_bytes, dtype, 0);
-    Py_DECREF(array_bytes);
-    if (array == NULL) {
-        // expected array to steal dtype reference
+
+    PyObject *array = PyArray_CastToType((PyArrayObject*)array_inter, dtype, 0);
+    Py_DECREF(array_inter);
+    if (array == NULL) { // dtype ref already stolen
         return NULL;
     }
     PyArray_CLEARFLAGS((PyArrayObject *)array, NPY_ARRAY_WRITEABLE);
@@ -1983,10 +1993,11 @@ AK_CPL_ToArray(AK_CodePointLine* cpl, PyArray_Descr* dtype, char tsep, char decc
         return AK_CPL_to_array_int(cpl, dtype, tsep);
     }
     else if (PyDataType_ISDATETIME(dtype)) {
-        return AK_CPL_to_array_via_cast(cpl, dtype);
+        return AK_CPL_to_array_via_cast(cpl, dtype, NPY_UNICODE);
     }
     else if (PyDataType_ISCOMPLEX(dtype)) {
-        return AK_CPL_to_array_via_cast(cpl, dtype); // no tsep, decc as using NumPy cast
+        // no tsep, decc as using NumPy cast
+        return AK_CPL_to_array_via_cast(cpl, dtype, NPY_STRING);
     }
 
     PyErr_Format(PyExc_NotImplementedError, "No handling for %R", dtype);

test/test_delimited_to_arrays.py

@@ -370,10 +370,14 @@ def test_iterable_str_to_array_1d_complex_4(self) -> None:
         self.assertEqual(a1.dtype, np.dtype(complex))
         self.assertEqual(a1.tolist(), [complex('-0+infj'), (0j)])
 
-    # NOTE: this causes a seg fault
-    # def test_iterable_str_to_array_1d_d4(self) -> None:
-    #     with self.assertRaises(ValueError):
-    #         a1 = iterable_str_to_array_1d(['-2+1.2j', '1.5+-4.2j'], complex)
+    def test_iterable_str_to_array_1d_complex_5(self) -> None:
+        with self.assertRaises(ValueError):
+            a1 = iterable_str_to_array_1d(['-2+1.2j', '1.5+-4.2j'], complex)
+
+    def test_iterable_str_to_array_1d_complex_6(self) -> None:
+        # NOTE: malformed complex raise Exception as expected
+        with self.assertRaises(ValueError):
+            a1 = iterable_str_to_array_1d(['-2+1.2asdfj', '1.5wer4.2j'], complex)
 
     #---------------------------------------------------------------------------
 
@@ -389,12 +393,16 @@ def test_iterable_str_to_array_1d_dt64_2(self) -> None:
         self.assertFalse(a1.flags.writeable)
         self.assertEqual(a1.tolist(), [datetime.date(2020, 1, 1), datetime.date(2020, 2, 1)])
 
-    def test_iterable_str_to_array_1d_dt64_2(self) -> None:
+    def test_iterable_str_to_array_1d_dt64_3(self) -> None:
         a1 = iterable_str_to_array_1d(['2020-01-01', '2020-02-01'], np.datetime64)
         self.assertEqual(a1.dtype, np.dtype('<M8[D]'))
         self.assertFalse(a1.flags.writeable)
         self.assertEqual(a1.tolist(), [datetime.date(2020, 1, 1), datetime.date(2020, 2, 1)])
 
+    def test_iterable_str_to_array_1d_dt64_4(self) -> None:
+        with self.assertRaises(ValueError):
+            _ = iterable_str_to_array_1d(['202.30', '202.20'], 'datetime64[D]')
+
     #---------------------------------------------------------------------------
 
     def test_iterable_str_to_array_1d_parse_1(self) -> None: