Formatting, unit tests, and ArrayGO updates

Author: brandtbucher

arraykit.c

@@ -8,7 +8,8 @@
 
 // Bug in NumPy < 1.16 (https://github.com/numpy/numpy/pull/12131):
 # undef PyDataType_ISBOOL
-# define PyDataType_ISBOOL(obj) PyTypeNum_ISBOOL(((PyArray_Descr*)(obj))->type_num)
+# define PyDataType_ISBOOL(obj) \
+    PyTypeNum_ISBOOL(((PyArray_Descr*)(obj))->type_num)
 
 # define AK_CHECK_NUMPY_ARRAY(O)                                              \
     if (!PyArray_Check(O)) {                                                  \
@@ -26,9 +27,8 @@
 
 typedef struct {
     PyObject_VAR_HEAD
-    PyArrayObject *array;
+    PyObject *array;
     PyObject *list;
-    PyArray_Descr* dtype;
 } ArrayGOObject;
 
 static PyTypeObject ArrayGOType;
@@ -172,17 +172,17 @@ update_array_cache(ArrayGOObject *self)
 {
     if (self->list) {
         if (self->array) {
-            PyObject *container = PyTuple_Pack(2, (PyObject *)self->array, self->list);
+            PyObject *container = PyTuple_Pack(2, self->array, self->list);
             if (!container) {
                 return -1;
             }
-            Py_SETREF(self->array, (PyArrayObject *)PyArray_Concatenate(container, 0));
+            Py_SETREF(self->array, PyArray_Concatenate(container, 0));
             Py_DECREF(container);
         }
         else {
-            self->array = (PyArrayObject *) PyArray_FROM_OT(self->list, self->dtype->type_num);
+            self->array = PyArray_FROM_OT(self->list, NPY_OBJECT);
         }
-        PyArray_CLEARFLAGS(self->array, NPY_ARRAY_WRITEABLE);
+        PyArray_CLEARFLAGS((PyArrayObject *)self->array, NPY_ARRAY_WRITEABLE);
         Py_CLEAR(self->list);
     }
     return 0;
@@ -194,41 +194,30 @@ update_array_cache(ArrayGOObject *self)
 static int
 ArrayGO_init(ArrayGOObject *self, PyObject *args, PyObject *kwargs)
 {
-
     PyObject *temp;
+    char* argnames[] = {"iterable", "own_iterable", NULL};
     PyObject *iterable;
     int own_iterable;
-    int parsed;
-
-    char* argnames[] = {"iterable", "dtype", "own_iterable", NULL};
-
-    Py_CLEAR(self->dtype);
-    parsed = PyArg_ParseTupleAndKeywords(
-        args, kwargs, "O|$O&p:ArrayGO", argnames,
-        &iterable, PyArray_DescrConverter, &self->dtype, &own_iterable
+    int parsed = PyArg_ParseTupleAndKeywords(
+        args, kwargs, "O|$p:ArrayGO", argnames, &iterable, &own_iterable
     );
     if (!parsed) {
         return -1;
     }
-    if (!self->dtype) {
-        self->dtype = PyArray_DescrFromType(NPY_OBJECT);
-    }
     if (PyArray_Check(iterable)) {
-        temp = (PyObject *) self->array;
+        temp = self->array;
         if (own_iterable) {
             PyArray_CLEARFLAGS((PyArrayObject *) iterable, NPY_ARRAY_WRITEABLE);
             Py_INCREF(iterable);
         } else {
             iterable = (PyObject *)AK_ImmutableFilter((PyArrayObject *) iterable);
         }
-        if (!PyArray_EquivTypes(PyArray_DESCR((PyArrayObject *) iterable), self->dtype)) {
-            PyErr_Format(
-                PyExc_TypeError, "bad dtype given to ArrayGO initializer (expected '%S', got '%S')",
-                PyArray_DESCR((PyArrayObject *) iterable), self->dtype
-            );
+        if (!PyDataType_ISOBJECT(PyArray_DESCR((PyArrayObject *)iterable))) {
+            PyErr_SetString(PyExc_NotImplementedError,
+                            "only object arrays are supported");
             return -1;
         }
-        self->array = (PyArrayObject *) iterable;
+        self->array = iterable;
         Py_XDECREF(temp);
 
     } else {
@@ -291,9 +280,7 @@ ArrayGO_copy(ArrayGOObject *self, PyObject *Py_UNUSED(unused))
     ArrayGOObject *copy = PyObject_New(ArrayGOObject, &ArrayGOType);
     copy->array = self->array;
     copy->list = self->list ? PySequence_List(self->list) : NULL;
-    copy->dtype = self->dtype;
     Py_XINCREF(copy->array);
-    Py_INCREF(copy->dtype);
     return (PyObject *)copy;
 }
 
@@ -303,7 +290,7 @@ ArrayGO_iter(ArrayGOObject *self)
     if (self->list && update_array_cache(self)) {
         return NULL;
     }
-    return PyObject_GetIter((PyObject *)self->array);
+    return PyObject_GetIter(self->array);
 }
 
 static PyObject *
@@ -312,13 +299,13 @@ ArrayGO_mp_subscript(ArrayGOObject *self, PyObject *key)
     if (self->list && update_array_cache(self)) {
         return NULL;
     }
-    return PyObject_GetItem((PyObject *)self->array, key);
+    return PyObject_GetItem(self->array, key);
 }
 
 static Py_ssize_t
 ArrayGO_mp_length(ArrayGOObject *self)
 {
-    return ((self->array ? PyArray_SIZE(self->array) : 0)
+    return ((self->array ? PyArray_SIZE((PyArrayObject *)self->array) : 0)
             + (self->list ? PyList_Size(self->list) : 0));
 }
 
@@ -329,13 +316,12 @@ ArrayGO_values_getter(ArrayGOObject *self, void* Py_UNUSED(closure))
         return NULL;
     }
     Py_INCREF(self->array);
-    return (PyObject *)self->array;
+    return self->array;
 }
 
 static void
 ArrayGO_dealloc(ArrayGOObject *self)
 {
-    Py_XDECREF(self->dtype);
     Py_XDECREF(self->array);
     Py_XDECREF(self->list);
     Py_TYPE(self)->tp_free((PyObject *)self);

arraykit.pyi

@@ -1,38 +1,24 @@
-# pylint: disable = all
-
 import typing
 
 import numpy  # type: ignore
 
-
 _T = typing.TypeVar('_T')
 
-
 class ArrayGO:
 
     values: numpy.array
-
-    def __init__(self, iterable: typing.Iterable[object], *, dtype: object = ..., own_iterable: bool = ...) -> None: ...
-
+    def __init__(
+        self, iterable: typing.Iterable[object], *, own_iterable: bool = ...
+    ) -> None: ...
     def __iter__(self) -> typing.Iterator[typing.Any]: ...
-
     def __getitem__(self, __key: object) -> typing.Any: ...
-
     def __len__(self) -> int: ...
-
     def append(self, __value: object) -> None: ...
-
     def copy(self: _T) -> _T: ...
-
     def extend(self, __values: typing.Iterable[object]) -> None: ...
 
-
 def immutable_filter(__array: numpy.array) -> numpy.array: ...
-
 def mloc(__array: numpy.array) -> int: ...
-
 def name_filter(__name: typing.Hashable) -> typing.Hashable: ...
-
 def resolve_dtype(__d1: numpy.dtype, __d2: numpy.dtype) -> numpy.dtype: ...
-
 def resolve_dtype_iter(__dtypes: typing.Iterable[numpy.dtype]) -> numpy.dtype: ...

setup.py

@@ -16,6 +16,7 @@ def get_long_description() -> str:
 Packages: https://pypi.org/project/arraykit
 '''
 
+
 setup(
     name='arraykit',
     version=AK_VERSION,
@@ -28,16 +29,16 @@ def get_long_description() -> str:
     license='MIT',
     # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
-            'Development Status :: 3 - Alpha',
-            'Intended Audience :: Developers',
-            'Topic :: Software Development',
-            'License :: OSI Approved :: MIT License',
-            'Operating System :: MacOS :: MacOS X',
-            'Operating System :: Microsoft :: Windows',
-            'Operating System :: POSIX',
-            'Programming Language :: Python :: 3.6',
-            'Programming Language :: Python :: 3.7',
-            'Programming Language :: Python :: 3.8',
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'Topic :: Software Development',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: MacOS :: MacOS X',
+        'Operating System :: Microsoft :: Windows',
+        'Operating System :: POSIX',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
     ],
     keywords='numpy array',
     packages=[],

tasks.py

@@ -4,7 +4,8 @@
 
 import invoke
 
-#-------------------------------------------------------------------------------
+# -------------------------------------------------------------------------------
+
 
 @invoke.task
 def clean(context):

test_property.py

@@ -0,0 +1,112 @@
+import pytest  # type: ignore
+import numpy as np  # type: ignore
+
+import arraykit as ak
+
+
+def test_array_init_a() -> None:
+    with pytest.raises(NotImplementedError):
+        ak.ArrayGO(np.array((3, 4, 5)))
+
+
+def test_array_append_a() -> None:
+    ag1 = ak.ArrayGO(('a', 'b', 'c', 'd'))
+    assert [*ag1] == ['a', 'b', 'c', 'd']
+    assert ag1.values.tolist() == ['a', 'b', 'c', 'd']
+    ag1.append('e')
+    ag1.extend(('f', 'g'))
+    assert [*ag1] == ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+    assert ag1.values.tolist() == ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+
+
+def test_array_append_b() -> None:
+    ag1 = ak.ArrayGO(np.array(('a', 'b', 'c', 'd'), object))
+    assert [*ag1] == ['a', 'b', 'c', 'd']
+    assert ag1.values.tolist() == ['a', 'b', 'c', 'd']
+    ag1.append('e')
+    ag1.extend(('f', 'g'))
+    assert [*ag1] == ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+    assert ag1.values.tolist() == ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+
+
+def test_array_getitem_a() -> None:
+    a = np.array(('a', 'b', 'c', 'd'), object)
+    a.flags.writeable = False
+    ag1 = ak.ArrayGO(a)
+    # Ensure no copy for immutable:
+    assert ak.mloc(ag1.values) == ak.mloc(a)
+    ag1.append('b')
+    post = ag1[ag1.values == 'b']
+    assert post.tolist() == ['b', 'b']
+    assert ag1[[2, 1, 1, 1]].tolist() == ['c', 'b', 'b', 'b']
+
+
+def test_array_copy_a() -> None:
+    ag1 = ak.ArrayGO(np.array(('a', 'b', 'c', 'd'), dtype=object))
+    ag1.append('e')
+    ag2 = ag1.copy()
+    ag1.extend(('f', 'g'))
+    assert ag1.values.tolist() == ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+    assert ag2.values.tolist() == ['a', 'b', 'c', 'd', 'e']
+
+
+def test_array_len_a() -> None:
+    ag1 = ak.ArrayGO(np.array(('a', 'b', 'c', 'd'), object))
+    ag1.append('e')
+    assert len(ag1) == 5
+
+
+def test_resolve_dtype_a() -> None:
+    a1 = np.array([1, 2, 3])
+    a2 = np.array([False, True, False])
+    a3 = np.array(['b', 'c', 'd'])
+    a4 = np.array([2.3, 3.2])
+    a5 = np.array(['test', 'test again'], dtype='S')
+    a6 = np.array([2.3, 5.4], dtype='float32')
+    assert ak.resolve_dtype(a1.dtype, a1.dtype) == a1.dtype
+    assert ak.resolve_dtype(a1.dtype, a2.dtype) == np.object_
+    assert ak.resolve_dtype(a2.dtype, a3.dtype) == np.object_
+    assert ak.resolve_dtype(a2.dtype, a4.dtype) == np.object_
+    assert ak.resolve_dtype(a3.dtype, a4.dtype) == np.object_
+    assert ak.resolve_dtype(a3.dtype, a6.dtype) == np.object_
+    assert ak.resolve_dtype(a1.dtype, a4.dtype) == np.float64
+    assert ak.resolve_dtype(a1.dtype, a6.dtype) == np.float64
+    assert ak.resolve_dtype(a4.dtype, a6.dtype) == np.float64
+
+
+def test_resolve_dtype_b() -> None:
+    a1 = np.array('a').dtype
+    a3 = np.array('aaa').dtype
+    assert ak.resolve_dtype(a1, a3) == np.dtype(('U', 3))
+
+
+def test_resolve_dtype_c() -> None:
+    a1 = np.array(['2019-01', '2019-02'], dtype=np.datetime64)
+    a2 = np.array(['2019-01-01', '2019-02-01'], dtype=np.datetime64)
+    a3 = np.array([0, 1], dtype='datetime64[ns]')
+    a4 = np.array([0, 1])
+    assert str(ak.resolve_dtype(a1.dtype, a2.dtype)) == 'datetime64[D]'
+    assert ak.resolve_dtype(a1.dtype, a3.dtype) == np.dtype('<M8[ns]')
+    assert ak.resolve_dtype(a1.dtype, a4.dtype) == np.dtype('O')
+
+
+def test_resolve_dtype_iter_a() -> None:
+    a1 = np.array([1, 2, 3])
+    a2 = np.array([False, True, False])
+    a3 = np.array(['b', 'c', 'd'])
+    a4 = np.array([2.3, 3.2])
+    a5 = np.array(['test', 'test again'], dtype='S')
+    a6 = np.array([2.3, 5.4], dtype='float32')
+    assert ak.resolve_dtype_iter((a1.dtype, a1.dtype)) == a1.dtype
+    assert ak.resolve_dtype_iter((a2.dtype, a2.dtype)) == a2.dtype
+    # Boolean with mixed types:
+    assert ak.resolve_dtype_iter((a2.dtype, a2.dtype, a3.dtype)) == np.object_
+    assert ak.resolve_dtype_iter((a2.dtype, a2.dtype, a5.dtype)) == np.object_
+    assert ak.resolve_dtype_iter((a2.dtype, a2.dtype, a6.dtype)) == np.object_
+    # Numeric types go to float64:
+    assert ak.resolve_dtype_iter((a1.dtype, a4.dtype, a6.dtype)) == np.float64
+    # Add in bool or str, goes to object:
+    assert ak.resolve_dtype_iter((a1.dtype, a4.dtype, a6.dtype, a2.dtype)) == np.object_
+    assert ak.resolve_dtype_iter((a1.dtype, a4.dtype, a6.dtype, a5.dtype)) == np.object_
+    # Mixed strings go to the largest:
+    assert ak.resolve_dtype_iter((a3.dtype, a5.dtype)) == np.dtype('<U10')