DE-9920 python 3.11 compatibility (#363)

* Update github actions and ubuntu

* Move 3.11 upgrade to new PR

* Intermediate comparison commit

* Final formatting

* Run test-postgres and lint action in 3.11

* Treat  type as length zero

* Try bumping spark version to 3.11 compatible one

* Remove unnnecessary list comp

* Add more context to Tuple length comment

* Revert to previous redun behavior

* Revert unit test change

.github/workflows/test_pull_request.yml

@@ -9,7 +9,7 @@ jobs:
         # Pinned to ubuntu-22.04 as that is the last version to include Python 3.8 support.
         # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md#python
         os: [ubuntu-22.04]
-        python: ['3.8', '3.9', '3.10']
+        python: ['3.8', '3.9', '3.10', '3.11']
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -27,7 +27,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - run: sudo apt-get install graphviz graphviz-dev
       - run: make test-postgres
   lint:
@@ -37,6 +37,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - run: sudo apt-get install graphviz graphviz-dev
       - run: make lint

docs/source/spark.md

@@ -105,7 +105,7 @@ def do_stuff():
 ### Accessing Spark contexts
 
 The `redun.glue` module contains helper functions to access the Spark Session and Context objects:
-`get_spark_context()` and `get_spark_session()`. [Full Documentation](spark.md)
+`get_spark_context()` and `get_spark_session()`.
 
 ### User-defined functions
 

redun/sphinx.py

@@ -31,10 +31,10 @@
 # This technique is inspired by:
 # https://docs.celeryq.dev/en/latest/_modules/celery/contrib/sphinx.html
 
-from inspect import formatargspec, getfullargspec
+from inspect import getfullargspec, signature
 from typing import Any, List, Optional
 
-from docutils import nodes  # type: ignore
+from docutils import nodes  # type: ignore[import]
 from sphinx.domains.python import PyFunction
 from sphinx.ext.autodoc import FunctionDocumenter
 from sphinx.util.docstrings import prepare_docstring
@@ -60,10 +60,7 @@ def format_args(self, **kwargs: Any) -> str:
         """
         wrapped = self.object.func
         if wrapped is not None:
-            argspec = getfullargspec(wrapped)
-            fmt = formatargspec(*argspec)
-            fmt = fmt.replace("\\", "\\\\")
-            return fmt
+            return str(signature(wrapped))
         return ""
 
     def get_doc(self, ignore: int = None) -> List[List[str]]:
@@ -119,16 +116,16 @@ def format_args(self, **kwargs: Any) -> str:
         """
         wrapped = self.object.func
         if wrapped is not None:
-            argspec = getfullargspec(wrapped)
+            sig = signature(wrapped)
+
             # Remove SchedulerTask-specific internal parameters.
-            del argspec.args[:3]
+            params = list(sig.parameters.values())[3:]
 
             # Unwrap the Promise return value.
-            argspec.annotations["return"] = argspec.annotations["return"].__parameters__[0]
+            argspec = getfullargspec(wrapped)
+            return_annotation = argspec.annotations["return"].__parameters__[0]
 
-            fmt = formatargspec(*argspec)
-            fmt = fmt.replace("\\", "\\\\")
-            return fmt
+            return str(sig.replace(parameters=params, return_annotation=return_annotation))
         return ""
 
 

redun/task.py

@@ -103,12 +103,18 @@ def get_tuple_type_length(tuple_type: Any) -> Optional[int]:
         else:
             tuple_type_args = tuple_type.__args__
 
-        if Ellipsis in tuple_type_args or len(tuple_type_args) == 0:
+        if Ellipsis in tuple_type_args or tuple_type is Tuple:
             # Tuple of unknown length.
             return None
-
-        if tuple_type_args == ((),):
+        elif tuple_type_args == ((),):
             # Special case for length zero.
+            # Note that the representation of empty tuple types changed in python 3.11
+            # Specifically: get_args(Tuple[()]) now evaluates to () instead of ((),)
+            # which is the same value that get_args(Tuple) evaluates to.
+            # https://docs.python.org/3/whatsnew/3.11.html
+            #
+            # Because of this we need to check for the type `Tuple` above before entering
+            # this branch.
             return 0
 
         return len(tuple_type_args)

redun/tests/test_tasks.py

@@ -459,8 +459,8 @@ def test_tuple_type_length() -> None:
     assert get_tuple_type_length(Tuple[int, int]) == 2
     assert get_tuple_type_length(Tuple[int, int, int]) == 3
     assert get_tuple_type_length(Tuple[()]) == 0
-    assert get_tuple_type_length(Tuple[Any, ...]) is None
     assert get_tuple_type_length(Tuple) is None
+    assert get_tuple_type_length(Tuple[Any, ...]) is None
 
     User = namedtuple("User", ["name", "age", "friend"])
     assert get_tuple_type_length(User) == 3

requirements-dev.txt

@@ -6,7 +6,7 @@ pandas
 pdbpp
 pre-commit
 pyarrow
-pyspark==3.1.3
+pyspark==3.4.3
 pytest==6.2.5
 tox
 vcrpy==4.1.1