upgrade to py3.6+ syntax

Author: mr-c

Makefile

@@ -25,10 +25,12 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8\
-	isort wheel autoflake flake8-bugbear pyupgrade bandit -rtest-requirements.txt -rmypy_requirements.txt
+DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 tox\
+	isort wheel autoflake flake8-bugbear pyupgrade bandit \
+	-rtest-requirements.txt -rmypy_requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
 	   python-flake8 python-mock shellcheck
+
 VERSION=3.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
 	--format=format:%cd --date=format-local:%Y%m%d%H%M%S)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -190,7 +192,7 @@ release: release-test
 		twine upload testenv2/src/${MODULE}/dist/* && \
 		git tag ${VERSION} && git push --tags
 
-flake8: $(filter-out setup.py,${PYSOURCES})
+flake8: ${PYSOURCES}
 	flake8 $^
 
 FORCE:

cwltool/argparser.py

@@ -694,7 +694,7 @@ def __init__(
         """Fail if nargs is used."""
         if nargs is not None:
             raise ValueError("nargs not allowed")
-        super(FSAction, self).__init__(option_strings, dest, **kwargs)
+        super().__init__(option_strings, dest, **kwargs)
 
     def __call__(
         self,
@@ -722,7 +722,7 @@ def __init__(
         """Initialize."""
         if nargs is not None:
             raise ValueError("nargs not allowed")
-        super(FSAppendAction, self).__init__(option_strings, dest, **kwargs)
+        super().__init__(option_strings, dest, **kwargs)
 
     def __call__(
         self,

cwltool/builder.py

@@ -130,7 +130,9 @@ def check_format(
         )
 
 
-class HasReqsHints(object):
+class HasReqsHints:
+    """Base class for get_requirement()."""
+
     def __init__(self) -> None:
         """Initialize this reqs decorator."""
         self.requirements = []  # type: List[CWLObjectType]
@@ -294,7 +296,7 @@ def bind_input(
                         bound_input = True
             if not bound_input:
                 raise ValidationException(
-                    "'%s' is not a valid union %s" % (datum, schema["type"])
+                    "'{}' is not a valid union {}".format(datum, schema["type"])
                 )
         elif isinstance(schema["type"], MutableMapping):
             st = copy.deepcopy(schema["type"])
@@ -398,7 +400,9 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
                             ) as f2:
                                 datum["contents"] = content_limit_respected_read(f2)
                         except Exception as e:
-                            raise Exception("Reading %s\n%s" % (datum["location"], e))
+                            raise Exception(
+                                "Reading {}\n{}".format(datum["location"], e)
+                            )
 
                 if "secondaryFiles" in schema:
                     if "secondaryFiles" not in datum:
@@ -545,7 +549,7 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str:
         ):
             if "path" not in value:
                 raise WorkflowException(
-                    u'%s object missing "path": %s' % (value["class"], value)
+                    '{} object missing "path": {}'.format(value["class"], value)
                 )
 
             # Path adjust for windows file path when passing to docker, docker accepts unix like path only

cwltool/checker.py

@@ -60,7 +60,7 @@ def check_types(
         return check_types(
             merge_flatten_type(_get_type(srctype)), _get_type(sinktype), None, None
         )
-    raise WorkflowException("Unrecognized linkMerge enum '{}'".format(linkMerge))
+    raise WorkflowException(f"Unrecognized linkMerge enum '{linkMerge}'")
 
 
 def merge_flatten_type(src: SinkType) -> CWLOutputType:
@@ -222,7 +222,7 @@ def static_checker(
         # by the source
         missing = missing_subset(srcsf, sinksf)
         if missing:
-            msg1 = "Parameter '%s' requires secondaryFiles %s but" % (
+            msg1 = "Parameter '{}' requires secondaryFiles {} but".format(
                 shortname(sink["id"]),
                 missing,
             )
@@ -241,7 +241,7 @@ def static_checker(
                 % shortname(sink["id"])
             )
             msg = SourceLine(sink).makeError(
-                "%s\n%s" % (msg1, bullets([msg3, msg4, msg5], "  "))
+                "{}\n{}".format(msg1, bullets([msg3, msg4, msg5], "  "))
             )
         elif sink.get("not_connected"):
             if not sink.get("used_by_step"):

cwltool/command_line_tool.py

@@ -104,7 +104,7 @@
 """
 
 
-class ExpressionJob(object):
+class ExpressionJob:
     """Job for ExpressionTools."""
 
     def __init__(
@@ -267,7 +267,9 @@ def revmap_file(
     )
 
 
-class CallbackJob(object):
+class CallbackJob:
+    """Callback Job class, used by CommandLine.job()."""
+
     def __init__(
         self,
         job: "CommandLineTool",
@@ -352,7 +354,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None:
 class ParameterOutputWorkflowException(WorkflowException):
     def __init__(self, msg: str, port: CWLObjectType, **kwargs: Any) -> None:
         """Exception for when there was an error collecting output for a parameter."""
-        super(ParameterOutputWorkflowException, self).__init__(
+        super().__init__(
             "Error collecting output for parameter '%s':\n%s"
             % (shortname(cast(str, port["id"])), msg),
             kwargs,
@@ -364,7 +366,7 @@ def __init__(
         self, toolpath_object: CommentedMap, loadingContext: LoadingContext
     ) -> None:
         """Initialize this CommandLineTool."""
-        super(CommandLineTool, self).__init__(toolpath_object, loadingContext)
+        super().__init__(toolpath_object, loadingContext)
         self.prov_obj = loadingContext.prov_obj
 
     def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
@@ -792,7 +794,7 @@ def calc_checksum(location: str) -> Optional[str]:
             jobcache = os.path.join(runtimeContext.cachedir, cachekey)
 
             # Create a lockfile to manage cache status.
-            jobcachepending = "{}.status".format(jobcache)
+            jobcachepending = f"{jobcache}.status"
             jobcachelock = None
             jobstatus = None
 

cwltool/context.py

@@ -27,7 +27,9 @@
     from .provenance_profile import ProvenanceProfile
 
 
-class ContextBase(object):
+class ContextBase:
+    """Shared kwargs based initilizer for {Runtime,Loading}Context."""
+
     def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         """Initialize."""
         if kwargs:
@@ -73,7 +75,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.jobdefaults = None  # type: Optional[CommentedMap]
         self.doc_cache = True  # type: bool
 
-        super(LoadingContext, self).__init__(kwargs)
+        super().__init__(kwargs)
 
     def copy(self):
         # type: () -> LoadingContext
@@ -149,7 +151,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.mpi_config = MpiConfig()  # type: MpiConfig
         self.default_stdout = None  # type: Optional[Union[IO[bytes], TextIO]]
         self.default_stderr = None  # type: Optional[Union[IO[bytes], TextIO]]
-        super(RuntimeContext, self).__init__(kwargs)
+        super().__init__(kwargs)
         if self.tmp_outdir_prefix == "":
             self.tmp_outdir_prefix = self.tmpdir_prefix
 

cwltool/cwlrdf.py

@@ -47,8 +47,8 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
 
     for step, run, _ in qres:
         stdout.write(
-            u'"%s" [label="%s"]\n'
-            % (lastpart(step), "%s (%s)" % (lastpart(step), lastpart(run)))
+            '"%s" [label="%s"]\n'
+            % (lastpart(step), "{} ({})".format(lastpart(step), lastpart(run)))
         )
 
     qres = g.query(
@@ -61,12 +61,12 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
     )
 
     for step, inp, source in qres:
-        stdout.write(u'"%s" [shape=box]\n' % (lastpart(inp)))
+        stdout.write('"%s" [shape=box]\n' % (lastpart(inp)))
         stdout.write(
-            u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(inp), "")
+            '"{}" -> "{}" [label="{}"]\n'.format(lastpart(source), lastpart(inp), "")
         )
         stdout.write(
-            u'"%s" -> "%s" [label="%s"]\n' % (lastpart(inp), lastpart(step), "")
+            '"{}" -> "{}" [label="{}"]\n'.format(lastpart(inp), lastpart(step), "")
         )
 
     qres = g.query(
@@ -78,9 +78,9 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
     )
 
     for step, out in qres:
-        stdout.write(u'"%s" [shape=box]\n' % (lastpart(out)))
+        stdout.write('"%s" [shape=box]\n' % (lastpart(out)))
         stdout.write(
-            u'"%s" -> "%s" [label="%s"]\n' % (lastpart(step), lastpart(out), "")
+            '"{}" -> "{}" [label="{}"]\n'.format(lastpart(step), lastpart(out), "")
         )
 
     qres = g.query(
@@ -92,9 +92,9 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
     )
 
     for out, source in qres:
-        stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(out)))
+        stdout.write('"%s" [shape=octagon]\n' % (lastpart(out)))
         stdout.write(
-            u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(out), "")
+            '"{}" -> "{}" [label="{}"]\n'.format(lastpart(source), lastpart(out), "")
         )
 
     qres = g.query(
@@ -106,7 +106,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
     )
 
     for (inp,) in qres:
-        stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(inp)))
+        stdout.write('"%s" [shape=octagon]\n' % (lastpart(inp)))
 
 
 def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
@@ -150,7 +150,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
                 if wf not in dotname:
                     dotname[wf] = "cluster_" + lastpart(wf)
                 stdout.write(
-                    u'subgraph "%s" { label="%s"\n' % (dotname[wf], lastpart(wf))
+                    'subgraph "{}" {{ label="{}"\n'.format(dotname[wf], lastpart(wf))
                 )
                 currentwf = wf
                 clusternode[wf] = step
@@ -159,7 +159,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
 
         if str(runtype) != "https://w3id.org/cwl/cwl#Workflow":
             stdout.write(
-                u'"%s" [label="%s"]\n'
+                '"%s" [label="%s"]\n'
                 % (dotname[step], urllib.parse.urldefrag(str(step))[1])
             )
 
@@ -182,12 +182,12 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
     for src, sink, srcrun, sinkrun in qres:
         attr = ""
         if srcrun in clusternode:
-            attr += u'ltail="%s"' % dotname[srcrun]
+            attr += 'ltail="%s"' % dotname[srcrun]
             src = clusternode[srcrun]
         if sinkrun in clusternode:
-            attr += u' lhead="%s"' % dotname[sinkrun]
+            attr += ' lhead="%s"' % dotname[sinkrun]
             sink = clusternode[sinkrun]
-        stdout.write(u'"%s" -> "%s" [%s]\n' % (dotname[src], dotname[sink], attr))
+        stdout.write('"{}" -> "{}" [{}]\n'.format(dotname[src], dotname[sink], attr))
 
 
 def printdot(

cwltool/docker.py

@@ -9,7 +9,7 @@
 import sys
 import threading
 from distutils import spawn
-from io import StringIO, open  # pylint: disable=redefined-builtin
+from io import StringIO  # pylint: disable=redefined-builtin
 from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast
 
 import requests
@@ -99,9 +99,7 @@ def __init__(
         name: str,
     ) -> None:
         """Initialize a command line builder using the Docker software container engine."""
-        super(DockerCommandLineJob, self).__init__(
-            builder, joborder, make_path_mapper, requirements, hints, name
-        )
+        super().__init__(builder, joborder, make_path_mapper, requirements, hints, name)
 
     @staticmethod
     def get_image(
@@ -261,7 +259,7 @@ def append_volume(
         output = StringIO()
         csv.writer(output).writerow(options)
         mount_arg = output.getvalue().strip()
-        runtime.append("--mount={}".format(mount_arg))
+        runtime.append(f"--mount={mount_arg}")
         # Unlike "--volume", "--mount" will fail if the volume doesn't already exist.
         if not os.path.exists(source):
             os.makedirs(source)
@@ -382,7 +380,7 @@ def create_runtime(
 
             if self.networkaccess:
                 if runtimeContext.custom_net:
-                    runtime.append("--net={0}".format(runtimeContext.custom_net))
+                    runtime.append(f"--net={runtimeContext.custom_net}")
             else:
                 runtime.append("--net=none")
 
@@ -437,7 +435,7 @@ def create_runtime(
             cidfile_path = os.path.join(cidfile_dir, cidfile_name)
             runtime.append("--cidfile=%s" % cidfile_path)
         for key, value in self.environment.items():
-            runtime.append("--env=%s=%s" % (key, value))
+            runtime.append(f"--env={key}={value}")
 
         if runtimeContext.strict_memory_limit and not user_space_docker_cmd:
             ram = self.builder.resources["ram"]

cwltool/executors.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Single and multi-threaded executors."""
 import datetime
 import functools
@@ -40,7 +39,7 @@
 TMPDIR_LOCK = Lock()
 
 
-class JobExecutor(object, metaclass=ABCMeta):
+class JobExecutor(metaclass=ABCMeta):
     """Abstract base job executor."""
 
     def __init__(self) -> None:
@@ -278,7 +277,7 @@ class MultithreadedJobExecutor(JobExecutor):
 
     def __init__(self) -> None:
         """Initialize."""
-        super(MultithreadedJobExecutor, self).__init__()
+        super().__init__()
         self.exceptions = []  # type: List[WorkflowException]
         self.pending_jobs = []  # type: List[JobsType]
         self.pending_jobs_lock = threading.Lock()
@@ -332,10 +331,10 @@ def _runner(self, job, runtime_context, TMPDIR_LOCK):
             )
             job.run(runtime_context, TMPDIR_LOCK)
         except WorkflowException as err:
-            _logger.exception("Got workflow error: {}".format(err))
+            _logger.exception(f"Got workflow error: {err}")
             self.exceptions.append(err)
         except Exception as err:  # pylint: disable=broad-except
-            _logger.exception("Got workflow error: {}".format(err))
+            _logger.exception(f"Got workflow error: {err}")
             self.exceptions.append(WorkflowException(str(err)))
         finally:
             if runtime_context.workflow_eval_lock: