Pass loadingContext to pack(). (#1263)

Ensures that all the document loading options get passed through.

Author: tetron

Makefile

@@ -29,7 +29,7 @@ DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 mypy\
 	pytest-xdist==1.27.0 isort wheel -rtest-requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
 	   python-flake8 python-mock shellcheck
-VERSION=2.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
+VERSION=3.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
 	--format=format:%cd --date=format-local:%Y%m%d%H%M%S)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 UNAME_S=$(shell uname -s)

cwltool/main.py

@@ -554,12 +554,11 @@ def loadref(base, uri):  # type: (str, str) -> Any
 
 
 def print_pack(
-    document_loader,  # type: Loader
+    loadingContext,  # type: LoadingContext
     uri,  # type: str
-    metadata,  # type: Dict[str, Any]
 ):  # type: (...) -> str
     """Return a CWL serialization of the CWL document in JSON."""
-    packed = pack(document_loader, uri, metadata)
+    packed = pack(loadingContext, uri)
     if len(packed["$graph"]) > 1:
         return json_dumps(packed, indent=4)
     return json_dumps(packed["$graph"][0], indent=4)
@@ -923,13 +922,13 @@ def main(
             processobj, metadata = loadingContext.loader.resolve_ref(uri)
             processobj = cast(CommentedMap, processobj)
             if args.pack:
-                stdout.write(print_pack(loadingContext.loader, uri, metadata))
+                stdout.write(print_pack(loadingContext, uri))
                 return 0
 
             if args.provenance and runtimeContext.research_obj:
                 # Can't really be combined with args.pack at same time
                 runtimeContext.research_obj.packed_workflow(
-                    print_pack(loadingContext.loader, uri, metadata)
+                    print_pack(loadingContext, uri)
                 )
 
             if args.print_pre:

cwltool/pack.py

@@ -122,10 +122,10 @@ def import_embed(d, seen):
 
 
 def pack(
-    document_loader: Loader,
+    loadingContext: LoadingContext,
     uri,  # type: str
-    metadata,  # type: Dict[str, str]
     rewrite_out=None,  # type: Optional[Dict[str, str]]
+    loader=None,  # type: Optional[Loader]
 ):  # type: (...) -> Dict[str, Any]
 
     # The workflow document we have in memory right now may have been
@@ -138,8 +138,8 @@ def pack(
     # loading process with an empty index and updating turned off
     # so we have the original un-updated documents.
     #
-    document_loader = SubLoader(document_loader)
-    loadingContext = LoadingContext()
+    loadingContext = loadingContext.copy()
+    document_loader = SubLoader(loader or loadingContext.loader or Loader({}))
     loadingContext.do_update = False
     loadingContext.loader = document_loader
     loadingContext.loader.idx = {}
@@ -223,7 +223,6 @@ def rewrite_id(r, mainuri):
             dcr = cast(CommentedMap, dcr)
         if not isinstance(dcr, MutableMapping):
             continue
-        metadata = cast(Dict[str, Any], metadata)
         if "$schemas" in metadata:
             for s in metadata["$schemas"]:
                 schemas.add(s)

cwltool/update.py

@@ -143,6 +143,7 @@ def v1_1_0dev1to1_1(doc, loader, baseuri):  # pylint: disable=unused-argument
     # type: (Any, Loader, str) -> Tuple[Any, str]
     return (doc, "v1.1")
 
+
 def v1_2_0dev1todev2(doc, loader, baseuri):  # pylint: disable=unused-argument
     # type: (Any, Loader, str) -> Tuple[Any, str]
     return (doc, "v1.2.0-dev2")

tests/test_pack.py

@@ -26,7 +26,7 @@ def test_pack():
     with open(get_data("tests/wf/expect_packed.cwl")) as packed_file:
         expect_packed = yaml.safe_load(packed_file)
 
-    packed = cwltool.pack.pack(loadingContext.loader, uri, loadingContext.metadata)
+    packed = cwltool.pack.pack(loadingContext, uri)
     adjustFileObjs(
         packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
     )
@@ -53,7 +53,7 @@ def test_pack_input_named_name():
     with open(get_data("tests/wf/expect_trick_packed.cwl")) as packed_file:
         expect_packed = yaml.round_trip_load(packed_file)
 
-    packed = cwltool.pack.pack(loadingContext.loader, uri, loadingContext.metadata)
+    packed = cwltool.pack.pack(loadingContext, uri)
     adjustFileObjs(
         packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
     )
@@ -77,7 +77,7 @@ def test_pack_single_tool():
     )
     processobj = loadingContext.loader.resolve_ref(uri)[0]
 
-    packed = cwltool.pack.pack(loadingContext.loader, uri, loadingContext.metadata)
+    packed = cwltool.pack.pack(loadingContext, uri)
     assert "$schemas" in packed
 
 
@@ -86,9 +86,7 @@ def test_pack_fragment():
         expect_packed = yaml.safe_load(packed_file)
 
     loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/scatter2.cwl"))
-    packed = cwltool.pack.pack(
-        loadingContext.loader, uri + "#scatterstep/mysub", loadingContext.metadata
-    )
+    packed = cwltool.pack.pack(loadingContext, uri + "#scatterstep/mysub")
     adjustFileObjs(
         packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
     )
@@ -112,7 +110,7 @@ def test_pack_rewrites():
     processobj = loadingContext.loader.resolve_ref(uri)[0]
 
     cwltool.pack.pack(
-        loadingContext.loader, uri, loadingContext.metadata, rewrite_out=rewrites,
+        loadingContext, uri, rewrite_out=rewrites,
     )
 
     assert len(rewrites) == 6
@@ -136,7 +134,7 @@ def test_pack_missing_cwlVersion(cwl_path):
     processobj = loadingContext.loader.resolve_ref(uri)[0]
 
     # generate pack output dict
-    packed = json.loads(print_pack(loadingContext.loader, uri, loadingContext.metadata))
+    packed = json.loads(print_pack(loadingContext, uri))
 
     assert packed["cwlVersion"] == "v1.0"
 
@@ -160,7 +158,7 @@ def _pack_idempotently(document):
     processobj = loadingContext.loader.resolve_ref(uri)[0]
 
     # generate pack output dict
-    packed_text = print_pack(loadingContext.loader, uri, loadingContext.metadata)
+    packed_text = print_pack(loadingContext, uri)
     packed = json.loads(packed_text)
 
     tmp = NamedTemporaryFile(mode="w", delete=False)
@@ -177,7 +175,7 @@ def _pack_idempotently(document):
         processobj = loadingContext.loader.resolve_ref(uri2)[0]
 
         # generate pack output dict
-        packed_text = print_pack(loadingContext.loader, uri2, loadingContext.metadata)
+        packed_text = print_pack(loadingContext, uri2)
         double_packed = json.loads(packed_text)
     finally:
         os.remove(tmp.name)
@@ -203,7 +201,7 @@ def test_packed_workflow_execution(wf_path, job_path, namespaced, tmpdir):
         loadingContext, workflowobj, uri
     )
     processobj = loadingContext.loader.resolve_ref(uri)[0]
-    packed = json.loads(print_pack(loadingContext.loader, uri, loadingContext.metadata))
+    packed = json.loads(print_pack(loadingContext, uri))
 
     assert not namespaced or "$namespaces" in packed