Skip to content

Commit 5ae5798

Browse files
tetrontetron
authored
Add test for packing a fragment of a document & fix. (#1261)
Bugfix and test for packing starting from a document fragment instead of whole document. Bump version to 3.0 because pack() API was changed in non-backwards compatible way.
1 parent 4700fbe commit 5ae5798

File tree

8 files changed

+199
-49
lines changed

8 files changed

+199
-49
lines changed

‎cwltool/load_tool.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,8 @@ def resolve_and_validate_document(
333333
if cwlVersion == "v1.0":
334334
_add_blank_ids(workflowobj)
335335

336-
processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
336+
document_loader.resolve_all(workflowobj, fileuri)
337+
processobj, metadata = document_loader.resolve_ref(uri)
337338
if loadingContext.metadata:
338339
metadata = loadingContext.metadata
339340
if not isinstance(processobj, (CommentedMap, CommentedSeq)):

‎cwltool/main.py‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ def generate_example_input(
159159
else:
160160
comment = "optional"
161161
else:
162-
example = yaml.comments.CommentedSeq()
162+
example = CommentedSeq()
163163
for index, entry in enumerate(inptype):
164164
value, e_comment = generate_example_input(entry, default)
165165
example.append(value)
@@ -555,12 +555,11 @@ def loadref(base, uri): # type: (str, str) -> Any
555555

556556
def print_pack(
557557
document_loader, # type: Loader
558-
processobj, # type: CommentedMap
559558
uri, # type: str
560559
metadata, # type: Dict[str, Any]
561560
): # type: (...) -> str
562561
"""Return a CWL serialization of the CWL document in JSON."""
563-
packed = pack(document_loader, processobj, uri, metadata)
562+
packed = pack(document_loader, uri, metadata)
564563
if len(packed["$graph"]) > 1:
565564
return json_dumps(packed, indent=4)
566565
return json_dumps(packed["$graph"][0], indent=4)
@@ -917,15 +916,13 @@ def main(
917916
processobj, metadata = loadingContext.loader.resolve_ref(uri)
918917
processobj = cast(CommentedMap, processobj)
919918
if args.pack:
920-
stdout.write(
921-
print_pack(loadingContext.loader, processobj, uri, metadata)
922-
)
919+
stdout.write(print_pack(loadingContext.loader, uri, metadata))
923920
return 0
924921

925922
if args.provenance and runtimeContext.research_obj:
926923
# Can't really be combined with args.pack at same time
927924
runtimeContext.research_obj.packed_workflow(
928-
print_pack(loadingContext.loader, processobj, uri, metadata)
925+
print_pack(loadingContext.loader, uri, metadata)
929926
)
930927

931928
if args.print_pre:

‎cwltool/pack.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,6 @@ def import_embed(d, seen):
123123

124124
def pack(
125125
document_loader: Loader,
126-
startingobj, # type: Union[Dict[str, Any], List[Dict[str, Any]]]
127126
uri, # type: str
128127
metadata, # type: Dict[str, str]
129128
rewrite_out=None, # type: Optional[Dict[str, str]]
@@ -146,7 +145,7 @@ def pack(
146145
loadingContext.loader.idx = {}
147146
loadingContext.metadata = {}
148147
loadingContext, docobj, uri = fetch_document(uri, loadingContext)
149-
loadingContext, uri = resolve_and_validate_document(
148+
loadingContext, fileuri = resolve_and_validate_document(
150149
loadingContext, docobj, uri, preprocess_only=True
151150
)
152151
if loadingContext.loader is None:

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
setup(
2323
name="cwltool",
24-
version="2.0",
24+
version="3.0",
2525
description="Common workflow language reference implementation",
2626
long_description=open(README).read(),
2727
long_description_content_type="text/x-rst",

‎tests/test_pack.py‎

Lines changed: 47 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import tempfile
44
from functools import partial
55
from io import StringIO
6+
from tempfile import NamedTemporaryFile
67

78
import pytest
89

@@ -21,18 +22,11 @@
2122

2223
def test_pack():
2324
loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/revsort.cwl"))
24-
loadingContext.do_update = False
25-
loadingContext, uri = resolve_and_validate_document(
26-
loadingContext, workflowobj, uri
27-
)
28-
processobj = loadingContext.loader.resolve_ref(uri)[0]
2925

3026
with open(get_data("tests/wf/expect_packed.cwl")) as packed_file:
3127
expect_packed = yaml.safe_load(packed_file)
3228

33-
packed = cwltool.pack.pack(
34-
loadingContext.loader, processobj, uri, loadingContext.metadata
35-
)
29+
packed = cwltool.pack.pack(loadingContext.loader, uri, loadingContext.metadata)
3630
adjustFileObjs(
3731
packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
3832
)
@@ -59,9 +53,7 @@ def test_pack_input_named_name():
5953
with open(get_data("tests/wf/expect_trick_packed.cwl")) as packed_file:
6054
expect_packed = yaml.round_trip_load(packed_file)
6155

62-
packed = cwltool.pack.pack(
63-
loadingContext.loader, processobj, uri, loadingContext.metadata
64-
)
56+
packed = cwltool.pack.pack(loadingContext.loader, uri, loadingContext.metadata)
6557
adjustFileObjs(
6658
packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
6759
)
@@ -85,10 +77,26 @@ def test_pack_single_tool():
8577
)
8678
processobj = loadingContext.loader.resolve_ref(uri)[0]
8779

80+
packed = cwltool.pack.pack(loadingContext.loader, uri, loadingContext.metadata)
81+
assert "$schemas" in packed
82+
83+
84+
def test_pack_fragment():
85+
with open(get_data("tests/wf/scatter2_subwf.cwl")) as packed_file:
86+
expect_packed = yaml.safe_load(packed_file)
87+
88+
loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/scatter2.cwl"))
8889
packed = cwltool.pack.pack(
89-
loadingContext.loader, processobj, uri, loadingContext.metadata
90+
loadingContext.loader, uri + "#scatterstep/mysub", loadingContext.metadata
91+
)
92+
adjustFileObjs(
93+
packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))
94+
)
95+
adjustDirObjs(packed, partial(make_relative, os.path.abspath(get_data("tests/wf"))))
96+
97+
assert json.dumps(packed, sort_keys=True, indent=2) == json.dumps(
98+
expect_packed, sort_keys=True, indent=2
9099
)
91-
assert "$schemas" in packed
92100

93101

94102
def test_pack_rewrites():
@@ -104,11 +112,7 @@ def test_pack_rewrites():
104112
processobj = loadingContext.loader.resolve_ref(uri)[0]
105113

106114
cwltool.pack.pack(
107-
loadingContext.loader,
108-
processobj,
109-
uri,
110-
loadingContext.metadata,
111-
rewrite_out=rewrites,
115+
loadingContext.loader, uri, loadingContext.metadata, rewrite_out=rewrites,
112116
)
113117

114118
assert len(rewrites) == 6
@@ -132,9 +136,7 @@ def test_pack_missing_cwlVersion(cwl_path):
132136
processobj = loadingContext.loader.resolve_ref(uri)[0]
133137

134138
# generate pack output dict
135-
packed = json.loads(
136-
print_pack(loadingContext.loader, processobj, uri, loadingContext.metadata)
137-
)
139+
packed = json.loads(print_pack(loadingContext.loader, uri, loadingContext.metadata))
138140

139141
assert packed["cwlVersion"] == "v1.0"
140142

@@ -158,19 +160,29 @@ def _pack_idempotently(document):
158160
processobj = loadingContext.loader.resolve_ref(uri)[0]
159161

160162
# generate pack output dict
161-
packed = json.loads(
162-
print_pack(loadingContext.loader, processobj, uri, loadingContext.metadata)
163-
)
164-
165-
loadingContext, workflowobj, uri2 = fetch_document(packed)
166-
loadingContext.do_update = False
167-
loadingContext, uri2 = resolve_and_validate_document(
168-
loadingContext, workflowobj, uri
169-
)
170-
processobj = loadingContext.loader.resolve_ref(uri2)[0]
171-
double_packed = json.loads(
172-
print_pack(loadingContext.loader, processobj, uri2, loadingContext.metadata)
173-
)
163+
packed_text = print_pack(loadingContext.loader, uri, loadingContext.metadata)
164+
packed = json.loads(packed_text)
165+
166+
tmp = NamedTemporaryFile(mode="w", delete=False)
167+
try:
168+
tmp.write(packed_text)
169+
tmp.flush()
170+
tmp.close()
171+
172+
loadingContext, workflowobj, uri2 = fetch_document(tmp.name)
173+
loadingContext.do_update = False
174+
loadingContext, uri2 = resolve_and_validate_document(
175+
loadingContext, workflowobj, uri2
176+
)
177+
processobj = loadingContext.loader.resolve_ref(uri2)[0]
178+
179+
# generate pack output dict
180+
packed_text = print_pack(loadingContext.loader, uri2, loadingContext.metadata)
181+
double_packed = json.loads(packed_text)
182+
finally:
183+
os.remove(tmp.name)
184+
185+
assert uri != uri2
174186
assert packed == double_packed
175187

176188

@@ -191,9 +203,7 @@ def test_packed_workflow_execution(wf_path, job_path, namespaced, tmpdir):
191203
loadingContext, workflowobj, uri
192204
)
193205
processobj = loadingContext.loader.resolve_ref(uri)[0]
194-
packed = json.loads(
195-
print_pack(loadingContext.loader, processobj, uri, loadingContext.metadata)
196-
)
206+
packed = json.loads(print_pack(loadingContext.loader, uri, loadingContext.metadata))
197207

198208
assert not namespaced or "$namespaces" in packed
199209

‎tests/test_procgenerator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,5 @@ def test_missing_enable_ext():
3535
finally:
3636
if opt is not None:
3737
os.environ["CWLTOOL_OPTIONS"] = opt
38-
else:
38+
elif "CWLTOOL_OPTIONS" in os.environ:
3939
del os.environ["CWLTOOL_OPTIONS"]

‎tests/wf/scatter2.cwl‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright (C) The Arvados Authors. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
class: Workflow
6+
cwlVersion: v1.0
7+
$namespaces:
8+
arv: "http://arvados.org/cwl#"
9+
inputs:
10+
sleeptime:
11+
type: int[]
12+
default: [5]
13+
fileblub:
14+
type: File
15+
default:
16+
class: File
17+
location: keep:99999999999999999999999999999999+118/token.txt
18+
outputs:
19+
out:
20+
type: string[]
21+
outputSource: scatterstep/out
22+
requirements:
23+
SubworkflowFeatureRequirement: {}
24+
ScatterFeatureRequirement: {}
25+
InlineJavascriptRequirement: {}
26+
StepInputExpressionRequirement: {}
27+
steps:
28+
scatterstep:
29+
in:
30+
sleeptime: sleeptime
31+
fileblub: fileblub
32+
out: [out]
33+
scatter: sleeptime
34+
hints:
35+
- class: arv:RunInSingleContainer
36+
run:
37+
class: Workflow
38+
id: mysub
39+
inputs:
40+
sleeptime: int
41+
fileblub: File
42+
outputs:
43+
out:
44+
type: string
45+
outputSource: sleep1/out
46+
steps:
47+
sleep1:
48+
in:
49+
sleeptime: sleeptime
50+
blurb:
51+
valueFrom: |
52+
${
53+
return String(inputs.sleeptime) + "b";
54+
}
55+
out: [out]
56+
run:
57+
class: CommandLineTool
58+
id: subtool
59+
inputs:
60+
sleeptime:
61+
type: int
62+
inputBinding: {position: 1}
63+
outputs:
64+
out:
65+
type: string
66+
outputBinding:
67+
outputEval: "out"
68+
baseCommand: sleep

‎tests/wf/scatter2_subwf.cwl‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Copyright (C) The Arvados Authors. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
{
6+
"$graph": [
7+
{
8+
"$namespaces": {
9+
"arv": "http://arvados.org/cwl#"
10+
},
11+
"class": "Workflow",
12+
"cwlVersion": "v1.0",
13+
"id": "#main",
14+
"inputs": [
15+
{
16+
"id": "#main/fileblub",
17+
"type": "File"
18+
},
19+
{
20+
"id": "#main/sleeptime",
21+
"type": "int"
22+
}
23+
],
24+
"outputs": [
25+
{
26+
"id": "#main/out",
27+
"outputSource": "#main/sleep1/out",
28+
"type": "string"
29+
}
30+
],
31+
"steps": [
32+
{
33+
"id": "#main/sleep1",
34+
"in": [
35+
{
36+
"id": "#main/sleep1/blurb",
37+
"valueFrom": "${\n return String(inputs.sleeptime) + \"b\";\n}\n"
38+
},
39+
{
40+
"id": "#main/sleep1/sleeptime",
41+
"source": "#main/sleeptime"
42+
}
43+
],
44+
"out": [
45+
"#main/sleep1/out"
46+
],
47+
"run": {
48+
"baseCommand": "sleep",
49+
"class": "CommandLineTool",
50+
"id": "#main/sleep1/subtool",
51+
"inputs": [
52+
{
53+
"id": "#main/sleep1/subtool/sleeptime",
54+
"inputBinding": {
55+
"position": 1
56+
},
57+
"type": "int"
58+
}
59+
],
60+
"outputs": [
61+
{
62+
"id": "#main/sleep1/subtool/out",
63+
"outputBinding": {
64+
"outputEval": "out"
65+
},
66+
"type": "string"
67+
}
68+
]
69+
}
70+
}
71+
]
72+
}
73+
],
74+
"cwlVersion": "v1.0"
75+
}

0 commit comments

Comments
 (0)