Change WDL annotation names, add new WDL attributes, add requiredOutputs, separate required/optional companions. (#165)

Author: cmnbroad

src/main/java/org/broadinstitute/barclay/argparser/CommandLineArgumentParser.java

@@ -55,6 +55,10 @@ public final class CommandLineArgumentParser implements CommandLineParser {
     protected static final Set<String> EXPANSION_FILE_EXTENSIONS = new HashSet<>(
             Arrays.asList(EXPANSION_FILE_EXTENSION_LIST, EXPANSION_FILE_EXTENSIONS_ARGS));
 
+    // Prefixes used by the opt parser for short/long prefixes
+    public static final String SHORT_OPTION_PREFIX = "-";
+    public static final String LONG_OPTION_PREFIX = "--";
+
     // This is the object that the caller has provided that contains annotations,
     // and into which the values will be assigned.
     private final Object callerArguments;

src/main/java/org/broadinstitute/barclay/argparser/TaggedArgumentParser.java

@@ -60,10 +60,6 @@ public final class TaggedArgumentParser {
     // Map of surrogate keys to Pair(tag_string, raw_argument_value)
     private Map<String, Pair<String, String>> tagSurrogates = new HashMap<>();
 
-    // Prefixes used by the opt parser for short/long prefixes
-    private static final String SHORT_OPTION_PREFIX = "-";
-    private static final String LONG_OPTION_PREFIX = "--";
-
     /**
      * Given an array of raw arguments provided by the user, return an array of args where tagged arguments
      * have been replaced with curated arguments containing a key to be used by the parser to retrieve the actual
@@ -78,9 +74,9 @@ public String[] preprocessTaggedOptions(final String[] argArray) {
         while (argIt.hasNext()) {
             final String arg = argIt.next();
             if (isShortOptionToken(arg)) {
-                replaceTaggedOption(SHORT_OPTION_PREFIX, arg.substring(SHORT_OPTION_PREFIX.length()), argIt, finalArgs);
+                replaceTaggedOption(CommandLineArgumentParser.SHORT_OPTION_PREFIX, arg.substring(CommandLineArgumentParser.SHORT_OPTION_PREFIX.length()), argIt, finalArgs);
             } else if (isLongOptionToken(arg)) {
-                replaceTaggedOption(LONG_OPTION_PREFIX, arg.substring(LONG_OPTION_PREFIX.length()), argIt, finalArgs);
+                replaceTaggedOption(CommandLineArgumentParser.LONG_OPTION_PREFIX, arg.substring(CommandLineArgumentParser.LONG_OPTION_PREFIX.length()), argIt, finalArgs);
             } else { // Positional arg, etc., just add it
                 finalArgs.add(arg);
             }
@@ -173,14 +169,14 @@ public Pair<String, String> getTaggedOptionForSurrogate(final String putativeSur
 
     // See if the opt parser would think this is a short option ("-")
     private static boolean isShortOptionToken(final String argument) {
-        return argument.startsWith( SHORT_OPTION_PREFIX )
-                && !SHORT_OPTION_PREFIX.equals( argument )
+        return argument.startsWith( CommandLineArgumentParser.SHORT_OPTION_PREFIX )
+                && !CommandLineArgumentParser.SHORT_OPTION_PREFIX.equals( argument )
                 && !isLongOptionToken( argument );
     }
 
     // See if the opt parser would think this is a long option ("--")
     private static boolean isLongOptionToken(final String argument) {
-        return argument.startsWith( LONG_OPTION_PREFIX );
+        return argument.startsWith( CommandLineArgumentParser.LONG_OPTION_PREFIX );
     }
 
     // Stores the option string and value in the tagSurrogates hash map and returns a surrogate key.

src/main/java/org/broadinstitute/barclay/argparser/WorkflowInput.java

@@ -0,0 +1,44 @@
+package org.broadinstitute.barclay.argparser;
+
+import java.lang.annotation.*;
+
+/**
+ * Used to annotate @Arguments of a CommandLineProgram that are a workflow input, used by auto WDL generation.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+@Documented
+public @interface WorkflowInput {
+    /**
+     * @return array of names of required companion files that should travel with this resource. For example,
+     * a (Genomic Reference) FASTA file might have two associated companion files that must be supplied as
+     * arguments to the workflow, so the WorkflowInput would use the attribute values:
+     *
+     * @WorkflowInput(requiredCompanions = {"referenceDictionary", "referenceIndex" }
+     *
+     * The associated workflow and workflow input JSON (but not the task) generated by Barclay will contain two
+     * additional required parameters with these names, allowing callers to optionally provide name values for
+     * the companion files.
+     */
+    String[] requiredCompanions() default {};
+
+    /**
+     * @return array of names of optional companion files that should travel with this resource. For example,
+     * a (Genomic Reference) FASTA file might have two associated companion files that can optionally be supplied as
+     * arguments to the workflow, so the WorkflowOutput would use the attribute values:
+     *
+     * @WorkflowInput(optionalCompanions = {"referenceDictionary", "referenceIndex" }
+     *
+     * The associated workflow and workflow input JSON (but not the task) generated by Barclay will contain two
+     * additional optional parameters with these names, allowing callers to optionally provide name values for
+     * the companion files.
+     */
+    String[] optionalCompanions() default {};
+
+    /**
+     * Return a boolean specifying whether this input should have the "localization optional" attribute set.
+     * Defaults to false. A value of true should only be used for input arguments on tools that can handle
+     * inputs on non-local/remote file systems. This attribute is propagated to this input's companion files.
+     */
+    boolean localizationOptional() default false;
+}

src/main/java/org/broadinstitute/barclay/argparser/WorkflowOutput.java

@@ -0,0 +1,42 @@
+package org.broadinstitute.barclay.argparser;
+
+import java.lang.annotation.*;
+
+/**
+ * Used to annotate @Arguments of a CommandLineProgram that are a workflow output, used by auto WDL generation.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+@Documented
+public @interface WorkflowOutput {
+    /**
+     * @return array of names of required companion files that should travel with this resource. For example,
+     * a (Genomic Reference) FASTA file might have two associated companion files that must be supplied as
+     * arguments to the workflow, so the WorkflowOutput would use the attribute values:
+     *
+     * @WorkflowOutput(requiredCompanions = {"referenceDictionary", "referenceIndex" }
+     *
+     * The associated workflow and workflow input JSON (but not the task) generated by Barclay will contain two
+     * additional parameters with these names (the parameters will only be required if the source of the companions
+     * is required, otherwise they will be optional), allowing callers to optionally provide name values for
+     * the companion files.
+     *
+     * The additional companion arguments will be included in the workflow output section of the associated WDL.
+     */
+    String[] requiredCompanions() default {};
+
+    /**
+     * @return array of names of optional companion files that should travel with this resource. For example,
+     * a (Genomic Reference) FASTA file might have two associated companion files that can optionally be supplied as
+     * arguments to the workflow, so the WorkflowOutput would use the attribute values:
+     *
+     * @WorkflowOutput(optionalCompanions = {"referenceDictionary", "referenceIndex" }
+     *
+     * The associated workflow and workflow input JSON (but not the task) generated by Barclay will contain two
+     * additional optional parameters with these names, allowing callers to optionally provide name values for
+     * the companion files.
+     *
+     * The additional companion arguments will be included in the workflow output section of the associated WDL.
+     */
+    String[] optionalCompanions() default {};
+}

src/main/java/org/broadinstitute/barclay/argparser/RuntimeProperties.java src/main/java/org/broadinstitute/barclay/argparser/WorkflowProperties.java

@@ -12,7 +12,7 @@
  */
 @Target(ElementType.TYPE)
 @Retention(RetentionPolicy.RUNTIME)
-public @interface RuntimeProperties {
+public @interface WorkflowProperties {
 
     // NOTE: When a new attribute is added here, code should also be added to WDLWorkUnitHandler.addCustomBindings
     // to propagate the value to the freemarker map, and the corresponding expansion of the value should be
@@ -26,7 +26,7 @@
     /**
      * @return number of CPUs to use for this tool
      */
-    int cpu() default 1;
+    int cpu() default 2;
 
     /**
      * @return maximum number of times the workflow execution engine should request a preemptible machine for this

src/main/java/org/broadinstitute/barclay/argparser/WorkflowResource.java

@@ -0,0 +1,78 @@
+package org.broadinstitute.barclay.help;
+
+/**
+ * String constants for properties used in the various freemarker templates.
+ */
+public class TemplateProperties {
+
+    // Properties used by the base doc gen system
+    public final static String ARGUMENTS_POSITIONAL = "positional";
+
+    // Single argument map properties
+    public final static String ARGUMENT_NAME = "name";
+    public final static String ARGUMENT_TYPE = "type";
+    public final static String ARGUMENT_SUMMARY = "summary";
+    public final static String ARGUMENT_REQUIRED = "required";
+    public final static String ARGUMENT_DEFAULT_VALUE = "defaultValue";
+
+    // Properties specific to WDL generation
+
+    // workflow/task argument names need to be wdl-conforming, but we need to preserve the actual arg name
+    // for use in the command block
+    public final static String WDL_ARGUMENT_ACTUAL_NAME = "actualArgName";
+
+    // The WDL input type (as opposed to the acutal/output type; these differ when an input is the name of
+    // an output file. The input type is (String) but the output type is File.
+    public final static String WDL_ARGUMENT_INPUT_TYPE = "wdlinputtype";
+
+    /**
+     * Name of the top level freemarker map entry for runtime properties.
+     *
+     * Note that the property names used in this map will appear as workflow and task argument names in the
+     * generated WDL, and should therefore not collide with any WDL reserved words.
+     */
+    public static final String WDL_WORKFLOW_PROPERTIES = "workflowProperties";
+    /**
+     * runtime memory property (stored in "workflowProperties", used to initialize arg value in JSON)
+     */
+    public static final String WDL_WORKFLOW_MEMORY = "memoryRequirements";
+    /**
+     * runtime disks property (stored in "workflowProperties", used to initialize arg value in JSON)
+     */
+    public static final String WDL_WORKFLOW_DISKS = "diskRequirements";
+    /**
+     * cpu property
+     */
+    public static final String WDL_WORKFLOW_CPU = "cpuRequirements";
+    /**
+     * bootDiskSizeGb property
+     */
+    public static final String WDL_WORKFLOW_BOOT_DISK_SIZE_GB = "bootdisksizegbRequirements";
+    /**
+     * preemptible property
+     */
+    public static final String WDL_WORKFLOW_PREEMPTIBLE = "preemptibleRequirements";
+
+    /**
+     * name of the top level freemarker map entry for runtime outputs
+     */
+    public static final String WDL_RUNTIME_OUTPUTS = "runtimeOutputs";
+    /**
+     * name of the top level freemarker map entry for required runtime outputs
+     */
+    public static final String WDL_REQUIRED_OUTPUTS = "requiredOutputs";
+    /**
+     * name of the top level freemarker map entry for required companion resources
+     */
+    public static final String WDL_REQUIRED_COMPANIONS = "requiredCompanions";
+    /**
+     * name of the top level freemarker map entry for optional companion resources
+     */
+    public static final String WDL_OPTIONAL_COMPANIONS = "optionalCompanions";
+
+    /**
+     * localization optional inputs
+     */
+    public static final String WDL_LOCALIZATION_OPTIONAL = "localizationOptional";
+
+}

src/main/java/org/broadinstitute/barclay/help/TemplateProperties.java

@@ -5,7 +5,7 @@
 import freemarker.template.Template;
 import freemarker.template.TemplateException;
 import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
-import org.broadinstitute.barclay.argparser.RuntimeProperties;
+import org.broadinstitute.barclay.argparser.WorkflowProperties;
 
 import java.io.*;
 import java.util.List;
@@ -28,12 +28,12 @@ public static boolean start(final com.sun.javadoc.RootDoc rootDoc) throws IOExce
     @Override
     public boolean includeInDocs(final DocumentedFeature documentedFeature, final ClassDoc classDoc, final Class<?> clazz) {
         if (super.includeInDocs(documentedFeature, classDoc, clazz)) {
-            boolean hasRuntimeProperties = clazz.getAnnotation(RuntimeProperties.class) != null;
+            boolean hasWorkflowProperties = clazz.getAnnotation(WorkflowProperties.class) != null;
             boolean isCommandLineProgram = clazz.getAnnotation(CommandLineProgramProperties.class) != null;
-            if (hasRuntimeProperties) {
+            if (hasWorkflowProperties) {
                 if (!isCommandLineProgram) {
                     throw new DocException(String.format(
-                            "RuntimeProperties can only be applied to classes that are annotated with CommandLineProgramProperties (%s)",
+                            "WorkflowProperties can only be applied to classes that are annotated with CommandLineProgramProperties (%s)",
                             clazz));
                 }
                 return true;