Merge branch 'release-0.17'

broadinstitute · Feb 2, 2016 · 42884d8 · 42884d8
2 parents 42b827e + 8a48eee
commit 42884d8
Show file tree

Hide file tree

Showing 78 changed files with 2,977 additions and 922 deletions.
diff --git a/README.md b/README.md
diff --git a/build.sbt b/build.sbt
@@ -3,15 +3,15 @@ import sbtrelease.ReleasePlugin._
 
 name := "cromwell"
 
-version := "0.16"
+version := "0.17"
 
 organization := "org.broadinstitute"
 
 scalaVersion := "2.11.7"
 
 val lenthallV = "0.15"
 
-val wdl4sV = "0.1"
+val wdl4sV = "0.2"
 
 val sprayV = "1.3.2"
 
@@ -61,6 +61,7 @@ libraryDependencies ++= Seq(
   "com.google.api-client" % "google-api-client-java6" % googleClientApiV,
   "com.google.api-client" % "google-api-client-jackson2" % googleClientApiV,
   "com.google.oauth-client" % "google-oauth-client" % googleClientApiV,
+  "com.google.cloud.bigdataoss" % "gcsio" % "1.4.3",
   "mysql" % "mysql-connector-java" % "5.1.36",
   "org.scalaz" % "scalaz-core_2.11" % "7.1.3",
   "com.github.pathikrit" %% "better-files" % "2.13.0",

diff --git a/src/main/config/cromwell-compose.yaml.ctmpl b/src/main/config/cromwell-compose.yaml.ctmpl
@@ -1,33 +1,47 @@
-{{with $environment := env "ENVIRONMENT"}}
-{{$keyname := printf "secret/dsde/%s/cromwell/cromwell-compose.yaml" $environment}}
-{{with vault $keyname}}
-
+{{with $environment := env "ENVIRONMENT"}}{{$cromkey := printf "secret/dsde/%s/cromwell/cromwell-compose.yaml" $environment}}{{with vault $cromkey}}
+{{$cromwell_image := .Data.cromwell_image}}
+{{$cromwell_dns := .Data.cromwell_dns}}
+{{$env_log_driver := .Data.env_log_driver}}
+{{$env_java_opts := .Data.env_java_opts}}
+{{$cromwell_volumes := .Data.cromwell_volumes}}
+{{$proxy_hostname := .Data.proxy_hostname}}
+{{$proxy_log_driver := .Data.proxy_log_driver}}
+{{$proxy_volumes := .Data.proxy_volumes}}
+{{$env_callback_uri := .Data.env_callback_uri}}
+{{$env_log_level := .Data.env_log_level}}
+{{$env_server_name := .Data.env_server_name}}
+{{$commonkey := printf "secret/dsde/%s/common/proxy-ldap" $environment}}{{with vault $commonkey}}
+{{$proxy_ldap_group := .Data.proxy_ldap_group}}
+{{$proxy_ldap_url := .Data.proxy_ldap_url}}
+{{$proxy_ldap_bind_dn := .Data.proxy_ldap_bind_dn}}
+{{$proxy_ldap_bind_password := .Data.proxy_ldap_bind_password}}
 app:
-  image: {{.Data.cromwell_image}}
-  {{.Data.cromwell_dns}}
-  log_driver: "{{.Data.env_log_driver}}"
+  image: {{ $cromwell_image }}
+  {{ $cromwell_dns }}
+  log_driver: "{{ $env_log_driver }}"
   environment:
-    JAVA_OPTS: {{.Data.env_java_opts}}
-  {{.Data.cromwell_volumes}}
+    JAVA_OPTS: {{ $env_java_opts }}
+  {{ $cromwell_volumes }}
 proxy:
   image: broadinstitute/openidc-proxy:latest
-  hostname: {{.Data.proxy_hostname}}
-  log_driver: "{{.Data.proxy_log_driver}}"
+  hostname: {{ $proxy_hostname }}
+  log_driver: "{{ $proxy_log_driver }}"
   links:
     - app:app
   ports:
     - "80:80"
     - "443:443"
-  {{.Data.proxy_volumes}}
+  {{ $proxy_volumes }}
   environment:
-    CALLBACK_URI: {{.Data.env_callback_uri}}
-    LOG_LEVEL: {{.Data.env_log_level}}
+    CALLBACK_URI: {{ $env_callback_uri }}
+    LOG_LEVEL: {{ $env_log_level }}
     PROXY_URL: http://app:8000/
     PROXY_URL2: http://app:8000/api
-    SERVER_NAME: {{.Data.env_server_name}}
-    AUTH_REQUIRE2: Require ldap-group {{.Data.proxy_ldap_group}}
-    AUTH_LDAP_URL2: 'AuthLDAPURL "{{.Data.proxy_ldap_url}}"'
+    SERVER_NAME: {{ $env_server_name }}
+    AUTH_REQUIRE2: Require ldap-group {{ $proxy_ldap_group }}
+    AUTH_LDAP_URL2: 'AuthLDAPURL "{{ $proxy_ldap_url }}"'
     AUTH_LDAP_GROUP_ATTR2: 'AuthLDAPGroupAttribute member'
+    AUTH_LDAP_BIND_DN2: 'AuthLDAPBindDN "{{ $proxy_ldap_bind_dn }}"'
+    AUTH_LDAP_BIND_PASSWORD2: 'AuthLDAPBindPassword {{ $proxy_ldap_bind_password }}'
     REMOTE_USER_CLAIM: sub
-{{end}}
-{{end}}
+{{end}}{{end}}{{end}}
diff --git a/src/main/config/cromwell.conf.ctmpl b/src/main/config/cromwell.conf.ctmpl
@@ -13,6 +13,14 @@ backend {
   }
 }
 
+workflow-options {
+  // These workflow options will be encrypted when stored in the database
+  encrypted-fields: ["refresh_token"]
+
+  // AES-256 key to use to encrypt the values in `encrypted-fields`
+  base64-encryption-key: "{{.Data.workflow_options_encryption_key}}"
+}
+
 docker {
   dockerAccount = "{{.Data.docker_account}}"
   dockerToken = "{{.Data.docker_token}}"

diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
@@ -96,7 +96,7 @@ google {
   // cromwellAuthenticationScheme can take the values "service_account",
   // "user_account", and "application_default". The first two are described
   // in the stanzas below.
-  // Use "application-default" to use the default service account credentials.
+  // Use "application_default" to use the default service account credentials.
   // This is useful if you are running on a GCE VM and if you don't need
   // user level access. See https://developers.google.com/identity/protocols/application-default-credentials
   // for more info. No further information is needed.
@@ -135,6 +135,7 @@ database {
     hsqldb {
       db.url = "jdbc:hsqldb:mem:${slick.uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
       db.driver = "org.hsqldb.jdbcDriver"
+      db.connectionTimeout = 1000 // NOTE: 1000ms (the default) is ok for a small hsqldb, but often too short for mysql
       driver = "slick.driver.HsqldbDriver$"
       slick.createSchema = true
     }
@@ -144,6 +145,7 @@ database {
     hsqldb {
       db.url = "jdbc:hsqldb:mem:testdb;shutdown=false;hsqldb.tx=mvcc"
       db.driver = "org.hsqldb.jdbcDriver"
+      db.connectionTimeout = 1000 // NOTE: 1000ms (the default) is ok for a small hsqldb, but often too short for mysql
       driver = "slick.driver.HsqldbDriver$"
       liquibase = {
         changelog = "src/main/migrations/changelog.xml"
@@ -156,6 +158,7 @@ database {
       db.user = "travis"
       db.password = ""
       db.driver = "com.mysql.jdbc.Driver"
+      db.connectionTimeout = 5000 // NOTE: The default 1000ms is often too short for production mysql use
       driver = "slick.driver.MySQLDriver$"
       liquibase = {
         changelog = "src/main/migrations/changelog.xml"

diff --git a/src/main/resources/swagger/cromwell.yaml b/src/main/resources/swagger/cromwell.yaml
@@ -105,7 +105,7 @@ paths:
           in: formData
         - name: workflowInputs
           description: WDL JSON
-          required: true
+          required: false
           type: file
           in: formData
       tags:
@@ -597,12 +597,12 @@ definitions:
     required:
       - valid
     properties:
-      valid:
-        type: boolean
-        description: The validation of the workflow
-      error:
+      status:
+        type: string
+        description: The validation status of the workflow
+      message:
         type: string
-        description: The validation error of the workflow
+        description: Details about the validation
   WorkflowSubmitResponse:
     description: ''
     required:

diff --git a/src/main/resources/workflowTimings/workflowTimings.html b/src/main/resources/workflowTimings/workflowTimings.html
@@ -7,6 +7,14 @@
 <script type="text/javascript">
 
     google.setOnLoadCallback(drawChart);
+    function addDataTableRow(dataTable, callName, callPartName, startDate, endDate) {
+        if (startDate <= endDate) {
+            dataTable.addRow([callName, callPartName, startDate, endDate]);
+        } else {
+            console.error("Unable to add '" + callName + "'s entry: '" + callPartName + "' because start-time '" + startDate + "'' is greater than end-time '" + endDate + "'");
+        }
+    }
+
     function drawChart() {
         $.getJSON("./metadata", function( data ) {
 
@@ -69,7 +77,7 @@
                                 finalEventEnd = endDate
                             }
                             // NB: See the column definitions above.
-                            dataTable.addRow([thisCallName, description, startDate, endDate]);
+                            addDataTableRow(dataTable, thisCallName, description, startDate, endDate);
                         }
 
                         if (callList[callIndex].hasOwnProperty("start")) {
@@ -86,19 +94,19 @@
                         }
 
                         if (firstEventStart == null || finalEventEnd == null) {
-                            dataTable.addRow([thisCallName, callList[callIndex].executionStatus, callStart, callEnd]);
+                            addDataTableRow(dataTable, thisCallName, callList[callIndex].executionStatus, callStart, callEnd);
                         } else {
-                            if (callStart < firstEventStart) dataTable.addRow([thisCallName, "cromwell starting overhead", callStart, firstEventStart]);
-                            if (callEnd > finalEventEnd) dataTable.addRow([thisCallName, "cromwell final overhead", finalEventEnd, callEnd]);
+                            if (callStart < firstEventStart) addDataTableRow(dataTable, thisCallName, "cromwell starting overhead", callStart, firstEventStart);
+                            if (callEnd > finalEventEnd) addDataTableRow(dataTable, thisCallName, "cromwell final overhead", finalEventEnd, callEnd);
                         }
                     } else if (callList[callIndex].executionStatus == "Running") {
                         executionCallsCount++;
                         var endDate = workflowEnd;
                         if(endDate == null) {
-                            dataTable.addRow([thisCallName, "Running", new Date(callList[callIndex].start), new Date(Date.now())]);
+                            addDataTableRow(dataTable, thisCallName, "Running", new Date(callList[callIndex].start), new Date(Date.now()));
                         }
                         else {
-                            dataTable.addRow([thisCallName, "Still running when workflow ended", new Date(callList[callIndex].start), endDate]);
+                            addDataTableRow(dataTable, thisCallName, "Still running when workflow ended", new Date(callList[callIndex].start), endDate);
                         }
                     }
                 }