Merge pull request #143 from broadinstitute/tj_DBScatterChanges

Scatter Gather Database schema changes

src/main/migrations/changelog.xml

@@ -11,4 +11,6 @@
     <include file="changesets/symbol_iteration_not_null.xml" relativeToChangelogFile="true" />
     <include file="changesets/add_unique_constraints.xml" relativeToChangelogFile="true" />
     <include file="changesets/lengthen_wdl_value.xml" relativeToChangelogFile="true" />
+    <include file="changesets/add_index_in_execution.xml" relativeToChangelogFile="true" />
+    <include file="changesets/rename_iteration_to_index.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>

src/main/migrations/changesets/add_index_in_execution.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="tjeandet" id="add-index-in-execution">
+        <addColumn
+            tableName="EXECUTION">
+            <column name="INDEX" type="INT" />
+        </addColumn>
+        <addNotNullConstraint
+                tableName="EXECUTION" columnName="INDEX"
+                columnDataType="INT" defaultNullValue="-1"/>
+    </changeSet>
+
+</databaseChangeLog>

src/main/migrations/changesets/rename_iteration_to_index.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="tjeandet" id="rename-iteration-to-index">
+        <renameColumn columnDataType="INT"
+                      newColumnName="INDEX"
+                      oldColumnName="ITERATION"
+                      tableName="SYMBOL"/>
+    </changeSet>
+
+</databaseChangeLog>

src/main/scala/cromwell/engine/db/slick/ExecutionComponent.scala

@@ -4,6 +4,7 @@ case class Execution
 (
   workflowExecutionId: Int,
   callFqn: String,
+  index: Int,
   status: String,
   executionId: Option[Int] = None
   )
@@ -20,9 +21,11 @@ trait ExecutionComponent {
 
     def callFqn = column[String]("CALL_FQN")
 
+    def index = column[Int]("INDEX")
+
     def status = column[String]("STATUS")
 
-    override def * = (workflowExecutionId, callFqn, status, executionId.?) <>
+    override def * = (workflowExecutionId, callFqn, index, status, executionId.?) <>
       (Execution.tupled, Execution.unapply)
 
     def workflowExecution = foreignKey(

src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala

@@ -8,7 +8,7 @@ import _root_.slick.util.ConfigExtensionMethods._
 import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory}
 import cromwell.binding._
 import cromwell.binding.types.{WdlPrimitiveType, WdlType}
-import cromwell.binding.values.{WdlValue, WdlPrimitive}
+import cromwell.binding.values.WdlValue
 import cromwell.engine._
 import cromwell.engine.backend.Backend
 import cromwell.engine.backend.jes.JesBackend
@@ -25,7 +25,7 @@ object SlickDataAccess {
   type IoValue = String
   val IoInput = "INPUT"
   val IoOutput = "OUTPUT"
-  val IterationNone = -1 // "It's a feature" https://bugs.mysql.com/bug.php?id=8173
+  val IndexNone = -1 // "It's a feature" https://bugs.mysql.com/bug.php?id=8173
 
   implicit class DateToTimestamp(val date: Date) extends AnyVal {
     def toTimestamp = new Timestamp(date.getTime)
@@ -183,7 +183,7 @@ class SlickDataAccess(databaseConfig: Config, val dataAccess: DataAccessComponen
         workflowExecution.workflowExecutionId.get,
         symbol.key.scope,
         symbol.key.name,
-        symbol.key.iteration.getOrElse(IterationNone),
+        symbol.key.index.getOrElse(IndexNone),
         if (symbol.isInput) IoInput else IoOutput,
         symbol.wdlType.toWdlString,
         symbol.wdlValue.map(v => wdlValueToDbValue(v).toClob))
@@ -206,6 +206,7 @@ class SlickDataAccess(databaseConfig: Config, val dataAccess: DataAccessComponen
         new Execution(
           workflowExecution.workflowExecutionId.get,
           call.fullyQualifiedName,
+          IndexNone,
           ExecutionStatus.NotStarted.toString)
 
       // Depending on the backend, insert a job specific row
@@ -388,7 +389,7 @@ class SlickDataAccess(databaseConfig: Config, val dataAccess: DataAccessComponen
       new SymbolStoreKey(
         symbolResult.scope,
         symbolResult.name,
-        Option(symbolResult.iteration).filterNot(_ == IterationNone),
+        Option(symbolResult.index).filterNot(_ == IndexNone),
         input = symbolResult.io == IoInput // input = true, if db contains "INPUT"
       ),
       wdlType,
@@ -455,7 +456,7 @@ class SlickDataAccess(databaseConfig: Config, val dataAccess: DataAccessComponen
             workflowExecution.workflowExecutionId.get,
             call.fullyQualifiedName,
             symbolLocallyQualifiedName,
-            IterationNone,
+            IndexNone,
             IoOutput,
             wdlValue.wdlType.toWdlString,
             Option(wdlValueToDbValue(wdlValue).toClob))

src/main/scala/cromwell/engine/db/slick/SymbolComponent.scala

@@ -7,7 +7,7 @@ case class Symbol
   workflowExecutionId: Int,
   scope: String,
   name: String,
-  iteration: Int, // https://bugs.mysql.com/bug.php?id=8173
+  index: Int, // https://bugs.mysql.com/bug.php?id=8173
   io: String,
   wdlType: String,
   wdlValue: Option[Clob],
@@ -28,22 +28,22 @@ trait SymbolComponent {
 
     def name = column[String]("NAME")
 
-    def iteration = column[Int]("ITERATION")
+    def index = column[Int]("INDEX")
 
     def io = column[String]("IO")
 
     def wdlType = column[String]("WDL_TYPE")
 
     def wdlValue = column[Option[Clob]]("WDL_VALUE")
 
-    override def * = (workflowExecutionId, scope, name, iteration, io, wdlType, wdlValue, symbolId.?) <>
+    override def * = (workflowExecutionId, scope, name, index, io, wdlType, wdlValue, symbolId.?) <>
       (Symbol.tupled, Symbol.unapply)
 
     def workflowExecution = foreignKey(
       "FK_SYMBOL_WORKFLOW_EXECUTION_ID", workflowExecutionId, workflowExecutions)(_.workflowExecutionId)
 
     def uniqueKey = index("UK_SYM_WORKFLOW_EXECUTION_ID_SCOPE_NAME_ITERATION_IO",
-      (workflowExecutionId, scope, name, iteration, io), unique = true)
+      (workflowExecutionId, scope, name, index, io), unique = true)
   }
 
   protected val symbols = TableQuery[Symbols]

src/main/scala/cromwell/engine/package.scala

@@ -82,7 +82,7 @@ package object engine {
 
     def apply(fullyQualifiedName: FullyQualifiedName, wdlValue: WdlValue, input: Boolean): SymbolStoreEntry = {
       val (scope, name) = splitFqn(fullyQualifiedName)
-      val key = SymbolStoreKey(scope, name, iteration = None, input)
+      val key = SymbolStoreKey(scope, name, index = None, input)
       SymbolStoreEntry(key, wdlValue.wdlType, Some(wdlValue))
     }
 
@@ -95,7 +95,7 @@ package object engine {
     }.toMap
   }
 
-  case class SymbolStoreKey(scope: String, name: String, iteration: Option[Int], input: Boolean)
+  case class SymbolStoreKey(scope: String, name: String, index: Option[Int], input: Boolean)
 
   case class SymbolStoreEntry(key: SymbolStoreKey, wdlType: WdlType, wdlValue: Option[WdlValue]) {
     def isInput: Boolean = key.input

src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala

@@ -293,7 +293,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures {
           val resultSymbolStoreKey = resultSymbol.key
           resultSymbolStoreKey.scope should be("call.fully.qualified.scope")
           resultSymbolStoreKey.name should be("symbol.fully.qualified.scope")
-          resultSymbolStoreKey.iteration should be(None)
+          resultSymbolStoreKey.index should be(None)
           resultSymbolStoreKey.input should be(right = true) // IntelliJ highlighting
           resultSymbol.wdlType should be(WdlStringType)
           resultSymbol.wdlValue shouldNot be(empty)
@@ -324,7 +324,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures {
           val resultSymbolStoreKey = resultSymbol.key
           resultSymbolStoreKey.scope should be("call.fully.qualified.scope")
           resultSymbolStoreKey.name should be("symbol.fully.qualified.scope")
-          resultSymbolStoreKey.iteration should be(None)
+          resultSymbolStoreKey.index should be(None)
           resultSymbolStoreKey.input should be(right = true) // IntelliJ highlighting
           resultSymbol.wdlType should be(WdlArrayType(WdlStringType))
           resultSymbol.wdlValue shouldNot be(empty)
@@ -364,7 +364,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures {
           val resultSymbolStoreKey = resultSymbol.key
           resultSymbolStoreKey.scope should be("call.fully.qualified.scope")
           resultSymbolStoreKey.name should be("symbol")
-          resultSymbolStoreKey.iteration should be(None)
+          resultSymbolStoreKey.index should be(None)
           resultSymbolStoreKey.input should be(right = false) // IntelliJ highlighting
           resultSymbol.wdlType should be(WdlStringType)
           resultSymbol.wdlValue shouldNot be(empty)
@@ -392,7 +392,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures {
           val resultSymbolStoreKey = resultSymbol.key
           resultSymbolStoreKey.scope should be("call.fully.qualified.scope")
           resultSymbolStoreKey.name should be("symbol")
-          resultSymbolStoreKey.iteration should be(None)
+          resultSymbolStoreKey.index should be(None)
           resultSymbolStoreKey.input should be(right = false) // IntelliJ highlighting
           resultSymbol.wdlType should be(WdlStringType)
           resultSymbol.wdlValue shouldNot be(empty)
@@ -459,7 +459,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures {
           val resultSymbolStoreKey = resultSymbol.key
           resultSymbolStoreKey.scope should be("call.fully.qualified.scope")
           resultSymbolStoreKey.name should be("symbol")
-          resultSymbolStoreKey.iteration should be(None)
+          resultSymbolStoreKey.index should be(None)
           resultSymbolStoreKey.input should be(right = false) // IntelliJ highlighting
           resultSymbol.wdlType should be(WdlStringType)
           resultSymbol.wdlValue shouldNot be(empty)