Merge branch 'release/0.4.1'

twitter · Mar 20, 2014 · 1ad11cf · 1ad11cf
2 parents ae86cf0 + 4000555
commit 1ad11cf
Show file tree

Hide file tree

Showing 102 changed files with 2,651 additions and 1,596 deletions.
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,4 @@ project/plugins/src_managed/
 */.settings
 */.cache
 */.classpath
+vagrant/
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,8 @@
 # summingbird #
 
+## 0.4.1
+* Release tool failure fix from 0.4.0 release.
+
 ## 0.4.0
 * Add/use the multiplier option: https://github.com/twitter/summingbird/pull/481
 * add variations on pipeFactory to allow a map function before time extrac...: https://github.com/twitter/summingbird/pull/482

diff --git a/README.md b/README.md
@@ -102,7 +102,7 @@ Follow [@summingbird](https://twitter.com/summingbird) on Twitter for updates.
 
 ## Maven
 
-Summingbird modules are published on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.4.0`.
+Summingbird modules are published on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.4.1`.
 
 Current published artifacts are
 

diff --git a/project/Build.scala b/project/Build.scala
@@ -18,9 +18,11 @@ object SummingbirdBuild extends Build {
       case version if version startsWith "2.10" => "org.specs2" %% "specs2" % "1.13" % "test"
   }
 
-  val sharedSettings = Project.defaultSettings ++ Seq(
+  val extraSettings = Project.defaultSettings ++ mimaDefaultSettings
+
+  val sharedSettings = extraSettings ++ Seq(
     organization := "com.twitter",
-    version := "0.4.0",
+    version := "0.4.1",
     scalaVersion := "2.9.3",
     crossScalaVersions := Seq("2.9.3", "2.10.0"),
     libraryDependencies ++= Seq(
@@ -110,22 +112,25 @@ object SummingbirdBuild extends Build {
   ).aggregate(
     summingbirdCore,
     summingbirdBatch,
+    summingbirdBatchHadoop,
     summingbirdOnline,
     summingbirdClient,
     summingbirdStorm,
+    summingbirdStormTest,
     summingbirdScalding,
+    summingbirdScaldingTest,
     summingbirdBuilder,
     summingbirdChill,
     summingbirdExample
   )
 
   val dfsDatastoresVersion = "1.3.4"
-  val bijectionVersion = "0.6.0"
-  val algebirdVersion = "0.3.1"
-  val scaldingVersion = "0.9.0rc4"
-  val storehausVersion = "0.8.0"
+  val bijectionVersion = "0.6.2"
+  val algebirdVersion = "0.5.0"
+  val scaldingVersion = "0.9.0rc15"
+  val storehausVersion = "0.9.0"
   val utilVersion = "6.3.8"
-  val chillVersion = "0.3.5"
+  val chillVersion = "0.3.6"
   val tormentaVersion = "0.7.0"
 
   lazy val slf4jVersion = "1.6.6"
@@ -139,7 +144,7 @@ object SummingbirdBuild extends Build {
   def youngestForwardCompatible(subProj: String) =
     Some(subProj)
       .filterNot(unreleasedModules.contains(_))
-      .map { s => "com.twitter" % ("summingbird-" + s + "_2.9.3") % "0.2.4" }
+      .map { s => "com.twitter" % ("summingbird-" + s + "_2.9.3") % "0.4.1" }
 
   def module(name: String) = {
     val id = "summingbird-%s".format(name)
@@ -152,7 +157,8 @@ object SummingbirdBuild extends Build {
   lazy val summingbirdBatch = module("batch").settings(
     libraryDependencies ++= Seq(
       "com.twitter" %% "algebird-core" % algebirdVersion,
-      "com.twitter" %% "bijection-core" % bijectionVersion
+      "com.twitter" %% "bijection-core" % bijectionVersion,
+      "com.twitter" %% "scalding-date" % scaldingVersion
     )
   )
 
@@ -190,8 +196,7 @@ object SummingbirdBuild extends Build {
       withCross("com.twitter" %% "util-core" % utilVersion)
     )
   ).dependsOn(
-    summingbirdCore % "test->test;compile->compile",
-    summingbirdBatch
+    summingbirdCore % "test->test;compile->compile"
   )
 
   lazy val summingbirdStorm = module("storm").settings(
@@ -216,6 +221,22 @@ object SummingbirdBuild extends Build {
     summingbirdBatch
   )
 
+  lazy val summingbirdStormTest = module("storm-test").settings(
+    parallelExecution in Test := false,
+    libraryDependencies ++= Seq(
+      "com.twitter" %% "algebird-core" % algebirdVersion,
+      "com.twitter" %% "bijection-core" % bijectionVersion,
+      "com.twitter" %% "storehaus-core" % storehausVersion,
+      "com.twitter" %% "storehaus-algebra" % storehausVersion,
+      "com.twitter" %% "tormenta-core" % tormentaVersion,
+      withCross("com.twitter" %% "util-core" % utilVersion),
+      "storm" % "storm" % "0.9.0-wip15" % "provided"
+    )
+  ).dependsOn(
+    summingbirdCore % "test->test;compile->compile",
+    summingbirdStorm
+  )
+
   lazy val summingbirdScalding = module("scalding").settings(
     libraryDependencies ++= Seq(
       "com.backtype" % "dfs-datastores" % dfsDatastoresVersion,
@@ -234,6 +255,30 @@ object SummingbirdBuild extends Build {
   ).dependsOn(
     summingbirdCore % "test->test;compile->compile",
     summingbirdChill,
+    summingbirdBatchHadoop,
+    summingbirdBatch
+  )
+
+  lazy val summingbirdScaldingTest = module("scalding-test").settings(
+    libraryDependencies ++= Seq(
+      "org.scalacheck" %% "scalacheck" % "1.10.0"
+    )
+  ).dependsOn(
+    summingbirdCore % "test->test;compile->compile",
+    summingbirdChill,
+    summingbirdBatchHadoop,
+    summingbirdScalding
+  )
+
+  lazy val summingbirdBatchHadoop = module("batch-hadoop").settings(
+    libraryDependencies ++= Seq(
+      "com.backtype" % "dfs-datastores" % dfsDatastoresVersion,
+      "com.twitter" %% "algebird-core" % algebirdVersion,
+      "com.twitter" %% "bijection-json" % bijectionVersion,
+      "com.twitter" %% "scalding-date" % scaldingVersion
+    )
+  ).dependsOn(
+    summingbirdCore % "test->test;compile->compile",
     summingbirdBatch
   )
 

diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,3 +1,5 @@
+resolvers += Resolver.url("sbt-plugin-releases", new URL("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases/"))(Resolver.ivyStylePatterns)
+
 resolvers ++= Seq(
   "jgit-repo" at "http://download.eclipse.org/jgit/maven",
   "sonatype-releases"  at "http://oss.sonatype.org/content/repositories/releases"

diff --git a/...summingbird/scalding/ScaldingConfig.scala → ...itter/summingbird/batch/BatchConfig.scala b/...summingbird/scalding/ScaldingConfig.scala → ...itter/summingbird/batch/BatchConfig.scala
@@ -14,13 +14,13 @@
  limitations under the License.
  */
 
-package com.twitter.summingbird.scalding
+package com.twitter.summingbird.batch
 
 import scala.collection.JavaConverters._
 import org.apache.hadoop.conf.Configuration
 import com.twitter.summingbird.{WrappingConfig, ReadableMap}
 
-object ScaldingConfig {
+object BatchConfig {
   def apply(backingConfig: Configuration) = WrappingConfig(new WrappedHadoopConfig(backingConfig))
 }
 

diff --git a/...ummingbird/scalding/ConfigBijection.scala → ...r/summingbird/batch/ConfigBijection.scala b/...ummingbird/scalding/ConfigBijection.scala → ...r/summingbird/batch/ConfigBijection.scala
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package com.twitter.summingbird.scalding
+package com.twitter.summingbird.batch
 
 import com.twitter.bijection.Bijection
 import java.util.{ Map => JMap }
@@ -27,7 +27,7 @@ import scala.collection.JavaConverters._
  * @author Ashu Singhal
  */
 
-object ConfigBijection {
+private[summingbird] object ConfigBijection {
   implicit val fromMap: Bijection[Map[String, AnyRef], Configuration] =
     new Bijection[Map[String, AnyRef], Configuration] {
       override def apply(config: Map[String, AnyRef]) = {

diff --git a/...r/summingbird/scalding/IteratorSums.scala → ...tter/summingbird/batch/IteratorSums.scala b/...r/summingbird/scalding/IteratorSums.scala → ...tter/summingbird/batch/IteratorSums.scala
@@ -14,12 +14,12 @@
  limitations under the License.
  */
 
-package com.twitter.summingbird.scalding
+package com.twitter.summingbird.batch
 import com.twitter.algebird.{Semigroup, StatefulSummer}
 
 import scala.collection.mutable.ArrayBuffer
 
-object IteratorSums extends java.io.Serializable {
+private[summingbird] object IteratorSums extends java.io.Serializable {
 
   def sumWith[T](it: Iterator[T], summer: StatefulSummer[T]): Iterator[T] =
     // this is for MAXIMUM speed. Any ideas to speed it up, say so + benchmark

diff --git a/...itter/summingbird/scalding/JobState.scala → .../twitter/summingbird/batch/JobState.scala b/...itter/summingbird/scalding/JobState.scala → .../twitter/summingbird/batch/JobState.scala
@@ -14,7 +14,7 @@
  limitations under the License.
  */
 
-package com.twitter.summingbird.scalding
+package com.twitter.summingbird.batch
 
 /**
  * Job state models the memory of when the next job should

diff --git a/...gbird/scalding/option/FlatMapShards.scala → ...mingbird/batch/option/FlatMapShards.scala b/...gbird/scalding/option/FlatMapShards.scala → ...mingbird/batch/option/FlatMapShards.scala
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package com.twitter.summingbird.scalding.option
+package com.twitter.summingbird.batch.option
 
 object FlatMapShards {
   val default = FlatMapShards(0)

diff --git a/...ummingbird/scalding/option/Reducers.scala → ...r/summingbird/batch/option/Reducers.scala b/...ummingbird/scalding/option/Reducers.scala → ...r/summingbird/batch/option/Reducers.scala
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package com.twitter.summingbird.scalding.option
+package com.twitter.summingbird.batch.option
 
 object Reducers {
   val default = Reducers(-1)

diff --git a/.../scalding/state/FileVersionTracking.scala → ...ird/batch/state/FileVersionTracking.scala b/.../scalding/state/FileVersionTracking.scala → ...ird/batch/state/FileVersionTracking.scala
@@ -14,7 +14,7 @@
   limitations under the License.
 */
 
-package com.twitter.summingbird.scalding.state
+package com.twitter.summingbird.batch.state
 
 import org.apache.hadoop.fs.FileSystem
 import org.apache.hadoop.fs.Path
@@ -24,14 +24,14 @@ import scala.util.{Try, Success, Failure}
 
 import org.slf4j.LoggerFactory
 
-object FileVersionTracking {
+private[summingbird] object FileVersionTracking {
   @transient private val logger = LoggerFactory.getLogger(classOf[FileVersionTracking])
   val FINISHED_VERSION_SUFFIX = ".version"
   implicit def path(strPath: String): Path = new Path(strPath)
   def path(basePath: String, fileName: String): Path = new Path(basePath, fileName)
 }
 
-case class FileVersionTracking(root: String, fs: FileSystem) {
+private[summingbird] case class FileVersionTracking(root: String, fs: FileSystem) {
   import FileVersionTracking._
 
   fs.mkdirs(root)

diff --git a/...ummingbird/scalding/state/HDFSState.scala → ...r/summingbird/batch/state/HDFSState.scala b/...ummingbird/scalding/state/HDFSState.scala → ...r/summingbird/batch/state/HDFSState.scala
@@ -13,22 +13,26 @@
  See the License for the specific language governing permissions and
  limitations under the License.
  */
-package com.twitter.summingbird.scalding.state
-
-import com.twitter.algebird.ExclusiveLower
-import com.twitter.algebird.InclusiveLower
-import com.twitter.algebird.InclusiveUpper
-import com.twitter.algebird.Lower
-import com.twitter.algebird.Upper
-import com.twitter.algebird.{ ExclusiveUpper, Intersection, Interval }
+package com.twitter.summingbird.batch.state
+
+import com.twitter.algebird.{
+  ExclusiveLower,
+  ExclusiveUpper,
+  InclusiveLower,
+  InclusiveUpper,
+  Intersection,
+  Interval,
+  Lower,
+  Upper
+}
 import com.twitter.bijection.Conversion.asMethod
 import com.twitter.summingbird.batch.{ BatchID, Batcher, Timestamp }
-import com.twitter.summingbird.scalding.{ PrepareState, RunningState, WaitingState, Scalding }
+import com.twitter.summingbird.batch.{ PrepareState, RunningState, WaitingState }
 import java.util.concurrent.atomic.AtomicBoolean
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.FileSystem
 
-import Scalding.dateRangeInjection
+// import Scalding.dateRangeInjection
 
 import org.slf4j.LoggerFactory
 /**
@@ -141,7 +145,7 @@ class HDFSState(config: HDFSState.Config)(implicit batcher: Batcher)
     override def fail(err: Throwable) = throw err
   }
 
-  private class Running(succeedPart: Intersection[Timestamp], bool: AtomicBoolean)
+  private class Running(succeedPart: Interval.GenIntersection[Timestamp], bool: AtomicBoolean)
     extends RunningState[Interval[Timestamp]] {
     def setStopped = assert(
       bool.compareAndSet(true, false),

diff --git a/...ingbird/scalding/store/HDFSMetadata.scala → ...ummingbird/batch/store/HDFSMetadata.scala b/...ingbird/scalding/store/HDFSMetadata.scala → ...ummingbird/batch/store/HDFSMetadata.scala
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package com.twitter.summingbird.scalding.store
+package com.twitter.summingbird.batch.store
 
 import com.backtype.hadoop.datastores.{ VersionedStore => BacktypeVersionedStore }
 import com.twitter.bijection.json.{ JsonInjection, JsonNodeInjection }
@@ -54,7 +54,7 @@ import scala.util.control.Exception.allCatch
 // pull req to scalding-commons that adds this generic capability to
 // VersionedKeyValSource.
 
-object HDFSMetadata {
+private[summingbird] object HDFSMetadata {
   val METADATA_FILE = "_summingbird.json"
 
   def apply(conf: Configuration, rootPath: String): HDFSMetadata =
@@ -132,7 +132,7 @@ class HDFSMetadata(conf: Configuration, rootPath: String) {
 
 /** Refers to a specific version on disk. Allows reading and writing metadata to specific locations
  */
-class HDFSVersionMetadata private[store] (val version: Long, conf: Configuration, val path: Path) {
+private[summingbird] class HDFSVersionMetadata private[store] (val version: Long, conf: Configuration, val path: Path) {
   private def getString: Try[String] =
     Try {
       val fs = FileSystem.get(conf)

diff --git a/.../summingbird/scalding/HDFSStateLaws.scala → ...ter/summingbird/batch/HDFSStateLaws.scala b/.../summingbird/scalding/HDFSStateLaws.scala → ...ter/summingbird/batch/HDFSStateLaws.scala
@@ -14,13 +14,13 @@
  limitations under the License.
  */
 
-package com.twitter.summingbird.scalding
+package com.twitter.summingbird.batch
 
 import com.twitter.algebird.Interval
 import com.twitter.scalding.{DateParser, RichDate}
 import com.twitter.summingbird._
 import com.twitter.summingbird.batch._
-import com.twitter.summingbird.scalding.state.HDFSState
+import com.twitter.summingbird.batch.state.HDFSState
 import java.io.File
 import java.util.{ TimeZone, UUID }
 import org.apache.hadoop.conf.Configuration
@@ -84,13 +84,15 @@ object HDFSStateLaws extends Specification {
     val config = HDFSState.Config(path, new Configuration, startDate, numBatches)
 
     // Not aligned with batch size
-    val partialIncompleteInterval: Interval[Timestamp] = Interval.leftClosedRightOpen(
+    val partialIncompleteInterval: Interval[Timestamp] = Interval.leftClosedRightOpen[Timestamp](
       batcher.earliestTimeOf(batcher.batchOf(startDate.get)),
-      RichDate("2012-12-26T10:40").value)
+      RichDate("2012-12-26T10:40").value
+    ).right.get
 
-    val partialCompleteInterval: Interval[Timestamp] = Interval.leftClosedRightOpen(
+    val partialCompleteInterval: Interval[Timestamp] = Interval.leftClosedRightOpen[Timestamp](
       batcher.earliestTimeOf(batcher.batchOf(startDate.get)),
-      RichDate("2012-12-26T11:30").value)
+      RichDate("2012-12-26T11:30").value
+    ).right.get
 
     val runningState = HDFSState(config).begin.willAccept(partialIncompleteInterval)
     val waitingState = runningState match {

diff --git a/...ummingbird/scalding/IteratorSumLaws.scala → ...r/summingbird/batch/IteratorSumLaws.scala b/...ummingbird/scalding/IteratorSumLaws.scala → ...r/summingbird/batch/IteratorSumLaws.scala
@@ -14,7 +14,7 @@
  limitations under the License.
  */
 
-package com.twitter.summingbird.scalding
+package com.twitter.summingbird.batch
 
 import org.scalacheck._
 import Gen._