Merge branch 'release/0.9.0'

twitter · Oct 22, 2015 · 7d44263 · 7d44263
2 parents 34611dd + f51c2b3
commit 7d44263
Show file tree

Hide file tree

Showing 78 changed files with 905 additions and 3,037 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,6 +5,7 @@ sudo: false
 before_install: umask 0022
 scala:
   - 2.10.5
+  - 2.11.7
 script:
 - "echo no op"
 
@@ -32,13 +33,9 @@ matrix:
       script: "scripts/run_tests.sh"
 
     - scala: 2.10.5
-      env: BUILD="base" TEST_TARGET="summingbird-core-java"
+      env: BUILD="base" TEST_TARGET="summingbird-core-test"
       script: "scripts/run_tests.sh"
 
-#    - scala: 2.10.5
-#      env: BUILD="base" TEST_TARGET="summingbird-core-test"
-#      script: "scripts/run_tests.sh"
-
     - scala: 2.10.5
       env: BUILD="base" TEST_TARGET="summingbird-example"
       script: "scripts/run_tests.sh"
@@ -56,18 +53,57 @@ matrix:
       script: "scripts/run_tests.sh"
 
     - scala: 2.10.5
-      env: BUILD="base" TEST_TARGET="summingbird-spark"
+      env: BUILD="base" TEST_TARGET="summingbird-storm"
       script: "scripts/run_tests.sh"
 
     - scala: 2.10.5
-      env: BUILD="base" TEST_TARGET="summingbird-storm"
+      env: BUILD="base" TEST_TARGET="summingbird-storm-test"
       script: "scripts/run_tests.sh"
 
-    - scala: 2.10.5
-      env: BUILD="base" TEST_TARGET="summingbird-storm-java"
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-batch"
       script: "scripts/run_tests.sh"
 
-    - scala: 2.10.5
-      env: BUILD="base" TEST_TARGET="summingbird-storm-test"
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-batch-hadoop"
       script: "scripts/run_tests.sh"
 
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-builder"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-chill"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-core"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-core-test"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-example"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-online"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-scalding"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-scalding-test"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-storm"
+      script: "scripts/run_tests.sh"
+
+    - scala: 2.11.7
+      env: BUILD="base" TEST_TARGET="summingbird-storm-test"
+      script: "scripts/run_tests.sh"
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,18 @@
 # Summingbird #
 
+## 0.9.0
+* Removing internal config setup from scalding platform: https://github.com/twitter/summingbird/pull/629
+* Remove store that no one seems to use and has no tests: https://github.com/twitter/summingbird/pull/630
+* Ianoc/optimization changes: https://github.com/twitter/summingbird/pull/628
+* Fixes tests: https://github.com/twitter/summingbird/pull/626
+* resilient to all stores being complete: https://github.com/twitter/summingbird/pull/627
+* Resilient to a store being ahead: https://github.com/twitter/summingbird/pull/625
+* move to scala test: https://github.com/twitter/summingbird/pull/621
+* Killing off Summingbird Java: https://github.com/twitter/summingbird/pull/624
+* Kill summingbird spark: https://github.com/twitter/summingbird/pull/622
+* Bumping sbt versions: https://github.com/twitter/summingbird/pull/623
+* Fixed erroneous comment in ClientStore&#39;s merge description: https://github.com/twitter/summingbird/pull/620
+
 ## 0.8.0
 * bumping Scalding to 0.15.0, algebird to 0.10.1
 

diff --git a/README.md b/README.md
@@ -108,21 +108,21 @@ Please feel free to use the beautiful [Summingbird logo](https://drive.google.co
 
 ## Maven
 
-Summingbird modules are published on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.8.0`.
+Summingbird modules are published on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.9.0`.
 
 Current published artifacts are
 
-* `summingbird-core_2.9.3`
+* `summingbird-core_2.11`
 * `summingbird-core_2.10`
-* `summingbird-batch_2.9.3`
+* `summingbird-batch_2.11`
 * `summingbird-batch_2.10`
-* `summingbird-client_2.9.3`
+* `summingbird-client_2.11`
 * `summingbird-client_2.10`
-* `summingbird-storm_2.9.3`
+* `summingbird-storm_2.11`
 * `summingbird-storm_2.10`
-* `summingbird-scalding_2.9.3`
+* `summingbird-scalding_2.11`
 * `summingbird-scalding_2.10`
-* `summingbird-builder_2.9.3`
+* `summingbird-builder_2.11`
 * `summingbird-builder_2.10`
 
 The suffix denotes the scala version.

diff --git a/project/Build.scala b/project/Build.scala
@@ -21,36 +21,33 @@ object SummingbirdBuild extends Build {
 
   val scalaCheckVersion = "1.12.2"
   val hadoopVersion = "1.2.1"
-  val algebirdVersion = "0.10.1"
-  val bijectionVersion = "0.8.0"
-  val chillVersion = "0.6.0"
+  val algebirdVersion = "0.11.0"
+  val bijectionVersion = "0.8.1"
+  val chillVersion = "0.7.0"
   val slf4jVersion = "1.6.6"
-  val parquetVersion = "1.6.0rc4"
 
   val dfsDatastoresVersion = "1.3.6"
-  val scaldingVersion = "0.15.0"
-  val storehausVersion = "0.11.0"
-  val utilVersion = "6.3.8"
+  val scaldingVersion = "0.15.1-RC13"
+  val storehausVersion = "0.12.0"
+  val utilVersion = "6.26.0"
 
-  val finagleVersion = "6.12.2"
-  val tormentaVersion = "0.10.0"
+  val finagleVersion = "6.27.0"
+  val tormentaVersion = "0.11.0"
   val junitVersion = "4.11"
   val log4jVersion = "1.2.16"
   val stormVersion = "0.9.0-wip15"
   val commonsLangVersion = "2.6"
   val novocodeJunitVersion = "0.10"
-  val specs2Version = "1.13"
+  val scalatestVersion = "2.2.4"
 
-  val sparkCoreVersion ="1.2.0"
   val commonsHttpClientVersion = "3.1"
 
   val extraSettings = Project.defaultSettings ++ mimaDefaultSettings ++ scalariformSettings
 
   val sharedSettings = extraSettings ++ Seq(
     organization := "com.twitter",
-    version := "0.8.0",
     scalaVersion := "2.10.5",
-    crossScalaVersions := Seq("2.10.5"),
+    crossScalaVersions := Seq("2.10.5", "2.11.7"),
     // To support hadoop 1.x
     javacOptions ++= Seq("-source", "1.6", "-target", "1.6"),
 
@@ -68,7 +65,7 @@ object SummingbirdBuild extends Build {
       // These satisify's scaldings log4j needs when in test mode
       "log4j" % "log4j" % log4jVersion % "test",
       "com.novocode" % "junit-interface" % novocodeJunitVersion % "test",
-      "org.specs2" %% "specs2" % specs2Version % "test"
+      "org.scalatest" %% "scalatest" % scalatestVersion % "test"
     ),
 
     resolvers ++= Seq(
@@ -155,17 +152,14 @@ object SummingbirdBuild extends Build {
     publishLocal := { }
   ).aggregate(
     summingbirdCore,
-    summingbirdCoreJava,
     summingbirdBatch,
     summingbirdBatchHadoop,
     summingbirdOnline,
     summingbirdClient,
     summingbirdStorm,
     summingbirdStormTest,
-    summingbirdStormJava,
     summingbirdScalding,
     summingbirdScaldingTest,
-    summingbirdSpark,
     summingbirdBuilder,
     summingbirdChill,
     summingbirdExample,
@@ -226,10 +220,6 @@ object SummingbirdBuild extends Build {
     libraryDependencies += "com.twitter" %% "algebird-core" % algebirdVersion
   )
 
-  lazy val summingbirdCoreJava = module("core-java").dependsOn(
-    summingbirdCore % "test->test;compile->compile"
-  )
-
   lazy val summingbirdOnline = module("online").settings(
     libraryDependencies ++= Seq(
       "com.twitter" %% "algebird-core" % algebirdVersion,
@@ -287,15 +277,6 @@ object SummingbirdBuild extends Build {
     summingbirdStorm
   )
 
-  lazy val summingbirdStormJava = module("storm-java").settings(
-    libraryDependencies ++= Seq(
-      "storm" % "storm" % stormVersion % "provided"
-    )
-  ).dependsOn(
-    summingbirdCore % "test->test;compile->compile",
-    summingbirdCoreJava % "test->test;compile->compile",
-    summingbirdStorm % "test->test;compile->compile"
-  )
   lazy val summingbirdScalding = module("scalding").settings(
     libraryDependencies ++= Seq(
       "com.backtype" % "dfs-datastores" % dfsDatastoresVersion,
@@ -368,62 +349,15 @@ object SummingbirdBuild extends Build {
       "com.twitter" %% "storehaus-memcache" % storehausVersion exclude("com.twitter.common", "dynamic-host-set") exclude("com.twitter.common", "service-thrift"),
       "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test"
     )
-  ).dependsOn(summingbirdCore, summingbirdCoreJava, summingbirdStorm, summingbirdStormJava)
-
-  lazy val sparkAssemblyMergeSettings = assemblySettings :+ {
-    mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
-      {
-        //case PathList("org", "w3c", xs @ _*) => MergeStrategy.first
-        //case "about.html"     => MergeStrategy.discard
-        case PathList("com", "esotericsoftware", "minlog", xs @ _*) => MergeStrategy.first
-        case PathList("org", "apache", "commons", "beanutils", xs @ _*) => MergeStrategy.first
-        case PathList("org", "apache", "commons", "collections", xs @ _*) => MergeStrategy.first
-        case PathList("org", "apache", "jasper", xs @ _*) => MergeStrategy.first
-        case "log4j.properties"     => MergeStrategy.concat
-        case x if x.endsWith(".xsd") || x.endsWith(".dtd") => MergeStrategy.first
-        case x => old(x)
-      }
-    }
-  }
-
-
-  val sparkDeps = Seq(
-    "com.twitter" %% "algebird-core" % algebirdVersion,
-    "com.twitter" %% "algebird-util" % algebirdVersion,
-    "com.twitter" %% "algebird-bijection" % algebirdVersion,
-    "com.twitter" %% "bijection-json" % bijectionVersion,
-    "com.twitter" %% "chill" % chillVersion,
-    "com.twitter" % "chill-hadoop" % chillVersion,
-    "com.twitter" %% "chill-bijection" % chillVersion,
-    "commons-lang" % "commons-lang" % commonsLangVersion,
-    "commons-httpclient" % "commons-httpclient" % commonsHttpClientVersion,
-    "org.apache.spark" %% "spark-core" % sparkCoreVersion % "provided"
-  )
-
-  def buildSparkDeps(scalaVersion: String) = if (isScala210x(scalaVersion)) sparkDeps else Seq()
-
-  lazy val summingbirdSpark = module("spark").settings(
-    resolvers += "Typesafe Repo" at "http://repo.typesafe.com/typesafe/releases/",
-    skip in compile := !isScala210x(scalaVersion.value),
-    skip in doc := !isScala210x(scalaVersion.value),
-    skip in test := !isScala210x(scalaVersion.value),
-    publishArtifact := isScala210x(scalaVersion.value),
-    libraryDependencies ++= buildSparkDeps(scalaVersion.value)
-  )
-  .settings(sparkAssemblyMergeSettings:_*)
-  .dependsOn(
-    summingbirdCore % "test->test;compile->compile",
-    summingbirdCoreTest % "test->test",
-    summingbirdChill
-  )
+  ).dependsOn(summingbirdCore, summingbirdStorm)
 
   lazy val summingbirdCoreTest = module("core-test").settings(
     parallelExecution in Test := false,
     libraryDependencies ++=Seq(
       "junit" % "junit" % junitVersion % "provided",
       "org.slf4j" % "slf4j-api" % slf4jVersion % "provided",
       "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "provided",
-      "org.specs2" %% "specs2" % specs2Version % "provided")
+      "org.scalatest" %% "scalatest" % scalatestVersion % "provided")
 
   ).dependsOn(
       summingbirdCore % "test->test;compile->compile"

diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=0.13.0
+sbt.version=0.13.8