predictionio cluster with hortonworks

.travis.yml

@@ -30,7 +30,7 @@ branches:
 language: scala
 
 jdk:
-  - oraclejdk8
+  - openjdk8
 
 services:
   - docker
@@ -47,15 +47,9 @@ env:
   matrix:
     - BUILD_TYPE=Unit
       METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
-    - BUILD_TYPE=Integration
-      METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=S3
-      PIO_ELASTICSEARCH_VERSION=1.7.3
-    - BUILD_TYPE=Integration
-      METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
-      PIO_ELASTICSEARCH_VERSION=5.6.9
     - BUILD_TYPE=Integration
       METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
-      PIO_ELASTICSEARCH_VERSION=6.4.2
+      PIO_ELASTICSEARCH_VERSION=6.8.1
     - BUILD_TYPE=Integration
       METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
       PIO_HBASE_VERSION=1.2.6
@@ -96,11 +90,6 @@ env:
       PIO_SCALA_VERSION=2.11.12
       PIO_SPARK_VERSION=2.2.3
       PIO_HADOOP_VERSION=2.7.7
-    - BUILD_TYPE=Integration
-      METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
-      PIO_SCALA_VERSION=2.11.12
-      PIO_SPARK_VERSION=2.3.3
-      PIO_HADOOP_VERSION=2.7.7
     - BUILD_TYPE=Integration
       METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
       PIO_SCALA_VERSION=2.11.12

LICENSE.txt

@@ -1708,7 +1708,6 @@ Binary distribution bundles
   org.scala-lang.modules # scala-parser-combinators_2.11 # 1.0.6 (http://scala-lang.org/)
   org.scala-lang.modules # scala-parser-combinators_2.11 # 1.1.0 (http://scala-lang.org/)
   org.scala-lang.modules # scala-xml_2.11 # 1.0.5 (http://scala-lang.org/)
-  org.scala-lang.modules # scala-xml_2.11 # 1.0.6 (http://scala-lang.org/)
 
   which is available under the BSD license (http://www.scala-lang.org/downloads/license.html)
 

PMC.md

@@ -27,38 +27,38 @@ http://apache.org/dev/openpgp.html#generate-key on how to generate a strong code
 signing key.
 2. Add your public key to the `KEYS` file at the root of the source code tree.
 3. Create a new release branch, with version bumped to the next release version.
-    * `git checkout -b release/0.14.0`
-    * Replace all `0.14.0-SNAPSHOT` in the code tree to `0.14.0`
-    * `git commit -am "Prepare 0.14.0-rc1"`
-    * `git tag -am "Apache PredictionIO 0.14.0-rc1" v0.14.0-rc1`
+    * `git checkout -b release/0.15.0`
+    * Replace all `0.15.0-SNAPSHOT` in the code tree to `0.15.0`
+    * `git commit -am "Prepare 0.15.0-rc1"`
+    * `git tag -am "Apache PredictionIO 0.15.0-rc1" v0.15.0-rc1`
 4. Push the release branch and tag to the apache git repo.
 5. Wait for Travis to pass build on the release branch.
 6. Package a clean tarball for staging a release candidate.
-    * `git archive --format tar v0.14.0-rc1 >
-  ../apache-predictionio-0.14.0-rc1.tar`
-    * `cd ..; gzip apache-predictionio-0.14.0-rc1.tar`
+    * `git archive --format tar v0.15.0-rc1 >
+  ../apache-predictionio-0.15.0-rc1.tar`
+    * `cd ..; gzip apache-predictionio-0.15.0-rc1.tar`
 7. Generate detached signature for the release candidate.
 (http://apache.org/dev/release-signing.html#openpgp-ascii-detach-sig)
-    * `gpg --armor --output apache-predictionio-0.14.0-rc1.tar.gz.asc
-  --detach-sig apache-predictionio-0.14.0-rc1.tar.gz`
+    * `gpg --armor --output apache-predictionio-0.15.0-rc1.tar.gz.asc
+  --detach-sig apache-predictionio-0.15.0-rc1.tar.gz`
 8. Generate SHA512 checksums for the release candidate.
-    * `gpg --print-md SHA512 apache-predictionio-0.14.0-rc1.tar.gz >
-  apache-predictionio-0.14.0-rc1.tar.gz.sha512`
+    * `gpg --print-md SHA512 apache-predictionio-0.15.0-rc1.tar.gz >
+  apache-predictionio-0.15.0-rc1.tar.gz.sha512`
 9. Run `./make-distribution.sh` and repeat steps 6 to 8 to create binary distribution release.
-    * `mv PredictionIO-0.14.0.tar.gz apache-predictionio-0.14.0-bin.tar.gz`
-    * `gpg --armor --output apache-predictionio-0.14.0-bin.tar.gz.asc
-  --detach-sig apache-predictionio-0.14.0-bin.tar.gz`
-    * `gpg --print-md SHA512 apache-predictionio-0.14.0-bin.tar.gz >
-  apache-predictionio-0.14.0-bin.tar.gz.sha512`
+    * `mv PredictionIO-0.15.0.tar.gz apache-predictionio-0.15.0-bin.tar.gz`
+    * `gpg --armor --output apache-predictionio-0.15.0-bin.tar.gz.asc
+  --detach-sig apache-predictionio-0.15.0-bin.tar.gz`
+    * `gpg --print-md SHA512 apache-predictionio-0.15.0-bin.tar.gz >
+  apache-predictionio-0.15.0-bin.tar.gz.sha512`
 10. If you have not done so, use SVN to checkout
 https://dist.apache.org/repos/dist/dev/predictionio. This is the area
 for staging release candidates for voting.
     * `svn co https://dist.apache.org/repos/dist/dev/predictionio`
 11. Create a subdirectory at the SVN staging area. The area should have a `KEYS` file.
-    * `mkdir apache-predictionio-0.14.0-rc1`
-    * `cp apache-predictionio-0.14.0-* apache-predictionio-0.14.0-rc1`
+    * `mkdir apache-predictionio-0.15.0-rc1`
+    * `cp apache-predictionio-0.15.0-* apache-predictionio-0.15.0-rc1`
 12. If you have updated the `KEYS` file, also copy that to the staging area.
-13. `svn commit -m "Apache PredictionIO 0.14.0-rc1"`
+13. `svn commit -m "Apache PredictionIO 0.15.0-rc1"`
 14. Set up credentials with Apache Nexus using the SBT Sonatype plugin. Put this
 in `~/.sbt/1.0/sonatype.sbt`.
 
@@ -78,26 +78,27 @@ Close the staged repository on Apache Nexus.
 16. Send out email for voting on PredictionIO dev mailing list.
 
   ```
-  Subject: [VOTE] Apache PredictionIO 0.14.0 Release (RC1)
+  Subject: [VOTE] Apache PredictionIO 0.15.0 Release (RC1)
 
-  This is the vote for 0.14.0 of Apache PredictionIO.
+  This is the vote for 0.15.0 of Apache PredictionIO.
 
   The vote will run for at least 72 hours and will close on Apr 7th, 2017.
 
-  The release candidate artifacts can be downloaded here: https://dist.apache.org/repos/dist/dev/predictionio/apache-predictionio-0.14.0-rc1/
+  The release candidate artifacts can be downloaded here: https://dist.apache.org/repos/dist/dev/predictionio/apache-predictionio-0.15.0-rc1/
 
   Test results of RC1 can be found here: https://travis-ci.org/apache/predictionio/builds/xxx
 
   Maven artifacts are built from the release candidate artifacts above, and are provided as convenience for testing with engine templates. The Maven artifacts are provided at the Maven staging repo here: https://repository.apache.org/content/repositories/orgapachepredictionio-nnnn/
 
-  All JIRAs completed for this release are tagged with 'FixVersion = 0.14.0'. You can view them here: https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12320420&version=12337844
+  All JIRAs completed for this release are tagged with 'FixVersion = 0.15.0'. You can view them here: https://issues.apache.org/jira/secure/ReleaseNote
+  .jspa?projectId=12320420&version=12337844
 
   The artifacts have been signed with Key : YOUR_KEY_ID
 
   Please vote accordingly:
 
-  [ ] +1, accept RC as the official 0.14.0 release
-  [ ] -1, do not accept RC as the official 0.14.0 release because...
+  [ ] +1, accept RC as the official 0.15.0 release
+  [ ] -1, do not accept RC as the official 0.15.0 release because...
   ```
 17. After the vote has been accepted, update `RELEASE.md`.
 18. Create a release tag
@@ -106,36 +107,36 @@ Close the staged repository on Apache Nexus.
 https://dist.apache.org/repos/dist/release/predictionio/. This is the area
 for staging actual releases.
 21. Create a subdirectory at the SVN staging area. The area should have a `KEYS` file.
-    * `mkdir 0.14.0`
+    * `mkdir 0.15.0`
     * Copy the binary distribution from the dev/ tree to the release/ tree
     * Copy the official release to the release/ tree
 22. If you have updated the `KEYS` file, also copy that to the staging area.
 23. Remove old releases from the ASF distribution mirrors.
 (https://www.apache.org/dev/mirrors.html#location)
-    * `svn delete 0.13.0`
-24. `svn commit -m "Apache PredictionIO 0.14.0"`
+    * `svn delete 0.14.0`
+24. `svn commit -m "Apache PredictionIO 0.15.0"`
 25. Document breaking changes in https://predictionio.apache.org/resources/upgrade/.
 26. Mark the version as released on JIRA.
 (https://issues.apache.org/jira/projects/PIO?selectedItem=com.atlassian.jira.jira-projects-plugin%3Arelease-page&status=no-filter)
 27. Send out an email to the following mailing lists: announce, user, dev.
 
   ```
-  Subject: [ANNOUNCE] Apache PredictionIO 0.14.0 Release
+  Subject: [ANNOUNCE] Apache PredictionIO 0.15.0 Release
 
-  The Apache PredictionIO team would like to announce the release of Apache PredictionIO 0.14.0.
+  The Apache PredictionIO team would like to announce the release of Apache PredictionIO 0.15.0.
 
   Release notes are here:
-  https://github.com/apache/predictionio/blob/release/0.14.0/RELEASE.md
+  https://github.com/apache/predictionio/blob/v0.15.0/RELEASE.md
 
   Apache PredictionIO is an open source Machine Learning Server built on top of state-of-the-art open source stack, that enables developers to manage and deploy production-ready predictive services for various kinds of machine learning tasks.
 
   More details regarding Apache PredictionIO can be found here:
   https://predictionio.apache.org/
 
   The release artifacts can be downloaded here:
-  https://www.apache.org/dyn/closer.lua/predictionio/0.14.0/apache-predictionio-0.14.0-bin.tar.gz
+  https://www.apache.org/dyn/closer.lua/predictionio/0.15.0/apache-predictionio-0.15.0-bin.tar.gz
 
-  All JIRAs completed for this release are tagged with 'FixVersion = 0.13.0'; the JIRA release notes can be found here:
+  All JIRAs completed for this release are tagged with 'FixVersion = 0.15.0'; the JIRA release notes can be found here:
   https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12320420&version=12337844
 
   Thanks!

build.sbt

@@ -16,17 +16,36 @@
  */
 import PIOBuild._
 
+lazy val scalaSparkDepsVersion = Map(
+  "2.11" -> Map(
+    "2.0" -> Map(
+      "akka" -> "2.5.16",
+      "hadoop" -> "2.7.7",
+      "json4s" -> "3.2.11"),
+    "2.1" -> Map(
+      "akka" -> "2.5.17",
+      "hadoop" -> "2.7.7",
+      "json4s" -> "3.2.11"),
+    "2.2" -> Map(
+      "akka" -> "2.5.17",
+      "hadoop" -> "2.7.7",
+      "json4s" -> "3.2.11"),
+    "2.3" -> Map(
+      "akka" -> "2.5.17",
+      "hadoop" -> "2.7.7",
+      "json4s" -> "3.2.11")))
+
 name := "apache-predictionio-parent"
 
-version in ThisBuild := "0.14.0"
+version in ThisBuild := "0.15.0-SNAPSHOT"
 
 organization in ThisBuild := "org.apache.predictionio"
 
 scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.11.12")
 
 scalaBinaryVersion in ThisBuild := binaryVersion(scalaVersion.value)
 
-crossScalaVersions in ThisBuild := Seq("2.11.12")
+crossScalaVersions in ThisBuild := Seq(scalaVersion.value)
 
 scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature")
 
@@ -45,9 +64,7 @@ hadoopVersion in ThisBuild := sys.props.getOrElse("hadoop.version", "2.7.7")
 
 akkaVersion in ThisBuild := sys.props.getOrElse("akka.version", "2.5.17")
 
-lazy val es = sys.props.getOrElse("elasticsearch.version", "5.6.9")
-
-elasticsearchVersion in ThisBuild := es
+elasticsearchVersion in ThisBuild := sys.props.getOrElse("elasticsearch.version", "6.8.1")
 
 hbaseVersion in ThisBuild := sys.props.getOrElse("hbase.version", "1.2.6")
 
@@ -72,10 +89,6 @@ val commonTestSettings = Seq(
     "org.postgresql"   % "postgresql"  % "9.4-1204-jdbc41" % "test",
     "org.scalikejdbc" %% "scalikejdbc" % "3.1.0" % "test"))
 
-val dataElasticsearch1 = (project in file("storage/elasticsearch1")).
-  settings(commonSettings: _*).
-  enablePlugins(GenJavadocPlugin)
-
 val dataElasticsearch = (project in file("storage/elasticsearch")).
   settings(commonSettings: _*)
 
@@ -145,19 +158,17 @@ val tools = (project in file("tools")).
   enablePlugins(GenJavadocPlugin).
   enablePlugins(SbtTwirl)
 
-val dataEs = if (majorVersion(es) == 1) dataElasticsearch1 else dataElasticsearch
-
-val storageSubprojects = Seq(
-    dataEs,
+val storageProjectReference = Seq(
+    dataElasticsearch,
     dataHbase,
     dataHdfs,
     dataJdbc,
     dataLocalfs,
-    dataS3)
+    dataS3) map Project.projectToRef
 
 val storage = (project in file("storage"))
   .settings(skip in publish := true)
-  .aggregate(storageSubprojects map Project.projectToRef: _*)
+  .aggregate(storageProjectReference: _*)
   .disablePlugins(sbtassembly.AssemblyPlugin)
 
 val assembly = (project in file("assembly")).
@@ -167,8 +178,8 @@ val root = (project in file(".")).
   settings(commonSettings: _*).
   enablePlugins(ScalaUnidocPlugin).
   settings(
-    unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(dataElasticsearch, dataElasticsearch1),
-    unidocProjectFilter in (JavaUnidoc, unidoc) := inAnyProject -- inProjects(dataElasticsearch, dataElasticsearch1),
+    unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(storageProjectReference: _*),
+    unidocProjectFilter in (JavaUnidoc, unidoc) := inAnyProject -- inProjects(storageProjectReference: _*),
     scalacOptions in (ScalaUnidoc, unidoc) ++= Seq(
       "-groups",
       "-skip-packages",
@@ -181,11 +192,6 @@ val root = (project in file(".")).
         "org.apache.predictionio.controller.java",
         "org.apache.predictionio.data.api",
         "org.apache.predictionio.data.storage.*",
-        "org.apache.predictionio.data.storage.hdfs",
-        "org.apache.predictionio.data.storage.jdbc",
-        "org.apache.predictionio.data.storage.localfs",
-        "org.apache.predictionio.data.storage.s3",
-        "org.apache.predictionio.data.storage.hbase",
         "org.apache.predictionio.data.view",
         "org.apache.predictionio.data.webhooks",
         "org.apache.predictionio.tools",

conf/pio-env.sh.template

@@ -24,7 +24,6 @@
 # you need to change these to fit your site.
 
 # SPARK_HOME: Apache Spark is a hard dependency and must be configured.
-# SPARK_HOME=$PIO_HOME/vendors/spark-2.0.2-bin-hadoop2.7
 SPARK_HOME=$PIO_HOME/vendors/spark-2.1.1-bin-hadoop2.6
 
 POSTGRES_JDBC_DRIVER=$PIO_HOME/lib/postgresql-42.0.0.jar
@@ -40,7 +39,7 @@ MYSQL_JDBC_DRIVER=$PIO_HOME/lib/mysql-connector-java-5.1.41.jar
 
 # HBASE_CONF_DIR: You must configure this if you intend to run PredictionIO
 #                 with HBase on a remote cluster.
-# HBASE_CONF_DIR=$PIO_HOME/vendors/hbase-1.0.0/conf
+# HBASE_CONF_DIR=$PIO_HOME/vendors/hbase-1.2.6/conf
 
 # Filesystem paths where PredictionIO uses as block storage.
 PIO_FS_BASEDIR=$HOME/.pio_store
@@ -89,24 +88,18 @@ PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
 # PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=localhost
 # PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
 # PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-5.6.9
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-6.8.1
 # Optional basic HTTP auth
 # PIO_STORAGE_SOURCES_ELASTICSEARCH_USERNAME=my-name
 # PIO_STORAGE_SOURCES_ELASTICSEARCH_PASSWORD=my-secret
-# Elasticsearch 1.x Example
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=<elasticsearch_cluster_name>
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=localhost
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-1.7.6
 
 # Local File System Example
 # PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs
 # PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/models
 
 # HBase Example
 # PIO_STORAGE_SOURCES_HBASE_TYPE=hbase
-# PIO_STORAGE_SOURCES_HBASE_HOME=$PIO_HOME/vendors/hbase-1.0.0
+# PIO_STORAGE_SOURCES_HBASE_HOME=$PIO_HOME/vendors/hbase-1.2.6
 
 # AWS S3 Example
 # PIO_STORAGE_SOURCES_S3_TYPE=s3

conf/pio-vendors.sh

@@ -32,22 +32,15 @@ if [ -z "$PIO_HADOOP_VERSION" ]; then
 fi
 
 if [ -z "$PIO_ELASTICSEARCH_VERSION" ]; then
-    PIO_ELASTICSEARCH_VERSION="5.6.9"
+    PIO_ELASTICSEARCH_VERSION="6.8.1"
 fi
 
 if [ -z "$PIO_HBASE_VERSION" ]; then
     PIO_HBASE_VERSION="1.2.6"
 fi
 
-ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'`
-
-if [ "$ES_MAJOR" = "1" ]; then
-    export ES_IMAGE="elasticsearch"
-    export ES_TAG="1"
-else
-    export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch"
-    export ES_TAG="$PIO_ELASTICSEARCH_VERSION"
-fi
+export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch"
+export ES_TAG="$PIO_ELASTICSEARCH_VERSION"
 
 HBASE_MAJOR=`echo $PIO_HBASE_VERSION | awk -F. '{print $1 "." $2}'`
 export HBASE_TAG="$HBASE_MAJOR"
@@ -60,6 +53,3 @@ SPARK_DIR=spark-${PIO_SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR}
 SPARK_ARCHIVE=${SPARK_DIR}.tgz
 SPARK_DOWNLOAD_MIRROR=https://www.apache.org/dyn/closer.lua\?action=download\&filename=spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
 SPARK_DOWNLOAD_ARCHIVE=https://archive.apache.org/dist/spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
-# ELASTICSEARCH_DOWNLOAD
-#   5.x https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
-#   1.x https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz

core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala

@@ -30,7 +30,7 @@ import org.apache.spark.SparkContext
   * @group Preparator
   */
 class IdentityPreparator[TD] extends BasePreparator[TD, TD] {
-  def prepareBase(sc: SparkContext, td: TD): TD = td
+  override def prepareBase(sc: SparkContext, td: TD): TD = td
 }
 
 /** Companion object of [[IdentityPreparator]] that conveniently returns an