Jdk11 class loading (#343)

* temp commit

* Had to monkey patch tzinfo, jruby bug fix scheduled for 9.2.8.0

* update travis.yml

* try a better matrix

* maybe this matrix will do

* try docker based travis

* final adjustments, version bump and docs changes

* review changes, add comments and jruby version guard

* more review led changes

Author: Guy Boertje

.travis.yml

@@ -1,21 +1,16 @@
 ---
-sudo: false
-language: ruby
-cache: bundler
+sudo: required
+services: docker
+addons:
+  apt:
+    packages:
+      - docker-ce
 matrix:
   include:
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=master
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=7.0
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.7
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.6
-  - rvm: jruby-1.7.27
-    env: LOGSTASH_BRANCH=5.6
+    - env: ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
+    - env: ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
+    - env: ELASTIC_STACK_VERSION=7.x
+    - env: ELASTIC_STACK_VERSION=6.x
   fast_finish: true
-install: true
-script: ci/build.sh
-jdk: oraclejdk8
-before_install: gem install bundler -v '< 2'
+install: ci/unit/docker-setup.sh
+script: ci/unit/docker-run.sh

ci/build.sh

@@ -0,0 +1,11 @@
+ARG ELASTIC_STACK_VERSION
+FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION
+COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile
+COPY --chown=logstash:logstash *.gemspec /usr/share/plugins/plugin/
+RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml
+ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin"
+ENV LOGSTASH_SOURCE="1"
+RUN gem install bundler -v '< 2'
+WORKDIR /usr/share/plugins/plugin
+RUN bundle install
+COPY --chown=logstash:logstash . /usr/share/plugins/plugin

ci/setup.sh

@@ -0,0 +1,17 @@
+version: '3'
+
+# run tests:  cd ci/unit; docker-compose up --build --force-recreate
+# manual:  cd ci/unit; docker-compose run logstash bash
+services:
+
+  logstash:
+    build:
+      context: ../../
+      dockerfile: ci/unit/Dockerfile
+      args:
+        - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+    command: /usr/share/plugins/plugin/ci/unit/run.sh
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+      LOGSTASH_SOURCE: 1
+    tty: true

ci/unit/Dockerfile

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+docker-compose -f ci/unit/docker-compose.yml up --exit-code-from logstash

ci/unit/docker-compose.yml

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+ # This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh`
+# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable.
+set -e
+
+ VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json"
+
+ if [ -z "${ELASTIC_STACK_VERSION}" ]; then
+    echo "Please set the ELASTIC_STACK_VERSION environment variable"
+    echo "For example: export ELASTIC_STACK_VERSION=6.2.4"
+    exit 1
+fi
+
+ echo "Fetching versions from $VERSION_URL"
+VERSIONS=$(curl $VERSION_URL)
+
+ if [[ "$SNAPSHOT" = "true" ]]; then
+  ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"')
+  echo $ELASTIC_STACK_RETRIEVED_VERSION
+else
+  ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"')
+fi
+
+ if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then
+  # remove starting and trailing double quotes
+  ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}"
+  ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}"
+  echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}"
+  export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION
+fi
+
+ echo "Testing against version: $ELASTIC_STACK_VERSION"
+
+ if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then
+    cd /tmp
+
+    jq=".build.projects.logstash.packages.\"logstash-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url"
+    result=$(curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq)
+    echo $result
+    curl $result > logstash-docker-image.tar.gz
+    tar xfvz logstash-docker-image.tar.gz  repositories
+    echo "Loading docker image: "
+    cat repositories
+    docker load < logstash-docker-image.tar.gz
+    rm logstash-docker-image.tar.gz
+    cd -
+fi
+
+ if [ -f Gemfile.lock ]; then
+    rm Gemfile.lock
+fi
+
+ docker-compose -f ci/unit/docker-compose.yml down
+docker-compose -f ci/unit/docker-compose.yml build

ci/unit/docker-run.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+
+export USER='logstash'
+
+bundle exec rspec -fd 2>/dev/null

ci/unit/docker-setup.sh

@@ -149,6 +149,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-plugin_timezone>> |<<string,string>>, one of `["local", "utc"]`|No
 | <<plugins-{type}s-{plugin}-clean_run>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-columns_charset>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-connection_retry_attempts>> |<<number,number>>|No
@@ -247,13 +248,35 @@ JDBC connection string
   * There is no default value for this setting.
 
 Timezone conversion.
-SQL does not allow for timezone data in timestamp fields.  This plugin will automatically
-convert your SQL timestamp fields to Logstash timestamps, in relative UTC time in ISO8601 format.
+Logstash (and Elasticsearch) expects that timestamps are expressed in UTC terms.
+If your database has recorded timestamps that are relative to another timezone,
+the database timezone if you will, then set this setting to be the timezone that
+the database is using. However, as SQL does not allow for timezone data in
+timestamp fields we can't figure this out on a record by record basis.  This plugin
+will automatically convert your SQL timestamp fields to Logstash timestamps,
+in relative UTC time in ISO8601 format.
 
 Using this setting will manually assign a specified timezone offset, instead
 of using the timezone setting of the local machine.  You must use a canonical
 timezone, *America/Denver*, for example.
 
+[id="plugins-{type}s-{plugin}-plugin_timezone"]
+===== `plugin_timezone`
+
+  * Value can be any of: `utc`, `local`
+  * Default value is `"utc"`
+
+If you want this plugin to offset timestamps to a timezone other than UTC, you
+can set this setting to `local` and the plugin will use the OS timezone for offset
+adjustments.
+
+Note: when specifying `plugin_timezone` and/or `jdbc_default_timezone`, offset
+adjustments are made in two places, if `sql_last_value` is a timestamp and it
+is used as a parameter in the statement then offset adjustment is done from the
+plugin timezone into the data timezone and while records are processed, timestamps
+are offset adjusted from the database timezone to the plugin timezone. If your
+database timezone is UTC then you do not need to set either of these settings.
+
 [id="plugins-{type}s-{plugin}-jdbc_driver_class"]
 ===== `jdbc_driver_class`
 

ci/unit/run.sh

@@ -3,6 +3,8 @@
 require "logstash/namespace"
 require "logstash/plugin_mixins/jdbc/jdbc"
 
+# this require_relative returns early unless the JRuby version is between 9.2.0.0 and 9.2.8.0
+require_relative "tzinfo_jruby_patch"
 
 # This plugin was created as a way to ingest data from any database
 # with a JDBC interface into Logstash. You can periodically schedule ingestion
@@ -84,19 +86,19 @@
 # The file option only supports one SQL statement. The plugin will only accept one of the options.
 # It cannot read a statement from a file as well as from the `statement` configuration parameter.
 #
-# ==== Configuring multiple SQL statements 
+# ==== Configuring multiple SQL statements
 #
-# Configuring multiple SQL statements is useful when there is a need to query and ingest data 
-# from different database tables or views. It is possible to define separate Logstash 
-# configuration files for each statement or to define multiple statements in a single configuration 
-# file. When using multiple statements in a single Logstash configuration file, each statement 
-# has to be defined as a separate jdbc input (including jdbc driver, connection string and other 
-# required parameters). 
+# Configuring multiple SQL statements is useful when there is a need to query and ingest data
+# from different database tables or views. It is possible to define separate Logstash
+# configuration files for each statement or to define multiple statements in a single configuration
+# file. When using multiple statements in a single Logstash configuration file, each statement
+# has to be defined as a separate jdbc input (including jdbc driver, connection string and other
+# required parameters).
 #
-# Please note that if any of the statements use the `sql_last_value` parameter (e.g. for 
-# ingesting only data changed since last run), each input should define its own 
+# Please note that if any of the statements use the `sql_last_value` parameter (e.g. for
+# ingesting only data changed since last run), each input should define its own
 # `last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as
-# all inputs will store their state to the same (default) metadata file, effectively 
+# all inputs will store their state to the same (default) metadata file, effectively
 # overwriting each other's `sql_last_value`.
 #
 # ==== Predefined Parameters
@@ -178,11 +180,11 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   # Whether to force the lowercasing of identifier fields
   config :lowercase_column_names, :validate => :boolean, :default => true
 
-  # The character encoding of all columns, leave empty if the columns are already properly UTF-8 
+  # The character encoding of all columns, leave empty if the columns are already properly UTF-8
   # encoded. Specific columns charsets using :columns_charset can override this setting.
   config :charset, :validate => :string
 
-  # The character encoding for specific columns. This option will override the `:charset` option 
+  # The character encoding for specific columns. This option will override the `:charset` option
   # for the specified columns.
   #
   # Example:
@@ -199,6 +201,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   # this will only convert column0 that has ISO-8859-1 as an original encoding.
   config :columns_charset, :validate => :hash, :default => {}
 
+  attr_reader :database # for test mocking/stubbing
+
   public
 
   def register

docs/index.asciidoc

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+# There is a bug in JRuby versions between 9.2.0.0 and 9.2.8.0
+# the TZinfo::Timestamp `new_datetime` can build Rational numbers having
+# numerators and denominators that are too large for Java longs.
+#
+# This patch reopens the TZinfo::Timestamp class and redefines
+# the `new_datetime method.
+# It scales down the numerator and denominator if they are larger than
+# Java Long. There is no appreciable precision loss at the microsecond level
+
+tzinfo_jruby_bugfixed_version = "9.2.8.0"
+tzinfo_jruby_bugadded_version = "9.2.0.0"
+
+current_jruby_version = Gem::Version.new(JRUBY_VERSION)
+broken_jruby_version = Gem::Version.new(tzinfo_jruby_bugadded_version)
+patched_jruby_version = Gem::Version.new(tzinfo_jruby_bugfixed_version)
+
+return unless current_jruby_version >= broken_jruby_version && current_jruby_version < patched_jruby_version
+
+require 'tzinfo'
+
+if defined?(TZInfo::VERSION) && TZInfo::VERSION > '2'
+  module TZInfo
+    # A time represented as an `Integer` number of seconds since 1970-01-01
+    # 00:00:00 UTC (ignoring leap seconds), the fraction through the second
+    # (sub_second as a `Rational`) and an optional UTC offset. Like Ruby's `Time`
+    # class, {Timestamp} can distinguish between a local time with a zero offset
+    # and a time specified explicitly as UTC.
+    class Timestamp
+
+      protected
+
+      def new_datetime(klass = DateTime)
+        val = JD_EPOCH + ((@value.to_r + @sub_second) / 86400)
+        datetime = klass.jd(jruby_scale_down_rational(val))
+        @utc_offset && @utc_offset != 0 ? datetime.new_offset(Rational(@utc_offset, 86400)) : datetime
+      end
+
+      private
+
+      # while this JRuby bug exists in 9.2.X.X https://github.com/jruby/jruby/issues/5791
+      # we must scale down the numerator and denominator to fit Java Long values.
+
+      def jruby_scale_down_rational(rat)
+        return rat if rat.numerator <= java.lang.Long::MAX_VALUE
+        [10, 100, 1000].each do |scale_by|
+          new_numerator = rat.numerator / scale_by
+          if new_numerator  <= java.lang.Long::MAX_VALUE
+            return Rational(new_numerator, rat.denominator / scale_by)
+          end
+        end
+      end
+    end
+  end
+end