ES|QL support: ESQL executor implementation, response type to accept esql option, validations to make sure both LS and ES support the ESQL execution.

Author: mashhurs

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.2.0
+  - ES|QL support [#233](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/233)
+
 ## 5.1.0
   - Add "cursor"-like index tracking [#205](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/205)
 

lib/logstash/inputs/elasticsearch.rb

@@ -74,6 +74,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   require 'logstash/inputs/elasticsearch/paginated_search'
   require 'logstash/inputs/elasticsearch/aggregation'
   require 'logstash/inputs/elasticsearch/cursor_tracker'
+  require 'logstash/inputs/elasticsearch/esql'
 
   include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
   include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
@@ -104,7 +105,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # This allows you to speccify the response type: either hits or aggregations
   # where hits: normal search request
   #       aggregations: aggregation request
-  config :response_type, :validate => ['hits', 'aggregations'], :default => 'hits'
+  config :response_type, :validate => %w[hits aggregations esql], :default => 'hits'
 
   # This allows you to set the maximum number of hits returned per scroll.
   config :size, :validate => :number, :default => 1000
@@ -302,10 +303,17 @@ def register
     fill_hosts_from_cloud_id
     setup_ssl_params!
 
-    @base_query = LogStash::Json.load(@query)
-    if @slices
-      @base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
-      @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
+    if @response_type == 'esql'
+      validate_ls_version_for_esql_support!
+      validate_esql_query!
+      inform_ineffective_esql_params
+    else
+      # for the ES|QL, plugin accepts raw string query but JSON for others
+      @base_query = LogStash::Json.load(@query)
+      if @slices
+        @base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
+        @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
+      end
     end
 
     @retries < 0 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `retries` option must be equal or greater than zero, got `#{@retries}`")
@@ -341,6 +349,9 @@ def register
 
     test_connection!
 
+    # make sure connected ES supports ES|QL (8.11+)
+    validate_es_for_esql_support! if @response_type == 'esql'
+
     setup_serverless
 
     setup_search_api
@@ -398,6 +409,12 @@ def event_from_hit(hit, root_field)
     return event_factory.new_event('event' => { 'original' => serialized_hit }, 'tags' => ['_elasticsearch_input_failure'])
   end
 
+  def decorate_and_push_to_queue(output_queue, mapped_entry)
+    event = targeted_event_factory.new_event mapped_entry
+    decorate(event)
+    output_queue << event
+  end
+
   def set_docinfo_fields(hit, event)
     # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
     docinfo_target = event.get(@docinfo_target) || {}
@@ -675,6 +692,8 @@ def setup_query_executor
                         end
                       when 'aggregations'
                         LogStash::Inputs::Elasticsearch::Aggregation.new(@client, self)
+                      when 'esql'
+                        LogStash::Inputs::Elasticsearch::Esql.new(@client, self)
                       end
   end
 
@@ -714,6 +733,31 @@ def get_transport_client_class
     ::Elastic::Transport::Transport::HTTP::Manticore
   end
 
+  def validate_ls_version_for_esql_support!
+    # LS 8.17.4+ has elasticsearch-ruby 8.17 client
+    # elasticsearch-ruby 8.11+ supports ES|QL
+    if Gem::Version.create(LOGSTASH_VERSION) < Gem::Version.create("8.17.4")
+      fail("Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least 8.17.4")
+    end
+  end
+
+  def validate_esql_query!
+    fail(LogStash::ConfigurationError, "`query` cannot be empty") if @query.strip.empty?
+    source_commands = %w[FROM ROW SHOW]
+    contains_source_command = source_commands.any? { |source_command| @query.strip.start_with?(source_command) }
+    fail(LogStash::ConfigurationError, "`query` needs to start with any of #{source_commands}") unless contains_source_command
+  end
+
+  def inform_ineffective_esql_params
+    ineffective_options = original_params.keys & %w(target size slices search_api)
+    @logger.info("Configured #{ineffective_options} params are ineffective in ES|QL mode") if ineffective_options.size > 1
+  end
+
+  def validate_es_for_esql_support!
+    es_supports_esql = Gem::Version.create(es_version) >= Gem::Version.create("8.11")
+    fail("Connected Elasticsearch #{es_version} version does not supports ES|QL. Please upgrade it.") unless es_supports_esql
+  end
+
   module URIOrEmptyValidator
     ##
     # @override to provide :uri_or_empty validator

lib/logstash/inputs/elasticsearch/esql.rb

@@ -0,0 +1,79 @@
+require 'logstash/helpers/loggable_try'
+
+module LogStash
+  module Inputs
+    class Elasticsearch
+      class Esql
+        include LogStash::Util::Loggable
+
+        ESQL_JOB = "ES|QL job"
+
+        # Initialize the ESQL query executor
+        # @param client [Elasticsearch::Client] The Elasticsearch client instance
+        # @param plugin [LogStash::Inputs::Elasticsearch] The parent plugin instance
+        def initialize(client, plugin)
+          @client = client
+          @plugin_params = plugin.params
+          @plugin = plugin
+          @retries = @plugin_params["retries"]
+          @query = @plugin_params["query"]
+        end
+
+        # Execute the ESQL query and process results
+        # @param output_queue [Queue] The queue to push processed events to
+        def do_run(output_queue)
+          logger.info("ES|QL executor starting")
+          response = retryable(ESQL_JOB) do
+            @client.esql.query({ body: { query: @query }, format: 'json' })
+          end
+          # retriable already printed error details
+          return if response == false
+
+          if response&.headers&.dig("warning")
+            logger.warn("ES|QL executor received warning", {:message => response.headers["warning"]})
+          end
+          if response['values'] && response['columns']
+            process_response(response['values'], response['columns'], output_queue)
+          end
+        end
+
+        # Execute a retryable operation with proper error handling
+        # @param job_name [String] Name of the job for logging purposes
+        # @yield The block to execute
+        # @return [Boolean] true if successful, false otherwise
+        def retryable(job_name, &block)
+          stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
+          stud_try.try((@retries + 1).times) { yield }
+        rescue => e
+          error_details = {:message => e.message, :cause => e.cause}
+          error_details[:backtrace] = e.backtrace if logger.debug?
+          logger.error("#{job_name} failed with ", error_details)
+          false
+        end
+
+        private
+
+        # Process the ESQL response and push events to the output queue
+        # @param values [Array[Array]] The ESQL query response hits
+        # @param columns [Array[Hash]] The ESQL query response columns
+        # @param output_queue [Queue] The queue to push processed events to
+        def process_response(values, columns, output_queue)
+          values.each do |value|
+            mapped_data = map_column_and_values(columns, value)
+            @plugin.decorate_and_push_to_queue(output_queue, mapped_data)
+          end
+        end
+
+        # Map column names to their corresponding values
+        # @param columns [Array] Array of column definitions
+        # @param values [Array] Array of values for the current row
+        # @return [Hash] Mapped data with column names as keys
+        def map_column_and_values(columns, values)
+          columns.each_with_index.with_object({}) do |(column, index), mapped_data|
+            mapped_data[column["name"]] = values[index]
+          end
+        end
+      end
+    end
+  end
+end

logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '5.1.0'
+  s.version         = '5.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/inputs/elasticsearch_esql_spec.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/elasticsearch"
+require "elasticsearch"
+
+describe LogStash::Inputs::Elasticsearch::Esql do
+  let(:client) { instance_double(Elasticsearch::Client) }
+  let(:esql_client) { double("esql-client") }
+  let(:plugin) { instance_double(LogStash::Inputs::Elasticsearch, params: plugin_config) }
+  let(:plugin_config) do
+    {
+      "query" => "FROM test-index | STATS count() BY field",
+      "retries" => 3
+    }
+  end
+  let(:esql_executor) { described_class.new(client, plugin) }
+
+  describe "when initializes" do
+    it "sets up the ESQL client with correct parameters" do
+      expect(esql_executor.instance_variable_get(:@query)).to eq(plugin_config["query"])
+      expect(esql_executor.instance_variable_get(:@retries)).to eq(plugin_config["retries"])
+    end
+  end
+
+  describe "when faces error while retrying" do
+    it "retries the given block the specified number of times" do
+      attempts = 0
+      result = esql_executor.retryable("Test Job") do
+        attempts += 1
+        raise StandardError if attempts < 3
+        "success"
+      end
+      expect(attempts).to eq(3)
+      expect(result).to eq("success")
+    end
+
+    it "returns false if the block fails all attempts" do
+      result = esql_executor.retryable("Test Job") do
+        raise StandardError
+      end
+      expect(result).to eq(false)
+    end
+  end
+
+  describe "when executing chain of processes" do
+    let(:output_queue) { Queue.new }
+    let(:response) { { 'values' => [%w[foo bar]], 'columns' => [{ 'name' => 'id'}, { 'name' => 'val'}] } }
+
+    before do
+      allow(esql_executor).to receive(:retryable).and_yield
+      allow(client).to receive_message_chain(:esql, :query).and_return(response)
+      allow(plugin).to receive(:decorate_and_push_to_queue)
+    end
+
+    it "executes the ESQL query and processes the results" do
+      allow(response).to receive(:headers).and_return({})
+      esql_executor.do_run(output_queue)
+      expect(plugin).to have_received(:decorate_and_push_to_queue).with(output_queue, {'id' => 'foo', 'val' => 'bar'})
+    end
+
+    it "logs a warning if the response contains a warning header" do
+      allow(response).to receive(:headers).and_return({"warning" => "some warning"})
+      expect(esql_executor.logger).to receive(:warn).with("ES|QL executor received warning", {:message => "some warning"})
+      esql_executor.do_run(output_queue)
+    end
+
+    it "does not log a warning if the response does not contain a warning header" do
+      allow(response).to receive(:headers).and_return({})
+      expect(esql_executor.logger).not_to receive(:warn)
+      esql_executor.do_run(output_queue)
+    end
+  end
+
+
+  describe "when starts processing the response" do
+    let(:output_queue) { Queue.new }
+    let(:values) { [%w[foo bar]] }
+    let(:columns) { [{'name' => 'id'}, {'name' => 'val'}] }
+
+    it "processes the ESQL response and pushes events to the output queue" do
+      allow(plugin).to receive(:decorate_and_push_to_queue)
+      esql_executor.send(:process_response, values, columns, output_queue)
+      expect(plugin).to have_received(:decorate_and_push_to_queue).with(output_queue, {'id' => 'foo', 'val' => 'bar'})
+    end
+  end
+
+  describe "when maps column and values" do
+    let(:columns) { [{'name' => 'id'}, {'name' => 'val'}] }
+    let(:values) { %w[foo bar] }
+
+    it "maps column names to their corresponding values" do
+      result = esql_executor.send(:map_column_and_values, columns, values)
+      expect(result).to eq({'id' => 'foo', 'val' => 'bar'})
+    end
+  end
+end