diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryTestBase.java b/api/src/testFixtures/java/org/opensearch/sql/api/UnifiedQueryTestBase.java similarity index 100% rename from api/src/test/java/org/opensearch/sql/api/UnifiedQueryTestBase.java rename to api/src/testFixtures/java/org/opensearch/sql/api/UnifiedQueryTestBase.java diff --git a/benchmarks/README.md b/benchmarks/README.md index d90eb850db0..6a720c7201c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -10,7 +10,18 @@ The microbenchmark suite is also handy for ad-hoc microbenchmarks but please rem ## Getting Started -Just run `./gradlew :benchmarks:jmh` from the project root directory or run specific benchmark via your IDE. It will build all microbenchmarks, execute them and print the result. +Run all benchmarks from the project root directory: + +```bash +./gradlew :benchmarks:jmh +``` + +Run specific benchmarks using the `-Pjmh.includes` parameter: + +```bash +./gradlew :benchmarks:jmh -Pjmh.includes='UnifiedQueryBenchmark' +./gradlew :benchmarks:jmh -Pjmh.includes='UnifiedQueryBenchmark.plan.*' +``` ## Adding Microbenchmarks diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 3b59b92f943..5f4f7baaba2 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -5,6 +5,7 @@ plugins { id 'java-library' + id "io.freefair.lombok" id "me.champeau.jmh" version "0.7.3" } @@ -15,6 +16,8 @@ repositories { dependencies { implementation project(':core') implementation project(':opensearch') + implementation project(':api') + jmhImplementation testFixtures(project(':api')) // Dependencies required by JMH micro benchmark api group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.36' @@ -30,4 +33,9 @@ spotless { } } +// JMH configuration passed via command line +jmh { + includes = project.hasProperty('jmh.includes') ? [project.property('jmh.includes')] : [] +} + compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"]) \ No newline at end of file diff --git a/benchmarks/src/jmh/java/org/opensearch/sql/api/UnifiedFunctionBenchmark.java b/benchmarks/src/jmh/java/org/opensearch/sql/api/UnifiedFunctionBenchmark.java new file mode 100644 index 00000000000..b333c8c7e1c --- /dev/null +++ b/benchmarks/src/jmh/java/org/opensearch/sql/api/UnifiedFunctionBenchmark.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.api; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.opensearch.sql.api.function.UnifiedFunction; +import org.opensearch.sql.api.function.UnifiedFunctionRepository; + +/** + * JMH benchmark for measuring {@link UnifiedFunction} performance. Tests one representative + * function per PPL category (json, math, conditional, collection, string). + * + *

Benchmarks: + * + *

+ */ +@Warmup(iterations = 2, time = 1) +@Measurement(iterations = 5, time = 1) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(value = 1) +public class UnifiedFunctionBenchmark extends UnifiedQueryTestBase { + + /** Number of function evaluations per benchmark invocation. */ + private static final int OPS = 1000; + + /** Benchmark specification selecting which function to test. */ + @Param public BenchmarkSpec benchmarkSpec; + + /** Repository for loading unified functions. */ + private UnifiedFunctionRepository repository; + + /** Pre-loaded function instance for evaluation benchmarks. */ + private UnifiedFunction function; + + /** Input arguments for function evaluation. */ + private List inputs; + + @Setup(Level.Trial) + public void setUpBenchmark() { + super.setUp(); + + repository = new UnifiedFunctionRepository(context); + function = benchmarkSpec.loadFunction(repository); + inputs = benchmarkSpec.getInputs(); + } + + @TearDown(Level.Trial) + public void tearDownBenchmark() throws Exception { + super.tearDown(); + } + + /** Benchmarks function loading from repository. */ + @Benchmark + public UnifiedFunction loadFunction() { + return benchmarkSpec.loadFunction(repository); + } + + /** Benchmarks function evaluation with pre-loaded function and inputs. */ + @Benchmark + @OperationsPerInvocation(OPS) + public void evalFunction(Blackhole bh) { + for (int i = 0; i < OPS; i++) { + bh.consume(function.eval(inputs)); + } + } + + /** Enum defining benchmark test cases - one representative function per PPL category. */ + @RequiredArgsConstructor + public enum BenchmarkSpec { + JSON_EXTRACT( + inputTypes("VARCHAR", "VARCHAR"), + sampleInputs("{\"name\":\"test\",\"value\":42}", "$.name")), + COALESCE( + inputTypes("VARCHAR", "VARCHAR", "VARCHAR"), + sampleInputs(null, "first_value", "default_value")), + MVFIND( + inputTypes("ARRAY", "VARCHAR"), + sampleInputs(List.of("debug", "error", "warn", "info"), "err.*")), + REX_EXTRACT( + inputTypes("VARCHAR", "VARCHAR", "INTEGER"), + sampleInputs("192.168.1.1 - GET /api", "(\\d+)", 1)); + + private final List inputTypes; + @Getter private final List inputs; + + UnifiedFunction loadFunction(UnifiedFunctionRepository repository) { + return repository + .loadFunction(name()) + .map(desc -> desc.getBuilder().build(inputTypes)) + .orElseThrow(); + } + + private static List inputTypes(String... types) { + return List.of(types); + } + + private static List sampleInputs(Object... args) { + return Arrays.asList(args); + } + } +} diff --git a/benchmarks/src/jmh/java/org/opensearch/sql/api/UnifiedQueryBenchmark.java b/benchmarks/src/jmh/java/org/opensearch/sql/api/UnifiedQueryBenchmark.java new file mode 100644 index 00000000000..d75a87ea8c3 --- /dev/null +++ b/benchmarks/src/jmh/java/org/opensearch/sql/api/UnifiedQueryBenchmark.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.api; + +import java.sql.PreparedStatement; +import java.util.concurrent.TimeUnit; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.sql.dialect.SparkSqlDialect; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.opensearch.sql.api.compiler.UnifiedQueryCompiler; +import org.opensearch.sql.api.transpiler.UnifiedQueryTranspiler; + +/** + * JMH benchmark for measuring the overhead of unified query API components when processing queries. + * This provides baseline metrics and guidance for API consumers during integration. + */ +@Warmup(iterations = 2, time = 1) +@Measurement(iterations = 5, time = 1) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(value = 1) +public class UnifiedQueryBenchmark extends UnifiedQueryTestBase { + + /** Common query patterns for benchmarking. */ + @Param({ + "source = catalog.employees", + "source = catalog.employees | where age > 30", + "source = catalog.employees | stats count() by department", + "source = catalog.employees | sort - age", + "source = catalog.employees | where age > 25 | stats avg(age) by department | sort - department" + }) + private String query; + + /** Transpiler for converting logical plans to SQL strings. */ + private UnifiedQueryTranspiler transpiler; + + /** Compiler for converting logical plans to executable statements. */ + private UnifiedQueryCompiler compiler; + + @Setup(Level.Trial) + public void setUpBenchmark() { + super.setUp(); + transpiler = UnifiedQueryTranspiler.builder().dialect(SparkSqlDialect.DEFAULT).build(); + compiler = new UnifiedQueryCompiler(context); + } + + @TearDown(Level.Trial) + public void tearDownBenchmark() throws Exception { + super.tearDown(); + } + + /** Benchmarks query parsing and Calcite logical plan generation. */ + @Benchmark + public RelNode planQuery() { + return planner.plan(query); + } + + /** Benchmarks the full transpilation pipeline: Query → logical plan → SQL string. */ + @Benchmark + public String transpileToSql() { + RelNode plan = planner.plan(query); + return transpiler.toSql(plan); + } + + /** + * Benchmarks the compilation pipeline: Query → logical plan → executable statement. The result + * includes both compile and close time; close overhead is negligible and avoids resource leaking + * during benchmark runs. + */ + @Benchmark + public void compileQuery() throws Exception { + RelNode plan = planner.plan(query); + try (PreparedStatement stmt = compiler.compile(plan)) { + // Statement is auto-closed after benchmark iteration + } + } +}