Skip to content

Commit 5ac0bb3

Browse files
estolfoestolfo
authored
Benchmarking (#565)
* [COMMON] Add Simple.ping and Simple.create_index benchmark tests * [COMMON] Add benchmarking tests for making simple requests * [COMMON] Add benchmarking tests for making complex requests * [COMMON] Define method that sets up and cleans up before/after a benchmarking test * [COMMON] Define dataset file location * [COMMON] Add benchmarking test using noop plugin * [COMMON] First step to using matrix for Benchmark run config * [COMMON] Index benchmarking results into Elasticsearch * [COMMON] Index slices of documents * [COMMON] Build results for Simple:Ping * [COMMON] Set up results for small document, with dataset info * [COMMON] Set up results for large document, with dataset info * [COMMON] Finish up simple benchmarking tests * [COMMON] Add license to file headers in Benchmarking * [COMMON] Don't include dataset details unless dataset is used * [COMMON] Move benchmarking tasks into their own rake file * [COMMON] Fill out search complex benchmarking test * [COMMON] Fix ordering of database cleanup in benchmarks * [COMMON] All benchmarking indices have \'benchmarking\' prefix * [COMMON] Ensure time is saved as iso8601" * [COMMON] gitignore json data files * Temporarily disable indexing ping benchmarking results * Updates after teammate review of results schema and test definitions * Fix documentation in Measurable module
1 parent 49868d0 commit 5ac0bb3

11 files changed

+1305
-0
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ Gemfile.lock
1111
.DS_Store
1212
*.log
1313
.idea/*
14+
profile/**/data/*.json

‎Rakefile

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import 'rake_tasks/elasticsearch_tasks.rake'
1919
import 'rake_tasks/test_tasks.rake'
20+
import 'profile/benchmarking/benchmarking_tasks.rake'
2021

2122
require 'pathname'
2223

‎profile/benchmarking.rb

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
require 'benchmark'
19+
require 'yaml'
20+
require 'erb'
21+
require 'elasticsearch'
22+
require 'elasticsearch-api'
23+
require 'elasticsearch-transport'
24+
require 'json'
25+
require_relative 'benchmarking/measurable'
26+
require_relative 'benchmarking/simple'
27+
require_relative 'benchmarking/complex'
28+
require_relative 'benchmarking/results'
29+
30+
module Elasticsearch
31+
32+
# Module with all functionality for running client benchmark tests.
33+
#
34+
# @since 7.0.0
35+
module Benchmarking
36+
37+
extend self
38+
39+
# The default number of test repetitions.
40+
#
41+
# @return [ Integer ] The number of test repetitions.
42+
#
43+
# @since 7.0.0
44+
DEFAULT_TEST_REPETITIONS = 10.freeze
45+
46+
# The number of default warmup repetitions of the test to do before
47+
# recording times.
48+
#
49+
# @return [ Integer ] The default number of warmup repetitions.
50+
#
51+
# @since 7.0.0
52+
DEFAULT_WARMUP_REPETITIONS = 1.freeze
53+
54+
# The default definition of a test run.
55+
#
56+
# @return [ Hash ] The default test run definition.
57+
#
58+
# @since 7.0.0
59+
DEFAULT_RUN = { 'description' => 'Default run',
60+
'repetitions' => {
61+
'measured' => DEFAULT_TEST_REPETITIONS,
62+
'warmup' => DEFAULT_WARMUP_REPETITIONS },
63+
'name' => 'default',
64+
'metrics' => ['mean'] }.freeze
65+
66+
# Parse a file of run definitions and yield each run.
67+
#
68+
# @params [ String ] file The YAML file containing the matrix of test run definitions.
69+
#
70+
# @yieldparam [ Hash ] A test run definition.
71+
#
72+
# @since 7.0.0
73+
def each_run(file)
74+
if file
75+
file = File.new(file)
76+
matrix = YAML.load(ERB.new(file.read).result)
77+
file.close
78+
79+
matrix.each_with_index do |run, i|
80+
DEFAULT_RUN.merge(run)
81+
yield(run, i)
82+
end
83+
else
84+
yield(DEFAULT_RUN)
85+
end
86+
end
87+
end
88+
end

‎profile/benchmarking/benchmarking_tasks.rake

+169
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
require_relative "../benchmarking"
19+
20+
namespace :benchmark do
21+
22+
namespace :simple do
23+
24+
desc "Run the \'ping\' benchmark test"
25+
task :ping do
26+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
27+
task = Elasticsearch::Benchmarking::Simple.new(run)
28+
puts "PING: "
29+
puts "#{task.run(:ping)}"
30+
end
31+
end
32+
33+
desc "Run the \'create index\' benchmark test"
34+
task :create_index do
35+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
36+
task = Elasticsearch::Benchmarking::Simple.new(run)
37+
puts "CREATE INDEX: #{task.run(:create_index)}"
38+
end
39+
end
40+
41+
desc "Run the \'index smal document\' benchmark test with patron adapter"
42+
task :index_document_small_patron do
43+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
44+
begin
45+
require 'patron'
46+
rescue LoadError
47+
puts "Patron not loaded, skipping test"
48+
else
49+
task = Elasticsearch::Benchmarking::Simple.new(run, :patron)
50+
puts "INDEX SMALL DOCUMENT, PATRON: #{task.run(:index_document_small)}"
51+
end
52+
end
53+
end
54+
55+
desc "Run the \'index small document\' benchmark test"
56+
task :index_document_small do
57+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
58+
task = Elasticsearch::Benchmarking::Simple.new(run)
59+
puts "INDEX SMALL DOCUMENT: #{task.run(:index_document_small)}"
60+
end
61+
end
62+
63+
desc "Run the \'index large document\' benchmark test"
64+
task :index_document_large do
65+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
66+
task = Elasticsearch::Benchmarking::Simple.new(run)
67+
puts "INDEX LARGE DOCUMENT: #{task.run(:index_document_large)}"
68+
end
69+
end
70+
71+
desc "Run the \'get small document\' benchmark test"
72+
task :get_document_small do
73+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
74+
task = Elasticsearch::Benchmarking::Simple.new(run)
75+
puts "GET SMALL DOCUMENT: #{task.run(:get_document_small)}"
76+
end
77+
end
78+
79+
desc "Run the \'get large document\' benchmark test"
80+
task :get_document_large do
81+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
82+
task = Elasticsearch::Benchmarking::Simple.new(run)
83+
puts "GET LARGE DOCUMENT: #{task.run(:get_document_large)}"
84+
end
85+
end
86+
87+
desc "Run the \'search small document\' benchmark test"
88+
task :search_document_small do
89+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
90+
task = Elasticsearch::Benchmarking::Simple.new(run)
91+
puts "SEARCH SMALL DOCUMENT: #{task.run(:search_document_small)}"
92+
end
93+
end
94+
95+
desc "Run the \'search small document\' benchmark test"
96+
task :search_document_large do
97+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
98+
task = Elasticsearch::Benchmarking::Simple.new(run)
99+
puts "SEARCH LARGE DOCUMENT: #{task.run(:search_document_large)}"
100+
end
101+
end
102+
103+
desc "Run the \'update document\' benchmark test"
104+
task :update_document do
105+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
106+
task = Elasticsearch::Benchmarking::Simple.new(run)
107+
puts "UPDATE DOCUMENT: #{task.run(:update_document)}"
108+
end
109+
end
110+
111+
desc "Run all simple benchmark tests"
112+
task :all, [:matrix] do |t, args|
113+
%w[ benchmark:simple:ping
114+
benchmark:simple:create_index
115+
benchmark:simple:index_document_small
116+
benchmark:simple:index_document_small_patron
117+
benchmark:simple:index_document_large
118+
benchmark:simple:get_document_small
119+
benchmark:simple:get_document_large
120+
benchmark:simple:search_document_small
121+
benchmark:simple:search_document_large
122+
benchmark:simple:update_document
123+
].each do |task_name|
124+
begin
125+
Rake::Task[task_name].invoke(*args)
126+
rescue => ex
127+
puts "Error in task [#{task_name}], #{ex.inspect}"
128+
next
129+
end
130+
end
131+
end
132+
133+
# namespace :noop do
134+
#
135+
# desc "Run the \'search small document\' benchmark test with the noop plugin"
136+
# task :search_document_small do
137+
# puts "SIMPLE REQUEST BENCHMARK:: SEARCH SMALL DOCUMENT WITH NOOP PLUGIN"
138+
# Elasticsearch::Benchmarking::Simple.new.run(:search_document_small, noop: true)
139+
# end
140+
# end
141+
end
142+
143+
namespace :complex do
144+
145+
desc "Run the \'index documents\' benchmark test"
146+
task :index_documents do
147+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
148+
task = Elasticsearch::Benchmarking::Complex.new(run)
149+
puts "INDEX DOCUMENTS: #{task.run(:index_documents)}"
150+
end
151+
end
152+
153+
desc "Run the \'search documents\' benchmark test"
154+
task :search_documents do
155+
Elasticsearch::Benchmarking.each_run(ENV['matrix']) do |run|
156+
task = Elasticsearch::Benchmarking::Complex.new(run)
157+
puts "SEARCH DOCUMENTS: #{task.run(:search_documents)}"
158+
end
159+
end
160+
161+
desc "Run all complex benchmark test"
162+
task :all do
163+
%w[ benchmark:complex:index_documents
164+
].each do |task_name|
165+
Rake::Task[task_name].invoke
166+
end
167+
end
168+
end
169+
end

‎profile/benchmarking/complex.rb

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# Licensed to Elasticsearch B.V. under one or more contributor
2+
# license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
module Elasticsearch
19+
module Benchmarking
20+
21+
# Class encapsulating all settings and functionality for running benchmarking
22+
# tests making complex requests.
23+
#
24+
# @since 7.0.0
25+
class Complex
26+
include Measurable
27+
28+
# Test sending a bulk request to index a large dataset.
29+
#
30+
# @example Test sending a bulk index request.
31+
# task.create_documents(opts)
32+
#
33+
# @param [ Hash ] opts The test run options.
34+
#
35+
# @results [ Hash ] The results documents.
36+
#
37+
# @since 7.0.0
38+
def index_documents(opts = {})
39+
start = 0
40+
end_time = 0
41+
results = []
42+
slices = dataset_slices
43+
44+
warmup_repetitions.times do
45+
slices.each do |slice|
46+
client.bulk(body: slice)
47+
end
48+
end
49+
50+
with_cleanup do
51+
start = Time.now
52+
results = measured_repetitions.times.collect do
53+
Benchmark.realtime do
54+
slices.each do |slice|
55+
client.bulk(body: slice)
56+
end
57+
end
58+
end
59+
end_time = Time.now
60+
results
61+
end
62+
63+
options = { duration: end_time - start,
64+
operation: __method__,
65+
dataset: File.basename(DATASET_FILE),
66+
dataset_size: ObjectSpace.memsize_of(dataset),
67+
dataset_n_documents: dataset.length }
68+
index_results!(results, options)
69+
end
70+
71+
# Test sending a request a search request.
72+
#
73+
# @example Test sending a search request.
74+
# Benchmarking::Complex.search_documents(10)
75+
#
76+
# @param [ Integer ] repetitions The number of test repetitions.
77+
#
78+
# @since 7.0.0
79+
def search_documents(opts = {})
80+
start = 0
81+
end_time = 0
82+
results = []
83+
84+
with_cleanup do
85+
slices = dataset_slices
86+
sample_slice = slices.collect do |slice|
87+
client.bulk(body: slice)
88+
slice
89+
end[rand(slices.size)-1]
90+
91+
sample_documment = sample_slice[rand(sample_slice.size)-1][:index][:data]
92+
search_criteria = sample_documment.find { |k,v| v.is_a?(String) }
93+
request = { body: { query: { match: { search_criteria[0] => search_criteria[1] } } } }
94+
95+
warmup_repetitions.times do
96+
client.search(request)
97+
end
98+
99+
start = Time.now
100+
results = measured_repetitions.times.collect do
101+
Benchmark.realtime do
102+
client.search(request)
103+
end
104+
end
105+
end_time = Time.now
106+
results
107+
end
108+
109+
options = { duration: end_time - start,
110+
operation: __method__,
111+
dataset: File.basename(DATASET_FILE),
112+
dataset_size: ObjectSpace.memsize_of(dataset),
113+
dataset_n_documents: dataset.length }
114+
index_results!(results, options)
115+
end
116+
117+
# def mixed_bulk_small(repetitions)
118+
#
119+
# end
120+
#
121+
# def mixed_bulk_large(repetitions)
122+
#
123+
# end
124+
end
125+
end
126+
end

0 commit comments

Comments
 (0)