Skip to content

Commit b124ecb

Browse files
authored
Adaptive Load exponential search step controller (#460)
A StepController plugin that searches exponentially for a bad input that causes metrics to go outside thresholds, then performs a binary search between the last known good input and the bad input. It should not exceed 2x the healthy capacity of the system under test. The exponent of the exponential search can be adjusted from 2. Note that this algorithm does not take into account the magnitude of the metric score, only the sign. This PR also adds FakeInputVariableSetter in order to use it in the unit tests for the StepController, and at the same time removes multiple implementation inheritance for all InputVariableSetters.
1 parent 65cca8b commit b124ecb

15 files changed

+1014
-11
lines changed

api/adaptive_load/step_controller_impl.proto

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ package nighthawk.adaptive_load;
77
import "envoy/config/core/v3/extension.proto";
88

99
// Configuration for ExponentialSearchStepController (plugin name:
10-
// "nighthawk.exponential-search") that performs an exponential search for the optimal
10+
// "nighthawk.exponential_search") that performs an exponential search for the optimal
1111
// value of a single Nighthawk input variable (e.g. RPS). Exponential search
1212
// starts with the input set to |initial_value| and increases the input by
1313
// |exponential_factor| until the metric goes outside thresholds at some input

include/nighthawk/adaptive_load/config_validator.h

+13-2
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,24 @@ class ConfigValidator {
2222
* plugin config factories follow this convention, the entire adaptive load session spec will be
2323
* recursively validated at load time.
2424
*
25-
* This method should not throw exceptions. Any error conditions should be encoded in the
26-
* absl::Status return object.
25+
* Any validation errors should be encoded in the absl::Status return object; do not throw an
26+
* exception.
27+
*
28+
* In the absence of fields to check, just return absl::OkStatus() immediately.
29+
*
30+
* This method is not responsible for checking the type of |message|. If |message| is the wrong
31+
* type, this will be detected elsewhere during plugin creation and handled cleanly.
32+
*
33+
* To inspect the content of |message|, directly attempt to unpack it to the plugin-specific proto
34+
* type, without specially checking for errors. If it is the wrong type, the unpacking will throw
35+
* EnvoyException, and the caller will handle it.
2736
*
2837
* @param message The Any config proto taken from the TypedExtensionConfig that activated this
2938
* plugin, to be checked for validity in plugin-specific ways.
3039
*
3140
* @return Status OK for valid config, InvalidArgument with detailed error message otherwise.
41+
*
42+
* @throw EnvoyException Only if unpacking |message| fails; otherwise return absl::Status.
3243
*/
3344
virtual absl::Status ValidateConfig(const Envoy::Protobuf::Message& message) const PURE;
3445
};

include/nighthawk/adaptive_load/input_variable_setter.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ using InputVariableSetterPtr = std::unique_ptr<InputVariableSetter>;
4545
* A factory that must be implemented for each InputVariableSetter plugin. It instantiates the
4646
* specific InputVariableSetter class after unpacking the plugin-specific config proto.
4747
*/
48-
class InputVariableSetterConfigFactory : public virtual Envoy::Config::TypedFactory,
49-
public virtual ConfigValidator {
48+
class InputVariableSetterConfigFactory : public Envoy::Config::TypedFactory,
49+
public ConfigValidator {
5050
public:
5151
std::string category() const override { return "nighthawk.input_variable_setter"; }
5252
/**

source/adaptive_load/BUILD

+21-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ envoy_cc_library(
3636
repository = "@envoy",
3737
visibility = ["//visibility:public"],
3838
deps = [
39-
":config_validator_impl",
4039
"//include/nighthawk/adaptive_load:input_variable_setter",
4140
"@envoy//source/common/config:utility_lib_with_external_headers",
4241
"@envoy//source/common/protobuf:protobuf_with_external_headers",
@@ -98,3 +97,24 @@ envoy_cc_library(
9897
"@envoy//source/common/protobuf:protobuf_with_external_headers",
9998
],
10099
)
100+
101+
envoy_cc_library(
102+
name = "step_controller_impl",
103+
srcs = [
104+
"step_controller_impl.cc",
105+
],
106+
hdrs = [
107+
"step_controller_impl.h",
108+
],
109+
repository = "@envoy",
110+
visibility = ["//visibility:public"],
111+
deps = [
112+
":input_variable_setter_impl",
113+
":plugin_loader",
114+
"//include/nighthawk/adaptive_load:input_variable_setter",
115+
"//include/nighthawk/adaptive_load:step_controller",
116+
"@envoy//source/common/common:assert_lib_with_external_headers",
117+
"@envoy//source/common/config:utility_lib_with_external_headers",
118+
"@envoy//source/common/protobuf:protobuf_with_external_headers",
119+
],
120+
)

source/adaptive_load/input_variable_setter_impl.cc

+5
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ InputVariableSetterPtr RequestsPerSecondInputVariableSetterConfigFactory::create
3737
return std::make_unique<RequestsPerSecondInputVariableSetter>(config);
3838
}
3939

40+
absl::Status RequestsPerSecondInputVariableSetterConfigFactory::ValidateConfig(
41+
const Envoy::Protobuf::Message&) const {
42+
return absl::OkStatus();
43+
}
44+
4045
REGISTER_FACTORY(RequestsPerSecondInputVariableSetterConfigFactory,
4146
InputVariableSetterConfigFactory);
4247

source/adaptive_load/input_variable_setter_impl.h

+2-5
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66

77
#include "api/adaptive_load/input_variable_setter_impl.pb.h"
88

9-
#include "adaptive_load/config_validator_impl.h"
10-
119
namespace Nighthawk {
1210

1311
/**
@@ -30,14 +28,13 @@ class RequestsPerSecondInputVariableSetter : public InputVariableSetter {
3028
* A factory that creates an RequestsPerSecondInputVariableSetter from a
3129
* RequestsPerSecondInputVariableSetterConfig proto.
3230
*/
33-
class RequestsPerSecondInputVariableSetterConfigFactory
34-
: public virtual InputVariableSetterConfigFactory,
35-
public virtual NullConfigValidator {
31+
class RequestsPerSecondInputVariableSetterConfigFactory : public InputVariableSetterConfigFactory {
3632
public:
3733
std::string name() const override;
3834
Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override;
3935
InputVariableSetterPtr
4036
createInputVariableSetter(const Envoy::Protobuf::Message& message) override;
37+
absl::Status ValidateConfig(const Envoy::Protobuf::Message& message) const override;
4138
};
4239

4340
// This factory is activated through LoadInputVariableSetterPlugin in plugin_util.h.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
#include "adaptive_load/step_controller_impl.h"
2+
3+
#include <memory>
4+
5+
#include "external/envoy/source/common/protobuf/protobuf.h"
6+
7+
#include "api/adaptive_load/adaptive_load.pb.h"
8+
#include "api/adaptive_load/benchmark_result.pb.h"
9+
#include "api/adaptive_load/metric_spec.pb.h"
10+
#include "api/adaptive_load/step_controller_impl.pb.h"
11+
12+
#include "adaptive_load/input_variable_setter_impl.h"
13+
#include "adaptive_load/plugin_loader.h"
14+
15+
namespace Nighthawk {
16+
17+
namespace {
18+
19+
using ::nighthawk::adaptive_load::BenchmarkResult;
20+
using ::nighthawk::adaptive_load::ExponentialSearchStepControllerConfig;
21+
using ::nighthawk::adaptive_load::MetricEvaluation;
22+
23+
/**
24+
* Checks if any non-informational metrics (weight > 0) were outside thresholds (score < 0).
25+
*
26+
* @param benchmark_result Metrics from the latest Nighthawk benchmark session.
27+
*
28+
* @return double -1.0 if any metric was outside its threshold or 1.0 if all metrics were within
29+
* thresholds.
30+
*/
31+
double TotalScore(const BenchmarkResult& benchmark_result) {
32+
for (const MetricEvaluation& evaluation : benchmark_result.metric_evaluations()) {
33+
if (evaluation.weight() > 0.0 && evaluation.threshold_score() < 0.0) {
34+
return -1.0;
35+
}
36+
}
37+
return 1.0;
38+
}
39+
40+
} // namespace
41+
42+
Envoy::ProtobufTypes::MessagePtr
43+
ExponentialSearchStepControllerConfigFactory::createEmptyConfigProto() {
44+
return std::make_unique<ExponentialSearchStepControllerConfig>();
45+
}
46+
47+
std::string ExponentialSearchStepControllerConfigFactory::name() const {
48+
return "nighthawk.exponential_search";
49+
}
50+
51+
absl::Status ExponentialSearchStepControllerConfigFactory::ValidateConfig(
52+
const Envoy::Protobuf::Message& message) const {
53+
const auto& any = dynamic_cast<const Envoy::ProtobufWkt::Any&>(message);
54+
ExponentialSearchStepControllerConfig config;
55+
Envoy::MessageUtil::unpackTo(any, config);
56+
if (config.has_input_variable_setter()) {
57+
return LoadInputVariableSetterPlugin(config.input_variable_setter()).status();
58+
}
59+
return absl::OkStatus();
60+
}
61+
62+
StepControllerPtr ExponentialSearchStepControllerConfigFactory::createStepController(
63+
const Envoy::Protobuf::Message& message,
64+
const nighthawk::client::CommandLineOptions& command_line_options_template) {
65+
const auto& any = dynamic_cast<const Envoy::ProtobufWkt::Any&>(message);
66+
ExponentialSearchStepControllerConfig config;
67+
Envoy::MessageUtil::unpackTo(any, config);
68+
return std::make_unique<ExponentialSearchStepController>(config, command_line_options_template);
69+
}
70+
71+
REGISTER_FACTORY(ExponentialSearchStepControllerConfigFactory, StepControllerConfigFactory);
72+
73+
ExponentialSearchStepController::ExponentialSearchStepController(
74+
const ExponentialSearchStepControllerConfig& config,
75+
nighthawk::client::CommandLineOptions command_line_options_template)
76+
: command_line_options_template_{std::move(command_line_options_template)},
77+
exponential_factor_{config.exponential_factor() > 0.0 ? config.exponential_factor() : 2.0},
78+
current_load_value_{config.initial_value()} {
79+
doom_reason_ = "";
80+
if (config.has_input_variable_setter()) {
81+
absl::StatusOr<InputVariableSetterPtr> input_variable_setter_or =
82+
LoadInputVariableSetterPlugin(config.input_variable_setter());
83+
RELEASE_ASSERT(input_variable_setter_or.ok(),
84+
absl::StrCat("InputVariableSetter plugin loading error should have been caught "
85+
"during input validation: ",
86+
input_variable_setter_or.status().message()));
87+
input_variable_setter_ = std::move(input_variable_setter_or.value());
88+
} else {
89+
input_variable_setter_ = std::make_unique<RequestsPerSecondInputVariableSetter>(
90+
nighthawk::adaptive_load::RequestsPerSecondInputVariableSetterConfig());
91+
}
92+
}
93+
94+
absl::StatusOr<nighthawk::client::CommandLineOptions>
95+
ExponentialSearchStepController::GetCurrentCommandLineOptions() const {
96+
nighthawk::client::CommandLineOptions options = command_line_options_template_;
97+
absl::Status status = input_variable_setter_->SetInputVariable(options, current_load_value_);
98+
if (!status.ok()) {
99+
return status;
100+
}
101+
return options;
102+
}
103+
104+
bool ExponentialSearchStepController::IsConverged() const {
105+
// Binary search has brought successive input values within 1% of each other.
106+
return doom_reason_.empty() && !is_range_finding_phase_ &&
107+
abs(current_load_value_ / previous_load_value_ - 1.0) < 0.01;
108+
}
109+
110+
bool ExponentialSearchStepController::IsDoomed(std::string& doom_reason) const {
111+
if (doom_reason_.empty()) {
112+
return false;
113+
}
114+
doom_reason = doom_reason_;
115+
return true;
116+
}
117+
118+
void ExponentialSearchStepController::UpdateAndRecompute(const BenchmarkResult& benchmark_result) {
119+
if (benchmark_result.status().code()) {
120+
doom_reason_ = "Nighthawk Service returned an error.";
121+
return;
122+
}
123+
const double score = TotalScore(benchmark_result);
124+
if (is_range_finding_phase_) {
125+
IterateRangeFindingPhase(score);
126+
} else {
127+
IterateBinarySearchPhase(score);
128+
}
129+
}
130+
131+
/**
132+
* Updates state variables based on the latest score. Exponentially increases the load in each step.
133+
* Transitions to the binary search phase when the load has caused metrics to go outside thresholds.
134+
*/
135+
void ExponentialSearchStepController::IterateRangeFindingPhase(double score) {
136+
if (score > 0.0) {
137+
// Have not reached the threshold yet; continue increasing the load exponentially.
138+
previous_load_value_ = current_load_value_;
139+
current_load_value_ *= exponential_factor_;
140+
} else {
141+
// We have found a value that exceeded the threshold.
142+
// Prepare for the binary search phase.
143+
if (std::isnan(previous_load_value_)) {
144+
doom_reason_ =
145+
"ExponentialSearchStepController cannot continue if the metrics values already exceed "
146+
"metric thresholds with the initial load. Check the initial load value in the "
147+
"ExponentialSearchStepControllerConfig, requested metrics, and thresholds.";
148+
return;
149+
}
150+
is_range_finding_phase_ = false;
151+
// Binary search is between previous load (ok) and current load (too high).
152+
bottom_load_value_ = previous_load_value_;
153+
top_load_value_ = current_load_value_;
154+
155+
previous_load_value_ = current_load_value_;
156+
current_load_value_ = (bottom_load_value_ + top_load_value_) / 2;
157+
}
158+
}
159+
160+
/**
161+
* Updates state variables based on the latest score. Performs one step of a binary search.
162+
*/
163+
void ExponentialSearchStepController::IterateBinarySearchPhase(double score) {
164+
if (score > 0.0) {
165+
// Within threshold, go higher.
166+
bottom_load_value_ = current_load_value_;
167+
} else {
168+
// Outside threshold, go lower.
169+
top_load_value_ = current_load_value_;
170+
}
171+
previous_load_value_ = current_load_value_;
172+
current_load_value_ = (bottom_load_value_ + top_load_value_) / 2;
173+
}
174+
175+
} // namespace Nighthawk
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#pragma once
2+
3+
#include "envoy/registry/registry.h"
4+
5+
#include "nighthawk/adaptive_load/input_variable_setter.h"
6+
#include "nighthawk/adaptive_load/step_controller.h"
7+
8+
#include "api/adaptive_load/adaptive_load.pb.h"
9+
#include "api/adaptive_load/benchmark_result.pb.h"
10+
#include "api/adaptive_load/step_controller_impl.pb.h"
11+
12+
namespace Nighthawk {
13+
14+
/**
15+
* A StepController that performs an exponential search for the highest load that keeps metrics
16+
* within thresholds. See https://en.wikipedia.org/wiki/Exponential_search.
17+
*
18+
* Converges when the binary search values are within 1%. Report doom if the initial load already
19+
* caused metrics to exceed thresholds, or if any Nighthawk result has an error status.
20+
*
21+
* Example usage in adaptive load session spec:
22+
* // ...
23+
* step_controller_config {
24+
* name: "nighthawk.exponential_search"
25+
* typed_config {
26+
* [type.googleapis.com/nighthawk.adaptive_load.ExponentialSearchStepControllerConfig] {
27+
* initial_value: 10.0
28+
* }
29+
* }
30+
* }
31+
* // ...
32+
*/
33+
class ExponentialSearchStepController : public StepController {
34+
public:
35+
explicit ExponentialSearchStepController(
36+
const nighthawk::adaptive_load::ExponentialSearchStepControllerConfig& config,
37+
nighthawk::client::CommandLineOptions command_line_options_template);
38+
absl::StatusOr<nighthawk::client::CommandLineOptions>
39+
GetCurrentCommandLineOptions() const override;
40+
bool IsConverged() const override;
41+
bool IsDoomed(std::string& doom_reason) const override;
42+
void UpdateAndRecompute(const nighthawk::adaptive_load::BenchmarkResult& result) override;
43+
44+
private:
45+
void IterateRangeFindingPhase(double score);
46+
void IterateBinarySearchPhase(double score);
47+
48+
// Proto defining the traffic request to be sent to Nighthawk, apart from what is set by the
49+
// InputVariableSetter.
50+
const nighthawk::client::CommandLineOptions command_line_options_template_;
51+
// A plugin that applies a numerical load value to the traffic definition, e.g by setting
52+
// requests_per_second.
53+
InputVariableSetterPtr input_variable_setter_;
54+
// Whether the algorithm is in the initial range finding phase, as opposed to the subsequent
55+
// binary search phase.
56+
bool is_range_finding_phase_{true};
57+
// The factor for increasing the load value in each recalculation during the range finding phase.
58+
double exponential_factor_;
59+
// The previous load the controller recommended before the most recent recalculation, in both
60+
// range finding and binary search phases.
61+
double previous_load_value_{std::numeric_limits<double>::signaling_NaN()};
62+
// The load the controller will currently recommend, until the next recalculation, in both range
63+
// finding and binary search phases.
64+
double current_load_value_;
65+
// The current bottom of the search range during the binary search phase.
66+
double bottom_load_value_{std::numeric_limits<double>::signaling_NaN()};
67+
// The current top of the search range during the binary search phase.
68+
double top_load_value_{std::numeric_limits<double>::signaling_NaN()};
69+
// Set when an error has been detected; exposed via IsDoomed().
70+
std::string doom_reason_;
71+
};
72+
73+
/**
74+
* Factory that creates an ExponentialSearchStepController from an
75+
* ExponentialSearchStepControllerConfig proto. Registered as an Envoy plugin.
76+
*/
77+
class ExponentialSearchStepControllerConfigFactory : public StepControllerConfigFactory {
78+
public:
79+
std::string name() const override;
80+
Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override;
81+
absl::Status ValidateConfig(const Envoy::Protobuf::Message& config) const override;
82+
StepControllerPtr createStepController(
83+
const Envoy::Protobuf::Message& config,
84+
const nighthawk::client::CommandLineOptions& command_line_options_template) override;
85+
};
86+
87+
// This factory is activated through LoadStepControllerPlugin in plugin_util.h.
88+
DECLARE_FACTORY(ExponentialSearchStepControllerConfigFactory);
89+
90+
} // namespace Nighthawk

0 commit comments

Comments
 (0)