Skip to content

Commit bf03463

Browse files
shashank-iitbhutenzen-y
shashank-iitbhu
and
tenzen-y
authored
[GSOC] hyperopt suggestion service logic update (#2412)
* resolved merge conflicts Signed-off-by: Shashank Mittal <[email protected]> * fix Signed-off-by: Shashank Mittal <[email protected]> * DISTRIBUTION_UNKNOWN enum set to 0 in gRPC api Signed-off-by: Shashank Mittal <[email protected]> * convert parameter method fix Signed-off-by: Shashank Mittal <[email protected]> validation fix add e2e tests for hyperopt added e2e test to workflow * convert feasibleSpace func updated Signed-off-by: Shashank Mittal <[email protected]> * renamed DISTRIBUTION_UNKNOWN to DISTRIBUTION_UNSPECIFIED Signed-off-by: Shashank Mittal <[email protected]> * fix Signed-off-by: Shashank Mittal <[email protected]> * added more test cases for hyperopt distributions Signed-off-by: Shashank Mittal <[email protected]> * added support for NORMAL and LOG_NORMAL in hyperopt suggestion service Signed-off-by: Shashank Mittal <[email protected]> * added e2e tests for NORMAL and LOG_NORMAL Signed-off-by: Shashank Mittal <[email protected]> sigma calculation fixed fix parse new arguments to mnist.py * hyperopt-suggestion example update Signed-off-by: Shashank Mittal <[email protected]> * updated logic for log distributions Signed-off-by: Shashank Mittal <[email protected]> * updated logic for log distributions Signed-off-by: Shashank Mittal <[email protected]> * e2e test fixed Signed-off-by: Shashank Mittal <[email protected]> * added support for parameter distributions for Parameter type INT Signed-off-by: Shashank Mittal <[email protected]> * unit test fixed Signed-off-by: Shashank Mittal <[email protected]> * Update pkg/suggestion/v1beta1/hyperopt/base_service.py Co-authored-by: Yuki Iwai <[email protected]> Signed-off-by: Shashank Mittal <[email protected]> * comment fixed Signed-off-by: Shashank Mittal <[email protected]> * added unit tests for INT parameter type Signed-off-by: Shashank Mittal <[email protected]> * completed param unit test cases Signed-off-by: Shashank Mittal <[email protected]> * handled default case for normal distributions when min or max are not specified Signed-off-by: Shashank Mittal <[email protected]> * fixed validation logic for min and max Signed-off-by: Shashank Mittal <[email protected]> * removed unnecessary test params Signed-off-by: Shashank Mittal <[email protected]> * fixes Signed-off-by: Shashank Mittal <[email protected]> * added comments Signed-off-by: Shashank Mittal <[email protected]> * fix Signed-off-by: Shashank Mittal <[email protected]> * set default distribution as uniform Signed-off-by: Shashank Mittal <[email protected]> * line omit Signed-off-by: Shashank Mittal <[email protected]> * removed empty spaces from yaml files Signed-off-by: Shashank Mittal <[email protected]> --------- Signed-off-by: Shashank Mittal <[email protected]> Co-authored-by: Yuki Iwai <[email protected]>
1 parent 741238d commit bf03463

File tree

14 files changed

+420
-144
lines changed

14 files changed

+420
-144
lines changed

‎.github/workflows/e2e-test-pytorch-mnist.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,6 @@ jobs:
4141
- "long-running-resume,from-volume-resume,median-stop"
4242
# others
4343
- "grid,bayesian-optimization,tpe,multivariate-tpe,cma-es,hyperband"
44+
- "hyperopt-distribution"
4445
- "file-metrics-collector,pytorchjob-mnist"
4546
- "median-stop-with-json-format,file-metrics-collector-with-json-format"

‎examples/v1beta1/hp-tuning/hyperopt-distribution.yaml

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
---
2+
apiVersion: kubeflow.org/v1beta1
3+
kind: Experiment
4+
metadata:
5+
namespace: kubeflow
6+
name: hyperopt-distribution
7+
spec:
8+
objective:
9+
type: minimize
10+
goal: 0.05
11+
objectiveMetricName: loss
12+
algorithm:
13+
algorithmName: random
14+
parallelTrialCount: 3
15+
maxTrialCount: 12
16+
maxFailedTrialCount: 3
17+
parameters:
18+
- name: lr
19+
parameterType: double
20+
feasibleSpace:
21+
min: "0.01"
22+
max: "0.05"
23+
step: "0.01"
24+
distribution: normal
25+
- name: momentum
26+
parameterType: double
27+
feasibleSpace:
28+
min: "0.001"
29+
max: "1"
30+
distribution: uniform
31+
- name: epochs
32+
parameterType: int
33+
feasibleSpace:
34+
min: "1"
35+
max: "3"
36+
distribution: logUniform
37+
- name: batch_size
38+
parameterType: int
39+
feasibleSpace:
40+
min: "32"
41+
max: "64"
42+
distribution: logNormal
43+
trialTemplate:
44+
primaryContainerName: training-container
45+
trialParameters:
46+
- name: learningRate
47+
description: Learning rate for the training model
48+
reference: lr
49+
- name: momentum
50+
description: Momentum for the training model
51+
reference: momentum
52+
- name: epochs
53+
description: Epochs
54+
reference: epochs
55+
- name: batchSize
56+
description: Batch Size
57+
reference: batch_size
58+
trialSpec:
59+
apiVersion: batch/v1
60+
kind: Job
61+
spec:
62+
template:
63+
spec:
64+
containers:
65+
- name: training-container
66+
image: docker.io/kubeflowkatib/pytorch-mnist-cpu:latest
67+
command:
68+
- "python3"
69+
- "/opt/pytorch-mnist/mnist.py"
70+
- "--epochs=${trialParameters.epochs}"
71+
- "--batch-size=${trialParameters.batchSize}"
72+
- "--lr=${trialParameters.learningRate}"
73+
- "--momentum=${trialParameters.momentum}"
74+
restartPolicy: Never

‎pkg/apis/controller/experiments/v1beta1/experiment_defaults.go

+9
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ func (e *Experiment) SetDefault() {
3030
e.setDefaultObjective()
3131
e.setDefaultTrialTemplate()
3232
e.setDefaultMetricsCollector()
33+
e.setDefaultParameterDistribution()
3334
}
3435

3536
func (e *Experiment) setDefaultParallelTrialCount() {
@@ -176,3 +177,11 @@ func (e *Experiment) setDefaultMetricsCollector() {
176177
}
177178
}
178179
}
180+
181+
func (e *Experiment) setDefaultParameterDistribution() {
182+
for i := range e.Spec.Parameters {
183+
if e.Spec.Parameters[i].FeasibleSpace.Distribution == "" {
184+
e.Spec.Parameters[i].FeasibleSpace.Distribution = DistributionUniform
185+
}
186+
}
187+
}

‎pkg/apis/manager/v1beta1/api.pb.go

+90-90
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/apis/manager/v1beta1/api.proto

+5-5
Original file line numberDiff line numberDiff line change
@@ -101,11 +101,11 @@ enum ParameterType {
101101
* Distribution types for HyperParameter.
102102
*/
103103
enum Distribution {
104-
UNIFORM = 0;
105-
LOG_UNIFORM = 1;
106-
NORMAL = 2;
107-
LOG_NORMAL = 3;
108-
DISTRIBUTION_UNKNOWN = 4;
104+
DISTRIBUTION_UNSPECIFIED = 0;
105+
UNIFORM = 1;
106+
LOG_UNIFORM = 2;
107+
NORMAL = 3;
108+
LOG_NORMAL = 4;
109109
}
110110

111111
/**

‎pkg/apis/manager/v1beta1/python/api_pb2.py

+12-12
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/apis/manager/v1beta1/python/api_pb2.pyi

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@ class ParameterType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
1616

1717
class Distribution(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
1818
__slots__ = ()
19+
DISTRIBUTION_UNSPECIFIED: _ClassVar[Distribution]
1920
UNIFORM: _ClassVar[Distribution]
2021
LOG_UNIFORM: _ClassVar[Distribution]
2122
NORMAL: _ClassVar[Distribution]
2223
LOG_NORMAL: _ClassVar[Distribution]
23-
DISTRIBUTION_UNKNOWN: _ClassVar[Distribution]
2424

2525
class ObjectiveType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
2626
__slots__ = ()
@@ -39,11 +39,11 @@ DOUBLE: ParameterType
3939
INT: ParameterType
4040
DISCRETE: ParameterType
4141
CATEGORICAL: ParameterType
42+
DISTRIBUTION_UNSPECIFIED: Distribution
4243
UNIFORM: Distribution
4344
LOG_UNIFORM: Distribution
4445
NORMAL: Distribution
4546
LOG_NORMAL: Distribution
46-
DISTRIBUTION_UNKNOWN: Distribution
4747
UNKNOWN: ObjectiveType
4848
MINIMIZE: ObjectiveType
4949
MAXIMIZE: ObjectiveType

‎pkg/controller.v1beta1/suggestion/suggestionclient/suggestionclient.go

+2-12
Original file line numberDiff line numberDiff line change
@@ -532,22 +532,12 @@ func convertParameterType(typ experimentsv1beta1.ParameterType) suggestionapi.Pa
532532
}
533533

534534
func convertFeasibleSpace(fs experimentsv1beta1.FeasibleSpace) *suggestionapi.FeasibleSpace {
535-
distribution := convertDistribution(fs.Distribution)
536-
if distribution == suggestionapi.Distribution_DISTRIBUTION_UNKNOWN {
537-
return &suggestionapi.FeasibleSpace{
538-
Max: fs.Max,
539-
Min: fs.Min,
540-
List: fs.List,
541-
Step: fs.Step,
542-
}
543-
}
544-
545535
return &suggestionapi.FeasibleSpace{
546536
Max: fs.Max,
547537
Min: fs.Min,
548538
List: fs.List,
549539
Step: fs.Step,
550-
Distribution: distribution,
540+
Distribution: convertDistribution(fs.Distribution),
551541
}
552542
}
553543

@@ -562,7 +552,7 @@ func convertDistribution(typ experimentsv1beta1.Distribution) suggestionapi.Dist
562552
case experimentsv1beta1.DistributionLogNormal:
563553
return suggestionapi.Distribution_LOG_NORMAL
564554
default:
565-
return suggestionapi.Distribution_DISTRIBUTION_UNKNOWN
555+
return suggestionapi.Distribution_DISTRIBUTION_UNSPECIFIED
566556
}
567557
}
568558

‎pkg/controller.v1beta1/suggestion/suggestionclient/suggestionclient_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ func TestConvertDistribution(t *testing.T) {
618618
},
619619
{
620620
inDistribution: experimentsv1beta1.DistributionUnknown,
621-
expectedDistribution: suggestionapi.Distribution_DISTRIBUTION_UNKNOWN,
621+
expectedDistribution: suggestionapi.Distribution_DISTRIBUTION_UNSPECIFIED,
622622
testDescription: "Convert unknown distribution",
623623
},
624624
}

‎pkg/suggestion/v1beta1/hyperopt/base_service.py

+83-8
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,12 @@
1313
# limitations under the License.
1414

1515
import logging
16+
import math
1617

1718
import hyperopt
1819
import numpy as np
1920

21+
from pkg.apis.manager.v1beta1.python import api_pb2
2022
from pkg.suggestion.v1beta1.internal.constant import (
2123
CATEGORICAL,
2224
DISCRETE,
@@ -62,14 +64,87 @@ def create_hyperopt_domain(self):
6264
# hyperopt.hp.uniform('x2', -10, 10)}
6365
hyperopt_search_space = {}
6466
for param in self.search_space.params:
65-
if param.type == INTEGER:
66-
hyperopt_search_space[param.name] = hyperopt.hp.quniform(
67-
param.name, float(param.min), float(param.max), float(param.step)
68-
)
69-
elif param.type == DOUBLE:
70-
hyperopt_search_space[param.name] = hyperopt.hp.uniform(
71-
param.name, float(param.min), float(param.max)
72-
)
67+
if param.type in [INTEGER, DOUBLE]:
68+
if param.distribution == api_pb2.UNIFORM or param.distribution is None:
69+
# Uniform distribution: values are sampled between min and max.
70+
# If step is defined, we use the quantized version quniform.
71+
if param.step:
72+
hyperopt_search_space[param.name] = hyperopt.hp.quniform(
73+
param.name,
74+
float(param.min),
75+
float(param.max),
76+
float(param.step),
77+
)
78+
elif param.type == INTEGER:
79+
hyperopt_search_space[param.name] = hyperopt.hp.uniformint(
80+
param.name, float(param.min), float(param.max)
81+
)
82+
else:
83+
hyperopt_search_space[param.name] = hyperopt.hp.uniform(
84+
param.name, float(param.min), float(param.max)
85+
)
86+
elif param.distribution == api_pb2.LOG_UNIFORM:
87+
# Log-uniform distribution: used for parameters that vary exponentially.
88+
# We convert min and max to their logarithmic scale using math.log, because
89+
# the log-uniform distribution is applied over the logarithmic range.
90+
if param.step:
91+
hyperopt_search_space[param.name] = hyperopt.hp.qloguniform(
92+
param.name,
93+
math.log(float(param.min)),
94+
math.log(float(param.max)),
95+
float(param.step),
96+
)
97+
else:
98+
hyperopt_search_space[param.name] = hyperopt.hp.loguniform(
99+
param.name,
100+
math.log(float(param.min)),
101+
math.log(float(param.max)),
102+
)
103+
elif param.distribution == api_pb2.NORMAL:
104+
# Normal distribution: used when values are centered around the mean (mu)
105+
# and spread out by sigma. We calculate mu as the midpoint between
106+
# min and max, and sigma as (max - min) / 6. This is based on the assumption
107+
# that 99.7% of the values in a normal distribution fall within ±3 sigma.
108+
mu = (float(param.min) + float(param.max)) / 2
109+
sigma = (float(param.max) - float(param.min)) / 6
110+
111+
if param.step:
112+
hyperopt_search_space[param.name] = hyperopt.hp.qnormal(
113+
param.name,
114+
mu,
115+
sigma,
116+
float(param.step),
117+
)
118+
else:
119+
hyperopt_search_space[param.name] = hyperopt.hp.normal(
120+
param.name,
121+
mu,
122+
sigma,
123+
)
124+
elif param.distribution == api_pb2.LOG_NORMAL:
125+
# Log-normal distribution: applies when the logarithm
126+
# of the parameter follows a normal distribution.
127+
# We convert min and max to logarithmic scale and calculate
128+
# mu and sigma similarly to the normal distribution,
129+
# but on the log-transformed values to ensure the distribution is correct.
130+
log_min = math.log(float(param.min))
131+
log_max = math.log(float(param.max))
132+
mu = (log_min + log_max) / 2
133+
sigma = (log_max - log_min) / 6
134+
135+
if param.step:
136+
hyperopt_search_space[param.name] = hyperopt.hp.qlognormal(
137+
param.name,
138+
mu,
139+
sigma,
140+
float(param.step),
141+
)
142+
else:
143+
hyperopt_search_space[param.name] = hyperopt.hp.lognormal(
144+
param.name,
145+
mu,
146+
sigma,
147+
)
73148
elif param.type == CATEGORICAL or param.type == DISCRETE:
74149
hyperopt_search_space[param.name] = hyperopt.hp.choice(
75150
param.name, param.list

‎pkg/suggestion/v1beta1/internal/constant.py

+5
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,8 @@
1919
DOUBLE = "DOUBLE"
2020
CATEGORICAL = "CATEGORICAL"
2121
DISCRETE = "DISCRETE"
22+
23+
UNIFORM = "UNIFORM"
24+
LOG_UNIFORM = "LOG_UNIFORM"
25+
NORMAL = "NORMAL"
26+
LOG_NORMAL = "LOG_NORMAL"

‎pkg/suggestion/v1beta1/internal/search_space.py

+26-13
Original file line numberDiff line numberDiff line change
@@ -82,25 +82,36 @@ def __str__(self):
8282

8383
@staticmethod
8484
def convert_parameter(p):
85+
distribution = (
86+
p.feasible_space.distribution
87+
if p.feasible_space.distribution != ""
88+
and p.feasible_space.distribution is not None
89+
and p.feasible_space.distribution != api.DISTRIBUTION_UNSPECIFIED
90+
else None
91+
)
92+
8593
if p.parameter_type == api.INT:
8694
# Default value for INT parameter step is 1
87-
step = 1
88-
if p.feasible_space.step is not None and p.feasible_space.step != "":
89-
step = p.feasible_space.step
95+
step = p.feasible_space.step if p.feasible_space.step else 1
9096
return HyperParameter.int(
91-
p.name, p.feasible_space.min, p.feasible_space.max, step
97+
p.name, p.feasible_space.min, p.feasible_space.max, step, distribution
9298
)
99+
93100
elif p.parameter_type == api.DOUBLE:
94101
return HyperParameter.double(
95102
p.name,
96103
p.feasible_space.min,
97104
p.feasible_space.max,
98105
p.feasible_space.step,
106+
distribution,
99107
)
108+
100109
elif p.parameter_type == api.CATEGORICAL:
101110
return HyperParameter.categorical(p.name, p.feasible_space.list)
111+
102112
elif p.parameter_type == api.DISCRETE:
103113
return HyperParameter.discrete(p.name, p.feasible_space.list)
114+
104115
else:
105116
logger.error(
106117
"Cannot get the type for the parameter: %s (%s)",
@@ -110,33 +121,35 @@ def convert_parameter(p):
110121

111122

112123
class HyperParameter(object):
113-
def __init__(self, name, type_, min_, max_, list_, step):
124+
def __init__(self, name, type_, min_, max_, list_, step, distribution=None):
114125
self.name = name
115126
self.type = type_
116127
self.min = min_
117128
self.max = max_
118129
self.list = list_
119130
self.step = step
131+
self.distribution = distribution
120132

121133
def __str__(self):
122-
if self.type == constant.INTEGER or self.type == constant.DOUBLE:
134+
if self.type in [constant.INTEGER, constant.DOUBLE]:
123135
return (
124-
"HyperParameter(name: {}, type: {}, min: {}, max: {}, step: {})".format(
125-
self.name, self.type, self.min, self.max, self.step
126-
)
136+
f"HyperParameter(name: {self.name}, type: {self.type}, min: {self.min}, "
137+
f"max: {self.max}, step: {self.step}, distribution: {self.distribution})"
127138
)
128139
else:
129140
return "HyperParameter(name: {}, type: {}, list: {})".format(
130141
self.name, self.type, ", ".join(self.list)
131142
)
132143

133144
@staticmethod
134-
def int(name, min_, max_, step):
135-
return HyperParameter(name, constant.INTEGER, min_, max_, [], step)
145+
def int(name, min_, max_, step, distribution=None):
146+
return HyperParameter(
147+
name, constant.INTEGER, min_, max_, [], step, distribution
148+
)
136149

137150
@staticmethod
138-
def double(name, min_, max_, step):
139-
return HyperParameter(name, constant.DOUBLE, min_, max_, [], step)
151+
def double(name, min_, max_, step, distribution=None):
152+
return HyperParameter(name, constant.DOUBLE, min_, max_, [], step, distribution)
140153

141154
@staticmethod
142155
def categorical(name, lst):

‎pkg/webhook/v1beta1/experiment/validator/validator.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ func (g *DefaultValidator) validateParameters(parameters []experimentsv1beta1.Pa
284284
allErrs = append(allErrs, field.Invalid(parametersPath.Index(i).Child("feasibleSpace").Child("list"),
285285
param.FeasibleSpace.List, fmt.Sprintf("feasibleSpace.list is not supported for parameterType: %v", param.ParameterType)))
286286
}
287-
if param.FeasibleSpace.Max == "" && param.FeasibleSpace.Min == "" {
287+
if param.FeasibleSpace.Max == "" || param.FeasibleSpace.Min == "" {
288288
allErrs = append(allErrs, field.Required(parametersPath.Index(i).Child("feasibleSpace").Child("max"),
289289
fmt.Sprintf("feasibleSpace.max or feasibleSpace.min must be specified for parameterType: %v", param.ParameterType)))
290290
}

0 commit comments

Comments
 (0)