Skip to content

Implements metric normalization in the HV Curve Converter #1228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Implements metric normalization in the HV Curve Converter #1228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions vizier/_src/algorithms/ensemble/ensemble_designer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ def curve_generator() -> analyzers.StatefulCurveConverter:
reference_value=self.reference_value,
num_vectors=self.num_vectors,
infer_origin_factor=0.1,
disable_metric_normalization=True,
)

stateful_curve_generator = analyzers.RestartingCurveConverter(
Expand Down
40 changes: 28 additions & 12 deletions vizier/_src/benchmarks/analyzers/convergence_curve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,7 @@ def __init__(
reference_value: Optional[np.ndarray] = None,
num_vectors: int = 10000,
infer_origin_factor: float = 0.0,
disable_metric_normalization: bool = False,
):
"""Init.

Expand All @@ -361,6 +362,7 @@ def __init__(
num_vectors: Number of vectors from which hypervolume is computed.
infer_origin_factor: When inferring the reference point, set origin to be
minimum value - factor * (range).
disable_metric_normalization: If True, disable metric normalization.
"""
if len(metric_informations) < 2:
raise ValueError(
Expand All @@ -386,8 +388,9 @@ def create_metric_converter(mc):
self._origin_value = reference_value
# TODO: Speed this up with hypervolume vector tracking.
self._min_trial_idx = 1
self._pareto_frontier = np.empty(shape=(0, len(metric_informations)))
self._infer_origin_factor = infer_origin_factor
self._disable_metric_normalization = disable_metric_normalization
self._all_metrics = np.empty(shape=(0, len(metric_informations)))

def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
"""Returns ConvergenceCurve with a curve of shape 1 x len(trials)."""
Expand All @@ -396,6 +399,28 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
raise ValueError(f'No trials provided {trials}')

metrics = self._converter.to_labels_array(trials)
self._all_metrics = np.vstack([self._all_metrics, metrics])
if self._disable_metric_normalization:
normalizer = 1.0
else:
normalizer = np.nanmedian(
np.absolute(
self._all_metrics
- np.nanmedian(self._all_metrics, axis=0, keepdims=True)
),
axis=0,
keepdims=True,
)
if np.any(normalizer <= 0):
raise ValueError(
'Some metric normalizers are zero. This is likely due to some'
' metrics being identical or contain a lot of duplicates across'
f' trials. Normalizers: {normalizer}'
)
metrics = metrics / normalizer
# shape is [num_existing_points + num_new_points, num_metrics]
all_metrics = self._all_metrics / normalizer

if self._origin_value is None:
# Set origin to the minimum of finite values.
origin = np.zeros(shape=(metrics.shape[1],))
Expand Down Expand Up @@ -428,10 +453,6 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
)
origin = self._origin_value

# Calculate cumulative hypervolume with the Pareto frontier.
all_metrics = np.vstack(
[self._pareto_frontier, metrics]
) # shape is [num_pareto_points + num_points, feature dimension]
front = multimetric.ParetoFrontier(
points=all_metrics,
origin=origin,
Expand All @@ -440,17 +461,12 @@ def convert(self, trials: Sequence[pyvizier.Trial]) -> ConvergenceCurve:
)
all_hv_curve = front.hypervolume(is_cumulative=True)

# Remove the Pareto frontier add-in and update state.
hv_curve = all_hv_curve[len(self._pareto_frontier) :]
# Extracts the hv points for the new trials.
hv_curve = all_hv_curve[-metrics.shape[0] :]
xs = np.asarray(
range(self._min_trial_idx, len(hv_curve) + self._min_trial_idx)
)
self._min_trial_idx += len(hv_curve)
algo = multimetric.FastParetoOptimalAlgorithm(
xla_pareto.JaxParetoOptimalAlgorithm()
)
pareto_points = algo.is_pareto_optimal(points=all_metrics)
self._pareto_frontier = all_metrics[pareto_points]

return ConvergenceCurve(
xs=xs,
Expand Down
51 changes: 27 additions & 24 deletions vizier/_src/benchmarks/analyzers/convergence_curve_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,25 +278,28 @@ def test_convert_with_origin_reference(self):
pytrials = []
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 4.0, 'min': 2.0})
pyvizier.Measurement(metrics={'max': 8.0, 'min': 0.0})
)
)
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 3.0, 'min': -1.0})
pyvizier.Measurement(metrics={'max': 6.0, 'min': -2.0})
)
)
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0})
pyvizier.Measurement(metrics={'max': 4.0, 'min': -4.0})
)
)

curve = generator.convert(pytrials)
np.testing.assert_array_equal(curve.xs, [1, 2, 3])
np.testing.assert_array_almost_equal(
curve.ys, [[0.0, 3.0, 8.0]], decimal=0.5
)
# After the sign of the minimization metric is flipped, the three metric
# points are (8, 0), (6, 2), (4, 4). After normalization, they become
# (4, 0), (3, 1), (2, 2). The origin is (0, 0). The sequence of hypervolume
# is expected to be (0, 3.5, 6.0). However, we allow a large error because
# the hypervolume computation is approximate.
np.testing.assert_array_almost_equal(curve.ys, [[0.0, 3.5, 6.0]], decimal=0)

def test_convert_with_reference(self):
generator = convergence.HypervolumeCurveConverter(
Expand All @@ -313,25 +316,29 @@ def test_convert_with_reference(self):
pytrials = []
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 4.0, 'min': 2.0})
pyvizier.Measurement(metrics={'max': 8.0, 'min': 0.0})
)
)
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 3.0, 'min': -1.0})
pyvizier.Measurement(metrics={'max': 6.0, 'min': -2.0})
)
)
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0})
pyvizier.Measurement(metrics={'max': 4.0, 'min': -4.0})
)
)

curve = generator.convert(pytrials)
np.testing.assert_array_equal(curve.xs, [1, 2, 3])
np.testing.assert_array_almost_equal(
curve.ys, [[0.0, 0.0, 2.0]], decimal=0.5
)
# After the sign of the minimization metric is flipped, the three metric
# points are (8, 0), (6, 2), (4, 4). After normalization, they become
# (4, 0), (3, 1), (2, 2). After accounting for the reference point (3, 0),
# they are (1, 0), (0, 1), (-1, 2). The sequence of hypervolume
# is expected to be (0, 0.5, 0.5). However, we allow a large error
# because the hypervolume computation is approximate.
np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.5, 0.5]], decimal=0)

def test_convert_with_none_reference(self):
generator = convergence.HypervolumeCurveConverter([
Expand Down Expand Up @@ -361,9 +368,7 @@ def test_convert_with_none_reference(self):

curve = generator.convert(pytrials)
np.testing.assert_array_equal(curve.xs, [1, 2, 3])
np.testing.assert_array_almost_equal(
curve.ys, [[0.0, 0.0, 1.0]], decimal=0.5
)
np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.0, 1.0]], decimal=0)

def test_convert_with_inf_none_reference(self):
generator = convergence.HypervolumeCurveConverter([
Expand Down Expand Up @@ -401,6 +406,7 @@ def test_convert_with_state(self):
),
],
reference_value=np.array([0.0]),
disable_metric_normalization=True,
)
pytrials = []
pytrials.append(
Expand All @@ -415,15 +421,13 @@ def test_convert_with_state(self):
)
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 4.0, 'min': -2.0})
pyvizier.Measurement(metrics={'max': 3.0, 'min': -2.0})
)
)

curve = generator.convert(pytrials)
np.testing.assert_array_equal(curve.xs, [1, 2, 3])
np.testing.assert_array_almost_equal(
curve.ys, [[0.0, 5.0, 9.0]], decimal=0.5
)
np.testing.assert_array_almost_equal(curve.ys, [[0.0, 5.0, 9.0]], decimal=0)

pytrials = []
pytrials.append(
Expand All @@ -439,7 +443,7 @@ def test_convert_with_state(self):

curve = generator.convert(pytrials)
np.testing.assert_array_equal(curve.xs, [4, 5])
np.testing.assert_array_almost_equal(curve.ys, [[9.0, 10.0]], decimal=0.5)
np.testing.assert_array_almost_equal(curve.ys, [[9.0, 10.0]], decimal=0)

def test_convert_factor_with_inf(self):
generator = convergence.HypervolumeCurveConverter(
Expand Down Expand Up @@ -534,7 +538,7 @@ def test_convert_multiobjective(self):
pytrials = []
pytrials.append(
pyvizier.Trial().complete(
pyvizier.Measurement(metrics={'max': 4.0, 'min': -1.0, 'safe': 1.0})
pyvizier.Measurement(metrics={'max': 2.0, 'min': 0.0, 'safe': 1.0})
)
)
pytrials.append(
Expand All @@ -552,9 +556,7 @@ def test_convert_multiobjective(self):

curve = generator.convert(pytrials)
np.testing.assert_array_equal(curve.xs, [1, 2, 3])
np.testing.assert_array_almost_equal(
curve.ys, [[4.0, 4.0, 8.0]], decimal=0.5
)
np.testing.assert_array_almost_equal(curve.ys, [[0.0, 0.0, 2.0]], decimal=0)


class RestartingCurveConverterTest(absltest.TestCase):
Expand All @@ -570,6 +572,7 @@ def converter_factory():
name='min', goal=pyvizier.ObjectiveMetricGoal.MINIMIZE
),
],
disable_metric_normalization=True,
)

restart_converter = convergence.RestartingCurveConverter(
Expand Down
Loading