Skip to content

Commit 097115d

Browse files
gmarouligmarouli
committed
Do not use DownsampleConfig in DataStreamLifecycle
1 parent 345346d commit 097115d

File tree

15 files changed

+68
-198
lines changed

15 files changed

+68
-198
lines changed

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -556,7 +556,7 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
556556
String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName(
557557
DOWNSAMPLED_INDEX_PREFIX,
558558
backingIndex,
559-
round.config().getFixedInterval()
559+
round.fixedInterval()
560560
);
561561
IndexMetadata targetDownsampleIndexMeta = project.index(downsampleIndexName);
562562
boolean targetDownsampleIndexExists = targetDownsampleIndexMeta != null;
@@ -601,7 +601,7 @@ private void downsampleIndexOnce(
601601
sourceIndex,
602602
downsampleIndexName,
603603
null,
604-
round.config()
604+
new DownsampleConfig(round.fixedInterval(), null)
605605
);
606606
transportActionsDeduplicator.executeOnce(
607607
Tuple.tuple(projectId, request),

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99

1010
package org.elasticsearch.datastreams;
1111

12-
import org.elasticsearch.action.downsample.DownsampleConfig;
1312
import org.elasticsearch.cluster.metadata.ComponentTemplate;
1413
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1514
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
@@ -255,7 +254,7 @@ private static List<DataStreamLifecycle.DownsamplingRound> randomRounds() {
255254
List<DataStreamLifecycle.DownsamplingRound> rounds = new ArrayList<>();
256255
var previous = new DataStreamLifecycle.DownsamplingRound(
257256
TimeValue.timeValueDays(randomIntBetween(1, 365)),
258-
new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h"))
257+
new DateHistogramInterval(randomIntBetween(1, 24) + "h")
259258
);
260259
rounds.add(previous);
261260
for (int i = 0; i < count; i++) {
@@ -268,9 +267,7 @@ private static List<DataStreamLifecycle.DownsamplingRound> randomRounds() {
268267

269268
private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) {
270269
var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10));
271-
var fixedInterval = new DownsampleConfig(
272-
new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms")
273-
);
270+
var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms");
274271
return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval);
275272
}
276273

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
1313
import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
1414
import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
15-
import org.elasticsearch.action.downsample.DownsampleConfig;
1615
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1716
import org.elasticsearch.cluster.metadata.DataStream;
1817
import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
@@ -164,7 +163,7 @@ private static List<DataStreamLifecycle.DownsamplingRound> randomDownsamplingRou
164163
List<DataStreamLifecycle.DownsamplingRound> rounds = new ArrayList<>();
165164
var previous = new DataStreamLifecycle.DownsamplingRound(
166165
TimeValue.timeValueDays(randomIntBetween(1, 365)),
167-
new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h"))
166+
new DateHistogramInterval(randomIntBetween(1, 24) + "h")
168167
);
169168
rounds.add(previous);
170169
for (int i = 0; i < count; i++) {
@@ -177,9 +176,7 @@ private static List<DataStreamLifecycle.DownsamplingRound> randomDownsamplingRou
177176

178177
private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) {
179178
var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10));
180-
var fixedInterval = new DownsampleConfig(
181-
new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms")
182-
);
179+
var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms");
183180
return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval);
184181
}
185182
}

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1228,9 +1228,7 @@ public void testDownsampling() throws Exception {
12281228
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
12291229
.put("index.routing_path", "@timestamp"),
12301230
DataStreamLifecycle.dataLifecycleBuilder()
1231-
.downsampling(
1232-
List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))))
1233-
)
1231+
.downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m"))))
12341232
.dataRetention(TimeValue.MAX_VALUE)
12351233
.build(),
12361234
now
@@ -1377,9 +1375,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc
13771375
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
13781376
.put("index.routing_path", "@timestamp"),
13791377
DataStreamLifecycle.dataLifecycleBuilder()
1380-
.downsampling(
1381-
List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))))
1382-
)
1378+
.downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m"))))
13831379
.dataRetention(TimeValue.MAX_VALUE)
13841380
.build(),
13851381
now
@@ -1662,9 +1658,7 @@ private ClusterState downsampleSetup(ProjectId projectId, String dataStreamName,
16621658
settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
16631659
.put("index.routing_path", "@timestamp"),
16641660
DataStreamLifecycle.dataLifecycleBuilder()
1665-
.downsampling(
1666-
List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))))
1667-
)
1661+
.downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m"))))
16681662
.dataRetention(TimeValue.timeValueMillis(1))
16691663
.build(),
16701664
now

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
5858

5959
// Versions over the wire
6060
public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X;
61+
public static final TransportVersion ADD_LAST_VALUE_DOWNSAMPLE_DLM = TransportVersion.fromName("add_last_value_downsample_dlm");
6162
public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention";
6263

6364
public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode";
@@ -522,9 +523,9 @@ public String displayName() {
522523
/**
523524
* A round represents the configuration for when and how elasticsearch will downsample a backing index.
524525
* @param after is a TimeValue configuring how old (based on generation age) should a backing index be before downsampling
525-
* @param config contains the interval that the backing index is going to be downsampled.
526+
* @param fixedInterval contains the interval that the backing index is going to be downsampled.
526527
*/
527-
public record DownsamplingRound(TimeValue after, DownsampleConfig config) implements Writeable, ToXContentObject {
528+
public record DownsamplingRound(TimeValue after, DateHistogramInterval fixedInterval) implements Writeable, ToXContentObject {
528529

529530
public static final ParseField AFTER_FIELD = new ParseField("after");
530531
public static final ParseField FIXED_INTERVAL_FIELD = new ParseField("fixed_interval");
@@ -533,7 +534,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem
533534
private static final ConstructingObjectParser<DownsamplingRound, Void> PARSER = new ConstructingObjectParser<>(
534535
"downsampling_round",
535536
false,
536-
(args, unused) -> new DownsamplingRound((TimeValue) args[0], new DownsampleConfig((DateHistogramInterval) args[1]))
537+
(args, unused) -> new DownsamplingRound((TimeValue) args[0], (DateHistogramInterval) args[1])
537538
);
538539

539540
static {
@@ -545,7 +546,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem
545546
PARSER.declareField(
546547
constructorArg(),
547548
p -> new DateHistogramInterval(p.text()),
548-
new ParseField(FIXED_INTERVAL_FIELD.getPreferredName()),
549+
FIXED_INTERVAL_FIELD,
549550
ObjectParser.ValueType.STRING
550551
);
551552
}
@@ -576,37 +577,42 @@ static void validateRounds(List<DownsamplingRound> rounds) {
576577
+ "."
577578
);
578579
}
579-
DownsampleConfig.validateSourceAndTargetIntervals(
580-
previous.config().getFixedInterval(),
581-
round.config().getFixedInterval()
582-
);
580+
DownsampleConfig.validateSourceAndTargetIntervals(previous.fixedInterval(), round.fixedInterval());
583581
}
584582
}
585583
}
586584

587585
public static DownsamplingRound read(StreamInput in) throws IOException {
588-
return new DownsamplingRound(in.readTimeValue(), new DownsampleConfig(in));
586+
TimeValue after = in.readTimeValue();
587+
DateHistogramInterval fixedInterval = in.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)
588+
? new DateHistogramInterval(in)
589+
: new DownsampleConfig(in).getFixedInterval();
590+
return new DownsamplingRound(after, fixedInterval);
589591
}
590592

591593
public DownsamplingRound {
592-
if (config.getFixedInterval().estimateMillis() < FIVE_MINUTES_MILLIS) {
594+
if (fixedInterval.estimateMillis() < FIVE_MINUTES_MILLIS) {
593595
throw new IllegalArgumentException(
594-
"A downsampling round must have a fixed interval of at least five minutes but found: " + config.getFixedInterval()
596+
"A downsampling round must have a fixed interval of at least five minutes but found: " + fixedInterval
595597
);
596598
}
597599
}
598600

599601
@Override
600602
public void writeTo(StreamOutput out) throws IOException {
601603
out.writeTimeValue(after);
602-
out.writeWriteable(config);
604+
if (out.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)) {
605+
out.writeWriteable(fixedInterval);
606+
} else {
607+
out.writeWriteable(new DownsampleConfig(fixedInterval, null));
608+
}
603609
}
604610

605611
@Override
606612
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
607613
builder.startObject();
608614
builder.field(AFTER_FIELD.getPreferredName(), after.getStringRep());
609-
config.toXContentFragment(builder);
615+
builder.field(FIXED_INTERVAL_FIELD.getPreferredName(), fixedInterval().toString());
610616
builder.endObject();
611617
return builder;
612618
}

‎server/src/main/resources/transport/definitions/referable/add_last_value_downsample_dlm.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
9201000

‎server/src/main/resources/transport/upper_bounds/9.3.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
inference_cached_tokens,9200000
1+
add_last_value_downsample_dlm,9201000

‎server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java‎

Lines changed: 8 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99

1010
package org.elasticsearch.cluster.metadata;
1111

12-
import org.elasticsearch.action.downsample.DownsampleConfig;
1312
import org.elasticsearch.common.Strings;
1413
import org.elasticsearch.common.io.stream.Writeable;
1514
import org.elasticsearch.core.TimeValue;
@@ -106,14 +105,8 @@ public void testInvalidDownsamplingConfiguration() {
106105
() -> DataStreamLifecycle.dataLifecycleBuilder()
107106
.downsampling(
108107
List.of(
109-
new DataStreamLifecycle.DownsamplingRound(
110-
TimeValue.timeValueDays(10),
111-
new DownsampleConfig(new DateHistogramInterval("2h"))
112-
),
113-
new DataStreamLifecycle.DownsamplingRound(
114-
TimeValue.timeValueDays(3),
115-
new DownsampleConfig(new DateHistogramInterval("2h"))
116-
)
108+
new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")),
109+
new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h"))
117110
)
118111
)
119112
.buildTemplate()
@@ -129,14 +122,8 @@ public void testInvalidDownsamplingConfiguration() {
129122
() -> DataStreamLifecycle.dataLifecycleBuilder()
130123
.downsampling(
131124
List.of(
132-
new DataStreamLifecycle.DownsamplingRound(
133-
TimeValue.timeValueDays(10),
134-
new DownsampleConfig(new DateHistogramInterval("2h"))
135-
),
136-
new DataStreamLifecycle.DownsamplingRound(
137-
TimeValue.timeValueDays(30),
138-
new DownsampleConfig(new DateHistogramInterval("2h"))
139-
)
125+
new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")),
126+
new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h"))
140127
)
141128
)
142129
.buildTemplate()
@@ -149,14 +136,8 @@ public void testInvalidDownsamplingConfiguration() {
149136
() -> DataStreamLifecycle.dataLifecycleBuilder()
150137
.downsampling(
151138
List.of(
152-
new DataStreamLifecycle.DownsamplingRound(
153-
TimeValue.timeValueDays(10),
154-
new DownsampleConfig(new DateHistogramInterval("2h"))
155-
),
156-
new DataStreamLifecycle.DownsamplingRound(
157-
TimeValue.timeValueDays(30),
158-
new DownsampleConfig(new DateHistogramInterval("3h"))
159-
)
139+
new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")),
140+
new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h"))
160141
)
161142
)
162143
.buildTemplate()
@@ -180,7 +161,7 @@ public void testInvalidDownsamplingConfiguration() {
180161
.map(
181162
i -> new DataStreamLifecycle.DownsamplingRound(
182163
TimeValue.timeValueDays(i),
183-
new DownsampleConfig(new DateHistogramInterval(i + "h"))
164+
new DateHistogramInterval(i + "h")
184165
)
185166
)
186167
.toList()
@@ -195,12 +176,7 @@ public void testInvalidDownsamplingConfiguration() {
195176
IllegalArgumentException.class,
196177
() -> DataStreamLifecycle.dataLifecycleBuilder()
197178
.downsampling(
198-
List.of(
199-
new DataStreamLifecycle.DownsamplingRound(
200-
TimeValue.timeValueDays(10),
201-
new DownsampleConfig(new DateHistogramInterval("2m"))
202-
)
203-
)
179+
List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m")))
204180
)
205181
.buildTemplate()
206182
);

0 commit comments

Comments
 (0)