Skip to content

Commit 74e4bc3

Browse files
author
jaserud
authored
Merge pull request #43 from overture-stack/rc/2.4.0
update to 2.4.0
2 parents 442a6f7 + b67d6b7 commit 74e4bc3

File tree

6 files changed

+134
-23
lines changed

6 files changed

+134
-23
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323

2424
<groupId>bio.overture</groupId>
2525
<artifactId>rollcall</artifactId>
26-
<version>2.3.0</version>
26+
<version>2.4.0</version>
2727
<packaging>jar</packaging>
2828

2929
<name>rollcall</name>

‎src/main/java/bio/overture/rollcall/config/RollcallConfig.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import lombok.Data;
2323
import lombok.NoArgsConstructor;
2424
import lombok.NonNull;
25+
import lombok.RequiredArgsConstructor;
2526
import org.springframework.boot.context.properties.ConfigurationProperties;
2627
import org.springframework.context.annotation.Configuration;
2728

@@ -50,10 +51,12 @@ public void setAliases(@NonNull List<ConfiguredAlias> aliases) {
5051
@Data
5152
@NoArgsConstructor
5253
@AllArgsConstructor
54+
@RequiredArgsConstructor
5355
public static class ConfiguredAlias {
54-
private String alias;
55-
private String entity;
56-
private String type;
56+
@NonNull private String alias;
57+
@NonNull private String entity;
58+
@NonNull private String type;
59+
private int releaseRotation = -1;
5760
}
5861

5962
}

‎src/main/java/bio/overture/rollcall/repository/IndexRepository.java

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import lombok.val;
2424
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
2525
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
26+
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
2627
import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
2728
import org.elasticsearch.action.admin.indices.shrink.ResizeType;
2829
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
@@ -37,8 +38,8 @@
3738
import org.springframework.beans.factory.annotation.Autowired;
3839
import org.springframework.stereotype.Repository;
3940

40-
import java.util.Collections;
41-
import java.util.List;
41+
import java.util.*;
42+
import java.util.stream.Collectors;
4243

4344
import static java.util.stream.Collectors.toUnmodifiableList;
4445
import static org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions.*;
@@ -132,4 +133,29 @@ public boolean cloneIndex(@NonNull String indexToClone, @NonNull String newIndex
132133
req.getTargetIndexRequest().settings(settings, XContentType.JSON);
133134
return client.indices().clone(req, RequestOptions.DEFAULT).isAcknowledged();
134135
}
136+
137+
@SneakyThrows
138+
public boolean deleteIndices(@NonNull String... indices) {
139+
if (indices.length == 0) {
140+
return true;
141+
}
142+
val request = new DeleteIndexRequest();
143+
request.indices(indices);
144+
return client.indices().delete(request, RequestOptions.DEFAULT).isAcknowledged();
145+
}
146+
147+
@SneakyThrows
148+
public Map<String, Date> getIndicesMappedToCreationDate(String... indices) {
149+
val response = client.indices().get(new GetIndexRequest(indices).indicesOptions(IndicesOptions.lenientExpand()), RequestOptions.DEFAULT);
150+
151+
val indicesSettings = response.getSettings();
152+
153+
return indicesSettings.entrySet().stream()
154+
.collect(
155+
Collectors.toMap(
156+
Map.Entry::getKey, // key is index name
157+
e -> new Date(e.getValue().getAsLong("index.creation_date", null))
158+
)
159+
);
160+
}
135161
}

‎src/main/java/bio/overture/rollcall/service/AliasService.java

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@
3535

3636
import java.util.ArrayList;
3737
import java.util.Arrays;
38+
import java.util.Collections;
3839
import java.util.List;
40+
import java.util.Map;
3941

4042
import static java.util.stream.Collectors.toList;
4143

@@ -100,9 +102,13 @@ public boolean release(@NonNull AliasRequest aliasRequest) {
100102
}
101103

102104
val indicesToRemoveFromAlias = getIndicesToRemoveFromAlias(aliasRequest);
105+
val indicesToDelete = getIndicesToDelete(candidates, shards, indicesToRelease);
103106

104-
repository.makeIndicesReadOnly(indicesToRelease);
105-
return repository.updateIndicesAliases(alias, indicesToRelease, indicesToRemoveFromAlias);
107+
val successfullyMadeReadOnly = repository.makeIndicesReadOnly(indicesToRelease);
108+
val successfullyUpdatedAliases = repository.updateIndicesAliases(alias, indicesToRelease, indicesToRemoveFromAlias);
109+
val successfullyDeletedOldIndices = repository.deleteIndices(indicesToDelete.toArray(String[]::new));
110+
111+
return successfullyMadeReadOnly && successfullyUpdatedAliases && successfullyDeletedOldIndices;
106112
}
107113

108114
public List<String> getIndicesToRemoveFromAlias(@NonNull AliasRequest aliasRequest) {
@@ -113,14 +119,12 @@ public List<String> getIndicesToRemoveFromAlias(@NonNull AliasRequest aliasReque
113119

114120
// TODO: UNIT TEST
115121
val existing = getIndicesWithAlias(alias);
116-
val indices = candidates.getIndices().stream()
122+
return candidates.getIndices().stream()
117123
.filter(i -> shards.stream().
118124
anyMatch(shard -> shard.matches(i.getShardPrefix(), i.getShard())))
119125
.filter(i -> existing.contains(i.getIndexName()))
120126
.map(ResolvedIndex::getIndexName)
121127
.collect(toList());
122-
123-
return indices;
124128
}
125129

126130
public boolean remove(@NonNull AliasRequest aliasRequest) {
@@ -129,11 +133,6 @@ public boolean remove(@NonNull AliasRequest aliasRequest) {
129133
return repository.removeAlias(alias, indices);
130134
}
131135

132-
public boolean removeAliasFromAllIndices(@NonNull String alias) {
133-
List<String> existing = getIndicesWithAlias(alias);
134-
return repository.removeAlias(alias, existing);
135-
}
136-
137136
private static List<Shard> getShardsFromRequest(AliasRequest aliasRequest) {
138137
return aliasRequest.getShards().stream()
139138
.map(String::toLowerCase)
@@ -174,6 +173,32 @@ private List<String> getIndicesForRelease(AliasCandidates candidates, String[] r
174173
.collect(toList());
175174
}
176175

176+
private List<String> getIndicesToDelete(AliasCandidates candidates, List<Shard> shards, List<String> releaseIndicesToIgnore) {
177+
val numOfRecentIndicesToKeep = candidates.getAlias().getReleaseRotation();
178+
if ( numOfRecentIndicesToKeep < 0) {
179+
return Collections.emptyList();
180+
}
181+
182+
val relevantIndices = candidates.getIndices().stream()
183+
.filter(i -> shards.stream().anyMatch(shard -> shard.matches(i.getShardPrefix(), i.getShard())))
184+
.map(ResolvedIndex::getIndexName)
185+
.filter(i -> !releaseIndicesToIgnore.contains(i))
186+
.collect(toList());
187+
188+
val sortedByDate = getIndicesSortedByCreationDate(relevantIndices);
189+
val deleteUpToIndex = Math.max(sortedByDate.size() - numOfRecentIndicesToKeep, 0);
190+
191+
return sortedByDate.subList(0, deleteUpToIndex);
192+
}
193+
194+
private List<String> getIndicesSortedByCreationDate(List<String> indexNames) {
195+
val indexMappedToCreationDate = repository.getIndicesMappedToCreationDate(indexNames.toArray(String[]::new));
196+
return indexMappedToCreationDate.entrySet().stream()
197+
.sorted(Map.Entry.comparingByValue()) // sort by map value, which is date
198+
.map(Map.Entry::getKey)
199+
.collect(toList());
200+
}
201+
177202
@Data
178203
public class AliasCandidates {
179204
private final ConfiguredAlias alias;

‎src/main/resources/application.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ rollcall:
1919
alias: file_centric
2020
entity: file
2121
type: centric
22+
releaseRotation: 3
2223
-
2324
alias: participant_centric
2425
entity: participant

‎src/test/java/bio/overture/rollcall/service/AliasServiceTest.java

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,13 @@ public class AliasServiceTest {
6565
@Before
6666
@SneakyThrows
6767
public void setUp() {
68-
client = new RestHighLevelClient( RestClient.builder(new HttpHost(InetAddress.getByName(esContainer.getIpAddress()), 10200)));
68+
client = new RestHighLevelClient( RestClient.builder(new HttpHost(InetAddress.getByName(esContainer.getContainerIpAddress()), 10200)));
6969
repository = new IndexRepository(client);
7070

71-
val config = new RollcallConfig(Lists.list(new RollcallConfig.ConfiguredAlias("file_centric", "file", "centric")));
71+
val config = new RollcallConfig(Lists.list(
72+
new RollcallConfig.ConfiguredAlias("file_centric", "file", "centric", 1),
73+
new RollcallConfig.ConfiguredAlias("participant_centric", "participant", "centric")
74+
));
7275
service = new AliasService(config, repository);
7376

7477
client.indices().create(new CreateIndexRequest(INDEX1), RequestOptions.DEFAULT);
@@ -80,16 +83,14 @@ public void setUp() {
8083
@After
8184
@SneakyThrows
8285
public void tearDown() {
83-
client.indices().delete(new DeleteIndexRequest(INDEX1), RequestOptions.DEFAULT);
84-
client.indices().delete(new DeleteIndexRequest(INDEX2), RequestOptions.DEFAULT);
85-
client.indices().delete(new DeleteIndexRequest(INDEX3), RequestOptions.DEFAULT);
86-
client.indices().delete(new DeleteIndexRequest("badindex"), RequestOptions.DEFAULT);
86+
// delete all indices
87+
client.indices().delete(new DeleteIndexRequest("*"), RequestOptions.DEFAULT);
8788
}
8889

8990
@Test
9091
public void getConfiguredTest() {
9192
val configured = service.getConfigured();
92-
assertThat(configured).hasSize(1);
93+
assertThat(configured).hasSize(2);
9394
}
9495

9596
@Test
@@ -125,4 +126,59 @@ public void testReleaseNonDestructiveFailurePreFlight() {
125126
assertThat(state2.get(INDEX2).get(0).alias()).isEqualTo("file_centric");
126127
}
127128

129+
@Test
130+
@SneakyThrows
131+
public void testReleaseAndDeleteOldIndices() {
132+
// release foobar2
133+
val request1 = new AliasRequest("file_centric", "RE_foobar2", Lists.list( "sd_preasa7s"));
134+
service.release(request1);
135+
136+
// verify aliases assigned to indices
137+
val state1 = repository.getAliasState();
138+
assertThat(state1.get(INDEX2).isEmpty()).isTrue();
139+
assertThat(state1.get(INDEX3).get(0).alias()).isEqualTo("file_centric");
140+
141+
// add new index and assert current indices
142+
final String INDEX4 = "file_centric_sd_preasa7s_re_foobar3";
143+
client.indices().create(new CreateIndexRequest(INDEX4), RequestOptions.DEFAULT);
144+
val indicesBeforeRelease = repository.getIndices();
145+
assertThat(indicesBeforeRelease).containsExactlyInAnyOrder(INDEX1, INDEX2, INDEX3, "badindex", INDEX4);
146+
147+
// release foobar3
148+
val request2 = new AliasRequest("file_centric", "RE_foobar3", Lists.list("SD_preasa7s"));
149+
service.release(request2);
150+
151+
// verify aliases assigned to indices
152+
val state2 = repository.getAliasState();
153+
assertThat(state2.get(INDEX2)).isNull(); // sd_preasa7s foobar3 release deleted old foobar1 since keeping `1` latestNonReleasedIndex
154+
assertThat(state2.get(INDEX3).isEmpty()).isTrue(); // sd_preasa7s unreleased index foobar2 is kept
155+
assertThat(state2.get(INDEX4).get(0).alias()).isEqualTo("file_centric"); // new released index
156+
157+
// assert current indices
158+
val indicesAfterRelease = repository.getIndices();
159+
assertThat(indicesAfterRelease).containsExactlyInAnyOrder(INDEX1, INDEX3, "badindex", INDEX4);
160+
}
161+
162+
@Test
163+
@SneakyThrows
164+
public void testIndicesNotDeletedIfNotConfigured() {
165+
final String INDEXA = "participant_centric_sd_preasa7s_re_foobar1";
166+
final String INDEXB = "participant_centric_sd_preasa7s_re_foobar2";
167+
final String INDEXC = "participant_centric_sd_preasa7s_re_foobar3";
168+
169+
client.indices().create(new CreateIndexRequest(INDEXA), RequestOptions.DEFAULT);
170+
client.indices().create(new CreateIndexRequest(INDEXB), RequestOptions.DEFAULT);
171+
client.indices().create(new CreateIndexRequest(INDEXC), RequestOptions.DEFAULT);
172+
173+
val indicesBeforeRelease = repository.getIndices();
174+
assertThat(indicesBeforeRelease).containsExactlyInAnyOrder(INDEX1, INDEX2, INDEX3, "badindex", INDEXA, INDEXB, INDEXC);
175+
176+
// release foobar3 in participant_centric, shouldn't delete any participant_centric index since it was not configured
177+
val request = new AliasRequest("participant_centric", "RE_foobar3", Lists.list( "sd_preasa7s"));
178+
service.release(request);
179+
180+
// assert current indices
181+
val indicesAfterRelease = repository.getIndices();
182+
assertThat(indicesAfterRelease).containsExactlyInAnyOrder(indicesBeforeRelease);
183+
}
128184
}

0 commit comments

Comments
 (0)