Skip to content

Commit e02a9fd

Browse files
rdettai-skrdettai-sk
authored
Support ignore_unavailable query param (quickwit-oss#5971)
* Support ignore_unavailable query param * Add unit tests and improve some wording
1 parent b7556f4 commit e02a9fd

9 files changed

Lines changed: 288 additions & 16 deletions

File tree

‎quickwit/quickwit-metastore/src/tests/index.rs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,9 +677,14 @@ pub async fn test_metastore_list_indexes<MetastoreToTest: MetastoreServiceExt +
677677
let index_uri_4 = format!("ram:///indexes/{index_id_4}");
678678
let index_config_4 = IndexConfig::for_test(&index_id_4, &index_uri_4);
679679

680+
let index_id_5 = format!("my-exact-index-{index_id_fragment}-5");
681+
let index_uri_5 = format!("ram:///indexes/{index_id_5}");
682+
let index_config_5 = IndexConfig::for_test(&index_id_5, &index_uri_5);
683+
680684
let index_id_patterns = vec![
681685
format!("prefix-*-{index_id_fragment}-suffix-*"),
682686
format!("prefix*{index_id_fragment}*suffix-*"),
687+
format!("my-exact-index-{index_id_fragment}-5"),
683688
];
684689
let indexes_count = metastore
685690
.list_indexes_metadata(ListIndexesMetadataRequest { index_id_patterns })
@@ -715,8 +720,17 @@ pub async fn test_metastore_list_indexes<MetastoreToTest: MetastoreServiceExt +
715720
.unwrap()
716721
.index_uid()
717722
.clone();
723+
let index_uid_5 = metastore
724+
.create_index(CreateIndexRequest::try_from_index_config(&index_config_5).unwrap())
725+
.await
726+
.unwrap()
727+
.index_uid()
728+
.clone();
718729

719-
let index_id_patterns = vec![format!("prefix-*-{index_id_fragment}-suffix-*")];
730+
let index_id_patterns = vec![
731+
format!("prefix-*-{index_id_fragment}-suffix-*"),
732+
format!("my-exact-index-{index_id_fragment}-5"),
733+
];
720734
let indexes_count = metastore
721735
.list_indexes_metadata(ListIndexesMetadataRequest { index_id_patterns })
722736
.await
@@ -725,12 +739,13 @@ pub async fn test_metastore_list_indexes<MetastoreToTest: MetastoreServiceExt +
725739
.await
726740
.unwrap()
727741
.len();
728-
assert_eq!(indexes_count, 2);
742+
assert_eq!(indexes_count, 3);
729743

730744
cleanup_index(&mut metastore, index_uid_1).await;
731745
cleanup_index(&mut metastore, index_uid_2).await;
732746
cleanup_index(&mut metastore, index_uid_3).await;
733747
cleanup_index(&mut metastore, index_uid_4).await;
748+
cleanup_index(&mut metastore, index_uid_5).await;
734749
}
735750

736751
pub async fn test_metastore_delete_index<

‎quickwit/quickwit-proto/protos/quickwit/search.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,10 @@ message SearchRequest {
243243
optional PartialHit search_after = 16;
244244

245245
CountHits count_hits = 17;
246+
247+
// When an exact index ID is provided (not a pattern), the query fails only if
248+
// that index is not found and this parameter is set to `false`.
249+
bool ignore_missing_indexes = 18;
246250
}
247251

248252
enum CountHits {

‎quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎quickwit/quickwit-search/src/root.rs‎

Lines changed: 172 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ fn simplify_search_request_for_scroll_api(req: &SearchRequest) -> crate::Result<
366366
// request is simplified after initial query, and we cache the hit count, so we don't need
367367
// to recompute it afterward.
368368
count_hits: quickwit_proto::search::CountHits::Underestimate as i32,
369+
ignore_missing_indexes: req.ignore_missing_indexes,
369370
})
370371
}
371372

@@ -1156,7 +1157,12 @@ async fn plan_splits_for_root_search(
11561157
.deserialize_indexes_metadata()
11571158
.await?;
11581159

1159-
check_all_index_metadata_found(&indexes_metadata[..], &search_request.index_id_patterns[..])?;
1160+
if !search_request.ignore_missing_indexes {
1161+
check_all_index_metadata_found(
1162+
&indexes_metadata[..],
1163+
&search_request.index_id_patterns[..],
1164+
)?;
1165+
}
11601166

11611167
if indexes_metadata.is_empty() {
11621168
return Ok((Vec::new(), HashMap::default()));
@@ -1243,7 +1249,12 @@ pub async fn search_plan(
12431249
.deserialize_indexes_metadata()
12441250
.await?;
12451251

1246-
check_all_index_metadata_found(&indexes_metadata[..], &search_request.index_id_patterns[..])?;
1252+
if !search_request.ignore_missing_indexes {
1253+
check_all_index_metadata_found(
1254+
&indexes_metadata[..],
1255+
&search_request.index_id_patterns[..],
1256+
)?;
1257+
}
12471258
if indexes_metadata.is_empty() {
12481259
return Ok(SearchPlanResponse {
12491260
result: serde_json::to_string(&SearchPlanResponseRest {
@@ -3240,6 +3251,102 @@ mod tests {
32403251
Ok(())
32413252
}
32423253

3254+
#[tokio::test]
3255+
async fn test_root_search_missing_index() -> anyhow::Result<()> {
3256+
let mut mock_metastore = MockMetastoreService::new();
3257+
let index_metadata = IndexMetadata::for_test("test-index1", "ram:///test-index");
3258+
let index_uid = index_metadata.index_uid.clone();
3259+
mock_metastore
3260+
.expect_list_indexes_metadata()
3261+
.returning(move |_index_ids_query| {
3262+
Ok(ListIndexesMetadataResponse::for_test(vec![
3263+
index_metadata.clone(),
3264+
]))
3265+
});
3266+
mock_metastore
3267+
.expect_list_splits()
3268+
.returning(move |_list_splits_request| {
3269+
let splits = vec![
3270+
MockSplitBuilder::new("split1")
3271+
.with_index_uid(&index_uid)
3272+
.build(),
3273+
];
3274+
let splits_response = ListSplitsResponse::try_from_splits(splits).unwrap();
3275+
Ok(ServiceStream::from(vec![Ok(splits_response)]))
3276+
});
3277+
let mock_metastore_client = MetastoreServiceClient::from_mock(mock_metastore);
3278+
let mut mock_search_service = MockSearchService::new();
3279+
mock_search_service.expect_leaf_search().returning(
3280+
|_leaf_search_req: quickwit_proto::search::LeafSearchRequest| {
3281+
Ok(quickwit_proto::search::LeafSearchResponse {
3282+
num_hits: 3,
3283+
partial_hits: vec![
3284+
mock_partial_hit("split1", 3, 1),
3285+
mock_partial_hit("split1", 2, 2),
3286+
mock_partial_hit("split1", 1, 3),
3287+
],
3288+
failed_splits: Vec::new(),
3289+
num_attempted_splits: 1,
3290+
..Default::default()
3291+
})
3292+
},
3293+
);
3294+
mock_search_service.expect_fetch_docs().returning(
3295+
|fetch_docs_req: quickwit_proto::search::FetchDocsRequest| {
3296+
Ok(quickwit_proto::search::FetchDocsResponse {
3297+
hits: get_doc_for_fetch_req(fetch_docs_req),
3298+
})
3299+
},
3300+
);
3301+
let searcher_pool = searcher_pool_for_test([("127.0.0.1:1001", mock_search_service)]);
3302+
let search_job_placer = SearchJobPlacer::new(searcher_pool);
3303+
let cluster_client = ClusterClient::new(search_job_placer.clone());
3304+
3305+
let searcher_context = SearcherContext::for_test();
3306+
3307+
// search with ignore_missing_indexes=true succeeds
3308+
let search_request = quickwit_proto::search::SearchRequest {
3309+
index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()],
3310+
query_ast: qast_json_helper("test", &["body"]),
3311+
max_hits: 10,
3312+
ignore_missing_indexes: true,
3313+
..Default::default()
3314+
};
3315+
let search_response = root_search(
3316+
&searcher_context,
3317+
search_request,
3318+
mock_metastore_client.clone(),
3319+
&cluster_client,
3320+
)
3321+
.await
3322+
.unwrap();
3323+
assert_eq!(search_response.num_hits, 3);
3324+
assert_eq!(search_response.hits.len(), 3);
3325+
3326+
// search with ignore_missing_indexes=false fails
3327+
let search_request = quickwit_proto::search::SearchRequest {
3328+
index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()],
3329+
query_ast: qast_json_helper("test", &["body"]),
3330+
max_hits: 10,
3331+
ignore_missing_indexes: false,
3332+
..Default::default()
3333+
};
3334+
let search_error = root_search(
3335+
&searcher_context,
3336+
search_request,
3337+
mock_metastore_client,
3338+
&cluster_client,
3339+
)
3340+
.await
3341+
.unwrap_err();
3342+
if let SearchError::IndexesNotFound { index_ids } = search_error {
3343+
assert_eq!(index_ids, vec!["test-index2".to_string()]);
3344+
} else {
3345+
panic!("unexpected error type: {search_error}");
3346+
}
3347+
Ok(())
3348+
}
3349+
32433350
#[tokio::test]
32443351
async fn test_root_search_multiple_splits_retry_on_other_node() -> anyhow::Result<()> {
32453352
let search_request = quickwit_proto::search::SearchRequest {
@@ -4112,6 +4219,69 @@ mod tests {
41124219
Ok(())
41134220
}
41144221

4222+
#[tokio::test]
4223+
async fn test_search_plan_missing_index() -> anyhow::Result<()> {
4224+
let mut mock_metastore = MockMetastoreService::new();
4225+
let index_metadata = IndexMetadata::for_test("test-index1", "ram:///test-index");
4226+
let index_uid = index_metadata.index_uid.clone();
4227+
mock_metastore
4228+
.expect_list_indexes_metadata()
4229+
.returning(move |_index_ids_query| {
4230+
Ok(ListIndexesMetadataResponse::for_test(vec![
4231+
index_metadata.clone(),
4232+
]))
4233+
});
4234+
mock_metastore
4235+
.expect_list_splits()
4236+
.returning(move |_filter| {
4237+
let splits = vec![
4238+
MockSplitBuilder::new("split1")
4239+
.with_index_uid(&index_uid)
4240+
.build(),
4241+
MockSplitBuilder::new("split2")
4242+
.with_index_uid(&index_uid)
4243+
.build(),
4244+
];
4245+
let splits_response = ListSplitsResponse::try_from_splits(splits).unwrap();
4246+
Ok(ServiceStream::from(vec![Ok(splits_response)]))
4247+
});
4248+
let mock_metastore_service = MetastoreServiceClient::from_mock(mock_metastore);
4249+
4250+
// plan with ignore_missing_indexes=true succeeds
4251+
search_plan(
4252+
quickwit_proto::search::SearchRequest {
4253+
index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()],
4254+
query_ast: qast_json_helper("test-query", &["body"]),
4255+
max_hits: 10,
4256+
ignore_missing_indexes: true,
4257+
..Default::default()
4258+
},
4259+
mock_metastore_service.clone(),
4260+
)
4261+
.await
4262+
.unwrap();
4263+
4264+
// plan with ignore_missing_indexes=false fails
4265+
let search_error = search_plan(
4266+
quickwit_proto::search::SearchRequest {
4267+
index_id_patterns: vec!["test-index1".to_string(), "test-index2".to_string()],
4268+
query_ast: qast_json_helper("test-query", &["body"]),
4269+
max_hits: 10,
4270+
ignore_missing_indexes: false,
4271+
..Default::default()
4272+
},
4273+
mock_metastore_service.clone(),
4274+
)
4275+
.await
4276+
.unwrap_err();
4277+
if let SearchError::IndexesNotFound { index_ids } = search_error {
4278+
assert_eq!(index_ids, vec!["test-index2".to_string()]);
4279+
} else {
4280+
panic!("unexpected error type: {search_error}");
4281+
}
4282+
Ok(())
4283+
}
4284+
41154285
#[test]
41164286
fn test_extract_timestamp_range_from_ast() {
41174287
use std::ops::Bound;

‎quickwit/quickwit-serve/src/elasticsearch_api/model/multi_search.rs‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ use crate::simple_list::{from_simple_list, to_simple_list};
2525

2626
// Multi search doc: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html
2727

28+
#[serde_as]
2829
#[serde_with::skip_serializing_none]
2930
#[derive(Default, Debug, Serialize, Deserialize)]
3031
#[serde(deny_unknown_fields)]
@@ -50,6 +51,10 @@ pub struct MultiSearchQueryParams {
5051
pub ignore_throttled: Option<bool>,
5152
#[serde(default)]
5253
pub ignore_unavailable: Option<bool>,
54+
/// List of indexes to search.
55+
#[serde_as(deserialize_as = "OneOrMany<_, PreferMany>")]
56+
#[serde(default, rename = "index")]
57+
pub indexes: Vec<String>,
5358
#[serde(default)]
5459
pub max_concurrent_searches: Option<u64>,
5560
#[serde(default)]
@@ -90,8 +95,8 @@ pub struct MultiSearchHeader {
9095
#[serde(default)]
9196
pub ignore_unavailable: Option<bool>,
9297
#[serde_as(deserialize_as = "OneOrMany<_, PreferMany>")]
93-
#[serde(default)]
94-
pub index: Vec<String>,
98+
#[serde(default, rename = "index")]
99+
pub indexes: Vec<String>,
95100
#[serde(default)]
96101
pub preference: Option<String>,
97102
#[serde(default)]
@@ -100,6 +105,26 @@ pub struct MultiSearchHeader {
100105
pub routing: Option<Vec<String>>,
101106
}
102107

108+
impl MultiSearchHeader {
109+
pub fn apply_query_param_defaults(&mut self, defaults: &MultiSearchQueryParams) {
110+
if self.allow_no_indices.is_none() {
111+
self.allow_no_indices = defaults.allow_no_indices;
112+
}
113+
if self.expand_wildcards.is_none() {
114+
self.expand_wildcards = defaults.expand_wildcards.clone();
115+
}
116+
if self.ignore_unavailable.is_none() {
117+
self.ignore_unavailable = defaults.ignore_unavailable;
118+
}
119+
if self.indexes.is_empty() {
120+
self.indexes = defaults.indexes.clone();
121+
}
122+
if self.routing.is_none() {
123+
self.routing = defaults.routing.clone();
124+
}
125+
}
126+
}
127+
103128
#[derive(Serialize)]
104129
pub struct MultiSearchResponse {
105130
pub responses: Vec<MultiSearchSingleResponse>,

‎quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,7 @@ fn build_request_for_es_api(
358358

359359
let max_hits = search_params.size.or(search_body.size).unwrap_or(10);
360360
let start_offset = search_params.from.or(search_body.from).unwrap_or(0);
361+
let ignore_missing_indexes = search_params.ignore_unavailable.unwrap_or(false);
361362
let count_hits = match search_params
362363
.track_total_hits
363364
.or(search_body.track_total_hits)
@@ -410,6 +411,7 @@ fn build_request_for_es_api(
410411
scroll_ttl_secs,
411412
search_after,
412413
count_hits,
414+
ignore_missing_indexes,
413415
},
414416
has_doc_id_field,
415417
))
@@ -814,26 +816,28 @@ async fn es_compat_index_multi_search(
814816
let mut payload_lines = str_lines(str_payload);
815817

816818
while let Some(line) = payload_lines.next() {
817-
let request_header = serde_json::from_str::<MultiSearchHeader>(line).map_err(|err| {
818-
SearchError::InvalidArgument(format!(
819-
"failed to parse request header `{}...`: {}",
820-
truncate_str(line, 20),
821-
err
822-
))
823-
})?;
824-
if request_header.index.is_empty() {
819+
let mut request_header =
820+
serde_json::from_str::<MultiSearchHeader>(line).map_err(|err| {
821+
SearchError::InvalidArgument(format!(
822+
"failed to parse request header `{}...`: {}",
823+
truncate_str(line, 20),
824+
err
825+
))
826+
})?;
827+
request_header.apply_query_param_defaults(&multi_search_params);
828+
if request_header.indexes.is_empty() {
825829
return Err(ElasticsearchError::from(SearchError::InvalidArgument(
826830
"`_msearch` request header must define at least one index".to_string(),
827831
)));
828832
}
829-
for index in &request_header.index {
833+
for index in &request_header.indexes {
830834
validate_index_id_pattern(index, true).map_err(|err| {
831835
SearchError::InvalidArgument(format!(
832836
"request header contains an invalid index: {err}"
833837
))
834838
})?;
835839
}
836-
let index_ids_patterns = request_header.index.clone();
840+
let index_ids_patterns = request_header.indexes.clone();
837841
let search_body = payload_lines
838842
.next()
839843
.ok_or_else(|| {

‎quickwit/quickwit-serve/src/search_api/rest_handler.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ pub fn search_request_from_api_request(
264264
scroll_ttl_secs: None,
265265
search_after: None,
266266
count_hits: search_request.count_all.into(),
267+
ignore_missing_indexes: false,
267268
};
268269
Ok(search_request)
269270
}

0 commit comments

Comments
 (0)