Use version 0.9 for versioning object

guilload · guilload · commit b750d771e920 · 2025-02-04T17:33:36.000-05:00
diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md
@@ -257,7 +257,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes --data @index_config.json -H "C
 
 ```json title="index_config.json
 {
-    "version": "0.8",
+    "version": "0.9",
     "index_id": "hdfs-logs",
     "doc_mapping": {
         "field_mappings": [
@@ -355,7 +355,7 @@ curl -XPUT http://localhost:7280/api/v1/indexes/hdfs-logs --data @updated_index_
 
 ```json title="updated_index_update.json
 {
-    "version": "0.8",
+    "version": "0.9",
     "index_id": "hdfs-logs",
     "doc_mapping": {
         "field_mappings": [
@@ -513,7 +513,7 @@ GET /api/v1/indexes/stackoverflow/splits?offset=0&limit=10
       "split_state": "Published",
       "update_timestamp": 1695642901,
       "publish_timestamp": 1695642901,
-      "version": "0.7",
+      "version": "0.9",
       "split_id": "01HB632HD8W6WHNM7CZFH3KG1X",
       "index_uid": "stackoverflow:01HB6321TDT3SP58D4EZP14KSX",
       "partition_id": 0,
@@ -617,7 +617,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes/my-index/sources --data @source
 
 ```json title="source_config.json
 {
-    "version": "0.8",
+    "version": "0.9",
     "source_id": "kafka-source",
     "source_type": "kafka",
     "params": {
@@ -664,7 +664,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes/my-index/sources --data @source
 
 ```json title="source_config.json
 {
-    "version": "0.8",
+    "version": "0.9",
     "source_id": "kafka-source",
     "source_type": "kafka",
     "params": {
diff --git a/quickwit/quickwit-cli/src/cli_doc_ext.toml b/quickwit/quickwit-cli/src/cli_doc_ext.toml
@@ -232,7 +232,7 @@ quickwit run --service metastore --config=./config/quickwit.yaml
 # Open a new terminal and run:
 cat << EOF > wikipedia-kafka-source.json
 {
-  "version": "0.7",
+  "version": "0.9",
   "source_id": "kafka-source",
   "source_type": "kafka",
   "params": {
diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs-create-config.yaml b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs-create-config.yaml
@@ -1,4 +1,4 @@
-version: 0.8
+version: 0.9
 
 doc_mapping:
   field_mappings:
diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.json b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.json
@@ -1,6 +1,6 @@
 # Comments are supported.
 {
-    "version": "0.7",
+    "version": "0.9",
     "index_id": "hdfs-logs",
     "index_uri": "s3://quickwit-indexes/hdfs-logs",
     "doc_mapping": {
diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml
@@ -1,4 +1,4 @@
-version = "0.7"
+version = "0.9"
 index_id = "hdfs-logs"
 index_uri = "s3://quickwit-indexes/hdfs-logs"
 
diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.yaml b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.yaml
@@ -1,4 +1,4 @@
-version: 0.8
+version: 0.9
 index_id: hdfs-logs
 index_uri: s3://quickwit-indexes/hdfs-logs
 
diff --git a/quickwit/quickwit-config/resources/tests/index_config/minimal-hdfs-logs.yaml b/quickwit/quickwit-config/resources/tests/index_config/minimal-hdfs-logs.yaml
@@ -1,4 +1,4 @@
-version: 0.8
+version: 0.9
 
 index_id: hdfs-logs
 index_uri: s3://quickwit-indexes/hdfs-logs
diff --git a/quickwit/quickwit-config/resources/tests/index_config/partial-hdfs-logs.yaml b/quickwit/quickwit-config/resources/tests/index_config/partial-hdfs-logs.yaml
@@ -1,4 +1,4 @@
-version: 0.8
+version: 0.9
 
 index_id: hdfs-logs
 index_uri: s3://quickwit-indexes/hdfs-logs
diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.json b/quickwit/quickwit-config/resources/tests/node_config/quickwit.json
@@ -1,6 +1,6 @@
 # Comments are supported.
 {
-    "version": "0.7",
+    "version": "0.9",
     "cluster_id": "quickwit-cluster",
     "node_id": "my-unique-node-id",
     "enabled_services": [
diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml
@@ -1,4 +1,4 @@
-version = "0.7"
+version = "0.9"
 
 cluster_id = "quickwit-cluster"
 node_id = "my-unique-node-id"
diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.wrongkey.yaml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.wrongkey.yaml
@@ -1,4 +1,4 @@
-version: 0.8
+version: 0.9
 searcher:
   fast_field_cache_capacity: 10G
   # Typo here. It is supposed to be searches.
diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml
@@ -1,4 +1,4 @@
-version: 0.8
+version: 0.9
 
 cluster_id: quickwit-cluster
 node_id: my-unique-node-id
diff --git a/quickwit/quickwit-config/resources/tests/source_config/ingest-api-source.json b/quickwit/quickwit-config/resources/tests/source_config/ingest-api-source.json
@@ -1,5 +1,5 @@
 {
-  "version": "0.7",
+  "version": "0.9",
   "source_id": "_ingest-api-source",
   "enabled": true,
   "source_type": "ingest-api",
diff --git a/quickwit/quickwit-config/resources/tests/source_config/kafka-source.json b/quickwit/quickwit-config/resources/tests/source_config/kafka-source.json
@@ -1,7 +1,7 @@
 {
-    "version": "0.7",
+    "version": "0.9",
     "source_id": "hdfs-logs-kafka-source",
-    "desired_num_pipelines": 2,
+    "num_pipelines": 2,
     "source_type": "kafka",
     "params": {
         "topic": "cloudera-cluster-logs",
diff --git a/quickwit/quickwit-config/resources/tests/source_config/kinesis-source.yaml b/quickwit/quickwit-config/resources/tests/source_config/kinesis-source.yaml
@@ -1,4 +1,4 @@
-version: 0.8
+version: 0.9
 source_id: hdfs-logs-kinesis-source
 source_type: kinesis
 params:
diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs
@@ -74,7 +74,7 @@ pub use crate::node_config::{
     IndexerConfig, IngestApiConfig, JaegerConfig, NodeConfig, SearcherConfig, SplitCacheLimits,
     StorageTimeoutPolicy, DEFAULT_QW_CONFIG_PATH,
 };
-use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig};
+use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_9, VersionedSourceConfig};
 pub use crate::storage_config::{
     AzureStorageConfig, FileStorageConfig, GoogleCloudStorageConfig, RamStorageConfig,
     S3StorageConfig, StorageBackend, StorageBackendFlavor, StorageConfig, StorageConfigs,
@@ -104,7 +104,7 @@ pub fn disable_ingest_v1() -> bool {
     DocMapping,
     VersionedSourceConfig,
     SourceConfigV0_7,
-    SourceConfigV0_8,
+    SourceConfigV0_9,
     VersionedIndexConfig,
     IndexConfigV0_8,
     VersionedIndexTemplate,
diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs
@@ -145,16 +145,16 @@ pub async fn load_node_config_with_env(
 #[derive(Debug, Deserialize)]
 #[serde(tag = "version")]
 enum VersionedNodeConfig {
-    #[serde(rename = "0.8")]
+    #[serde(rename = "0.9")]
     // Retro compatibility.
-    #[serde(alias = "0.7")]
-    V0_8(NodeConfigBuilder),
+    #[serde(alias = "0.7", alias = "0.8")]
+    V0_9(NodeConfigBuilder),
 }
 
 impl From<VersionedNodeConfig> for NodeConfigBuilder {
     fn from(versioned_node_config: VersionedNodeConfig) -> Self {
         match versioned_node_config {
-            VersionedNodeConfig::V0_8(node_config_builder) => node_config_builder,
+            VersionedNodeConfig::V0_9(node_config_builder) => node_config_builder,
         }
     }
 }
diff --git a/quickwit/quickwit-config/src/source_config/mod.rs b/quickwit/quickwit-config/src/source_config/mod.rs
@@ -861,41 +861,24 @@ mod tests {
         {
             let content = r#"
             {
-                "version": "0.7",
+                "version": "0.9",
                 "source_id": "hdfs-logs-void-source",
-                "desired_num_pipelines": 0,
-                "max_num_pipelines_per_indexer": 1,
+                "num_pipelines": 0,
                 "source_type": "void",
                 "params": {}
             }
             "#;
             let error = load_source_config_from_user_config(ConfigFormat::Json, content.as_bytes())
                 .unwrap_err();
+            println!("error: {error}");
             assert!(error
                 .to_string()
-                .contains("`desired_num_pipelines` must be"));
+                .contains("`num_pipelines` must be"));
         }
-        // {
-        //     let content = r#"
-        //     {
-        //         "version": "0.7",
-        //         "source_id": "hdfs-logs-void-source",
-        //         "desired_num_pipelines": 1,
-        //         "max_num_pipelines_per_indexer": 0,
-        //         "source_type": "void",
-        //         "params": {}
-        //     }
-        //     "#;
-        //     let error = load_source_config_from_user_config(ConfigFormat::Json,
-        // content.as_bytes())         .unwrap_err();
-        //     assert!(error
-        //         .to_string()
-        //         .contains("`max_num_pipelines_per_indexer` must be"));
-        // }
         {
             let content = r#"
             {
-                "version": "0.8",
+                "version": "0.9",
                 "source_id": "hdfs-logs-void-source",
                 "num_pipelines": 2,
                 "source_type": "void",
@@ -909,10 +892,9 @@ mod tests {
         {
             let content = r#"
             {
-                "version": "0.7",
+                "version": "0.9",
                 "source_id": "hdfs-logs-void-source",
-                "desired_num_pipelines": 2,
-                "max_num_pipelines_per_indexer": 1,
+                "num_pipelines": 2,
                 "source_type": "void",
                 "params": {}
             }
@@ -970,7 +952,7 @@ mod tests {
         {
             let content = r#"
             {
-                "version": "0.8",
+                "version": "0.9",
                 "source_id": "hdfs-logs-kafka-source",
                 "num_pipelines": 3,
                 "source_type": "kafka",
@@ -1461,10 +1443,9 @@ mod tests {
     #[tokio::test]
     async fn test_source_config_plain_text_input_format() {
         let file_content = r#"{
-            "version": "0.7",
+            "version": "0.9",
             "source_id": "logs-file-source",
-            "desired_num_pipelines": 1,
-            "max_num_pipelines_per_indexer": 1,
+            "num_pipelines": 1,
             "source_type": "file",
             "params": {
               "filepath": "s3://mybucket/test_non_json_corpus.txt"
diff --git a/quickwit/quickwit-config/src/source_config/serialize.rs b/quickwit/quickwit-config/src/source_config/serialize.rs
@@ -24,15 +24,15 @@ use crate::{
     SourceParams,
 };
 
-type SourceConfigForSerialization = SourceConfigV0_8;
+type SourceConfigForSerialization = SourceConfigV0_9;
 
 #[derive(Serialize, Deserialize, utoipa::ToSchema)]
 #[serde(deny_unknown_fields)]
 #[serde(tag = "version")]
 pub enum VersionedSourceConfig {
     #[serde(rename = "0.9")]
     #[serde(alias = "0.8")]
-    V0_8(SourceConfigV0_8),
+    V0_9(SourceConfigV0_9),
     // Retro compatibility.
     #[serde(rename = "0.7")]
     V0_7(SourceConfigV0_7),
@@ -42,7 +42,7 @@ impl From<VersionedSourceConfig> for SourceConfigForSerialization {
     fn from(versioned_source_config: VersionedSourceConfig) -> Self {
         match versioned_source_config {
             VersionedSourceConfig::V0_7(v0_7) => v0_7.into(),
-            VersionedSourceConfig::V0_8(v0_8) => v0_8,
+            VersionedSourceConfig::V0_9(v0_9) => v0_9,
         }
     }
 }
@@ -102,7 +102,7 @@ impl SourceConfigForSerialization {
             validate_identifier("source", &self.source_id)?;
         }
         let num_pipelines = NonZeroUsize::new(self.num_pipelines)
-            .ok_or_else(|| anyhow::anyhow!("`desired_num_pipelines` must be strictly positive"))?;
+            .ok_or_else(|| anyhow::anyhow!("`num_pipelines` must be strictly positive"))?;
         match &self.source_params {
             SourceParams::Stdin => {
                 bail!(
@@ -158,9 +158,9 @@ impl SourceConfigForSerialization {
     }
 }
 
-impl From<SourceConfig> for SourceConfigV0_8 {
+impl From<SourceConfig> for SourceConfigV0_9 {
     fn from(source_config: SourceConfig) -> Self {
-        SourceConfigV0_8 {
+        SourceConfigV0_9 {
             source_id: source_config.source_id,
             num_pipelines: source_config.num_pipelines.get(),
             enabled: source_config.enabled,
@@ -173,15 +173,15 @@ impl From<SourceConfig> for SourceConfigV0_8 {
 
 impl From<SourceConfig> for VersionedSourceConfig {
     fn from(source_config: SourceConfig) -> Self {
-        VersionedSourceConfig::V0_8(source_config.into())
+        VersionedSourceConfig::V0_9(source_config.into())
     }
 }
 
 impl TryFrom<VersionedSourceConfig> for SourceConfig {
     type Error = anyhow::Error;
 
     fn try_from(versioned_source_config: VersionedSourceConfig) -> anyhow::Result<Self> {
-        let v1: SourceConfigV0_8 = versioned_source_config.into();
+        let v1: SourceConfigV0_9 = versioned_source_config.into();
         v1.validate_and_build()
     }
 }
@@ -230,7 +230,7 @@ pub struct SourceConfigV0_7 {
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
 #[serde(deny_unknown_fields)]
-pub struct SourceConfigV0_8 {
+pub struct SourceConfigV0_9 {
     #[schema(value_type = String)]
     pub source_id: SourceId,
 
@@ -252,7 +252,7 @@ pub struct SourceConfigV0_8 {
     pub input_format: SourceInputFormat,
 }
 
-impl From<SourceConfigV0_7> for SourceConfigV0_8 {
+impl From<SourceConfigV0_7> for SourceConfigV0_9 {
     fn from(source_config_v0_7: SourceConfigV0_7) -> Self {
         let SourceConfigV0_7 {
             source_id,
@@ -263,7 +263,7 @@ impl From<SourceConfigV0_7> for SourceConfigV0_8 {
             transform,
             input_format,
         } = source_config_v0_7;
-        SourceConfigV0_8 {
+        SourceConfigV0_9 {
             source_id,
             num_pipelines: desired_num_pipelines,
             enabled,
diff --git a/quickwit/quickwit-integration-tests/src/tests/update_tests/restart_indexer_tests.rs b/quickwit/quickwit-integration-tests/src/tests/update_tests/restart_indexer_tests.rs
@@ -64,7 +64,7 @@ async fn test_update_doc_mapping_restart_indexing_pipeline() {
         .indexes()
         .create(
             json!({
-                "version": "0.8",
+                "version": "0.9",
                 "index_id": index_id,
                 "doc_mapping": {
                     "doc_mapping_uid": initial_mapping_uid,
@@ -141,7 +141,7 @@ async fn test_update_doc_mapping_restart_indexing_pipeline() {
         .update(
             index_id,
             json!({
-                "version": "0.8",
+                "version": "0.9",
                 "index_id": index_id,
                 "doc_mapping": {
                     "doc_mapping_uid": final_mapping_uid,
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json
diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json
diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.9.json b/quickwit/quickwit-metastore/test-data/manifest/v0.9.json
diff --git a/quickwit/quickwit-serve/src/index_api/rest_handler.rs b/quickwit/quickwit-serve/src/index_api/rest_handler.rs
diff --git a/quickwit/quickwit-serve/src/template_api/rest_handler.rs b/quickwit/quickwit-serve/src/template_api/rest_handler.rs
diff --git a/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml

Original file line number	Diff line number	Diff line change
`@@ -232,7 +232,7 @@ quickwit run --service metastore --config=./config/quickwit.yaml`
`232`	`232`	`# Open a new terminal and run:`
`233`	`233`	`cat << EOF > wikipedia-kafka-source.json`
`234`	`234`	`{`
`235`		`- "version": "0.7",`
	`235`	`+ "version": "0.9",`
`236`	`236`	`"source_id": "kafka-source",`
`237`	`237`	`"source_type": "kafka",`
`238`	`238`	`"params": {`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-version: 0.8`
	`1`	`+version: 0.9`
`2`	`2`
`3`	`3`	`doc_mapping:`
`4`	`4`	`field_mappings:`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`# Comments are supported.`
`2`	`2`	`{`
`3`		`- "version": "0.7",`
	`3`	`+ "version": "0.9",`
`4`	`4`	`"index_id": "hdfs-logs",`
`5`	`5`	`"index_uri": "s3://quickwit-indexes/hdfs-logs",`
`6`	`6`	`"doc_mapping": {`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-version = "0.7"`
	`1`	`+version = "0.9"`
`2`	`2`	`index_id = "hdfs-logs"`
`3`	`3`	`index_uri = "s3://quickwit-indexes/hdfs-logs"`
`4`	`4`