Index catalog

Cargo.toml

@@ -28,7 +28,8 @@ opendal = { version = "0.46.0", features = ["services-fs"] }
 pretty_env_logger = "0.5.0"
 serde = { version = "1.0.201", features = ["derive", "rc"] }
 serde_json = "1.0.117"
-sqlx = { version = "0.7.4", features = ["postgres", "macros", "runtime-tokio"] }
+serde_yaml = "0.9.34"
+sqlx = { version = "0.7.4", features = ["postgres", "macros", "runtime-tokio", "json"] }
 tantivy = "0.22.0"
 tokio = { version = "1.37.0", features = ["full"] }
 tokio-util = { version = "0.7.11", features = ["compat"] }

example_config.yaml

@@ -0,0 +1,13 @@
+version: 1
+name: test
+path: /tmp/toshokan
+schema:
+  time_field: timestamp
+  mappings:
+    timestamp:
+      type: !datetime
+      formats:
+        - timestamp
+      indexed: true
+      stored: true
+      fast: true

migrations/0001_index_files.down.sql → migrations/0001_indexes.down.sql

@@ -1 +1,2 @@
 DROP TABLE index_files;
+DROP TABLE indexes;

migrations/0001_indexes.up.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS indexes(
+    name TEXT PRIMARY KEY,
+    config jsonb NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS index_files(
+    id VARCHAR(36) PRIMARY KEY,
+    index_name TEXT NOT NULL REFERENCES indexes(name) ON DELETE CASCADE,
+    file_name TEXT NOT NULL,
+    footer_len BIGINT NOT NULL
+);

src/args.rs

@@ -3,14 +3,6 @@ use clap::Parser;
 #[derive(Parser, Debug, Clone)]
 #[command(author, version, about, long_about = None)]
 pub struct Args {
-    #[clap(
-        short,
-        long,
-        help = "Path to the index dir.",
-        default_value = "/tmp/toshokan"
-    )]
-    pub index_dir: String,
-
     #[clap(
         long,
         help = "Postgres DB connection url.
@@ -24,6 +16,12 @@ Can also be provided by a DATABASE_URL env var, but only if this arg is not prov
 
 #[derive(Parser, Debug, Clone)]
 pub enum SubCommand {
+    #[clap(name = "create")]
+    Create(CreateArgs),
+
+    #[clap(name = "drop")]
+    Drop(DropArgs),
+
     #[clap(name = "index")]
     Index(IndexArgs),
 
@@ -34,8 +32,23 @@ pub enum SubCommand {
     Search(SearchArgs),
 }
 
+#[derive(Parser, Debug, Clone)]
+pub struct CreateArgs {
+    #[clap(help = "Path to the input config file.")]
+    pub config_path: String,
+}
+
+#[derive(Parser, Debug, Clone)]
+pub struct DropArgs {
+    #[clap(help = "The index name.")]
+    pub name: String,
+}
+
 #[derive(Parser, Debug, Clone)]
 pub struct IndexArgs {
+    #[clap(help = "The index name.")]
+    pub name: String,
+
     #[clap(help = "Path to the input jsonl file you want to index.")]
     pub input_path: String,
 
@@ -58,6 +71,9 @@ The memory is split evenly between all indexing threads, once a thread reaches i
 
 #[derive(Parser, Debug, Clone)]
 pub struct MergeArgs {
+    #[clap(help = "The index name.")]
+    pub name: String,
+
     #[clap(
         short,
         long,
@@ -69,6 +85,9 @@ pub struct MergeArgs {
 
 #[derive(Parser, Debug, Clone)]
 pub struct SearchArgs {
+    #[clap(help = "The index name.")]
+    pub name: String,
+
     #[clap(help = "Query in tantivy syntax.")]
     pub query: String,
 

src/index_config.rs

@@ -0,0 +1,196 @@
+use std::{collections::HashMap, ops::Deref, path::Path};
+
+use color_eyre::eyre::Result;
+use serde::{Deserialize, Serialize};
+use tantivy::{
+    schema::{IndexRecordOption, TextFieldIndexing, TextOptions},
+    DateOptions, DateTimePrecision,
+};
+use tokio::fs::read_to_string;
+
+const VERSION: u32 = 1;
+
+fn default_version() -> u32 {
+    VERSION
+}
+
+fn default_true() -> bool {
+    true
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(rename_all = "snake_case")]
+pub enum FastTextFieldType {
+    #[default]
+    Disabled,
+    Raw,
+    Lowercase,
+}
+
+impl From<FastTextFieldType> for Option<&str> {
+    fn from(value: FastTextFieldType) -> Self {
+        match value {
+            FastTextFieldType::Disabled => None,
+            FastTextFieldType::Raw => Some("raw"),
+            FastTextFieldType::Lowercase => Some("lowercase"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct TextMappingConfig {
+    #[serde(default = "default_true")]
+    pub stored: bool,
+
+    #[serde(default)]
+    pub fast: FastTextFieldType,
+
+    #[serde(default)]
+    pub record: IndexRecordOption,
+
+    #[serde(default = "default_true")]
+    pub fieldnorms: bool,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(rename_all = "snake_case")]
+pub enum DateTimeFastPrecisionType {
+    #[default]
+    False,
+    True,
+    Seconds,
+    Milliseconds,
+    Microseconds,
+    Nanoseconds,
+}
+
+impl From<DateTimeFastPrecisionType> for Option<DateTimePrecision> {
+    fn from(value: DateTimeFastPrecisionType) -> Self {
+        use DateTimeFastPrecisionType::*;
+        match value {
+            False => None,
+            True | Seconds => Some(DateTimePrecision::Seconds),
+            Milliseconds => Some(DateTimePrecision::Milliseconds),
+            Microseconds => Some(DateTimePrecision::Microseconds),
+            Nanoseconds => Some(DateTimePrecision::Nanoseconds),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DateTimeFormatType {
+    Iso8601,
+    Rfc2822,
+    Rfc3339,
+    Timestamp,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct DateTimeFormats(Vec<DateTimeFormatType>);
+
+impl Default for DateTimeFormats {
+    fn default() -> Self {
+        Self(vec![
+            DateTimeFormatType::Rfc3339,
+            DateTimeFormatType::Timestamp,
+        ])
+    }
+}
+
+impl Deref for DateTimeFormats {
+    type Target = Vec<DateTimeFormatType>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct DateTimeMappingConfig {
+    #[serde(default = "default_true")]
+    pub stored: bool,
+
+    #[serde(default = "default_true")]
+    pub indexed: bool,
+
+    #[serde(default)]
+    pub fast: DateTimeFastPrecisionType,
+
+    #[serde(default)]
+    pub formats: DateTimeFormats,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum MappingFieldType {
+    Text(TextMappingConfig),
+    Datetime(DateTimeMappingConfig),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct MappingConfig {
+    #[serde(rename = "type")]
+    pub type_: MappingFieldType,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct IndexSchema {
+    #[serde(default)]
+    pub mappings: HashMap<String, MappingConfig>,
+
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    time_field: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct IndexConfig {
+    pub name: String,
+    pub path: String,
+
+    #[serde(default = "default_version")]
+    version: u32,
+
+    #[serde(default)]
+    pub schema: IndexSchema,
+}
+
+impl IndexConfig {
+    pub async fn from_path<P: AsRef<Path>>(path: P) -> Result<Self> {
+        let config_str = read_to_string(path).await?;
+        Ok(serde_yaml::from_str(&config_str)?)
+    }
+}
+
+impl From<TextMappingConfig> for TextOptions {
+    fn from(config: TextMappingConfig) -> Self {
+        let mut text_options = TextOptions::default();
+        if config.stored {
+            text_options = text_options.set_stored();
+        }
+        text_options = text_options.set_fast(config.fast.into());
+        text_options = text_options.set_indexing_options(
+            TextFieldIndexing::default()
+                .set_index_option(config.record)
+                .set_fieldnorms(config.fieldnorms),
+        );
+        text_options
+    }
+}
+
+impl From<DateTimeMappingConfig> for DateOptions {
+    fn from(config: DateTimeMappingConfig) -> Self {
+        let mut date_options = DateOptions::default();
+        if config.stored {
+            date_options = date_options.set_stored();
+        }
+        if config.indexed {
+            date_options = date_options.set_indexed();
+        }
+        if let Some(precision) = config.fast.into() {
+            date_options = date_options.set_fast().set_precision(precision);
+        }
+        date_options
+    }
+}