Skip to content

Commit

Permalink
config: text: Put all fast text field normalizers tativy supports
Browse files Browse the repository at this point in the history
  • Loading branch information
tontinton committed Jun 1, 2024
1 parent fec0eb4 commit f6cb364
Showing 1 changed file with 20 additions and 4 deletions.
24 changes: 20 additions & 4 deletions src/config/text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,33 @@ use super::default_true;
#[serde(rename_all = "snake_case")]
pub enum FastTextFieldType {
#[default]
Disabled,
False,

/// Chops the text on according to whitespace and
/// punctuation, removes tokens that are too long, and lowercases
/// tokens.
True,

/// Does not process nor tokenize the text.
Raw,
Lowercase,

/// Like `true`, but also applies stemming on the
/// resulting tokens. Stemming can improve the recall of your
/// search engine.
EnStem,

/// Splits the text on whitespaces.
Whitespace,
}

impl From<FastTextFieldType> for Option<&str> {
fn from(value: FastTextFieldType) -> Self {
match value {
FastTextFieldType::Disabled => None,
FastTextFieldType::False => None,
FastTextFieldType::True => Some("default"),
FastTextFieldType::Raw => Some("raw"),
FastTextFieldType::Lowercase => Some("lowercase"),
FastTextFieldType::EnStem => Some("en_stem"),
FastTextFieldType::Whitespace => Some("whitespace"),
}
}
}
Expand Down

0 comments on commit f6cb364

Please sign in to comment.