feat: handle multiple derivations for words in the metadata

harper-cli/src/main.rs

@@ -103,6 +103,8 @@ enum Args {
         /// The document to mine words from.
         file: PathBuf,
     },
+    /// Get the word associated with a particular word id.
+    WordFromId { hash: u64 },
     #[cfg(feature = "training")]
     TrainBrillTagger {
         #[arg(short, long, default_value = "1.0")]
@@ -368,7 +370,29 @@ fn main() -> anyhow::Result<()> {
             let mut results = BTreeMap::new();
             for word in words {
                 let metadata = dictionary.get_word_metadata_str(&word);
-                results.insert(word, metadata);
+                let mut metadata_value = serde_json::to_value(metadata).unwrap_or_default();
+
+                // If there are derived words, add them to the metadata
+                if let Some(metadata) = dictionary.get_word_metadata_str(&word)
+                    && let Some(derived_from) = &metadata.derived_from
+                {
+                    let derived_words: Vec<String> = derived_from
+                        .iter()
+                        .filter_map(|wordid| dictionary.get_word_from_id(wordid))
+                        .map(|word| word.iter().collect())
+                        .collect();
+
+                    if !derived_words.is_empty()
+                        && let Some(obj) = metadata_value.as_object_mut()
+                    {
+                        obj.insert(
+                            "derived_from_words".to_string(),
+                            serde_json::json!(derived_words),
+                        );
+                    }
+                }
+
+                results.insert(word, metadata_value);
             }
             let json = serde_json::to_string_pretty(&results).unwrap();
             println!("{json}");
@@ -502,6 +526,11 @@ fn main() -> anyhow::Result<()> {
 
             Ok(())
         }
+        Args::WordFromId { hash } => {
+            let id = WordId::from_hash(hash);
+            println!("{:?}", dictionary.get_word_from_id(&id));
+            Ok(())
+        }
         Args::CoreVersion => {
             println!("harper-core v{}", harper_core::core_version());
             Ok(())
@@ -853,9 +882,14 @@ fn print_word_derivations(word: &str, annot: &str, dictionary: &impl Dictionary)
 
     let id = WordId::from_word_str(word);
 
-    let children = dictionary
-        .words_iter()
-        .filter(|e| dictionary.get_word_metadata(e).unwrap().derived_from == Some(id));
+    let children = dictionary.words_iter().filter(|e| {
+        dictionary
+            .get_word_metadata(e)
+            .unwrap()
+            .derived_from
+            .as_ref()
+            .is_some_and(|derived| derived.contains(&id))
+    });
 
     println!(" - {word}");
 

harper-core/src/fat_token.rs

@@ -4,7 +4,7 @@ use crate::{CharStringExt, TokenKind};
 
 /// A [`Token`](crate::Token) that holds its content as a fat [`Vec<char>`] rather than as a
 /// [`Span`](crate::Span).
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
 pub struct FatToken {
     pub content: Vec<char>,
     pub kind: TokenKind,
@@ -20,7 +20,7 @@ impl From<FatStringToken> for FatToken {
 }
 
 /// Similar to a [`FatToken`], but uses a [`String`] as the underlying store.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct FatStringToken {
     pub content: String,
     pub kind: TokenKind,

harper-core/src/ignored_lints/lint_context.rs

@@ -9,7 +9,7 @@ use crate::{
 
 /// A location-agnostic structure that attempts to captures the context and content that a [`Lint`]
 /// occurred.
-#[derive(Debug, Hash, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct LintContext {
     pub lint_kind: LintKind,
     pub suggestions: Vec<Suggestion>,
@@ -18,6 +18,16 @@ pub struct LintContext {
     pub tokens: Vec<FatToken>,
 }
 
+impl Hash for LintContext {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.lint_kind.hash(state);
+        self.suggestions.hash(state);
+        self.message.hash(state);
+        self.priority.hash(state);
+        self.tokens.hash(state);
+    }
+}
+
 impl LintContext {
     pub fn from_lint(lint: &Lint, document: &Document) -> Self {
         let Lint {

harper-core/src/spell/fst_dictionary.rs

@@ -306,52 +306,55 @@ mod tests {
     #[test]
     fn plural_llamas_derived_from_llama() {
         let dict = FstDictionary::curated();
-
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("llamas")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("llama")
-        )
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("llama"))
+        );
     }
 
     #[test]
     fn plural_cats_derived_from_cat() {
         let dict = FstDictionary::curated();
 
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("cats")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("cat")
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("cat"))
         );
     }
 
     #[test]
     fn unhappy_derived_from_happy() {
         let dict = FstDictionary::curated();
 
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("unhappy")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("happy")
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("happy"))
         );
     }
 
     #[test]
     fn quickly_derived_from_quick() {
         let dict = FstDictionary::curated();
 
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("quickly")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("quick")
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("quick"))
         );
     }
 }

harper-core/src/spell/mod.rs

@@ -17,7 +17,7 @@ mod rune;
 mod word_id;
 mod word_map;
 
-#[derive(PartialEq, Debug, Hash, Eq)]
+#[derive(PartialEq, Debug, Eq)]
 pub struct FuzzyMatchResult<'a> {
     pub word: &'a [char],
     pub edit_distance: u8,

harper-core/src/spell/rune/attribute_list.rs

@@ -1,4 +1,4 @@
-use hashbrown::HashMap;
+use hashbrown::{HashMap, HashSet};
 use serde::{Deserialize, Serialize};
 use smallvec::ToSmallVec;
 
@@ -124,11 +124,17 @@ impl AttributeList {
                     );
                     let t_metadata = dest.get_metadata_mut_chars(&new_word).unwrap();
                     t_metadata.append(&metadata);
-                    t_metadata.derived_from = Some(WordId::from_word_chars(&word.letters))
+                    t_metadata
+                        .derived_from
+                        .get_or_insert_with(HashSet::new)
+                        .insert(WordId::from_word_chars(&word.letters));
                 }
             } else {
                 for (key, mut value) in new_words.into_iter() {
-                    value.derived_from = Some(WordId::from_word_chars(&word.letters));
+                    value
+                        .derived_from
+                        .get_or_insert_with(HashSet::new)
+                        .insert(WordId::from_word_chars(&word.letters));
 
                     if let Some(val) = dest.get_metadata_mut_chars(&key) {
                         val.append(&value);

harper-core/src/spell/word_id.rs

@@ -31,4 +31,8 @@ impl WordId {
         let chars: CharString = text.as_ref().chars().collect();
         Self::from_word_chars(chars)
     }
+
+    pub fn from_hash(hash: u64) -> Self {
+        Self { hash }
+    }
 }

harper-core/src/token_kind.rs

@@ -2,7 +2,9 @@ use harper_brill::UPOS;
 use is_macro::Is;
 use serde::{Deserialize, Serialize};
 
-use crate::{Number, Punctuation, Quote, TokenKind::Word, WordMetadata};
+use crate::TokenKind::Word;
+use crate::{Number, Punctuation, Quote, WordMetadata};
+use std::hash::{Hash, Hasher};
 
 /// Generate wrapper code to pass a function call to the inner [`WordMetadata`],  
 /// if the token is indeed a word, while also emitting method-level documentation.
@@ -29,7 +31,7 @@ macro_rules! delegate_to_metadata {
 /// Has a variety of queries available.
 /// If there is a query missing, it may be easy to implement by just calling the
 /// `delegate_to_metadata` macro.
-#[derive(Debug, Is, Clone, Serialize, Deserialize, Default, PartialOrd, Hash, Eq, PartialEq)]
+#[derive(Debug, Is, Clone, Serialize, Deserialize, Default, PartialOrd, Eq, PartialEq)]
 #[serde(tag = "kind", content = "value")]
 pub enum TokenKind {
     /// `None` if the word does not exist in the dictionary.
@@ -52,6 +54,49 @@ pub enum TokenKind {
     Regexish,
 }
 
+impl Hash for TokenKind {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            TokenKind::Word(metadata) => {
+                metadata.hash(state);
+            }
+            TokenKind::Punctuation(punct) => {
+                punct.hash(state);
+            }
+            TokenKind::Decade => {
+                0.hash(state);
+            }
+            TokenKind::Number(number) => {
+                number.hash(state);
+            }
+            TokenKind::Space(space) => {
+                space.hash(state);
+            }
+            TokenKind::Newline(newline) => {
+                newline.hash(state);
+            }
+            TokenKind::EmailAddress => {
+                0.hash(state);
+            }
+            TokenKind::Url => {
+                0.hash(state);
+            }
+            TokenKind::Hostname => {
+                0.hash(state);
+            }
+            TokenKind::Unlintable => {
+                0.hash(state);
+            }
+            TokenKind::ParagraphBreak => {
+                0.hash(state);
+            }
+            TokenKind::Regexish => {
+                0.hash(state);
+            }
+        }
+    }
+}
+
 impl TokenKind {
     // Word metadata delegation methods grouped by part of speech
     delegate_to_metadata! {