Merge pull request #16 from NuschtOS/index-meta

Author: SuperSandro2000

fixx/src/lib.rs

@@ -1,3 +1,5 @@
+use std::string::FromUtf8Error;
+
 use wasm_bindgen::prelude::*;
 
 #[wasm_bindgen]
@@ -17,8 +19,28 @@ impl Index {
       .map_err(|err| format!("{:?}", err))
   }
 
-  pub fn search(&self, query: String, max_results: usize) -> Result<Vec<SearchedOption>, String> {
-    match self.0.search(&query, max_results) {
+  pub fn chunk_size(&self) -> u32 {
+    self.0.meta().chunk_size
+  }
+
+  pub fn scopes(&self) -> Result<Vec<String>, String> {
+    self
+      .0
+      .meta()
+      .scopes
+      .iter()
+      .map(|scope| String::try_from(scope.clone()))
+      .collect::<Result<Vec<String>, FromUtf8Error>>()
+      .map_err(|err| format!("{:?}", err))
+  }
+
+  pub fn search(
+    &self,
+    scope_id: Option<u8>,
+    query: String,
+    max_results: usize,
+  ) -> Result<Vec<SearchedOption>, String> {
+    match self.0.search(scope_id, &query, max_results) {
       Ok(options) => Ok(
         options
           .into_iter()
@@ -29,10 +51,6 @@ impl Index {
     }
   }
 
-  pub fn all(&self, max: usize) -> Result<Vec<String>, String> {
-    self.0.all(max).map_err(|err| format!("{:?}", err))
-  }
-
   pub fn get_idx_by_name(&self, name: String) -> Result<Option<usize>, String> {
     self
       .0

ixx/src/action/index.rs

@@ -23,22 +23,32 @@ pub(crate) struct Config {
 #[derive(Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct Scope {
+  name: Option<String>,
   options_json: PathBuf,
   url_prefix: Url,
   options_prefix: Option<String>,
 }
 
 pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> {
-  let mut raw_options: BTreeMap<String, libixx::Option> = BTreeMap::new();
+  let mut raw_options: BTreeMap<String, (u8, libixx::Option)> = BTreeMap::new();
 
   let config_file = File::open(module.config)?;
   let config: Config = serde_json::from_reader(config_file)?;
 
+  let mut index = Index::new(module.chunk_size);
+
   for scope in config.scopes {
     println!("Parsing {}", scope.options_json.to_string_lossy());
     let file = File::open(scope.options_json)?;
     let options: HashMap<String, option::Option> = serde_json::from_reader(file)?;
 
+    let scope_idx = index.push_scope(
+      scope
+        .name
+        .map(|x| x.to_string())
+        .unwrap_or_else(|| scope.url_prefix.to_string()),
+    );
+
     for (name, option) in options {
       // internal options which cannot be hidden when importing existing options.json
       if name == "_module.args" {
@@ -50,14 +60,15 @@ pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> {
         None => name,
       };
       let option = into_option(&scope.url_prefix, &name, option)?;
-      raw_options.insert(name, option);
+      raw_options.insert(name.clone(), (scope_idx, option));
     }
   }
 
   println!("Read {} options", raw_options.len());
 
-  let mut index = Index::default();
-  raw_options.keys().for_each(|name| index.push(name));
+  for (name, (scope_idx, _)) in &raw_options {
+    index.push(*scope_idx, name);
+  }
 
   println!("Writing index to {}", module.index_output.to_string_lossy());
 
@@ -70,8 +81,12 @@ pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> {
     std::fs::create_dir(&module.meta_output)?;
   }
 
-  let options: Vec<libixx::Option> = raw_options.into_values().collect();
-  for (idx, chunk) in options.chunks(module.chunk_size).enumerate() {
+  let options = raw_options
+    .into_values()
+    .map(|(_, options)| options)
+    .collect::<Vec<_>>();
+
+  for (idx, chunk) in options.chunks(module.chunk_size as usize).enumerate() {
     let mut file = File::create(module.meta_output.join(format!("{}.json", idx)))?;
     serde_json::to_writer(&mut file, &chunk)?;
   }
@@ -91,7 +106,7 @@ fn into_option(
       .map(|declaration| update_declaration(url_prefix, declaration))
       .collect::<anyhow::Result<_>>()?,
     default: option.default.map(|option| option.render()),
-    description: option.description,
+    description: markdown::to_html(&option.description),
     example: option.example.map(|example| example.render()),
     read_only: option.read_only,
     r#type: option.r#type,

ixx/src/action/search.rs

@@ -8,7 +8,7 @@ pub(crate) fn search(module: SearchModule) -> anyhow::Result<()> {
   let mut file = File::open(module.index)?;
   let index = Index::read_from(&mut file)?;
 
-  let result = index.search(&module.query, module.max_results as usize)?;
+  let result = index.search(module.scope_id, &module.query, module.max_results as usize)?;
   for (idx, name) in result {
     println!("idx: {}, name: {}", idx, name);
   }

ixx/src/args.rs

@@ -24,7 +24,7 @@ pub(super) struct IndexModule {
   pub(crate) meta_output: PathBuf,
 
   #[clap(short, long, default_value = "100")]
-  pub(super) chunk_size: usize,
+  pub(super) chunk_size: u32,
 }
 
 #[derive(Parser)]
@@ -34,6 +34,9 @@ pub(super) struct SearchModule {
   #[clap(short, long, default_value = "index.ixx")]
   pub(super) index: PathBuf,
 
+  #[clap(short, long)]
+  pub(super) scope_id: Option<u8>,
+
   #[clap(short, long, default_value = "10")]
   pub(super) max_results: u32,
 }

libixx/src/index.rs

@@ -4,18 +4,32 @@ use binrw::{binrw, BinRead, BinWrite, Endian, NullString};
 
 use crate::IxxError;
 
-#[binrw(magic = b"ixx01", little)]
-#[derive(Default, Debug, Clone, PartialEq)]
+#[binrw]
+#[brw(magic = b"ixx01")]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Index {
-  #[bw(calc = entries.len() as u32)]
+  meta: Meta,
+  #[bw(calc = options.len() as u32)]
   count: u32,
   #[br(count = count)]
-  entries: Vec<Entry>,
+  options: Vec<OptionEntry>,
+}
+
+#[binrw]
+#[derive(Debug, Clone, PartialEq)]
+pub struct Meta {
+  pub chunk_size: u32,
+  #[bw(calc = scopes.len() as u8)]
+  scope_count: u8,
+  #[br(count = scope_count)]
+  pub scopes: Vec<NullString>,
 }
 
 #[binrw]
 #[derive(Default, Debug, Clone, PartialEq)]
-pub struct Entry {
+pub struct OptionEntry {
+  /// index in the scopes Vec
+  scope_id: u8,
   #[bw(calc = labels.len() as u16)]
   count: u16,
   #[br(count = count)]
@@ -32,11 +46,23 @@ struct Reference {
 #[binrw]
 #[derive(Debug, Clone, PartialEq)]
 enum Label {
+  #[brw(magic = b"0")]
   InPlace(NullString),
+  #[brw(magic = b"1")]
   Reference(Reference),
 }
 
 impl Index {
+  pub fn new(chunk_size: u32) -> Self {
+    Self {
+      meta: Meta {
+        chunk_size,
+        scopes: vec![],
+      },
+      options: vec![],
+    }
+  }
+
   pub fn read(buf: &[u8]) -> Result<Self, IxxError> {
     Self::read_from(&mut Cursor::new(buf))
   }
@@ -49,13 +75,13 @@ impl Index {
     Ok(BinWrite::write_options(self, write, Endian::Little, ())?)
   }
 
-  pub fn push(&mut self, option: &str) {
+  pub fn push(&mut self, scope_id: u8, option: &str) {
     let labels = option
       .split('.')
       .map(|segment| {
         let segment = segment.into();
 
-        for (option_idx, Entry { labels: option }) in self.entries.iter().enumerate() {
+        for (option_idx, OptionEntry { labels: option, .. }) in self.options.iter().enumerate() {
           for (label_idx, label) in option.iter().enumerate() {
             if let Label::InPlace(inplace) = label {
               if inplace != &segment {
@@ -74,17 +100,17 @@ impl Index {
       })
       .collect();
 
-    self.entries.push(Entry { labels });
+    self.options.push(OptionEntry { scope_id, labels });
   }
 
   fn resolve_reference(&self, reference: &Reference) -> Result<&NullString, IxxError> {
     let option_idx = reference.option_idx as usize;
 
-    if self.entries.len() <= option_idx {
+    if self.options.len() <= option_idx {
       return Err(IxxError::ReferenceOutOfBounds);
     }
 
-    let entry = &self.entries[option_idx].labels;
+    let entry = &self.options[option_idx].labels;
 
     let label_idx = reference.label_idx as usize;
 
@@ -106,7 +132,7 @@ impl Index {
       let segment = segment.into();
 
       'outer: {
-        for (option_idx, Entry { labels: option }) in self.entries.iter().enumerate() {
+        for (option_idx, OptionEntry { labels: option, .. }) in self.options.iter().enumerate() {
           for (label_idx, label) in option.iter().enumerate() {
             if let Label::InPlace(inplace) = label {
               if inplace != &segment {
@@ -128,27 +154,41 @@ impl Index {
 
     Ok(
       self
-        .entries
+        .options
         .iter()
         .enumerate()
-        .find(|(idx, Entry { labels: option })| do_labels_match(*idx, option, &labels))
+        .find(|(idx, OptionEntry { labels: option, .. })| do_labels_match(*idx, option, &labels))
         .map(|(idx, _)| idx),
     )
   }
 
-  pub fn search(&self, query: &str, max_results: usize) -> Result<Vec<(usize, String)>, IxxError> {
+  pub fn search(
+    &self,
+    scope_id: Option<u8>,
+    query: &str,
+    max_results: usize,
+  ) -> Result<Vec<(usize, String)>, IxxError> {
     let search = query
       .split('*')
       .map(|segment| segment.to_lowercase())
       .collect::<Vec<_>>();
 
-    if search.is_empty() {
-      return Ok(vec![]);
-    }
-
     let mut results = Vec::new();
 
-    for (idx, Entry { labels: option }) in self.entries.iter().enumerate() {
+    for (
+      idx,
+      OptionEntry {
+        scope_id: option_scope_id,
+        labels: option,
+      },
+    ) in self.options.iter().enumerate()
+    {
+      if let Some(scope_id) = scope_id {
+        if *option_scope_id != scope_id {
+          continue;
+        }
+      }
+
       let mut option_name = String::new();
       for label in option {
         option_name.push_str(&String::try_from(
@@ -185,28 +225,18 @@ impl Index {
     Ok(results)
   }
 
-  pub fn all(&self, max: usize) -> Result<Vec<String>, IxxError> {
-    let mut options = Vec::new();
-
-    for Entry { labels: option } in &self.entries[..max] {
-      let mut option_name = String::new();
-      for label in option {
-        option_name.push_str(&String::try_from(
-          match label {
-            Label::InPlace(data) => data,
-            Label::Reference(reference) => self.resolve_reference(reference)?,
-          }
-          .clone(),
-        )?);
-        option_name.push('.')
-      }
-      // remove last dot...
-      option_name.pop();
+  pub fn meta(&self) -> &Meta {
+    &self.meta
+  }
 
-      options.push(option_name);
+  pub fn push_scope(&mut self, scope: String) -> u8 {
+    if self.meta.scopes.len() == u8::MAX.into() {
+      panic!("You reached the limit of 256 scopes. Please contact the developers for further assistance.");
     }
 
-    Ok(options)
+    let idx = self.meta.scopes.len();
+    self.meta.scopes.push(scope.into());
+    idx as u8
   }
 }