Plugins: Resolve TODOs & Refactoring & Documentation.

* Plugin api: Remove checks for building with all features since they
  are not conflicting anymore.
* Remove all code for forcing parser plugin via environment variables
  since it's not needed anymore for parser plugins.
* Resolve smaller todos in plugins host and api.
* Documentation and comments about plugins in Rust code.

Author: AmmarAbouZor

application/apps/indexer/indexer_cli/src/interactive.rs

@@ -99,19 +99,22 @@ pub(crate) async fn handle_interactive_session(input: Option<PathBuf>) {
                         println!("plugin command received");
                         const PLUGIN_PATH_ENV: &str = "WASM_PLUGIN_PATH";
 
-                        //TODO AAZ: Find a better way to deliver plugin path than environment variables
+                        // Delivering the plugins info via env variables is good enough since this
+                        // tool is going to be deprecated soon.
                         let plugin_path = match std::env::var(PLUGIN_PATH_ENV) {
                             Ok(path) => path,
-                            Err(err) => panic!("Retrieving plugin path environment variable failed. Err {err}") ,
+                            Err(err) => panic!("Retrieving plugin path environment variable failed.\n\
+                                    Please set the plugin path via the environment variable {PLUGIN_PATH_ENV}.\n\
+                                    Err {err}") ,
                         };
                         start = Instant::now();
                         let uuid = Uuid::new_v4();
                         let file_path = input.clone().expect("input must be present");
                         let proto_plugin_path = PathBuf::from(plugin_path);
 
-                        //TODO AAZ: plugins configuration aren't delivered here.
+                        //NOTE: plugins configuration aren't delivered here.
                         let plugin_configs = Vec::new();
-                        let plugin_parser_settings = stypes::PluginParserSettings::prototyping(proto_plugin_path, plugin_configs);
+                        let plugin_parser_settings = stypes::PluginParserSettings::new(proto_plugin_path, Default::default() ,plugin_configs);
                         session.observe(uuid, stypes::ObserveOptions::file(file_path, stypes::FileFormat::Binary, stypes::ParserType::Plugin(plugin_parser_settings))).expect("observe failed");
                     }
                     Some(Command::Grab) => {

application/apps/indexer/plugins_api/build.rs

@@ -180,7 +180,8 @@ macro_rules! bytesource_export {
                 // SAFETY: Bytesource host implements read trait, which takes a mutable reference
                 // to self when called. Therefor it's not possible to have multiple references on
                 // the static bytesource instance here at once.
-                //TODO AAZ: Find better way than denying the warning.
+                //TODO AAZ: Measure the impact on this unsafe function and provide explanation for
+                // suppressing the warning here.
                 #[allow(static_mut_refs)]
                 let source =
                     unsafe { BYTESOURCE.as_mut().expect("Bytesource already initialized") };

application/apps/indexer/plugins_api/src/bytesource/mod.rs

@@ -10,7 +10,7 @@
 //! The provided types and functions match the definitions from the `WIT` files defined in Chipmunk
 //! repository.
 //!
-//! TODO: This is basic documentation that need a lot of improvements and examples
+//! TODO AAZ: This is basic documentation that need a lot of improvements and examples
 //!
 
 mod shared;
@@ -36,10 +36,6 @@ pub use shared::{
     plugin_logger::{LogSend as __LogSend, PluginLogger as __PluginLogger},
 };
 
-//TODO AAZ: Check if we can remove the features, after moving shared types.
-// NOTE: Calling generate! Macro multiple time on the same crate causes compilation errors with `cargo
-// component` in release mode.
-
 #[cfg(feature = "bytesource")]
 #[cfg_attr(docsrs, doc(cfg(feature = "bytesource")))]
 pub mod bytesource;

application/apps/indexer/plugins_api/src/lib.rs

@@ -262,7 +262,8 @@ macro_rules! parser_export {
                 use $crate::parser::Parser;
                 // SAFETY: Parse method has mutable reference to self and can't be called more than
                 // once on the same time on host
-                //TODO AAZ: Find better way than denying the warning.
+                //TODO AAZ: Measure the impact on this unsafe function and provide explanation for
+                // suppressing the warning here.
                 #[allow(static_mut_refs)]
                 let parser = unsafe { PARSER.as_mut().expect("parser already initialized") };
                 parser.parse(&data, timestamp).map(|items| items.collect())

application/apps/indexer/plugins_api/src/parser/mod.rs

@@ -24,8 +24,9 @@ fn plugin_parser_init(c: &mut Criterion) {
     //TODO AAZ: Deliver plugin configurations for benchmarks
     let plugin_configs = Vec::new();
 
-    let settings = black_box(stypes::PluginParserSettings::prototyping(
+    let settings = black_box(stypes::PluginParserSettings::new(
         plugin_file,
+        stypes::PluginParserGeneralSettings::default(),
         plugin_configs,
     ));
 

application/apps/indexer/plugins_host/benches/plugin_parser_init.rs

@@ -11,6 +11,7 @@ use crate::{
     PluginHostInitError, WasmPlugin,
 };
 
+/// Interface name for the byte-source plugin with the package name as defined in WIT file.
 const BYTESOURCE_INTERFACE_NAME: &str = "chipmunk:bytesource/byte-source";
 
 /// The maximum number of consecutive returns with empty bytes allowed from a plugin.
@@ -139,6 +140,7 @@ impl PluginsByteSource {
         }
     }
 
+    /// Calls the function read on the plugin with provided length return the read byte.
     async fn read_next(&mut self, len: usize) -> io::Result<Vec<u8>> {
         let res = match &mut self.source {
             PlugVerByteSource::Ver010(source) => source.read_next(len).await,
@@ -183,6 +185,7 @@ impl WasmPlugin for PluginsByteSource {
     }
 }
 
+// PluginsByteSource implements Read trait so we can use it with `BinaryByteSource`
 impl Read for PluginsByteSource {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         let len = buf.len();

application/apps/indexer/plugins_host/src/bytesource_shared/mod.rs

@@ -1,3 +1,7 @@
+//! Library contains implementation for plugins system in Chipmunk using WASM with
+//! the component model approach.
+//! The used runtime for the plugin is [wasmtime](https://docs.rs/wasmtime/latest/wasmtime/)
+
 mod bytesource_shared;
 mod parser_shared;
 pub mod plugins_manager;
@@ -14,7 +18,7 @@ pub use bytesource_shared::PluginsByteSource;
 pub use plugins_shared::plugin_errors::{PluginGuestInitError, PluginHostInitError};
 use stypes::{PluginType, SemanticVersion};
 
-// Trait is used with Chipmunk only.
+/// Provided needed method and definitions for all WASM plugins in Chipmunk.
 #[allow(async_fn_in_trait)]
 pub trait WasmPlugin {
     /// Provides the Type of the plugin.

application/apps/indexer/plugins_host/src/lib.rs

@@ -1,6 +1,6 @@
 use std::path::{Path, PathBuf};
 
-use stypes::{ParserRenderOptions, SemanticVersion, ValidPluginInfo};
+use stypes::{SemanticVersion, ValidPluginInfo};
 use wasmtime::component::Component;
 
 use crate::{
@@ -10,6 +10,7 @@ use crate::{
 
 pub mod plugin_parse_message;
 
+/// Interface name for the parser plugin with the package name as defined in WIT file.
 const PARSER_INTERFACE_NAME: &str = "chipmunk:parser/parser";
 
 /// Marker for a column separator in the output string.
@@ -139,12 +140,6 @@ impl PluginsParser {
             ))),
         }
     }
-
-    pub async fn get_render_options(&mut self) -> Result<ParserRenderOptions, PluginError> {
-        match &mut self.parser {
-            PlugVerParser::Ver010(parser) => parser.get_render_options().await,
-        }
-    }
 }
 
 impl WasmPlugin for PluginsParser {

application/apps/indexer/plugins_host/src/parser_shared/mod.rs

@@ -3,29 +3,13 @@ use std::fmt::Display;
 use parsers::LogMessage;
 use serde::Serialize;
 
+/// Represent the message of the parsed item returned by plugins.
 #[derive(Debug, Serialize)]
 pub struct PluginParseMessage {
+    /// The content of the message as string.
     pub content: String,
 }
 
-impl PluginParseMessage {
-    pub fn new(content: String) -> Self {
-        Self { content }
-    }
-}
-
-impl From<String> for PluginParseMessage {
-    fn from(content: String) -> Self {
-        Self::new(content)
-    }
-}
-
-impl From<PluginParseMessage> for String {
-    fn from(value: PluginParseMessage) -> Self {
-        value.content
-    }
-}
-
 impl Display for PluginParseMessage {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.content)
@@ -37,6 +21,7 @@ impl LogMessage for PluginParseMessage {
         &self,
         writer: &mut W,
     ) -> Result<usize, std::io::Error> {
+        //TODO AAZ: Make sure plugins messages should support export as binary.
         let bytes = self.content.as_bytes();
         let len = bytes.len();
         writer.write_all(bytes)?;

application/apps/indexer/plugins_host/src/parser_shared/plugin_parse_message.rs

@@ -5,14 +5,19 @@ use thiserror::Error;
 
 use crate::wasm_host::WasmHostInitError;
 
+/// Plugins manager initialization Error.
 #[derive(Debug, Error)]
 pub enum InitError {
+    /// Errors while initializing wasm host for plugins.
     #[error("Initialization of WASM host failed. {0}")]
     WasmHost(#[from] WasmHostInitError),
+    /// Errors from WASM runtime.
     #[error(transparent)]
     WasmRunTimeError(#[from] anyhow::Error),
+    /// IO Errors.
     #[error("IO Error. {0}")]
     IO(#[from] io::Error),
+    /// Errors not included in the other error types.
     #[error("Error during initialization. {0}")]
     Other(String),
 }

application/apps/indexer/plugins_host/src/plugins_manager/errors.rs

@@ -1,6 +1,4 @@
 mod paths;
-#[cfg(test)]
-mod tests;
 
 use std::{
     fs::{self, read_to_string},
@@ -63,9 +61,13 @@ fn get_dirs(dir_path: &PathBuf) -> Result<impl Iterator<Item = PathBuf>, io::Err
     Ok(dirs)
 }
 
+/// Loads parser infos and metadata from the provided parser directory.
 async fn load_parser(dir: PathBuf) -> Result<PluginEntity, InitError> {
     let (wasm_file, metadata_file) = match validate_plugin_files(&dir)? {
-        PluginValidationState::Valid { wasm, metadata } => (wasm, metadata),
+        PluginValidationState::Valid {
+            wasm_path: wasm,
+            metadata,
+        } => (wasm, metadata),
         PluginValidationState::Invalid { err_msg } => {
             let invalid_entity = PluginEntity {
                 dir_path: dir,
@@ -127,17 +129,25 @@ async fn load_parser(dir: PathBuf) -> Result<PluginEntity, InitError> {
     Ok(valid_plugin)
 }
 
+/// Represents Plugins State and the corresponding infos.
 #[derive(Debug, Clone)]
 enum PluginValidationState {
+    /// Represents valid plugin with its infos and metadata.
     Valid {
-        wasm: PathBuf,
+        /// The path for the plugin wasm file.
+        wasm_path: PathBuf,
+        /// Metadata of the plugins found in plugins metadata toml file.
         metadata: Option<PathBuf>,
     },
+    /// Represents an invalid plugin with infos about validation error.
     Invalid {
+        /// Error message explaining why the plugin is invalid.
         err_msg: String,
     },
 }
 
+/// Loads the files inside plugin directory and validate their content return the state
+/// of the plugin.
 fn validate_plugin_files(dir: &PathBuf) -> Result<PluginValidationState, InitError> {
     use PluginValidationState as Re;
     let mut wasm_file = None;
@@ -172,7 +182,7 @@ fn validate_plugin_files(dir: &PathBuf) -> Result<PluginValidationState, InitErr
 
     let res = match wasm_file {
         Some(wasm) => Re::Valid {
-            wasm,
+            wasm_path: wasm,
             metadata: metadata_file,
         },
         None => {
@@ -185,13 +195,15 @@ fn validate_plugin_files(dir: &PathBuf) -> Result<PluginValidationState, InitErr
     Ok(res)
 }
 
+/// Parser the plugin metadata from the provided toml file.
 fn parse_metadata(file: &PathBuf) -> Result<PluginMetadata, String> {
     let content = read_to_string(file)
         .map_err(|err| format!("Reading metadata file fail. Error {err:#?}"))?;
 
     toml::from_str(&content).map_err(|err| format!("Parsing metadata file fail. Error {err:#?}"))
 }
 
+/// Loads all byte-source plugins from their main directory.
 async fn load_all_bytesources() -> Result<Vec<PluginEntity>, InitError> {
     let mut bytesources = Vec::new();
 
@@ -211,9 +223,13 @@ async fn load_all_bytesources() -> Result<Vec<PluginEntity>, InitError> {
     Ok(bytesources)
 }
 
+/// Loads byte-source infos and metadata from the provided parser directory.
 async fn load_bytesource(dir: PathBuf) -> Result<PluginEntity, InitError> {
     let (wasm_file, metadata_file) = match validate_plugin_files(&dir)? {
-        PluginValidationState::Valid { wasm, metadata } => (wasm, metadata),
+        PluginValidationState::Valid {
+            wasm_path: wasm,
+            metadata,
+        } => (wasm, metadata),
         PluginValidationState::Invalid { err_msg } => {
             let invalid_entity = PluginEntity {
                 dir_path: dir,

application/apps/indexer/plugins_host/src/plugins_manager/load/mod.rs

@@ -1,11 +1,14 @@
-//TODO: Remove after prototyping
-#![allow(unused)]
 use std::path::PathBuf;
 
 use crate::plugins_manager::InitError;
 
+/// The name of the plugins directory in Chipmunk home directory.
 const PLUGINS_DIR: &str = "plugins";
+
+/// The name of the parser plugins directory in plugins directory.
 const PARSER_DIR: &str = "parsers";
+
+/// The name of the byte-source plugins directory in plugins directory.
 const BYTESOURCE_DIR: &str = "bytesources";
 
 /// Copied from `session/src/paths.rs`
@@ -17,14 +20,17 @@ pub fn get_home_dir() -> Result<PathBuf, InitError> {
         .ok_or_else(|| InitError::Other(String::from("Failed to find home directory")))
 }
 
+/// Return plugins directory in Chipmunk home directory.
 pub fn plugins_dir() -> Result<PathBuf, InitError> {
     get_home_dir().map(|home| home.join(PLUGINS_DIR))
 }
 
+/// Returns parser plugins directory within Chipmunk home directory.
 pub fn parser_dir() -> Result<PathBuf, InitError> {
     plugins_dir().map(|plugins| plugins.join(PARSER_DIR))
 }
 
+/// Returns byte-source plugins directory within Chipmunk home directory.
 pub fn bytesource_dir() -> Result<PathBuf, InitError> {
     plugins_dir().map(|plugins| plugins.join(BYTESOURCE_DIR))
 }