[inventory] Add unhealthy zpools from each sled

common/src/lib.rs

@@ -26,6 +26,7 @@ pub mod disk;
 pub mod ledger;
 pub mod policy;
 pub mod progenitor_operation_retry;
+pub mod snake_case_option_result;
 pub mod snake_case_result;
 pub mod update;
 pub mod vlan;

common/src/snake_case_option_result.rs

@@ -0,0 +1,114 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! A serializable Option<Result> that plays nicely with OpenAPI lints.
+
+use crate::snake_case_result::SnakeCaseResult;
+use schemars::JsonSchema;
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(rename = "OptionResult{T}Or{E}")]
+pub enum SnakeCaseOptionResult<T, E> {
+    Some(SnakeCaseResult<T, E>),
+    None,
+}
+
+impl<T, E> JsonSchema for SnakeCaseOptionResult<T, E>
+where
+    T: JsonSchema,
+    E: JsonSchema,
+{
+    fn schema_name() -> String {
+        format!("OptionResult{}Or{}", T::schema_name(), E::schema_name())
+    }
+
+    fn json_schema(
+        generator: &mut schemars::r#gen::SchemaGenerator,
+    ) -> schemars::schema::Schema {
+        let mut ok_schema = schemars::schema::SchemaObject {
+            instance_type: Some(schemars::schema::InstanceType::Object.into()),
+            ..Default::default()
+        };
+        let obj = ok_schema.object();
+        obj.required.insert("ok".to_owned());
+        obj.properties.insert("ok".to_owned(), generator.subschema_for::<T>());
+
+        let mut err_schema = schemars::schema::SchemaObject {
+            instance_type: Some(schemars::schema::InstanceType::Object.into()),
+            ..Default::default()
+        };
+        let obj = err_schema.object();
+        obj.required.insert("err".to_owned());
+        obj.properties.insert("err".to_owned(), generator.subschema_for::<E>());
+
+        let mut schema = schemars::schema::SchemaObject::default();
+        schema.subschemas().one_of =
+            Some(vec![ok_schema.into(), err_schema.into()]);
+
+        schema
+            .extensions
+            .insert(String::from("nullable"), serde_json::json!(true));
+        schema.extensions.insert(
+            String::from("x-rust-type"),
+            serde_json::json!({
+                "crate": "std",
+                "version": "*",
+                "path": "::std::result::Result",
+                "parameters": [
+                    generator.subschema_for::<T>(),
+                    generator.subschema_for::<E>(),
+                ],
+            }),
+        );
+        schema.into()
+    }
+}
+
+/// Serialize an Option<Result<T, E>> as a `SnakeCaseOptionResult`.
+pub fn serialize<S, T, E>(
+    value: &Option<Result<T, E>>,
+    serializer: S,
+) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+    T: Serialize,
+    E: Serialize,
+{
+    match value {
+        None => serializer.serialize_none(),
+        Some(Ok(val)) => {
+            SnakeCaseOptionResult::<&T, &E>::Some(SnakeCaseResult::Ok(val))
+                .serialize(serializer)
+        }
+        Some(Err(err)) => {
+            SnakeCaseOptionResult::<&T, &E>::Some(SnakeCaseResult::Err(err))
+                .serialize(serializer)
+        }
+    }
+}
+
+/// Deserialize a `SnakeCaseOptionResult` into an Option<Result>.
+pub fn deserialize<'de, D, T, E>(
+    deserializer: D,
+) -> Result<Option<Result<T, E>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: Deserialize<'de>,
+    E: Deserialize<'de>,
+{
+    SnakeCaseOptionResult::<T, E>::deserialize(deserializer).map(|snek| {
+        match snek {
+            SnakeCaseOptionResult::Some(SnakeCaseResult::Ok(val)) => {
+                Some(Ok(val))
+            }
+            SnakeCaseOptionResult::Some(SnakeCaseResult::Err(err)) => {
+                Some(Err(err))
+            }
+            SnakeCaseOptionResult::None => None,
+        }
+    })
+}

illumos-utils/src/svcs.rs

@@ -121,7 +121,7 @@ impl SvcsInMaintenanceResult {
                             ));
                             error!(
                                 log,
-                                "unable to parse; output line missing zone:";
+                                "unable to parse; output line missing zone";
                                 "line" => line,
                             );
                             continue;

illumos-utils/src/zpool.rs

@@ -6,6 +6,14 @@
 
 use crate::{ExecutionError, PFEXEC, execute_async};
 use camino::{Utf8Path, Utf8PathBuf};
+use chrono::DateTime;
+use chrono::Utc;
+use schemars::JsonSchema;
+use serde::Deserialize;
+use serde::Serialize;
+use slog::Logger;
+use slog::error;
+use slog::info;
 use std::str::FromStr;
 use tokio::process::Command;
 
@@ -60,7 +68,10 @@ pub struct GetInfoError {
     err: Error,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(
+    Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema,
+)]
+#[serde(rename_all = "snake_case")]
 pub enum ZpoolHealth {
     /// The device is online and functioning.
     Online,
@@ -198,6 +209,82 @@ pub struct PathInPool {
     pub path: Utf8PathBuf,
 }
 
+/// Lists unhealthy zpools, parsing errors if any, and the time the health check
+/// for zpools ran.
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub struct UnhealthyZpoolsResult {
+    pub zpools: Vec<String>,
+    pub errors: Vec<String>,
+    pub time_of_status: DateTime<Utc>,
+}
+
+impl UnhealthyZpoolsResult {
+    pub fn new() -> Self {
+        Self { zpools: vec![], errors: vec![], time_of_status: Utc::now() }
+    }
+
+    #[cfg_attr(not(target_os = "illumos"), allow(dead_code))]
+    fn parse(log: &Logger, data: &[u8]) -> Self {
+        let mut zpools = vec![];
+        let mut errors = vec![];
+        if data.is_empty() {
+            return Self { zpools, errors, time_of_status: Utc::now() };
+        }
+
+        // Example of the response from running `zpool list -Hpo health,name`
+        //
+        // FAULTED fakepool1
+        // FAULTED fakepool2
+        // ONLINE  rpool
+        let s = String::from_utf8_lossy(data);
+        let lines = s.trim().lines();
+
+        for line in lines {
+            let line = line.trim();
+            let mut pool = line.split_whitespace();
+
+            if let Some(state_str) = pool.next() {
+                // Only attempt to parse a zpool that is in a non-functional
+                // state.
+                match ZpoolHealth::from_str(state_str) {
+                    Ok(ZpoolHealth::Faulted)
+                    | Ok(ZpoolHealth::Degraded)
+                    | Ok(ZpoolHealth::Offline)
+                    | Ok(ZpoolHealth::Removed)
+                    | Ok(ZpoolHealth::Unavailable) => {
+                        if let Some(name) = pool.next() {
+                            zpools.push(name.to_string());
+                        } else {
+                            errors.push(format!(
+                                "Unexpected output line: {line}"
+                            ));
+                            error!(
+                                log,
+                                "unable to parse; output line missing zpool name";
+                                "line" => line,
+                            );
+                            continue;
+                        }
+                    }
+                    // Pool is in a healthy state, skip it.
+                    Ok(ZpoolHealth::Online) => {}
+                    Err(e) => {
+                        errors.push(format!("{e}"));
+                        info!(
+                            log,
+                            "output from 'zpool list' contains a zpool with \
+                            an unknown state: {state_str}",
+                        );
+                    }
+                }
+            }
+        }
+
+        Self { zpools, errors, time_of_status: Utc::now() }
+    }
+}
+
 /// Wraps commands for interacting with ZFS pools.
 pub struct Zpool(());
 
@@ -330,11 +417,133 @@ impl Zpool {
         })?;
         Ok(zpool)
     }
+
+    /// Lists zpools that are in a unhealthy non-functional state. Specifically
+    /// if they are in the following states:
+    ///
+    ///  - Faulted
+    ///  - Offline
+    ///  - Removed
+    ///  - Unavailable
+    #[cfg(target_os = "illumos")]
+    pub async fn status_unhealthy(
+        log: &Logger,
+    ) -> Result<UnhealthyZpoolsResult, ExecutionError> {
+        let mut command = Command::new(ZPOOL);
+        let cmd = command.args(&["list", "-Hpo", "health,name"]);
+        info!(log, "Retrieving information from zpools");
+        let output = execute_async(cmd).await?;
+        let zpool_result = UnhealthyZpoolsResult::parse(&log, &output.stdout);
+        info!(log, "Successfully retrieved unhealthy zpools");
+        Ok(zpool_result)
+    }
+
+    #[cfg(not(target_os = "illumos"))]
+    pub async fn status_unhealthy(
+        log: &Logger,
+    ) -> Result<UnhealthyZpoolsResult, ExecutionError> {
+        info!(log, "OS not illumos, will not retrieve zpool information");
+        let zpool_result = UnhealthyZpoolsResult::new();
+        Ok(zpool_result)
+    }
 }
 
 #[cfg(test)]
 mod test {
     use super::*;
+    use slog::Drain;
+    use slog::o;
+    use slog_term::FullFormat;
+    use slog_term::PlainDecorator;
+    use slog_term::TestStdoutWriter;
+
+    fn log() -> slog::Logger {
+        let decorator = PlainDecorator::new(TestStdoutWriter);
+        let drain = FullFormat::new(decorator).build().fuse();
+        let drain = slog_async::Async::new(drain).build().fuse();
+        slog::Logger::root(drain, o!())
+    }
+
+    #[test]
+    fn test_unhealthy_zpool_parse_success() {
+        let output = r#"FAULTED fakepool1
+UNAVAIL fakepool2
+ONLINE  rpool
+"#;
+
+        let log = log();
+        let result = UnhealthyZpoolsResult::parse(&log, output.as_bytes());
+
+        // We want to make sure we only have two unhealthy pools
+        assert_eq!(
+            result.zpools,
+            vec!["fakepool1".to_string(), "fakepool2".to_string()]
+        );
+        assert_eq!(result.errors.len(), 0);
+    }
+
+    #[test]
+    fn test_unhealthy_zpool_parse_none_success() {
+        let output = r#"ONLINE fakepool1
+ONLINE   fakepool2
+ONLINE   rpool
+"#;
+
+        let log = log();
+        let result = UnhealthyZpoolsResult::parse(&log, output.as_bytes());
+
+        // We want to make sure we only have zero unhealthy pools
+        assert_eq!(result.zpools.len(), 0);
+        assert_eq!(result.errors.len(), 0);
+    }
+
+    #[test]
+    fn test_unhealthy_zpool_empty_success() {
+        let output = r#""#;
+
+        let log = log();
+        let result = UnhealthyZpoolsResult::parse(&log, output.as_bytes());
+
+        // We want to make sure we have zero unhealthy pools
+        assert_eq!(result.zpools.len(), 0);
+        assert_eq!(result.errors.len(), 0);
+    }
+
+    #[test]
+    fn test_unhealthy_zpool_parse_unknown_status_fail() {
+        let output = r#"BARNACLES! fakepool1
+FAULTED fakepool2
+ONLINE  rpool
+"#;
+
+        let log = log();
+        let result = UnhealthyZpoolsResult::parse(&log, output.as_bytes());
+
+        assert_eq!(result.zpools, vec!["fakepool2".to_string()]);
+        assert_eq!(
+            result.errors,
+            vec![
+                "Failed to parse output: Unrecognized zpool 'health': BARNACLES!"
+                .to_string(),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_unhealthy_zpool_parse_zpool_fail() {
+        let output = r#"FAULTED
+ONLINE  rpool
+"#;
+
+        let log = log();
+        let result = UnhealthyZpoolsResult::parse(&log, output.as_bytes());
+
+        assert_eq!(result.zpools.len(), 0);
+        assert_eq!(
+            result.errors,
+            vec!["Unexpected output line: FAULTED".to_string(),],
+        );
+    }
 
     #[test]
     fn test_parse_zpool() {