From 85362d9ea091ae606518698be308b6e53532aa99 Mon Sep 17 00:00:00 2001 From: Shawn Hartsock Date: Tue, 9 Jun 2026 15:22:46 -0400 Subject: [PATCH 1/2] A2: board card markdown round-trip + directory sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WHAT: Add board_md (card_to_markdown / card_from_markdown) and the Store::import_dir / export_dir directory sync, so the operational SQLite cards round-trip to/from the portable markdown+frontmatter form. - board_md.rs: emits the canonical frontmatter field order (id, project, created, updated, summary, size, status, recurs, expires, blocked_on, refs, author/source) + body; parses tolerantly (free-text status, trailing `# comments` on list items, a `blocked_by` alias for blocked_on, refs label:url map). lane/context are directory facts, left empty by the parser and filled by the walker. - Store::import_dir walks /[/]/*.md (lanes p0|p1|p2|done|dropped), follows symlinks (the lane-view convention) but dedups by card_id so a source file and its lane symlink import once; returns {added, updated, skipped}. - Store::export_dir writes flat //.md real files (non-destructive; never deletes files it didn't write). - BoardConfig gains sync_dir / default_lane / default_context (all #[serde(default)], existing configs parse unchanged). - New dependency: serde_yaml_ng (maintained drop-in for the deprecated serde_yaml) — parses the frontmatter map + list cleanly. WHY: Markdown stays the portable, sovereign artifact (no lock-in); SQLite is the fast operational store. v1 round-trip is at the card-content level (frontmatter + body survive SQLite -> md -> SQLite) — it does NOT reconstruct the numbered relative-symlink lane views; that's a follow-up. Disclosure tier: internal capability + one new dependency. No MCP surface change, no listed step yet. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 20 ++ Cargo.toml | 4 + crates/modulex-core/Cargo.toml | 1 + crates/modulex-core/src/board_md.rs | 378 ++++++++++++++++++++++++++++ crates/modulex-core/src/config.rs | 14 +- crates/modulex-core/src/lib.rs | 2 + crates/modulex-core/src/store.rs | 148 +++++++++++ 7 files changed, 565 insertions(+), 2 deletions(-) create mode 100644 crates/modulex-core/src/board_md.rs diff --git a/Cargo.lock b/Cargo.lock index 283c7a0..7e17637 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1507,6 +1507,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "serde_yaml_ng", "shell-words", "thiserror 2.0.18", "tokio", @@ -2422,6 +2423,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "servo_arc" version = "0.4.3" @@ -2917,6 +2931,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 7f91e7a..9df69f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,10 @@ clap = { version = "4", features = ["derive"] } rusqlite = { version = "0.32", features = ["bundled"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +# Maintained drop-in for the deprecated/unmaintained serde_yaml — parses the +# knowledge-board card frontmatter (refs map + blocked_on list) for the +# markdown round-trip (board_md). +serde_yaml_ng = "0.10" shell-words = "1.1" thiserror = "2.0" tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "time", "io-util", "io-std"] } diff --git a/crates/modulex-core/Cargo.toml b/crates/modulex-core/Cargo.toml index 7ab3023..480ef57 100644 --- a/crates/modulex-core/Cargo.toml +++ b/crates/modulex-core/Cargo.toml @@ -27,6 +27,7 @@ async-trait = { workspace = true } chrono = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +serde_yaml_ng = { workspace = true } shell-words = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/modulex-core/src/board_md.rs b/crates/modulex-core/src/board_md.rs new file mode 100644 index 0000000..2c7b262 --- /dev/null +++ b/crates/modulex-core/src/board_md.rs @@ -0,0 +1,378 @@ +//! Markdown round-trip for board cards. +//! +//! SQLite is the operational store; the portable, sovereign form of a card is +//! a markdown file with YAML frontmatter: +//! +//! ```text +//! --- +//! id: homelab-2026-06-09-vpn-cert +//! project: homelab +//! created: 2026-06-09 +//! summary: Renew the VPN certificate +//! size: 1d +//! status: blocked +//! blocked_on: +//! - https://example.com/issues/1 # waiting on upstream +//! refs: +//! issue: https://example.com/issues/42 +//! --- +//! +//! # Body markdown … +//! ``` +//! +//! [`card_from_markdown`] is tolerant of real-world variance (free-text +//! `status`, trailing `# comments` on list items — stripped by the YAML +//! parser, a `blocked_by` alias for `blocked_on`). `lane` and `context` are +//! NOT frontmatter — they are directory facts, so the parser leaves them empty +//! and the directory walker ([`crate::store::Store::import_dir`]) fills them in. + +use crate::store::{Card, CardInput, CardRef}; + +/// Errors from parsing a card's markdown. +#[derive(Debug, thiserror::Error)] +pub enum BoardMdError { + /// No `--- … ---` frontmatter block. + #[error("board card: missing or malformed YAML frontmatter (--- ... ---)")] + MissingFrontmatter, + /// Frontmatter parsed but had no `id`. + #[error("board card: frontmatter is missing the required `id`")] + MissingId, + /// The frontmatter was not valid YAML. + #[error("board card: {0}")] + Yaml(#[from] serde_yaml_ng::Error), +} + +/// Render a card to its `--- frontmatter --- + body` markdown form, emitting +/// the canonical knowledge-repo field order. `lane`/`context` are omitted +/// (they are encoded by where the file is written). +#[must_use] +pub fn card_to_markdown(card: &Card) -> String { + let mut out = String::from("---\n"); + push_scalar(&mut out, "id", Some(&card.card_id)); + push_scalar(&mut out, "project", non_empty(&card.project)); + push_scalar(&mut out, "created", card.created.as_deref()); + push_scalar(&mut out, "updated", card.updated.as_deref()); + push_scalar(&mut out, "summary", non_empty(&card.summary)); + push_scalar(&mut out, "size", card.size.as_deref()); + push_scalar(&mut out, "status", card.status.as_deref()); + push_scalar(&mut out, "recurs", card.recurs.as_deref()); + push_scalar(&mut out, "expires", card.expires.as_deref()); + + let blocked: Vec<&CardRef> = card + .refs + .iter() + .filter(|r| r.kind == "blocked_on") + .collect(); + if !blocked.is_empty() { + out.push_str("blocked_on:\n"); + for b in blocked { + out.push_str(&format!(" - {}\n", yaml_scalar(&b.value))); + } + } + let refs: Vec<&CardRef> = card.refs.iter().filter(|r| r.kind == "ref").collect(); + if !refs.is_empty() { + out.push_str("refs:\n"); + for r in refs { + out.push_str(&format!(" {}: {}\n", r.label, yaml_scalar(&r.value))); + } + } + + push_scalar(&mut out, "author", card.author.as_deref()); + push_scalar(&mut out, "source", card.source.as_deref()); + push_scalar(&mut out, "source_id", card.source_id.as_deref()); + out.push_str("---\n"); + + if !card.body.is_empty() { + out.push('\n'); + out.push_str(&card.body); + if !card.body.ends_with('\n') { + out.push('\n'); + } + } + out +} + +/// Parse a card markdown file into a [`CardInput`]. `lane`/`context` are left +/// empty for the caller to fill from the file's directory position. +/// +/// # Errors +/// [`BoardMdError`] when the frontmatter is missing, malformed, or has no `id`. +pub fn card_from_markdown(text: &str) -> Result { + let (frontmatter, body) = split_frontmatter(text)?; + let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(&frontmatter)?; + let map = value.as_mapping().ok_or(BoardMdError::MissingFrontmatter)?; + + let get = |key: &str| map.get(key).and_then(scalar_to_string); + let card_id = get("id").ok_or(BoardMdError::MissingId)?; + + let mut refs = Vec::new(); + if let Some(blocked) = map.get("blocked_on").or_else(|| map.get("blocked_by")) { + for (i, value) in normalize_seq(blocked).into_iter().enumerate() { + refs.push(CardRef { + kind: "blocked_on".into(), + label: String::new(), + value, + ordinal: i as i64, + }); + } + } + if let Some(refs_map) = map.get("refs").and_then(serde_yaml_ng::Value::as_mapping) { + for (k, v) in refs_map { + if let (Some(label), Some(value)) = (k.as_str(), scalar_to_string(v)) { + refs.push(CardRef { + kind: "ref".into(), + label: label.to_string(), + value, + ordinal: 0, + }); + } + } + } + + Ok(CardInput { + card_id, + project: get("project").unwrap_or_default(), + lane: String::new(), + context: String::new(), + summary: get("summary").unwrap_or_default(), + size: get("size"), + status: get("status"), + recurs: get("recurs"), + expires: get("expires"), + created: get("created"), + updated: get("updated"), + body, + author: get("author"), + source: get("source"), + source_id: get("source_id"), + refs, + }) +} + +/// Split off the leading `--- … ---` frontmatter block; returns +/// `(frontmatter_yaml, body)`. +fn split_frontmatter(text: &str) -> Result<(String, String), BoardMdError> { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut lines = text.lines(); + if lines.next().map(str::trim_end) != Some("---") { + return Err(BoardMdError::MissingFrontmatter); + } + let mut frontmatter = String::new(); + let mut body_lines: Vec<&str> = Vec::new(); + let mut closed = false; + for line in lines { + if !closed && line.trim_end() == "---" { + closed = true; + continue; + } + if closed { + body_lines.push(line); + } else { + frontmatter.push_str(line); + frontmatter.push('\n'); + } + } + if !closed { + return Err(BoardMdError::MissingFrontmatter); + } + let body = body_lines.join("\n").trim_start_matches('\n').to_string(); + Ok((frontmatter, body)) +} + +/// A YAML scalar can be `value`, or a list of values; normalize to a `Vec`. +fn normalize_seq(value: &serde_yaml_ng::Value) -> Vec { + match value { + serde_yaml_ng::Value::Sequence(items) => { + items.iter().filter_map(scalar_to_string).collect() + } + other => scalar_to_string(other).into_iter().collect(), + } +} + +/// Stringify a scalar YAML value (string/number/bool); `None` for collections. +fn scalar_to_string(value: &serde_yaml_ng::Value) -> Option { + match value { + serde_yaml_ng::Value::String(s) => Some(s.clone()), + serde_yaml_ng::Value::Number(n) => Some(n.to_string()), + serde_yaml_ng::Value::Bool(b) => Some(b.to_string()), + _ => None, + } +} + +fn non_empty(s: &str) -> Option<&str> { + (!s.is_empty()).then_some(s) +} + +fn push_scalar(out: &mut String, key: &str, value: Option<&str>) { + if let Some(v) = value { + out.push_str(&format!("{key}: {}\n", yaml_scalar(v))); + } +} + +/// Emit a YAML-safe scalar: plain when unambiguous, else double-quoted. Keeps +/// URLs (which contain `:` but not `: `) unquoted, matching the source format. +fn yaml_scalar(s: &str) -> String { + let needs_quote = s.is_empty() + || s.contains(": ") + || s.contains(" #") + || s.ends_with(':') + || s.starts_with(' ') + || s.ends_with(' ') + || s.starts_with(|c: char| "!&*?|>%@`\"'#,[]{}".contains(c)) + || s.starts_with("- "); + if needs_quote { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } else { + s.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // A synthetic but format-faithful fixture (no real board content). + const FIXTURE: &str = r#"--- +id: homelab-2026-06-09-vpn-cert +project: homelab +created: 2026-06-09 +updated: 2026-06-09 +summary: "Renew VPN cert: blocked on upstream" +size: 1d +status: blocked +blocked_on: + - https://example.com/issues/1 # waiting on upstream + - https://example.com/issues/2 +refs: + issue: https://example.com/issues/42 + doc: docs/vpn.md +--- + +# Renew the VPN certificate + +Steps go here. +"#; + + fn card_of(input: CardInput, id: i64) -> Card { + Card { + id, + card_id: input.card_id, + project: input.project, + lane: input.lane, + context: input.context, + summary: input.summary, + size: input.size, + status: input.status, + recurs: input.recurs, + expires: input.expires, + created: input.created, + updated: input.updated, + body: input.body, + author: input.author, + source: input.source, + source_id: input.source_id, + created_gen: 1, + updated_gen: 1, + closed_gen: None, + refs: input.refs, + } + } + + #[test] + fn parses_real_world_fixture() { + let input = card_from_markdown(FIXTURE).unwrap(); + assert_eq!(input.card_id, "homelab-2026-06-09-vpn-cert"); + assert_eq!(input.project, "homelab"); + assert_eq!(input.summary, "Renew VPN cert: blocked on upstream"); + assert_eq!(input.size.as_deref(), Some("1d")); + assert_eq!(input.status.as_deref(), Some("blocked")); + assert_eq!(input.created.as_deref(), Some("2026-06-09")); + assert!(input.lane.is_empty(), "lane is a directory fact"); + assert!(input.body.starts_with("# Renew the VPN certificate")); + + let blocked: Vec<_> = input + .refs + .iter() + .filter(|r| r.kind == "blocked_on") + .collect(); + assert_eq!(blocked.len(), 2); + assert_eq!( + blocked[0].value, "https://example.com/issues/1", + "trailing # comment is stripped" + ); + assert_eq!(blocked[0].ordinal, 0); + assert_eq!(blocked[1].ordinal, 1); + + let refs: Vec<_> = input.refs.iter().filter(|r| r.kind == "ref").collect(); + assert_eq!(refs.len(), 2); + assert_eq!(refs[0].label, "issue"); + } + + #[test] + fn round_trips_semantically() { + let from_fixture = card_from_markdown(FIXTURE).unwrap(); + let markdown = card_to_markdown(&card_of(from_fixture.clone(), 1)); + let reparsed = card_from_markdown(&markdown).unwrap(); + + assert_eq!(reparsed.card_id, from_fixture.card_id); + assert_eq!( + reparsed.summary, from_fixture.summary, + "colon summary survives" + ); + assert_eq!(reparsed.status, from_fixture.status); + assert_eq!(reparsed.created, from_fixture.created); + assert_eq!(reparsed.body, from_fixture.body); + assert_eq!( + reparsed.refs, from_fixture.refs, + "refs + blocked_on survive" + ); + } + + #[test] + fn blocked_by_alias_and_freetext_status() { + let text = r#"--- +id: x-1 +status: scoped — not started +blocked_by: a free-text reason +summary: s +--- + +body +"#; + let input = card_from_markdown(text).unwrap(); + assert_eq!(input.status.as_deref(), Some("scoped — not started")); + let blocked: Vec<_> = input + .refs + .iter() + .filter(|r| r.kind == "blocked_on") + .collect(); + assert_eq!(blocked.len(), 1, "blocked_by aliases blocked_on"); + assert_eq!(blocked[0].value, "a free-text reason"); + } + + #[test] + fn missing_frontmatter_is_an_error() { + assert!(matches!( + card_from_markdown("# just a heading\n"), + Err(BoardMdError::MissingFrontmatter) + )); + } + + #[test] + fn missing_id_is_an_error() { + assert!(matches!( + card_from_markdown("---\nproject: x\n---\nbody\n"), + Err(BoardMdError::MissingId) + )); + } + + #[test] + fn yaml_scalar_quotes_only_when_needed() { + assert_eq!( + yaml_scalar("https://example.com/x"), + "https://example.com/x" + ); + assert_eq!(yaml_scalar("plain text"), "plain text"); + assert_eq!(yaml_scalar("has: colon"), "\"has: colon\""); + } +} diff --git a/crates/modulex-core/src/config.rs b/crates/modulex-core/src/config.rs index aa5cc51..a6241e9 100644 --- a/crates/modulex-core/src/config.rs +++ b/crates/modulex-core/src/config.rs @@ -143,12 +143,22 @@ pub struct SharedConfig { /// Board/lane scan configuration. #[derive(Clone, Debug, Default, Deserialize)] pub struct BoardConfig { - /// Board root directory. + /// Board root directory (the filesystem `board-scan` step). #[serde(default)] pub path: String, - /// Lane subdirectories to scan. + /// Lane subdirectories to scan (the filesystem `board-scan` step). #[serde(default)] pub lanes: Vec, + /// Board directory the store-backed card model syncs to/from + /// (`import_dir`/`export_dir`). May equal `path`; empty = no dir sync. + #[serde(default)] + pub sync_dir: String, + /// Default lane for new cards when unspecified (falls back to `p2`). + #[serde(default)] + pub default_lane: String, + /// Default context for new cards; empty = board root. + #[serde(default)] + pub default_context: String, } /// Chores manifest location. diff --git a/crates/modulex-core/src/lib.rs b/crates/modulex-core/src/lib.rs index c2084fc..5c7b914 100644 --- a/crates/modulex-core/src/lib.rs +++ b/crates/modulex-core/src/lib.rs @@ -23,6 +23,7 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] +pub mod board_md; pub mod caveats; pub mod config; pub mod credentials; @@ -34,6 +35,7 @@ pub mod step; pub mod steps; pub mod store; +pub use board_md::{card_from_markdown, card_to_markdown, BoardMdError}; pub use caveats::{CaveatsSource, GrantedCaveats}; pub use config::{Config, RoutineSpec, StepSpec}; pub use credentials::{CredentialRef, Secret}; diff --git a/crates/modulex-core/src/store.rs b/crates/modulex-core/src/store.rs index 48ee039..4da1be3 100644 --- a/crates/modulex-core/src/store.rs +++ b/crates/modulex-core/src/store.rs @@ -949,8 +949,112 @@ impl Store { } Ok(()) } + + // ── board directory sync (markdown <-> cards) ────────────────────── + + /// Import a board directory tree into the cards table. Walks + /// `/[/]/*.md` (lanes `p0|p1|p2|done|dropped`), + /// parses each file, derives lane/context from the path, and upserts by + /// `card_id`. Follows symlinks (the lane-view convention) but dedups by + /// `card_id`, so a source file and its lane symlink import once. + /// + /// # Errors + /// [`StoreError`] on SQLite failure (parse/read errors are counted as + /// `skipped`, never fatal). + pub fn import_dir(&self, root: &Path, generation: u64) -> Result { + let mut report = ImportReport::default(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + + // (context, lane, dir) + let mut lane_dirs: Vec<(String, String, PathBuf)> = Vec::new(); + if let Ok(entries) = std::fs::read_dir(root) { + for entry in entries.filter_map(Result::ok) { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let name = entry.file_name().to_string_lossy().into_owned(); + if BOARD_LANES.contains(&name.as_str()) { + lane_dirs.push((String::new(), name, path)); + } else if let Ok(sub) = std::fs::read_dir(&path) { + for s in sub.filter_map(Result::ok) { + let sp = s.path(); + let sname = s.file_name().to_string_lossy().into_owned(); + if sp.is_dir() && BOARD_LANES.contains(&sname.as_str()) { + lane_dirs.push((name.clone(), sname, sp)); + } + } + } + } + } + lane_dirs.sort(); + + for (context, lane, dir) in lane_dirs { + let mut files: Vec = std::fs::read_dir(&dir) + .map(|es| { + es.filter_map(Result::ok) + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "md")) + .collect() + }) + .unwrap_or_default(); + files.sort(); + for path in files { + let Ok(text) = std::fs::read_to_string(&path) else { + report.skipped += 1; + continue; + }; + let mut input = match crate::board_md::card_from_markdown(&text) { + Ok(input) => input, + Err(_) => { + report.skipped += 1; + continue; + } + }; + if !seen.insert(input.card_id.clone()) { + report.skipped += 1; // a source file and its symlink + continue; + } + input.lane.clone_from(&lane); + input.context.clone_from(&context); + let existed = self.card_by_card_id(&input.card_id)?.is_some(); + self.card_add(&input, generation)?; + if existed { + report.updated += 1; + } else { + report.added += 1; + } + } + } + Ok(report) + } + + /// Export every card to `///.md` as flat real + /// files (not symlink lane-views). Non-destructive: writes/overwrites the + /// files it owns, never deletes others. Returns the number written. + /// + /// # Errors + /// [`StoreError::Io`] when a directory or file cannot be written. + pub fn export_dir(&self, root: &Path) -> Result { + let cards = self.cards_query(None, None, None)?; + for card in &cards { + let dir = if card.context.is_empty() { + root.join(&card.lane) + } else { + root.join(&card.context).join(&card.lane) + }; + std::fs::create_dir_all(&dir).map_err(|e| StoreError::Io(dir.clone(), e))?; + let file = dir.join(format!("{}.md", card.card_id)); + let markdown = crate::board_md::card_to_markdown(card); + std::fs::write(&file, markdown).map_err(|e| StoreError::Io(file.clone(), e))?; + } + Ok(cards.len()) + } } +/// The recognized board lanes, in priority order. +const BOARD_LANES: &[&str] = &["p0", "p1", "p2", "done", "dropped"]; + /// Column list for `cards` SELECTs, matching [`row_to_card`]'s field order. const CARD_COLS: &str = "rowid_id, card_id, project, lane, context, summary, size, status, \ recurs, expires, created, updated, body, author, source, source_id, \ @@ -1467,4 +1571,48 @@ mod tests { ); assert_eq!(b.card_by_card_id("c-1").unwrap().unwrap().refs.len(), 1); } + + #[test] + fn dir_sync_round_trips() { + let root = std::env::temp_dir().join(format!( + "modulex-board-dir-{}-{}", + std::process::id(), + COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + // Lay out a board: a context with a lane and a card. + let lane = root.join("homelab").join("p0"); + std::fs::create_dir_all(&lane).unwrap(); + std::fs::write( + lane.join("vpn.md"), + "---\nid: homelab-2026-06-09-vpn\nproject: homelab\nsummary: renew cert\nrefs:\n issue: https://example.com/1\n---\n\nbody\n", + ) + .unwrap(); + + let a = Store::in_memory().unwrap(); + let report = a.import_dir(&root, 1).unwrap(); + assert_eq!(report.added, 1); + let cards = a.cards_in_lane("p0", Some("homelab")).unwrap(); + assert_eq!(cards.len(), 1); + assert_eq!(cards[0].card_id, "homelab-2026-06-09-vpn"); + assert_eq!(cards[0].refs.len(), 1); + + // Export to a second tree, re-import into a fresh store: equal card set. + let out = root.join("export"); + a.export_dir(&out).unwrap(); + assert!(out + .join("homelab") + .join("p0") + .join("homelab-2026-06-09-vpn.md") + .is_file()); + + let b = Store::in_memory().unwrap(); + b.import_dir(&out, 1).unwrap(); + let bcards = b.cards_query(None, None, None).unwrap(); + assert_eq!(bcards.len(), 1); + assert_eq!(bcards[0].lane, "p0"); + assert_eq!(bcards[0].context, "homelab"); + assert_eq!(bcards[0].summary, "renew cert"); + + std::fs::remove_dir_all(&root).ok(); + } } From 15f18c3e9bd8e9eba28181eb5fa4a5340f099b21 Mon Sep 17 00:00:00 2001 From: Shawn Hartsock Date: Tue, 9 Jun 2026 15:25:37 -0400 Subject: [PATCH 2/2] A3: store-backed `board` step for the morning report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WHAT: Add a `board` step type that reads open knowledge-board cards from the agent state store and renders them grouped by lane — the operational counterpart of the filesystem `board-scan` step (which stays as-is). - steps/board::Board: pure step (no subprocess), soft-skips without a store. Params (optional): `lane` (a single lane), `project` (filter). With no `lane`, shows the open work lanes p0/p1/p2 (closed lanes only on request). Flags cards carrying blocked_on refs as `[blocked]`. - data_schema(): {lanes:[{lane, cards:[{card_id, project, summary, status, size, blocked}]}], open} — a new versioned contract (golden pinned). - Registered in builtin_registry; documented-types test + module table updated. board-scan vs board distinction documented (filesystem vs store). - data-contract harness: the `board` step joins CONTRACT_CONFIG and the golden set; a seeded card proves executed output validates against the schema. WHY: The store-backed board is what makes "good morning" surface the local knowledge board with no git or filesystem layout required — cards live in SQLite (synced from/to markdown via A2). Preferred disclosure path per FOUNDATION: a read-only step, zero MCP tools/list cost. Disclosure tier: step. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/modulex-core/src/steps/board.rs | 264 +++++++++++++++++++- crates/modulex-core/src/steps/mod.rs | 5 +- crates/modulex-core/tests/data_contract.rs | 16 ++ crates/modulex-core/tests/golden/board.json | 64 +++++ 4 files changed, 343 insertions(+), 6 deletions(-) create mode 100644 crates/modulex-core/tests/golden/board.json diff --git a/crates/modulex-core/src/steps/board.rs b/crates/modulex-core/src/steps/board.rs index cd33381..9a21348 100644 --- a/crates/modulex-core/src/steps/board.rs +++ b/crates/modulex-core/src/steps/board.rs @@ -1,9 +1,11 @@ -//! Filesystem board steps: `board-scan` and `chores-check`. +//! Board steps. //! -//! Pure directory scans — no subprocesses. `board-scan` lists `*.md` tasks -//! per configured lane. `chores-check` looks for `due: YYYY-MM-DD` lines in -//! the chores directory's markdown files and reports what's due or overdue -//! (today's date is display math, never coordination). +//! - `board-scan` and `chores-check` are pure **filesystem** directory scans — +//! no subprocesses, no store. `board-scan` lists `*.md` task stems per +//! configured lane; `chores-check` reports `due:` lines that are due/overdue. +//! - `board` is the **store-backed** view: open cards grouped by lane from the +//! agent state store (the operational knowledge board). It is the counterpart +//! of `board-scan` for boards synced into the store via `import_dir`. use async_trait::async_trait; use chrono::NaiveDate; @@ -11,6 +13,7 @@ use chrono::NaiveDate; use crate::config::{expand_tilde, StepSpec}; use crate::report::StepResult; use crate::step::{RunContext, StepHandler}; +use crate::store::Card; /// `board-scan`: `### lane (N tasks)` plus the task stems, per lane. pub struct BoardScan; @@ -287,6 +290,153 @@ fn render_chores(items: &[DueItem], today: NaiveDate) -> String { lines.join("\n") } +/// The default lanes shown when a `board` step has no `lane` param: the open +/// work lanes, in priority order (closed lanes are surfaced only on request). +const OPEN_LANES: &[&str] = &["p0", "p1", "p2"]; + +/// `board`: open cards grouped by lane, from the agent state store. +/// +/// Params (all optional): `lane` (a single lane to show), `project` (filter). +/// With no `lane`, shows the open lanes (`p0`/`p1`/`p2`). +pub struct Board; + +#[async_trait] +impl StepHandler for Board { + fn type_name(&self) -> &'static str { + "board" + } + + fn description(&self) -> &'static str { + "Open knowledge-board cards grouped by lane, from the agent state store" + } + + fn data_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "required": ["lanes", "open"], + "properties": { + "open": { "type": "integer", "minimum": 0 }, + "lanes": { + "type": "array", + "items": { + "type": "object", + "required": ["lane", "cards"], + "properties": { + "lane": { "type": "string" }, + "cards": { + "type": "array", + "items": { + "type": "object", + "required": ["card_id", "summary"], + "properties": { + "card_id": { "type": "string" }, + "project": { "type": "string" }, + "summary": { "type": "string" }, + "status": { "type": ["string", "null"] }, + "size": { "type": ["string", "null"] }, + "blocked": { "type": "boolean" } + } + } + } + } + } + } + } + }) + } + + fn required_programs(&self, _spec: &StepSpec) -> Vec { + vec![] + } + + async fn run(&self, spec: &StepSpec, cx: &RunContext) -> StepResult { + let Some(store) = &cx.store else { + return StepResult::skip(&spec.name, &spec.step_type, "agent state store unavailable"); + }; + let lane = spec.param_str("lane"); + let project = spec.param_str("project"); + + if cx.dry_run { + return StepResult::ok( + &spec.name, + &spec.step_type, + "[dry-run] would list open board cards from the store", + ); + } + + let cards = match store.cards_query(project, None, lane) { + Ok(cards) => cards, + Err(e) => return StepResult::fail(&spec.name, &spec.step_type, e.to_string()), + }; + + // Which lanes to show, in order: the requested one, else the open lanes. + let lanes: Vec = match lane { + Some(l) => vec![l.to_string()], + None => OPEN_LANES.iter().map(ToString::to_string).collect(), + }; + + let mut result = StepResult::ok(&spec.name, &spec.step_type, render(&cards, &lanes)); + result.data = Some(board_data(&cards, &lanes)); + result + } +} + +/// Cards in a given lane, preserving store order. +fn cards_in<'a>(cards: &'a [Card], lane: &str) -> Vec<&'a Card> { + cards.iter().filter(|c| c.lane == lane).collect() +} + +/// Typed payload for the data contract. +fn board_data(cards: &[Card], lanes: &[String]) -> serde_json::Value { + let lane_views: Vec = lanes + .iter() + .map(|lane| { + let entries: Vec = cards_in(cards, lane) + .into_iter() + .map(|c| { + serde_json::json!({ + "card_id": c.card_id, + "project": c.project, + "summary": c.summary, + "status": c.status, + "size": c.size, + "blocked": c.closed_gen.is_none() + && c.refs.iter().any(|r| r.kind == "blocked_on"), + }) + }) + .collect(); + serde_json::json!({ "lane": lane, "cards": entries }) + }) + .collect(); + let open = cards + .iter() + .filter(|c| lanes.iter().any(|l| l == &c.lane)) + .count(); + serde_json::json!({ "lanes": lane_views, "open": open }) +} + +/// Pure renderer, factored so tests pin the output shape. +fn render(cards: &[Card], lanes: &[String]) -> String { + let mut lines = Vec::new(); + for lane in lanes { + let entries = cards_in(cards, lane); + lines.push(format!("### {lane} ({} cards)", entries.len())); + for c in entries { + let blocked = if c.refs.iter().any(|r| r.kind == "blocked_on") { + " [blocked]" + } else { + "" + }; + lines.push(format!(" - {}: {}{blocked}", c.card_id, c.summary)); + } + } + if lines.is_empty() { + "(no board cards)".to_string() + } else { + lines.join("\n") + } +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -296,6 +446,7 @@ mod tests { use super::*; use crate::config::Config; use crate::exec::test_support::{gate_with, MockSpawner}; + use crate::store::{CardInput, Store}; fn cx_with(config: Config) -> RunContext { RunContext { @@ -407,4 +558,107 @@ mod tests { let result = ChoresCheck.run(&spec, &cx_with(config)).await; assert!(result.skipped); } + + // ── board (store-backed) ─────────────────────────────────────────── + + fn cx_with_store(store: Option>) -> RunContext { + RunContext { + config: Arc::new(Config::default()), + dry_run: false, + generation: 1, + exec: gate_with(&Caveats::top(), Arc::new(MockSpawner::default())), + prior: Vec::new(), + store, + } + } + + fn seed(store: &Store, card_id: &str, lane: &str, summary: &str) { + store + .card_add( + &CardInput { + card_id: card_id.into(), + project: "homelab".into(), + lane: lane.into(), + summary: summary.into(), + ..Default::default() + }, + 1, + ) + .unwrap(); + } + + #[tokio::test] + async fn board_step_missing_store_soft_skips() { + let spec: StepSpec = toml::from_str("name=\"b\"\ntype=\"board\"").unwrap(); + let result = Board.run(&spec, &cx_with_store(None)).await; + assert!(result.skipped); + } + + #[tokio::test] + async fn board_step_lists_open_lanes_only() { + let store = Arc::new(Store::in_memory().unwrap()); + seed(&store, "a", "p0", "active thing"); + seed(&store, "b", "p2", "backlog thing"); + seed(&store, "c", "done", "finished thing"); + + let spec: StepSpec = toml::from_str("name=\"b\"\ntype=\"board\"").unwrap(); + let result = Board.run(&spec, &cx_with_store(Some(store))).await; + assert!(result.success); + assert!(result.output.contains("### p0 (1 cards)")); + assert!(result.output.contains("active thing")); + assert!(result.output.contains("### p2 (1 cards)")); + assert!( + !result.output.contains("finished thing"), + "done lane excluded by default" + ); + + let data = result.data.unwrap(); + assert_eq!(data["open"], 2, "p0 + p2, not done"); + } + + #[tokio::test] + async fn board_step_honors_lane_param() { + let store = Arc::new(Store::in_memory().unwrap()); + seed(&store, "a", "p0", "active thing"); + seed(&store, "c", "done", "finished thing"); + + let spec: StepSpec = toml::from_str("name=\"b\"\ntype=\"board\"\nlane=\"done\"").unwrap(); + let result = Board.run(&spec, &cx_with_store(Some(store))).await; + assert!(result.output.contains("### done (1 cards)")); + assert!(result.output.contains("finished thing")); + } + + #[test] + fn board_render_flags_blocked_cards() { + let cards = vec![Card { + id: 1, + card_id: "x".into(), + project: "p".into(), + lane: "p0".into(), + context: String::new(), + summary: "do the thing".into(), + size: None, + status: Some("blocked".into()), + recurs: None, + expires: None, + created: None, + updated: None, + body: String::new(), + author: None, + source: None, + source_id: None, + created_gen: 1, + updated_gen: 1, + closed_gen: None, + refs: vec![crate::store::CardRef { + kind: "blocked_on".into(), + label: String::new(), + value: "https://x/1".into(), + ordinal: 0, + }], + }]; + let body = render(&cards, &["p0".to_string()]); + assert!(body.contains("### p0 (1 cards)")); + assert!(body.contains("- x: do the thing [blocked]")); + } } diff --git a/crates/modulex-core/src/steps/mod.rs b/crates/modulex-core/src/steps/mod.rs index 8c92883..ec52314 100644 --- a/crates/modulex-core/src/steps/mod.rs +++ b/crates/modulex-core/src/steps/mod.rs @@ -14,8 +14,9 @@ //! | `gitlab-mr-review` | [`gitlab`] | glab | //! | `gitlab-group-mrs` | [`gitlab`] | glab | //! | `mr-sla-check` | [`gitlab`] | — (derived from prior results) | -//! | `board-scan` | [`board`] | — | +//! | `board-scan` | [`board`] | — (filesystem lane dirs) | //! | `chores-check` | [`board`] | — | +//! | `board` | [`board`] | — (store-backed cards) | //! | `python` | [`python`] | the configured interpreter (plugin protocol) | //! | `reminders` | [`reminders`] | — (agent state store) | //! | `url-watch` | [`web`] | — (leashed in-proc fetch; feature `web`) | @@ -53,6 +54,7 @@ pub fn builtin_registry() -> StepRegistry { registry.register(Arc::new(gitlab::MrSlaCheck)); registry.register(Arc::new(board::BoardScan)); registry.register(Arc::new(board::ChoresCheck)); + registry.register(Arc::new(board::Board)); registry.register(Arc::new(python::PythonPlugin)); registry.register(Arc::new(reminders::Reminders)); #[cfg(feature = "web")] @@ -82,6 +84,7 @@ mod tests { "mr-sla-check", "board-scan", "chores-check", + "board", "python", "reminders", ] { diff --git a/crates/modulex-core/tests/data_contract.rs b/crates/modulex-core/tests/data_contract.rs index 6cc4ce9..f9bb158 100644 --- a/crates/modulex-core/tests/data_contract.rs +++ b/crates/modulex-core/tests/data_contract.rs @@ -159,6 +159,10 @@ type = "chores-check" [[routines.contract.steps]] name = "agenda" type = "reminders" + +[[routines.contract.steps]] +name = "cards" +type = "board" "#; /// Guarantee 2: executed builtins emit `data` that validates against their @@ -195,6 +199,18 @@ async fn executed_step_data_validates_against_schema() { store .reminder_add("validate me", Some("2999-01-01"), None, 0) .unwrap(); + store + .card_add( + &modulex_core::store::CardInput { + card_id: "contract-1".into(), + project: "p".into(), + lane: "p0".into(), + summary: "validate the board step".into(), + ..Default::default() + }, + 0, + ) + .unwrap(); let engine = Engine::with_spawner(config, registry, granted, spawner).with_store(store); let report = engine diff --git a/crates/modulex-core/tests/golden/board.json b/crates/modulex-core/tests/golden/board.json new file mode 100644 index 0000000..92381f4 --- /dev/null +++ b/crates/modulex-core/tests/golden/board.json @@ -0,0 +1,64 @@ +{ + "properties": { + "lanes": { + "items": { + "properties": { + "cards": { + "items": { + "properties": { + "blocked": { + "type": "boolean" + }, + "card_id": { + "type": "string" + }, + "project": { + "type": "string" + }, + "size": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "string", + "null" + ] + }, + "summary": { + "type": "string" + } + }, + "required": [ + "card_id", + "summary" + ], + "type": "object" + }, + "type": "array" + }, + "lane": { + "type": "string" + } + }, + "required": [ + "lane", + "cards" + ], + "type": "object" + }, + "type": "array" + }, + "open": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "lanes", + "open" + ], + "type": "object" +}