diff --git a/.changeset/add-replay-command.md b/.changeset/add-replay-command.md new file mode 100644 index 000000000..148f58933 --- /dev/null +++ b/.changeset/add-replay-command.md @@ -0,0 +1,14 @@ +--- +"agent-browser": minor +--- + +Add `replay` command for interactive DOM session recording via rrweb + +New commands: +- `replay start` - Inject rrweb recorder into the current page (auto re-injects on navigation) +- `replay stop [path]` - Stop recording and generate self-contained replay HTML +- `replay status` - Show event count and recording state + +Unlike video recording (`record`), DOM replays are lightweight, inspectable, and produce +self-contained HTML files with play/pause, timeline scrubbing, and speed controls (1x-8x). +The export automatically extracts CSS custom properties for accurate visual replay. diff --git a/cli/src/commands.rs b/cli/src/commands.rs index a802c0647..4cdb8f816 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -1184,6 +1184,26 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result { + const VALID: &[&str] = &["start", "stop", "status"]; + match rest.first().copied() { + Some("start") => Ok(json!({ "id": id, "action": "replay_start" })), + Some("stop") => { + let path = rest.get(1).unwrap_or(&"/tmp/replay"); + Ok(json!({ "id": id, "action": "replay_stop", "path": path })) + } + Some("status") => Ok(json!({ "id": id, "action": "replay_status" })), + Some(sub) => Err(ParseError::UnknownSubcommand { + subcommand: sub.to_string(), + valid_options: VALID, + }), + None => Err(ParseError::MissingArguments { + context: "replay".to_string(), + usage: "replay [path]", + }), + } + } "console" => { let clear = rest.contains(&"--clear"); Ok(json!({ "id": id, "action": "console", "clear": clear })) diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 4c140043e..1eb8b69a0 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -29,6 +29,7 @@ use super::network::{self, DomainFilter, EventTracker}; use super::policy::{ActionPolicy, ConfirmActions, PolicyResult}; use super::providers; use super::recording::{self, RecordingState}; +use super::replay::{self, ReplayState}; use super::screenshot::{self, ScreenshotOptions}; use super::snapshot::{self, SnapshotOptions}; use super::state; @@ -179,6 +180,7 @@ pub struct DaemonState { pub session_id: String, pub tracing_state: TracingState, pub recording_state: RecordingState, + pub replay_state: ReplayState, event_rx: Option>, pub screencasting: bool, pub policy: Option, @@ -236,6 +238,7 @@ impl DaemonState { session_id: env::var("AGENT_BROWSER_SESSION").unwrap_or_else(|_| "default".to_string()), tracing_state: TracingState::new(), recording_state: RecordingState::new(), + replay_state: ReplayState::new(), event_rx: None, screencasting: false, policy: ActionPolicy::load_if_exists(), @@ -1131,6 +1134,9 @@ pub async fn execute_command(cmd: &Value, state: &mut DaemonState) -> Value { "recording_start" => handle_recording_start(cmd, state).await, "recording_stop" => handle_recording_stop(state).await, "recording_restart" => handle_recording_restart(cmd, state).await, + "replay_start" => handle_replay_start(state).await, + "replay_stop" => handle_replay_stop(cmd, state).await, + "replay_status" => handle_replay_status(state).await, "pdf" => handle_pdf(cmd, state).await, "tab_list" => handle_tab_list(state).await, "tab_new" => handle_tab_new(cmd, state).await, @@ -3746,6 +3752,29 @@ async fn handle_recording_restart(cmd: &Value, state: &mut DaemonState) -> Resul Ok(result) } +// --------------------------------------------------------------------------- +// Replay (rrweb DOM recording) +// --------------------------------------------------------------------------- + +async fn handle_replay_start(state: &mut DaemonState) -> Result { + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + replay::replay_start(&mut state.replay_state, mgr).await +} + +async fn handle_replay_stop(cmd: &Value, state: &mut DaemonState) -> Result { + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + let path = cmd + .get("path") + .and_then(|v| v.as_str()) + .unwrap_or("/tmp/replay"); + replay::replay_stop(&mut state.replay_state, mgr, path).await +} + +async fn handle_replay_status(state: &DaemonState) -> Result { + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + replay::replay_status(&state.replay_state, mgr).await +} + async fn handle_pdf(cmd: &Value, state: &DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); diff --git a/cli/src/native/mod.rs b/cli/src/native/mod.rs index f86c816e8..f42c87dbc 100644 --- a/cli/src/native/mod.rs +++ b/cli/src/native/mod.rs @@ -27,6 +27,8 @@ pub mod providers; #[allow(dead_code)] pub mod recording; #[allow(dead_code)] +pub mod replay; +#[allow(dead_code)] pub mod screenshot; #[allow(dead_code)] pub mod snapshot; diff --git a/cli/src/native/replay.rs b/cli/src/native/replay.rs new file mode 100644 index 000000000..f2db6790f --- /dev/null +++ b/cli/src/native/replay.rs @@ -0,0 +1,483 @@ +use serde_json::{json, Value}; +use std::fs; +use std::path::Path; + +/// rrweb CDN URL -- fetched at runtime and cached in-page. +const RRWEB_CDN_URL: &str = "https://cdn.jsdelivr.net/npm/rrweb@latest/dist/rrweb.min.js"; + +/// Maximum gap (ms) between events before compression. +/// Gaps larger than this are compressed down to this value, removing +/// idle periods caused by navigation redirects, network waits, etc. +const MAX_EVENT_GAP_MS: i64 = 200; + +/// JavaScript that fetches rrweb from CDN, injects it, and starts recording. +/// Uses `inlineImages`, `collectFonts`, and `inlineStylesheet` for high-fidelity replay. +/// Safe to call multiple times -- skips if already injected. +fn build_inject_js() -> String { + format!( + r#"fetch('{cdn}').then(r=>r.text()).then(src=>{{ +var el=document.createElement('script'); +el.textContent=src; +document.head.appendChild(el); +window.__rrwebEvents=window.__rrwebEvents||[]; +window.__rrwebInjected=true; +window.rrweb.record({{ +emit:function(e){{window.__rrwebEvents.push(e)}}, +inlineImages:true, +collectFonts:true, +inlineStylesheet:true +}}); +return 'recording started, '+window.__rrwebEvents.length+' prior events' +}})"#, + cdn = RRWEB_CDN_URL + ) +} + +/// JavaScript that returns the current event count. +const STATUS_JS: &str = "(window.__rrwebEvents||[]).length"; + +/// JavaScript that extracts all recorded events as a JSON string. +const EXTRACT_EVENTS_JS: &str = "JSON.stringify(window.__rrwebEvents||[])"; + +/// JavaScript that extracts resolved CSS custom properties from :root. +/// This fixes replay visual bugs where var(--x) references don't resolve +/// because the replay runs in a different context without the original stylesheets. +const EXTRACT_CSS_VARS_JS: &str = r#"(function(){ +var s=getComputedStyle(document.documentElement); +var vars=[]; +for(var i=0;i, +} + +impl ReplayState { + pub fn new() -> Self { + Self { + active: false, + event_count: 0, + auto_inject_id: None, + } + } +} + +// --------------------------------------------------------------------------- +// Command handlers (called from actions.rs) +// --------------------------------------------------------------------------- + +/// Start rrweb recording: inject the script into the current page and +/// register it for automatic re-injection on navigation. +pub async fn replay_start( + state: &mut ReplayState, + mgr: &super::browser::BrowserManager, +) -> Result { + if state.active { + return Err("Replay recording already active. Use 'replay stop' to save.".to_string()); + } + + let inject_js = build_inject_js(); + + // Inject into the current page + mgr.evaluate(&inject_js, None).await?; + + // Register for auto-injection on future navigations + let identifier = mgr.add_script_to_evaluate(&inject_js).await?; + state.auto_inject_id = Some(identifier); + state.active = true; + state.event_count = 0; + + Ok(json!({ + "started": true, + "message": "Recording started. Navigate and interact -- events are captured automatically." + })) +} + +/// Check recording status and event count. +pub async fn replay_status( + state: &ReplayState, + mgr: &super::browser::BrowserManager, +) -> Result { + if !state.active { + return Ok(json!({ + "active": false, + "events": 0, + "message": "Not recording. Use 'replay start' to begin." + })); + } + + let result = mgr.evaluate(STATUS_JS, None).await?; + let count = result + .as_str() + .and_then(|s| s.parse::().ok()) + .or_else(|| result.as_u64()) + .unwrap_or(0); + + Ok(json!({ + "active": true, + "events": count, + })) +} + +/// Stop recording, extract events + CSS variables, generate replay HTML. +pub async fn replay_stop( + state: &mut ReplayState, + mgr: &super::browser::BrowserManager, + output_path: &str, +) -> Result { + if !state.active { + return Err("No replay recording in progress. Use 'replay start' first.".to_string()); + } + + // Remove auto-inject script so future navigations don't keep recording + if let Some(ref id) = state.auto_inject_id { + let _ = mgr + .client + .send_command( + "Page.removeScriptToEvaluateOnNewDocument", + Some(json!({ "identifier": id })), + Some(mgr.active_session_id()?), + ) + .await; + } + + // Extract event count + let count_result = mgr.evaluate(STATUS_JS, None).await?; + let event_count = count_result + .as_str() + .and_then(|s| s.parse::().ok()) + .or_else(|| count_result.as_u64()) + .unwrap_or(0); + + if event_count == 0 { + state.active = false; + state.auto_inject_id = None; + let _ = mgr.evaluate(CLEANUP_JS, None).await; + return Err("No events captured.".to_string()); + } + + // Extract CSS custom properties (for accurate replay of var() references) + let css_vars = mgr + .evaluate(EXTRACT_CSS_VARS_JS, None) + .await + .unwrap_or_else(|_| Value::String(":root{}".to_string())); + let css_vars_str = css_vars.as_str().unwrap_or(":root{}"); + + // Extract events JSON + let events_value = mgr.evaluate(EXTRACT_EVENTS_JS, None).await?; + let events_json = events_value.as_str().unwrap_or("[]"); + + // Validate we got real JSON + if events_json == "[]" || events_json.len() < 10 { + state.active = false; + state.auto_inject_id = None; + let _ = mgr.evaluate(CLEANUP_JS, None).await; + return Err("Failed to extract events from page.".to_string()); + } + + // Compress gaps: remove idle periods caused by navigation, network waits, etc. + // This prevents the rrweb-player timeline from showing grey inactive zones + // that block the UI controls (a known rrweb-player bug). + let compressed_json = compress_event_gaps(events_json); + let final_events = compressed_json.as_deref().unwrap_or(events_json); + + let size_kb = final_events.len() as f64 / 1024.0; + state.event_count = event_count; + + // Ensure output directory exists + if let Some(parent) = Path::new(output_path).parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create output directory: {}", e))?; + } + + // Determine paths + let base_path = if output_path.ends_with(".html") { + output_path.trim_end_matches(".html").to_string() + } else { + output_path.to_string() + }; + let html_path = format!("{}.html", base_path); + let json_path = format!("{}.json", base_path); + + // Write recording.json (raw, uncompressed for programmatic use) + fs::write(&json_path, events_json) + .map_err(|e| format!("Failed to write {}: {}", json_path, e))?; + + // Escape sequences to prevent broken HTML + let escaped_events = final_events.replace("", "<\\/script>"); + let escaped_css = css_vars_str.replace(" Option { + let mut events: Vec = serde_json::from_str(events_json).ok()?; + if events.len() < 2 { + return None; + } + + let mut time_saved: i64 = 0; + for i in 1..events.len() { + let prev_ts = events[i - 1].get("timestamp")?.as_i64()?; + let curr_ts = events[i].get("timestamp")?.as_i64()?; + let gap = curr_ts - prev_ts; + + if gap > MAX_EVENT_GAP_MS { + time_saved += gap - MAX_EVENT_GAP_MS; + } + + let new_ts = curr_ts - time_saved; + events[i]["timestamp"] = json!(new_ts); + } + + serde_json::to_string(&events).ok() +} + +// --------------------------------------------------------------------------- +// HTML template +// --------------------------------------------------------------------------- + +fn generate_replay_html(events_json: &str, css_vars: &str, count: u64, size_kb: f64) -> String { + format!( + r##" + + + +Session Replay - agent-browser + + + + +

Session Replay

+

+ {count} events · + {size:.1} KB +

+
+
+ + +
+ + + + +"##, + count = count, + size = size_kb, + events = events_json, + css_vars = serde_json::to_string(css_vars).unwrap_or_else(|_| "\"\"".to_string()), + ) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_replay_state_new() { + let state = ReplayState::new(); + assert!(!state.active); + assert_eq!(state.event_count, 0); + assert!(state.auto_inject_id.is_none()); + } + + #[test] + fn test_build_inject_js_contains_rrweb_url() { + let js = build_inject_js(); + assert!(js.contains(RRWEB_CDN_URL)); + assert!(js.contains("inlineImages:true")); + assert!(js.contains("collectFonts:true")); + } + + #[test] + fn test_generate_replay_html_structure() { + let html = generate_replay_html("[{\"type\":4}]", ":root{}", 1, 0.1); + assert!(html.contains("rrweb-player")); + assert!(html.contains("Session Replay")); + assert!(html.contains("1 events")); + assert!(html.contains("insertStyleRules")); + assert!(html.contains("Record this tab as video")); + } + + #[test] + fn test_generate_replay_html_escapes_css() { + let html = generate_replay_html("[]", ":root{--bg: #fff;}", 0, 0.0); + assert!(html.contains("--bg")); + } + + #[test] + fn test_compress_event_gaps_removes_idle() { + let events = json!([ + {"type": 4, "timestamp": 1000}, + {"type": 2, "timestamp": 1050}, + {"type": 3, "timestamp": 5000}, // 3950ms gap -> compressed + {"type": 3, "timestamp": 5100}, + ]); + let json_str = serde_json::to_string(&events).unwrap(); + let compressed = compress_event_gaps(&json_str).unwrap(); + let result: Vec = serde_json::from_str(&compressed).unwrap(); + + // Gap between event 1 (1050) and event 2 should be MAX_EVENT_GAP_MS, not 3950 + let ts2 = result[2]["timestamp"].as_i64().unwrap(); + let ts1 = result[1]["timestamp"].as_i64().unwrap(); + assert_eq!(ts2 - ts1, MAX_EVENT_GAP_MS); + } + + #[test] + fn test_compress_event_gaps_preserves_small_gaps() { + let events = json!([ + {"type": 4, "timestamp": 1000}, + {"type": 3, "timestamp": 1050}, + {"type": 3, "timestamp": 1100}, + ]); + let json_str = serde_json::to_string(&events).unwrap(); + let compressed = compress_event_gaps(&json_str).unwrap(); + // Small gaps should be unchanged + assert_eq!(json_str, compressed); + } +} diff --git a/cli/src/output.rs b/cli/src/output.rs index e357cff66..2e2210b13 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -2200,6 +2200,50 @@ Examples: "## } + // === Replay (rrweb DOM recording) === + "replay" => { + r##" +agent-browser replay - Record interactive DOM session replays + +Usage: agent-browser replay start + agent-browser replay stop [path] + agent-browser replay status + +Record browser sessions as interactive HTML replays using rrweb. +Unlike video recording, DOM replays are lightweight, inspectable, +and produce self-contained HTML files with playback controls. + +Operations: + start Inject rrweb recorder into the current page. + Automatically re-injects on navigation. + stop [path] Stop recording and generate replay HTML + JSON. + Default path: /tmp/replay (.html and .json appended) + status Show event count and recording state. + +Features: + - Captures DOM mutations, mouse movements, scrolls, inputs + - Inlines images and stylesheets for self-contained replay + - Extracts CSS custom properties for accurate visual replay + - Play/pause, timeline scrubbing, speed controls (1x-8x) + - No ffmpeg required (unlike video recording) + +Global Options: + --json Output as JSON + --session Use specific session + +Examples: + # Record a user flow + agent-browser open https://app.example.com + agent-browser replay start + agent-browser click @e3 + agent-browser fill @e5 "hello" + agent-browser replay stop ./my-session + + # Open the replay + open ./my-session.html +"## + } + // === Console/Errors === "console" => { r##" @@ -2737,6 +2781,9 @@ Debug: profiler start|stop [path] Record Chrome DevTools profile record start [url] Start video recording (WebM) record stop Stop and save video + replay start Start DOM replay recording (rrweb) + replay stop [path] Stop and save interactive replay HTML + replay status Show replay event count console [--clear] View console logs errors [--clear] View page errors highlight Highlight element