diff --git a/.beads/.gitignore b/.beads/.gitignore new file mode 100644 index 00000000..8ff02f42 --- /dev/null +++ b/.beads/.gitignore @@ -0,0 +1,17 @@ +# SQLite databases +*.db +*.db-journal +*.db-wal +*.db-shm + +# Daemon runtime files +daemon.log +daemon.pid +bd.sock + +# Legacy database files +db.sqlite +bd.db + +# Keep JSONL exports (source of truth for git) +!*.jsonl diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl new file mode 100644 index 00000000..3f06a8cf --- /dev/null +++ b/.beads/issues.jsonl @@ -0,0 +1,31 @@ +{"id":"scotty-1","title":"Unified Output System - Complete Implementation","description":"Parent epic tracking the unified output system for logs and interactive shell access. Backend is complete, CLI logs working, but some frontend features and CLI shell command remain.","design":"See docs/prds/unified-output-system.md for complete technical specification. Current status: Phase 1-3.7 and Phase 5 complete, Phase 4 partially complete.","status":"open","priority":1,"issue_type":"epic","created_at":"2025-10-25T00:58:21.660841+02:00","updated_at":"2025-10-25T00:58:21.660841+02:00"} +{"id":"scotty-10","title":"Instrument shell service with metrics","description":"Add metrics recording to ShellService for session counts, durations, errors, and timeouts.","design":"Instrument ShellService:\n- Track shell_sessions_active gauge\n- Increment shell_sessions_total counter\n- Record shell_session_duration histogram\n- Track shell_session_errors and timeouts","acceptance_criteria":"- Session metrics recorded correctly\n- Duration tracking works\n- Timeout cases tracked separately\n- Memory leak tests pass","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-25T01:28:15.964474+02:00","updated_at":"2025-10-26T18:37:15.004569+01:00","closed_at":"2025-10-26T18:37:15.004569+01:00","dependencies":[{"issue_id":"scotty-10","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:30.18956+02:00","created_by":"daemon"},{"issue_id":"scotty-10","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T01:28:42.15994+02:00","created_by":"daemon"}]} +{"id":"scotty-11","title":"Instrument WebSocket connections with metrics","description":"Add metrics to WebSocket client management for connection counts, message throughput, and authentication failures.","design":"Instrument WebSocket layer:\n- Track websocket_connections_active\n- Count messages sent/received\n- Track authentication failures\n- Monitor disconnect events","acceptance_criteria":"- Connection lifecycle tracked\n- Message counters increment correctly\n- Auth failure tracking works\n- No overhead on message path","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-25T01:28:16.086525+02:00","updated_at":"2025-10-26T17:38:41.091932+01:00","closed_at":"2025-10-26T17:38:41.091932+01:00","dependencies":[{"issue_id":"scotty-11","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:30.272345+02:00","created_by":"daemon"},{"issue_id":"scotty-11","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T01:28:42.23292+02:00","created_by":"daemon"}]} +{"id":"scotty-12","title":"Add observability stack to docker-compose","description":"Create docker-compose.observability.yml with OTel Collector, VictoriaMetrics, and Grafana services. Create OTel Collector configuration file.","design":"Create docker-compose.observability.yml with:\n- otel-collector service (ports 4317/4318)\n- victoriametrics service (port 8428)\n- grafana service (port 3000)\n- Create otel-collector-config.yaml with trace/metrics pipelines\n- Add Traefik labels for ddev.site domains","acceptance_criteria":"- docker-compose up works with observability stack\n- OTel Collector receives metrics on port 4317\n- VictoriaMetrics stores metrics\n- Grafana accessible at grafana.ddev.site\n- No port conflicts with existing services","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-25T01:28:16.207816+02:00","updated_at":"2025-10-25T01:38:25.612801+02:00","closed_at":"2025-10-25T01:38:25.612801+02:00","dependencies":[{"issue_id":"scotty-12","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:30.352479+02:00","created_by":"daemon"}]} +{"id":"scotty-13","title":"Create Grafana dashboards for Scotty metrics","description":"Create Grafana dashboard JSON and provisioning config showing unified output system metrics with panels for log streams, shell sessions, WebSocket connections, and tasks.","design":"Create Grafana assets:\n- grafana/provisioning/datasources/datasources.yaml (VictoriaMetrics + Jaeger)\n- grafana/provisioning/dashboards/dashboards.yaml\n- grafana/dashboards/scotty-unified-output.json\n- Panels: active streams, session durations, message rates, error rates","acceptance_criteria":"- Grafana dashboard loads automatically\n- All panels show live data\n- VictoriaMetrics datasource works\n- Dashboard is intuitive and useful\n- Export JSON to git","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-25T01:28:16.355104+02:00","updated_at":"2025-10-25T15:21:28.683425+02:00","closed_at":"2025-10-25T15:21:28.683425+02:00","dependencies":[{"issue_id":"scotty-13","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:30.434812+02:00","created_by":"daemon"},{"issue_id":"scotty-13","depends_on_id":"scotty-12","type":"blocks","created_at":"2025-10-25T01:28:42.307778+02:00","created_by":"daemon"}]} +{"id":"scotty-14","title":"Document metrics and observability setup","description":"Write documentation for setting up and using the observability stack, including metrics available, dashboard usage, and troubleshooting.","design":"Documentation:\n- README section on observability\n- docs/observability/setup.md - How to run the stack\n- docs/observability/metrics.md - Available metrics reference\n- docs/observability/dashboards.md - Dashboard guide\n- Troubleshooting common issues","acceptance_criteria":"- Setup instructions are clear and complete\n- Metrics reference documents all instruments\n- Dashboard guide has screenshots\n- Troubleshooting covers common issues\n- Updates to main README.md","status":"closed","priority":3,"issue_type":"task","created_at":"2025-10-25T01:28:16.527864+02:00","updated_at":"2025-10-26T18:43:52.600539+01:00","closed_at":"2025-10-26T18:43:52.600539+01:00","dependencies":[{"issue_id":"scotty-14","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:30.517999+02:00","created_by":"daemon"}]} +{"id":"scotty-15","title":"Upgrade OpenTelemetry dependencies to latest versions","description":"Upgrade opentelemetry, opentelemetry_sdk, and opentelemetry-otlp crates to their latest stable versions to get bug fixes, performance improvements, and new features.","design":"Check current versions in Cargo.toml workspace dependencies:\n- opentelemetry = \"0.28.0\"\n- opentelemetry_sdk = \"0.28\"\n- opentelemetry-otlp = \"0.28.0\"\n\nResearch latest versions on crates.io and upgrade incrementally. Update any API changes in:\n- scotty/src/metrics/init.rs\n- scotty/src/metrics/instruments.rs\n- scotty/src/init_telemetry.rs\n\nTest that traces and metrics still export correctly after upgrade.","acceptance_criteria":"- All opentelemetry crates upgraded to latest stable versions\n- Cargo.toml updated with new versions\n- Code compiles without errors\n- Metrics initialization works\n- Traces export successfully\n- No breaking changes in existing functionality","notes":"Successfully upgraded OpenTelemetry from 0.28 to 0.31 (latest stable version):\n\nUpgraded crates:\n- opentelemetry: 0.28.0 → 0.31.0\n- opentelemetry_sdk: 0.28 → 0.31\n- opentelemetry-otlp: 0.28.0 → 0.31.0 (added grpc-tonic feature)\n- tracing-opentelemetry: 0.29 → 0.32\n- axum-tracing-opentelemetry: 0.26.0 → 0.32.1\n- init-tracing-opentelemetry: 0.27.0 → 0.32.1\n\nAPI changes handled:\n- Fixed TraceError import (moved to opentelemetry_sdk::trace)\n- Updated init_tracerprovider imports (now in otlp::traces module)\n- Fixed TracingConfig import (moved to config module)\n- Added error conversion for init_tracerprovider\n- Fixed build_layer() result handling\n\nRemoved axum-otel-metrics due to version conflict (required 0.30 while ecosystem moved to 0.31). Implemented custom HTTP metrics middleware instead.\n\nImplementation notes:\n- Created scotty/src/metrics/http.rs with custom middleware\n- Tracks http_requests_total, http_request_duration, http_requests_active\n- Fixed telemetry_enabled flag to check for both 'traces' and 'metrics'\n- All metrics exporting correctly to VictoriaMetrics\n- Grafana dashboard updated with correct metric names\n\nCommits: bb28b5cd, 075c755b, 8e297537","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-25T01:51:52.305855+02:00","updated_at":"2025-10-25T18:54:58.935792+02:00","closed_at":"2025-10-25T18:54:58.935792+02:00","dependencies":[{"issue_id":"scotty-15","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T01:51:52.306614+02:00","created_by":"daemon"}]} +{"id":"scotty-16","title":"Instrument task output streaming with metrics","description":"Add metrics recording to TaskOutputStreamingService for task counts, durations, errors, and output volume.","design":"Instrument TaskOutputStreamingService similar to LogStreamingService:\n- Track tasks_active gauge (current running tasks)\n- Track tasks_total counter (cumulative task count)\n- Add task_output_lines counter (throughput tracking)\n- Add task_errors counter (error tracking)\n- Consider task_duration histogram if tasks have clear lifecycle\n\nLocation: scotty/src/tasks/output_streaming.rs\nPattern: Use metrics::get_metrics() to access global metrics instance","acceptance_criteria":"- Metrics recorded at task start/end\n- Output lines counted\n- Error cases tracked\n- No performance degradation\n- Code compiles and tests pass","status":"closed","priority":3,"issue_type":"task","created_at":"2025-10-25T02:29:59.988921+02:00","updated_at":"2025-10-26T18:10:42.53396+01:00","closed_at":"2025-10-26T18:10:42.53396+01:00","dependencies":[{"issue_id":"scotty-16","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T02:29:59.990577+02:00","created_by":"daemon"}]} +{"id":"scotty-17","title":"Add memory usage metrics","description":"Track scotty's memory usage (heap allocated, RSS, etc.) to monitor resource consumption and detect memory leaks.","design":"Add memory metrics to ScottyMetrics struct:\n- memory_heap_bytes (Gauge) - heap allocated memory\n- memory_rss_bytes (Gauge) - resident set size\n- Consider using jemalloc or system metrics crate\n\nOptions:\n1. Use `sysinfo` crate for cross-platform process metrics\n2. Use jemalloc stats if using jemalloc allocator\n3. Sample memory every 30s-60s to avoid overhead\n\nLocation: \n- Add metrics to scotty/src/metrics/instruments.rs\n- Add sampling task to scotty/src/main.rs or metrics/mod.rs\n- Record metrics periodically in background task","acceptance_criteria":"- Memory metrics exported to OTLP\n- Metrics update at reasonable interval (30-60s)\n- Minimal performance overhead\n- Works on all platforms (Linux, macOS)\n- Dashboard panel created for memory tracking","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-25T02:30:11.101617+02:00","updated_at":"2025-10-25T02:49:17.419136+02:00","closed_at":"2025-10-25T02:49:17.419136+02:00","dependencies":[{"issue_id":"scotty-17","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T02:30:11.102361+02:00","created_by":"daemon"}]} +{"id":"scotty-18","title":"Add Tokio runtime metrics","description":"Track Tokio async runtime statistics like active tasks, worker threads, idle time, and task queue depth to monitor async performance.","design":"Add Tokio runtime metrics to ScottyMetrics:\n- tokio_tasks_active (Gauge) - currently spawned tasks\n- tokio_workers_count (Gauge) - number of worker threads\n- tokio_workers_idle (Gauge) - idle workers\n- tokio_tasks_spawned_total (Counter) - total tasks spawned\n\nImplementation options:\n1. Use tokio-metrics crate (official tokio metrics)\n2. Use tokio::runtime::Handle::metrics() (requires unstable features)\n3. Use tokio-console integration via console-subscriber\n\nRecommended: tokio-metrics crate\n- Add tokio-metrics to Cargo.toml\n- Create background task to sample runtime metrics\n- Record every 10-30s\n\nLocation:\n- Add metrics to scotty/src/metrics/instruments.rs\n- Add tokio metrics sampler in scotty/src/metrics/tokio.rs\n- Spawn sampling task in main.rs after runtime creation","acceptance_criteria":"- Tokio runtime metrics exported to OTLP\n- Metrics show active tasks and worker state\n- Minimal performance overhead\n- Code compiles with stable Rust\n- Dashboard panel created for runtime monitoring","status":"closed","priority":3,"issue_type":"task","created_at":"2025-10-25T02:30:22.11133+02:00","updated_at":"2025-10-25T16:17:30.994921+02:00","closed_at":"2025-10-25T16:17:30.994921+02:00","dependencies":[{"issue_id":"scotty-18","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T02:30:22.112175+02:00","created_by":"daemon"}]} +{"id":"scotty-19","title":"Add TaskManager metrics for task tracking","description":"Instrument TaskManager to track number of tasks, their states (running/finished/failed), durations, and success rates.","design":"Add TaskManager metrics to ScottyMetrics struct:\n- tasks_total (Counter) - Total tasks created\n- tasks_by_state (Gauge) - Current tasks grouped by state labels:\n * state=\"running\"\n * state=\"finished\"\n * state=\"failed\"\n- task_duration_seconds (Histogram) - Task execution time\n- task_failures_total (Counter) - Failed tasks counter\n\nImplementation approach:\n1. Add metrics to scotty/src/metrics/instruments.rs\n2. Instrument scotty/src/tasks/manager.rs:\n - add_task(): increment tasks_total, update state gauge\n - set_task_finished(): update state gauges, record duration histogram\n - Optional: background sampler every 30s to sync gauges with HashMap state\n\nData sources:\n- TaskManager.processes HashMap size = total active tasks\n- TaskDetails.state: Running | Finished | Failed\n- TaskDetails.start_time + finish_time for duration calculation\n- TaskDetails.last_exit_code for success/failure tracking","acceptance_criteria":"- Task metrics exported to OTLP\n- Metrics accurately reflect task states\n- Duration tracking works correctly\n- Failed vs successful tasks distinguishable\n- Dashboard panels created for task monitoring\n- No performance degradation","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-25T02:37:07.454119+02:00","updated_at":"2025-10-25T15:29:19.632177+02:00","closed_at":"2025-10-25T15:29:19.632177+02:00","dependencies":[{"issue_id":"scotty-19","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T02:37:07.454889+02:00","created_by":"daemon"}]} +{"id":"scotty-2","title":"Implement app:shell CLI command in scottyctl","description":"Add the app:shell command to scottyctl CLI. Backend ShellService is fully implemented and tested, but the CLI command is missing. Need to create scottyctl/src/commands/apps/shell.rs with WebSocket-based terminal integration.","design":"- Create ShellCommand struct in cli.rs\n- Implement WebSocket-based shell handler in commands/apps/shell.rs\n- Add TTY resize handling and raw terminal mode\n- Support interactive input/output with proper escape sequences\n- Reuse AuthenticatedWebSocket pattern from logs command","acceptance_criteria":"- scottyctl app:shell myapp web opens interactive shell\n- Terminal escape sequences work correctly\n- Ctrl+C and Ctrl+D handled properly\n- Session cleanup on disconnect\n- Help text and examples documented","status":"in_progress","priority":1,"issue_type":"feature","created_at":"2025-10-25T00:58:21.798713+02:00","updated_at":"2025-10-26T18:37:31.697764+01:00","dependencies":[{"issue_id":"scotty-2","depends_on_id":"scotty-1","type":"parent-child","created_at":"2025-10-25T00:58:32.439042+02:00","created_by":"daemon"}]} +{"id":"scotty-20","title":"Add AppList metrics for application monitoring","description":"Instrument SharedAppList to track number of apps, their states (stopped/starting/running/etc), services per app, and health check age.","design":"Add AppList metrics to ScottyMetrics struct:\n- apps_total (Gauge) - Total number of managed apps\n- apps_by_status (Gauge) - Apps grouped by status labels:\n * status=\"stopped\"\n * status=\"starting\"\n * status=\"running\"\n * status=\"creating\"\n * status=\"destroying\"\n * status=\"unsupported\"\n- app_services_count (Histogram) - Distribution of services per app\n- app_last_check_age_seconds (Histogram) - Time since last health check\n\nImplementation approach:\n1. Add metrics to scotty/src/metrics/instruments.rs\n2. Instrument scotty-core/src/apps/shared_app_list.rs:\n - add_app() / remove_app(): update apps_total\n - Background sampler task (30-60s interval):\n * Iterate apps HashMap\n * Count by status\n * Sample services.len() distribution\n * Calculate last_checked age\n3. Spawn sampler task in scotty/src/main.rs or via AppState\n\nData sources:\n- SharedAppList.apps HashMap size = total apps\n- AppData.status: Stopped | Starting | Running | Creating | Destroying | Unsupported\n- AppData.services.len() = service count per app\n- AppData.last_checked timestamp for age calculation\n\nNote: Requires access to SharedAppList from metrics, may need to pass via background task or add to AppState.","acceptance_criteria":"- App metrics exported to OTLP\n- Metrics accurately reflect app states\n- Status distribution correct\n- Service count distribution tracked\n- Health check age visible\n- Dashboard panels created for app monitoring\n- Minimal performance overhead from sampling","status":"closed","priority":2,"issue_type":"task","created_at":"2025-10-25T02:37:26.668911+02:00","updated_at":"2025-10-26T17:20:02.191752+01:00","closed_at":"2025-10-26T17:20:02.191752+01:00","dependencies":[{"issue_id":"scotty-20","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T02:37:26.669776+02:00","created_by":"daemon"}]} +{"id":"scotty-21","title":"Add Traefik to observability docker-compose for .ddev.site URLs","description":"The observability stack uses Traefik labels for routing (grafana.ddev.site, jaeger.ddev.site, vm.ddev.site) but requires external Traefik to be running from apps/traefik. This creates a dependency and extra setup step.\n\nAdd Traefik service to observability/docker-compose.yml so the stack works out-of-the-box without requiring apps/traefik to be running.","design":"Options:\n1. Add Traefik service to observability/docker-compose.yml\n - Include network configuration\n - Configure dashboard access\n - Ensure no port conflicts with main Traefik\n\n2. Share network with existing apps/traefik\n - Create external network\n - Less duplication but still requires apps/traefik\n\nRecommended: Option 1 - self-contained observability stack\n\nImplementation:\n- Add Traefik service to observability/docker-compose.yml\n- Use different ports than apps/traefik (80/443 vs 8080/8443)\n- Configure proxy network\n- Update README.md to mention this works standalone","acceptance_criteria":"- grafana.ddev.site, jaeger.ddev.site, vm.ddev.site work without apps/traefik running\n- cd observability \u0026\u0026 docker-compose up -d brings up full working stack\n- No port conflicts with apps/traefik\n- Documentation updated","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-25T02:59:32.096433+02:00","updated_at":"2025-10-25T02:59:32.096433+02:00"} +{"id":"scotty-22","title":"Refactor metrics instrumentation to use dedicated helper functions","description":"Review and refactor existing metrics code in log streaming and other services to move metrics recording into dedicated helper functions, keeping business logic clean and separated from instrumentation code.","design":"Pattern established in scotty-19:\n- Create dedicated helper functions like `record_task_added_metrics()` and `record_task_finished_metrics()`\n- Move all `if let Some(m) = metrics::get_metrics()` blocks into these helpers\n- Keep business logic methods focused on their primary responsibility\n\nFiles to review and refactor:\n- scotty/src/docker/services/logs.rs (log streaming metrics)\n- scotty/src/docker/services/shell.rs (shell session metrics - if implemented)\n- Any other services with inline metrics code\n\nBenefits:\n- Cleaner separation of concerns\n- Easier to test business logic without metrics\n- Consistent metrics instrumentation pattern\n- Better code readability","status":"closed","priority":3,"issue_type":"chore","created_at":"2025-10-25T15:28:09.285832+02:00","updated_at":"2025-10-25T16:07:00.885661+02:00","closed_at":"2025-10-25T16:07:00.885661+02:00"} +{"id":"scotty-23","title":"Reduce cloning overhead by wrapping AppData in Arc","description":"AppData is cloned on every access from SharedAppList. Wrapping AppData in Arc would make cloning cheap (just reference count increment) instead of copying all nested structures.","design":"Location: scotty-core/src/apps/shared_app_list.rs:56-58\n\nCurrent code clones entire AppData structure on every get_app() call. AppData contains multiple nested structures (containers, services, settings) making clones expensive.\n\nProposed solution:\n```rust\npub type SharedAppData = Arc\u0026lt;AppData\u0026gt;;\n\npub async fn get_app(\u0026amp;self, app_name: \u0026amp;str) -\u0026gt; Option\u0026lt;SharedAppData\u0026gt; {\n let t = self.apps.read().await;\n t.get(app_name).map(Arc::clone) // Only clones Arc, not data\n}\n```\n\nImpact: Major performance improvement for app data access paths\nEffort: 2-4 hours","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-26T21:21:10.34027+01:00","updated_at":"2025-10-26T21:21:10.34027+01:00","labels":["performance","refactoring"]} +{"id":"scotty-24","title":"Add timeout to Docker command execution","description":"Docker commands currently have no timeout and could hang indefinitely if Docker daemon becomes unresponsive.","design":"Location: scotty/src/docker/docker_compose.rs:88\n\nCurrent code:\n```rust\nlet output = cmd.output()?; // No timeout\n```\n\nProposed solution:\n```rust\nuse tokio::time::timeout;\n\nlet output = timeout(\n Duration::from_secs(300), // 5 minute timeout\n tokio::process::Command::new(\"docker-compose\")\n .args(command)\n .output()\n).await??;\n```\n\nImpact: Prevents indefinite hangs on Docker failures\nEffort: 1-2 hours","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-26T21:21:10.406194+01:00","updated_at":"2025-10-26T21:21:10.406194+01:00","labels":["docker","stability"]} +{"id":"scotty-25","title":"Improve authorization error handling in Casbin enforcement","description":"Authorization service silently denies access on Casbin errors using unwrap_or(false), potentially hiding system issues.","design":"Location: scotty/src/services/authorization/service.rs:182\n\nCurrent code:\n```rust\nlet result = enforcer\n .enforce(vec![user, app, action_str])\n .unwrap_or(false);\n```\n\nProposed solution:\n```rust\nlet result = enforcer\n .enforce(vec![user, app, action_str])\n .map_err(|e| {\n error!(\"Casbin enforce error: {}\", e);\n crate::metrics::authorization::record_error();\n })\n .unwrap_or(false);\n```\n\nImpact: Better visibility into authorization failures\nEffort: 30 minutes","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-26T21:21:10.471975+01:00","updated_at":"2025-10-26T21:21:10.471975+01:00","labels":["observability","security"]} +{"id":"scotty-26","title":"Fix wildcard dependency for tracing-subscriber","description":"tracing-subscriber uses wildcard version \"*\" which prevents reproducible builds.","design":"Location: Cargo.toml:36\n\nCurrent: `tracing-subscriber = \"*\"`\nReplace with: `tracing-subscriber = \"0.3\"`\n\nImpact: Reproducible builds\nEffort: 5 minutes","status":"open","priority":1,"issue_type":"chore","created_at":"2025-10-26T21:21:10.53848+01:00","updated_at":"2025-10-26T21:21:10.53848+01:00","labels":["dependencies"]} +{"id":"scotty-27","title":"Wrap large configuration structures in Arc","description":"Settings struct contains large nested HashMaps (blueprints, registries) that get cloned unnecessarily. Wrapping them in Arc would reduce clone overhead.","design":"Location: scotty-core/src/settings/apps.rs\n\nSettings contains large, rarely modified structures that are cloned when Settings is cloned.\n\nProposed solution:\n```rust\n#[derive(Clone)]\npub struct Apps {\n pub root_folder: String,\n pub blueprints: Arc\u0026lt;HashMap\u0026lt;String, AppBlueprint\u0026gt;\u0026gt;, // Large, rarely modified\n // ...\n}\n```\n\nImpact: Reduce Settings clone overhead\nEffort: 1-2 hours","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-26T21:21:10.606337+01:00","updated_at":"2025-10-26T21:21:10.606337+01:00","labels":["performance","refactoring"]} +{"id":"scotty-28","title":"Consider DashMap for WebSocket client management","description":"WebSocket client HashMap uses Arc\u0026lt;Mutex\u0026lt;HashMap\u0026gt;\u0026gt; which causes lock contention during broadcasts. DashMap provides lock-free reads.","design":"Location: scotty/src/api/websocket/messaging.rs\n\nCurrent implementation uses mutex-protected HashMap which can cause contention during broadcasts to multiple clients.\n\nProposed solution:\n```rust\nuse dashmap::DashMap;\n\npub struct WebSocketMessenger {\n clients: Arc\u0026lt;DashMap\u0026lt;Uuid, WebSocketClient\u0026gt;\u0026gt;,\n}\n```\n\nBenefits:\n- Lock-free concurrent reads\n- Automatic sharding for better concurrency\n- Reduced contention during broadcasts\n\nImpact: Reduced lock contention during broadcasts\nEffort: 2-3 hours","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-26T21:21:10.680572+01:00","updated_at":"2025-10-26T21:21:10.680572+01:00","labels":["performance","websocket"]} +{"id":"scotty-29","title":"Split large authorization service file into submodules","description":"The authorization service.rs file is 754 lines and could be split into more focused submodules for better maintainability.","design":"Location: scotty/src/services/authorization/service.rs (754 lines)\n\nProposed structure:\n- service.rs - Core service and main authorization logic\n- permissions.rs - Permission checking and management\n- scopes.rs - Scope-related operations\n- roles.rs - Role management\n- assignments.rs - Assignment handling\n\nImpact: Better code organization and maintainability\nEffort: 2-4 hours","status":"open","priority":1,"issue_type":"chore","created_at":"2025-10-26T21:21:10.754914+01:00","updated_at":"2025-10-26T21:21:10.754914+01:00","labels":["maintainability","refactoring"]} +{"id":"scotty-3","title":"Frontend container log viewer UI","description":"Add UI to view container logs in the web frontend. Backend log streaming API is complete with WebSocket support, but frontend has no log viewer component.","design":"- Create log viewer component similar to unified-output.svelte\n- Add WebSocket handlers for LogLineReceived/LogStreamStarted/LogStreamEnded messages\n- Add log viewer page or modal accessible from app detail page\n- Support follow mode, timestamps, line limits\n- Reuse webSocketStore.ts infrastructure","acceptance_criteria":"- Can view historical logs for any service\n- Follow mode for real-time streaming\n- Toggle timestamps on/off\n- Auto-scroll control\n- Copy logs to clipboard\n- Integration from app detail page","status":"closed","priority":2,"issue_type":"feature","created_at":"2025-10-25T00:58:21.90526+02:00","updated_at":"2025-10-25T01:06:56.395577+02:00","closed_at":"2025-10-25T01:06:56.395577+02:00","dependencies":[{"issue_id":"scotty-3","depends_on_id":"scotty-1","type":"parent-child","created_at":"2025-10-25T00:58:32.507014+02:00","created_by":"daemon"}]} +{"id":"scotty-30","title":"Add metrics for clone operations in hot paths","description":"Add tracing/metrics to performance-critical clone operations to identify actual hotspots with real usage data.","design":"Add instrumentation to measure clone operations in:\n- AppData access patterns\n- Settings propagation\n- State machine handler contexts\n\nUse tracing spans with timing information to identify which clones actually impact performance in production.\n\nThis data will help prioritize which clone operations to optimize first.\n\nImpact: Data-driven optimization decisions\nEffort: 2-3 hours","status":"open","priority":1,"issue_type":"task","created_at":"2025-10-26T21:21:10.829701+01:00","updated_at":"2025-10-26T21:21:10.829701+01:00","labels":["observability","performance"]} +{"id":"scotty-31","title":"Refactor test code organization and extract common helpers","description":"Extract common test helpers and utilities into a dedicated test utilities module to reduce duplication across test files.","design":"Current state: Test code has duplicated setup/teardown logic and helper functions across multiple test files.\n\nProposed:\n- Create `scotty/tests/common/mod.rs` for shared test utilities\n- Extract common fixtures (test users, apps, configurations)\n- Create builder patterns for test data\n- Consolidate mock setup functions\n\nImpact: Reduce test code duplication, easier to maintain tests\nEffort: 4-6 hours","status":"open","priority":1,"issue_type":"chore","created_at":"2025-10-26T21:21:10.90688+01:00","updated_at":"2025-10-26T21:21:10.90688+01:00","labels":["maintainability","testing"]} +{"id":"scotty-4","title":"Frontend interactive shell UI with xterm.js","description":"Add interactive shell terminal to web frontend using xterm.js. Backend ShellService is complete with WebSocket support, but frontend has no shell UI.","design":"- Integrate xterm.js for terminal emulation\n- WebSocket handlers for ShellSession* messages\n- TTY resize support on window resize\n- Copy/paste support\n- Terminal settings (font size, theme)\n- Shell session management UI (list, create, terminate)\n- Session timeout indicator","acceptance_criteria":"- Can open interactive shell to any service from web UI\n- Terminal emulation works correctly (colors, escape sequences)\n- Copy/paste functional\n- Terminal resizes properly\n- Session list shows active shells\n- Clean session termination\n- Security: requires Shell permission","status":"open","priority":2,"issue_type":"feature","created_at":"2025-10-25T00:58:22.023108+02:00","updated_at":"2025-10-25T00:58:22.023108+02:00","dependencies":[{"issue_id":"scotty-4","depends_on_id":"scotty-1","type":"parent-child","created_at":"2025-10-25T00:58:32.576268+02:00","created_by":"daemon"},{"issue_id":"scotty-4","depends_on_id":"scotty-3","type":"related","created_at":"2025-10-25T00:58:39.410575+02:00","created_by":"daemon"}]} +{"id":"scotty-5","title":"Enhanced monitoring and observability","description":"Add comprehensive monitoring for the unified output system. Basic logging exists but metrics and tracing are incomplete.","design":"Implementation using OpenTelemetry Collector + VictoriaMetrics architecture.\n\nArchitecture:\n- Scotty exports OTLP metrics to OTel Collector (port 4317)\n- OTel Collector routes traces to Jaeger, metrics to VictoriaMetrics\n- Grafana visualizes metrics from VictoriaMetrics\n- Total memory overhead: ~180-250 MB\n\nSee docs/research/otel-metrics-backend-evaluation.md for complete research and rationale.","acceptance_criteria":"- Prometheus metrics exported\n- Grafana dashboard created\n- Tracing spans for log/shell operations\n- Memory usage tracked\n- Error rates monitored","notes":"Chosen solution: OTel Collector + VictoriaMetrics for lightweight, OpenTelemetry-native metrics collection. Integrates with existing Jaeger setup.","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-25T00:58:22.135422+02:00","updated_at":"2025-10-25T01:28:15.468691+02:00","dependencies":[{"issue_id":"scotty-5","depends_on_id":"scotty-1","type":"parent-child","created_at":"2025-10-25T00:58:32.652182+02:00","created_by":"daemon"}]} +{"id":"scotty-6","title":"End-user documentation for unified output system","description":"Write comprehensive end-user documentation for the logs and shell features. Technical PRD exists but user-facing docs are missing.","design":"- User guide for app:logs command with examples\n- User guide for app:shell command (once CLI implemented)\n- Web UI documentation for log viewer\n- Web UI documentation for shell access\n- Security best practices for shell access\n- Troubleshooting guide\n- Add to main documentation site","acceptance_criteria":"- app:logs documented with all options\n- app:shell documented (when available)\n- Screenshots of web UI features\n- Security guidelines clear\n- Common issues documented\n- Published to docs site","status":"open","priority":3,"issue_type":"task","created_at":"2025-10-25T00:58:22.241151+02:00","updated_at":"2025-10-25T00:58:22.241151+02:00","dependencies":[{"issue_id":"scotty-6","depends_on_id":"scotty-1","type":"parent-child","created_at":"2025-10-25T00:58:32.723437+02:00","created_by":"daemon"},{"issue_id":"scotty-6","depends_on_id":"scotty-2","type":"related","created_at":"2025-10-25T00:58:39.470922+02:00","created_by":"daemon"}]} +{"id":"scotty-7","title":"Add OpenTelemetry metrics dependencies to workspace","description":"Add metrics feature to opentelemetry and opentelemetry-otlp crates in workspace Cargo.toml. Enable metrics support in opentelemetry_sdk.","design":"Update workspace Cargo.toml:\n- opentelemetry: Add \"metrics\" feature\n- opentelemetry_sdk: Add \"metrics\" to features array\n- opentelemetry-otlp: Add \"metrics\" feature","acceptance_criteria":"- Cargo.toml updated with metrics features\n- cargo check passes\n- No breaking changes to existing trace functionality","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-25T01:28:15.598548+02:00","updated_at":"2025-10-25T01:32:41.399679+02:00","closed_at":"2025-10-25T01:32:41.399679+02:00","dependencies":[{"issue_id":"scotty-7","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:29.940257+02:00","created_by":"daemon"}]} +{"id":"scotty-8","title":"Create metrics module with ScottyMetrics struct","description":"Create scotty/src/metrics/mod.rs with ScottyMetrics struct containing all metric instruments (counters, gauges, histograms) for unified output system monitoring.","design":"Create metrics module with:\n- ScottyMetrics struct with all instruments\n- init_metrics() function to set up OTLP exporter\n- Metrics for: log streams, shell sessions, WebSocket, tasks, system health\n- Uses opentelemetry::metrics API (Counter, Gauge, Histogram)","acceptance_criteria":"- metrics/mod.rs created and compiles\n- ScottyMetrics struct has all planned metrics\n- init_metrics() successfully initializes MeterProvider\n- Metrics can be recorded without panics","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-25T01:28:15.721881+02:00","updated_at":"2025-10-25T01:50:03.687196+02:00","closed_at":"2025-10-25T01:50:03.687196+02:00","dependencies":[{"issue_id":"scotty-8","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:30.027423+02:00","created_by":"daemon"},{"issue_id":"scotty-8","depends_on_id":"scotty-7","type":"blocks","created_at":"2025-10-25T01:28:42.019675+02:00","created_by":"daemon"}]} +{"id":"scotty-9","title":"Instrument log streaming service with metrics","description":"Add metrics recording to LogStreamingService for stream counts, durations, errors, and bytes transferred.","design":"Instrument LogStreamingService:\n- Increment log_streams_active on stream start\n- Increment log_streams_total counter\n- Record log_stream_duration on completion\n- Track log_lines_received and log_stream_bytes\n- Increment log_stream_errors on failures","acceptance_criteria":"- Metrics recorded at stream start/end\n- Duration measured accurately\n- Error cases increment error counter\n- No performance degradation","status":"closed","priority":1,"issue_type":"task","created_at":"2025-10-25T01:28:15.847014+02:00","updated_at":"2025-10-25T01:57:04.063722+02:00","closed_at":"2025-10-25T01:57:04.063722+02:00","dependencies":[{"issue_id":"scotty-9","depends_on_id":"scotty-5","type":"parent-child","created_at":"2025-10-25T01:28:30.109241+02:00","created_by":"daemon"},{"issue_id":"scotty-9","depends_on_id":"scotty-8","type":"blocks","created_at":"2025-10-25T01:28:42.089191+02:00","created_by":"daemon"}]} diff --git a/.env.1password b/.env.1password new file mode 100644 index 00000000..94c20a52 --- /dev/null +++ b/.env.1password @@ -0,0 +1,3 @@ +SCOTTY__DOCKER__REGISTRIES__FACTORIAL__PASSWORD=op://Kubernetes/Deploybot Kubernetes token/password +SCOTTY__API__OAUTH__CLIENT_ID=op://Scotty/scotty local oauth gitlab/application_id +SCOTTY__API__OAUTH__CLIENT_SECRET=op://Scotty/scotty local oauth gitlab/Secret diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93bed177..bf8b2016 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,17 @@ jobs: steps: - uses: actions/checkout@v4 + + # Setup Rust toolchain for TypeScript generation + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + # Generate TypeScript types from Rust code + - name: Generate TypeScript types + run: cargo run + working-directory: ts-generator + - uses: oven-sh/setup-bun@v2 # run any `bun` or `bunx` command diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 3808dbee..205b0fe2 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -17,8 +17,6 @@ jobs: # github.event.pull_request.user.login == 'external-contributor' || # github.event.pull_request.user.login == 'new-developer' || # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' - - if: github.event.pull_request.user.type != 'Bot' runs-on: ubuntu-latest permissions: @@ -26,7 +24,7 @@ jobs: pull-requests: read issues: read id-token: write - + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -35,46 +33,25 @@ jobs: - name: Run Claude Code Review id: claude-review - uses: anthropics/claude-code-action@beta + uses: anthropics/claude-code-action@v1 with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - - # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1) - # model: "claude-opus-4-1-20250805" + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + prompt: | + REPO: ${{ github.repository }} + PR NUMBER: ${{ github.event.pull_request.number }} - # Direct prompt for automated review (no @claude mention needed) - direct_prompt: | Please review this pull request and provide feedback on: - Code quality and best practices - Potential bugs or issues - Performance considerations - Security concerns - Test coverage - - Be constructive and helpful in your feedback. - # Optional: Use sticky comments to make Claude reuse the same comment on subsequent pushes to the same PR - # use_sticky_comment: true - - # Optional: Customize review based on file types - # direct_prompt: | - # Review this PR focusing on: - # - For TypeScript files: Type safety and proper interface usage - # - For API endpoints: Security, input validation, and error handling - # - For React components: Performance, accessibility, and best practices - # - For tests: Coverage, edge cases, and test quality - - # Optional: Different prompts for different authors - # direct_prompt: | - # ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' && - # 'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' || - # 'Please provide a thorough code review focusing on our coding standards and best practices.' }} - - # Optional: Add specific tools for running tests or linting - # allowed_tools: "Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)" - - # Optional: Skip review for certain conditions - # if: | - # !contains(github.event.pull_request.title, '[skip-review]') && - # !contains(github.event.pull_request.title, '[WIP]') + Use the repository's CLAUDE.md for guidance on style and conventions. Be constructive and helpful in your feedback. + + Use `gh pr comment` with your Bash tool to leave your review as a comment on the PR. + + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://docs.claude.com/en/docs/claude-code/cli-reference for available options + claude_args: '--allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 196aea73..412cef9e 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -32,33 +32,19 @@ jobs: - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@beta + uses: anthropics/claude-code-action@v1 with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read - - # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1) - # model: "claude-opus-4-1-20250805" - - # Optional: Customize the trigger phrase (default: @claude) - # trigger_phrase: "/claude" - - # Optional: Trigger when specific user is assigned to an issue - # assignee_trigger: "claude-bot" - - # Optional: Allow Claude to run specific commands - # allowed_tools: "Bash(npm install),Bash(npm run build),Bash(npm run test:*),Bash(npm run lint:*)" - - # Optional: Add custom instructions for Claude to customize its behavior for your project - # custom_instructions: | - # Follow our coding standards - # Ensure all new code has tests - # Use TypeScript for new files - - # Optional: Custom environment variables for Claude - # claude_env: | - # NODE_ENV: test + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://docs.claude.com/en/docs/claude-code/cli-reference for available options + # claude_args: '--allowed-tools Bash(gh pr:*)' diff --git a/.gitignore b/.gitignore index 3bb4b45a..5698b4bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ /target +/ts-generator/target frontend/node_modules +frontend/src/generated/ +scotty-types/bindings/ docs/node_modules docs/_site apps/test-nginx-creation diff --git a/.zed/debug.json b/.zed/debug.json new file mode 100644 index 00000000..1c6170ec --- /dev/null +++ b/.zed/debug.json @@ -0,0 +1,18 @@ +// Project-local debug tasks +// +// For more documentation on how to configure debug tasks, +// see: https://zed.dev/docs/debugger +[ + { + "label": "Build & Debug scotty server", + "build": { + "command": "cargo", + "args": ["build", "--bin", "scotty"] + }, + "program": "$ZED_WORKTREE_ROOT/target/debug/scotty", + // sourceLanguages is required for CodeLLDB (not GDB) when using Rust + "sourceLanguages": ["rust"], + "request": "launch", + "adapter": "CodeLLDB" + } +] diff --git a/CLAUDE.md b/AGENTS.md similarity index 74% rename from CLAUDE.md rename to AGENTS.md index b2725adc..3d07cd1a 100644 --- a/CLAUDE.md +++ b/AGENTS.md @@ -1,4 +1,6 @@ -# CLAUDE.md +# AGENTS.md + +BEFORE ANYTHING ELSE: run 'bd onboard' and follow the instructions This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. @@ -7,7 +9,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Scotty is a Rust-based Micro-PaaS (Platform as a Service) that provides an API to manage Docker Compose-based applications. It consists of three main components: - **scotty**: HTTP server providing REST API and web UI for managing applications -- **scottyctl**: CLI application for interacting with the Scotty server +- **scottyctl**: CLI application for interacting with the Scotty server - **scotty-core**: Core library containing shared data structures and settings ## Architecture @@ -23,7 +25,7 @@ Scotty manages applications by: Scotty supports three authentication modes (configured via `auth_mode`): - **Development**: No authentication required, uses fixed dev user -- **OAuth**: Authentication via oauth2-proxy with GitLab OIDC integration +- **OAuth**: Authentication via oauth2-proxy with GitLab OIDC integration - **Bearer**: Traditional token-based authentication Authentication is handled by the `basic_auth.rs` middleware which extracts user information based on the configured mode. @@ -56,7 +58,7 @@ cd frontend # Install dependencies (use bun instead of npm) bun install -# Run development server +# Run development server bun run dev # Build for production @@ -109,7 +111,7 @@ cd examples/oauth2-proxy # Start in development mode (no auth) ./start-dev.sh dev -# Start with OAuth (requires GitLab app configuration) +# Start with OAuth (requires GitLab app configuration) op run --env-file="./.env.1password" -- ./start-dev.sh oauth --build # Start in bearer token mode @@ -137,7 +139,7 @@ op run --env-file="./.env.1password" -- ./start-dev.sh oauth --build apps/ ├── my-app/ │ ├── docker-compose.yml # Required -│ ├── .scotty.yml # Optional app settings +│ ├── .scotty.yml # Optional app settings │ ├── docker-compose.override.yml # Generated by Scotty │ └── ... (other app files) ``` @@ -162,15 +164,41 @@ Scotty generates appropriate configurations for: - Supports custom middlewares, basic auth, robots.txt prevention - Automatic SSL via Let's Encrypt integration -### HAProxy-Config (Legacy) +### HAProxy-Config (Legacy) - Uses environment variables for configuration - Limited feature set compared to Traefik ## Development Notes - Use workspace-level Cargo.toml for shared dependencies -- Frontend uses Bun instead of npm for package management +- Frontend uses Bun instead of npm for package management (62% faster builds) +- TypeScript generation optimized with standalone `ts-generator` crate (6s vs 27s) +- All shared types consolidated in `scotty-types` crate (single source of truth) +- Docker builds support multiple platforms (ARM64, x86_64, glibc, musl) - Conventional commits are enforced via git-cliff - Pre-push hooks via cargo-husky perform quality checks - Container apps directory must have identical paths on host and container for bind mounts -- Use conventional commit messages \ No newline at end of file +- Use conventional commit messages +- **IMPORTANT**: Always run quality checks before committing: + - Rust code: `cargo fmt` and `cargo clippy` + - Frontend code: `cd frontend && bun run lint` (runs Prettier and ESLint) + - If linting fails, run `bun run format` to auto-fix formatting issues + +## Current Work in Progress + +### Unified Output System Implementation + +**Branch:** `feat/better-logs-and-shell` + +See beads issues (scotty-1 and children) for current implementation status and remaining work. + +**Reference Documents:** +- `docs/prds/unified-output-system.md` - Complete PRD and technical specifications +- `docs/technical-spike-bollard-findings.md` - Bollard API validation results + +**Key Notes:** +- The build files of scotty_frontend will be embedded into scotty, so restart scotty after the frontend files got rebuilt +- Backend log streaming and shell services are fully implemented and tested +- CLI `app:logs` command is fully functional with real-time streaming +- Frontend task output viewer uses WebSocket integration +- Breaking change: TaskDetails no longer contains stdout/stderr fields (uses WebSocket streaming instead) diff --git a/Cargo.lock b/Cargo.lock index 6018272c..61d3a5fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,40 +197,13 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper 1.0.2", - "tower 0.5.2", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" dependencies = [ - "axum-core 0.5.5", + "axum-core", "axum-macros", "base64 0.22.1", "bytes", @@ -242,7 +215,7 @@ dependencies = [ "hyper 1.7.0", "hyper-util", "itoa", - "matchit 0.8.4", + "matchit", "memchr", "mime", "percent-encoding", @@ -255,32 +228,12 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-tungstenite", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper 1.0.2", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.5" @@ -320,7 +273,7 @@ dependencies = [ "anyhow", "assert-json-diff", "auto-future", - "axum 0.8.6", + "axum", "bytes", "bytesize", "cookie", @@ -337,23 +290,24 @@ dependencies = [ "serde_urlencoded", "smallvec", "tokio", - "tower 0.5.2", + "tower", "url", ] [[package]] name = "axum-tracing-opentelemetry" -version = "0.26.1" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248d7a5925bf89cc9997882d3d0768061477c9b5998959fcebaa5bf8ba4d50dc" +checksum = "328c8ddd5ca871b2a5acb00be0b4f103aa62f5d6b6db4071ccf3b12b0629e7c1" dependencies = [ - "axum 0.8.6", + "axum", "futures-core", "futures-util", "http 1.3.1", "opentelemetry", + "opentelemetry-semantic-conventions", "pin-project-lite", - "tower 0.5.2", + "tower", "tracing", "tracing-opentelemetry", "tracing-opentelemetry-instrumentation-sdk", @@ -790,6 +744,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1583,7 +1556,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -1746,9 +1719,9 @@ dependencies = [ [[package]] name = "init-tracing-opentelemetry" -version = "0.27.1" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa368e50bd23d6445105987d7dc4fed10d93b3dc4fc9f38b25f691edb7c3b352" +checksum = "3405b8beafca487bde01d88e05e6646c4e5c6a3c9830e7b937290ea8f7f5e5d9" dependencies = [ "opentelemetry", "opentelemetry-otlp", @@ -1867,6 +1840,16 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +[[package]] +name = "libyml" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980" +dependencies = [ + "anyhow", + "version_check", +] + [[package]] name = "libz-rs-sys" version = "0.5.2" @@ -1930,12 +1913,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -2053,6 +2030,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2179,9 +2165,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.28.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", @@ -2193,26 +2179,23 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.28.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8863faf2910030d139fb48715ad5ff2f35029fc5f244f6d5f689ddcf4d26253" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", "http 1.3.1", "opentelemetry", "reqwest 0.12.24", - "tracing", ] [[package]] name = "opentelemetry-otlp" -version = "0.28.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bef114c6d41bea83d6dc60eb41720eedd0261a67af57b66dd2b84ac46c01d91" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" dependencies = [ - "async-trait", - "futures-core", "http 1.3.1", "opentelemetry", "opentelemetry-http", @@ -2223,40 +2206,40 @@ dependencies = [ "thiserror 2.0.17", "tokio", "tonic", + "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.28.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8870d3024727e99212eb3bb1762ec16e255e3e6f58eeb3dc8db1aa226746d" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", "prost", "tonic", + "tonic-prost", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.28.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fb3a2f78c2d55362cd6c313b8abedfbc0142ab3c2676822068fd2ab7d51f9b7" +checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" [[package]] name = "opentelemetry_sdk" -version = "0.28.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ - "async-trait", "futures-channel", "futures-executor", "futures-util", - "glob", "opentelemetry", "percent-encoding", - "rand 0.8.5", + "rand 0.9.2", "thiserror 2.0.17", "tokio", "tokio-stream", @@ -2496,9 +2479,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", "prost-derive", @@ -2506,9 +2489,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", "itertools", @@ -2657,6 +2640,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "readonly" version = "0.2.13" @@ -2804,7 +2807,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls 0.26.4", - "tower 0.5.2", + "tower", "tower-http", "tower-service", "url", @@ -2975,6 +2978,7 @@ version = "0.23.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -3101,7 +3105,7 @@ version = "0.2.0-alpha.1" dependencies = [ "anyhow", "async-trait", - "axum 0.8.6", + "axum", "axum-test", "axum-tracing-opentelemetry", "base64 0.22.1", @@ -3118,20 +3122,27 @@ dependencies = [ "init-tracing-opentelemetry", "maplit", "mime_guess", + "num_cpus", "oauth2", + "once_cell", "opentelemetry", + "opentelemetry-otlp", "opentelemetry_sdk", "path-clean", "readonly", "regex", "reqwest 0.12.24", "scotty-core", + "scotty-types", "serde", "serde_json", "serde_norway", + "serde_yml", + "sysinfo", "tempfile", "thiserror 2.0.17", "tokio", + "tokio-metrics", "tokio-stream", "tokio-test", "tower-http", @@ -3155,7 +3166,7 @@ version = "0.2.0-alpha.1" dependencies = [ "anyhow", "async-trait", - "axum 0.8.6", + "axum", "bollard", "cargo-husky", "chrono", @@ -3164,6 +3175,7 @@ dependencies = [ "deunicode", "readonly", "reqwest 0.12.24", + "scotty-types", "secrecy", "semver", "serde", @@ -3179,6 +3191,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "scotty-ts-generator" +version = "0.2.0-alpha.1" +dependencies = [ + "scotty-types", +] + +[[package]] +name = "scotty-types" +version = "0.2.0-alpha.1" +dependencies = [ + "chrono", + "serde", + "ts-rs", + "utoipa", + "uuid", +] + [[package]] name = "scottyctl" version = "0.2.0-alpha.1" @@ -3190,18 +3220,23 @@ dependencies = [ "clap_complete", "crossterm", "dotenvy", + "futures-util", "open", "owo-colors", "reqwest 0.12.24", "scotty-core", + "scotty-types", "semver", "serde", "serde_json", "tabled", "thiserror 1.0.69", "tokio", + "tokio-tungstenite", "tracing", "tracing-subscriber", + "url", + "uuid", "walkdir", ] @@ -3400,6 +3435,21 @@ dependencies = [ "time", ] +[[package]] +name = "serde_yml" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" +dependencies = [ + "indexmap 2.12.0", + "itoa", + "libyml", + "memchr", + "ryu", + "serde", + "version_check", +] + [[package]] name = "sha1" version = "0.10.6" @@ -3602,6 +3652,20 @@ dependencies = [ "syn", ] +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -3689,6 +3753,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "testing_table" version = "0.3.0" @@ -3849,6 +3922,18 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-metrics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-native-tls" version = "0.3.1" @@ -3961,16 +4046,13 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.3" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", "base64 0.22.1", "bytes", - "h2 0.4.12", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -3979,34 +4061,25 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", - "socket2 0.5.10", + "sync_wrapper 1.0.2", "tokio", + "tokio-rustls 0.26.4", "tokio-stream", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", ] [[package]] -name = "tower" -version = "0.4.13" +name = "tonic-prost" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", + "bytes", + "prost", + "tonic", ] [[package]] @@ -4017,9 +4090,12 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap 2.12.0", "pin-project-lite", + "slab", "sync_wrapper 1.0.2", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -4047,7 +4123,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", @@ -4111,15 +4187,16 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.29.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721f2d2569dce9f3dfbbddee5906941e953bfcdf736a62da3377f5751650cc36" +checksum = "1e6e5658463dd88089aba75c7791e1d3120633b1bfde22478b28f625a9bb1b8e" dependencies = [ "js-sys", - "once_cell", "opentelemetry", "opentelemetry_sdk", + "rustversion", "smallvec", + "thiserror 2.0.17", "tracing", "tracing-core", "tracing-log", @@ -4129,12 +4206,13 @@ dependencies = [ [[package]] name = "tracing-opentelemetry-instrumentation-sdk" -version = "0.26.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cae2c7a01582abc7b0a4672f92c47411b69cd3967b8b79bb743d5d0991c9089" +checksum = "7a1a4dcfb798af2cef9e47c30a14e13c108b4b40e057120401b2025ec622c416" dependencies = [ "http 1.3.1", "opentelemetry", + "opentelemetry-semantic-conventions", "tracing", "tracing-opentelemetry", ] @@ -4182,6 +4260,31 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ts-rs" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6" +dependencies = [ + "chrono", + "lazy_static", + "thiserror 2.0.17", + "ts-rs-macros", + "uuid", +] + +[[package]] +name = "ts-rs-macros" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9d8656589772eeec2cf7a8264d9cda40fb28b9bc53118ceb9e8c07f8f38730" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "termcolor", +] + [[package]] name = "tungstenite" version = "0.28.0" @@ -4320,7 +4423,7 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5f8f5abd341cce16bb4f09a8bafc087d4884a004f25fb980e538d51d6501dab" dependencies = [ - "axum 0.8.6", + "axum", "serde", "serde_json", "utoipa", @@ -4332,7 +4435,7 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6427547f6db7ec006cbbef95f7565952a16f362e298b416d2d497d9706fef72d" dependencies = [ - "axum 0.8.6", + "axum", "serde", "serde_json", "utoipa", @@ -4344,7 +4447,7 @@ version = "9.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d047458f1b5b65237c2f6dc6db136945667f40a7668627b3490b9513a3d43a55" dependencies = [ - "axum 0.8.6", + "axum", "base64 0.22.1", "mime_guess", "regex", @@ -4593,19 +4696,52 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement 0.57.0", + "windows-interface 0.57.0", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link 0.2.1", "windows-result 0.4.1", "windows-strings 0.5.1", ] +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -4617,6 +4753,17 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-interface" version = "0.59.3" @@ -4651,6 +4798,15 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.3.4" diff --git a/Cargo.toml b/Cargo.toml index 178284db..e24eda0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["scotty-core", "scotty", "scottyctl"] +members = ["scotty-core", "scotty", "scottyctl", "scotty-types", "ts-generator"] resolver = "2" [workspace.package] @@ -31,23 +31,28 @@ tokio = { version = "1.44.2", features = [ "signal", "process", ] } +tokio-metrics = "0.3" tracing = "0.1.40" tracing-subscriber = "*" -opentelemetry = { version = "0.28.0", features = ["trace"] } -opentelemetry_sdk = { version = "0.28", features = [ +ts-rs = "10.0" +opentelemetry = { version = "0.31.0", features = ["trace", "metrics"] } +opentelemetry_sdk = { version = "0.31", features = [ "rt-tokio", + "metrics", ], default-features = false } -tracing-opentelemetry = "0.29" -init-tracing-opentelemetry = { version = "0.27.0", features = [ +tracing-opentelemetry = "0.32" +init-tracing-opentelemetry = { version = "0.32.1", features = [ "otlp", "tracing_subscriber_ext", ] } -axum-tracing-opentelemetry = "0.26.0" -opentelemetry-otlp = { version = "0.28.0", features = [ +axum-tracing-opentelemetry = "0.32.1" +opentelemetry-otlp = { version = "0.31.0", features = [ + "grpc-tonic", "reqwest-client", "reqwest-rustls", "http-proto", "tls", + "metrics", ] } utoipa = { version = "5.3", features = ["axum_extras", "uuid", "chrono"] } @@ -85,6 +90,9 @@ semver = "1.0" casbin = { version = "2.8", default-features = false, features = ["runtime-tokio", "cached"] } secrecy = { version = "0.10", features = ["serde"] } zeroize = "1.8" +sysinfo = "0.33" +once_cell = "1.20" +num_cpus = "1.16" [workspace.metadata.release] pre-release-hook = ["sh", "-c", "git cliff -o CHANGELOG.md --tag {{version}}"] diff --git a/Dockerfile b/Dockerfile index 96bd2315..624974c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,5 @@ -# First build the frontend -FROM node:22 as frontend-builder -WORKDIR /app -COPY ./frontend /app -RUN yarn install && yarn build - -# Now build the backend with the frontend files embedded -FROM rust:1.89-slim-bookworm as chef +# Base Rust environment with cargo-chef +FROM rust:1.89-slim-bookworm AS chef RUN apt-get update -y && \ apt-get install --no-install-recommends -y pkg-config make g++ libssl-dev curl jq && \ rustup target add x86_64-unknown-linux-gnu && \ @@ -16,10 +10,34 @@ RUN cargo install cargo-chef WORKDIR /app FROM chef AS planner -COPY --from=frontend-builder /app/build /app/frontend/build COPY . . RUN cargo chef prepare --recipe-path recipe.json +# TypeScript generation stage using cargo-chef +FROM chef AS ts-generator +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --recipe-path recipe.json --package scotty-ts-generator +COPY . . +RUN cargo run --package scotty-ts-generator + +# Frontend build stage +FROM oven/bun:1 AS frontend-builder +WORKDIR /app +COPY frontend/package.json frontend/bun.lockb ./ +RUN bun install --frozen-lockfile +# Install all potential Rollup platform binaries as optional dependencies +# This ensures the build works across different target platforms +RUN bun add \ + @rollup/rollup-linux-x64-gnu \ + @rollup/rollup-linux-arm64-gnu \ + @rollup/rollup-linux-x64-musl \ + @rollup/rollup-linux-arm64-musl \ + --optional +COPY frontend/ ./ +COPY --from=ts-generator /app/frontend/src/generated ./src/generated +RUN bun run build + +# Main application build stage FROM chef AS builder COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json diff --git a/README.md b/README.md index 0bdb3c31..536aacad 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,49 @@ scottyctl --server https://localhost:21342 --access-token your_secure_bearer_tok **Security Note**: Server administrators should **never store actual bearer tokens in configuration files**. Instead, use placeholder values in config files and set actual secure tokens via environment variables like `SCOTTY__API__BEARER_TOKENS__ADMIN=your_secure_token`. See the [configuration documentation](docs/content/configuration.md) for security best practices. +## Observability + +Scotty includes a comprehensive observability stack with metrics, distributed tracing, and pre-built dashboards for monitoring application health and performance. + +### Quick Start + +Start the observability stack (Grafana, Jaeger, VictoriaMetrics, OpenTelemetry Collector): + +```shell +cd observability +docker-compose up -d +``` + +Enable telemetry in Scotty: + +```shell +SCOTTY__TELEMETRY=metrics,traces cargo run --bin scotty +``` + +Access the services: +- **Grafana Dashboard**: http://grafana.ddev.site (admin/admin) +- **Jaeger Tracing**: http://jaeger.ddev.site +- **VictoriaMetrics**: http://vm.ddev.site + +### What's Monitored + +Scotty exports 40+ metrics covering: +- Log streaming (active streams, throughput, errors) +- Shell sessions (active connections, timeouts) +- WebSocket connections and message rates +- Task execution and output streaming +- HTTP server performance by endpoint +- Memory usage (RSS and virtual) +- Application fleet metrics +- Tokio async runtime health + +### Documentation + +For complete setup instructions, metrics reference, and production deployment guide: + +📖 **[Observability Documentation](docs/content/observability.md)** +📖 **[Observability Setup Guide](observability/README.md)** + ## Developing/Contributing We welcome contributions! Please fork the repository, create a diff --git a/config/casbin/policy.yaml b/config/casbin/policy.yaml index e42a16e4..83ed1b28 100644 --- a/config/casbin/policy.yaml +++ b/config/casbin/policy.yaml @@ -1,65 +1,73 @@ scopes: - client-b: - description: Client B - created_at: '2024-01-01T00:00:00Z' - qa: - description: QA - created_at: '2024-01-01T00:00:00Z' client-a: description: Client A created_at: '2024-01-01T00:00:00Z' + client-b: + description: Client B + created_at: '2024-01-01T00:00:00Z' default: description: Default scope for unassigned apps created_at: '2024-01-01T00:00:00Z' + qa: + description: QA + created_at: '2024-01-01T00:00:00Z' roles: - viewer: - permissions: - - view - description: Read-only access - developer: + operator: permissions: - view - manage - - shell - logs - - create - description: Developer access - all except destroy + description: Operations team - no shell or destroy admin: permissions: - '*' description: Full system access - operator: + viewer: + permissions: + - view + description: Read-only access + developer: permissions: - view - manage + - shell - logs - description: Operations team - no shell or destroy + - create + description: Developer access - all except destroy assignments: - identifier:client-a: - - role: developer + dev:system:internal: + - role: admin + scopes: + - '*' + identifier:test-bearer-token-123: + - role: admin scopes: - client-a + - client-b + - qa + - default + stephan@factorial.io: + - role: admin + scopes: + - '*' identifier:hello-world: - role: developer scopes: - client-a - client-b - qa - identifier:test-bearer-token-123: + identifier:admin: - role: admin scopes: - client-a - client-b - qa - default + identifier:client-a: + - role: developer + scopes: + - client-a '*': - role: viewer scopes: - default - identifier:admin: - - role: admin - scopes: - - client-a - - client-b - - qa - - default diff --git a/config/default.yaml b/config/default.yaml index 8c6b605b..73a4d626 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -15,7 +15,7 @@ scheduler: running_app_check: "15m" ttl_check: "10m" task_cleanup: "3m" -telemetry: None +telemetry: None # Options: None, "traces", "metrics", or "traces,metrics" for both apps: domain_suffix: "ddev.site" root_folder: "./apps" # Path to the folder where the apps are stored diff --git a/config/local.yaml b/config/local.yaml index 3e4ff37f..7f0da0c3 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -1,6 +1,6 @@ api: bind_address: "127.0.0.1:21342" - auth_mode: bearer + auth_mode: oauth oauth: oidc_issuer_url: "https://source.factorial.io" client_id: "your_client_id" diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 535009cd..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -#ddev-generated -services: - jaeger: - image: jaegertracing/all-in-one:${JAEGER_VERSION:-latest} - ports: - - "4318:4318" - environment: - - LOG_LEVEL=debug - labels: - - "traefik.enable=true" - - "traefik.http.routers.jaeger.rule=Host(`jaeger.ddev.site`)" - - "traefik.http.services.jaeger.loadbalancer.server.port=16686" diff --git a/docs/.fdocs.js b/docs/.fdocs.js index 4eceabef..5815d711 100644 --- a/docs/.fdocs.js +++ b/docs/.fdocs.js @@ -28,8 +28,10 @@ export default function (defaultConfig) { "architecture", "installation", "configuration", + "observability", "cli", "oauth-authentication", + "authorization", "changelog", ], }); diff --git a/docs/content/configuration.md b/docs/content/configuration.md index e096e9fc..103d8f2d 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -61,7 +61,7 @@ api: deployment: "placeholder-will-be-overridden" create_app_max_size: "50M" auth_mode: "bearer" # "dev", "oauth", or "bearer" - dev_user_email: "dev@localhost" + dev_user_email: "dev:system:internal" dev_user_name: "Dev User" oauth: oidc_issuer_url: "https://gitlab.com" @@ -138,16 +138,38 @@ roles: created_at: "2023-12-01T00:00:00Z" # User/token assignments to roles within scopes +# Assignment key format depends on authentication mode: +# +# OAuth Mode (auth_mode: "oauth"): +# - Use email addresses directly: "user@example.com" +# - Email is extracted from OIDC token claims during authentication +# +# Bearer Mode (auth_mode: "bearer"): +# - Use identifier prefix: "identifier:token_name" +# - Maps to configured bearer_tokens (api.bearer_tokens.token_name) +# +# Dev Mode (auth_mode: "dev"): +# - Uses fixed dev user from api.dev_user_* configuration +# - Authorization assignments are not applicable assignments: + # OAuth user assignments (when auth_mode: "oauth") "alice@example.com": - role: "admin" - scopes: ["*"] # Global access + scopes: ["*"] # Global access to all scopes + "stephan@factorial.io": + - role: "admin" + scopes: ["*"] # Global admin access "bob@example.com": - role: "developer" scopes: ["frontend", "backend"] + + # Bearer token assignments (when auth_mode: "bearer") "identifier:deployment": # Maps to bearer_tokens.deployment - role: "developer" scopes: ["staging"] + "identifier:admin": # Maps to bearer_tokens.admin + - role: "admin" + scopes: ["*"] # App scope mappings (managed automatically from .scotty.yml) apps: diff --git a/docs/content/observability.md b/docs/content/observability.md new file mode 100644 index 00000000..4c1efd1d --- /dev/null +++ b/docs/content/observability.md @@ -0,0 +1,486 @@ +# Observability + +Scotty includes a comprehensive observability stack for monitoring application health, performance, and behavior. The stack provides metrics, distributed tracing, and visualization through industry-standard tools. + +## Architecture + +``` +Scotty Application + ↓ (OTLP over gRPC) +OpenTelemetry Collector (port 4317) + ├─→ Jaeger (distributed traces) + └─→ VictoriaMetrics (metrics storage) + ↓ + Grafana (visualization & dashboards) +``` + +### Components + +- **OpenTelemetry Collector**: Receives telemetry data from Scotty via OTLP protocol and routes it to appropriate backends +- **VictoriaMetrics**: High-performance time-series database for metrics storage (30-day retention) +- **Jaeger**: Distributed tracing backend for request traces and spans +- **Grafana**: Visualization platform with pre-configured dashboards + +### Resource Usage + +The observability stack requires approximately: +- **Memory**: 180-250 MB total +- **CPU**: Minimal (< 5% on modern systems) +- **Disk**: ~1-2 GB for 30 days of metrics retention + +## Prometheus Compatibility & Flexibility + +**All metrics are fully Prometheus-compatible.** The stack uses open standards (OTLP, PromQL, W3C Trace Context) and components are interchangeable. + +### Metric Format + +- OpenTelemetry format (`scotty.metric.name`) → Prometheus format (`scotty_metric_name_total`) +- Standard types: Counter, Gauge, Histogram, UpDownCounter +- Attributes become labels (`method`, `status`, `path`) + +### Replace Components as Needed + +**Use Prometheus instead of VictoriaMetrics:** +Update `otel-collector-config.yaml` exporter from `prometheusremotewrite` to `prometheus` endpoint, then swap VictoriaMetrics for Prometheus in docker-compose. + +**Alternative backends:** Thanos, Cortex, M3DB, InfluxDB, Datadog, New Relic, Honeycomb, Grafana Cloud + +**Alternative visualization:** Prometheus UI, VictoriaMetrics vmui, Chronograf, commercial dashboards + +**Alternative tracing:** Zipkin, Tempo, Elasticsearch + Jaeger, Lightstep, Honeycomb + +**Multi-backend export example:** +```yaml +# otel-collector-config.yaml - export to multiple destinations +service: + pipelines: + metrics: + exporters: [prometheusremotewrite/victoria, prometheusremotewrite/thanos, otlp/datadog] +``` + +### Integration Patterns + +**Remote write to existing Prometheus:** +```yaml +exporters: + prometheusremotewrite: + endpoint: "https://your-prometheus.company.com/api/v1/write" +``` + +**Federation from VictoriaMetrics:** +```yaml +# prometheus.yml +scrape_configs: + - job_name: 'scotty' + metrics_path: '/api/v1/export/prometheus' + params: + match[]: ['{__name__=~"scotty_.*"}'] + static_configs: + - targets: ['victoriametrics:8428'] +``` + +**Service discovery:** Standard Kubernetes/Consul Prometheus SD works with VictoriaMetrics API. + +### Why VictoriaMetrics Default + +Chosen for development convenience: lower memory usage, single binary, Prometheus-compatible, free. Swap for Prometheus in production if preferred. + +## Quick Start + +### Prerequisites + +The observability stack requires Traefik for .ddev.site domain routing. Start Traefik first: + +```bash +cd apps/traefik +docker-compose up -d +``` + +### Starting the Observability Stack + +```bash +cd observability +docker-compose up -d +``` + +This will start all four services: +- OpenTelemetry Collector +- VictoriaMetrics +- Jaeger +- Grafana + +### Enabling Metrics in Scotty + +Configure Scotty to export telemetry data using the `SCOTTY__TELEMETRY` environment variable: + +**Enable both metrics and traces:** +```bash +SCOTTY__TELEMETRY=metrics,traces cargo run --bin scotty +``` + +**Enable only metrics:** +```bash +SCOTTY__TELEMETRY=metrics cargo run --bin scotty +``` + +**Production deployment** (in docker-compose.yml or .env): +```yaml +environment: + - SCOTTY__TELEMETRY=metrics,traces +``` + +### Accessing Services + +Once running, access the services at: + +| Service | URL | Credentials | +|---------|-----|-------------| +| Grafana | http://grafana.ddev.site | admin/admin | +| Jaeger UI | http://jaeger.ddev.site | (none) | +| VictoriaMetrics | http://vm.ddev.site | (none) | + +## Available Metrics + +Scotty exports comprehensive metrics covering all major subsystems. All metrics use the `scotty.` prefix. + +### Log Streaming Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_log_streams_active` | Gauge | Number of active log streams | +| `scotty_log_streams_total` | Counter | Total log streams created | +| `scotty_log_stream_duration_seconds` | Histogram | Duration of log streaming sessions | +| `scotty_log_stream_lines_total` | Counter | Total log lines streamed to clients | +| `scotty_log_stream_errors_total` | Counter | Log streaming errors | + +**Use Cases:** +- Monitor concurrent log stream load +- Detect log streaming errors +- Analyze log stream duration patterns + +### Shell Session Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_shell_sessions_active` | Gauge | Number of active shell sessions | +| `scotty_shell_sessions_total` | Counter | Total shell sessions created | +| `scotty_shell_session_duration_seconds` | Histogram | Shell session duration | +| `scotty_shell_session_errors_total` | Counter | Shell session errors | +| `scotty_shell_session_timeouts_total` | Counter | Sessions ended due to timeout | + +**Use Cases:** +- Monitor active shell connections +- Track session timeout rates +- Identify shell session errors + +### WebSocket Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_websocket_connections_active` | Gauge | Active WebSocket connections | +| `scotty_websocket_messages_sent_total` | Counter | Messages sent to clients | +| `scotty_websocket_messages_received_total` | Counter | Messages received from clients | +| `scotty_websocket_auth_failures_total` | Counter | WebSocket authentication failures | + +**Use Cases:** +- Monitor real-time connection count +- Track message throughput +- Detect authentication issues + +### Task Output Streaming Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_tasks_active` | Gauge | Active task output streams | +| `scotty_tasks_total` | Counter | Total tasks executed | +| `scotty_task_duration_seconds` | Histogram | Task execution duration | +| `scotty_task_failures_total` | Counter | Failed tasks | +| `scotty_task_output_lines_total` | Counter | Task output lines streamed | + +**Use Cases:** +- Monitor task execution load +- Track task failure rates +- Analyze output streaming performance + +### HTTP Server Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_http_requests_active` | UpDownCounter | Currently processing requests | +| `scotty_http_requests_total` | Counter | Total HTTP requests | +| `scotty_http_request_duration_seconds` | Histogram | Request processing time | + +**Attributes:** +- `method`: HTTP method (GET, POST, etc.) +- `path`: Request path +- `status`: HTTP status code + +**Use Cases:** +- Monitor API endpoint performance +- Track request rates by endpoint +- Identify slow requests + +### Memory Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_memory_rss_bytes` | Gauge | Resident Set Size (RSS) in bytes | +| `scotty_memory_virtual_bytes` | Gauge | Virtual memory size in bytes | + +**Use Cases:** +- Monitor memory consumption +- Detect memory leaks +- Capacity planning + +### Application Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_apps_total` | Gauge | Total managed applications | +| `scotty_apps_by_status` | Gauge | Apps grouped by status | +| `scotty_app_services_count` | Histogram | Services per application distribution | +| `scotty_app_last_check_age_seconds` | Histogram | Time since last health check | + +**Attributes:** +- `status`: Application status (running, stopped, etc.) + +**Use Cases:** +- Monitor application fleet size +- Track application health check timeliness +- Analyze service distribution + +### Tokio Runtime Metrics + +| Metric Name | Type | Description | +|-------------|------|-------------| +| `scotty_tokio_workers_count` | Gauge | Number of Tokio worker threads | +| `scotty_tokio_tasks_active` | Gauge | Active instrumented tasks | +| `scotty_tokio_tasks_dropped_total` | Counter | Completed/dropped tasks | +| `scotty_tokio_poll_count_total` | Counter | Total task polls | +| `scotty_tokio_poll_duration_seconds` | Histogram | Task poll duration | +| `scotty_tokio_slow_poll_count_total` | Counter | Slow task polls (>1ms) | +| `scotty_tokio_idle_duration_seconds` | Histogram | Task idle time between polls | +| `scotty_tokio_scheduled_count_total` | Counter | Task scheduling events | +| `scotty_tokio_first_poll_delay_seconds` | Histogram | Delay from creation to first poll | + +**Use Cases:** +- Monitor async runtime health +- Detect slow tasks blocking the runtime +- Optimize task scheduling + +## Grafana Dashboard + +Scotty includes a pre-configured Grafana dashboard (`scotty-metrics.json`) that visualizes all available metrics. + +### Dashboard Sections + +1. **Log Streaming**: Active streams, throughput, duration percentiles, errors +2. **Shell Sessions**: Active sessions, creation rate, duration, errors & timeouts +3. **WebSocket & Tasks**: Connection metrics, message rates, task execution +4. **Memory Usage**: RSS and virtual memory trends +5. **HTTP Server**: Request rates, active requests, latencies +6. **Tokio Runtime**: Worker threads, task lifecycle, poll metrics +7. **Application Metrics**: App count, status distribution, health checks + +### Accessing the Dashboard + +1. Open Grafana: http://grafana.ddev.site +2. Login with `admin` / `admin` (change on first login) +3. Navigate to **Dashboards** → **Scotty Metrics** + +The dashboard auto-refreshes every 5 seconds and shows data from the last hour by default. + +## PromQL Query Examples + +### Request Rate by HTTP Status + +```promql +sum by (status) (rate(scotty_http_requests_total[5m])) +``` + +### P95 Request Latency + +```promql +histogram_quantile(0.95, rate(scotty_http_request_duration_seconds_bucket[5m])) +``` + +### WebSocket Connection Churn + +```promql +rate(scotty_websocket_connections_total[5m]) +``` + +### Memory Growth Rate + +```promql +deriv(scotty_memory_rss_bytes[10m]) +``` + +### Active Resources Summary + +```promql +# All active resources +scotty_log_streams_active + +scotty_shell_sessions_active + +scotty_websocket_connections_active + +scotty_tasks_active +``` + +## Distributed Tracing + +When traces are enabled (`SCOTTY__TELEMETRY=traces` or `metrics,traces`), Scotty exports distributed traces to Jaeger. + +### Viewing Traces + +1. Open Jaeger UI: http://jaeger.ddev.site +2. Select **scotty** service +3. Search for traces by operation or timeframe + +### Key Operations + +- `HTTP POST /apps/create`: Application creation +- `HTTP GET /apps/info/{name}`: Application info retrieval +- `log_stream_handler`: Log streaming operations +- `shell_session_handler`: Shell session management + +Traces include timing information, error status, and contextual metadata for debugging request flows. + +## Troubleshooting + +### No Metrics Appearing in Grafana + +1. **Check Scotty is exporting metrics:** + ```bash + # Verify SCOTTY__TELEMETRY is set + echo $SCOTTY__TELEMETRY + + # Should be 'metrics' or 'metrics,traces' + ``` + +2. **Verify OpenTelemetry Collector is receiving data:** + ```bash + docker logs otel-collector + # Look for: "Trace received" + ``` + +3. **Check VictoriaMetrics has data:** + ```bash + curl http://vm.ddev.site/api/v1/label/__name__/values | jq + # Should list scotty_* metrics + ``` + +4. **Restart the stack:** + ```bash + cd observability + docker-compose restart + ``` + +### High Memory Usage + +If VictoriaMetrics uses too much memory, adjust retention: + +```yaml +# observability/docker-compose.yml +services: + victoriametrics: + command: + - '-retentionPeriod=14d' # Reduce from 30d +``` + +### Connection Refused Errors + +Ensure Traefik is running: +```bash +docker ps | grep traefik +cd apps/traefik +docker-compose up -d +``` + +### Grafana Dashboard Not Loading + +1. Check dashboard file exists: `observability/grafana/dashboards/scotty-metrics.json` +2. Restart Grafana: `docker-compose restart grafana` +3. Check Grafana logs: `docker logs grafana` + +## Configuration + +### OpenTelemetry Collector + +Configuration file: `observability/otel-collector-config.yaml` + +Key settings: +- **OTLP Receiver**: Port 4317 (gRPC) +- **Exporters**: Jaeger (traces), Prometheus Remote Write (metrics to VictoriaMetrics) +- **Batch Processor**: Batches telemetry for efficiency + +### VictoriaMetrics + +Configuration via docker-compose environment: +- **Retention**: 30 days (`-retentionPeriod=30d`) +- **Storage path**: `/victoria-metrics-data` +- **HTTP port**: 8428 + +### Grafana + +Configuration in `observability/grafana/provisioning/`: +- **Datasources**: VictoriaMetrics (Prometheus type) +- **Dashboards**: Auto-provisioned from `dashboards/` directory + +## Production Recommendations + +### Resource Allocation + +For production deployments, allocate resources based on scale: + +**Small deployment** (< 10 apps): +- VictoriaMetrics: 256 MB memory +- OpenTelemetry Collector: 128 MB memory +- Grafana: 256 MB memory + +**Medium deployment** (10-50 apps): +- VictoriaMetrics: 512 MB memory +- OpenTelemetry Collector: 256 MB memory +- Grafana: 512 MB memory + +**Large deployment** (50+ apps): +- VictoriaMetrics: 1 GB+ memory +- OpenTelemetry Collector: 512 MB memory +- Grafana: 512 MB memory + +### Alerting + +Configure Grafana alerts for critical metrics: + +- **High error rate**: `rate(scotty_http_requests_total{status="500"}[5m]) > 0.1` +- **Memory leak**: `deriv(scotty_memory_rss_bytes[30m]) > 1000000` +- **High WebSocket failures**: `rate(scotty_websocket_auth_failures_total[5m]) > 1` +- **Task failures**: `rate(scotty_task_failures_total[5m]) > 0.5` + +### Data Retention + +Adjust retention based on compliance and capacity: + +```yaml +# observability/docker-compose.yml +services: + victoriametrics: + command: + - '-retentionPeriod=90d' # 3 months for compliance +``` + +### Security + +**Production checklist:** +- [ ] Change Grafana default password +- [ ] Enable Grafana authentication (OAuth, LDAP, etc.) +- [ ] Use TLS for Grafana access +- [ ] Restrict Jaeger UI access +- [ ] Firewall VictoriaMetrics port (8428) +- [ ] Use secure networks for OTLP traffic + +## Further Reading + +- [OpenTelemetry Documentation](https://opentelemetry.io/docs/) +- [VictoriaMetrics Documentation](https://docs.victoriametrics.com/) +- [Grafana Documentation](https://grafana.com/docs/) +- [PromQL Tutorial](https://prometheus.io/docs/prometheus/latest/querying/basics/) diff --git a/docs/prds/unified-output-system.md b/docs/prds/unified-output-system.md new file mode 100644 index 00000000..85649b32 --- /dev/null +++ b/docs/prds/unified-output-system.md @@ -0,0 +1,482 @@ +# PRD: Unified Output System for Logs and Interactive Shell Access + +## Overview + +This document outlines the design for a unified output system that addresses the current time-synchronicity issues with stdout/stderr handling and introduces new capabilities for real-time log streaming and interactive shell access to Docker services. + +## Current Problems + +### Time Synchronicity Issues +- TaskDetails collects stdout and stderr in separate async tasks without coordination +- Temporal order of interleaved output is lost +- Simple string accumulation without timestamps +- Frontend displays two separate output sections, losing the actual execution flow + +### Limited Functionality +- No real-time log streaming capability +- No interactive shell access to services +- Output accumulation without size limits can cause memory issues +- WebSocket infrastructure exists but isn't leveraged for real-time output + +## Goals + +### Primary Goals +1. **Unified Time-Synchronized Output**: Preserve the chronological order of stdout/stderr streams +2. **Real-time Log Streaming**: Enable `app:logs ` command with live following capability +3. **Interactive Shell Access**: Enable `app:shell ` command for direct container access +4. **Memory Management**: Implement configurable output limits with proper cleanup +5. **Permission Integration**: Leverage existing Shell and Logs permissions + +### Secondary Goals +1. **Frontend Integration**: Unified log viewer in the web UI +2. **CLI WebSocket Support**: Enable real-time streaming in CLI tools +3. **Future Shell UI**: Plan for xterm.js integration (implementation later) + +## Technical Approach + +### Docker Integration Strategy + +**Primary: Bollard** +- Use bollard's Container Logs API for streaming logs +- Use bollard's Exec API for interactive shell sessions +- Leverage existing bollard dependency and async streaming capabilities +- Better integration with Rust ecosystem and existing Docker client + +**Fallback: Docker Compose Commands** +- Use `docker-compose logs -f` for log streaming if bollard approach faces issues +- Use `docker-compose exec -it` for shell access + +### WebSocket Architecture + +**Current WebSocket Infrastructure**: +- Existing broadcast-based WebSocket system in `api/ws.rs` +- Client management with UUID-based connection tracking +- Message types: Ping, AppListUpdated, TaskListUpdated, TaskInfoUpdated +- Frontend integration in `lib/ws.ts` + +**Extensions Needed**: +- New message types for logs and shell data +- Per-client session management for shell connections +- Binary data support for shell escape sequences + +**CLI WebSocket Integration**: +- Use existing WebSocket library (tokio-tungstenite recommended) +- Real-time streaming will work for both web UI and CLI +- CLI handles terminal escape sequences using crossterm or similar + +## Data Models + +### Unified Output Model + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OutputLine { + pub timestamp: DateTime, + pub stream: OutputStreamType, + pub content: String, + pub sequence: u64, // For ordering guarantee +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum OutputStreamType { + Stdout, + Stderr, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StreamingTaskOutput { + pub task_id: Uuid, + pub app_name: String, + pub service_name: Option, + pub lines: VecDeque, + pub max_lines: usize, + pub total_lines: u64, // For pagination +} +``` + +### Separated Task Management + +```rust +// Existing TaskDetails updated for task state management only +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TaskDetails { + pub id: Uuid, + pub command: String, + pub state: State, + pub start_time: DateTime, + pub finish_time: Option>, + pub last_exit_code: Option, + pub app_name: Option, + // stdout/stderr fields removed - breaking change +} + +// New separate structure for command output +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TaskOutput { + pub task_id: Uuid, + pub output_lines: VecDeque, + pub max_lines: usize, +} +``` + +### Shell Session Management + +```rust +#[derive(Debug, Clone)] +pub struct ShellSession { + pub id: Uuid, + pub app_name: String, + pub service_name: String, + pub user_id: String, + pub created_at: DateTime, + pub last_activity: DateTime, + pub ttl: Duration, +} + +#[derive(Debug)] +pub struct ShellManager { + sessions: Arc>>, + // bollard exec handles per session + exec_handles: Arc>>, +} +``` + +## API Design + +### New REST Endpoints + +```rust +// Logs API +GET /apps/{app_name}/logs/{service}?follow=true&lines=100&since=timestamp +GET /apps/{app_name}/logs/{service}/download // Download full logs + +// Shell API +POST /apps/{app_name}/shell/{service} // Create shell session +DELETE /apps/{app_name}/shell/{session_id} // Terminate session +GET /apps/{app_name}/shell/sessions // List active sessions + +// Task Output API - NOT IMPLEMENTED +// Decision: Use WebSocket-only approach for unified experience +// Task output is streamed via WebSocket messages (TaskOutputData) +``` + +### WebSocket Protocol Extensions + +```rust +#[derive(Serialize, Deserialize, Debug)] +pub enum WebSocketMessage { + // Existing messages... + Ping, + Pong, + AppListUpdated, + TaskListUpdated, + TaskInfoUpdated(TaskDetails), + Error(String), + + // New log streaming messages + LogLineReceived { + app_name: String, + service_name: String, + line: OutputLine, + }, + LogStreamStarted { + app_name: String, + service_name: String, + session_id: Uuid, + }, + LogStreamEnded { + session_id: Uuid, + reason: String, + }, + + // New shell messages + ShellSessionCreated { + session_id: Uuid, + app_name: String, + service_name: String, + }, + ShellDataReceived { + session_id: Uuid, + data: Vec, // Raw terminal data with escape sequences + }, + ShellDataSend { + session_id: Uuid, + data: Vec, // User input to send to shell + }, + ShellSessionEnded { + session_id: Uuid, + exit_code: Option, + }, +} +``` + +## CLI Commands + +### app:logs Command + +```bash +scottyctl app:logs [OPTIONS] + +OPTIONS: + -f, --follow Follow log output (default: false) + -n, --lines Number of lines to show (default: all available) + --since Show logs since timestamp + --until Show logs until timestamp + -t, --timestamps Show timestamps in log output + +EXAMPLES: + scottyctl app:logs my-app web # Show all available logs + scottyctl app:logs my-app web -f # Follow logs in real-time + scottyctl app:logs my-app web -t # Show logs with timestamps + scottyctl app:logs my-app web -n 500 # Show last 500 lines + scottyctl app:logs my-app web --since 1h # Logs from last hour +``` + +### app:shell Command (NOT YET IMPLEMENTED) + +**Note**: The backend shell service is fully implemented, but the CLI command is not yet available in scottyctl. + +**Planned Command:** +```bash +scottyctl app:shell [OPTIONS] + +OPTIONS: + -u, --user User to run shell as (default: root) + -w, --workdir Working directory (default: container default) + --shell Shell to use (default: /bin/bash) + --timeout Session timeout (default: from config) + +PLANNED EXAMPLES: + scottyctl app:shell my-app web # Open bash shell + scottyctl app:shell my-app web -u www-data # Shell as www-data user + scottyctl app:shell my-app db --shell /bin/sh # Use sh instead of bash +``` + +**Current Workaround**: Use the REST API directly or wait for CLI implementation (Phase 6). + +## Configuration Options + +### New Configuration Structure + +```yaml +# config/default.yaml additions +output: + # Task output limits + max_lines_per_task: 10000 # Max lines to keep per task + max_line_length: 4096 # Max characters per line + cleanup_interval: "5m" # How often to cleanup old output + + # Log streaming limits + max_log_lines_streaming: 1000 # Max lines for real-time streaming + log_buffer_size: 10000 # Buffer size for log collection + +shell: + # Shell session management + default_ttl: "30m" # Default session timeout + max_concurrent_sessions: 10 # Max shells per service + cleanup_interval: "1m" # How often to check for expired sessions + allowed_shells: # Allowed shell executables + - "/bin/bash" + - "/bin/sh" + - "/bin/zsh" + +websocket: + # WebSocket specific settings + max_message_size: 1048576 # 1MB max message size + ping_interval: "30s" # Ping interval for connection health +``` + +### Environment Variable Overrides + +```bash +SCOTTY__OUTPUT__MAX_LINES_PER_TASK=5000 +SCOTTY__SHELL__DEFAULT_TTL=60m +SCOTTY__OUTPUT__MAX_LOG_LINES_STREAMING=2000 +``` + +## Implementation Status + +See beads issues for current implementation status and remaining work. The core design is complete and the backend infrastructure is implemented. + +## Implementation Plan + +### Phase 1: Core Infrastructure +1. **Unified Output Data Model**: Implement OutputLine and StreamingTaskOutput structures +2. **Bollard Integration**: Research and implement bollard container logs and exec APIs +3. **WebSocket Protocol**: Extend message types for logs and shell data +4. **Configuration System**: Add new configuration options with validation + +### Phase 2: Log Streaming +1. **Backend Log API**: Implement REST endpoints and WebSocket handlers for log streaming +2. **CLI Logs Command**: Implement `app:logs` with WebSocket integration +3. **Permission Integration**: Add authorization checks for logs access +4. **Testing**: Unit and integration tests for log streaming + +### Phase 3: CLI Commands +1. **CLI Logs Command**: Implement `app:logs` with WebSocket integration, follow mode, timestamps, line limits +2. **Permission Integration**: Authorization checks for logs access +3. **Authentication Enhancement**: Centralized auth system for WebSocket connections +4. **Stream Cleanup**: Proactive client disconnect cleanup for proper resource management +5. **User Experience**: Improved completion timing and removed duplicate messages +6. **Shell Backend**: Complete ShellService implementation with session management, REST/WebSocket APIs +7. **CLI Shell Command**: Implement `app:shell` command in scottyctl (backend ready) + +### Phase 3.5: WebSocket Message Consolidation +1. **Message Type Consolidation**: Move all WebSocket message types to `scotty-core/src/websocket/message.rs` +2. **Code Deduplication**: Eliminate duplicate message definitions from scottyctl +3. **Type Consistency**: Server and client use identical message types +4. **Import Updates**: Update files across server and client to use consolidated types +5. **Testing**: Verify all tests pass with new consolidated message structure +6. **Single Source of Truth**: All WebSocket communication types defined once and shared + +### Phase 3.6: Task Output WebSocket Streaming +1. **Hybrid WebSocket Implementation**: Update `wait_for_task` function to use REST polling for task status + WebSocket for real-time output +2. **WebSocketMessenger Architecture**: Create centralized abstraction for WebSocket client management and message broadcasting +3. **Task Output Display**: Implement unified output display with colored stderr output during task execution +4. **Real-time Feedback**: Show task progress with live stdout/stderr output during app operations +5. **Stack Overflow Resolution**: Fix circular reference issues in TaskManager data structures +6. **Resource Management**: Proper WebSocket subscription cleanup and client management +7. **Unified Experience**: Consistent streaming experience across logs, shell, and task operations +8. **Status Integration**: Use `set_status` for proper UI status updates + +### Phase 3.7: Infrastructure Optimization +1. **TypeScript Generation Optimization**: Create standalone `ts-generator` crate reducing build time from 27s to 6s +2. **Type System Consolidation**: Move all shared types to `scotty-types` crate as single source of truth +3. **Import Cleanup**: Update all files to use direct imports from `scotty-types` instead of re-exports +4. **Frontend Build Migration**: Switch from npm to bun for 62% faster frontend builds (3.2s vs 5.2s) +5. **Docker Build Optimization**: Implement platform-agnostic Docker builds with multi-platform Rollup binaries +6. **Workspace Integration**: Add new crates to workspace for better tooling and dependency management +7. **Legacy Cleanup**: Remove duplicate dependencies, old package manager files, and unused code +8. **Multi-Platform Support**: Docker builds work on ARM64, x86_64, and different libc implementations + +### Phase 4: Frontend Integration +1. **Unified Output Viewer**: Create `unified-output.svelte` component with chronological display +2. **WebSocket-Only Streaming**: Task output uses WebSocket streaming (TaskOutputData messages) +3. **Real-time Updates**: Live task output streaming during execution via WebSocket +4. **Task Output Store**: Implement taskOutputStore.ts for managing streaming data +5. **WebSocket Store**: Connection management and message handling in webSocketStore.ts +6. **Task Detail Page**: Enhanced with real-time output streaming and WebSocket status indicator +7. **Container Log Viewer UI**: Frontend UI for viewing service logs +8. **Interactive Shell UI**: xterm.js or shell UI in frontend + +### Phase 5: Performance and Reliability +1. **Deadlock Resolution**: Fix critical lock contention causing API hangs +2. **Performance Optimization**: Reduce write lock frequency 20-100x (1000/sec → 10-50/sec) +3. **TimedBuffer System**: Generic batching utility with configurable size and time thresholds +4. **Memory Management**: Proper output limits, cleanup intervals, and resource management +5. **Lock Pattern Improvements**: Helper methods to ensure consistent, minimal lock holding times +6. **Security Enhancements**: MaskedSecret and SecretHashMap for memory-safe secret handling +7. **Error Handling**: Robust error handling with enum-based errors (LogStreamError, ShellServiceError) +8. **Test Coverage**: Comprehensive tests with CI-friendly Docker handling +9. **Monitoring**: Metrics and logging for the new system +10. **Documentation**: User and developer documentation + +### Phase 6: Remaining Features +1. **CLI Shell Command**: Implement `app:shell` command in scottyctl with terminal integration + - Add ShellCommand struct to cli.rs + - Implement WebSocket-based shell handler + - Add TTY resize handling and raw terminal mode + - Interactive command input/output with proper escape sequences +2. **Frontend Log Viewer**: Add container log viewing UI to web interface + - Create log viewer component (similar to unified-output.svelte) + - Add WebSocket handlers for LogLineReceived/LogStreamStarted/LogStreamEnded + - Add log viewer page or modal for each service + - Integration with app detail page +3. **Frontend Shell UI**: Add interactive shell terminal to web interface + - Integrate xterm.js for terminal emulation + - WebSocket handlers for ShellSession* messages + - TTY resize support, copy/paste, terminal settings + - Shell session management UI (list, create, terminate) +4. **Monitoring & Observability**: + - Add metrics for log/shell session counts, durations, errors + - OpenTelemetry integration for tracing + - Performance dashboards +5. **End-User Documentation**: + - User guide for app:logs command + - User guide for app:shell command (once implemented) + - Web UI documentation for log viewer and shell access + - Security best practices for shell access + +## Security Considerations + +### Permission Model +- **Logs Permission**: Required for `app:logs` command and log streaming +- **Shell Permission**: Required for `app:shell` command and shell access +- **Scope-based Access**: Permissions checked against app-specific scopes +- **Session Isolation**: Shell sessions are isolated per user and app + +### Input Validation +- **Service Name Validation**: Ensure service exists in app's docker-compose +- **Shell Path Validation**: Only allow pre-configured shell executables +- **Command Injection Prevention**: Use bollard APIs directly, avoid shell command construction +- **Rate Limiting**: Implement rate limits for shell session creation + +### Data Security +- **Log Data Sanitization**: Option to filter sensitive data from logs +- **Shell Session Logging**: Optional audit logging of shell commands +- **WebSocket Security**: Ensure proper authentication for WebSocket connections +- **TTL Enforcement**: Strict enforcement of session timeouts + +## Migration Notes + +### Breaking Changes +- `TaskDetails.stdout` and `TaskDetails.stderr` fields will be removed +- Frontend components using separate stdout/stderr displays need updates +- CLI output formatting will change to unified display +- Manual migration required for existing installations + +### Migration Strategy +1. **Database/Storage**: No persistent storage migration needed (tasks are ephemeral) +2. **Frontend**: Update components to use new unified output display +3. **CLI**: Update output formatting in scottyctl +4. **API**: Maintain task state endpoints, add new output endpoints + +## Testing Strategy + +### Unit Tests +- OutputLine serialization/deserialization +- ShellSession management logic +- Configuration validation +- Permission checking logic + +### Integration Tests +- Bollard container logs streaming +- Bollard exec session creation and management +- WebSocket message flow +- CLI command functionality + +### End-to-End Tests +- Complete log streaming workflow (backend → WebSocket → CLI) +- Complete shell session workflow (CLI → WebSocket → backend → container) +- Permission enforcement across all components +- Frontend integration + +## Future Enhancements + +### Frontend Shell Terminal +- **xterm.js Integration**: Full terminal emulator in web UI +- **Terminal Sharing**: Multiple users can view same shell session +- **Terminal Recording**: Save and replay shell sessions + +### Advanced Log Features +- **Log Search**: Full-text search across historical logs +- **Log Alerts**: Real-time alerts based on log patterns +- **Log Export**: Export logs in various formats (JSON, CSV, etc.) + +### Performance Optimizations (Later) +- **Log Compression**: Compress historical logs to save space +- **Streaming Optimization**: Optimized WebSocket streaming for large outputs +- **Caching Layer**: Cache frequently accessed logs + +## Next Steps + +1. **Review and Approval**: Stakeholder review of this PRD +2. **Technical Spike**: Investigate bollard logs and exec APIs in detail +3. **Architecture Review**: Validate WebSocket protocol design +4. **Implementation Start**: Begin with Phase 1 core infrastructure + +## Open Questions + +1. **Service Discovery**: How should we map service names to actual container IDs when multiple instances exist? +2. **Log Persistence**: Should we implement any form of log persistence beyond the current task system? +3. **Error Recovery**: How should we handle bollard connection failures and automatic reconnection? \ No newline at end of file diff --git a/docs/research/otel-metrics-backend-evaluation.md b/docs/research/otel-metrics-backend-evaluation.md new file mode 100644 index 00000000..b2f0892c --- /dev/null +++ b/docs/research/otel-metrics-backend-evaluation.md @@ -0,0 +1,661 @@ +# OpenTelemetry Metrics Backend Research + +## Context + +Scotty already uses OpenTelemetry for distributed tracing with: +- ✅ `opentelemetry` 0.28.0 with trace features +- ✅ `opentelemetry-otlp` for OTLP protocol +- ✅ Jaeger all-in-one running in observability/docker-compose.yml (port 4318 OTLP receiver) +- ✅ `tracing-opentelemetry` integration + +**Goal**: Extend OpenTelemetry usage to include metrics while: +- Using OpenTelemetry metrics API (not Prometheus directly) +- Keeping resource requirements low (no Prometheus) +- Integrating with existing observability/docker-compose.yml and Jaeger setup +- Visualizing with Grafana + +## Architecture Options + +### Current Setup +``` +Scotty (Rust) + └─> OTLP/gRPC (traces) ─> Jaeger (port 4318) +``` + +### Target Architecture +``` +Scotty (Rust) + ├─> OTLP/gRPC (traces) ─────────────┐ + └─> OTLP/gRPC (metrics) ────────────┤ + ▼ + OpenTelemetry Collector + ├─> Jaeger (traces) + └─> Metrics Backend ─> Grafana +``` + +## Metrics to Track + +### Unified Output System Metrics + +```rust +// Using opentelemetry::metrics API +use opentelemetry::metrics::{Counter, Gauge, Histogram}; + +// Log streaming +log_streams_active: Gauge +log_streams_total: Counter +log_stream_duration: Histogram +log_lines_received: Counter +log_stream_errors: Counter +log_stream_bytes: Counter + +// Shell sessions +shell_sessions_active: Gauge +shell_sessions_total: Counter +shell_session_duration: Histogram +shell_session_errors: Counter +shell_session_timeouts: Counter + +// WebSocket +websocket_connections_active: Gauge +websocket_connections_total: Counter +websocket_messages_sent: Counter +websocket_messages_received: Counter +websocket_auth_failures: Counter + +// Tasks +tasks_active: Gauge +tasks_total: Counter +task_duration: Histogram +task_output_lines: Counter +task_output_buffer_bytes: Gauge + +// System health +memory_usage_bytes: Gauge +cpu_usage_percent: Gauge +uptime_seconds: Counter +``` + +## Lightweight OTLP-Compatible Options + +### Option 1: OpenTelemetry Collector + VictoriaMetrics ⭐ RECOMMENDED + +**Architecture:** +``` +Scotty + └─> OTLP ─> OTel Collector ┬─> Jaeger (traces) + └─> VictoriaMetrics (metrics, Prometheus format) + └─> Grafana +``` + +**Pros:** +- OTel Collector is lightweight (~30MB memory) +- VictoriaMetrics very efficient (~50-100MB) +- Single unified pipeline for traces + metrics +- Grafana works with Prometheus datasource +- Can extend to logs later (full observability stack) +- Flexible exporters (can swap backends easily) + +**Cons:** +- Two components instead of one +- Extra hop for metrics + +**Total Resource Requirements:** +- OTel Collector: ~30-50 MB +- VictoriaMetrics: ~50-100 MB +- **Total: ~100-150 MB** (still lighter than Prometheus alone) + +**observability/docker-compose.yml addition:** +```yaml +services: + otel-collector: + image: otel/opentelemetry-collector:0.95.0 + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver + - "8888:8888" # Metrics endpoint + depends_on: + - victoriametrics + + victoriametrics: + image: victoriametrics/victoria-metrics:latest + ports: + - "8428:8428" + volumes: + - vm-data:/victoria-metrics-data + command: + - "--storageDataPath=/victoria-metrics-data" + - "--retentionPeriod=30d" + + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - grafana-data:/var/lib/grafana + - ./grafana-provisioning:/etc/grafana/provisioning + depends_on: + - victoriametrics + - jaeger + +volumes: + vm-data: + grafana-data: +``` + +### Option 2: Grafana Alloy (formerly Grafana Agent) + +**Architecture:** +``` +Scotty + └─> OTLP ─> Grafana Alloy ┬─> Jaeger (traces) + └─> Grafana Cloud / Mimir (metrics) +``` + +**Pros:** +- Single lightweight agent +- Native OTLP support +- Grafana-native solution +- Can output to multiple backends + +**Cons:** +- Requires Grafana Cloud or Mimir for metrics +- Mimir is resource-heavy for local dev +- Less flexible than OTel Collector + +**Not recommended** - Too Grafana-ecosystem specific + +### Option 3: VictoriaMetrics with Native OTLP + +**Architecture:** +``` +Scotty + ├─> OTLP ─> Jaeger (traces, port 4318) + └─> OTLP ─> VictoriaMetrics (metrics, different port) +``` + +**Pros:** +- Minimal components (reuse existing Jaeger container) +- VictoriaMetrics v1.93+ has experimental OTLP support +- Simple architecture + +**Cons:** +- OTLP support in VictoriaMetrics is experimental +- Still need separate endpoints for traces vs metrics +- Less flexibility + +**Resource Requirements:** +- VictoriaMetrics: ~50-100 MB +- **Total: ~50-100 MB** (lightest option) + +**observability/docker-compose.yml addition:** +```yaml +services: + victoriametrics: + image: victoriametrics/victoria-metrics:latest + ports: + - "8428:8428" # HTTP API + - "4318:4318" # OTLP receiver (conflicts with Jaeger!) + volumes: + - vm-data:/victoria-metrics-data + command: + - "--storageDataPath=/victoria-metrics-data" + - "--retentionPeriod=30d" + - "--opentelemetry.http.listenAddr=:4318" + +volumes: + vm-data: +``` + +**Issue:** Port 4318 conflict with Jaeger! + +### Option 4: SigNoz (All-in-One Observability) + +**Architecture:** +``` +Scotty + └─> OTLP ─> SigNoz (traces + metrics + logs) + └─> Built-in UI +``` + +**Pros:** +- All-in-one solution (traces, metrics, logs) +- Native OTLP support +- Built-in dashboards +- ClickHouse backend (efficient) + +**Cons:** +- Heavy (~1GB memory for full stack) +- Complex deployment (multiple containers) +- Overkill for just metrics +- Not using Grafana + +**Not recommended** - Too heavy, doesn't use Grafana + +### Option 5: Jaeger + Prometheus Exporter + +**Architecture:** +``` +Scotty + └─> OTLP ─> Jaeger (with metrics) ─> Prometheus Remote Write ─> VictoriaMetrics +``` + +**Pros:** +- Reuse existing Jaeger container +- Jaeger can expose metrics + +**Cons:** +- Jaeger is primarily for traces, metrics support limited +- Complex configuration +- Not the intended use case + +**Not recommended** - Wrong tool for the job + +## Detailed Comparison + +| Feature | OTel Collector + VM | VM Native OTLP | Grafana Alloy | SigNoz | +|---------|---------------------|----------------|---------------|--------| +| **Memory (MB)** | 100-150 | 50-100 | 80-120 | 1000+ | +| **Components** | 2 + Jaeger | 1 + Jaeger | 1 + Backend | Many | +| **OTLP Maturity** | ✅ Stable | 🟡 Experimental | ✅ Stable | ✅ Stable | +| **Port Conflict** | ❌ No | ⚠️ Yes (4318) | ❌ No | ❌ No | +| **Grafana Ready** | ✅ Yes | ✅ Yes | ✅ Yes | ❌ No | +| **Flexibility** | ✅ High | 🟡 Medium | 🟡 Medium | ❌ Low | +| **Setup Complexity** | Medium | Low | Medium | High | +| **Production Ready** | ✅ Yes | 🟡 Experimental | ✅ Yes | ✅ Yes | + +## Recommendation: OpenTelemetry Collector + VictoriaMetrics + +### Why This Combination? + +1. **Unified Pipeline**: Single OTLP endpoint for both traces and metrics +2. **Proven Stack**: Both components are production-ready and widely used +3. **Lightweight**: Total ~100-150MB (much less than Prometheus) +4. **Flexible**: Easy to swap backends or add exporters +5. **Future-Proof**: Can add logs to same pipeline later +6. **No Port Conflicts**: Clean separation of concerns + +### Implementation Plan + +#### Step 1: Add OpenTelemetry Metrics Dependencies + +```toml +# Cargo.toml - workspace dependencies +[workspace.dependencies] +opentelemetry = { version = "0.28.0", features = ["trace", "metrics"] } +opentelemetry_sdk = { version = "0.28", features = [ + "trace", + "metrics", + "rt-tokio", +] } +opentelemetry-otlp = { version = "0.28.0", features = [ + "trace", + "metrics", + "http-proto", + "reqwest-client", +] } +``` + +#### Step 2: Create OTel Collector Configuration + +```yaml +# observability/otel-collector-config.yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 10s + send_batch_size: 1024 + +exporters: + # Traces to Jaeger + jaeger: + endpoint: jaeger:14250 + tls: + insecure: true + + # Metrics to VictoriaMetrics (Prometheus Remote Write) + prometheusremotewrite: + endpoint: http://victoriametrics:8428/api/v1/write + tls: + insecure: true + + # Debug exporter for testing + logging: + loglevel: debug + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [jaeger, logging] + + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheusremotewrite, logging] +``` + +#### Step 3: Update observability/docker-compose.yml + +```yaml +services: + # Existing Jaeger service + jaeger: + image: jaegertracing/all-in-one:${JAEGER_VERSION:-latest} + ports: + - "16686:16686" # Jaeger UI + - "14250:14250" # Model proto + environment: + - LOG_LEVEL=debug + labels: + - "traefik.enable=true" + - "traefik.http.routers.jaeger.rule=Host(`jaeger.ddev.site`)" + - "traefik.http.services.jaeger.loadbalancer.server.port=16686" + + # NEW: OpenTelemetry Collector + otel-collector: + image: otel/opentelemetry-collector:0.95.0 + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "8888:8888" # Prometheus metrics about the collector itself + depends_on: + - jaeger + - victoriametrics + labels: + - "traefik.enable=false" + + # NEW: VictoriaMetrics + victoriametrics: + image: victoriametrics/victoria-metrics:latest + ports: + - "8428:8428" + volumes: + - vm-data:/victoria-metrics-data + command: + - "--storageDataPath=/victoria-metrics-data" + - "--retentionPeriod=30d" + labels: + - "traefik.enable=true" + - "traefik.http.routers.victoriametrics.rule=Host(`vm.ddev.site`)" + - "traefik.http.services.victoriametrics.loadbalancer.server.port=8428" + + # NEW: Grafana + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - victoriametrics + - jaeger + labels: + - "traefik.enable=true" + - "traefik.http.routers.grafana.rule=Host(`grafana.ddev.site`)" + - "traefik.http.services.grafana.loadbalancer.server.port=3000" + +volumes: + vm-data: + grafana-data: +``` + +#### Step 4: Create Metrics Module in Scotty + +```rust +// scotty/src/metrics/mod.rs +use opentelemetry::{ + metrics::{Counter, Gauge, Histogram, Meter, MeterProvider}, + KeyValue, +}; +use opentelemetry_sdk::metrics::SdkMeterProvider; + +pub struct ScottyMetrics { + // Log streaming + pub log_streams_active: Gauge, + pub log_streams_total: Counter, + pub log_stream_duration: Histogram, + + // Shell sessions + pub shell_sessions_active: Gauge, + pub shell_sessions_total: Counter, + + // WebSocket + pub websocket_connections: Gauge, + pub websocket_messages_sent: Counter, + + // Tasks + pub tasks_active: Gauge, + pub tasks_total: Counter, +} + +impl ScottyMetrics { + pub fn new(meter: Meter) -> Self { + Self { + log_streams_active: meter + .u64_gauge("scotty.log_streams.active") + .with_description("Current number of active log streams") + .init(), + + log_streams_total: meter + .u64_counter("scotty.log_streams.total") + .with_description("Total log streams created") + .init(), + + log_stream_duration: meter + .f64_histogram("scotty.log_stream.duration") + .with_description("Log stream duration in seconds") + .with_unit("s") + .init(), + + shell_sessions_active: meter + .u64_gauge("scotty.shell_sessions.active") + .with_description("Current number of active shell sessions") + .init(), + + shell_sessions_total: meter + .u64_counter("scotty.shell_sessions.total") + .with_description("Total shell sessions created") + .init(), + + websocket_connections: meter + .u64_gauge("scotty.websocket.connections") + .with_description("Current WebSocket connections") + .init(), + + websocket_messages_sent: meter + .u64_counter("scotty.websocket.messages_sent") + .with_description("Total WebSocket messages sent") + .init(), + + tasks_active: meter + .u64_gauge("scotty.tasks.active") + .with_description("Current number of active tasks") + .init(), + + tasks_total: meter + .u64_counter("scotty.tasks.total") + .with_description("Total tasks executed") + .init(), + } + } +} + +// Initialize in main.rs +pub fn init_metrics() -> Result> { + let meter_provider = opentelemetry_otlp::new_pipeline() + .metrics(opentelemetry_sdk::runtime::Tokio) + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint("http://otel-collector:4317"), + ) + .build()?; + + let meter = meter_provider.meter("scotty"); + Ok(ScottyMetrics::new(meter)) +} +``` + +#### Step 5: Instrument Services + +```rust +// In LogStreamingService +impl LogStreamingService { + pub async fn start_stream(&self, ...) -> Result<...> { + // Increment counters + self.metrics.log_streams_total.add(1, &[]); + self.metrics.log_streams_active.add(1, &[]); + + let start_time = std::time::Instant::now(); + + // ... existing stream logic ... + + // On completion + self.metrics.log_streams_active.add(-1, &[]); + self.metrics.log_stream_duration.record( + start_time.elapsed().as_secs_f64(), + &[] + ); + } +} +``` + +#### Step 6: Grafana Datasource Provisioning + +```yaml +# grafana/provisioning/datasources/datasources.yaml +apiVersion: 1 + +datasources: + - name: VictoriaMetrics + type: prometheus + access: proxy + url: http://victoriametrics:8428 + isDefault: true + jsonData: + timeInterval: 15s + + - name: Jaeger + type: jaeger + access: proxy + url: http://jaeger:16686 + jsonData: + tracesToLogs: + datasourceUid: 'loki' +``` + +#### Step 7: Create Grafana Dashboard + +```json +// grafana/dashboards/scotty-metrics.json +{ + "dashboard": { + "title": "Scotty - Unified Output System", + "panels": [ + { + "title": "Active Log Streams", + "targets": [ + { + "expr": "scotty_log_streams_active" + } + ] + }, + { + "title": "Active Shell Sessions", + "targets": [ + { + "expr": "scotty_shell_sessions_active" + } + ] + }, + { + "title": "WebSocket Connections", + "targets": [ + { + "expr": "scotty_websocket_connections" + } + ] + } + ] + } +} +``` + +## Resource Requirements Summary + +| Component | Memory | CPU | Storage | +|-----------|--------|-----|---------| +| OTel Collector | 30-50 MB | < 2% | Minimal | +| VictoriaMetrics | 50-100 MB | < 5% | ~100MB/30 days | +| Grafana | 100-150 MB | < 5% | ~50MB | +| Jaeger (existing) | 200-300 MB | < 5% | Varies | +| **Total New** | **180-250 MB** | **< 10%** | **~150MB** | + +## Timeline Estimate + +- **Day 1**: Add OTel metrics dependencies and create metrics module +- **Day 2**: Instrument log streaming and shell services +- **Day 3**: Add OTel Collector and VictoriaMetrics to docker-compose +- **Day 4**: Set up Grafana with dashboards +- **Day 5**: Testing, documentation, and refinement +- **Total**: ~5 days + +## Success Criteria + +- [x] Uses OpenTelemetry metrics API (native integration) +- [x] Total memory overhead < 250 MB +- [x] Integrates with existing Jaeger in observability/docker-compose.yml +- [x] No port conflicts +- [x] Grafana dashboards showing real-time metrics +- [x] 30-day retention configured +- [x] Simple docker-compose up experience + +## Open Questions + +1. Should we use gRPC (port 4317) or HTTP (port 4318) for OTLP? + - **Recommendation**: gRPC (4317) - better performance, smaller payloads + +2. Should metrics be always-on or configurable? + - **Recommendation**: Always on, but allow disabling via env var + +3. What scrape interval for OTel Collector? + - **Recommendation**: 15s (balance between freshness and overhead) + +4. Should we include system metrics (CPU, memory) from the host? + - **Recommendation**: Yes, but via separate receiver in OTel Collector + +5. Bundle observability stack in separate compose file or main one? + - **Recommendation**: Separate `docker-compose.observability.yml` for opt-in + +## Next Steps + +1. ✅ Update beads issue scotty-5 with this research +2. Create proof-of-concept branch +3. Add OTel metrics dependencies +4. Create basic metrics module +5. Test with OTel Collector + VictoriaMetrics locally +6. Benchmark memory usage +7. Create initial Grafana dashboard +8. Document setup in README diff --git a/docs/technical-spike-bollard-findings.md b/docs/technical-spike-bollard-findings.md new file mode 100644 index 00000000..0b2ff690 --- /dev/null +++ b/docs/technical-spike-bollard-findings.md @@ -0,0 +1,326 @@ +# Bollard Technical Spike - Findings and Recommendations + +## Executive Summary + +Successfully validated bollard's capabilities for implementing unified output system with real-time log streaming and interactive shell access. Bollard provides all necessary features to replace the current docker-compose command approach with a more robust, native Rust solution. + +## Key Findings + +### ✅ Container Logs API Capabilities + +**Perfect for Unified Output System:** +- **Stream Separation**: Clear distinction between `LogOutput::StdOut` and `LogOutput::StdErr` +- **Timestamps**: Native timestamp support with `timestamps: true` option +- **Real-time Streaming**: `follow: true` enables live log following +- **Historical Limits**: `tail` parameter controls log history +- **Time-Synchronized**: Streams maintain chronological order when timestamps enabled + +**API Structure:** +```rust +use bollard::query_parameters::LogsOptions; + +let options = LogsOptions { + stdout: true, + stderr: true, + follow: true, // Real-time streaming + tail: "100".to_string(), // Last N lines + timestamps: true, // Include timestamps + ..Default::default() +}; + +let mut stream = docker.logs(&container_id, Some(options)); +while let Some(log_result) = stream.next().await { + match log_result? { + LogOutput::StdOut { message } => { + // Handle stdout with timestamp + } + LogOutput::StdErr { message } => { + // Handle stderr with timestamp + } + _ => {} + } +} +``` + +### ✅ Container Exec API Capabilities + +**Fully Supports Interactive Shell Requirements:** +- **Session Management**: Each exec gets unique session ID +- **TTY Support**: `tty: true` enables proper terminal behavior +- **Bidirectional Communication**: `attach_stdin/stdout/stderr: true` +- **Command Flexibility**: Can specify any shell (`/bin/bash`, `/bin/sh`, etc.) + +**API Structure:** +```rust +use bollard::exec::{CreateExecOptions, StartExecOptions, StartExecResults}; + +// Create interactive shell session +let exec_options = CreateExecOptions { + cmd: Some(vec!["/bin/bash"]), + attach_stdin: Some(true), + attach_stdout: Some(true), + attach_stderr: Some(true), + tty: Some(true), + ..Default::default() +}; + +let exec = docker.create_exec(&container_id, exec_options).await?; + +// Start session for bidirectional communication +match docker.start_exec(&exec.id, Some(StartExecOptions { + detach: false, + tty: true, + output_capacity: None, +})).await? { + StartExecResults::Attached { mut output, mut input } => { + // Handle bidirectional stream + } +} +``` + +### ✅ Service Discovery and Container Mapping + +**Existing Pattern Analysis:** +Scotty already has robust service discovery through: + +1. **Docker Compose Integration**: Uses `docker-compose ps -q -a` to find containers +2. **Label-Based Service Mapping**: Extracts service name from `com.docker.compose.service` label +3. **Container Inspection**: Uses `bollard::Docker::inspect_container()` for detailed info + +**Service → Container ID Mapping Process:** +```rust +// 1. Get containers for docker-compose app +let output = run_docker_compose_now(file, &["ps", "-q", "-a"], None, false)?; +let containers: Vec = output.lines().map(String::from).collect(); + +// 2. Inspect each container to get service mapping +for container_id in containers { + let insights = docker.inspect_container(&container_id, None).await?; + let labels = insights.config.unwrap().labels.unwrap(); + let service_name = labels.get("com.docker.compose.service").unwrap(); + + // service_name -> container_id mapping established +} +``` + +## Bollard vs Docker-Compose Commands Comparison + +| Feature | Bollard (Recommended) | Docker-Compose Commands | +|---------|----------------------|------------------------| +| **Logs Streaming** | ✅ Native async streams | ⚠️ Subprocess management | +| **Stdout/Stderr Separation** | ✅ Built-in `LogOutput` enum | ⚠️ Manual stream handling | +| **Timestamps** | ✅ Native support | ⚠️ Docker-compose dependent | +| **Interactive Shell** | ✅ Full TTY support | ⚠️ Complex pseudo-TTY setup | +| **Error Handling** | ✅ Rust Result types | ⚠️ Exit codes + stderr parsing | +| **Performance** | ✅ Direct Docker API | ⚠️ Process spawn overhead | +| **Resource Usage** | ✅ Single connection pool | ⚠️ Multiple processes | +| **Integration** | ✅ Already used in codebase | ⚠️ Mixed approach | + +## Service Discovery Strategy + +**For app:logs and app:shell commands:** + +```rust +pub async fn find_container_for_service( + app_state: &SharedAppState, + app_name: &str, + service_name: &str, +) -> anyhow::Result> { + // 1. Get app data (already contains service->container mapping) + let apps = app_state.apps.read().await; + let app = apps.apps.iter() + .find(|a| a.name == app_name) + .ok_or_else(|| anyhow!("App not found: {}", app_name))?; + + // 2. Find service and get container ID + let service = app.services.iter() + .find(|s| s.service == service_name) + .ok_or_else(|| anyhow!("Service not found: {}", service_name))?; + + Ok(service.id.clone()) +} +``` + +**Key Benefits:** +- ✅ Reuses existing service discovery logic +- ✅ No additional docker-compose calls needed +- ✅ Consistent with current app management +- ✅ Handles multiple instances per service + +## Implementation Recommendations + +### Phase 1: Core Unified Output System + +1. **Replace TaskDetails stdout/stderr** with new unified model: +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OutputLine { + pub timestamp: DateTime, + pub stream: OutputStreamType, // Stdout | Stderr + pub content: String, + pub sequence: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TaskOutput { + pub task_id: Uuid, + pub lines: VecDeque, + pub max_lines: usize, +} +``` + +2. **Update TaskManager** to use unified output collection: +```rust +async fn collect_unified_output( + task_details: Arc>, + stdout: impl AsyncRead + Unpin, + stderr: impl AsyncRead + Unpin, +) { + // Use tokio::select! to coordinate both streams + // Maintain chronological order with timestamps + // Apply line limits and cleanup +} +``` + +### Phase 2: Bollard Log Streaming Service + +```rust +pub struct LogStreamingService { + docker: Docker, + active_streams: Arc>>, +} + +pub struct LogStreamSession { + session_id: Uuid, + app_name: String, + service_name: String, + container_id: String, + // WebSocket sender for real-time updates +} + +impl LogStreamingService { + pub async fn start_log_stream( + &self, + app_name: &str, + service_name: &str, + options: LogStreamOptions, + ) -> Result { + // 1. Find container ID for service + // 2. Create bollard LogsOptions + // 3. Start streaming with WebSocket integration + // 4. Return session ID for client tracking + } +} +``` + +### Phase 3: Bollard Shell Service + +```rust +pub struct ShellService { + docker: Docker, + active_sessions: Arc>>, +} + +pub struct ShellSession { + session_id: Uuid, + exec_id: String, + app_name: String, + service_name: String, + container_id: String, + created_at: DateTime, + last_activity: DateTime, + ttl: Duration, +} + +impl ShellService { + pub async fn create_shell_session( + &self, + app_name: &str, + service_name: &str, + shell_options: ShellOptions, + ) -> Result { + // 1. Find container ID for service + // 2. Create bollard exec session + // 3. Setup TTY and bidirectional communication + // 4. Return session for WebSocket integration + } +} +``` + +## WebSocket Integration Strategy + +**Message Types for Real-time Communication:** +```rust +#[derive(Serialize, Deserialize, Debug)] +pub enum WebSocketMessage { + // Existing messages preserved... + + // Log streaming + LogStreamStart { app_name: String, service_name: String, session_id: Uuid }, + LogLine { session_id: Uuid, line: OutputLine }, + LogStreamEnd { session_id: Uuid, reason: String }, + + // Shell sessions + ShellSessionCreated { session_id: Uuid, app_name: String, service_name: String }, + ShellData { session_id: Uuid, data: Vec }, + ShellInput { session_id: Uuid, data: Vec }, + ShellSessionEnded { session_id: Uuid, exit_code: Option }, +} +``` + +## CLI Integration with WebSocket + +**For real-time streaming in CLI:** +- Use `tokio-tungstenite` for WebSocket client +- Handle terminal I/O with `crossterm` +- Coordinate WebSocket messages with local terminal state + +## Performance Considerations + +**Memory Management:** +- Implement circular buffers for log lines +- Configurable limits per stream/session +- Automatic cleanup of expired sessions + +**Connection Pooling:** +- Reuse existing `SharedAppState.docker` client +- No additional connection overhead + +**Concurrency:** +- Multiple log streams can run simultaneously +- Shell sessions are independent per container +- WebSocket broadcasts scale to multiple clients + +## Risk Assessment + +**Low Risk ✅:** +- Bollard already used extensively in codebase +- Service discovery logic already proven +- WebSocket infrastructure already exists +- Breaking changes are acceptable per requirements + +**Medium Risk ⚠️:** +- CLI WebSocket integration complexity +- Terminal escape sequence handling in shells +- Session cleanup and TTL management + +**Mitigation Strategies:** +- Start with log streaming (simpler) +- Extensive testing with different container types +- Fallback to docker-compose commands if needed +- Comprehensive session management testing + +## Next Steps + +1. **Immediate**: Begin implementing unified output data model +2. **Week 1**: Bollard log streaming service with WebSocket integration +3. **Week 2**: CLI app:logs command with real-time support +4. **Week 3**: Bollard shell service and session management +5. **Week 4**: CLI app:shell command with terminal integration +6. **Week 5**: Frontend unified log viewer and shell preparation + +## Conclusion + +Bollard provides all necessary capabilities for implementing the unified output system. The existing service discovery mechanism is robust and ready for integration. The technical spike confirms feasibility with low implementation risk and significant benefits over the docker-compose command approach. + +**Recommendation: Proceed with bollard-based implementation as outlined in the PRD.** \ No newline at end of file diff --git a/examples/log-demo/Dockerfile b/examples/log-demo/Dockerfile new file mode 100644 index 00000000..c7331d7c --- /dev/null +++ b/examples/log-demo/Dockerfile @@ -0,0 +1,32 @@ +# Simple Dockerfile for log generator demo +FROM alpine:3.18 + +# Install bash (Alpine uses ash by default) +RUN apk add --no-cache bash + +# Create a directory for our app +WORKDIR /app + +# Copy the log generator script +COPY log-generator.sh /app/log-generator.sh + +# Make sure the script is executable +RUN chmod +x /app/log-generator.sh + +# Add a non-root user for security +RUN addgroup -g 1001 -S appgroup && \ + adduser -u 1001 -S appuser -G appgroup + +# Change ownership of the app directory +RUN chown -R appuser:appgroup /app + +# Switch to non-root user +USER appuser + +# Set the default command +CMD ["/app/log-generator.sh"] + +# Add labels for documentation +LABEL maintainer="Scotty Demo" +LABEL description="Log generator for testing scotty logs functionality" +LABEL version="1.0" \ No newline at end of file diff --git a/examples/log-demo/README.md b/examples/log-demo/README.md new file mode 100644 index 00000000..910a029d --- /dev/null +++ b/examples/log-demo/README.md @@ -0,0 +1,90 @@ +# Log Demo Example + +This example provides a simple logging service for testing Scotty's unified logs functionality. + +## What it does + +- **log-generator**: Continuously generates log messages to both stdout and stderr +- **web-logger**: A second instance of the same logger (simulates a multi-service app) + +## Log Output Features + +The log generator produces various types of messages: + +- **Regular INFO messages**: Every message to stdout +- **ERROR messages**: Every 3rd message goes to stderr +- **Multi-line messages**: Every 5th message spans multiple lines +- **WARNING messages**: Every 10th message is a warning to stderr +- **Exception simulation**: Every 20th message simulates a stack trace +- **Progress indicators**: Every 50th message shows progress + +## Usage with Scotty + +### 1. Create the app + +```bash +scottyctl app:create log-demo --folder examples/log-demo --service log-generator:80 --service web-logger:80 +``` + +### 2. View logs + +```bash +# View recent logs for the log-generator service +scottyctl app:logs log-demo log-generator + +# Follow logs in real-time +scottyctl app:logs log-demo log-generator --follow + +# View logs with timestamps +scottyctl app:logs log-demo log-generator --timestamps + +# View logs from the web-logger service +scottyctl app:logs log-demo web-logger + +# View specific number of lines +scottyctl app:logs log-demo log-generator --lines 200 +``` + +### 3. Test error scenarios + +```bash +# Test with wrong service name (should show available services) +scottyctl app:logs log-demo wrong-service + +# Test with non-existent app +scottyctl app:logs non-existent wrong-service +``` + +## Manual Testing + +You can also run the services manually for testing: + +```bash +# Build and run +cd examples/log-demo +docker-compose up --build + +# View logs +docker-compose logs -f log-generator +docker-compose logs -f web-logger + +# Clean up +docker-compose down +``` + +## Expected Output + +The log generator will produce output like: + +``` +[2024-01-01 10:00:00] INFO: This is info message #1 from log-generator +[2024-01-01 10:00:02] INFO: This is info message #2 from log-generator +[2024-01-01 10:00:04] ERROR: This is error message #3 from log-generator +[2024-01-01 10:00:06] INFO: This is info message #4 from log-generator +[2024-01-01 10:00:08] INFO: Multi-line message #5 + ├── This is line 2 of the multi-line message + ├── This is line 3 with some JSON: {"counter": 5, "type": "demo"} + └── End of multi-line message #5 +``` + +This provides a realistic testing environment for the unified output system, with both stdout and stderr messages, timestamps, and various message types that help verify the logs functionality works correctly. \ No newline at end of file diff --git a/examples/log-demo/docker-compose.yml b/examples/log-demo/docker-compose.yml new file mode 100644 index 00000000..f4316075 --- /dev/null +++ b/examples/log-demo/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3.8' + +services: + log-generator: + build: . + container_name: log-demo-generator + restart: unless-stopped + environment: + - LOG_LEVEL=INFO + - DEMO_MODE=true + # Uncomment below to see logs with docker-compose logs + # stdin_open: true + # tty: true + + web-logger: + build: . + container_name: log-demo-web + restart: unless-stopped + environment: + - LOG_LEVEL=DEBUG + - SERVICE_NAME=web-logger + # This service could expose a port if needed for web access + # ports: + # - "8080:80" + +networks: + default: + name: log-demo-network \ No newline at end of file diff --git a/examples/log-demo/log-generator.sh b/examples/log-demo/log-generator.sh new file mode 100755 index 00000000..19046296 --- /dev/null +++ b/examples/log-demo/log-generator.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Log generator script for testing scotty logs functionality +# This script outputs messages to both stdout and stderr at regular intervals + +echo "Starting log generator..." +echo "Log generator started at $(date)" >&2 + +counter=1 + +while true; do + # Generate timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + # Every 3rd message goes to stderr, others to stdout + if [ $((counter % 3)) -eq 0 ]; then + echo "[$timestamp] ERROR: This is error message #$counter from log-generator" >&2 + else + echo "[$timestamp] INFO: This is info message #$counter from log-generator" + fi + + # Every 5th message is a longer multi-line message + if [ $((counter % 5)) -eq 0 ]; then + echo "[$timestamp] INFO: Multi-line message #$counter" + echo " ├── This is line 2 of the multi-line message" + echo " ├── This is line 3 with some JSON: {\"counter\": $counter, \"type\": \"demo\"}" + echo " └── End of multi-line message #$counter" + fi + + # Every 10th message is a warning + if [ $((counter % 10)) -eq 0 ]; then + echo "[$timestamp] WARN: This is warning message #$counter - something might be wrong!" >&2 + fi + + # Every 20th message simulates an exception + if [ $((counter % 20)) -eq 0 ]; then + echo "[$timestamp] ERROR: Exception occurred in log-generator!" >&2 + echo " Stack trace simulation:" >&2 + echo " at log-generator.sh:line_$counter" >&2 + echo " at main_loop (log-generator.sh:$(($LINENO - 5)))" >&2 + echo " Error details: Counter reached $counter" >&2 + fi + + # Progress indicator every 50 messages + if [ $((counter % 50)) -eq 0 ]; then + echo "[$timestamp] INFO: Log generator progress: $counter messages generated" + fi + + counter=$((counter + 1)) + + # Sleep for 2 seconds between messages + sleep 2 +done \ No newline at end of file diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 62dbd03c..d58b710f 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -25,6 +25,10 @@ export default [ parserOptions: { parser: ts.parser } + }, + rules: { + // Disable infinite-reactive-loop as it has false positives with proper guards + 'svelte/infinite-reactive-loop': 'off' } }, { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b8155aa4..2bc1c3a4 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1223,49 +1223,49 @@ } }, "node_modules/@tailwindcss/node": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.15.tgz", - "integrity": "sha512-HF4+7QxATZWY3Jr8OlZrBSXmwT3Watj0OogeDvdUY/ByXJHQ+LBtqA2brDb3sBxYslIFx6UP94BJ4X6a4L9Bmw==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.16.tgz", + "integrity": "sha512-BX5iaSsloNuvKNHRN3k2RcCuTEgASTo77mofW0vmeHkfrDWaoFAFvNHpEgtu0eqyypcyiBkDWzSMxJhp3AUVcw==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.4", "enhanced-resolve": "^5.18.3", - "jiti": "^2.6.0", + "jiti": "^2.6.1", "lightningcss": "1.30.2", "magic-string": "^0.30.19", "source-map-js": "^1.2.1", - "tailwindcss": "4.1.15" + "tailwindcss": "4.1.16" } }, "node_modules/@tailwindcss/oxide": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.1.15.tgz", - "integrity": "sha512-krhX+UOOgnsUuks2SR7hFafXmLQrKxB4YyRTERuCE59JlYL+FawgaAlSkOYmDRJdf1Q+IFNDMl9iRnBW7QBDfQ==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.1.16.tgz", + "integrity": "sha512-2OSv52FRuhdlgyOQqgtQHuCgXnS8nFSYRp2tJ+4WZXKgTxqPy7SMSls8c3mPT5pkZ17SBToGM5LHEJBO7miEdg==", "dev": true, "license": "MIT", "engines": { "node": ">= 10" }, "optionalDependencies": { - "@tailwindcss/oxide-android-arm64": "4.1.15", - "@tailwindcss/oxide-darwin-arm64": "4.1.15", - "@tailwindcss/oxide-darwin-x64": "4.1.15", - "@tailwindcss/oxide-freebsd-x64": "4.1.15", - "@tailwindcss/oxide-linux-arm-gnueabihf": "4.1.15", - "@tailwindcss/oxide-linux-arm64-gnu": "4.1.15", - "@tailwindcss/oxide-linux-arm64-musl": "4.1.15", - "@tailwindcss/oxide-linux-x64-gnu": "4.1.15", - "@tailwindcss/oxide-linux-x64-musl": "4.1.15", - "@tailwindcss/oxide-wasm32-wasi": "4.1.15", - "@tailwindcss/oxide-win32-arm64-msvc": "4.1.15", - "@tailwindcss/oxide-win32-x64-msvc": "4.1.15" + "@tailwindcss/oxide-android-arm64": "4.1.16", + "@tailwindcss/oxide-darwin-arm64": "4.1.16", + "@tailwindcss/oxide-darwin-x64": "4.1.16", + "@tailwindcss/oxide-freebsd-x64": "4.1.16", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.1.16", + "@tailwindcss/oxide-linux-arm64-gnu": "4.1.16", + "@tailwindcss/oxide-linux-arm64-musl": "4.1.16", + "@tailwindcss/oxide-linux-x64-gnu": "4.1.16", + "@tailwindcss/oxide-linux-x64-musl": "4.1.16", + "@tailwindcss/oxide-wasm32-wasi": "4.1.16", + "@tailwindcss/oxide-win32-arm64-msvc": "4.1.16", + "@tailwindcss/oxide-win32-x64-msvc": "4.1.16" } }, "node_modules/@tailwindcss/oxide-android-arm64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.1.15.tgz", - "integrity": "sha512-TkUkUgAw8At4cBjCeVCRMc/guVLKOU1D+sBPrHt5uVcGhlbVKxrCaCW9OKUIBv1oWkjh4GbunD/u/Mf0ql6kEA==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.1.16.tgz", + "integrity": "sha512-8+ctzkjHgwDJ5caq9IqRSgsP70xhdhJvm+oueS/yhD5ixLhqTw9fSL1OurzMUhBwE5zK26FXLCz2f/RtkISqHA==", "cpu": [ "arm64" ], @@ -1280,9 +1280,9 @@ } }, "node_modules/@tailwindcss/oxide-darwin-arm64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.1.15.tgz", - "integrity": "sha512-xt5XEJpn2piMSfvd1UFN6jrWXyaKCwikP4Pidcf+yfHTSzSpYhG3dcMktjNkQO3JiLCp+0bG0HoWGvz97K162w==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.1.16.tgz", + "integrity": "sha512-C3oZy5042v2FOALBZtY0JTDnGNdS6w7DxL/odvSny17ORUnaRKhyTse8xYi3yKGyfnTUOdavRCdmc8QqJYwFKA==", "cpu": [ "arm64" ], @@ -1297,9 +1297,9 @@ } }, "node_modules/@tailwindcss/oxide-darwin-x64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.1.15.tgz", - "integrity": "sha512-TnWaxP6Bx2CojZEXAV2M01Yl13nYPpp0EtGpUrY+LMciKfIXiLL2r/SiSRpagE5Fp2gX+rflp/Os1VJDAyqymg==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.1.16.tgz", + "integrity": "sha512-vjrl/1Ub9+JwU6BP0emgipGjowzYZMjbWCDqwA2Z4vCa+HBSpP4v6U2ddejcHsolsYxwL5r4bPNoamlV0xDdLg==", "cpu": [ "x64" ], @@ -1314,9 +1314,9 @@ } }, "node_modules/@tailwindcss/oxide-freebsd-x64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.1.15.tgz", - "integrity": "sha512-quISQDWqiB6Cqhjc3iWptXVZHNVENsWoI77L1qgGEHNIdLDLFnw3/AfY7DidAiiCIkGX/MjIdB3bbBZR/G2aJg==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.1.16.tgz", + "integrity": "sha512-TSMpPYpQLm+aR1wW5rKuUuEruc/oOX3C7H0BTnPDn7W/eMw8W+MRMpiypKMkXZfwH8wqPIRKppuZoedTtNj2tg==", "cpu": [ "x64" ], @@ -1331,9 +1331,9 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.1.15.tgz", - "integrity": "sha512-ObG76+vPlab65xzVUQbExmDU9FIeYLQ5k2LrQdR2Ud6hboR+ZobXpDoKEYXf/uOezOfIYmy2Ta3w0ejkTg9yxg==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.1.16.tgz", + "integrity": "sha512-p0GGfRg/w0sdsFKBjMYvvKIiKy/LNWLWgV/plR4lUgrsxFAoQBFrXkZ4C0w8IOXfslB9vHK/JGASWD2IefIpvw==", "cpu": [ "arm" ], @@ -1348,9 +1348,9 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.1.15.tgz", - "integrity": "sha512-4WbBacRmk43pkb8/xts3wnOZMDKsPFyEH/oisCm2q3aLZND25ufvJKcDUpAu0cS+CBOL05dYa8D4U5OWECuH/Q==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.1.16.tgz", + "integrity": "sha512-DoixyMmTNO19rwRPdqviTrG1rYzpxgyYJl8RgQvdAQUzxC1ToLRqtNJpU/ATURSKgIg6uerPw2feW0aS8SNr/w==", "cpu": [ "arm64" ], @@ -1365,9 +1365,9 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm64-musl": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.1.15.tgz", - "integrity": "sha512-AbvmEiteEj1nf42nE8skdHv73NoR+EwXVSgPY6l39X12Ex8pzOwwfi3Kc8GAmjsnsaDEbk+aj9NyL3UeyHcTLg==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.1.16.tgz", + "integrity": "sha512-H81UXMa9hJhWhaAUca6bU2wm5RRFpuHImrwXBUvPbYb+3jo32I9VIwpOX6hms0fPmA6f2pGVlybO6qU8pF4fzQ==", "cpu": [ "arm64" ], @@ -1382,9 +1382,9 @@ } }, "node_modules/@tailwindcss/oxide-linux-x64-gnu": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.1.15.tgz", - "integrity": "sha512-+rzMVlvVgrXtFiS+ES78yWgKqpThgV19ISKD58Ck+YO5pO5KjyxLt7AWKsWMbY0R9yBDC82w6QVGz837AKQcHg==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.1.16.tgz", + "integrity": "sha512-ZGHQxDtFC2/ruo7t99Qo2TTIvOERULPl5l0K1g0oK6b5PGqjYMga+FcY1wIUnrUxY56h28FxybtDEla+ICOyew==", "cpu": [ "x64" ], @@ -1399,9 +1399,9 @@ } }, "node_modules/@tailwindcss/oxide-linux-x64-musl": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.1.15.tgz", - "integrity": "sha512-fPdEy7a8eQN9qOIK3Em9D3TO1z41JScJn8yxl/76mp4sAXFDfV4YXxsiptJcOwy6bGR+70ZSwFIZhTXzQeqwQg==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.1.16.tgz", + "integrity": "sha512-Oi1tAaa0rcKf1Og9MzKeINZzMLPbhxvm7rno5/zuP1WYmpiG0bEHq4AcRUiG2165/WUzvxkW4XDYCscZWbTLZw==", "cpu": [ "x64" ], @@ -1416,9 +1416,9 @@ } }, "node_modules/@tailwindcss/oxide-wasm32-wasi": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.1.15.tgz", - "integrity": "sha512-sJ4yd6iXXdlgIMfIBXuVGp/NvmviEoMVWMOAGxtxhzLPp9LOj5k0pMEMZdjeMCl4C6Up+RM8T3Zgk+BMQ0bGcQ==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.1.16.tgz", + "integrity": "sha512-B01u/b8LteGRwucIBmCQ07FVXLzImWESAIMcUU6nvFt/tYsQ6IHz8DmZ5KtvmwxD+iTYBtM1xwoGXswnlu9v0Q==", "bundleDependencies": [ "@napi-rs/wasm-runtime", "@emnapi/core", @@ -1447,8 +1447,6 @@ }, "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.5.0.tgz", - "integrity": "sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==", "dev": true, "inBundle": true, "license": "MIT", @@ -1460,8 +1458,6 @@ }, "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz", - "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==", "dev": true, "inBundle": true, "license": "MIT", @@ -1518,9 +1514,9 @@ "optional": true }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.15.tgz", - "integrity": "sha512-sJGE5faXnNQ1iXeqmRin7Ds/ru2fgCiaQZQQz3ZGIDtvbkeV85rAZ0QJFMDg0FrqsffZG96H1U9AQlNBRLsHVg==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.16.tgz", + "integrity": "sha512-zX+Q8sSkGj6HKRTMJXuPvOcP8XfYON24zJBRPlszcH1Np7xuHXhWn8qfFjIujVzvH3BHU+16jBXwgpl20i+v9A==", "cpu": [ "arm64" ], @@ -1535,9 +1531,9 @@ } }, "node_modules/@tailwindcss/oxide-win32-x64-msvc": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.1.15.tgz", - "integrity": "sha512-NLeHE7jUV6HcFKS504bpOohyi01zPXi2PXmjFfkzTph8xRxDdxkRsXm/xDO5uV5K3brrE1cCwbUYmFUSHR3u1w==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.1.16.tgz", + "integrity": "sha512-m5dDFJUEejbFqP+UXVstd4W/wnxA4F61q8SoL+mqTypId2T2ZpuxosNSgowiCnLp2+Z+rivdU0AqpfgiD7yCBg==", "cpu": [ "x64" ], @@ -1552,17 +1548,17 @@ } }, "node_modules/@tailwindcss/postcss": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/postcss/-/postcss-4.1.15.tgz", - "integrity": "sha512-IZh8IT76KujRz6d15wZw4eoeViT4TqmzVWNNfpuNCTKiaZUwgr5vtPqO4HjuYDyx3MgGR5qgPt1HMzTeLJyA3g==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/@tailwindcss/postcss/-/postcss-4.1.16.tgz", + "integrity": "sha512-Qn3SFGPXYQMKR/UtqS+dqvPrzEeBZHrFA92maT4zijCVggdsXnDBMsPFJo1eArX3J+O+Gi+8pV4PkqjLCNBk3A==", "dev": true, "license": "MIT", "dependencies": { "@alloc/quick-lru": "^5.2.0", - "@tailwindcss/node": "4.1.15", - "@tailwindcss/oxide": "4.1.15", + "@tailwindcss/node": "4.1.16", + "@tailwindcss/oxide": "4.1.16", "postcss": "^8.4.41", - "tailwindcss": "4.1.15" + "tailwindcss": "4.1.16" } }, "node_modules/@tailwindcss/typography": { @@ -2107,9 +2103,9 @@ } }, "node_modules/daisyui": { - "version": "5.3.7", - "resolved": "https://registry.npmjs.org/daisyui/-/daisyui-5.3.7.tgz", - "integrity": "sha512-0+8PaSGift0HlIQABCeZzWOBV5Nx/vsI2TihB9hbaEyZENPlZZz+se2JnAH5rz9gBYTyDLB7NJup8hkREr6WBw==", + "version": "5.3.8", + "resolved": "https://registry.npmjs.org/daisyui/-/daisyui-5.3.8.tgz", + "integrity": "sha512-ihDXb07IzM/2ugkwBWdy2LFCaepdn1oGsKIsR3gNG/VuTAmS60+HUG9rskjR5BzyJOVVUDDpWoiX3PBDIT3DYQ==", "dev": true, "license": "MIT", "funding": { @@ -3757,9 +3753,9 @@ } }, "node_modules/svelte": { - "version": "5.41.1", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.41.1.tgz", - "integrity": "sha512-0a/huwc8e2es+7KFi70esqsReRfRbrT8h1cJSY/+z1lF0yKM6TT+//HYu28Yxstr50H7ifaqZRDGd0KuKDxP7w==", + "version": "5.41.3", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.41.3.tgz", + "integrity": "sha512-bkHg+whEnVVNcK3XP8Dy4NHujn5mU/+at9z09PXM5THKm+E73AwiKFoRMMTfyAzAj1yExKtudvGHq8UqOh8kMQ==", "license": "MIT", "peer": true, "dependencies": { @@ -3837,9 +3833,9 @@ } }, "node_modules/tailwindcss": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.15.tgz", - "integrity": "sha512-k2WLnWkYFkdpRv+Oby3EBXIyQC8/s1HOFMBUViwtAh6Z5uAozeUSMQlIsn/c6Q2iJzqG6aJT3wdPaRNj70iYxQ==", + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.16.tgz", + "integrity": "sha512-pONL5awpaQX4LN5eiv7moSiSPd/DLDzKVRJz8Q9PgzmAdd1R4307GQS2ZpfiN7ZmekdQrfhZZiSE5jkLR4WNaA==", "license": "MIT", "peer": true }, diff --git a/frontend/package.json b/frontend/package.json index 654b1290..51fb970f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -33,7 +33,8 @@ "tailwindcss": "^4.1.12", "typescript": "^5.7.3", "typescript-eslint": "^8.23.0", - "vite": "^6.4.1" + "vite": "^6.4.1", + "vite-plugin-devtools-json": "^1.0.0" }, "type": "module", "dependencies": { diff --git a/frontend/src/components/app-service-button.svelte b/frontend/src/components/app-service-button.svelte index 5c94c99f..10bd062f 100644 --- a/frontend/src/components/app-service-button.svelte +++ b/frontend/src/components/app-service-button.svelte @@ -17,12 +17,15 @@ {#if status !== 'Running'} {:else} + {#each service.domains as domain (domain)} {title(domain)}{title(domain)} {/each} + {/if} diff --git a/frontend/src/components/custom-actions-dropdown.svelte b/frontend/src/components/custom-actions-dropdown.svelte index ecf95159..8f869d4a 100644 --- a/frontend/src/components/custom-actions-dropdown.svelte +++ b/frontend/src/components/custom-actions-dropdown.svelte @@ -1,50 +1,78 @@ -{#if hasAvailableActions()} -
+{#if hasActions} +
diff --git a/frontend/src/components/start-stop-app-action.svelte b/frontend/src/components/start-stop-app-action.svelte index 69a77b15..8dde8fa0 100644 --- a/frontend/src/components/start-stop-app-action.svelte +++ b/frontend/src/components/start-stop-app-action.svelte @@ -40,7 +40,10 @@ {#if failed_task} -
+
diff --git a/frontend/src/components/tasks-table.svelte b/frontend/src/components/tasks-table.svelte index 1f30826d..9ee0aa34 100644 --- a/frontend/src/components/tasks-table.svelte +++ b/frontend/src/components/tasks-table.svelte @@ -1,4 +1,5 @@ + + +
+
+

{heading}

+
+ {#if lines.length > 0} + {lines.length} lines + {/if} + {#if loading} + + Streaming... + {/if} +
+
+ +
+ {#if loading && lines.length === 0} +
+ +
Streaming...
+
+ {:else} + {#each lines as line, index (line.sequence)} + 
{#if showTimestamps}{formatTimestamp(line.timestamp)}{/if}{#if getStreamTypeIcon(line.stream)}{:else}{/if}{#if showTimestamps}{getStreamTypePrefix(line.stream)}{/if}{line.content}
+ {/each} + + {#if lines.length === 0 && !loading} +
No output available
+ {/if} + {/if} +
+ + {#if lines.length > 0} +
+ + + + + +
+ {/if} +
+ + diff --git a/frontend/src/components/user-info.svelte b/frontend/src/components/user-info.svelte index fe961d70..872e123b 100644 --- a/frontend/src/components/user-info.svelte +++ b/frontend/src/components/user-info.svelte @@ -1,23 +1,10 @@ @@ -33,6 +20,7 @@
- scotty @ {site_info.domain} + scotty @ {site_info.domain}
-