From e6c26d7d656d090eb26b6ce5cf1cbc7b9142406d Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Sun, 1 Mar 2026 23:12:12 -0500 Subject: [PATCH] feat: add trogonstack-datadog plugin Signed-off-by: Yordis Prieto --- .claude-plugin/marketplace.json | 6 + .github/release-please-config.json | 3 + .github/release-please-manifest.json | 3 +- .../.claude-plugin/plugin.json | 9 + plugins/trogonstack-datadog/CHANGELOG.md | 1 + .../skills/datadog-design-dashboard/SKILL.md | 147 ++++++++ .../references/frameworks.md | 132 +++++++ .../references/layouts.md | 330 ++++++++++++++++++ .../references/thresholds.md | 62 ++++ .../references/widgets.md | 297 ++++++++++++++++ .../skills/datadog-review-dashboard/SKILL.md | 230 ++++++++++++ 11 files changed, 1219 insertions(+), 1 deletion(-) create mode 100644 plugins/trogonstack-datadog/.claude-plugin/plugin.json create mode 100644 plugins/trogonstack-datadog/CHANGELOG.md create mode 100644 plugins/trogonstack-datadog/skills/datadog-design-dashboard/SKILL.md create mode 100644 plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/frameworks.md create mode 100644 plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/layouts.md create mode 100644 plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/thresholds.md create mode 100644 plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/widgets.md create mode 100644 plugins/trogonstack-datadog/skills/datadog-review-dashboard/SKILL.md diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f7f9a9e..fb3b6a7 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -24,6 +24,12 @@ "description": "NATS messaging skills for designing subject hierarchies with naming conventions and wildcard patterns", "source": "./plugins/trogonstack-nats", "category": "development" + }, + { + "name": "trogonstack-datadog", + "description": "Datadog observability skills for designing dashboards with proper widget selection, layout patterns, and template variables using the pup CLI", + "source": "./plugins/trogonstack-datadog", + "category": "development" } ] } diff --git a/.github/release-please-config.json b/.github/release-please-config.json index e9baf2a..2a79a15 100644 --- a/.github/release-please-config.json +++ b/.github/release-please-config.json @@ -32,6 +32,9 @@ }, "plugins/trogonstack-nats": { "component": "trogonstack-nats" + }, + "plugins/trogonstack-datadog": { + "component": "trogonstack-datadog" } }, "plugins": [ diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index 1413f1a..669b561 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1,5 +1,6 @@ { "plugins/trogonstack-diataxis": "0.0.2", "plugins/trogonstack-gh": "0.0.2", - "plugins/trogonstack-nats": "0.0.2" + "plugins/trogonstack-nats": "0.0.2", + "plugins/trogonstack-datadog": "0.0.1" } diff --git a/plugins/trogonstack-datadog/.claude-plugin/plugin.json b/plugins/trogonstack-datadog/.claude-plugin/plugin.json new file mode 100644 index 0000000..6fc7acb --- /dev/null +++ b/plugins/trogonstack-datadog/.claude-plugin/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "trogonstack-datadog", + "description": "Datadog observability skills for designing dashboards with proper widget selection, layout patterns, and template variables using the pup CLI", + "version": "0.0.1", + "author": { + "name": "TrogonStack", + "url": "https://github.com/TrogonStack" + } +} diff --git a/plugins/trogonstack-datadog/CHANGELOG.md b/plugins/trogonstack-datadog/CHANGELOG.md new file mode 100644 index 0000000..825c32f --- /dev/null +++ b/plugins/trogonstack-datadog/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/plugins/trogonstack-datadog/skills/datadog-design-dashboard/SKILL.md b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/SKILL.md new file mode 100644 index 0000000..e3264b6 --- /dev/null +++ b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/SKILL.md @@ -0,0 +1,147 @@ +--- +name: datadog-design-dashboard +description: >- + Design Datadog dashboards with proper widget selection, layout organization, + template variables, and group structure. Uses pup CLI for inspecting existing + dashboards and validating designs. Use when designing new dashboards, auditing + existing ones, or planning observability layouts for services. Do not use for: + (1) Datadog agent installation or configuration, (2) monitor/alert rule design, + (3) APM instrumentation or tracing setup, (4) log pipeline configuration. +allowed-tools: AskUserQuestion, Write, Read, Shell +--- + +# Design Datadog Dashboard + +Design a dashboard layout that tells a clear story — from high-level health signals down to granular diagnostics — using proper widget types, group organization, and template variables for reusability. + +**Important**: Always check for existing dashboards first with `pup dashboards list`. Do not create a new dashboard if one already exists for the same service or purpose — update the existing one instead. Only create a new dashboard when no relevant one exists or the user explicitly asks for a new one. + +**Philosophy**: The frameworks, layouts, and widget guides in this skill are starting points — not rigid rules. Every product and business is different. Understand the domain first, then adapt the frameworks to fit. The best dashboards reflect how the business actually works, not how a generic template says they should. + +## Domain Discovery + +Before designing, understand what you are building observability for. The metrics that matter depend entirely on the product and business context. + +**Ask the user**: +- "Can you tell me about the product and what this service does for the business? What does a customer experience when they interact with it?" +- "What does a bad day look like for this service? What breaks, and how do customers feel it?" +- "Are there codebases, architecture docs, or README files I can read to understand the service and its dependencies?" + +**If the user points you to a codebase**: Read it. Look at the entry points, the API routes, the database models, the queue consumers, the external service calls. Understanding the code gives you the context to choose metrics that actually matter — not just generic RED/USE signals. + +**If the user describes the business**: Use that context to tailor the Customer-Facing section. An e-commerce service cares about checkout success rates. A messaging service cares about delivery latency. A payment service cares about transaction completion. Generic "request rate" and "error rate" are a starting point, but the real value comes from metrics that map to business outcomes. + +**Skip domain discovery if**: You already have deep context about the service from prior conversations or the user has provided detailed specifications. + +## Interview + +**Skip if ALL of these are already specified**: dashboard purpose, target audience, data sources, template variable needs, dashboard strategy. + +**Always interview if**: Auditing or redesigning an existing dashboard (needs current state review first). + +1. **Purpose** — "What is this dashboard for? Service overview, infrastructure, executive KPIs, debugging, or SLO tracking?" +2. **Audience** — "Who will use this? On-call engineers, platform team, leadership, or mixed?" +3. **Data Sources** — "Which Datadog products are involved? Metrics only, APM + Metrics, Logs + Metrics, or full stack?" +4. **Scope** — "Is this for a single service, a group of services, or infrastructure-wide?" +5. **Dashboard Strategy** — "One dashboard per service, or a consolidated view?" — share the trade-offs from [references/layouts.md](references/layouts.md) to help them decide. If unsure, ask: "During an outage, does your team investigate one service at a time, or do they need to see all services simultaneously?" +6. **Existing Dashboard** — "Is there an existing dashboard to audit or redesign?" If yes, fetch with `pup dashboards get ` before designing. + +--- + +## Workflow + +### 1. Gather existing context + +```bash +pup dashboards list +pup dashboards get +pup dashboards url +``` + +If auditing an existing dashboard, fetch its definition first and analyze its current structure before redesigning. + +### 2. Choose a framework + +Match the dashboard purpose to a framework. Read [references/frameworks.md](references/frameworks.md) for detailed metric mappings and group structures. + +| Purpose | Framework | +|---------|-----------| +| Service overview | RED (Rate, Errors, Duration) | +| Infrastructure | USE (Utilization, Saturation, Errors) | +| Executive/business | Golden Signals | +| SLO tracking | SLI/SLO | +| Debugging | Drill-down | + +### 3. Design the layout + +Using your domain understanding and the chosen framework, design the group structure and select widgets. Read these references as needed: + +- **[layouts.md](references/layouts.md)** — Template variable conventions, group structure patterns, dashboard strategy trade-offs, grid sizing, anti-patterns +- **[widgets.md](references/widgets.md)** — Widget selection guide, display options, sizing, naming conventions +- **[thresholds.md](references/thresholds.md)** — Alert threshold markers, threshold proximity, Y-axis configuration + +**Key principles** (not rigid rules — use judgment): +- **Prefix every widget title** with its layer and priority: `I0:` (most critical infra), `P0:` (most critical platform), `D0:` (most critical domain). See [widgets.md](references/widgets.md) for the full prefix system. +- Start with a **Customer-Facing** group (5-8 metrics) so someone with zero service knowledge can tell if customers are affected within 5 seconds. Tailor the metrics to the domain. +- Timeseries widgets should have **alert threshold markers** (red lines) with thresholds close to normal traffic. If a metric doesn't warrant an alert, question whether it belongs — but context-providing metrics can earn their place. +- Set **Y-axis max** explicitly near the threshold — don't let auto-scaling compress the normal range. +- Order groups macro-to-micro: customer-facing → overview → domain-specific → infrastructure. + +### 4. Write the output + +Present the design using this template: + +```markdown +# Dashboard Design: [Dashboard Title] + +## Purpose +[1-2 sentences: what this monitors, who uses it] + +## Template Variables +| Variable | Tag | Default | +|----------|-----|---------| +| ... | ... | `*` | + +## Layout + +### Group: [Group Title] +| Widget | Type | Query/Metric | Width | Alert Threshold | +|--------|------|-------------|-------|----------------| +| ... | ... | ... | ... | ... | + +[Repeat for each group] + +## Quality Validation +[Run quality principles below] +``` + +### 5. Validate + +```bash +pup dashboards list +pup dashboards get +pup metrics list --filter="trace.http.request.*" +``` + +--- + +## Quality Principles + +- [ ] Dashboard reflects the actual product and business — metrics tailored to the domain +- [ ] Dashboard title is concise (no environment, region, or version) +- [ ] Template variables defined for env, service, and relevant scopes (default `*`) +- [ ] **Customer-Facing group** with 5-8 metrics tailored to the service's business impact +- [ ] Groups ordered macro-to-micro (customer-facing → overview → details) +- [ ] **Timeseries widgets have alert threshold markers** (red lines) where the metric is alertable +- [ ] **Thresholds close to normal traffic** — no excessive whitespace +- [ ] **Zero-knowledge readability** — someone with no service knowledge can spot problems via red indicators +- [ ] **Widget titles prefixed** with layer and priority (`I0:`, `P1:`, `D0:`, etc.) +- [ ] Widget titles use sentence case, don't repeat group name +- [ ] Every metric earns its place — if it spikes, someone can act on it + +## References + +- **[Observability Frameworks](references/frameworks.md)** — RED, USE, Golden Signals, SLI/SLO with metric mappings +- **[Layout & Structure](references/layouts.md)** — Template variables, group patterns, dashboard strategy, grid sizing, anti-patterns +- **[Widgets](references/widgets.md)** — Widget types, display options, sizing, naming conventions +- **[Alert Thresholds](references/thresholds.md)** — Threshold markers, proximity guide, Y-axis configuration diff --git a/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/frameworks.md b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/frameworks.md new file mode 100644 index 0000000..b2f6f4e --- /dev/null +++ b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/frameworks.md @@ -0,0 +1,132 @@ +# Observability Frameworks + +Choose the framework that matches the dashboard purpose. Each framework defines which metrics to prioritize and how to organize them. + +--- + +## RED Method (Request-Driven Services) + +Best for: microservices, APIs, web applications — anything that handles requests. + +| Signal | Metric | Example Query | +|--------|--------|---------------| +| **Rate** | Requests per second | `sum:trace.http.request.hits{$service,$env}.as_rate()` | +| **Errors** | Error rate (%) | `sum:trace.http.request.errors{$service,$env} / sum:trace.http.request.hits{$service,$env} * 100` | +| **Duration** | Latency percentiles | `p50:trace.http.request.duration{$service,$env}`, `p90:trace.http.request.duration{$service,$env}`, `p99:trace.http.request.duration{$service,$env}` (one query per percentile, overlaid on the same timeseries widget) | + +### Group Structure + +```text +Overview → Rate → Errors → Duration → [Dependencies] → [Infrastructure] +``` + +### When to Use RED + +- Service receives external or internal HTTP/gRPC requests +- You care about user-facing latency and availability +- On-call engineers need to quickly identify degradation + +### Drill-Down Pattern + +1. **Overview**: Query Value widgets showing current Rate, Error %, p99 Latency +2. **Rate**: Timeseries of request rate, Top List by endpoint +3. **Errors**: Timeseries of error rate, Top List of failing endpoints, Log Stream of errors +4. **Duration**: Timeseries of latency percentiles, Heatmap of latency distribution, Top List of slowest endpoints + +--- + +## USE Method (Resource-Oriented) + +Best for: infrastructure, databases, queues — anything with finite capacity. + +| Signal | Metric | Example Query | +|--------|--------|---------------| +| **Utilization** | Resource usage (%) | `avg:system.cpu.user{$host,$env}` | +| **Saturation** | Queue depth / waiting | `avg:system.load.1{$host,$env}` | +| **Errors** | Hardware/resource errors | `sum:system.disk.error{$host,$env}` | + +### Group Structure + +```text +Overview → CPU → Memory → Disk → Network → [Application-specific] +``` + +### When to Use USE + +- Monitoring hosts, containers, or VMs +- Database or cache performance +- Queue/worker infrastructure +- Capacity planning + +### Resource Mapping + +| Resource | Utilization | Saturation | Errors | +|----------|-------------|------------|--------| +| CPU | `system.cpu.user` | `system.load.1` | — | +| Memory | `system.mem.pct_usable` | `system.swap.used` | OOM events | +| Disk | `system.disk.in_use` | `system.io.await` | `system.disk.error` | +| Network | `system.net.bytes_sent` | `system.net.packets_dropped` | `system.net.errors` | + +--- + +## Golden Signals (SRE / Executive) + +Best for: executive dashboards, cross-service views, SRE-level monitoring. + +| Signal | Description | Example Metric | +|--------|-------------|----------------| +| **Latency** | Time to serve requests | `p99:trace.http.request.duration{$service,$env}` | +| **Traffic** | Request volume | `sum:trace.http.request.hits{$service,$env}.as_rate()` | +| **Errors** | Rate of failed requests | `sum:trace.http.request.errors{$service,$env} / sum:trace.http.request.hits{$service,$env}` | +| **Saturation** | Resource fullness | `avg:system.cpu.user{$service,$env}`, `avg:system.mem.pct_usable{$service,$env}` | + +### Group Structure + +```text +Executive Summary → Latency → Traffic → Errors → Saturation → [Business KPIs] +``` + +### When to Use Golden Signals + +- Cross-service or platform-wide view +- Executive or leadership audience +- SRE team situational awareness +- Incident command dashboards + +### Presentation Style + +- Executive dashboards favor **Query Value** and **Timeseries** over detailed Top Lists +- Use wider time windows (1h, 4h, 1d) for trend visibility +- Include week-over-week comparisons where possible + +--- + +## SLI/SLO Tracking + +Best for: SLO compliance, error budget monitoring, reliability reviews. + +| Signal | Description | Example Metric | +|--------|-------------|----------------| +| **SLI** | Service Level Indicator | Availability %, latency p99 < threshold | +| **Error Budget** | Remaining budget before SLO breach | `(1 - SLI) / (1 - SLO_target)` | +| **Burn Rate** | How fast budget is consumed | Error budget consumed / time elapsed | + +### Group Structure + +```text +SLO Summary → Error Budget → Burn Rate Alerts → Historical Compliance +``` + +### When to Use SLI/SLO + +- Tracking reliability commitments +- Error budget-driven development decisions +- Reliability review meetings +- Incident post-mortem context + +### Widget Recommendations + +- **SLO widget**: Native Datadog SLO widget for compliance tracking +- **Query Value**: Current error budget remaining (color-coded) +- **Timeseries**: Burn rate over time with threshold lines +- **Change Widget**: Week-over-week SLI comparison diff --git a/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/layouts.md b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/layouts.md new file mode 100644 index 0000000..8a9280e --- /dev/null +++ b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/layouts.md @@ -0,0 +1,330 @@ +# Layout & Structure + +Template variable conventions, group structure patterns, dashboard strategy trade-offs, and grid sizing. + +--- + +## Template Variables + +Template variables make one dashboard serve many contexts. Define them before laying out widgets. + +**Standard variables** (include when applicable): + +| Variable | Tag | Use Case | +|----------|-----|----------| +| `env` | `env` | Environment filtering (prod, staging, dev) | +| `service` | `service` | Service scoping | +| `region` | `region` | Regional filtering | +| `availability_zone` | `availability_zone` | AZ-level drill-down | +| `host` | `host` | Host-level investigation | + +- Start with the broadest scope (`env`) and narrow down +- Use `*` as the default value so dashboards load with full scope +- Never put environment or region in the dashboard title — that is what template variables are for +- Name variables after the tag they filter on + +--- + +## Dashboard Strategy + +| Approach | Strengths | Weaknesses | +|----------|-----------|------------| +| **Per-service** | Focused, fast to scan during incidents. Each team owns their dashboard. Customer-Facing section is specific and actionable. Ops reviews can go service-by-service. | More dashboards to maintain. Cross-service correlation requires switching dashboards. | +| **Consolidated** | Single pane of glass for multiple services. Good for seeing cross-service dependencies. Fewer dashboards to maintain. | Can become overwhelming (100+ metrics). Customer-Facing section becomes diluted. Slower to load and harder to scan during incidents. | +| **Hybrid** | Per-service dashboards for depth + one top-level dashboard with only the Customer-Facing section from each service. Best of both worlds. | Requires maintaining both levels. Customer-Facing metrics duplicated across dashboards. | + +--- + +## Group Structure + +Organize widgets into collapsible groups. Groups are the primary navigation mechanism. + +**Recommended groups** (in order): + +1. **Customer-Facing** — 5-8 metrics that answer "are customers affected?" within 5 seconds. Should be the first group. The specific metrics depend on the product. Design so someone with zero service knowledge can spot problems via red indicators. +2. **Overview** — Service checks, key health indicators, monitor summaries. +3. **Domain-specific groups** — Organized by the chosen framework (e.g., Rate / Errors / Duration for RED), adapted to the service's actual architecture and concerns. + +**Optional groups**: Logs, Infrastructure, Dependencies, Deployment. + +- Group titles use **Title Case** +- Keep groups to 4-8 widgets each (collapse if more) +- Order groups macro-to-micro (health overview → detailed diagnostics) +- Color-code group headers for visual scanning + +--- + +## Layout Templates + +Standard group structures by dashboard type. Use these as starting points and customize based on specific needs. + +### Service Overview (RED) + +The most common dashboard type. Monitors a single service's request-level health. + +```text +┌─────────────────────────────────────────────────────┐ +│ Template Variables: env | service | region │ +├─────────────────────────────────────────────────────┤ +│ Group: Customer-Facing │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │ P0:Req/s │ P0:Errs │ P0:p99 │ P0:Apdex │ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ ├──────────────────────┬────────────────────┤ │ +│ │ D0: Key txn success │ I0: DB conn pool │ │ +│ │ (TS + red threshold) │ (TS + red thresh.) │ │ +│ └──────────────────────┴────────────────────┘ │ +│ 5-8 metrics · zero-knowledge readable │ +├─────────────────────────────────────────────────────┤ +│ Group: Overview │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │ P0:Req/s │ P0:Err % │ P0:p99 │ P0:Apdex │ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ └──────────┴──────────┴──────────┴──────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Rate │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ P0: Requests/s (TS) │ P1: By endpoint (TL) │ │ +│ │ ── red threshold ── │ │ │ +│ └────────────────────────┴──────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Errors │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ P0: Error rate (TS) │ P1: Top errors (TL) │ │ +│ │ ── red threshold ── │ │ │ +│ ├────────────────────────────────────────────────┤ │ +│ │ P1: Error logs (log stream, full width) │ │ +│ └────────────────────────────────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Duration │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ P0: Latency p50/90/99 │ P1: Latency heatmap │ │ +│ │ (TS + red threshold) │ (heatmap) │ │ +│ ├────────────────────────────────────────────────┤ │ +│ │ P1: Slowest endpoints (toplist, full width) │ │ +│ └────────────────────────────────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Infrastructure (collapsed by default) │ +│ ┌────────────┬────────────┬────────────┐ │ +│ │ I0: CPU │ I0: Memory │ I1: Disk │ │ +│ │(TS + red) │(TS + red) │(TS + red) │ │ +│ └────────────┴────────────┴────────────┘ │ +└─────────────────────────────────────────────────────┘ +``` + +All widget titles use the layer-priority prefix system (`I0:`, `P0:`, `D0:`, etc.) — see [widgets.md](widgets.md) for details. +All timeseries widgets include red alert threshold markers set close to normal traffic. + +**Widget count**: 20-24 +**Target audience**: On-call engineers, service owners + +--- + +### Infrastructure (USE) + +Monitors host, container, or VM resource health. + +```text +┌─────────────────────────────────────────────────────┐ +│ Template Variables: env | host | availability_zone │ +├─────────────────────────────────────────────────────┤ +│ Group: Customer-Facing │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │P0:Svc Rq │P0:Svc Err│P0:Svc p99│P0:Apdex │ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ ├──────────────────────┬────────────────────┤ │ +│ │ I0: Host avail. │ I0: Network errors │ │ +│ │ (TS + red threshold) │ (TS + red thresh.) │ │ +│ └──────────────────────┴────────────────────┘ │ +│ 5-8 metrics · zero-knowledge readable │ +├─────────────────────────────────────────────────────┤ +│ Group: Overview │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │ I0:CPU % │ I0:Mem % │ I0:Disk% │ I1:NetBps│ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ └──────────┴──────────┴──────────┴──────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: CPU │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ I0: CPU by core │ I1: Load avg │ │ +│ │ (TS + red threshold) │ (TS + red threshold) │ │ +│ └────────────────────────┴──────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Memory │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ I0: Memory usage │ I1: Swap usage │ │ +│ │ (TS + red threshold) │ (TS + red threshold) │ │ +│ └────────────────────────┴──────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Disk │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ I0: Disk utilization │ I1: I/O wait │ │ +│ │ (TS + red threshold) │ (TS + red threshold) │ │ +│ └────────────────────────┴──────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Network │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ I0: Bytes in/out │ I0: Errors + drops │ │ +│ │ (TS + red threshold) │ (TS + red threshold) │ │ +│ └────────────────────────┴──────────────────────┘ │ +└─────────────────────────────────────────────────────┘ +``` + +All widget titles use the layer-priority prefix system — see [widgets.md](widgets.md) for details. +All timeseries widgets include red alert threshold markers set close to normal traffic. + +**Widget count**: 18-22 +**Target audience**: Platform/SRE team + +--- + +### Executive / Golden Signals + +High-level view across multiple services for leadership. + +```text +┌─────────────────────────────────────────────────────┐ +│ Template Variables: env | team | region │ +├─────────────────────────────────────────────────────┤ +│ Group: Customer-Facing │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │P0:Uptime │P0:Cst Err│P0:p99 Lat│P0:Traffic│ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ ├──────────────────────┬────────────────────┤ │ +│ │ D0: Revenue txn succ │ D0: Checkout lat │ │ +│ │ (TS + red threshold) │ (TS + red thresh.) │ │ +│ └──────────────────────┴────────────────────┘ │ +│ 5-8 metrics · zero-knowledge readable │ +├─────────────────────────────────────────────────────┤ +│ Group: Executive Summary │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │P0:Uptime │P1:Avg Lat│P1:Tot Rq │P0:Err % │ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ └──────────┴──────────┴──────────┴──────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Traffic │ +│ ┌──────────────────────────────────────────────┐ │ +│ │ P0: Request volume by svc (TS + red thresh.) │ │ +│ └──────────────────────────────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Latency │ +│ ┌──────────────────────────────────────────────┐ │ +│ │ P0: p99 latency by svc (TS + red threshold) │ │ +│ └──────────────────────────────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Errors │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ P0: Err rate by svc │ P1: Svcs w/ errors │ │ +│ │ (TS + red threshold) │ (toplist) │ │ +│ └────────────────────────┴──────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Saturation │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ I0: CPU across fleet │ I0: Memory fleet │ │ +│ │ (TS + red threshold) │ (TS + red threshold) │ │ +│ └────────────────────────┴──────────────────────┘ │ +└─────────────────────────────────────────────────────┘ +``` + +All widget titles use the layer-priority prefix system — see [widgets.md](widgets.md) for details. +All timeseries widgets include red alert threshold markers set close to normal traffic. + +**Widget count**: 16-20 +**Target audience**: Engineering leadership, SRE + +--- + +### Debugging / Investigation + +Deep-dive dashboard for active incident investigation. + +```text +┌─────────────────────────────────────────────────────┐ +│ Template Variables: env | service | host | endpoint │ +├─────────────────────────────────────────────────────┤ +│ Group: Customer-Facing │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │P0:Cst Err│P0:p99 Lat│P0:Req/s │P0:Apdex │ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ ├──────────────────────┬────────────────────┤ │ +│ │ D0: Key txn success │ I0: DB conn pool │ │ +│ │ (TS + red threshold) │ (TS + red thresh.) │ │ +│ └──────────────────────┴────────────────────┘ │ +│ 5-8 metrics · zero-knowledge readable │ +├─────────────────────────────────────────────────────┤ +│ Group: Current State │ +│ ┌──────────┬──────────┬──────────┬──────────┐ │ +│ │P0:Err % │P0:p99 ms │P1:Req/s │I1:Hosts │ │ +│ │ (QV+bg) │ (QV+bg) │ (QV+bg) │ (QV+bg) │ │ +│ └──────────┴──────────┴──────────┴──────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Error Analysis │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ P0: Errors by type │ P1: By endpoint (TL) │ │ +│ │ (TS + red threshold) │ (toplist) │ │ +│ ├────────────────────────────────────────────────┤ │ +│ │ P1: Error logs (log stream, full width) │ │ +│ └────────────────────────────────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Latency Breakdown │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ P0: Latency by endpt │ P1: Latency distrib. │ │ +│ │ (TS + red threshold) │ (heatmap) │ │ +│ └────────────────────────┴──────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Dependencies │ +│ ┌────────────────────────┬──────────────────────┐ │ +│ │ P0: Downstream latency │ P0: Downstream errs │ │ +│ │ (TS + red threshold) │ (TS + red threshold) │ │ +│ └────────────────────────┴──────────────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Resource Pressure │ +│ ┌────────────┬────────────┬────────────┐ │ +│ │ I0: CPU │ I0: Memory │ P1: Conns │ │ +│ │(TS + red) │(TS + red) │(TS + red) │ │ +│ └────────────┴────────────┴────────────┘ │ +├─────────────────────────────────────────────────────┤ +│ Group: Recent Events │ +│ ┌──────────────────────────────────────────────┐ │ +│ │ Deploy + event stream (full width) │ │ +│ └──────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────┘ +``` + +All widget titles use the layer-priority prefix system — see [widgets.md](widgets.md) for details. +All timeseries widgets include red alert threshold markers set close to normal traffic. + +**Widget count**: 24-30 +**Target audience**: On-call engineers during incidents +**Note**: More template variables for deeper filtering; `endpoint` variable enables per-route investigation + +--- + +## Grid System + +Datadog uses a 12-column grid. + +| Widget Type | Minimum Width | Recommended Width | +|-------------|--------------|-------------------| +| Query Value | 2 col | 3 col | +| Timeseries | 4 col | 6 col | +| Top List | 4 col | 6 col | +| Heatmap | 4 col | 6 col | +| Log Stream | 6 col | 12 col | +| Event Stream | 6 col | 12 col | +| Note | 2 col | 3-4 col | +| Check Status | 2 col | 3 col | + +--- + +## Anti-Patterns + +| Anti-Pattern | Problem | Fix | +|-------------|---------|-----| +| Separate dashboards per environment | Dashboard sprawl, inconsistent layouts | Use `env` template variable | +| 50+ widgets in a single group | Overwhelming, slow to load | Split into focused groups, collapse secondary groups | +| Query Value without timeseries background | No trend context, just a number | Enable timeseries background | +| Mixing unrelated metrics in one group | Unclear narrative, hard to scan | One concern per group | +| Dashboard title with environment/region | Forces duplication | Put context in template variables | +| Identical widgets with different filters | Redundant, hard to maintain | Use template variables + saved views | +| Y-axis auto-scaling with distant threshold | Normal traffic compressed into flat band | Set `yaxis.max` near threshold — see [thresholds.md](thresholds.md) | diff --git a/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/thresholds.md b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/thresholds.md new file mode 100644 index 0000000..567f0e8 --- /dev/null +++ b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/thresholds.md @@ -0,0 +1,62 @@ +# Alert Thresholds + +Guidance for configuring alert threshold markers and Y-axis settings on timeseries widgets. + +--- + +## Threshold Markers + +Timeseries widgets should have a red threshold marker line so anomalies are immediately visible. + +- The threshold should correspond to an actionable alert — if the metric crosses this line, someone investigates +- Use `markers` in the widget definition with `display_type: "error dashed"` and `value: "y = "` +- The threshold should match (or be derived from) an actual Datadog monitor +- Use red color for instant visual identification + +Not every metric needs a threshold. Context-providing metrics (deployment markers, dependency traffic patterns) may earn their place on a dashboard without one. Use judgment based on the service's domain. + +--- + +## Threshold Proximity + +Set thresholds close to normal traffic. Large gaps between normal values and the alert line create blind spots where anomalies go unnoticed. + +**Bad**: Normal CPU is 20%, alert threshold at 95% — the graph is mostly empty space and a slow climb from 20% to 80% looks flat. + +**Good**: Normal CPU is 20%, alert threshold at 45% — anomalies visually stand out immediately. + +| Normal Value | Bad Threshold | Good Threshold | Why | +|-------------|--------------|----------------|-----| +| CPU ~20% | 95% | 40-50% | 75% gap hides slow climbs | +| Error rate ~0.1% | 10% | 1-2% | 10% gap masks gradual increase | +| Latency p99 ~50ms | 5000ms | 100-150ms | 100x gap makes 200ms look normal | +| Queue depth ~10 | 10000 | 50-100 | 1000x gap hides backpressure | + +**Design test**: If the metric slowly climbs from normal to 2x normal, the change should be visually obvious on the graph. If the threshold is too far away, the graph compresses the normal range and the climb looks flat. + +--- + +## Y-Axis Configuration + +Do not rely on Y-axis auto-scaling. Set `yaxis.max` explicitly to slightly above the alert threshold. + +**Why**: Auto-scaling fits the entire value range (normal traffic + threshold), which compresses normal traffic into a flat band at the bottom of the graph. Anomalies become invisible because the Y-axis range is too wide. + +**Rule**: The Y-axis max should frame the normal-to-threshold range so that deviations are visually obvious. The alert threshold should sit near the top of the graph, and normal traffic should occupy the visible area. + +| Metric | Normal | Threshold | Y-Axis Max | +|--------|--------|-----------|------------| +| CPU | ~20% | 45% | 55% | +| Error rate | ~0.1% | 2% | 3% | +| Latency p99 | ~50ms | 150ms | 175ms | +| Queue depth | ~10 | 50 | 65 | + +--- + +## Metrics Near Zero + +Some metrics hover close to zero under normal conditions (e.g., error counts, retry rates, dead letter queue depth). For these: + +- A threshold still makes sense — even a small absolute increase can indicate a problem +- Set the threshold at a level that represents "something changed" rather than a percentage of capacity +- The Y-axis max matters even more here — auto-scaling on a metric that's normally 0 and spikes to 5 will look the same as one that spikes to 5000 diff --git a/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/widgets.md b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/widgets.md new file mode 100644 index 0000000..61e642f --- /dev/null +++ b/plugins/trogonstack-datadog/skills/datadog-design-dashboard/references/widgets.md @@ -0,0 +1,297 @@ +# Widget Reference + +Configuration details, display options, and best practices for each widget type. + +--- + +## Widget Selection + +| Data Type | Widget | Display Type | +|-----------|--------|-------------| +| **Current value** (latency p99, error rate) | Query Value | With timeseries background | +| **Trend over time** (request rate, CPU) | Timeseries | `lines` for multiple groups, `area` for volume, `bars` for counts | +| **Ranked comparison** (top endpoints, busiest hosts) | Top List | Descending order | +| **Distribution** (latency percentiles) | Heatmap | Color-coded density | +| **Multiple related metrics** | Group | Collapsible container | +| **Status at a glance** (service checks) | Check Status | Color-coded grid | +| **Log volume** | Timeseries | `bars` grouped by status | +| **Log entries** | Log Stream | Filtered to error/critical | +| **Recent events** | Event Stream | Filtered by source | +| **Text context** | Note | Runbook links, team ownership | + +```text +What are you showing? +│ +├── Single current value? → Query Value (with timeseries bg) +├── Trend over time? → Timeseries +│ ├── Volume/composition? → area display +│ ├── Counts? → bars display +│ └── Multiple series? → lines display +├── Ranked comparison? → Top List +├── Distribution density? → Heatmap +├── Up/down status? → Check Status +├── Log entries? → Log Stream +├── Events/deploys? → Event Stream +├── Period comparison? → Change +├── SLO compliance? → SLO widget +└── Context/links? → Note +``` + +--- + +## Query Value + +Displays a single numeric value with optional conditional formatting and timeseries background. + +**Use for**: Current metric values — error rate, latency, throughput, count. + +**Configuration**: +- Enable timeseries background (a bare number without trend context is rarely useful) +- Use conditional formatting to color-code thresholds (green/yellow/red) +- Set appropriate precision (2 decimals for percentages, 0 for counts) + +**Sizing**: 3 columns wide, 2 rows tall (standard), or 2 columns for compact layouts. + +**Conditional Formatting Example**: + +| Condition | Color | Meaning | +|-----------|-------|---------| +| value < 1% | Green | Healthy error rate | +| value >= 1% AND value < 5% | Yellow | Warning | +| value >= 5% | Red | Critical | + +--- + +## Timeseries + +Time-based line, area, or bar chart. The most versatile widget. + +**Use for**: Any metric over time — trends, comparisons, correlations. + +**Display Types**: + +| Display | Use When | +|---------|----------| +| `lines` | Multiple groups/series, general trends | +| `area` | Volume metrics (stacked area for composition) | +| `bars` | Count-based metrics, log volume by status | + +**Configuration**: +- Add a legend (automatic mode for > 5 series) +- Alias formula expressions for readable legends +- Set y-axis minimum to 0 unless negative values are expected +- Set `yaxis.max` explicitly — see [thresholds.md](thresholds.md) for Y-axis guidance +- Use markers for threshold lines — see [thresholds.md](thresholds.md) for threshold configuration + +**Sizing**: Minimum 4 columns, recommended 6 columns. Use 12 columns for high-detail single-metric views. + +**Multiple Queries**: Overlay related metrics (e.g., p50, p90, p99 on one chart) rather than creating separate widgets. + +--- + +## Top List + +Ranked list of values by a dimension. + +**Use for**: Identifying outliers — busiest endpoints, highest error sources, slowest queries. + +**Configuration**: +- Order descending by default (highest values first) +- Limit to top 10-25 entries +- Use conditional formatting to highlight problematic entries +- Show both absolute value and percentage where useful + +**Sizing**: Minimum 4 columns, recommended 6 columns. + +--- + +## Heatmap + +Color-coded density visualization. + +**Use for**: Distribution data — latency distributions, request size distributions. + +**Configuration**: +- Use for metrics with many unique values (percentile distributions) +- Color palette should represent density (lighter = fewer, darker = more) +- Works well paired with a timeseries showing the same metric's percentiles + +**Sizing**: Minimum 4 columns, recommended 6 columns. + +--- + +## Group + +Container widget that organizes other widgets into a collapsible section. + +**Use for**: Organizing dashboard into logical sections. + +**Configuration**: +- Title in Title Case +- Color-code headers for visual scanning (consistent across similar groups) +- Collapse non-critical groups by default (Infrastructure, Dependencies) +- Keep 4-8 widgets per group + +--- + +## Check Status + +Color-coded grid showing service check results. + +**Use for**: Binary health status — service up/down, integration connected/disconnected. + +**Configuration**: +- One check per cell +- Green = OK, Red = Critical, Yellow = Warning, Grey = Unknown +- Group by host or service tag + +**Sizing**: 2-3 columns wide. + +--- + +## Log Stream + +Live-updating list of log entries matching a query. + +**Use for**: Viewing actual log lines — error details, request traces, debug output. + +**Configuration**: +- Filter to relevant log level (error, critical) or service +- Show timestamp, status, message columns at minimum +- Sort by timestamp descending (newest first) +- Link to Log Explorer for deeper investigation + +**Sizing**: Minimum 6 columns, recommended 12 columns (full width). Needs horizontal space for message readability. + +--- + +## Event Stream + +Timeline of Datadog events. + +**Use for**: Deployment events, alert triggers, infrastructure changes. + +**Configuration**: +- Filter by source (deploy, monitor, integration) +- Useful in debugging dashboards to correlate changes with metric shifts + +**Sizing**: Minimum 6 columns, recommended 12 columns. + +--- + +## Note + +Markdown text widget. + +**Use for**: Context that is not metric data — runbook links, team ownership, on-call rotation, usage instructions. + +**Configuration**: +- Keep concise — dashboards are for data, not documentation +- Include runbook links for on-call dashboards +- Use sparingly — if you need many notes, the dashboard structure might be unclear + +**Sizing**: 2-4 columns wide. + +--- + +## Change + +Shows the change in a metric value over a time period. + +**Use for**: Week-over-week or day-over-day comparisons. + +**Configuration**: +- Compare against previous period (1d, 1w) +- Use conditional formatting for increase/decrease +- Pairs well with Query Value for current value + Change for trend + +**Sizing**: 3-4 columns wide. + +--- + +## SLO + +Native Datadog SLO tracking widget. + +**Use for**: SLO compliance, error budget remaining. + +**Configuration**: +- Reference an existing SLO definition +- Show target, current value, and error budget +- Time window matching SLO period (7d, 30d, 90d) + +**Sizing**: 4-6 columns wide. + +--- + +## Widget Title Prefix System + +Every widget title starts with a layer-priority prefix so anyone can immediately tell what layer the metric belongs to and how critical it is. + +### Layers + +| Prefix | Layer | What it covers | +|--------|-------|---------------| +| `I` | **Infrastructure** | Load balancers, databases, networks, DNS, CDN, storage — shared infrastructure that the service depends on but doesn't own | +| `P` | **Platform** | Service-specific platform components from the codebase — gRPC servers, connection pools, cache clients, queue consumers, circuit breakers | +| `D` | **Domain** | Business and domain metrics — checkout success rate, order throughput, payment completion, delivery latency, user sign-ups | + +### Priority Numbers + +The number after the layer letter indicates priority within that layer. `0` is the most critical — the metric you look at first during an outage. Higher numbers are progressively less critical. + +| Priority | Meaning | +|----------|---------| +| `0` | Most critical — look at this first during an outage | +| `1` | Important — check after priority 0 is clear | +| `2+` | Supporting context — useful for investigation | + +### Examples + +```text +Group: "Customer-Facing" + D0: Checkout success rate + D0: Order throughput + I0: Load balancer 5xx + P0: API p99 latency + I1: Database connection pool + P1: gRPC client errors + +Group: "Rate" + P0: Requests per second + P1: By endpoint + +Group: "Errors" + P0: Error rate over time + D1: Failed payment rate + P2: Top errors by endpoint + +Group: "Infrastructure" + I0: CPU usage + I0: Memory usage + I1: Disk I/O + I2: Network errors +``` + +### Classification Guide + +When assigning prefixes, use the domain discovery context: + +- **I (Infrastructure)**: Would this metric exist even if your code didn't? Load balancer, database engine, OS resources, network — things the ops team manages. +- **P (Platform)**: Is this about how your code runs? Connection pools your code configures, gRPC channels your code opens, cache hit rates for caches your code uses — the technical platform layer. +- **D (Domain)**: Does this metric map to a business outcome? Checkout completions, delivery SLA, payment success — things a product manager would understand. + +The priority number comes from the ops review order: what do you look at first when paged at 3am? That's `0`. + +--- + +## General Naming Rules + +**Dashboard title**: Concise, purpose-driven. Example: `Order Service`, not `Order Service Production US-East Dashboard v2`. + +**Widget titles**: Prefix + sentence case, concise, action-oriented. + +- Always start with the layer-priority prefix (`I0:`, `P1:`, `D0:`, etc.) +- Do not repeat the group title after the prefix +- Do not repeat the integration name if it is obvious from context +- Alias all formulas so legends are readable diff --git a/plugins/trogonstack-datadog/skills/datadog-review-dashboard/SKILL.md b/plugins/trogonstack-datadog/skills/datadog-review-dashboard/SKILL.md new file mode 100644 index 0000000..f98b330 --- /dev/null +++ b/plugins/trogonstack-datadog/skills/datadog-review-dashboard/SKILL.md @@ -0,0 +1,230 @@ +--- +name: datadog-review-dashboard +description: >- + Review existing Datadog dashboards for operational readiness. Audits alert + threshold markers, threshold proximity to normal traffic, customer-facing + section completeness, and zero-knowledge readability. Uses pup CLI to fetch + dashboard definitions. Use when auditing dashboards before on-call handoff, + after dashboard changes, or during operational reviews. Do not use for: + (1) designing new dashboards from scratch, (2) monitor/alert rule design, + (3) APM instrumentation or tracing setup, (4) log pipeline configuration. +allowed-tools: AskUserQuestion, Read, Shell +--- + +# Review Datadog Dashboard + +Audit an existing Datadog dashboard against operational readiness principles. The core principles are: graphs should earn their place with alert thresholds, thresholds should sit close to normal traffic, a customer-facing section should exist, and the dashboard should be readable by someone with zero service knowledge. + +These are guiding principles — not a rigid checklist. Apply judgment based on the product and business context. A context-providing metric (like deployment events) may earn its place without a threshold. A service with unusual traffic patterns may need different proximity rules. + +## Interview Phase + +**Skip interview if ALL of these are already specified:** +- Dashboard ID or URL +- Service name or team context + +**Always interview if**: No dashboard ID is provided or multiple dashboards may be relevant. + +### Questions + +1. **Dashboard** — "Which dashboard should I review? Provide a dashboard ID, URL, or service name to search for." + - Impact: Determines which dashboard definition to fetch + +2. **Business Context** — "Can you tell me what this service does for customers? Are there codebases or docs I can read to understand the product?" + - Impact: Understanding the domain lets the review focus on whether the right metrics are being tracked, not just whether generic rules are followed + +3. **Focus** — "Is there anything specific you want me to focus on? (A) Full review, (B) Alert thresholds only, (C) Customer-facing section, (D) Layout and readability" + - Impact: Determines review scope — default to full review if unspecified + +--- + +## Workflow + +### 1. Fetch Dashboard Definition + +```bash +# If given a service name, search for matching dashboards +pup dashboards list --filter="" + +# If given a URL, extract the dashboard ID from the path (e.g., /dashboard/abc-def-ghi/...) + +# Get the full dashboard definition +pup dashboards get + +# Get the dashboard URL for reference +pup dashboards url +``` + +Parse the response to build an inventory of all widgets, groups, and their configurations. + +### 2. Build Widget Inventory + +Catalog every widget in the dashboard: + +| Widget Title | Prefix | Type | Group | Has Alert Threshold | Threshold Value | Notes | +|-------------|--------|------|-------|--------------------:|----------------|-------| +| ... | I0/P1/D0/— | ... | ... | ... | ... | ... | + +Check that every widget title uses the layer-priority prefix system: +- `I0-N:` for infrastructure (load balancers, databases, networks) +- `P0-N:` for platform (service-specific components from the codebase) +- `D0-N:` for domain (business metrics) +- The number indicates priority within the layer (`0` = most critical) + +Focus on timeseries and query value widgets — these are the primary candidates for alert threshold markers. + +### 3. Audit Alert Thresholds + +**Principle**: Timeseries graphs should generally have an alert threshold (red line). If a metric doesn't warrant an alert, question whether it belongs — but use judgment. Some metrics provide valuable context (deployment markers, dependency traffic patterns) without needing a threshold. + +For each timeseries widget, check: +- Does it have a marker/threshold line configured? +- Is the marker colored red for visibility? +- Does the threshold correspond to an actual monitor/alert? + +**Findings format**: + +```markdown +#### Alert Threshold Audit + +| Widget | Group | Status | Finding | +|--------|-------|--------|---------| +| Requests/s | Rate | MISSING | No threshold marker — add alert line or remove widget | +| Error rate | Errors | OK | Red line at 5% | +| CPU usage | Infra | MISSING | No threshold — is this metric alertable? | +``` + +### 4. Audit Threshold Proximity + +**Principle**: Alert thresholds must be close to normal traffic. Large gaps between normal values and the alert line create blind spots where anomalies go unnoticed. + +For each widget with a threshold: +- What is the typical (normal) value range? +- Where is the threshold set? +- Is there excessive whitespace between the normal line and the alert line? +- Is the Y-axis auto-scaled or explicitly set? Auto-scaled Y-axes compress normal traffic into a flat band when the threshold is far above normal — the Y-axis max should be set to slightly above the alert threshold + +**Bad example**: Normal CPU is 20%, alert threshold at 95% — the graph is mostly empty space and a slow climb from 20% to 80% looks flat. + +**Good example**: Normal CPU is 20%, alert threshold at 45% — anomalies visually stand out immediately. + +**Findings format**: + +```markdown +#### Threshold Proximity Audit + +| Widget | Normal Range | Threshold | Gap | Y-Axis | Status | +|--------|-------------|-----------|-----|--------|--------| +| CPU usage | ~20% | 95% | 75% | auto | TOO FAR — lower to 40-50%, set Y-max to 55% | +| Error rate | ~0.1% | 5% | ~5% | auto | OK gap — but set Y-max to 6% | +| p99 latency | ~50ms | 500ms | 10x | auto | TOO FAR — lower to 100-150ms, set Y-max to 175ms | +``` + +### 5. Audit Customer-Facing Section + +**Principle**: A dedicated "Customer-Facing" group should exist at the top of the dashboard with 5-8 key metrics for immediate outage identification. The specific metrics should reflect the product's business — not just generic traffic and error rates. + +Check: +- Does a "Customer-Facing" group exist? +- Is it the first group on the dashboard? +- Does it contain 5-8 metrics covering: traffic volume, API latency, error rates, key business transactions, and database health? +- Can someone determine "are customers affected?" within 5 seconds of opening the dashboard? + +**Findings format**: + +```markdown +#### Customer-Facing Section Audit + +**Status**: MISSING / INCOMPLETE / OK + +**Current state**: [Description of what exists] + +**Recommended metrics** (if missing or incomplete): +1. Total request rate (are we receiving traffic?) +2. Customer-facing error rate (are requests failing?) +3. API p99 latency (are responses slow?) +4. Key transaction success rate (are critical flows working?) +5. Database connection pool usage (is the data layer healthy?) +6. Queue depth or processing lag (is async work backing up?) +``` + +### 6. Apply Zero-Knowledge Viewer Test + +**Principle**: Someone with zero knowledge of the service should be able to spot problems by looking for red indicators. + +Evaluate: +- Can you identify a problem in under 10 seconds without reading widget titles? +- Are thresholds visible as red lines on every graph? +- Is conditional formatting applied to query value widgets (green/yellow/red)? +- Are group names self-explanatory? +- Is there a note widget with runbook links or team ownership? + +**Findings format**: + +```markdown +#### Zero-Knowledge Readability Audit + +| Check | Status | Finding | +|-------|--------|---------| +| Problems visible in <10s | FAIL | No red lines on 8 of 12 graphs | +| Conditional formatting on QV widgets | PARTIAL | 2 of 4 QV widgets have thresholds | +| Group names self-explanatory | OK | All groups use clear names | +| Runbook/ownership note | MISSING | No note widget with team info | +``` + +### 7. Generate Review Report + +Compile all findings into a structured report: + +```markdown +# Dashboard Review: [Dashboard Title] + +**Dashboard ID**: [id] +**URL**: [url] +**Review date**: [date] + +## Summary + +[2-3 sentence summary: overall health of the dashboard, critical issues count] + +## Critical Issues + +[List issues that must be fixed before the dashboard is production-ready] + +## Alert Threshold Audit +[From step 3] + +## Threshold Proximity Audit +[From step 4] + +## Customer-Facing Section Audit +[From step 5] + +## Zero-Knowledge Readability Audit +[From step 6] + +## Recommended Actions + +### Must Fix +1. [Action item with specific widget and group reference] + +### Should Fix +1. [Action item] + +### Nice to Have +1. [Action item] +``` + +--- + +## Quality Checklist + +- [ ] Every widget title uses the layer-priority prefix (`I0:`, `P1:`, `D0:`, etc.) +- [ ] Every timeseries widget audited for alert threshold markers +- [ ] Threshold proximity checked (no large gaps between normal values and alert lines) +- [ ] Customer-Facing group exists with 5-8 key metrics at the top +- [ ] Zero-knowledge viewer test applied (red indicators visible without context) +- [ ] Query Value widgets checked for conditional formatting (green/yellow/red) +- [ ] All findings include specific widget names and group references +- [ ] Recommended actions categorized by priority (must/should/nice-to-have) +- [ ] Dashboard URL included in report for easy reference