diff --git a/go.mod b/go.mod index f92afd3a5..87ea4489c 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/wailsapp/wails/v2 v2.12.0 golang.org/x/net v0.56.0 golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.21.0 golang.org/x/sys v0.46.0 golang.org/x/term v0.44.0 google.golang.org/grpc v1.81.0 @@ -158,7 +159,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect - golang.org/x/sync v0.21.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 328bdc321..d0911f271 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -2,7 +2,6 @@ package server import ( "context" - "encoding/json" "fmt" "log" "net/http" @@ -1442,11 +1441,6 @@ func (s *Server) countWarningEvents(cache *k8s.ResourceCache, namespace string) return count } -// metricsServerTimeout bounds the metrics-server query so a slow or unreachable -// metrics endpoint can't consume the dashboard's whole request budget (the -// handler runs this in a goroutine and blocks on it in wg.Wait()). -const metricsServerTimeout = 8 * time.Second - func (s *Server) getDashboardMetrics(ctx context.Context, allowedNamespaces []string) *DashboardMetrics { // Node capacity and pod requests come from the cache — they don't need // metrics-server. Live usage does. Compute the cached values first and @@ -1465,84 +1459,21 @@ func (s *Server) getDashboardMetrics(ctx context.Context, allowedNamespaces []st return nil } - // Sum capacity across all nodes (cached) - var cpuCapacityMillis int64 - var memCapacityBytes int64 - for _, n := range nodes { - cpuCapacityMillis += n.Status.Capacity.Cpu().MilliValue() - memCapacityBytes += n.Status.Capacity.Memory().Value() - } - - // Sum requests across pods the user can see (cached). For namespace- - // restricted users this scopes to allowedNamespaces — without it, the - // dashboard would expose aggregate pod-resource totals from namespaces they - // have no read access to. - var cpuRequestsMillis int64 - var memRequestsBytes int64 - var metricPods []*corev1.Pod - if podLister := cache.Pods(); podLister != nil { - if allowedNamespaces == nil { - metricPods, _ = podLister.List(labels.Everything()) - } else { - for _, ns := range allowedNamespaces { - items, _ := podLister.Pods(ns).List(labels.Everything()) - metricPods = append(metricPods, items...) - } - } - } - for _, pod := range metricPods { - // Skip completed/failed pods - if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { - continue - } - for _, container := range pod.Spec.Containers { - if container.Resources.Requests != nil { - if cpu, ok := container.Resources.Requests[corev1.ResourceCPU]; ok { - cpuRequestsMillis += cpu.MilliValue() - } - if mem, ok := container.Resources.Requests[corev1.ResourceMemory]; ok { - memRequestsBytes += mem.Value() - } - } - } - } + // Capacity + scheduled-pod requests via the shared informer-derived + // computation (node_metrics.go). Namespace scoping keeps restricted users + // from seeing aggregate totals of namespaces they can't read. + cr := computeCapacityRequests(nodes, listPodsScoped(cache.Pods(), allowedNamespaces)) + cpuCapacityMillis := cr.cpuCapMillis + memCapacityBytes := cr.memCapBytes + cpuRequestsMillis := cr.cpuReqMillis + memRequestsBytes := cr.memReqBytes // Live usage from metrics-server — best-effort. Query via raw REST to avoid // adding k8s.io/metrics dependency; metrics-server forwards impersonation // headers, so a user without metrics.k8s.io/nodes access gets a 403. A // missing, forbidden, or slow metrics-server leaves usageAvailable false // rather than discarding the capacity/request data computed above. - var cpuUsageMillis int64 - var memUsageBytes int64 - usageAvailable := false - if client := k8s.ClientFromContext(ctx); client != nil { - mctx, cancel := context.WithTimeout(ctx, metricsServerTimeout) - defer cancel() - data, err := client.CoreV1().RESTClient().Get(). - AbsPath("/apis/metrics.k8s.io/v1beta1/nodes"). - DoRaw(mctx) - if err != nil { - log.Printf("[dashboard] node metrics unavailable (showing requests/capacity only): %v", err) - } else { - var nodeMetricsList struct { - Items []struct { - Usage struct { - CPU string `json:"cpu"` - Memory string `json:"memory"` - } `json:"usage"` - } `json:"items"` - } - if err := json.Unmarshal(data, &nodeMetricsList); err != nil { - log.Printf("[dashboard] failed to parse node metrics: %v", err) - } else if len(nodeMetricsList.Items) > 0 { - for _, item := range nodeMetricsList.Items { - cpuUsageMillis += parseCPUToMillis(item.Usage.CPU) - memUsageBytes += parseMemoryToBytes(item.Usage.Memory) - } - usageAvailable = true - } - } - } + cpuUsageMillis, memUsageBytes, usageAvailable := s.fetchNodeUsage(ctx) metrics := &DashboardMetrics{UsageAvailable: usageAvailable} if cpuCapacityMillis > 0 { @@ -1571,12 +1502,6 @@ func (s *Server) getDashboardMetrics(ctx context.Context, allowedNamespaces []st return metrics } -// parseCPUToMillis delegates to k8s.ParseCPUToMillis. -func parseCPUToMillis(s string) int64 { return k8s.ParseCPUToMillis(s) } - -// parseMemoryToBytes delegates to k8s.ParseMemoryToBytes. -func parseMemoryToBytes(s string) int64 { return k8s.ParseMemoryToBytes(s) } - // Helper functions // collectClusterScopedCRDCounts returns counts of cluster-scoped CRD instances diff --git a/internal/server/namespace_scope.go b/internal/server/namespace_scope.go index 105686363..5f4c0cfcb 100644 --- a/internal/server/namespace_scope.go +++ b/internal/server/namespace_scope.go @@ -121,6 +121,7 @@ func (s *Server) finalizePostContextSwitch() { k8s.InvalidateUserCapabilitiesCache() clearPackagesCache() clearApplicationsCache() + s.vitalsMetrics.clear() s.clearAllNamespacePreferences() } diff --git a/internal/server/node_metrics.go b/internal/server/node_metrics.go new file mode 100644 index 000000000..574993927 --- /dev/null +++ b/internal/server/node_metrics.go @@ -0,0 +1,124 @@ +package server + +import ( + "context" + "encoding/json" + "log" + "time" + + "github.com/skyhook-io/radar/internal/k8s" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + v1listers "k8s.io/client-go/listers/core/v1" +) + +// Shared metrics-server plumbing — consumed by both /api/dashboard and +// /api/vitals. Lives in neither handler file on purpose: the probe is a +// cluster-metrics concern, not a feature of either endpoint. + +// metricsServerTimeout bounds the metrics-server query so a slow or unreachable +// metrics endpoint can't consume the dashboard's whole request budget (the +// handler runs this in a goroutine and blocks on it in wg.Wait()). +const metricsServerTimeout = 8 * time.Second + +// fetchNodeUsage probes metrics-server for live node usage (impersonated — +// a caller without metrics.k8s.io access gets ok=false, not someone else's +// numbers). Extracted so /api/vitals can memoize JUST this probe while +// computing capacity/requests fresh from the informer cache on every call. +func (s *Server) fetchNodeUsage(ctx context.Context) (cpuMillis, memBytes int64, ok bool) { + client := k8s.ClientFromContext(ctx) + if client == nil { + return 0, 0, false + } + mctx, cancel := context.WithTimeout(ctx, metricsServerTimeout) + defer cancel() + data, err := client.CoreV1().RESTClient().Get(). + AbsPath("/apis/metrics.k8s.io/v1beta1/nodes"). + DoRaw(mctx) + if err != nil { + log.Printf("[dashboard] node metrics unavailable (showing requests/capacity only): %v", err) + return 0, 0, false + } + var nodeMetricsList struct { + Items []struct { + Usage struct { + CPU string `json:"cpu"` + Memory string `json:"memory"` + } `json:"usage"` + } `json:"items"` + } + if err := json.Unmarshal(data, &nodeMetricsList); err != nil { + log.Printf("[dashboard] failed to parse node metrics: %v", err) + return 0, 0, false + } + if len(nodeMetricsList.Items) == 0 { + return 0, 0, false + } + for _, item := range nodeMetricsList.Items { + cpuMillis += parseCPUToMillis(item.Usage.CPU) + memBytes += parseMemoryToBytes(item.Usage.Memory) + } + return cpuMillis, memBytes, true +} + +// parseCPUToMillis delegates to k8s.ParseCPUToMillis. +func parseCPUToMillis(s string) int64 { return k8s.ParseCPUToMillis(s) } + +// parseMemoryToBytes delegates to k8s.ParseMemoryToBytes. +func parseMemoryToBytes(s string) int64 { return k8s.ParseMemoryToBytes(s) } + +// listPodsScoped lists pods either cluster-wide (namespaces nil) or across +// the caller's allowed namespaces — the scoping shape every metrics/vitals +// consumer shares. +func listPodsScoped(podLister v1listers.PodLister, namespaces []string) []*corev1.Pod { + if podLister == nil { + return nil + } + // Sentinel contract (parseNamespacesForUser): nil = all namespaces; + // non-nil EMPTY = no namespace access — zero pods, never cluster-wide. + if namespaces == nil { + pods, _ := podLister.List(labels.Everything()) + return pods + } + var pods []*corev1.Pod + for _, ns := range namespaces { + items, _ := podLister.Pods(ns).List(labels.Everything()) + pods = append(pods, items...) + } + return pods +} + +// capacityRequests carries the informer-derived halves of the capacity +// picture: node capacity plus scheduled-pod requests (completed pods +// excluded). Usage is the metrics-server probe's job (fetchNodeUsage). +type capacityRequests struct { + cpuCapMillis int64 + memCapBytes int64 + cpuReqMillis int64 + memReqBytes int64 +} + +func computeCapacityRequests(nodes []*corev1.Node, pods []*corev1.Pod) capacityRequests { + var cr capacityRequests + for _, n := range nodes { + cr.cpuCapMillis += n.Status.Capacity.Cpu().MilliValue() + cr.memCapBytes += n.Status.Capacity.Memory().Value() + } + for _, pod := range pods { + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + for _, c := range pod.Spec.Containers { + if c.Resources.Requests == nil { + continue + } + if cpu, ok := c.Resources.Requests[corev1.ResourceCPU]; ok { + cr.cpuReqMillis += cpu.MilliValue() + } + if mem, ok := c.Resources.Requests[corev1.ResourceMemory]; ok { + cr.memReqBytes += mem.Value() + } + } + } + return cr +} diff --git a/internal/server/node_metrics_test.go b/internal/server/node_metrics_test.go new file mode 100644 index 000000000..d065c0cef --- /dev/null +++ b/internal/server/node_metrics_test.go @@ -0,0 +1,70 @@ +package server + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +func TestListPodsScoped_SentinelContract(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "a", Namespace: "ns1"}}, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "ns2"}}, + ) + f := informers.NewSharedInformerFactory(client, 0) + informer := f.Core().V1().Pods() + _ = informer.Informer() + stop := make(chan struct{}) + defer close(stop) + f.Start(stop) + f.WaitForCacheSync(stop) + lister := informer.Lister() + + if got := listPodsScoped(lister, nil); len(got) != 2 { + t.Errorf("nil scope = %d pods, want 2 (cluster-wide)", len(got)) + } + if got := listPodsScoped(lister, []string{}); len(got) != 0 { + t.Errorf("empty scope = %d pods, want 0 (no namespace access)", len(got)) + } + if got := listPodsScoped(lister, []string{"ns1"}); len(got) != 1 { + t.Errorf("ns1 scope = %d pods, want 1", len(got)) + } + if got := listPodsScoped(nil, nil); got != nil { + t.Errorf("nil lister must yield nil") + } +} + +func TestComputeCapacityRequests_ExcludesCompletedPods(t *testing.T) { + nodes := []*corev1.Node{{Status: corev1.NodeStatus{Capacity: corev1.ResourceList{ + corev1.ResourceCPU: mustQty(t, "4"), corev1.ResourceMemory: mustQty(t, "8Gi"), + }}}} + req := corev1.ResourceList{corev1.ResourceCPU: mustQty(t, "500m"), corev1.ResourceMemory: mustQty(t, "1Gi")} + running := &corev1.Pod{ + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: req}}}}, + } + done := running.DeepCopy() + done.Status.Phase = corev1.PodSucceeded + + cr := computeCapacityRequests(nodes, []*corev1.Pod{running, done}) + if cr.cpuCapMillis != 4000 { + t.Errorf("cpuCap = %d, want 4000", cr.cpuCapMillis) + } + if cr.cpuReqMillis != 500 { + t.Errorf("cpuReq = %d, want 500 (completed pod excluded)", cr.cpuReqMillis) + } +} + +func mustQty(t *testing.T, s string) resource.Quantity { + t.Helper() + q, err := resource.ParseQuantity(s) + if err != nil { + t.Fatalf("parse %q: %v", s, err) + } + return q +} diff --git a/internal/server/server.go b/internal/server/server.go index 2aaeae88f..cdfce45e4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -59,6 +59,7 @@ import ( type Server struct { router *chi.Mux broadcaster *SSEBroadcaster + vitalsMetrics vitalsMetricsMemo port int devMode bool staticFS fs.FS @@ -291,6 +292,7 @@ func (s *Server) setupRoutes() { r.Get("/auth/me", s.handleAuthMe) r.Get("/version-check", s.handleVersionCheck) r.Get("/dashboard", s.handleDashboard) + r.Get("/vitals", s.handleVitals) r.Get("/dashboard/crds", s.handleDashboardCRDs) r.Get("/dashboard/helm", s.handleDashboardHelm) r.Get("/cluster-info", s.handleClusterInfo) diff --git a/internal/server/vitals.go b/internal/server/vitals.go new file mode 100644 index 000000000..01f5486f6 --- /dev/null +++ b/internal/server/vitals.go @@ -0,0 +1,322 @@ +package server + +import ( + "net/http" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + + "golang.org/x/sync/singleflight" + + "github.com/skyhook-io/radar/internal/auth" + "github.com/skyhook-io/radar/internal/k8s" + "github.com/skyhook-io/radar/pkg/health" + "github.com/skyhook-io/radar/pkg/k8score" +) + +// GET /api/vitals — a cluster's vital signs: a compact scale + capacity +// snapshot (node/pod phase counts, CPU/memory capacity vs requests vs live +// usage) meant to be polled cheaply, distinct from /api/dashboard's full +// home payload. Purpose-built for +// Radar Cloud's fleet fan-out (the hub previously projected this corner out +// of the full /api/dashboard payload and discarded the rest); equally usable +// by any consumer that wants counts + capacity without the home-page +// kitchen sink. +// +// Completeness is TYPED, never stringly: the sentinel fields carry k8score +// group-qualified Kind values ("Pod", "Node") and an explicit derived `complete`, +// so consumers don't re-derive authoritativeness from zeros. + +type VitalsResponse struct { + Nodes NodeCount `json:"nodes"` + Pods VitalsPodCount `json:"pods"` + CPU *MetricSummary `json:"cpu,omitempty"` + Memory *MetricSummary `json:"memory,omitempty"` + // True only when live usage came from metrics-server; requests/capacity + // are informer-derived and present regardless (same semantics as the + // dashboard's flag). + MetricsServerAvailable bool `json:"metricsServerAvailable"` + Completeness VitalsCompleteness `json:"completeness"` +} + +// VitalsPodCount is ResourceCount without omitempty — fleet consumers want +// stable JSON shape, not fields that vanish at zero. +type VitalsPodCount struct { + Total int `json:"total"` + Running int `json:"running"` + Pending int `json:"pending"` + Failed int `json:"failed"` + Succeeded int `json:"succeeded"` +} + +// VitalsCompleteness is a PROVISIONAL contract. `complete` and +// `accessRestricted` are stable primitives (a future shared completeness +// envelope keeps both meanings unchanged); `pending`/`restricted` are +// coarse per-kind hints that the shared envelope will subsume into a +// {kind, reason} model (rbac_denied vs unavailable vs syncing). No consumer +// reads the detail lists today — only `complete` — so that later change is +// non-breaking. Kind values use group-qualified Kind casing ("Pod") to +// match /api/resource-counts, the closest existing shape. +type VitalsCompleteness struct { + // Caller's identity has no namespace access at all — counts are + // meaningless, not zero. + AccessRestricted bool `json:"accessRestricted"` + // Vitals-relevant kinds ("Pod", "Node") whose critical informers are + // still syncing. Other kinds' warm-up doesn't gate vitals. + Pending []string `json:"pending,omitempty"` + // Vitals-relevant kinds the current identity (or the SA cache) cannot + // list. Includes "Node" when node RBAC is denied — the dashboard has no + // such sentinel and consumers had to infer it from zero totals. + Restricted []string `json:"restricted,omitempty"` + // Derived: !accessRestricted && no pending && no restricted. Metrics + // availability is deliberately NOT part of completeness — it's a + // capability, reported separately. + Complete bool `json:"complete"` +} + +// The live metrics-server probe (fetchNodeUsage) can block up to 8s. This +// 15s memo absorbs BURSTS — retries, concurrent consumers, page refreshes, +// and repeated 8s waits when metrics-server is slow/absent — rather than +// materially cutting steady-state load at the 30–60s fleet poll cadence. +// Keyed by contextName+username (the probe impersonates the caller, so +// results must never cross identities or clusters) and cleared on context +// switch (finalizePostContextSwitch). Only the live-usage probe is memoized; +// capacity/requests are recomputed fresh every call. +type vitalsMetricsMemo struct { + mu sync.Mutex + entries map[string]vitalsMetricsEntry + // Collapses concurrent same-key misses into a single probe — without it + // a burst of /api/vitals requests (the case this memo exists to absorb) + // would each run the 8s metrics-server probe and last-writer-wins could + // leave a stale metricsServerAvailable for the TTL. + group singleflight.Group +} + +type vitalsMetricsEntry struct { + cpuMillis int64 + memBytes int64 + ok bool + at time.Time +} + +const vitalsMetricsTTL = 15 * time.Second + +func (m *vitalsMetricsMemo) get(key string) (vitalsMetricsEntry, bool) { + m.mu.Lock() + defer m.mu.Unlock() + e, ok := m.entries[key] + if !ok || time.Since(e.at) > vitalsMetricsTTL { + return vitalsMetricsEntry{}, false + } + return e, true +} + +func (m *vitalsMetricsMemo) put(key string, e vitalsMetricsEntry) { + m.mu.Lock() + defer m.mu.Unlock() + if m.entries == nil { + m.entries = map[string]vitalsMetricsEntry{} + } + // Bound the map: keys are per-user; a runaway cardinality here would + // indicate a bug, not a workload. Reset rather than grow. + if len(m.entries) > 256 { + m.entries = map[string]vitalsMetricsEntry{} + } + e.at = time.Now() + m.entries[key] = e +} + +// loadOrFetch returns the memoized usage for key, running fetch at most once +// per key across concurrent callers. Cached entries within the TTL skip the +// probe entirely. +func (m *vitalsMetricsMemo) loadOrFetch(key string, fetch func() vitalsMetricsEntry) vitalsMetricsEntry { + if e, ok := m.get(key); ok { + return e + } + v, _, _ := m.group.Do(key, func() (any, error) { + // Re-check under the flight: a prior leader may have just filled it. + if e, ok := m.get(key); ok { + return e, nil + } + e := fetch() + m.put(key, e) + return e, nil + }) + return v.(vitalsMetricsEntry) +} + +func (m *vitalsMetricsMemo) clear() { + m.mu.Lock() + defer m.mu.Unlock() + m.entries = nil +} + +func (s *Server) handleVitals(w http.ResponseWriter, r *http.Request) { + // Connection guards FIRST, matching /api/dashboard: while disconnected, + // per-user namespace discovery fails closed, and answering that state + // with accessRestricted would disguise an infrastructure problem as an + // RBAC denial. 503 is the honest signal until the cluster is reachable. + if !s.requireConnected(w) { + return + } + cache := k8s.GetResourceCache() + if cache == nil { + s.writeError(w, http.StatusServiceUnavailable, "cluster cache not ready") + return + } + namespaces := s.parseNamespacesForUser(r) + if noNamespaceAccess(namespaces) { + s.writeJSON(w, VitalsResponse{ + Completeness: VitalsCompleteness{AccessRestricted: true}, + }) + return + } + + resp := VitalsResponse{} + var restricted []string + + // Pods — namespace VISIBILITY (parseNamespacesForUser) is not pod-READ + // authority: discovery admits a namespace on list-pods OR + // list-deployments, so a deployments-only user must not learn pod + // counts, phases, or request totals from the SA cache. Authorize the + // pods read explicitly per scope. + podNamespaces := namespaces + podsReadable := true + podsPartial := false + if namespaces == nil { + podsReadable = s.canRead(r, "", "pods", "", "list") + } else { + podNamespaces = s.filterNamespacesByCanRead(r, "", "pods", "list", namespaces) + podsReadable = len(podNamespaces) > 0 + // Some visible namespaces dropped at the pod-read gate: whatever we + // count below is a subset — flag it so consumers don't treat the + // numbers as the caller's whole world. + podsPartial = len(podNamespaces) < len(namespaces) + } + // The SA cache itself can be namespace-scoped (startup RBAC fallback); + // a cluster-wide caller reading a scoped cache is partial too. + if perm := k8s.GetCachedPermissionResult(); perm != nil { + if scope, ok := perm.Scopes[k8score.Pods]; ok && scope.Enabled && scope.Namespace != "" { + podsPartial = true + } + } + var scopedPods []*corev1.Pod + if podsReadable { + scopedPods = listPodsScoped(cache.Pods(), podNamespaces) + } + if podsReadable && cache.Pods() != nil { + pods := scopedPods + resp.Pods.Total = len(pods) + for _, pod := range pods { + switch pod.Status.Phase { + case corev1.PodRunning: + resp.Pods.Running++ + case corev1.PodPending: + resp.Pods.Pending++ + case corev1.PodFailed: + resp.Pods.Failed++ + case corev1.PodSucceeded: + resp.Pods.Succeeded++ + } + } + } else { + restricted = append(restricted, "Pod") + } + if podsReadable && podsPartial { + restricted = append(restricted, "Pod") + } + + // Nodes — cluster-scoped; gate on the caller's own RBAC like the + // dashboard does, then on the SA cache. + canReadNodes := s.canRead(r, "", "nodes", "", "list") + if canReadNodes && cache.Nodes() != nil { + nodes, _ := cache.Nodes().List(labels.Everything()) + resp.Nodes.Total = len(nodes) + for _, n := range nodes { + h := health.Node(n) + if h.Ready { + if h.Unschedulable { + resp.Nodes.Cordoned++ + } else { + resp.Nodes.Ready++ + } + } else { + resp.Nodes.NotReady++ + } + } + } else { + restricted = append(restricted, "Node") + } + + // Pending critical informers, filtered to the two vitals-relevant kinds. + // PendingPromotedKinds already emits group-qualified Kind names ("Pod", + // "Node") — the same vocabulary restricted uses above. + for _, k := range cache.PendingPromotedKinds() { + if k == "Pod" || k == "Node" { + resp.Completeness.Pending = append(resp.Completeness.Pending, k) + } + } + + // Capacity + usage — node-derived data, so it sits behind the same node + // RBAC gate as the counts (the dashboard computes metrics inside its + // canReadNodes branch; a caller denied node list must not learn cluster + // capacity here either). Requests/capacity are informer-derived; live + // usage is the metrics-server probe, memoized (see vitalsMetricsMemo). + if canReadNodes && cache.Nodes() != nil { + // Capacity + requests are informer-derived — computed FRESH on every + // call (memoizing them would freeze counts for the TTL). Only the + // live metrics-server probe is memoized, keyed by user (the probe + // impersonates the caller; results must never cross identities). + // Context switches clear the memo wholesale (finalizePostContextSwitch). + nodes, _ := cache.Nodes().List(labels.Everything()) + cr := computeCapacityRequests(nodes, scopedPods) + cpuCap, memCapBytes := cr.cpuCapMillis, cr.memCapBytes + cpuReq, memReqBytes := cr.cpuReqMillis, cr.memReqBytes + + username := "" + if user := auth.UserFromContext(r.Context()); user != nil { + username = user.Username + } + // Key carries the kube context too: finalizePostContextSwitch clears + // the memo, but a request racing the switch window could otherwise + // resurrect the previous cluster's usage for this user (same class + // of race PermissionCache stamps against). + memoKey := k8s.GetContextName() + "\x00" + username + usage := s.vitalsMetrics.loadOrFetch(memoKey, func() vitalsMetricsEntry { + cpu, mem, avail := s.fetchNodeUsage(r.Context()) + return vitalsMetricsEntry{cpuMillis: cpu, memBytes: mem, ok: avail} + }) + + if cpuCap > 0 { + resp.CPU = &MetricSummary{ + UsageMillis: usage.cpuMillis, + RequestsMillis: cpuReq, + CapacityMillis: cpuCap, + UsagePercent: int(usage.cpuMillis * 100 / cpuCap), + RequestPercent: int(cpuReq * 100 / cpuCap), + } + } + if memCapBytes > 0 { + memCapMiB := memCapBytes / (1024 * 1024) + memUseMiB := usage.memBytes / (1024 * 1024) + memReqMiB := memReqBytes / (1024 * 1024) + m := &MetricSummary{ + UsageMillis: memUseMiB, + RequestsMillis: memReqMiB, + CapacityMillis: memCapMiB, + UsagePercent: int(memUseMiB * 100 / memCapMiB), + RequestPercent: int(memReqMiB * 100 / memCapMiB), + } + resp.Memory = m + } + resp.MetricsServerAvailable = usage.ok + } + + resp.Completeness.Restricted = restricted + resp.Completeness.Complete = !resp.Completeness.AccessRestricted && + len(resp.Completeness.Pending) == 0 && len(restricted) == 0 + + s.writeJSON(w, resp) +} diff --git a/internal/server/vitals_test.go b/internal/server/vitals_test.go new file mode 100644 index 000000000..a83a01ff5 --- /dev/null +++ b/internal/server/vitals_test.go @@ -0,0 +1,175 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + pkgauth "github.com/skyhook-io/radar/pkg/auth" +) + +func TestVitals_HappyPath(t *testing.T) { + resp, err := http.Get(testServer.URL + "/api/vitals") + if err != nil { + t.Fatalf("GET /api/vitals: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var got VitalsResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + // The smoke fixture seeds one Running pod and no nodes. + if got.Pods.Total != 1 || got.Pods.Running != 1 { + t.Errorf("pods = %+v, want total 1 running 1", got.Pods) + } + if got.Nodes.Total != 0 { + t.Errorf("nodes.total = %d, want 0 (fixture seeds none)", got.Nodes.Total) + } + if !got.Completeness.Complete { + t.Errorf("completeness = %+v, want complete", got.Completeness) + } + if got.MetricsServerAvailable { + t.Errorf("metricsServerAvailable = true, want false (fake client has no metrics API)") + } +} + +func TestVitals_NodeRBACDeniedSurfacesRestricted(t *testing.T) { + if testServerSrv.permCache == nil { + testServerSrv.permCache = pkgauth.NewPermissionCache() + } + const username = "vitals-node-denied" + perms := &pkgauth.UserPermissions{} + perms.SetCanI("list", "", "nodes", "", false) + testServerSrv.permCache.Set(username, perms) + + req := httptest.NewRequest(http.MethodGet, "/api/vitals", nil) + req = req.WithContext(pkgauth.ContextWithUser(req.Context(), &pkgauth.User{Username: username})) + rec := httptest.NewRecorder() + testServerSrv.Handler().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200. body: %s", rec.Code, rec.Body.String()) + } + var got VitalsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + foundNodes := false + for _, k := range got.Completeness.Restricted { + if k == "Node" { + foundNodes = true + } + } + if !foundNodes { + t.Errorf("restricted = %v, want to contain \"nodes\" (RBAC-denied node list)", got.Completeness.Restricted) + } + if got.Completeness.Complete { + t.Errorf("complete = true despite restricted nodes") + } + if got.Nodes.Total != 0 { + t.Errorf("nodes.total = %d, want 0 when restricted", got.Nodes.Total) + } + if got.CPU != nil || got.Memory != nil || got.MetricsServerAvailable { + t.Errorf("metrics leaked past node RBAC denial: cpu=%v mem=%v msa=%v", got.CPU, got.Memory, got.MetricsServerAvailable) + } +} + +func TestVitals_DeploymentsOnlyUserGetsNoPodData(t *testing.T) { + if testServerSrv.permCache == nil { + testServerSrv.permCache = pkgauth.NewPermissionCache() + } + const username = "vitals-deploys-only" + perms := &pkgauth.UserPermissions{} + // Namespace visibility can come from deployments; pods are denied. + perms.SetCanI("list", "", "pods", "", false) + perms.SetCanI("list", "apps", "deployments", "", true) + perms.SetCanI("list", "", "nodes", "", true) + testServerSrv.permCache.Set(username, perms) + + req := httptest.NewRequest(http.MethodGet, "/api/vitals", nil) + req = req.WithContext(pkgauth.ContextWithUser(req.Context(), &pkgauth.User{Username: username})) + rec := httptest.NewRecorder() + testServerSrv.Handler().ServeHTTP(rec, req) + + var got VitalsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("unmarshal: %v (body %s)", err, rec.Body.String()) + } + if got.Pods.Total != 0 { + t.Errorf("pods leaked to a deployments-only user: %+v", got.Pods) + } + foundPods := false + for _, k := range got.Completeness.Restricted { + if k == "Pod" { + foundPods = true + } + } + if !foundPods { + t.Errorf("restricted = %v, want to contain \"pods\"", got.Completeness.Restricted) + } + if got.CPU != nil && got.CPU.RequestsMillis != 0 { + t.Errorf("pod request totals leaked: %+v", got.CPU) + } +} + +func TestVitals_MixedNamespacePodAccessMarksPartial(t *testing.T) { + if testServerSrv.permCache == nil { + testServerSrv.permCache = pkgauth.NewPermissionCache() + } + const username = "vitals-mixed-ns" + perms := &pkgauth.UserPermissions{} + // Visible in two namespaces (via deployments), pod-readable in one. + perms.SetCanI("list", "", "pods", "default", true) + perms.SetCanI("list", "", "pods", "kube-system", false) + perms.SetCanI("list", "apps", "deployments", "default", true) + perms.SetCanI("list", "apps", "deployments", "kube-system", true) + perms.SetCanI("list", "", "nodes", "", true) + perms.AllowedNamespaces = []string{"default", "kube-system"} + testServerSrv.permCache.Set(username, perms) + + req := httptest.NewRequest(http.MethodGet, "/api/vitals", nil) + req = req.WithContext(pkgauth.ContextWithUser(req.Context(), &pkgauth.User{Username: username})) + rec := httptest.NewRecorder() + testServerSrv.Handler().ServeHTTP(rec, req) + + var got VitalsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("unmarshal: %v (body %s)", err, rec.Body.String()) + } + foundPods := false + for _, k := range got.Completeness.Restricted { + if k == "Pod" { + foundPods = true + } + } + if !foundPods || got.Completeness.Complete { + t.Errorf("partial pod scope not flagged: completeness=%+v", got.Completeness) + } +} + +func TestVitalsMetricsMemo_SingleFlightCollapsesConcurrentMisses(t *testing.T) { + var m vitalsMetricsMemo + var calls int32 + fetch := func() vitalsMetricsEntry { + atomic.AddInt32(&calls, 1) + time.Sleep(20 * time.Millisecond) // widen the race window + return vitalsMetricsEntry{cpuMillis: 100, ok: true} + } + const n = 20 + var wg sync.WaitGroup + wg.Add(n) + for i := 0; i < n; i++ { + go func() { defer wg.Done(); m.loadOrFetch("ctx\x00user", fetch) }() + } + wg.Wait() + if got := atomic.LoadInt32(&calls); got != 1 { + t.Errorf("fetch ran %d times, want 1 (concurrent misses must share one probe)", got) + } +}