feat(observability): collect process system metrics

AGENTS.md

@@ -387,9 +387,9 @@ python sdk/python/benchmarks/sandbox_pressure.py \
   in the same change so future agents receive current guidance.
 - Use Conventional Commits with a concise title and a prose body explaining
   what changed and why; do not create title-only commits. Keep a blank line
-  between title and body, wrap the body for terminal readability, and sign
-  commits with `git commit -s` to include the Developer Certificate of Origin
-  sign-off.
+  between title and body, limit every commit-message line to 72 characters,
+  and sign commits with `git commit -s` to include the Developer Certificate
+  of Origin sign-off. Mechanically check line lengths before pushing.
 - Keep unrelated dirty files out of commits, especially local deployment state,
   generated binaries, Terraform state, kubeconfigs, tokens, and private
   registry configuration.

builder/config/otel-collector-config.yaml

@@ -26,6 +26,55 @@ receivers:
       http:
         endpoint: 0.0.0.0:4318
 
+  # ──────────────────────────────────────────────────────────────────────
+  # hostmetrics/process: system metrics for long-lived AKernel/YuanRong processes
+  # Linux exposes process names through /proc/<pid>/status and truncates them
+  # to 15 characters, hence the optional suffixes below. Per-sandbox runsc
+  # processes are deliberately excluded to keep metric cardinality bounded.
+  # ─────────────────────────────────────────────────────────────────────
+  hostmetrics/process:
+    collection_interval: 15s
+    scrapers:
+      process:
+        include:
+          names:
+            - '^(function_(master|proxy)|datasystem_work(er)?|goruntime|meta_service|iam_server|sandboxd|distill_fs|etcd|otelcol-contrib)$'
+          match_type: regexp
+        scrape_process_delay: 10s
+        mute_process_all_errors: true
+        resource_attributes:
+          process.parent_pid:
+            enabled: false
+          process.executable.path:
+            enabled: false
+          process.command:
+            enabled: false
+          process.command_line:
+            # Used transiently by transform/process_component to distinguish
+            # frontend and scheduler goruntime processes, then removed before export.
+            enabled: true
+          process.owner:
+            enabled: false
+        metrics:
+          process.cpu.time:
+            enabled: true
+          process.cpu.utilization:
+            enabled: true
+          process.memory.usage:
+            enabled: true
+          process.memory.virtual:
+            enabled: true
+          process.memory.utilization:
+            enabled: true
+          process.disk.io:
+            enabled: false
+          process.open_file_descriptors:
+            enabled: true
+          process.threads:
+            enabled: true
+          process.uptime:
+            enabled: true
+
   # ─────────────────────────────────────────────────────────────────────
   # filelog/spdlog: C++ glog-style logs
   # ─────────────────────────────────────────────────────────────────────
@@ -373,6 +422,36 @@ processors:
         value: ${env:AKERNEL_ENV:-default}
         action: upsert
 
+  resource/process_context:
+    attributes:
+      - key: service.name
+        value: akernel-process
+        action: upsert
+      - key: service.instance.id
+        value: ${env:POD_NAME:-unknown}
+        action: upsert
+      - key: akernel_role
+        value: ${env:AKERNEL_ROLE:-unknown}
+        action: upsert
+      - key: k8s.pod.name
+        value: ${env:POD_NAME:-unknown}
+        action: upsert
+      - key: k8s.namespace.name
+        value: ${env:POD_NAMESPACE:-default}
+        action: upsert
+      - key: k8s.node.name
+        value: ${env:NODE_NAME:-unknown}
+        action: upsert
+
+  transform/process_component:
+    error_mode: ignore
+    metric_statements:
+      - 'set(resource.attributes["component.name"], resource.attributes["process.executable.name"])'
+      - 'set(resource.attributes["component.name"], "ds_worker") where IsMatch(resource.attributes["process.executable.name"], "^datasystem_work(er)?$")'
+      - 'set(resource.attributes["component.name"], "frontend") where resource.attributes["process.executable.name"] == "goruntime" and IsMatch(resource.attributes["process.command_line"], "(^|\\s)-runtimeId=frontend_libruntime(\\s|$)")'
+      - 'set(resource.attributes["component.name"], "scheduler") where resource.attributes["process.executable.name"] == "goruntime" and IsMatch(resource.attributes["process.command_line"], "(^|\\s)-runtimeId=scheduler_libruntime(\\s|$)")'
+      - 'delete_key(resource.attributes, "process.command_line")'
+
 exporters:
   otlphttp/loki:
     endpoint: http://${env:LOKI_ENDPOINT:-loki:3100}/otlp
@@ -417,3 +496,7 @@ service:
       receivers: [otlp]
       processors: [memory_limiter, resource, batch]
       exporters: [prometheusremotewrite]
+    metrics/process:
+      receivers: [hostmetrics/process]
+      processors: [memory_limiter, resource, resource/process_context, transform/process_component, batch]
+      exporters: [prometheusremotewrite]

builder/systemd_services/otel_collector.service

@@ -9,7 +9,7 @@ Type=simple
 # memory-limiter threshold so GC can react before telemetry is refused.
 Environment=GOMAXPROCS=8
 Environment=GOMEMLIMIT=1400MiB
-PassEnvironment=AKERNEL_ENV YR_LOG_PATH PROMETHEUS_ENDPOINT LOKI_ENDPOINT TEMPO_ENDPOINT
+PassEnvironment=AKERNEL_ENV AKERNEL_ROLE NODE_NAME POD_NAME POD_NAMESPACE YR_LOG_PATH PROMETHEUS_ENDPOINT LOKI_ENDPOINT TEMPO_ENDPOINT
 ExecStartPre=/bin/bash -c 'for i in 1 2 3 4 5; do [ -f /etc/otel-collector/otel_config.yaml ] && break; sleep 1; done; [ -f /etc/otel-collector/otel_config.yaml ] && ln -sf /etc/otel-collector/otel_config.yaml /home/yuanrong/otel_config.yaml || true'
 ExecStart=/usr/local/bin/otelcol-contrib --config=/home/yuanrong/otel_config.yaml
 ExecReload=/bin/kill -HUP $MAINPID

deploy/akernel/charts/core/files/otel-collector/config.yaml

@@ -26,6 +26,55 @@ receivers:
       http:
         endpoint: 0.0.0.0:4318
 
+  # ─────────────────────────────────────────────────────────────────────
+  # hostmetrics/process: system metrics for long-lived AKernel/YuanRong processes
+  # Linux exposes process names through /proc/<pid>/status and truncates them
+  # to 15 characters, hence the optional suffixes below. Per-sandbox runsc
+  # processes are deliberately excluded to keep metric cardinality bounded.
+  # ───────────────────────────────────────────────────────────────────
+  hostmetrics/process:
+    collection_interval: 15s
+    scrapers:
+      process:
+        include:
+          names:
+            - '^(function_(master|proxy)|datasystem_work(er)?|goruntime|meta_service|iam_server|sandboxd|distill_fs|etcd|otelcol-contrib)$'
+          match_type: regexp
+        scrape_process_delay: 10s
+        mute_process_all_errors: true
+        resource_attributes:
+          process.parent_pid:
+            enabled: false
+          process.executable.path:
+            enabled: false
+          process.command:
+            enabled: false
+          process.command_line:
+            # Used transiently by transform/process_component to distinguish
+            # frontend and scheduler goruntime processes, then removed before export.
+            enabled: true
+          process.owner:
+            enabled: false
+        metrics:
+          process.cpu.time:
+            enabled: true
+          process.cpu.utilization:
+            enabled: true
+          process.memory.usage:
+            enabled: true
+          process.memory.virtual:
+            enabled: true
+          process.memory.utilization:
+            enabled: true
+          process.disk.io:
+            enabled: false
+          process.open_file_descriptors:
+            enabled: true
+          process.threads:
+            enabled: true
+          process.uptime:
+            enabled: true
+
   # ─────────────────────────────────────────────────────────────────────
   # filelog/spdlog: C++ glog-style logs
   # ─────────────────────────────────────────────────────────────────────
@@ -369,6 +418,36 @@ processors:
         value: ${env:AKERNEL_ENV:-default}
         action: upsert
 
+  resource/process_context:
+    attributes:
+      - key: service.name
+        value: akernel-process
+        action: upsert
+      - key: service.instance.id
+        value: ${env:POD_NAME:-unknown}
+        action: upsert
+      - key: akernel_role
+        value: ${env:AKERNEL_ROLE:-unknown}
+        action: upsert
+      - key: k8s.pod.name
+        value: ${env:POD_NAME:-unknown}
+        action: upsert
+      - key: k8s.namespace.name
+        value: ${env:POD_NAMESPACE:-default}
+        action: upsert
+      - key: k8s.node.name
+        value: ${env:NODE_NAME:-unknown}
+        action: upsert
+
+  transform/process_component:
+    error_mode: ignore
+    metric_statements:
+      - 'set(resource.attributes["component.name"], resource.attributes["process.executable.name"])'
+      - 'set(resource.attributes["component.name"], "ds_worker") where IsMatch(resource.attributes["process.executable.name"], "^datasystem_work(er)?$")'
+      - 'set(resource.attributes["component.name"], "frontend") where resource.attributes["process.executable.name"] == "goruntime" and IsMatch(resource.attributes["process.command_line"], "(^|\\s)-runtimeId=frontend_libruntime(\\s|$)")'
+      - 'set(resource.attributes["component.name"], "scheduler") where resource.attributes["process.executable.name"] == "goruntime" and IsMatch(resource.attributes["process.command_line"], "(^|\\s)-runtimeId=scheduler_libruntime(\\s|$)")'
+      - 'delete_key(resource.attributes, "process.command_line")'
+
 exporters:
   otlphttp/loki:
     endpoint: http://${env:LOKI_ENDPOINT:-loki:3100}/otlp
@@ -413,3 +492,7 @@ service:
       receivers: [otlp]
       processors: [memory_limiter, resource, batch]
       exporters: [prometheusremotewrite]
+    metrics/process:
+      receivers: [hostmetrics/process]
+      processors: [memory_limiter, resource, resource/process_context, transform/process_component, batch]
+      exporters: [prometheusremotewrite]

deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml

@@ -89,6 +89,21 @@ spec:
           env:
             - name: AKERNEL_ROLE
               value: "frontend"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
             - name: LITEBUS_DATA_KEY
               valueFrom:
                 secretKeyRef:

deploy/akernel/charts/core/templates/master/akernel_master.yaml

@@ -79,6 +79,21 @@ spec:
           env:
             - name: AKERNEL_ROLE
               value: "master"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
             {{- if .Values.frontend.enabled }}
             - name: ENABLE_FAAS_FRONTEND
               value: "false"