c384ab4b43
Prometheus was configured to scrape pods via prometheus.io/scrape annotations, but all services and training jobs used gitlab.com/ prefix from legacy GitLab-managed Prometheus — causing Prometheus to never discover any foxhunt pods. This resulted in stale/missing metrics on the Grafana training dashboard. 12 files updated across services/, gpu-overlays/, training/, and monitoring/ (node-exporter, dcgm-exporter cleanup). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
126 lines
3.9 KiB
YAML
126 lines
3.9 KiB
YAML
# NVIDIA DCGM Exporter — GPU metrics for Cockpit
|
|
# Runs on all GPU node pools (ci-training, gpu-dev)
|
|
# Alloy auto-discovers and scrapes these pods via annotations
|
|
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: monitoring
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: dcgm-counters
|
|
namespace: monitoring
|
|
data:
|
|
counters.csv: |
|
|
# Basic GPU metrics (no DCP/profiling — avoids SYS_ADMIN requirement)
|
|
dcgm_sm_clock, gauge, SM clock frequency (in MHz).
|
|
dcgm_memory_clock, gauge, Memory clock frequency (in MHz).
|
|
dcgm_memory_temp, gauge, Memory temperature (in C).
|
|
dcgm_gpu_temp, gauge, GPU temperature (in C).
|
|
dcgm_power_usage, gauge, Power draw (in W).
|
|
dcgm_total_energy_consumption, counter, Total energy consumption since boot (in mJ).
|
|
dcgm_gpu_utilization, gauge, GPU utilization (in %).
|
|
dcgm_mem_copy_utilization, gauge, Memory utilization (in %).
|
|
dcgm_enc_utilization, gauge, Encoder utilization (in %).
|
|
dcgm_dec_utilization, gauge, Decoder utilization (in %).
|
|
dcgm_xid_errors, gauge, Value of the last XID error encountered.
|
|
dcgm_fb_free, gauge, Framebuffer memory free (in MiB).
|
|
dcgm_fb_used, gauge, Framebuffer memory used (in MiB).
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: dcgm-exporter
|
|
namespace: monitoring
|
|
labels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "9400"
|
|
prometheus.io/path: "/metrics"
|
|
spec:
|
|
# Only schedule on nodes that have GPUs
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: nvidia.com/gpu.present
|
|
operator: In
|
|
values:
|
|
- "true"
|
|
# NOTE: runtimeClassName: nvidia removed — Scaleway Kapsule GPU pools use
|
|
# the nvidia container runtime as the default containerd handler. The
|
|
# RuntimeClass CRD is only created by the full NVIDIA GPU Operator, not
|
|
# the device plugin alone, so requiring it prevents pod scheduling.
|
|
tolerations:
|
|
# Tolerate GPU taints so the exporter can run on GPU nodes
|
|
- operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: dcgm-exporter
|
|
image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
|
|
ports:
|
|
- name: metrics
|
|
containerPort: 9400
|
|
args:
|
|
- "-f"
|
|
- "/etc/dcgm-exporter/custom/counters.csv"
|
|
env:
|
|
- name: DCGM_EXPORTER_KUBERNETES
|
|
value: "true"
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 1Gi
|
|
securityContext:
|
|
runAsNonRoot: false
|
|
runAsUser: 0
|
|
volumeMounts:
|
|
- name: device-metrics
|
|
mountPath: /var/lib/dcgm
|
|
- name: custom-counters
|
|
mountPath: /etc/dcgm-exporter/custom
|
|
readOnly: true
|
|
volumes:
|
|
- name: device-metrics
|
|
emptyDir: {}
|
|
- name: custom-counters
|
|
configMap:
|
|
name: dcgm-counters
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: dcgm-exporter
|
|
namespace: monitoring
|
|
labels:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "9400"
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app.kubernetes.io/name: dcgm-exporter
|
|
ports:
|
|
- name: metrics
|
|
port: 9400
|
|
targetPort: 9400
|