From 16db63d71eb1611b1a8f618194004738bdafc7ff Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sun, 1 Mar 2026 14:15:19 +0100
Subject: [PATCH] =?UTF-8?q?fix(monitoring):=20DCGM=20exporter=20=E2=80=94?=
 =?UTF-8?q?=20nvidia=20runtime,=20custom=20counters,=20GitLab=20annotation?=
 =?UTF-8?q?s?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three fixes for GPU metrics not appearing in dashboards:

1. Added runtimeClassName: nvidia so DCGM can access NVML/GPU devices
2. Custom counters ConfigMap without DCP/profiling metrics (avoids
   SYS_ADMIN requirement that caused fatal crash)
3. Added gitlab.com/prometheus_* annotations so GitLab's Prometheus
   discovers and scrapes the DCGM pods

Increased memory limit 128Mi → 1Gi (was OOMKilled).
Metrics now flowing: gpu_utilization, fb_used/free, power_usage, temps.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 infra/k8s/monitoring/dcgm-exporter.yaml | 45 ++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/infra/k8s/monitoring/dcgm-exporter.yaml b/infra/k8s/monitoring/dcgm-exporter.yaml
index 7541281e6..4885fcb1d 100644
--- a/infra/k8s/monitoring/dcgm-exporter.yaml
+++ b/infra/k8s/monitoring/dcgm-exporter.yaml
@@ -9,6 +9,28 @@ metadata:
   labels:
     app.kubernetes.io/part-of: foxhunt
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dcgm-counters
+  namespace: monitoring
+data:
+  counters.csv: |
+    # Basic GPU metrics (no DCP/profiling — avoids SYS_ADMIN requirement)
+    dcgm_sm_clock,     gauge, SM clock frequency (in MHz).
+    dcgm_memory_clock, gauge, Memory clock frequency (in MHz).
+    dcgm_memory_temp,  gauge, Memory temperature (in C).
+    dcgm_gpu_temp,     gauge, GPU temperature (in C).
+    dcgm_power_usage,              gauge, Power draw (in W).
+    dcgm_total_energy_consumption, counter, Total energy consumption since boot (in mJ).
+    dcgm_gpu_utilization,      gauge, GPU utilization (in %).
+    dcgm_mem_copy_utilization, gauge, Memory utilization (in %).
+    dcgm_enc_utilization,      gauge, Encoder utilization (in %).
+    dcgm_dec_utilization,      gauge, Decoder utilization (in %).
+    dcgm_xid_errors,           gauge, Value of the last XID error encountered.
+    dcgm_fb_free, gauge, Framebuffer memory free (in MiB).
+    dcgm_fb_used, gauge, Framebuffer memory used (in MiB).
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -29,6 +51,9 @@ spec:
         prometheus.io/scrape: "true"
         prometheus.io/port: "9400"
         prometheus.io/path: "/metrics"
+        gitlab.com/prometheus_scrape: "true"
+        gitlab.com/prometheus_port: "9400"
+        gitlab.com/prometheus_path: "/metrics"
     spec:
       # Only schedule on nodes that have GPUs
       affinity:
@@ -40,6 +65,7 @@ spec:
                     operator: In
                     values:
                       - "true"
+      runtimeClassName: nvidia
       tolerations:
         # Tolerate GPU taints so the exporter can run on GPU nodes
         - operator: Exists
@@ -50,25 +76,34 @@ spec:
           ports:
             - name: metrics
               containerPort: 9400
+          args:
+            - "-f"
+            - "/etc/dcgm-exporter/custom/counters.csv"
           env:
             - name: DCGM_EXPORTER_KUBERNETES
               value: "true"
           resources:
             requests:
-              cpu: 25m
-              memory: 64Mi
+              cpu: 50m
+              memory: 256Mi
             limits:
-              cpu: 100m
-              memory: 128Mi
+              cpu: 200m
+              memory: 1Gi
           securityContext:
             runAsNonRoot: false
             runAsUser: 0
           volumeMounts:
             - name: device-metrics
               mountPath: /var/lib/dcgm
+            - name: custom-counters
+              mountPath: /etc/dcgm-exporter/custom
+              readOnly: true
       volumes:
         - name: device-metrics
           emptyDir: {}
+        - name: custom-counters
+          configMap:
+            name: dcgm-counters
 ---
 apiVersion: v1
 kind: Service
@@ -80,6 +115,8 @@ metadata:
   annotations:
     prometheus.io/scrape: "true"
     prometheus.io/port: "9400"
+    gitlab.com/prometheus_scrape: "true"
+    gitlab.com/prometheus_port: "9400"
 spec:
   type: ClusterIP
   selector: