# NVIDIA DCGM Exporter — GPU metrics for Cockpit
# Runs on all GPU node pools (ci-training, gpu-dev)
# Alloy auto-discovers and scrapes these pods via annotations
---
apiVersion: v1
kind: Namespace
metadata:
  name: monitoring
  labels:
    app.kubernetes.io/part-of: foxhunt
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: dcgm-counters
  namespace: monitoring
data:
  counters.csv: |
    # Basic GPU metrics (no DCP/profiling — avoids SYS_ADMIN requirement)
    dcgm_sm_clock,     gauge, SM clock frequency (in MHz).
    dcgm_memory_clock, gauge, Memory clock frequency (in MHz).
    dcgm_memory_temp,  gauge, Memory temperature (in C).
    dcgm_gpu_temp,     gauge, GPU temperature (in C).
    dcgm_power_usage,              gauge, Power draw (in W).
    dcgm_total_energy_consumption, counter, Total energy consumption since boot (in mJ).
    dcgm_gpu_utilization,      gauge, GPU utilization (in %).
    dcgm_mem_copy_utilization, gauge, Memory utilization (in %).
    dcgm_enc_utilization,      gauge, Encoder utilization (in %).
    dcgm_dec_utilization,      gauge, Decoder utilization (in %).
    dcgm_xid_errors,           gauge, Value of the last XID error encountered.
    dcgm_fb_free, gauge, Framebuffer memory free (in MiB).
    dcgm_fb_used, gauge, Framebuffer memory used (in MiB).
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: dcgm-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: dcgm-exporter
    app.kubernetes.io/part-of: foxhunt
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: dcgm-exporter
  template:
    metadata:
      labels:
        app.kubernetes.io/name: dcgm-exporter
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9400"
        prometheus.io/path: "/metrics"
    spec:
      # Only schedule on nodes that have GPUs
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: nvidia.com/gpu.present
                    operator: In
                    values:
                      - "true"
      # NOTE: runtimeClassName: nvidia removed — Scaleway Kapsule GPU pools use
      # the nvidia container runtime as the default containerd handler. The
      # RuntimeClass CRD is only created by the full NVIDIA GPU Operator, not
      # the device plugin alone, so requiring it prevents pod scheduling.
      tolerations:
        # Tolerate GPU taints so the exporter can run on GPU nodes
        - operator: Exists
          effect: NoSchedule
      containers:
        - name: dcgm-exporter
          image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
          ports:
            - name: metrics
              containerPort: 9400
          args:
            - "-f"
            - "/etc/dcgm-exporter/custom/counters.csv"
          env:
            - name: DCGM_EXPORTER_KUBERNETES
              value: "true"
          resources:
            requests:
              cpu: 50m
              memory: 256Mi
            limits:
              cpu: 200m
              memory: 1Gi
          securityContext:
            runAsNonRoot: false
            runAsUser: 0
          volumeMounts:
            - name: device-metrics
              mountPath: /var/lib/dcgm
            - name: custom-counters
              mountPath: /etc/dcgm-exporter/custom
              readOnly: true
      volumes:
        - name: device-metrics
          emptyDir: {}
        - name: custom-counters
          configMap:
            name: dcgm-counters
---
apiVersion: v1
kind: Service
metadata:
  name: dcgm-exporter
  namespace: monitoring
  labels:
    app.kubernetes.io/name: dcgm-exporter
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "9400"
spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: dcgm-exporter
  ports:
    - name: metrics
      port: 9400
      targetPort: 9400