foxhunt/infra/k8s/monitoring/prometheus.yaml

# Standalone Prometheus for foxhunt services
# Runs on the foxhunt node pool alongside all other services.
# Replaces dependency on gitlab-prometheus-server (gitlab pool).
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: foxhunt-prometheus
rules:
  - apiGroups: [""]
    resources: [nodes, nodes/proxy, nodes/metrics, services, endpoints, pods]
    verbs: [get, list, watch]
  - apiGroups: [discovery.k8s.io]
    resources: [endpointslices]
    verbs: [get, list, watch]
  - nonResourceURLs: [/metrics]
    verbs: [get]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: foxhunt-prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: foxhunt-prometheus
subjects:
  - kind: ServiceAccount
    name: prometheus
    namespace: foxhunt
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: foxhunt
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
      scrape_timeout: 10s

    scrape_configs:
      # Self-monitoring
      - job_name: prometheus
        static_configs:
          - targets: ['localhost:9090']

      # Foxhunt services (metrics port)
      - job_name: foxhunt-services
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of]
            action: keep
            regex: foxhunt
          - source_labels: [__meta_kubernetes_service_name]
            action: drop
            regex: argo-workflows-workflow-controller-metrics
          - source_labels: [__meta_kubernetes_endpoint_port_name]
            action: keep
            regex: metrics
          - source_labels: [__meta_kubernetes_service_name]
            target_label: service
          - source_labels: [__meta_kubernetes_namespace]
            target_label: namespace

      # Node-exporter (host metrics)
      - job_name: node-exporter
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
            action: keep
            regex: node-exporter
          - source_labels: [__meta_kubernetes_endpoints_name]
            action: keep
            regex: node-exporter
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # kube-state-metrics
      - job_name: kube-state-metrics
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
            action: keep
            regex: kube-state-metrics
          - source_labels: [__meta_kubernetes_endpoint_port_name]
            action: keep
            regex: http

      # GPU metrics (DCGM exporter)
      - job_name: dcgm-exporter
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [kube-system, monitoring]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_name]
            action: keep
            regex: .*dcgm.*
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # Argo Workflows controller metrics (HTTPS endpoint)
      - job_name: argo-workflows
        scheme: https
        tls_config:
          insecure_skip_verify: true
        static_configs:
          - targets: ['argo-workflows-workflow-controller-metrics.foxhunt.svc.cluster.local:9090']
        relabel_configs:
          - target_label: service
            replacement: argo-workflows

      # Training workflow pods (ephemeral Argo pods exposing metrics on :9094)
      - job_name: training-pods
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
            action: keep
            regex: training-workflow
          # Only scrape Running pods — completed/failed Argo steps no longer serve metrics
          - source_labels: [__meta_kubernetes_pod_phase]
            action: keep
            regex: Running
          - source_labels: [__meta_kubernetes_pod_ip]
            target_label: __address__
            replacement: ${1}:9094
          - source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_label_workflows_argoproj_io_workflow]
            target_label: workflow

      # QuestDB (ML metrics store + DB health)
      - job_name: questdb
        metrics_path: /metrics
        static_configs:
          - targets: ['questdb.foxhunt.svc.cluster.local:9003']
        relabel_configs:
          - target_label: service
            replacement: questdb

      # Pushgateway (training job metrics)
      - job_name: pushgateway
        honor_labels: true
        static_configs:
          - targets: ['pushgateway.foxhunt.svc.cluster.local:9091']
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: foxhunt
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus
  strategy:
    type: Recreate  # single-replica, avoid PVC contention
  template:
    metadata:
      labels:
        app.kubernetes.io/name: prometheus
        app.kubernetes.io/part-of: foxhunt
    spec:
      serviceAccountName: prometheus
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      securityContext:
        fsGroup: 65534
        runAsNonRoot: true
        runAsUser: 65534
      containers:
        - name: prometheus
          image: quay.io/prometheus/prometheus:v3.8.1
          args:
            - --config.file=/etc/prometheus/prometheus.yml
            - --storage.tsdb.path=/prometheus
            - --storage.tsdb.retention.time=14d
            - --storage.tsdb.retention.size=1500MB
            - --web.enable-lifecycle
            - --web.console.libraries=/usr/share/prometheus/console_libraries
            - --web.console.templates=/usr/share/prometheus/consoles
          ports:
            - containerPort: 9090
              name: http
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: 500m
              memory: 512Mi
          volumeMounts:
            - name: config
              mountPath: /etc/prometheus
            - name: data
              mountPath: /prometheus
          readinessProbe:
            httpGet:
              path: /-/ready
              port: 9090
            initialDelaySeconds: 5
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /-/healthy
              port: 9090
            initialDelaySeconds: 10
            periodSeconds: 15
      volumes:
        - name: config
          configMap:
            name: prometheus-config
        - name: data
          persistentVolumeClaim:
            claimName: prometheus-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-data
  namespace: foxhunt
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: scw-bssd
  resources:
    requests:
      storage: 2Gi
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: foxhunt
spec:
  selector:
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: foxhunt
  ports:
    - name: http
      port: 80
      targetPort: 9090