foxhunt/infra/k8s/monitoring/prometheus-values.yaml

# Prometheus override values for GitLab Helm chart
# Removes Cockpit remote_write, adds comprehensive cluster scrape configs
#
# Apply with:
#   helm upgrade gitlab gitlab/gitlab -n foxhunt --reuse-values \
#     -f infra/k8s/monitoring/prometheus-values.yaml
prometheus:
  server:
    # Remove Cockpit remote_write — all metrics stay local
    remoteWrite: []

  # Extra scrape configs for full cluster observability
  # NOTE: this is at prometheus.extraScrapeConfigs (chart root), NOT prometheus.server.extraScrapeConfigs
  extraScrapeConfigs: |
      # --- Host metrics from node-exporter ---
      - job_name: node-exporter
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
            action: keep
            regex: node-exporter
          - source_labels: [__meta_kubernetes_endpoints_name]
            action: keep
            regex: node-exporter
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # --- Kubernetes object state ---
      - job_name: kube-state-metrics
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
            action: keep
            regex: kube-state-metrics
          - source_labels: [__meta_kubernetes_endpoint_port_name]
            action: keep
            regex: http

      # --- Kubelet metrics (container resource usage) ---
      - job_name: kubelet
        scheme: https
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        kubernetes_sd_configs:
          - role: node
        relabel_configs:
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/$1/proxy/metrics

      # --- cAdvisor (container CPU, memory, network, filesystem) ---
      - job_name: cadvisor
        scheme: https
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        kubernetes_sd_configs:
          - role: node
        relabel_configs:
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
        metric_relabel_configs:
          - source_labels: [namespace]
            action: keep
            regex: (foxhunt|monitoring|kube-system)

      # --- GPU metrics from NVIDIA DCGM Exporter ---
      - job_name: dcgm-exporter
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [kube-system, monitoring]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_name]
            action: keep
            regex: .*dcgm.*
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # --- Foxhunt services (app metrics ports) ---
      - job_name: foxhunt-services
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of]
            action: keep
            regex: foxhunt
          - source_labels: [__meta_kubernetes_endpoint_port_name]
            action: keep
            regex: metrics
          - source_labels: [__meta_kubernetes_service_name]
            target_label: service
          - source_labels: [__meta_kubernetes_namespace]
            target_label: namespace

      # --- General prometheus.io annotation-based scraping ---
      - job_name: prometheus-annotated-pods
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
            action: keep
            regex: "true"
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
            action: replace
            regex: (https?)
            target_label: __scheme__
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
            action: replace
            regex: (.+)
            target_label: __metrics_path__
          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
            action: replace
            regex: ([^:]+)(?::\d+)?;(\d+)
            replacement: $1:$2
            target_label: __address__
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # --- Loki metrics ---
      - job_name: loki
        static_configs:
          - targets: ['loki.foxhunt.svc.cluster.local:3100']

      # --- Tempo metrics ---
      - job_name: tempo
        static_configs:
          - targets: ['tempo.foxhunt.svc.cluster.local:3200']

      # --- Promtail metrics ---
      - job_name: promtail
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
            action: keep
            regex: promtail
          - source_labels: [__meta_kubernetes_pod_ip]
            target_label: __address__
            replacement: $1:9080

      # --- Pushgateway (training job metrics) ---
      - job_name: pushgateway
        honor_labels: true
        static_configs:
          - targets: ['pushgateway.foxhunt.svc.cluster.local:9091']