fix(infra): rewrite dashboards with verified metrics, fix KSM + internal DNS

Rewrite all 5 Grafana dashboards using only confirmed-existing Prometheus metric names (1023 metrics inventoried). Dashboards now use hardcoded datasource UIDs instead of unresolvable ${DS_*} template variables. - Cockpit: 38 panels (service health, gRPC, Argo CI/CD, trading, metrics) - Training: 33 panels (epochs, loss, Sharpe, Q-values, GPU, hyperopt) - Trading: 5 rows (overview, latency, gRPC pipeline, data acq, backtesting) - Infrastructure: 25 panels (cluster, nodes, workloads, storage, GPU, Prometheus) - Observability: 22 panels (API gateway auth/security/routing, Loki, Tempo) Fix kube-state-metrics: add part-of label for netpol, fix nodeSelector (infra→platform), increase memory limit (128→256Mi for OOM), add K8s API egress rules covering both 10.32.0.0/16 and 172.16.0.0/16 CIDR ranges. Add training-pods pod-based scrape job for ephemeral Argo workflow pods exposing metrics on port 9094. Migrate all image references from localhost:30500 to internal DNS (gitlab-registry.foxhunt.svc.cluster.local:5000) across 14 YAML files. Fix import.sh ConfigMap name (grafana-dashboards-infra → infrastructure). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 18:55:06 +01:00
parent b3c85d4f7c
commit 9dc4d2585f
24 changed files with 5468 additions and 6882 deletions
--- a/infra/k8s/argo/ci-pipeline-template.yaml
+++ b/infra/k8s/argo/ci-pipeline-template.yaml
@@ -129,7 +129,7 @@ spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
-        image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        env:
          - name: COMMITS_JSON
@@ -305,7 +305,7 @@ spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
-        image: localhost:30500/root/foxhunt/ci-builder-cpu:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
        imagePullPolicy: Always
        command: ["/bin/sh", "-c"]
        env:
@@ -433,7 +433,7 @@ spec:
          operator: Exists
          effect: NoSchedule
      container:
-        image: localhost:30500/root/foxhunt/ci-builder:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
        imagePullPolicy: Always
        command: ["/bin/sh", "-c"]
        env:
--- a/infra/k8s/argo/training-workflow-template.yaml
+++ b/infra/k8s/argo/training-workflow-template.yaml
@@ -83,7 +83,7 @@ spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
-        image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -133,7 +133,7 @@ spec:
          operator: Exists
          effect: NoSchedule
      container:
-        image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -229,7 +229,7 @@ spec:
          operator: Exists
          effect: NoSchedule
      container:
-        image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -319,7 +319,7 @@ spec:
          operator: Exists
          effect: NoSchedule
      container:
-        image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -392,7 +392,7 @@ spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
-        image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
--- a/infra/k8s/gitlab/values.yaml
+++ b/infra/k8s/gitlab/values.yaml
@@ -95,7 +95,7 @@ gitlab-runner:

 # Registry — S3-backed, NodePort for kubelet image pulls
 # Token auth disabled (cluster-internal only, secured by network policies)
-# Pods pull via localhost:30500; Kaniko pushes via ClusterIP gitlab-registry:5000
+# Pods pull via gitlab-registry.foxhunt.svc.cluster.local:5000 (internal DNS)
 registry:
  replicaCount: 1
  hpa:
--- a/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml
+++ b/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml
@@ -49,7 +49,7 @@ spec:
        - name: gitlab-registry
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -88,7 +88,7 @@ spec:
              memory: 128Mi
      containers:
        - name: ml-training-service
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/gpu-overlays/trading-service-gpu.yaml
+++ b/infra/k8s/gpu-overlays/trading-service-gpu.yaml
@@ -48,7 +48,7 @@ spec:
        - name: gitlab-registry
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -87,7 +87,7 @@ spec:
              memory: 128Mi
      containers:
        - name: trading-service
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json
+++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json
--- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json
+++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json
--- a/infra/k8s/monitoring/dashboards/foxhunt-observability.json
+++ b/infra/k8s/monitoring/dashboards/foxhunt-observability.json
--- a/infra/k8s/monitoring/dashboards/foxhunt-trading.json
+++ b/infra/k8s/monitoring/dashboards/foxhunt-trading.json
--- a/infra/k8s/monitoring/dashboards/foxhunt-training.json
+++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json
--- a/infra/k8s/monitoring/dashboards/import.sh
+++ b/infra/k8s/monitoring/dashboards/import.sh
@@ -126,7 +126,7 @@ declare -A CONFIGMAP_GROUPS=(
  [grafana-dashboards-cockpit]="foxhunt-cockpit.json"
  [grafana-dashboards-trading]="foxhunt-trading.json"
  [grafana-dashboards-training]="foxhunt-training.json"
-  [grafana-dashboards-infra]="foxhunt-infrastructure.json"
+  [grafana-dashboards-infrastructure]="foxhunt-infrastructure.json"
  [grafana-dashboards-observability]="foxhunt-observability.json"
 )

@@ -135,7 +135,7 @@ declare -A CONFIGMAP_FOLDERS=(
  [grafana-dashboards-cockpit]="Foxhunt"
  [grafana-dashboards-trading]="Foxhunt"
  [grafana-dashboards-training]="Foxhunt"
-  [grafana-dashboards-infra]="Foxhunt"
+  [grafana-dashboards-infrastructure]="Foxhunt"
  [grafana-dashboards-observability]="Foxhunt"
 )

--- a/infra/k8s/monitoring/kube-state-metrics.yaml
+++ b/infra/k8s/monitoring/kube-state-metrics.yaml
@@ -106,16 +106,11 @@ spec:
    metadata:
      labels:
        app.kubernetes.io/name: kube-state-metrics
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        gitlab.com/prometheus_scrape: "true"
-        gitlab.com/prometheus_port: "8080"
-        gitlab.com/prometheus_path: "/metrics"
+        app.kubernetes.io/part-of: foxhunt
    spec:
      serviceAccountName: kube-state-metrics
      nodeSelector:
-        k8s.scaleway.com/pool-name: infra
+        k8s.scaleway.com/pool-name: platform
      containers:
        - name: kube-state-metrics
          image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0
@@ -127,10 +122,10 @@ spec:
          resources:
            requests:
              cpu: 25m
-              memory: 64Mi
-            limits:
-              cpu: 100m
              memory: 128Mi
+            limits:
+              cpu: 200m
+              memory: 256Mi
          securityContext:
            readOnlyRootFilesystem: true
            allowPrivilegeEscalation: false
@@ -156,9 +151,6 @@ metadata:
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: kube-state-metrics
-  annotations:
-    prometheus.io/scrape: "true"
-    prometheus.io/port: "8080"
 spec:
  selector:
    app.kubernetes.io/name: kube-state-metrics
--- a/infra/k8s/monitoring/prometheus.yaml
+++ b/infra/k8s/monitoring/prometheus.yaml
@@ -128,6 +128,24 @@ data:
          - target_label: service
            replacement: argo-workflows

+      # Training workflow pods (ephemeral Argo pods exposing metrics on :9094)
+      - job_name: training-pods
+        kubernetes_sd_configs:
+          - role: pod
+            namespaces:
+              names: [foxhunt]
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
+            action: keep
+            regex: training-workflow
+          - source_labels: [__meta_kubernetes_pod_ip]
+            target_label: __address__
+            replacement: ${1}:9094
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - source_labels: [__meta_kubernetes_pod_label_workflows_argoproj_io_workflow]
+            target_label: workflow
+
      # Pushgateway (training job metrics)
      - job_name: pushgateway
        honor_labels: true
--- a/infra/k8s/network-policies/allow-monitoring.yaml
+++ b/infra/k8s/network-policies/allow-monitoring.yaml
@@ -27,6 +27,8 @@ spec:
            matchLabels:
              app.kubernetes.io/name: prometheus
      ports:
+        - protocol: TCP
+          port: 8080
        - protocol: TCP
          port: 9091
        - protocol: TCP
@@ -55,3 +57,18 @@ spec:
      ports:
        - protocol: TCP
          port: 4317
+    # Kubernetes API (for kube-state-metrics list/watch)
+    - to:
+        - ipBlock:
+            cidr: 10.32.0.0/16
+      ports:
+        - protocol: TCP
+          port: 443
+        - protocol: TCP
+          port: 6443
+    - to:
+        - ipBlock:
+            cidr: 172.16.0.0/16
+      ports:
+        - protocol: TCP
+          port: 6443
--- a/infra/k8s/services/api.yaml
+++ b/infra/k8s/services/api.yaml
@@ -40,7 +40,7 @@ spec:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -79,7 +79,7 @@ spec:
              memory: 128Mi
      containers:
        - name: api
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/services/backtesting-service.yaml
+++ b/infra/k8s/services/backtesting-service.yaml
@@ -39,7 +39,7 @@ spec:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -78,7 +78,7 @@ spec:
              memory: 128Mi
      containers:
        - name: backtesting-service
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/services/broker-gateway.yaml
+++ b/infra/k8s/services/broker-gateway.yaml
@@ -39,7 +39,7 @@ spec:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -78,7 +78,7 @@ spec:
              memory: 128Mi
      containers:
        - name: broker-gateway
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/services/data-acquisition-service.yaml
+++ b/infra/k8s/services/data-acquisition-service.yaml
@@ -39,7 +39,7 @@ spec:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -78,7 +78,7 @@ spec:
              memory: 128Mi
      containers:
        - name: data-acquisition-service
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/services/ml-training-service.yaml
+++ b/infra/k8s/services/ml-training-service.yaml
@@ -82,7 +82,7 @@ spec:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -121,7 +121,7 @@ spec:
              memory: 128Mi
      containers:
        - name: ml-training-service
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
@@ -169,7 +169,7 @@ spec:
                  name: minio-credentials
                  key: secret-key
            - name: TRAINING_RUNTIME_IMAGE
-              value: "localhost:30500/root/foxhunt/foxhunt-training-runtime:latest"
+              value: "gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest"
            - name: RUST_LOG
              value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn"
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
--- a/infra/k8s/services/trading-agent-service.yaml
+++ b/infra/k8s/services/trading-agent-service.yaml
@@ -39,7 +39,7 @@ spec:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -78,7 +78,7 @@ spec:
              memory: 128Mi
      containers:
        - name: trading-agent-service
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/services/trading-service.yaml
+++ b/infra/k8s/services/trading-service.yaml
@@ -39,7 +39,7 @@ spec:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -78,7 +78,7 @@ spec:
              memory: 128Mi
      containers:
        - name: trading-service
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
--- a/infra/k8s/services/web-dashboard.yaml
+++ b/infra/k8s/services/web-dashboard.yaml
@@ -28,7 +28,7 @@ spec:
        - name: gitlab-registry
      initContainers:
        - name: fetch-assets
-          image: localhost:30500/root/foxhunt/foxhunt-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
--- a/infra/k8s/training/image-prepuller.yaml
+++ b/infra/k8s/training/image-prepuller.yaml
@@ -39,7 +39,7 @@ spec:
        - name: gitlab-registry
      initContainers:
        - name: pull-training-runtime
-          image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
          command: ["echo", "foxhunt-training-runtime image pulled"]
          resources:
            requests:
--- a/infra/k8s/training/job-template.yaml
+++ b/infra/k8s/training/job-template.yaml
@@ -38,7 +38,7 @@ spec:
      # Replaces PVC-based copy — no node affinity constraint, no L40S autoscale for writes.
      initContainers:
        - name: fetch-binary
-          image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -79,7 +79,7 @@ spec:
              memory: 128Mi
      containers:
        - name: training
-          image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
+          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
          # Available binaries (copied by initContainer):
          #   train_baseline_rl (for dqn, ppo)
          #   train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion)