diff --git a/infra/k8s/argo/archive-rbac.yaml b/infra/k8s/argo/archive-rbac.yaml deleted file mode 100644 index fdbef7d0c..000000000 --- a/infra/k8s/argo/archive-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# RBAC for Argo workflow archive — grants controller and server access -# to the PostgreSQL archive secret and MinIO credentials. ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: argo-archive-secrets - namespace: foxhunt - labels: - app.kubernetes.io/part-of: argo-workflows -rules: -- apiGroups: [""] - resources: ["secrets"] - resourceNames: ["argo-postgres-config", "minio-credentials"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: argo-archive-secrets - namespace: foxhunt - labels: - app.kubernetes.io/part-of: argo-workflows -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: argo-archive-secrets -subjects: -- kind: ServiceAccount - name: argo-workflows-workflow-controller - namespace: foxhunt -- kind: ServiceAccount - name: argo-workflows-server - namespace: foxhunt diff --git a/infra/k8s/argo/argo-workflow-netpol.yaml b/infra/k8s/argo/argo-workflow-netpol.yaml deleted file mode 100644 index eb249659a..000000000 --- a/infra/k8s/argo/argo-workflow-netpol.yaml +++ /dev/null @@ -1,518 +0,0 @@ -# Base egress for ALL Argo workflow pods: DNS, K8s API, MinIO (logging), Mattermost (notifications). -# Applies to every pod with app.kubernetes.io/part-of: foxhunt. -# Component-specific policies below add extra rules (GitLab SSH, external HTTPS, etc.). -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argo-base-egress - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/part-of: foxhunt - policyTypes: - - Egress - egress: - # DNS - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Kubernetes API — Argo executor reports task results - - ports: - - port: 443 - protocol: TCP - - port: 6443 - protocol: TCP - to: - - ipBlock: - cidr: 10.32.0.0/16 - - ipBlock: - cidr: 172.16.0.4/32 - # MinIO — log archival (Argo wait sidecar) - - ports: - - port: 9000 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - # Mattermost — exit handler webhook notifications - - ports: - - port: 8065 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argo-training-workflow - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/component: training-workflow - policyTypes: - - Egress - egress: - # DNS — required for hostname resolution (minio, gitlab, etc.) - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Kubernetes API — Argo executor reports task results - - ports: - - port: 443 - protocol: TCP - - port: 6443 - protocol: TCP - to: - - ipBlock: - cidr: 10.32.0.0/16 - - ipBlock: - cidr: 172.16.0.4/32 - # HTTPS (external registries, etc.) - - ports: - - port: 443 - protocol: TCP - # GitLab SSH — git clone - - ports: - - port: 2222 - protocol: TCP - to: - - ipBlock: - cidr: 100.90.76.85/32 - - podSelector: - matchLabels: - app: gitlab-shell - # GitLab API — binary fetch from packages - - ports: - - port: 8181 - protocol: TCP - to: - - podSelector: - matchLabels: - app: webservice - # MinIO — log archival + result upload - - ports: - - port: 9000 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - # Pushgateway — Prometheus metrics - - ports: - - port: 9091 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: pushgateway - # Tempo — OTLP traces - - ports: - - port: 4317 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: tempo - # Mattermost — exit handler webhook notifications - - ports: - - port: 8065 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost - # Container registry — image pulls - - ports: - - port: 5000 - protocol: TCP - to: - - podSelector: - matchLabels: - app: registry ---- -# Compile workflow pods: need GitLab SSH, MinIO, k8s API, external HTTPS (cargo deps) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argo-compile-workflow - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/component: compile - policyTypes: - - Egress - egress: - # DNS — resolve external hostnames (crates.io, github.com) - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Kubernetes API — Argo executor - - ports: - - port: 443 - protocol: TCP - - port: 6443 - protocol: TCP - to: - - ipBlock: - cidr: 10.32.0.0/16 - - ipBlock: - cidr: 172.16.0.4/32 - # External HTTPS — cargo fetches crates.io, github.com (candle git dep) - - ports: - - port: 443 - protocol: TCP - # GitLab SSH — git clone (Tailscale IP + cluster service) - - ports: - - port: 2222 - protocol: TCP - to: - - ipBlock: - cidr: 100.90.76.85/32 - - podSelector: - matchLabels: - app: gitlab-shell - # MinIO — binary upload - - ports: - - port: 9000 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - # Mattermost — exit handler webhook notifications - - ports: - - port: 8065 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost ---- -# Compile-and-deploy workflow pods: need GitLab HTTP (tagging/releases), MinIO, -# k8s API (deploy rollouts + Argo executor), DNS, GitLab SSH, external HTTPS. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argo-compile-and-deploy-workflow - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/component: compile-and-deploy - policyTypes: - - Egress - egress: - # DNS - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Kubernetes API — Argo executor + kubectl rollout - - ports: - - port: 443 - protocol: TCP - - port: 6443 - protocol: TCP - to: - - ipBlock: - cidr: 10.32.0.0/16 - - ipBlock: - cidr: 172.16.0.4/32 - # External HTTPS — cargo fetches, kubectl download - - ports: - - port: 443 - protocol: TCP - # GitLab SSH — git clone - - ports: - - port: 2222 - protocol: TCP - to: - - ipBlock: - cidr: 100.90.76.85/32 - - podSelector: - matchLabels: - app: gitlab-shell - # GitLab webservice — CalVer tagging, releases, package uploads - - ports: - - port: 8181 - protocol: TCP - to: - - podSelector: - matchLabels: - app: webservice - # MinIO — binary upload + log archival - - ports: - - port: 9000 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - # Mattermost — exit handler webhook notifications - - ports: - - port: 8065 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost ---- -# Compile-and-train workflow pods: compile step needs GitLab SSH + external HTTPS + DNS, -# training steps need GitLab HTTP (package fetch/upload), MinIO, Tempo, k8s API. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argo-train-workflow - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/component: train - policyTypes: - - Egress - egress: - # DNS - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Kubernetes API — Argo executor - - ports: - - port: 443 - protocol: TCP - - port: 6443 - protocol: TCP - to: - - ipBlock: - cidr: 10.32.0.0/16 - - ipBlock: - cidr: 172.16.0.4/32 - # External HTTPS — cargo fetches crates.io, github.com (candle git dep) - - ports: - - port: 443 - protocol: TCP - # GitLab SSH — git clone - - ports: - - port: 2222 - protocol: TCP - to: - - ipBlock: - cidr: 100.90.76.85/32 - - podSelector: - matchLabels: - app: gitlab-shell - # GitLab webservice — package upload/download - - ports: - - port: 8181 - protocol: TCP - to: - - podSelector: - matchLabels: - app: webservice - # MinIO — binary upload - - ports: - - port: 9000 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - # Tempo — OTLP traces - - ports: - - port: 4317 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: tempo - # Mattermost — exit handler webhook notifications - - ports: - - port: 8065 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost ---- -# CI pipeline pods (detect-changes + compile-service inline templates) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argo-ci-pipeline - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/component: ci-pipeline - policyTypes: - - Egress - egress: - # DNS — resolve external hostnames - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Kubernetes API — Argo executor - - ports: - - port: 443 - protocol: TCP - - port: 6443 - protocol: TCP - to: - - ipBlock: - cidr: 10.32.0.0/16 - - ipBlock: - cidr: 172.16.0.4/32 - # External HTTP/HTTPS — Kaniko base image pulls, apt-get, cargo fetches - - ports: - - port: 80 - protocol: TCP - - port: 443 - protocol: TCP - # GitLab SSH — git clone (compile-service uses ssh://git@git.fxhnt.ai) - - ports: - - port: 2222 - protocol: TCP - to: - - ipBlock: - cidr: 100.90.76.85/32 - - podSelector: - matchLabels: - app: gitlab-shell - # MinIO — binary upload from compile-service - - ports: - - port: 9000 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - # GitLab registry — Kaniko image push - - ports: - - port: 5000 - protocol: TCP - to: - - podSelector: - matchLabels: - app: registry - # GitLab webservice — JWT auth for registry tokens - - ports: - - port: 8181 - protocol: TCP - to: - - podSelector: - matchLabels: - app: webservice - # Mattermost — exit handler webhook notifications - - ports: - - port: 8065 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost ---- -# ── sanitizer-test + nsys-test + smoke-test workflows: same egress as ci-pipeline ── -# (git fetch over SSH, MinIO log archive + nsys-rep upload, registry image pull, -# external HTTPS for mc download) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: argo-sanitizer-nsys-workflow - namespace: foxhunt -spec: - podSelector: - matchExpressions: - - key: app.kubernetes.io/component - operator: In - values: - - sanitizer-test - - nsys-test - - smoke-test - policyTypes: - - Egress - egress: - # DNS - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # Kubernetes API - - ports: - - port: 443 - protocol: TCP - - port: 6443 - protocol: TCP - to: - - ipBlock: - cidr: 10.32.0.0/16 - - ipBlock: - cidr: 172.16.0.4/32 - # External HTTPS — mc binary download from dl.min.io - - ports: - - port: 80 - protocol: TCP - - port: 443 - protocol: TCP - # GitLab SSH — git fetch - - ports: - - port: 2222 - protocol: TCP - to: - - ipBlock: - cidr: 100.90.76.85/32 - - podSelector: - matchLabels: - app: gitlab-shell - # MinIO — log archive + nsys profile upload - - ports: - - port: 9000 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - # GitLab registry — ci-builder image pull - - ports: - - port: 5000 - protocol: TCP - to: - - podSelector: - matchLabels: - app: registry - # GitLab webservice — JWT auth for registry tokens - - ports: - - port: 8181 - protocol: TCP - to: - - podSelector: - matchLabels: - app: webservice diff --git a/infra/k8s/argo/ci-deploy-rbac.yaml b/infra/k8s/argo/ci-deploy-rbac.yaml deleted file mode 100644 index c41ac8ac5..000000000 --- a/infra/k8s/argo/ci-deploy-rbac.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# RBAC for CI pipeline — deploy step + self-apply Argo templates -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: ci-deploy - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -rules: - # apps: roll + apply Deployments. watch is required by `kubectl rollout status`. - - apiGroups: ["apps"] - resources: [deployments] - verbs: [get, list, watch, patch] - - apiGroups: ["argoproj.io"] - resources: [workflowtemplates, eventsources, sensors, eventbus] - verbs: [get, list, create, update, patch] - # core: services/configmaps, plus serviceaccounts (tailscale-dashboard, dagster) and the forward-track PVC - # — added so the fxhnt-cockpit deploy can `kubectl apply` its full manifest set without a partial-apply failure. - - apiGroups: [""] - resources: [services, configmaps, serviceaccounts, persistentvolumeclaims] - verbs: [get, list, create, update, patch] - - apiGroups: ["networking.k8s.io"] - resources: [networkpolicies] - verbs: [get, list, create, update, patch] - # batch: the fxhnt-forward CronJob - - apiGroups: ["batch"] - resources: [cronjobs] - verbs: [get, list, create, update, patch] - - apiGroups: ["rbac.authorization.k8s.io"] - resources: [roles, rolebindings] - verbs: [get, list, create, update, patch, bind, escalate] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: ci-deploy - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ci-deploy -subjects: - - kind: ServiceAccount - name: argo-workflow - namespace: foxhunt diff --git a/infra/k8s/argo/databento-download-template.yaml b/infra/k8s/argo/databento-download-template.yaml deleted file mode 100644 index bd059f9a9..000000000 --- a/infra/k8s/argo/databento-download-template.yaml +++ /dev/null @@ -1,135 +0,0 @@ -# Databento data download via streaming API (get_range). -# -# Parameterized Argo WorkflowTemplate — replaces the hardcoded K8s Job. -# Binary pre-uploaded to /data/bin/download_baseline on training-data-pvc. -# Universe configs loaded from configmap databento-universe-all. -# -# Usage: -# argo submit -n foxhunt --from=wftmpl/databento-download -# argo submit -n foxhunt --from=wftmpl/databento-download -p schema=trades -p output-dir=/data/futures-baseline-trades -# ./scripts/argo-download.sh mbp-10 --parallel 9 --pool ci-compile-cpu -# -# Databento rate limits: 100 concurrent connections per IP, 100 req/s. ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: databento-download - namespace: foxhunt - labels: - app.kubernetes.io/name: databento-download - app.kubernetes.io/part-of: foxhunt -spec: - entrypoint: download - serviceAccountName: argo-workflow - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: databento-download - securityContext: - fsGroup: 1000 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 86400 # 24 hours - - arguments: - parameters: - - name: schema - value: mbp-10 - description: "Databento schema: ohlcv-1m, ohlcv-1s, trades, mbp-10" - - name: output-dir - value: /data/futures-baseline-mbp10 - description: "Output directory on training-data PVC" - - name: parallel - value: "9" - description: "Number of parallel quarterly downloads (max ~100 per Databento rate limit)" - - name: node-pool - value: ci-compile-cpu - description: "K8s node pool: ci-compile-cpu (32c/64G) or platform (4c/8G)" - - volumes: - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - - name: universe-configs - configMap: - name: databento-universe-all - - templates: - - name: download - inputs: - parameters: - - name: schema - - name: output-dir - - name: parallel - - name: node-pool - nodeSelector: - k8s.scaleway.com/pool-name: "{{inputs.parameters.node-pool}}" - topology.kubernetes.io/zone: fr-par-2 - container: - image: ubuntu:24.04 - command: ["/bin/bash", "-c"] - env: - - name: RUST_LOG - value: info - - name: DATABENTO_API_KEY - valueFrom: - secretKeyRef: - name: databento-credentials - key: api-key - resources: - requests: - cpu: 500m - memory: 512Mi - limits: - cpu: "2" - memory: 4Gi - volumeMounts: - - name: training-data - mountPath: /data - - name: universe-configs - mountPath: /configs - args: - - | - set -e - apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq ca-certificates > /dev/null 2>&1 - - BINARY=/data/bin/download_baseline - chmod +x "$BINARY" - - SCHEMA="{{inputs.parameters.schema}}" - OUTPUT_DIR="{{inputs.parameters.output-dir}}" - PARALLEL="{{inputs.parameters.parallel}}" - - # Map schema name to configmap key - CONFIG="/configs/${SCHEMA}.toml" - if [ ! -f "$CONFIG" ]; then - echo "ERROR: Config not found: $CONFIG" - echo "Available configs:" - ls /configs/ - exit 1 - fi - - mkdir -p "$OUTPUT_DIR" - - echo "=== Databento Download ===" - echo "Schema: $SCHEMA" - echo "Config: $CONFIG" - echo "Output: $OUTPUT_DIR" - echo "Parallel: $PARALLEL" - echo "Node: $(hostname), CPUs: $(nproc), RAM: $(free -g | awk '/Mem:/{print $2}')G" - echo "" - - $BINARY \ - --universe-config "$CONFIG" \ - --output-dir "$OUTPUT_DIR" \ - --parallel "$PARALLEL" \ - --yes - - echo "" - echo "=== Download complete ===" - echo "Files:" - find "$OUTPUT_DIR" -name '*.dbn.zst' -exec ls -lh {} \; - echo "" - du -sh "$OUTPUT_DIR" - df -h /data diff --git a/infra/k8s/argo/dns-bootstrap-policy.yaml b/infra/k8s/argo/dns-bootstrap-policy.yaml deleted file mode 100644 index ef73cc12b..000000000 --- a/infra/k8s/argo/dns-bootstrap-policy.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# DNS Bootstrap Prevention — Prevent CoreDNS chicken-and-egg deadlock -# -# ROOT CAUSE (2026-03-18 incident): -# 1. Rolling replacement of ALL platform nodes simultaneously -# 2. New nodes boot with /etc/resolv.conf → 10.32.0.10 (kube-dns only) -# 3. CoreDNS pods need to pull images → need DNS → need CoreDNS -# 4. Cilium BPF socket LB returns EPERM when kube-dns has no endpoints -# 5. Complete cluster DNS deadlock — nothing can pull images -# -# PREVENTION: -# 1. VPC DefaultRoutePropagation MUST be enabled (API, not Terraform yet) -# 2. NEVER replace ALL nodes at once — one at a time, wait for CoreDNS -# 3. Do NOT create CiliumClusterwideNetworkPolicy with egress rules -# (enables default deny, makes the problem worse) -# 4. bpf-lb-sock MUST remain "true" — disabling breaks ALL service routing -# 5. Keep bastion enabled on VPC gateway for emergency SSH access -# -# RECOVERY PROCEDURE: -# 1. SSH via bastion: ssh -o ProxyCommand="ssh -W %h:%p -p 61000 bastion@" root@ -# 2. On the node: /usr/local/bin/crictl pull -# 3. kubectl delete pods -l k8s-app=kube-dns --force -# 4. CoreDNS starts from cached image → DNS recovers → cascade -# -# DATA PROTECTION: -# All PVs set to Retain reclaim policy — data survives cluster deletion diff --git a/infra/k8s/argo/events/eventbus.yaml b/infra/k8s/argo/events/eventbus.yaml deleted file mode 100644 index 1119581a2..000000000 --- a/infra/k8s/argo/events/eventbus.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: EventBus -metadata: - name: default - namespace: foxhunt -spec: - nats: - native: - replicas: 3 - auth: token diff --git a/infra/k8s/argo/events/eventsource-svc.yaml b/infra/k8s/argo/events/eventsource-svc.yaml deleted file mode 100644 index 50ba4d388..000000000 --- a/infra/k8s/argo/events/eventsource-svc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: gitlab-push-eventsource-svc - namespace: foxhunt - labels: - app.kubernetes.io/name: gitlab-push-eventsource - app.kubernetes.io/part-of: foxhunt -spec: - type: ClusterIP - ports: - - port: 12000 - targetPort: 12000 - protocol: TCP - selector: - eventsource-name: gitlab-push diff --git a/infra/k8s/argo/events/gitea-deploy-sensor.yaml b/infra/k8s/argo/events/gitea-deploy-sensor.yaml deleted file mode 100644 index 0719b97bb..000000000 --- a/infra/k8s/argo/events/gitea-deploy-sensor.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# On push to fxhnt main/master in Gitea, submit the fxhnt-cockpit build+deploy workflow (Phase 2B). -# Gitea push payload: { "ref": "refs/heads/main", "after": "", ... } -apiVersion: argoproj.io/v1alpha1 -kind: Sensor -metadata: - name: gitea-deploy - namespace: foxhunt - labels: - app.kubernetes.io/name: gitea-deploy - app.kubernetes.io/part-of: foxhunt -spec: - template: - # pod intentionally NOT part-of:foxhunt (avoids default-deny-all; matches working sensors) - serviceAccountName: argo-workflow - nodeSelector: - k8s.scaleway.com/pool-name: platform - dependencies: - - name: gitea-push-dep - eventSourceName: gitea-push - eventName: push - filters: - data: - - path: body.ref - type: string - value: - - "refs/heads/main" - - "refs/heads/master" - triggers: - - template: - name: deploy-cockpit - conditions: gitea-push-dep - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: fxhnt-cockpit- - namespace: foxhunt - spec: - serviceAccountName: argo-workflow - podGC: - strategy: OnPodCompletion - ttlStrategy: - secondsAfterCompletion: 3600 - workflowTemplateRef: - name: fxhnt-cockpit - arguments: - parameters: - - name: commit-sha - parameters: - - src: - dependencyName: gitea-push-dep - dataKey: body.after - dest: spec.arguments.parameters.0.value diff --git a/infra/k8s/argo/events/gitea-push-eventsource.yaml b/infra/k8s/argo/events/gitea-push-eventsource.yaml deleted file mode 100644 index 670d3ca1a..000000000 --- a/infra/k8s/argo/events/gitea-push-eventsource.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Gitea push webhook → Argo Events. Replaces the old gitlab-push eventsource (Phase 2B). -# Gitea posts to gitea-push-eventsource-svc:12000/push on push to the fxhnt repo. -apiVersion: argoproj.io/v1alpha1 -kind: EventSource -metadata: - name: gitea-push - namespace: foxhunt - labels: - app.kubernetes.io/name: gitea-push - app.kubernetes.io/part-of: foxhunt -spec: - template: - # NOTE: pod is intentionally NOT part-of:foxhunt — that label triggers default-deny-all. The working - # workflow-trigger/gitlab-push eventsource pods have no part-of, so they stay default-allow. - metadata: - labels: - app.kubernetes.io/name: gitea-push - nodeSelector: - k8s.scaleway.com/pool-name: platform - service: - ports: - - port: 12000 - targetPort: 12000 - webhook: - push: - port: "12000" - endpoint: /push - method: POST diff --git a/infra/k8s/argo/events/rbac.yaml b/infra/k8s/argo/events/rbac.yaml deleted file mode 100644 index c11075a19..000000000 --- a/infra/k8s/argo/events/rbac.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: argo-sensor-submit - namespace: foxhunt -rules: - - apiGroups: ["argoproj.io"] - resources: ["workflows"] - verbs: ["create", "get", "list", "watch"] - - apiGroups: ["argoproj.io"] - resources: ["workflowtemplates"] - verbs: ["get", "list"] - - apiGroups: ["argoproj.io"] - resources: ["workflowtaskresults"] - verbs: ["create", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: argo-sensor-submit - namespace: foxhunt -subjects: - - kind: ServiceAccount - name: argo-workflow - namespace: foxhunt -roleRef: - kind: Role - name: argo-sensor-submit - apiGroup: rbac.authorization.k8s.io diff --git a/infra/k8s/argo/events/workflow-trigger-eventsource.yaml b/infra/k8s/argo/events/workflow-trigger-eventsource.yaml deleted file mode 100644 index 59e29dc22..000000000 --- a/infra/k8s/argo/events/workflow-trigger-eventsource.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# Manual workflow trigger EventSource. -# -# Webhook endpoints for triggering compile/deploy, training, and image builds. -# Each endpoint accepts a JSON POST body with workflow-specific parameters. -# -# Endpoints (all on port 12001): -# POST /compile-deploy — {"commit_sha": "HEAD"} -# POST /train/dqn — {"binary_tag": "latest"} -# POST /train/ppo — {"binary_tag": "latest"} -# POST /train/supervised — {"model": "tft", "binary_tag": "latest"} -# POST /build-image — {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"} -# -# Example: -# curl -X POST http://workflow-trigger-eventsource-svc.foxhunt:12001/build-image \ -# -H 'Content-Type: application/json' \ -# -d '{"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}' ---- -apiVersion: argoproj.io/v1alpha1 -kind: EventSource -metadata: - name: workflow-trigger - namespace: foxhunt - labels: - app.kubernetes.io/name: workflow-trigger - app.kubernetes.io/part-of: foxhunt -spec: - eventBusName: default - template: - nodeSelector: - k8s.scaleway.com/pool-name: platform - webhook: - compile-deploy: - port: "12001" - endpoint: /compile-deploy - method: POST - train-dqn: - port: "12001" - endpoint: /train/dqn - method: POST - train-ppo: - port: "12001" - endpoint: /train/ppo - method: POST - train-supervised: - port: "12001" - endpoint: /train/supervised - method: POST - build-image: - port: "12001" - endpoint: /build-image - method: POST diff --git a/infra/k8s/argo/events/workflow-trigger-sensor.yaml b/infra/k8s/argo/events/workflow-trigger-sensor.yaml deleted file mode 100644 index a4405d817..000000000 --- a/infra/k8s/argo/events/workflow-trigger-sensor.yaml +++ /dev/null @@ -1,217 +0,0 @@ -# Sensor for manual workflow triggers. -# -# Listens on workflow-trigger EventSource and submits the appropriate workflow -# when a webhook endpoint is hit. Each trigger maps JSON body fields to -# workflow parameters. -# -# Trigger reference: -# compile-deploy — POST /compile-deploy {"commit_sha": "..."} -# train-dqn — POST /train/dqn {"binary_tag": "..."} -# train-ppo — POST /train/ppo {"binary_tag": "..."} -# train-supervised — POST /train/supervised {"model": "...", "binary_tag": "..."} -# build-image — POST /build-image {"dockerfile": "...", "image_name": "...", "commit_sha": "..."} ---- -apiVersion: argoproj.io/v1alpha1 -kind: Sensor -metadata: - name: workflow-trigger - namespace: foxhunt - labels: - app.kubernetes.io/name: workflow-trigger - app.kubernetes.io/part-of: foxhunt -spec: - template: - serviceAccountName: argo-workflow - nodeSelector: - k8s.scaleway.com/pool-name: platform - eventBusName: default - - dependencies: - - name: compile-deploy-dep - eventSourceName: workflow-trigger - eventName: compile-deploy - - name: train-dqn-dep - eventSourceName: workflow-trigger - eventName: train-dqn - - name: train-ppo-dep - eventSourceName: workflow-trigger - eventName: train-ppo - - name: train-supervised-dep - eventSourceName: workflow-trigger - eventName: train-supervised - - name: build-image-dep - eventSourceName: workflow-trigger - eventName: build-image - - triggers: - # ── Compile + Deploy ── - # POST /compile-deploy {"commit_sha": "abc1234"} - # Omit service_packages to build all 7 services (default). - - template: - name: compile-deploy - conditions: compile-deploy-dep - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: compile-deploy- - namespace: foxhunt - spec: - serviceAccountName: argo-workflow - podGC: - strategy: OnPodCompletion - ttlStrategy: - secondsAfterCompletion: 3600 - workflowTemplateRef: - name: compile-and-deploy - arguments: - parameters: - - name: commit-sha - parameters: - - src: - dependencyName: compile-deploy-dep - dataKey: body.commit_sha - dest: spec.arguments.parameters.0.value - - # ── Train DQN ── - # POST /train/dqn {"binary_tag": "latest"} - - template: - name: train-dqn - conditions: train-dqn-dep - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: train-dqn- - namespace: foxhunt - spec: - serviceAccountName: argo-workflow - podGC: - strategy: OnPodCompletion - ttlStrategy: - secondsAfterCompletion: 3600 - workflowTemplateRef: - name: train-dqn - arguments: - parameters: - - name: binary-tag - parameters: - - src: - dependencyName: train-dqn-dep - dataKey: body.binary_tag - dest: spec.arguments.parameters.0.value - - # ── Train PPO ── - # POST /train/ppo {"binary_tag": "latest"} - - template: - name: train-ppo - conditions: train-ppo-dep - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: train-ppo- - namespace: foxhunt - spec: - serviceAccountName: argo-workflow - podGC: - strategy: OnPodCompletion - ttlStrategy: - secondsAfterCompletion: 3600 - workflowTemplateRef: - name: train-ppo - arguments: - parameters: - - name: binary-tag - parameters: - - src: - dependencyName: train-ppo-dep - dataKey: body.binary_tag - dest: spec.arguments.parameters.0.value - - # ── Train Supervised ── - # POST /train/supervised {"model": "tft", "binary_tag": "latest"} - # model: one of tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion - - template: - name: train-supervised - conditions: train-supervised-dep - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: train-supervised- - namespace: foxhunt - spec: - serviceAccountName: argo-workflow - podGC: - strategy: OnPodCompletion - ttlStrategy: - secondsAfterCompletion: 3600 - workflowTemplateRef: - name: train-supervised - arguments: - parameters: - - name: model - - name: binary-tag - parameters: - - src: - dependencyName: train-supervised-dep - dataKey: body.model - dest: spec.arguments.parameters.0.value - - src: - dependencyName: train-supervised-dep - dataKey: body.binary_tag - dest: spec.arguments.parameters.1.value - - # ── Build Image ── - # POST /build-image {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"} - - template: - name: build-image - conditions: build-image-dep - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: build-image- - namespace: foxhunt - spec: - serviceAccountName: argo-workflow - podGC: - strategy: OnPodCompletion - ttlStrategy: - secondsAfterCompletion: 3600 - workflowTemplateRef: - name: build-ci-image - arguments: - parameters: - - name: commit-sha - - name: dockerfile - - name: image-name - parameters: - - src: - dependencyName: build-image-dep - dataKey: body.commit_sha - dest: spec.arguments.parameters.0.value - - src: - dependencyName: build-image-dep - dataKey: body.dockerfile - dest: spec.arguments.parameters.1.value - - src: - dependencyName: build-image-dep - dataKey: body.image_name - dest: spec.arguments.parameters.2.value diff --git a/infra/k8s/argo/events/workflow-trigger-svc.yaml b/infra/k8s/argo/events/workflow-trigger-svc.yaml deleted file mode 100644 index 0ac7e0717..000000000 --- a/infra/k8s/argo/events/workflow-trigger-svc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: workflow-trigger-eventsource-svc - namespace: foxhunt - labels: - app.kubernetes.io/name: workflow-trigger - app.kubernetes.io/part-of: foxhunt -spec: - type: ClusterIP - ports: - - port: 12001 - targetPort: 12001 - protocol: TCP - selector: - eventsource-name: workflow-trigger diff --git a/infra/k8s/argo/feature-cache-pvc.yaml b/infra/k8s/argo/feature-cache-pvc.yaml deleted file mode 100644 index b464a1b4f..000000000 --- a/infra/k8s/argo/feature-cache-pvc.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Dedicated PVC for precomputed .fxcache files. -# Survives workflow completions (unlike workspace volumeClaimTemplate). -# Precompute writes here, all training steps read from here. -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: feature-cache-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: feature-cache - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: - - ReadWriteOnce - storageClassName: scw-bssd-retain - resources: - requests: - storage: 10Gi diff --git a/infra/k8s/argo/kustomization.yaml b/infra/k8s/argo/kustomization.yaml deleted file mode 100644 index c8dfc3777..000000000 --- a/infra/k8s/argo/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -# Rust train/CI/cache templates removed 2026-06-21 (decommission-rust-infra). -# Remaining: platform RBAC + network policy used by the active fxhnt-cockpit deploys. -resources: - - argo-workflow-netpol.yaml - - ci-deploy-rbac.yaml - - archive-rbac.yaml diff --git a/infra/k8s/argo/notification-webhook.yaml b/infra/k8s/argo/notification-webhook.yaml deleted file mode 100644 index 496909dd4..000000000 --- a/infra/k8s/argo/notification-webhook.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: notification-webhook - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - # Mattermost incoming webhook URL — update after Mattermost is deployed - webhook-url: "http://mattermost.foxhunt.svc.cluster.local:8065/hooks/PLACEHOLDER" diff --git a/infra/k8s/argo/values.yaml b/infra/k8s/argo/values.yaml deleted file mode 100644 index a8b18b046..000000000 --- a/infra/k8s/argo/values.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# Argo Workflows Helm values — foxhunt cluster -# Chart: argo/argo-workflows v1.0.2 (Argo v4.0.2) -# Install: helm upgrade argo-workflows argo/argo-workflows -n foxhunt -f infra/k8s/argo/values.yaml - -controller: - nodeSelector: - k8s.scaleway.com/pool-name: platform - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - # Only watch foxhunt namespace (not cluster-wide) - workflowNamespaces: - - foxhunt - # Workflow archive — persists workflow state + logs to PostgreSQL after pod GC - persistence: - archive: true - postgresql: - host: postgres.foxhunt.svc.cluster.local - port: 5432 - database: argo_archive - tableName: argo_workflows - userNameSecret: - name: argo-postgres-config - key: username - passwordSecret: - name: argo-postgres-config - key: password - ssl: false - sslMode: disable - -server: - nodeSelector: - k8s.scaleway.com/pool-name: platform - resources: - requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - ingress: - enabled: false - extraArgs: - - --auth-mode=server - - --secure=false - -# Default artifact repository — MinIO (in-cluster) -artifactRepository: - archiveLogs: true - s3: - endpoint: minio.foxhunt.svc.cluster.local:9000 - insecure: true - bucket: argo-logs - accessKeySecret: - name: minio-credentials - key: access-key - secretKeySecret: - name: minio-credentials - key: secret-key - -executor: - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - -workflow: - serviceAccount: - create: true - name: argo-workflow - -crds: - install: true diff --git a/infra/k8s/cert-manager/fxhnt-acme.yaml b/infra/k8s/cert-manager/fxhnt-acme.yaml deleted file mode 100644 index 63c2b3615..000000000 --- a/infra/k8s/cert-manager/fxhnt-acme.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# ACME (Let's Encrypt) wildcard cert for *.fxhnt.ai via cert-manager + Scaleway DNS-01. -# Replaces the prior MANUAL, unrenewed LE cert that expired 2026-05-26 (the tailscale-proxy -# fell back to the GitLab self-signed cert). DNS-01 is required: the services are tailnet-only -# (*.fxhnt.ai -> 100.x Tailscale CGNAT) so HTTP-01 is unreachable, and wildcards need DNS-01. -# -# Solver creds: cert-manager/scaleway-dns-credentials (SCW_ACCESS_KEY/SCW_SECRET_KEY, copied from -# foxhunt/scaleway-credentials). Webhook: scaleway-certmanager-webhook (helm, cert-manager ns). -# The issued secret gitlab-tls-cert is mounted by infra/k8s/gitlab/tailscale-proxy.yaml. ---- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: letsencrypt-staging -spec: - acme: - email: jeroen@grusewski.nl - server: https://acme-staging-v02.api.letsencrypt.org/directory - privateKeySecretRef: - name: letsencrypt-staging-account-key - solvers: - - dns01: - webhook: - groupName: acme.scaleway.com - solverName: scaleway - config: - accessKeySecretRef: - name: scaleway-dns-credentials - key: SCW_ACCESS_KEY - secretKeySecretRef: - name: scaleway-dns-credentials - key: SCW_SECRET_KEY ---- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: letsencrypt-prod -spec: - acme: - email: jeroen@grusewski.nl - server: https://acme-v02.api.letsencrypt.org/directory - privateKeySecretRef: - name: letsencrypt-prod-account-key - solvers: - - dns01: - webhook: - groupName: acme.scaleway.com - solverName: scaleway - config: - accessKeySecretRef: - name: scaleway-dns-credentials - key: SCW_ACCESS_KEY - secretKeySecretRef: - name: scaleway-dns-credentials - key: SCW_SECRET_KEY ---- -# Wildcard cert for the tailscale-proxy. issuerRef flips staging->prod after staging validates; -# secretName flips to gitlab-tls-cert (the secret the proxy mounts) for the prod issuance. -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: fxhnt-wildcard - namespace: foxhunt -spec: - secretName: gitlab-tls-cert # takes over the secret the tailscale-proxy mounts - privateKey: - rotationPolicy: Always # old manual key had a mismatching algorithm; regenerate on issue/renew - dnsNames: - - fxhnt.ai - - "*.fxhnt.ai" - issuerRef: - name: letsencrypt-prod - kind: ClusterIssuer diff --git a/infra/k8s/databases/postgres-backup.yaml b/infra/k8s/databases/postgres-backup.yaml deleted file mode 100644 index 23f8454c5..000000000 --- a/infra/k8s/databases/postgres-backup.yaml +++ /dev/null @@ -1,249 +0,0 @@ -# Postgres backup — daily pg_dump to MinIO S3 -# Uses initContainer (postgres:16-alpine) for pg_dump, then main container -# (python:3.12-alpine, already cached) uploads to MinIO via S3 API. -# Retains 7 daily + 4 weekly backups. -# Apply: kubectl apply -f postgres-backup.yaml ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: postgres-backup-script - namespace: foxhunt - labels: - app.kubernetes.io/name: postgres-backup - app.kubernetes.io/part-of: foxhunt -data: - upload.py: | - #!/usr/bin/env python3 - """Upload pg_dump backup to MinIO, prune old backups.""" - import hashlib, hmac, os, sys, glob, urllib.request, urllib.error - from datetime import datetime, timezone - - ENDPOINT = os.environ["MINIO_ENDPOINT"] # http://minio.foxhunt...:9000 - ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"] - SECRET_KEY = os.environ["MINIO_SECRET_KEY"] - BUCKET = "foxhunt-backups" - BACKUP_DIR = "/backup" - - def s3_sign(method, path, headers, payload_hash): - """AWS Signature V4 for MinIO.""" - now = datetime.now(timezone.utc) - date_stamp = now.strftime("%Y%m%d") - amz_date = now.strftime("%Y%m%dT%H%M%SZ") - region = "us-east-1" - service = "s3" - scope = f"{date_stamp}/{region}/{service}/aws4_request" - - headers["x-amz-date"] = amz_date - headers["x-amz-content-sha256"] = payload_hash - - signed_headers = ";".join(sorted(headers.keys())) - canonical_headers = "".join(f"{k}:{headers[k]}\n" for k in sorted(headers.keys())) - canonical_request = f"{method}\n{path}\n\n{canonical_headers}\n{signed_headers}\n{payload_hash}" - - string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{hashlib.sha256(canonical_request.encode()).hexdigest()}" - - def sign(key, msg): - return hmac.new(key, msg.encode(), hashlib.sha256).digest() - - signing_key = sign(sign(sign(sign( - f"AWS4{SECRET_KEY}".encode(), date_stamp), region), service), "aws4_request") - signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest() - - headers["Authorization"] = ( - f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/{scope}, " - f"SignedHeaders={signed_headers}, Signature={signature}" - ) - return headers - - def s3_put(key, data): - path = f"/{BUCKET}/{key}" - url = f"{ENDPOINT}{path}" - payload_hash = hashlib.sha256(data).hexdigest() - host = ENDPOINT.split("//")[1] - headers = {"host": host, "content-length": str(len(data))} - headers = s3_sign("PUT", path, headers, payload_hash) - req = urllib.request.Request(url, data=data, method="PUT", headers=headers) - urllib.request.urlopen(req, timeout=120) - - def s3_list(prefix): - path = f"/{BUCKET}?prefix={prefix}&list-type=2" - url = f"{ENDPOINT}{path}" - host = ENDPOINT.split("//")[1] - headers = {"host": host} - headers = s3_sign("GET", f"/{BUCKET}", headers, hashlib.sha256(b"").hexdigest()) - req = urllib.request.Request(f"{url}", method="GET", headers=headers) - try: - resp = urllib.request.urlopen(req, timeout=30).read().decode() - import re - return sorted(re.findall(r"([^<]+)", resp)) - except Exception: - return [] - - def s3_delete(key): - path = f"/{BUCKET}/{key}" - url = f"{ENDPOINT}{path}" - host = ENDPOINT.split("//")[1] - headers = {"host": host} - headers = s3_sign("DELETE", path, headers, hashlib.sha256(b"").hexdigest()) - req = urllib.request.Request(url, method="DELETE", headers=headers) - urllib.request.urlopen(req, timeout=30) - - def s3_ensure_bucket(): - path = f"/{BUCKET}" - url = f"{ENDPOINT}{path}" - host = ENDPOINT.split("//")[1] - headers = {"host": host} - headers = s3_sign("PUT", path, headers, hashlib.sha256(b"").hexdigest()) - req = urllib.request.Request(url, method="PUT", headers=headers) - try: - urllib.request.urlopen(req, timeout=30) - except urllib.error.HTTPError as e: - if e.code not in (409, 200): # 409 = bucket exists - raise - - # Find the backup file - files = glob.glob(f"{BACKUP_DIR}/*.sql.gz") - if not files: - print("ERROR: No backup file found in /backup/") - sys.exit(1) - - backup_file = files[0] - filename = os.path.basename(backup_file) - size_mb = os.path.getsize(backup_file) / (1024 * 1024) - print(f"Uploading {filename} ({size_mb:.1f} MB)") - - s3_ensure_bucket() - - with open(backup_file, "rb") as f: - data = f.read() - - # Upload daily - s3_put(f"daily/{filename}", data) - print(f" Uploaded daily/{filename}") - - # On Sundays, also upload weekly - if datetime.now(timezone.utc).isoweekday() == 7: - s3_put(f"weekly/{filename}", data) - print(f" Uploaded weekly/{filename}") - - # Prune old dailies (keep 7) - dailies = s3_list("daily/") - if len(dailies) > 7: - for old in dailies[:len(dailies) - 7]: - s3_delete(old) - print(f" Pruned {old}") - - # Prune old weeklies (keep 4) - weeklies = s3_list("weekly/") - if len(weeklies) > 4: - for old in weeklies[:len(weeklies) - 4]: - s3_delete(old) - print(f" Pruned {old}") - - print("=== Backup upload complete ===") ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: postgres-backup - namespace: foxhunt - labels: - app.kubernetes.io/name: postgres-backup - app.kubernetes.io/part-of: foxhunt -spec: - schedule: "30 2 * * *" - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 2 - activeDeadlineSeconds: 600 - template: - metadata: - labels: - app: postgres-backup - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - restartPolicy: OnFailure - # Step 1: pg_dump in initContainer → shared /backup volume - initContainers: - - name: pg-dump - image: postgres:16-alpine - command: - - sh - - -c - - | - TIMESTAMP=$(date -u +%Y%m%d-%H%M%S) - echo "=== Postgres Backup ${TIMESTAMP} ===" - PGPASSWORD="${POSTGRES_PASSWORD}" pg_dumpall \ - -h "${POSTGRES_HOST}" \ - -p 5432 \ - -U "${POSTGRES_USER}" \ - --clean --if-exists \ - | gzip > "/backup/foxhunt-${TIMESTAMP}.sql.gz" - ls -lh /backup/ - echo "Dump complete" - env: - - name: POSTGRES_HOST - value: "postgres.foxhunt.svc.cluster.local" - - name: POSTGRES_USER - value: "foxhunt" - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - volumeMounts: - - name: backup - mountPath: /backup - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi - # Step 2: Upload to MinIO in main container (python cached on nodes) - containers: - - name: upload - image: python:3.12-alpine - command: ["python3", "-u", "/scripts/upload.py"] - env: - - name: MINIO_ENDPOINT - value: "http://minio.foxhunt.svc.cluster.local:9000" - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: backup - mountPath: /backup - - name: script - mountPath: /scripts - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - volumes: - - name: backup - emptyDir: - sizeLimit: 2Gi - - name: script - configMap: - name: postgres-backup-script - items: - - key: upload.py - path: upload.py - defaultMode: 0755 diff --git a/infra/k8s/databases/postgres.yaml b/infra/k8s/databases/postgres.yaml deleted file mode 100644 index faa17ddcb..000000000 --- a/infra/k8s/databases/postgres.yaml +++ /dev/null @@ -1,96 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgres-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: postgres - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres - namespace: foxhunt - labels: - app.kubernetes.io/name: postgres - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: postgres - strategy: - type: Recreate - template: - metadata: - labels: - app.kubernetes.io/name: postgres - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: infra - containers: - - name: postgres - image: timescale/timescaledb:latest-pg16 - ports: - - containerPort: 5432 - name: postgres - args: - - -c - - max_connections=100 - env: - - name: POSTGRES_DB - value: foxhunt - - name: POSTGRES_USER - value: foxhunt - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - volumeMounts: - - name: postgres-data - mountPath: /var/lib/postgresql/data - subPath: pgdata - readinessProbe: - exec: - command: - - pg_isready - - -U - - foxhunt - initialDelaySeconds: 10 - periodSeconds: 10 - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: 500m - memory: 1Gi - volumes: - - name: postgres-data - persistentVolumeClaim: - claimName: postgres-pvc ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres - namespace: foxhunt - labels: - app.kubernetes.io/name: postgres - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: postgres - ports: - - port: 5432 - targetPort: 5432 - name: postgres diff --git a/infra/k8s/databases/questdb.yaml b/infra/k8s/databases/questdb.yaml deleted file mode 100644 index 5924722c0..000000000 --- a/infra/k8s/databases/questdb.yaml +++ /dev/null @@ -1,95 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: questdb-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: questdb - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: questdb - namespace: foxhunt - labels: - app.kubernetes.io/name: questdb - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: questdb - strategy: - type: Recreate - template: - metadata: - labels: - app.kubernetes.io/name: questdb - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: questdb - image: questdb/questdb:8.2.3 - ports: - - containerPort: 9009 - name: ilp - - containerPort: 8812 - name: pg - - containerPort: 9003 - name: metrics - env: - - name: QDB_PG_ENABLED - value: "true" - - name: QDB_METRICS_ENABLED - value: "TRUE" - volumeMounts: - - name: questdb-data - mountPath: /var/lib/questdb - readinessProbe: - httpGet: - path: /exec?query=SELECT%201 - port: 9003 - initialDelaySeconds: 10 - periodSeconds: 10 - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: 500m - memory: 1Gi - volumes: - - name: questdb-data - persistentVolumeClaim: - claimName: questdb-pvc ---- -apiVersion: v1 -kind: Service -metadata: - name: questdb - namespace: foxhunt - labels: - app.kubernetes.io/name: questdb - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: questdb - ports: - - port: 9009 - targetPort: 9009 - name: ilp - - port: 8812 - targetPort: 8812 - name: pg - - port: 9003 - targetPort: 9003 - name: metrics diff --git a/infra/k8s/databases/redis.yaml b/infra/k8s/databases/redis.yaml deleted file mode 100644 index 5f2716f7b..000000000 --- a/infra/k8s/databases/redis.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: redis - namespace: foxhunt - labels: - app.kubernetes.io/name: redis - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: redis - template: - metadata: - labels: - app.kubernetes.io/name: redis - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: infra - containers: - - name: redis - image: redis:7-alpine - command: - - redis-server - - --maxmemory - - 512mb - - --maxmemory-policy - - allkeys-lru - ports: - - containerPort: 6379 - name: redis - readinessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 250m - memory: 640Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: foxhunt - labels: - app.kubernetes.io/name: redis - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: redis - ports: - - port: 6379 - targetPort: 6379 - name: redis diff --git a/infra/k8s/dns/node-local-dns.yaml b/infra/k8s/dns/node-local-dns.yaml deleted file mode 100644 index 8ebb9bcb1..000000000 --- a/infra/k8s/dns/node-local-dns.yaml +++ /dev/null @@ -1,182 +0,0 @@ -# Node Bootstrap DaemonSet for Kapsule -# -# Two problems on Kapsule nodes: -# 1. Nodes have nameserver 127.0.53.53 which can't resolve .svc.cluster.local -# 2. Containerd defaults to HTTPS for non-localhost registries -# -# Fixes: -# 1. Point /etc/resolv.conf to kube-dns ClusterIP (10.32.0.10) -# 2. Create /etc/containerd/certs.d/ config for HTTP GitLab registry -# 3. On GPU nodes (containerd v3 with conf.d/), add registry drop-in -# -# Init container sets up containerd registry config (one-time). -# Main container manages resolv.conf (watches for Kapsule resets). ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: node-bootstrap - namespace: kube-system - labels: - app.kubernetes.io/name: node-bootstrap -spec: - selector: - matchLabels: - app.kubernetes.io/name: node-bootstrap - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - app.kubernetes.io/name: node-bootstrap - spec: - hostNetwork: true - dnsPolicy: Default - priorityClassName: system-node-critical - tolerations: - - operator: Exists - hostPID: true - initContainers: - # Configure containerd to use HTTP for the GitLab registry. - # On GPU nodes (NVIDIA overlay), also creates a v3-compatible conf.d drop-in - # and restarts containerd so it picks up the new config. - - name: setup-registry - image: alpine:3.19 - securityContext: - privileged: true - command: ["/bin/sh", "-c"] - args: - - | - set -e - NEEDS_RESTART=false - - # 1. Create per-host containerd registry config (HTTP for GitLab) - REGISTRY_DIR="/host-etc/containerd/certs.d/gitlab-registry.foxhunt.svc.cluster.local:5000" - mkdir -p "$REGISTRY_DIR" - DESIRED='server = "http://gitlab-registry.foxhunt.svc.cluster.local:5000" - - [host."http://gitlab-registry.foxhunt.svc.cluster.local:5000"] - capabilities = ["pull", "resolve"]' - DESIRED=$(printf '%s' "$DESIRED" | sed 's/^ //') - if [ ! -f "$REGISTRY_DIR/hosts.toml" ] || [ "$(cat "$REGISTRY_DIR/hosts.toml")" != "$DESIRED" ]; then - printf '%s\n' "$DESIRED" > "$REGISTRY_DIR/hosts.toml" - echo "Created hosts.toml for HTTP gitlab-registry" - NEEDS_RESTART=true - else - echo "hosts.toml already up to date" - fi - - # 2. Ensure conf.d/ exists and always write the v3 registry drop-in. - # containerd 2.x may ignore the v2-style grpc.v1.cri.registry.config_path - # even when it's set in config.toml. The v1 images drop-in is the reliable path. - # Create conf.d/ if missing (base config already has imports = ["conf.d/*.toml"]). - CONF_D="/host-etc/containerd/conf.d" - mkdir -p "$CONF_D" - if [ -d "$CONF_D" ]; then - DROPIN='version = 3 - - [plugins."io.containerd.cri.v1.images".registry] - config_path = "/etc/containerd/certs.d"' - DROPIN=$(printf '%s' "$DROPIN" | sed 's/^ //') - if [ ! -f "$CONF_D/50-registry.toml" ] || [ "$(cat "$CONF_D/50-registry.toml")" != "$DROPIN" ]; then - printf '%s\n' "$DROPIN" > "$CONF_D/50-registry.toml" - echo "GPU node: created v3 registry config drop-in" - NEEDS_RESTART=true - else - echo "GPU node: v3 drop-in already up to date" - fi - fi - - # 3. Restart containerd only if we changed config files. - # Uses /proc/1/root chroot to access host systemctl (requires hostPID + privileged). - # On a new node this will kill the DaemonSet pod; kubelet recreates it, - # second run finds files unchanged (NEEDS_RESTART=false), proceeds normally. - if [ "$NEEDS_RESTART" = "true" ]; then - echo "Restarting containerd to pick up config changes..." - chroot /proc/1/root systemctl restart containerd || \ - echo "WARNING: containerd restart failed (node may need manual restart)" - echo "containerd restart issued" - fi - resources: - requests: - cpu: 10m - memory: 8Mi - limits: - cpu: 50m - memory: 16Mi - volumeMounts: - - name: host-etc - mountPath: /host-etc - containers: - - name: resolv-manager - image: busybox:1.37 - command: ["/bin/sh", "-c"] - args: - - | - set -e - RESOLV="/host-etc/resolv.conf" - BACKUP="/host-etc/resolv.conf.pre-kube-dns" - KUBE_DNS="10.32.0.10" - - restore() { - if [ -f "$BACKUP" ]; then - cp "$BACKUP" "$RESOLV" - echo "$(date): Restored original resolv.conf" - fi - exit 0 - } - trap restore TERM INT - - # Save original - if [ ! -f "$BACKUP" ]; then - cp "$RESOLV" "$BACKUP" - echo "Saved original: $(cat "$BACKUP")" - fi - - apply() { - # CRITICAL: keep VPC upstream resolver as FALLBACK after kube-dns. - # If resolv.conf ONLY has 10.32.0.10, node DNS is dead when CoreDNS - # is down — containerd can't pull images, CoreDNS can't start, - # entire cluster deadlocks (incident 2026-03-18, 2026-04-12). - # Scaleway VPC gateway pushes 172.16.0.13 as DNS via DHCP. - UPSTREAM="" - if [ -f "$BACKUP" ]; then - UPSTREAM=$(grep "^nameserver" "$BACKUP" | grep -v "127.0.53.53\|$KUBE_DNS" | head -1 | awk '{print $2}') - fi - if [ -z "$UPSTREAM" ]; then - UPSTREAM="169.254.169.254" # Scaleway metadata DNS fallback - fi - printf 'nameserver %s\nnameserver %s\nsearch foxhunt.svc.cluster.local svc.cluster.local cluster.local\noptions ndots:5 timeout:2 attempts:3\n' "$KUBE_DNS" "$UPSTREAM" > "$RESOLV" - echo "$(date): resolv.conf → $KUBE_DNS + $UPSTREAM (fallback)" - } - - # Initial apply - apply - - # Watch for resets - while true; do - sleep 30 & - wait $! - if ! grep -q "$KUBE_DNS" "$RESOLV" 2>/dev/null; then - echo "$(date): resolv.conf was reset, re-applying" - apply - fi - done - resources: - requests: - cpu: 5m - memory: 8Mi - limits: - cpu: 10m - memory: 16Mi - volumeMounts: - - name: host-etc - mountPath: /host-etc - terminationGracePeriodSeconds: 10 - volumes: - - name: host-etc - hostPath: - path: /etc - type: Directory diff --git a/infra/k8s/gitea/clusterip-services.yaml b/infra/k8s/gitea/clusterip-services.yaml deleted file mode 100644 index 6056b2eb4..000000000 --- a/infra/k8s/gitea/clusterip-services.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# The gitea chart's gitea-http/gitea-ssh services are HEADLESS (clusterIP: None), so they resolve to the -# pod IP (100.64.x.x). That range OVERLAPS the Tailscale CGNAT range (100.64.0.0/10), so the tailscale -# sidecar in the tailscale-gitlab-proxy pod swallows traffic to the pod IP. These normal ClusterIP services -# give Gitea a service IP in the service CIDR (10.32.x.x, outside the tailscale range) — the proxy nginx + -# socat target these instead. (GitLab worked because its webservice svc was a normal ClusterIP.) -apiVersion: v1 -kind: Service -metadata: - name: gitea-web - namespace: foxhunt - labels: - app.kubernetes.io/name: gitea -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: gitea - app.kubernetes.io/instance: gitea - ports: - - name: http - port: 3000 - targetPort: 3000 - protocol: TCP ---- -apiVersion: v1 -kind: Service -metadata: - name: gitea-sshd - namespace: foxhunt - labels: - app.kubernetes.io/name: gitea -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: gitea - app.kubernetes.io/instance: gitea - ports: - - name: ssh - port: 22 - targetPort: 2222 # rootless gitea listens on 2222 internally (can't bind privileged 22) - protocol: TCP diff --git a/infra/k8s/gitea/networkpolicy.yaml b/infra/k8s/gitea/networkpolicy.yaml deleted file mode 100644 index 0cef92109..000000000 --- a/infra/k8s/gitea/networkpolicy.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Gitea NetworkPolicy (foxhunt ns has default-deny-all). Egress: postgres + DNS + in-cluster webhook -# target + general HTTPS. Ingress: from platform/argo pods + the tailscale proxy → :3000 (web) and :22 (ssh). -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: gitea - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: gitea - policyTypes: - - Ingress - - Egress - ingress: - # web (3000) + ssh (22) from platform + argo-workflow pods (cockpit build clones over HTTP) - - from: - - podSelector: - matchExpressions: - - key: app.kubernetes.io/part-of - operator: In - values: ["foxhunt", "argo-workflows"] - ports: - - { port: 3000, protocol: TCP } - - { port: 2222, protocol: TCP } - # web + ssh from the tailscale proxy (nginx + socat) that fronts git.fxhnt.ai - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: tailscale-gitlab-proxy - ports: - - { port: 3000, protocol: TCP } - - { port: 2222, protocol: TCP } - egress: - # PostgreSQL - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - { port: 5432, protocol: TCP } - # DNS - - ports: - - { port: 53, protocol: UDP } - - { port: 53, protocol: TCP } - # Argo Events webhook target (gitea push hook → gitea-push eventsource :12000). The eventsource pod is - # not part-of:foxhunt (stays default-allow), so allow egress to any in-namespace pod on 12000. - - to: - - podSelector: {} - ports: - - { port: 12000, protocol: TCP } - # general HTTPS (avatars, external clones) — external only, exclude cluster CIDRs - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: ["10.32.0.0/16", "172.16.0.0/16"] - ports: - - { port: 443, protocol: TCP } diff --git a/infra/k8s/gitea/values.yaml b/infra/k8s/gitea/values.yaml deleted file mode 100644 index 13f6775b5..000000000 --- a/infra/k8s/gitea/values.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# Gitea — lightweight git host replacing GitLab (Phase 2B). External Postgres (existing in-cluster -# `postgres`), no bundled DB/redis/memcached, Actions off (Argo does CI). Internal-only until cutover. -replicaCount: 1 -image: - rootless: true - -# RWO PVC (sbs-default-retain) → only one pod can mount it. RollingUpdate deadlocks (new pod can't -# attach while old holds it). Recreate terminates old before starting new. -strategy: - type: Recreate - -# Disable all bundled subcharts — reuse the existing in-cluster postgres, no redis/memcached. -postgresql: - enabled: false -postgresql-ha: - enabled: false -redis-cluster: - enabled: false -redis: - enabled: false -# chart 12.x replaced redis with valkey — disable both, gitea uses embedded queue/cache/session -valkey-cluster: - enabled: false -valkey: - enabled: false - -persistence: - enabled: true - size: 5Gi - storageClass: sbs-default-retain - -resources: - requests: { cpu: 100m, memory: 256Mi } - limits: { cpu: "1", memory: 2Gi } # 2Gi headroom: indexing the 876MB foxhunt pack OOMed at 512Mi - -service: - http: { type: ClusterIP, port: 3000 } - ssh: { type: ClusterIP, port: 22 } - -gitea: - admin: - existingSecret: gitea-admin - config: - actions: - ENABLED: "false" # Argo does CI/CD; Gitea Actions off (subchart removed in chart 12.x) - server: - ROOT_URL: https://git.fxhnt.ai/ - DOMAIN: git.fxhnt.ai - SSH_DOMAIN: git.fxhnt.ai - SSH_PORT: "22" # clean git@git.fxhnt.ai clone URLs; :2222 retired (Task 5). - DISABLE_SSH: "false" - database: - DB_TYPE: postgres - HOST: postgres.foxhunt.svc.cluster.local:5432 - NAME: gitea - USER: gitea - service: - DISABLE_REGISTRATION: "true" - webhook: - # allow delivery to in-cluster targets (Argo Events eventsource on the 10.32.x service CIDR); - # default blocks private hosts. Argo auto-deploy webhook needs this. - ALLOWED_HOST_LIST: "private" - # No redis/memcached — use embedded adapters - cache: - ADAPTER: memory - session: - PROVIDER: db - queue: - TYPE: level - additionalConfigFromEnvs: - - name: GITEA__database__PASSWD - valueFrom: - secretKeyRef: { name: gitea-db, key: password } diff --git a/infra/k8s/gitlab/grafana-values.yaml b/infra/k8s/gitlab/grafana-values.yaml deleted file mode 100644 index c5278278e..000000000 --- a/infra/k8s/gitlab/grafana-values.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# Grafana — Foxhunt monitoring dashboard -# Accessed via Tailscale proxy on port 3000 - -replicas: 1 - -nodeSelector: - k8s.scaleway.com/pool-name: platform - -resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - -persistence: - enabled: false - -admin: - existingSecret: grafana-admin - userKey: admin-user - passwordKey: admin-password - -envFromSecrets: - - name: db-credentials - -grafana.ini: - server: - root_url: https://grafana.fxhnt.ai - unified_alerting: - enabled: true - alerting: - enabled: false - database: - type: postgres - host: postgres:5432 - name: grafana - user: foxhunt - password: $__env{password} - ssl_mode: disable - -datasources: - datasources.yaml: - apiVersion: 1 - datasources: - - name: Prometheus - type: prometheus - url: http://prometheus-stack-kube-prom-prometheus.foxhunt.svc.cluster.local:9090 - access: proxy - isDefault: true - - name: Alertmanager - type: alertmanager - uid: alertmanager - access: proxy - url: http://prometheus-stack-kube-prom-alertmanager.foxhunt.svc.cluster.local:9093 - isDefault: false - jsonData: - implementation: prometheus - handleGrafanaManagedAlerts: true - - name: Loki - type: loki - access: proxy - url: http://loki.foxhunt.svc.cluster.local:3100 - isDefault: false - jsonData: - derivedFields: - - datasourceUid: tempo - matcherRegex: '"trace_id":"(\w+)"' - name: TraceID - url: "$${__value.raw}" - - name: Tempo - type: tempo - uid: tempo - access: proxy - url: http://tempo.foxhunt.svc.cluster.local:3200 - isDefault: false - jsonData: - tracesToLogsV2: - datasourceUid: loki - filterByTraceID: true - nodeGraph: - enabled: true - serviceMap: - datasourceUid: prometheus - -sidecar: - dashboards: - enabled: true - folder: /tmp/dashboards - folderAnnotation: grafana_folder - label: grafana_dashboard - labelValue: "1" - searchNamespace: foxhunt - provider: - foldersFromFilesStructure: true - datasources: - enabled: false - label: grafana_datasource - labelValue: "" - -dashboardProviders: - dashboardproviders.yaml: - apiVersion: 1 - providers: - - name: default - orgId: 1 - folder: Foxhunt - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards/default - -dashboards: - default: - node-exporter: - gnetId: 1860 - revision: 37 - datasource: Prometheus diff --git a/infra/k8s/gitlab/tailscale-proxy.yaml b/infra/k8s/gitlab/tailscale-proxy.yaml deleted file mode 100644 index ed9c4bb3b..000000000 --- a/infra/k8s/gitlab/tailscale-proxy.yaml +++ /dev/null @@ -1,325 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: tailscale-gitlab - namespace: foxhunt ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: tailscale-gitlab - namespace: foxhunt -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "get", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: tailscale-gitlab - namespace: foxhunt -subjects: - - kind: ServiceAccount - name: tailscale-gitlab - namespace: foxhunt -roleRef: - kind: Role - name: tailscale-gitlab - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tailscale-gitlab-proxy - namespace: foxhunt - labels: - app.kubernetes.io/name: tailscale-gitlab-proxy - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: tailscale-gitlab-proxy - template: - metadata: - labels: - app.kubernetes.io/name: tailscale-gitlab-proxy - spec: - serviceAccountName: tailscale-gitlab - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: nginx - image: nginx:alpine - ports: - - containerPort: 80 - - containerPort: 443 - volumeMounts: - - name: nginx-conf - mountPath: /etc/nginx/conf.d - - name: tls-certs - mountPath: /etc/nginx/certs - readOnly: true - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - - name: ssh-proxy - image: alpine/socat:latest - args: - - "TCP-LISTEN:22,fork,reuseaddr,sndbuf=1048576,rcvbuf=1048576" - - "TCP:gitea-sshd.foxhunt.svc.cluster.local:22,sndbuf=1048576,rcvbuf=1048576" - ports: - - containerPort: 22 - resources: - requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi - - name: imap-proxy - image: alpine/socat:latest - args: - - "TCP-LISTEN:993,fork,reuseaddr" - - "TCP:stalwart.foxhunt.svc.cluster.local:993" - ports: - - containerPort: 993 - resources: - requests: - cpu: 10m - memory: 16Mi - limits: - cpu: 50m - memory: 32Mi - - name: tailscale - image: ghcr.io/tailscale/tailscale:latest - env: - - name: TS_AUTHKEY - valueFrom: - secretKeyRef: - name: tailscale-auth - key: TS_AUTHKEY - - name: TS_KUBE_SECRET - value: tailscale-gitlab-state - - name: TS_USERSPACE - value: "false" - - name: TS_HOSTNAME - value: "foxhunt-gitlab" - - name: TS_ACCEPT_DNS - value: "false" - - name: TS_EXTRA_ARGS - value: "" - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumes: - - name: nginx-conf - configMap: - name: tailscale-gitlab-nginx - - name: tls-certs - secret: - secretName: gitlab-tls-cert ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: tailscale-gitlab-nginx - namespace: foxhunt -data: - default.conf: | - # HTTP → HTTPS redirect for all subdomains - server { - listen 80; - server_name *.fxhnt.ai; - return 301 https://$host$request_uri; - } - - # Gitea — git.fxhnt.ai (replaced GitLab, Phase 2B cutover) - server { - listen 443 ssl; - server_name git.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - client_max_body_size 0; - - location / { - proxy_pass http://gitea-web.foxhunt.svc.cluster.local:3000; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - - proxy_buffering off; - proxy_request_buffering off; - proxy_http_version 1.1; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - proxy_connect_timeout 300s; - } - } - - # Grafana — grafana.fxhnt.ai - server { - listen 443 ssl; - server_name grafana.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - location / { - proxy_pass http://grafana.foxhunt.svc.cluster.local:80; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - } - - # fxhnt cockpit — dashboard.fxhnt.ai - # Proxies to the cockpit's Tailscale node (peer-to-peer over the tailnet) rather than the cluster Service: - # the pod CIDR 100.64.0.0/15 overlaps Tailscale CGNAT, so this kernel-mode proxy can't reach platform-pool - # pods via ClusterIP, but it CAN reach another tailnet node. IP is stable while the cockpit's TS state - # secret (fxhnt-dashboard-ts-state) persists. (Was web-dashboard.foxhunt.svc — replaced by the fxhnt cockpit.) - server { - listen 443 ssl; - server_name dashboard.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - location / { - proxy_pass http://100.81.150.18:80; # fxhnt-dashboard tailnet node (TCP:80 -> cockpit :8080) - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - } - - # API Gateway (gRPC-Web + gRPC) — api.fxhnt.ai - # Browser dashboard uses gRPC-web (HTTP/1.1 POST), so proxy_pass not grpc_pass - server { - listen 443 ssl; - http2 on; - server_name api.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - location / { - proxy_pass http://api.foxhunt.svc.cluster.local:50051; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_http_version 1.1; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - proxy_connect_timeout 30s; - } - } - - # Stalwart Admin — mail.fxhnt.ai - server { - listen 443 ssl; - server_name mail.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - location / { - proxy_pass https://stalwart.foxhunt.svc.cluster.local:443; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_http_version 1.1; - proxy_ssl_verify off; - } - } - - # Mattermost — chat.fxhnt.ai - server { - listen 443 ssl; - server_name chat.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - client_max_body_size 50m; - - location / { - proxy_pass http://mattermost.foxhunt.svc.cluster.local:8065; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - } - } - - # MinIO S3 API — minio.fxhnt.ai - server { - listen 443 ssl; - server_name minio.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - client_max_body_size 0; - - location / { - proxy_pass http://minio.foxhunt.svc.cluster.local:9000; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - - proxy_buffering off; - proxy_request_buffering off; - proxy_http_version 1.1; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - proxy_connect_timeout 300s; - - # MinIO uses self-signed TLS internally - proxy_ssl_verify off; - } - } - diff --git a/infra/k8s/jobs/migrate.yaml b/infra/k8s/jobs/migrate.yaml deleted file mode 100644 index ad6f12da4..000000000 --- a/infra/k8s/jobs/migrate.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: foxhunt-migrate - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: migration -spec: - backoffLimit: 3 - template: - spec: - restartPolicy: Never - imagePullSecrets: - - name: gitlab-registry - containers: - - name: migrate - image: rg.fr-par.scw.cloud/foxhunt/api:latest - command: ["sqlx", "migrate", "run"] - env: - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DB_PASSWORD)@postgres:5432/foxhunt" - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "200m" diff --git a/infra/k8s/mattermost/mattermost.yaml b/infra/k8s/mattermost/mattermost.yaml deleted file mode 100644 index b7a67babb..000000000 --- a/infra/k8s/mattermost/mattermost.yaml +++ /dev/null @@ -1,121 +0,0 @@ -# Mattermost Team Edition — self-hosted Slack alternative for foxhunt. -# Runs on platform nodes. PostgreSQL is the shared foxhunt instance. -# Access: https://chat.fxhnt.ai (Tailscale only) ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mattermost-data - namespace: foxhunt - labels: - app.kubernetes.io/name: mattermost - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: [ReadWriteOnce] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mattermost - namespace: foxhunt - labels: - app.kubernetes.io/name: mattermost - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: mattermost - strategy: - type: Recreate # RWO PVC - template: - metadata: - labels: - app.kubernetes.io/name: mattermost - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: mattermost - image: mattermost/mattermost-team-edition:10.5 - ports: - - containerPort: 8065 - name: http - env: - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: MM_SQLSETTINGS_DRIVERNAME - value: postgres - - name: MM_SQLSETTINGS_DATASOURCE - value: "postgres://foxhunt:$(DB_PASSWORD)@postgres.foxhunt.svc.cluster.local:5432/mattermost?sslmode=disable" - - name: MM_SERVICESETTINGS_SITEURL - value: "https://chat.fxhnt.ai" - - name: MM_SERVICESETTINGS_LISTENADDRESS - value: ":8065" - - name: MM_SERVICESETTINGS_ENABLEINCOMINGWEBHOOKS - value: "true" - - name: MM_SERVICESETTINGS_ENABLEPOSTUSERNAMEOVERRIDE - value: "true" - - name: MM_SERVICESETTINGS_ENABLEPOSTICONOVERRIDE - value: "true" - - name: MM_LOGSETTINGS_ENABLECONSOLE - value: "true" - - name: MM_LOGSETTINGS_CONSOLELEVEL - value: INFO - - name: MM_LOGSETTINGS_ENABLEFILE - value: "false" - - name: MM_FILESETTINGS_DRIVERNAME - value: local - - name: MM_FILESETTINGS_DIRECTORY - value: /mattermost/data - # Email via Stalwart — uncomment after Stalwart is deployed: - # - name: MM_EMAILSETTINGS_SMTPSERVER - # value: stalwart.foxhunt.svc.cluster.local - # - name: MM_EMAILSETTINGS_SMTPPORT - # value: "25" - # - name: MM_EMAILSETTINGS_SENDEMAILNOTIFICATIONS - # value: "true" - volumeMounts: - - name: mattermost-data - mountPath: /mattermost/data - readinessProbe: - httpGet: - path: /api/v4/system/ping - port: 8065 - initialDelaySeconds: 15 - periodSeconds: 10 - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: "1" - memory: 1Gi - volumes: - - name: mattermost-data - persistentVolumeClaim: - claimName: mattermost-data ---- -apiVersion: v1 -kind: Service -metadata: - name: mattermost - namespace: foxhunt - labels: - app.kubernetes.io/name: mattermost - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: mattermost - ports: - - port: 8065 - targetPort: 8065 - name: http diff --git a/infra/k8s/minio/kustomization.yaml b/infra/k8s/minio/kustomization.yaml deleted file mode 100644 index 6a5695f9c..000000000 --- a/infra/k8s/minio/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - minio.yaml diff --git a/infra/k8s/minio/minio.yaml b/infra/k8s/minio/minio.yaml deleted file mode 100644 index 6597ec095..000000000 --- a/infra/k8s/minio/minio.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# MinIO — in-cluster S3 for compiled binaries and trained models -# Replaces Scaleway public S3 to keep proprietary code/IP in-cluster -# Plain HTTP only — internal cluster traffic, no TLS needed -apiVersion: v1 -kind: Secret -metadata: - name: minio-credentials - namespace: foxhunt - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - root-user: foxhunt-minio - root-password: CHANGE_ME_BEFORE_DEPLOY - # rclone-compatible keys (same values, different key names for initContainers) - access-key: foxhunt-minio - secret-key: CHANGE_ME_BEFORE_DEPLOY ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: minio-data-new - namespace: foxhunt - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: - - ReadWriteOnce - storageClassName: scw-bssd - resources: - requests: - storage: 150Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: foxhunt - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - strategy: - type: Recreate # single-node MinIO — no rolling update - selector: - matchLabels: - app.kubernetes.io/name: minio - template: - metadata: - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: minio - image: minio/minio:latest - args: ["server", "/data", "--console-address", ":9001"] - ports: - - containerPort: 9000 - name: api - - containerPort: 9001 - name: console - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio-credentials - key: root-user - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio-credentials - key: root-password - volumeMounts: - - name: data - mountPath: /data - readinessProbe: - httpGet: - path: /minio/health/live - port: 9000 - initialDelaySeconds: 5 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /minio/health/live - port: 9000 - initialDelaySeconds: 10 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: "1" - memory: 2Gi - volumes: - - name: data - persistentVolumeClaim: - claimName: minio-data-new ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: foxhunt - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: minio - ports: - - port: 9000 - targetPort: 9000 - name: api - - port: 9001 - targetPort: 9001 - name: console ---- -# Create buckets on first deploy -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-init-buckets - namespace: foxhunt - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/part-of: foxhunt -spec: - backoffLimit: 5 - ttlSecondsAfterFinished: 300 - template: - metadata: - labels: - app.kubernetes.io/name: minio-init - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - restartPolicy: OnFailure - containers: - - name: mc - image: minio/mc:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - echo "Waiting for MinIO to be ready..." - until mc alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do - echo "MinIO not ready, retrying in 3s..." - sleep 3 - done - echo "Creating buckets..." - mc mb --ignore-existing foxhunt/foxhunt-binaries - mc mb --ignore-existing foxhunt/foxhunt-models - mc mb --ignore-existing foxhunt/foxhunt-training-results - mc mb --ignore-existing foxhunt/argo-logs - mc mb --ignore-existing foxhunt/foxhunt-sccache - mc mb --ignore-existing foxhunt/foxhunt-training-data - # Plan 5 Task 3 (A.4.1): nsys profile artefacts bucket. - # Stores .nsys-rep files keyed by commit SHA under - # profiles//profile-seed*-fold*-*.nsys-rep. - mc mb --ignore-existing foxhunt/foxhunt-training-artifacts - echo "Buckets created:" - mc ls foxhunt/ - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio-credentials - key: root-user - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio-credentials - key: root-password - resources: - requests: - cpu: 50m - memory: 32Mi - limits: - cpu: 200m - memory: 64Mi diff --git a/infra/k8s/monitoring/APPLY-ORDER.md b/infra/k8s/monitoring/APPLY-ORDER.md deleted file mode 100644 index 167391e85..000000000 --- a/infra/k8s/monitoring/APPLY-ORDER.md +++ /dev/null @@ -1,104 +0,0 @@ -# Cockpit + NAT Gateway — Apply Order - -## Prerequisites -- `scw` CLI authenticated -- `terragrunt` installed -- `kubectl` access to foxhunt cluster -- `helm` installed - -## Phase 1: Cockpit (metrics backend) - -1. Apply Cockpit Terraform module: - ```bash - cd infra/live/production/cockpit - terragrunt apply - ``` - -2. Note the outputs: - ```bash - terragrunt output metrics_push_url - terragrunt output -raw push_token - terragrunt output grafana_url - terragrunt output -raw grafana_password - ``` - -3. Create the K8s secret for the push token: - ```bash - kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - - kubectl -n monitoring create secret generic cockpit-push-token \ - --from-literal=token=$(cd infra/live/production/cockpit && terragrunt output -raw push_token) - ``` - -4. Update placeholders in `infra/k8s/monitoring/alloy-values.yaml`: - - Replace `COCKPIT_METRICS_PUSH_URL` with the `metrics_push_url` output - -5. Deploy DCGM Exporter: - ```bash - kubectl apply -f infra/k8s/monitoring/dcgm-exporter.yaml - ``` - -6. Deploy Grafana Alloy (k8s-monitoring): - ```bash - helm repo add grafana https://grafana.github.io/helm-charts - helm repo update - helm install k8s-monitoring grafana/k8s-monitoring \ - --namespace monitoring \ - -f infra/k8s/monitoring/alloy-values.yaml - ``` - -7. Verify metrics flow: - - Open Cockpit Grafana (URL from step 2) - - Login with foxhunt-admin / (password from step 2) - - Check "Kubernetes / Compute Resources / Cluster" dashboard - -## Phase 2: GitLab Prometheus remote_write - -1. Update placeholders in `infra/k8s/gitlab/values.yaml`: - - Replace `COCKPIT_METRICS_PUSH_URL` and `COCKPIT_PUSH_TOKEN` - -2. Upgrade GitLab Helm release: - ```bash - helm upgrade gitlab gitlab/gitlab \ - --namespace foxhunt \ - -f infra/k8s/gitlab/values.yaml - ``` - -## Phase 3: NAT Gateway - -1. Apply Public Gateway module: - ```bash - cd infra/live/production/public-gateway - terragrunt apply - ``` - This creates the gateway and attaches it to the private network. - Existing nodes still have public IPs — nothing breaks yet. - -2. Apply Kapsule pool changes (disables public IPs): - ```bash - cd infra/live/production/kapsule - terragrunt apply - ``` - **WARNING**: This may recreate nodes. For zero-downtime: - -3. Zero-downtime node migration for services pool: - ```bash - # Temporarily allow 2 nodes - scw k8s pool update max-size=2 size=2 - # Wait for new node (private IP) to be Ready - kubectl get nodes -w - # Drain old node - kubectl drain --ignore-daemonsets --delete-emptydir-data - # Scale back - scw k8s pool update max-size=1 size=1 - ``` - -4. Repeat for gitlab pool (if not autoscaling, same process). - -5. Autoscaling pools (ci-training, gpu-dev): - - No action needed — next scale-up automatically uses private IP. - -6. Verify Tailscale still works: - ```bash - tailscale status # should show foxhunt-kapsule and foxhunt-gitlab nodes - curl -k https://git.fxhnt.ai # should still resolve via Tailscale - ``` diff --git a/infra/k8s/monitoring/alloy-values.yaml b/infra/k8s/monitoring/alloy-values.yaml deleted file mode 100644 index c8e68262b..000000000 --- a/infra/k8s/monitoring/alloy-values.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# Grafana k8s-monitoring Helm chart values (v1.x format) -# Chart: grafana/k8s-monitoring --version 1.6.50 -# Deploys: Alloy (DaemonSet) + node-exporter + kube-state-metrics -# -# Install: -# helm repo add grafana https://grafana.github.io/helm-charts -# PUSH_TOKEN=$(cd infra/live/production/cockpit && terragrunt output -raw push_token) -# helm install k8s-monitoring grafana/k8s-monitoring \ -# --namespace monitoring --version 1.6.50 \ -# -f infra/k8s/monitoring/alloy-values.yaml \ -# --set "externalServices.prometheus.extraHeaders.X-Token=${PUSH_TOKEN}" - -cluster: - name: foxhunt - -externalServices: - prometheus: - host: "https://d8cd4c5d-6d3b-4258-9f92-2ad0eaa36ba0.metrics.cockpit.fr-par.scw.cloud" - writeEndpoint: "/api/v1/push" - authMode: none - # X-Token header passed via --set to keep secret out of git - # Loki host required by chart validation even with logs disabled - loki: - host: "https://noop.localhost" - authMode: none - -metrics: - enabled: true - cost: - enabled: false - node-exporter: - enabled: true - kube-state-metrics: - enabled: true - kubelet: - enabled: true - cadvisor: - enabled: true - apiserver: - enabled: true - -# Alloy DaemonSet — runs on every node -alloy-metrics: - alloy: - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - -# node-exporter — host metrics (CPU, memory, disk, network) -prometheus-node-exporter: - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - -# kube-state-metrics — K8s object state -kube-state-metrics: - resources: - requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - # Scheduled on gitlab node (GP1-XS 16GB) — services (DEV1-M 4GB) too small - -# Disable components we don't need -logs: - enabled: false - cluster_events: - enabled: false -traces: - enabled: false -opencost: - enabled: false - -# Disable log/event collectors -alloy-logs: - enabled: false -alloy-events: - enabled: false diff --git a/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json deleted file mode 100644 index a5f24f455..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json +++ /dev/null @@ -1,809 +0,0 @@ -{ - "description": "Loki-powered CI pipeline log viewer \u2014 test gate, clippy, builds, errors", - "editable": true, - "graphTooltip": 2, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "title": "Pipeline Overview", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"ERROR\"} [$__range]))", - "queryType": "instant", - "refId": "A" - } - ], - "title": "Errors", - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "yellow", - "mode": "fixed" - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"WARN\"} [$__range]))", - "queryType": "instant", - "refId": "A" - } - ], - "title": "Warnings", - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "noValue": "0" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__range]))", - "queryType": "instant", - "refId": "A" - } - ], - "title": "Total Log Lines", - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "lines/interval", - "drawStyle": "bars", - "fillOpacity": 80, - "lineWidth": 0, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ERROR" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "WARN" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "INFO" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "DEBUG" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 5, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))", - "legendFormat": "{{level}}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Log Volume by Level", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "drawStyle": "bars", - "fillOpacity": 60, - "lineWidth": 0, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 6, - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum by (container) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))", - "legendFormat": "{{container}}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Log Volume by Container", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 10, - "title": "Full Pipeline Log Stream", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=~\"$level\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Pipeline Logs \u2014 $pipeline", - "type": "logs" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 20, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 21, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"main\"} |~ \"(clippy|cargo test|warning\\\\[|error\\\\[|FAILED|test result|running \\\\d+ test|failures:)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Clippy + Test Output", - "type": "logs" - } - ], - "title": "Test Gate (clippy + cargo test)", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 30, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 31, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"redis\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Redis Sidecar Logs", - "type": "logs" - } - ], - "title": "Redis Sidecar", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 40, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 41, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "All Errors & Failures", - "type": "logs" - } - ], - "title": "Errors & Failures", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 50, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 51, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy|compile-and-train)-.*\", container=\"main\"} |~ \"(Compiling|Finished|warning|error|Downloading|Building|cargo build|rclone)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Compile Logs", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 52, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy)-.*\", container=\"main\"} |~ \"(kubectl|apply|rollout|deploy|configured|unchanged|created|image|kaniko)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Deploy & Argo Template Logs", - "type": "logs" - } - ], - "title": "Build & Deploy", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 60, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 61, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"gpu-test-.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "GPU Test Logs (DQN/PPO/TFT + Supervised)", - "type": "logs" - } - ], - "title": "GPU Test Pipeline", - "type": "row" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "foxhunt", - "ci", - "loki", - "logs" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "ci-pipeline", - "value": "ci-pipeline" - }, - "label": "Pipeline", - "name": "pipeline", - "options": [], - "query": "ci-pipeline,compile-and-deploy,compile-and-train,gpu-test,gpu-diag,gpu-walltime,build-ci-image,build-training-runtime,evaluate", - "type": "custom" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "includeAll": true, - "label": "Level", - "multi": true, - "name": "level", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "ERROR", - "value": "ERROR" - }, - { - "selected": false, - "text": "WARN", - "value": "WARN" - }, - { - "selected": false, - "text": "INFO", - "value": "INFO" - }, - { - "selected": false, - "text": "DEBUG", - "value": "DEBUG" - }, - { - "selected": false, - "text": "TRACE", - "value": "TRACE" - } - ], - "query": "ERROR,WARN,INFO,DEBUG,TRACE", - "type": "custom" - }, - { - "current": { - "selected": false, - "text": "", - "value": "" - }, - "label": "Search", - "name": "search", - "type": "textbox" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "CI Pipeline Logs", - "uid": "foxhunt-ci-logs", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json deleted file mode 100644 index 38fd68b15..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ /dev/null @@ -1,2762 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Operations overview: service health, gRPC traffic, CI/CD pipelines, API gateway, trading metrics, and metrics pipeline health.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of foxhunt service targets currently up", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "green", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "count(up{job=\"foxhunt-services\"} == 1)", - "legendFormat": "up", - "refId": "A" - } - ], - "title": "Services Up", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of foxhunt service targets currently down", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "count(up{job=\"foxhunt-services\"} == 0) OR on() vector(0)", - "legendFormat": "down", - "refId": "A" - } - ], - "title": "Services Down", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 300 - }, - { - "color": "green", - "value": 3600 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "trading_service_uptime_seconds", - "legendFormat": "trading", - "refId": "A" - } - ], - "title": "Trading Service Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 300 - }, - { - "color": "green", - "value": 3600 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "backtesting_service_uptime_seconds", - "legendFormat": "backtesting", - "refId": "A" - } - ], - "title": "Backtesting Service Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 300 - }, - { - "color": "green", - "value": 3600 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "ml_training_service_uptime_seconds", - "legendFormat": "ml_training", - "refId": "A" - } - ], - "title": "ML Training Service Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 300 - }, - { - "color": "green", - "value": 3600 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_data_acquisition_uptime_seconds", - "legendFormat": "data_acquisition", - "refId": "A" - } - ], - "title": "Data Acquisition Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 300 - }, - { - "color": "green", - "value": 3600 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "max(time() - process_start_time_seconds{job=\"foxhunt-services\", instance=~\".*:9091\"})", - "legendFormat": "api", - "refId": "A" - } - ], - "title": "API Gateway Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(rate(api_auth_requests_total[$__rate_interval]))", - "legendFormat": "req/s", - "refId": "A" - } - ], - "title": "API Auth Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current up/down status for each service target", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "color-background-solid" - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "1": { - "color": "green", - "text": "UP" - } - }, - "type": "value" - }, - { - "options": { - "0": { - "color": "red", - "text": "DOWN" - } - }, - "type": "value" - } - ] - }, - { - "id": "custom.displayMode", - "value": "color-background-solid" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 10, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "service" - } - ] - }, - "targets": [ - { - "expr": "up{job=\"foxhunt-services\"}", - "format": "table", - "instant": true, - "legendFormat": "{{service}}", - "refId": "A" - } - ], - "title": "Per-Service Status", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "job": true - }, - "renameByName": { - "Value": "Status", - "instance": "Instance", - "service": "Service" - } - } - } - ], - "type": "table" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 20, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of gRPC calls started, grouped by exported_service", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "sum by (exported_service) (rate(grpc_server_started_total[$__rate_interval]))", - "legendFormat": "{{exported_service}}", - "refId": "A" - } - ], - "title": "gRPC Request Rate (by service)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of gRPC calls that returned a non-OK code", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 22, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval]))", - "legendFormat": "{{exported_service}}", - "refId": "A" - } - ], - "title": "gRPC Error Rate (by service)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "99th percentile gRPC server handling duration", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket[$__rate_interval])))", - "legendFormat": "{{exported_service}} p99", - "refId": "A" - } - ], - "title": "gRPC p99 Latency (by service)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Percentage of gRPC calls returning non-OK status", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.05 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 18 - }, - "id": 24, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))", - "legendFormat": "error ratio", - "refId": "A" - } - ], - "title": "gRPC Error Ratio", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total gRPC requests per second across all services", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - }, - { - "color": "yellow", - "value": 500 - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 18 - }, - "id": 25, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(rate(grpc_server_started_total[$__rate_interval]))", - "legendFormat": "total rps", - "refId": "A" - } - ], - "title": "gRPC Total RPS", - "type": "stat" - } - ], - "title": "gRPC Overview", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 30, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 15, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 11 - }, - "id": 31, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "running", - "refId": "A" - } - ], - "title": "Running Workflows", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 15, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 8 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 11 - }, - "id": 32, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Pending\"}", - "legendFormat": "pending", - "refId": "A" - } - ], - "title": "Pending Workflows", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 3 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 11 - }, - "id": 33, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Failed\"}", - "legendFormat": "failed", - "refId": "A" - } - ], - "title": "Failed Workflows", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 11 - }, - "id": 34, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Succeeded\"}", - "legendFormat": "succeeded", - "refId": "A" - } - ], - "title": "Succeeded Workflows", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - }, - { - "color": "yellow", - "value": 20 - }, - { - "color": "red", - "value": 50 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 11 - }, - "id": 35, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "argo_workflows_pods_gauge", - "legendFormat": "pods", - "refId": "A" - } - ], - "title": "Workflow Pods", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of errors from Argo Workflows controller", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 20, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.1 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 11 - }, - "id": 36, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "rate(argo_workflows_error_count[$__rate_interval])", - "legendFormat": "errors/s", - "refId": "A" - } - ], - "title": "Argo Error Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "95th percentile Argo operation duration", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "red", - "value": 120 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 37, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p95", - "refId": "A" - } - ], - "title": "Argo Operation Duration p95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of Kubernetes API requests from Argo controller", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "fixed" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 38, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "sum(rate(argo_workflows_k8s_request_total[$__rate_interval]))", - "legendFormat": "k8s req/s", - "refId": "A" - } - ], - "title": "Argo K8s API Requests", - "type": "timeseries" - } - ], - "title": "Argo CI/CD", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 40, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of authentication requests (total, success, failure)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "unit": "reqps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "failure" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 41, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "rate(api_auth_requests_total[$__rate_interval])", - "legendFormat": "total", - "refId": "A" - }, - { - "expr": "rate(api_auth_requests_success[$__rate_interval])", - "legendFormat": "success", - "refId": "B" - }, - { - "expr": "rate(api_auth_requests_failure[$__rate_interval])", - "legendFormat": "failure", - "refId": "C" - } - ], - "title": "Auth Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current number of entries in the API gateway rate limiter", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 100 - }, - { - "color": "red", - "value": 500 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 12 - }, - "id": 42, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "api_rate_limiter_entries", - "legendFormat": "entries", - "refId": "A" - } - ], - "title": "Rate Limiter Entries", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ratio of JWT cache hits to total lookups", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 12 - }, - "id": 43, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)", - "legendFormat": "hit ratio", - "refId": "A" - } - ], - "title": "JWT Cache Hit Ratio", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "API gateway routing duration percentiles", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 5, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5000 - }, - { - "color": "red", - "value": 50000 - } - ] - }, - "unit": "us" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 44, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))", - "legendFormat": "p50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))", - "legendFormat": "p95", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "Routing Latency p50 / p95 / p99", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Percentage of auth requests that fail", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.2 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 16 - }, - "id": 45, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "rate(api_auth_requests_failure[$__rate_interval]) / rate(api_auth_requests_total[$__rate_interval])", - "legendFormat": "failure rate", - "refId": "A" - } - ], - "title": "Auth Failure Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of JWT cache misses", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 16 - }, - "id": 46, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "rate(api_jwt_cache_misses[$__rate_interval])", - "legendFormat": "misses/s", - "refId": "A" - } - ], - "title": "JWT Cache Misses/s", - "type": "stat" - } - ], - "title": "API Gateway", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 50, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current portfolio value in USD", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 10000 - }, - { - "color": "green", - "value": 100000 - } - ] - }, - "unit": "currencyUSD" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 13 - }, - "id": 51, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "trading_agent_portfolio_value_usd", - "legendFormat": "portfolio", - "refId": "A" - } - ], - "title": "Portfolio Value", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of assets currently selected for trading", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 6, - "y": 13 - }, - "id": 52, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "trading_agent_assets_selected", - "legendFormat": "selected", - "refId": "A" - } - ], - "title": "Assets Selected", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total instruments in the trading universe", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 9, - "y": 13 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "trading_agent_universe_instruments", - "legendFormat": "instruments", - "refId": "A" - } - ], - "title": "Universe Instruments", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of trading measurements emitted", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 13 - }, - "id": 54, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "rate(trading_measurements_total[$__rate_interval])", - "legendFormat": "measurements/s", - "refId": "A" - } - ], - "title": "Trading Measurements Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "End-to-end trading latency", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "fixed" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 13 - }, - "id": 55, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "trading_total_latency_seconds", - "legendFormat": "total latency", - "refId": "A" - } - ], - "title": "Total Trading Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Breakdown of trading pipeline latency components", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "order processing" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "risk check" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "market data" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 56, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "trading_order_processing_seconds", - "legendFormat": "order processing", - "refId": "A" - }, - { - "expr": "trading_risk_check_seconds", - "legendFormat": "risk check", - "refId": "B" - }, - { - "expr": "trading_market_data_seconds", - "legendFormat": "market data", - "refId": "C" - } - ], - "title": "Order Processing vs Risk Check vs Market Data", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Historical portfolio value trend", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "scheme", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "decimals": 2, - "unit": "currencyUSD" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 57, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "trading_agent_portfolio_value_usd", - "legendFormat": "portfolio USD", - "refId": "A" - } - ], - "title": "Portfolio Value Over Time", - "type": "timeseries" - } - ], - "title": "Trading", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 60, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Percentage of the metrics buffer currently in use", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 60 - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 14 - }, - "id": 61, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "expr": "metrics_buffer_utilization_percent", - "legendFormat": "buffer %", - "refId": "A" - } - ], - "title": "Metrics Buffer Utilization", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate at which metrics are being dropped", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 20, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 14 - }, - "id": 62, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "rate(metrics_dropped_total[$__rate_interval])", - "legendFormat": "dropped/s", - "refId": "A" - } - ], - "title": "Metrics Dropped Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total count of metrics dropped since service start", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 100 - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 14 - }, - "id": 63, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "metrics_dropped_total", - "legendFormat": "total dropped", - "refId": "A" - } - ], - "title": "Total Metrics Dropped", - "type": "stat" - } - ], - "title": "Metrics Health", - "type": "row" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "foxhunt", - "cockpit" - ], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "label": "Interval", - "name": "interval", - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timezone": "browser", - "title": "Operations Cockpit", - "uid": "foxhunt-cockpit", - "weekStart": "", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json deleted file mode 100644 index 418f33696..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json +++ /dev/null @@ -1,2206 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "K8s cluster health, node resources, GPU, storage, and Prometheus health", - "editable": true, - "fiscalYearStartMonth": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Cluster Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "count(kube_node_info)", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Nodes Ready", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Pods Running", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 12 - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(kube_node_status_capacity{resource=\"cpu\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total CPU Capacity", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(kube_node_status_capacity{resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Memory Capacity", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sum(kube_node_status_capacity{resource=\"nvidia_com_gpu\"}) or vector(0)", - "legendFormat": "", - "refId": "A" - } - ], - "title": "GPUs Available", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Node Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisSoftMax": 1, - "axisSoftMin": 0, - "drawStyle": "line", - "fillOpacity": 30, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }}", - "refId": "A" - } - ], - "title": "CPU Usage by Node (stacked)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 20, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.8 - }, - { - "color": "red", - "value": 0.95 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }}", - "refId": "A" - } - ], - "title": "Memory Usage by Node", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 15, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 14 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} read", - "refId": "A" - }, - { - "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} write", - "refId": "B" - } - ], - "title": "Disk I/O (Read / Write)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 15, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 14 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|cali.*|flannel.*\"}[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} rx", - "refId": "A" - }, - { - "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|cali.*|flannel.*\"}[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} tx", - "refId": "B" - } - ], - "title": "Network I/O (Receive / Transmit)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 14 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "node_load1 * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} load1", - "refId": "A" - }, - { - "expr": "node_load5 * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} load5", - "refId": "B" - }, - { - "expr": "node_load15 * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} load15", - "refId": "C" - } - ], - "title": "Load Average (1m / 5m / 15m)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 20, - "panels": [], - "title": "Kubernetes Workloads", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "stepAfter", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*desired" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", - "legendFormat": "{{ deployment }} desired", - "refId": "A" - }, - { - "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", - "legendFormat": "{{ deployment }} available", - "refId": "B" - } - ], - "title": "Deployment Replicas (Desired vs Available)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Running" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pending" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Succeeded" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 23 - }, - "id": 22, - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "expr": "sum by (phase) (kube_pod_status_phase{namespace=\"foxhunt\"})", - "legendFormat": "{{ phase }}", - "refId": "A" - } - ], - "title": "Pod Phase Distribution", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 23 - }, - "id": 23, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[$__rate_interval])", - "legendFormat": "{{ pod }} / {{ container }}", - "refId": "A" - } - ], - "title": "Container Restarts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 24, - "options": { - "footer": { - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Value" - } - ] - }, - "targets": [ - { - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", phase=~\"Pending|Failed\"} > 0", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Pending / Failed Pods", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "instance": true, - "job": true - }, - "indexByName": {}, - "renameByName": { - "Value": "Count", - "namespace": "Namespace", - "phase": "Phase", - "pod": "Pod" - } - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 30, - "panels": [], - "title": "Storage", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "orange", - "value": 0.85 - }, - { - "color": "red", - "value": 0.95 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 31, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "targets": [ - { - "expr": "(1 - (node_filesystem_avail_bytes{fstype=~\"ext4|xfs|btrfs\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\"})) * on(instance) group_left(nodename) node_uname_info", - "instant": true, - "legendFormat": "{{ nodename }} {{ mountpoint }}", - "refId": "A" - } - ], - "title": "Disk Usage by Mountpoint", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Storage" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 32, - "options": { - "footer": { - "show": false - }, - "showHeader": true - }, - "targets": [ - { - "expr": "kube_persistentvolumeclaim_status_phase{namespace=\"foxhunt\"}", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - }, - { - "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=\"foxhunt\"}", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "B" - } - ], - "title": "PVC Status & Requested Storage", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "persistentvolumeclaim" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time 1": true, - "Time 2": true, - "__name__ 1": true, - "__name__ 2": true, - "instance 1": true, - "instance 2": true, - "job 1": true, - "job 2": true, - "namespace 1": true, - "namespace 2": true, - "phase": false - }, - "renameByName": { - "Value 1": "Phase Value", - "Value 2": "Storage", - "persistentvolumeclaim": "PVC", - "phase": "Phase" - } - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 40, - "panels": [], - "title": "GPU Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 25, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 47 - }, - "id": 41, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL", - "legendFormat": "GPU {{ gpu }} ({{ instance }})", - "refId": "A" - } - ], - "title": "GPU Utilization %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 20, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decmbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*free" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 5 - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 47 - }, - "id": 42, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED", - "legendFormat": "GPU {{ gpu }} used", - "refId": "A" - }, - { - "expr": "DCGM_FI_DEV_FB_FREE", - "legendFormat": "GPU {{ gpu }} free", - "refId": "B" - } - ], - "title": "VRAM Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 0, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 47 - }, - "id": 43, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP", - "legendFormat": "GPU {{ gpu }} core", - "refId": "A" - }, - { - "expr": "DCGM_FI_DEV_MEMORY_TEMP", - "legendFormat": "GPU {{ gpu }} mem", - "refId": "B" - } - ], - "title": "GPU Temperature", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 15, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "watt" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 55 - }, - "id": 44, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE", - "legendFormat": "GPU {{ gpu }}", - "refId": "A" - } - ], - "title": "GPU Power Draw", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 15, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 55 - }, - "id": 45, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_PROF_PCIE_RX_BYTES", - "legendFormat": "GPU {{ gpu }} rx", - "refId": "A" - }, - { - "expr": "DCGM_FI_PROF_PCIE_TX_BYTES", - "legendFormat": "GPU {{ gpu }} tx", - "refId": "B" - } - ], - "title": "GPU PCIe Throughput", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 63 - }, - "id": 50, - "panels": [], - "title": "Prometheus Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 500000 - }, - { - "color": "red", - "value": 1000000 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 64 - }, - "id": 51, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "prometheus_tsdb_head_series", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Active Time Series", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5368709120 - }, - { - "color": "red", - "value": 10737418240 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 64 - }, - "id": 52, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "prometheus_tsdb_storage_blocks_bytes", - "legendFormat": "", - "refId": "A" - } - ], - "title": "TSDB Storage Size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "down" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 64 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value_and_name" - }, - "targets": [ - { - "expr": "count(up == 1)", - "legendFormat": "up", - "refId": "A" - }, - { - "expr": "count(up == 0)", - "legendFormat": "down", - "refId": "B" - } - ], - "title": "Scrape Targets Up", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 68 - }, - "id": 54, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "scrape_duration_seconds", - "legendFormat": "{{ job }} / {{ instance }}", - "refId": "A" - } - ], - "title": "Scrape Duration by Job", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "text": "DOWN" - }, - "1": { - "color": "green", - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 68 - }, - "id": 55, - "options": { - "footer": { - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Value" - } - ] - }, - "targets": [ - { - "expr": "up", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Scrape Targets Status", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true - }, - "renameByName": { - "Value": "Status", - "instance": "Instance", - "job": "Job" - } - } - } - ], - "type": "table" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "style": "dark", - "tags": [ - "foxhunt", - "infrastructure", - "k8s" - ], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "hide": 0, - "name": "interval", - "query": "1m,5m,15m,30m,1h", - "refresh": 2, - "type": "interval" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "hide": 0, - "includeAll": true, - "multi": true, - "name": "node", - "query": "label_values(node_uname_info, nodename)", - "refresh": 2, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timezone": "browser", - "title": "Infrastructure & Cluster", - "uid": "foxhunt-infrastructure", - "weekStart": "", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-observability.json b/infra/k8s/monitoring/dashboards/foxhunt-observability.json deleted file mode 100644 index df6a0a03b..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-observability.json +++ /dev/null @@ -1,1621 +0,0 @@ -{ - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "title": "API Gateway Auth", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "ops" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "failure" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "rate(api_auth_requests_success[$interval])", - "legendFormat": "success", - "refId": "A" - }, - { - "expr": "rate(api_auth_requests_failure[$interval])", - "legendFormat": "failure", - "refId": "B" - }, - { - "expr": "rate(api_auth_requests_total[$interval])", - "legendFormat": "total", - "refId": "C" - } - ], - "title": "Auth Request Rate (Success / Failure)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "bars", - "fillOpacity": 50, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "rate(api_auth_errors_expired_jwt[$interval])", - "legendFormat": "expired_jwt", - "refId": "A" - }, - { - "expr": "rate(api_auth_errors_invalid_jwt[$interval])", - "legendFormat": "invalid_jwt", - "refId": "B" - }, - { - "expr": "rate(api_auth_errors_missing_jwt[$interval])", - "legendFormat": "missing_jwt", - "refId": "C" - }, - { - "expr": "rate(api_auth_errors_permission_denied[$interval])", - "legendFormat": "permission_denied", - "refId": "D" - }, - { - "expr": "rate(api_auth_errors_rate_limited[$interval])", - "legendFormat": "rate_limited", - "refId": "E" - } - ], - "title": "Auth Error Breakdown", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 3, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(api_jwt_validation_duration_microseconds_bucket[$interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, rate(api_jwt_validation_duration_microseconds_bucket[$interval]))", - "legendFormat": "p95", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, rate(api_jwt_validation_duration_microseconds_bucket[$interval]))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "JWT Validation Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "api_active_jwt_tokens", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Active JWT Tokens", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 101, - "title": "API Gateway Security", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 500 - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 10 - }, - "id": 5, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "api_rate_limiter_entries", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Rate Limiter Entries", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "max": 1, - "min": 0, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 10 - }, - "id": 6, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "api_rbac_cache_hits / (api_rbac_cache_hits + api_rbac_cache_misses)", - "legendFormat": "hit ratio", - "refId": "A" - } - ], - "title": "RBAC Cache Hit Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 50 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 10 - }, - "id": 7, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "api_revoked_tokens_cached", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Revoked Tokens Cached", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 10 - }, - "id": 8, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(api_mfa_verification_duration_microseconds_bucket[$interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, rate(api_mfa_verification_duration_microseconds_bucket[$interval]))", - "legendFormat": "p95", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, rate(api_mfa_verification_duration_microseconds_bucket[$interval]))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "MFA Verification Latency", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 102, - "title": "API Gateway Routing", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "\u00b5s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 9, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(api_routing_duration_microseconds_bucket[$interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, rate(api_routing_duration_microseconds_bucket[$interval]))", - "legendFormat": "p95", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, rate(api_routing_duration_microseconds_bucket[$interval]))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "Routing Duration p50/p95/p99", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "bars", - "fillOpacity": 50, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 10, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "rate(api_config_updates_auth[$interval])", - "legendFormat": "auth", - "refId": "A" - }, - { - "expr": "rate(api_config_updates_backend[$interval])", - "legendFormat": "backend", - "refId": "B" - }, - { - "expr": "rate(api_config_updates_rate_limit[$interval])", - "legendFormat": "rate_limit", - "refId": "C" - }, - { - "expr": "rate(api_config_updates_routing[$interval])", - "legendFormat": "routing", - "refId": "D" - } - ], - "title": "Config Updates Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "max": 1, - "min": 0, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "api_config_cache_hits / (api_config_cache_hits + api_config_cache_misses)", - "legendFormat": "config hit ratio", - "refId": "A" - }, - { - "expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)", - "legendFormat": "jwt hit ratio", - "refId": "B" - } - ], - "title": "Config Cache Hit Ratio", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 103, - "title": "API Gateway Events", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "ops" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "failure" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 25 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "rate(api_notify_events_total[$interval])", - "legendFormat": "total", - "refId": "A" - }, - { - "expr": "rate(api_notify_events_success[$interval])", - "legendFormat": "success", - "refId": "B" - }, - { - "expr": "rate(api_notify_events_failure[$interval])", - "legendFormat": "failure", - "refId": "C" - } - ], - "title": "Notify Events Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 25 - }, - "id": 13, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "api_notify_listener_connected", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Notify Listener Connected", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "errors" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "reconnections" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "rate(api_notify_listener_errors[$interval])", - "legendFormat": "errors", - "refId": "A" - }, - { - "expr": "rate(api_notify_listener_reconnections[$interval])", - "legendFormat": "reconnections", - "refId": "B" - } - ], - "title": "Listener Errors & Reconnections", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 104, - "title": "Logs (Loki)", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "bars", - "fillOpacity": 50, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "error" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "warn" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "info" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "debug" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 33 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\"} | json [$__interval]))", - "legendFormat": "{{level}}", - "refId": "A" - } - ], - "title": "Log Volume by Level", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 20, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 33 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"error\" [$__interval]))", - "legendFormat": "{{app}}", - "refId": "A" - } - ], - "title": "Error Log Rate by App", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 33 - }, - "id": 17, - "options": { - "enableLogDetails": true, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\"} |= \"error\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Recent Errors", - "type": "logs" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 105, - "title": "Traces (Tempo)", - "type": "row" - }, - { - "datasource": { - "type": "${DS_TEMPO}", - "uid": "${DS_TEMPO}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 18, - "options": {}, - "targets": [ - { - "queryType": "traceqlSearch", - "refId": "A" - } - ], - "title": "Trace Search", - "type": "traces" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 106, - "title": "Prometheus Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "color-text" - }, - "inspect": false - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "text": "DOWN" - }, - "1": { - "color": "green", - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.width", - "value": 80 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 51 - }, - "id": 19, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Value" - } - ] - }, - "targets": [ - { - "expr": "up", - "format": "table", - "instant": true, - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Scrape Targets Up/Down", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true - }, - "renameByName": { - "Value": "Status", - "instance": "Instance", - "job": "Job" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 51 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "scrape_duration_seconds", - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Scrape Duration by Job", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 50000 - }, - { - "color": "red", - "value": 100000 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 51 - }, - "id": 21, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "prometheus_tsdb_head_series", - "legendFormat": "", - "refId": "A" - } - ], - "title": "TSDB Head Series", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1073741824 - }, - { - "color": "red", - "value": 5368709120 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 51 - }, - "id": 22, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "expr": "prometheus_tsdb_storage_blocks_bytes", - "legendFormat": "", - "refId": "A" - } - ], - "title": "TSDB Storage Bytes", - "type": "stat" - } - ], - "refresh": "15s", - "schemaVersion": 39, - "tags": [ - "foxhunt", - "observability", - "api-gateway", - "logs", - "traces" - ], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Observability & API Gateway", - "uid": "foxhunt-observability", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-trading.json b/infra/k8s/monitoring/dashboards/foxhunt-trading.json deleted file mode 100644 index 4acb6900f..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-trading.json +++ /dev/null @@ -1,2167 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Trading overview, execution latency, gRPC pipeline health, data acquisition, and backtesting for the foxhunt HFT platform", - "editable": true, - "fiscalYearStartMonth": 0, - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Trading Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current portfolio value in USD from the trading agent", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 10000 - }, - { - "color": "green", - "value": 100000 - } - ] - }, - "unit": "currencyUSD" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "trading_agent_portfolio_value_usd", - "instant": true, - "legendFormat": "portfolio", - "refId": "A" - } - ], - "title": "Portfolio Value", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of assets currently selected by the trading agent", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - }, - { - "color": "yellow", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "trading_agent_assets_selected", - "instant": true, - "legendFormat": "selected", - "refId": "A" - } - ], - "title": "Assets Selected", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total instruments in the trading universe", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "trading_agent_universe_instruments", - "instant": true, - "legendFormat": "instruments", - "refId": "A" - } - ], - "title": "Universe Instruments", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "How long the trading service has been running", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "Down", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "green", - "value": 86400 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "trading_service_uptime_seconds", - "instant": true, - "legendFormat": "uptime", - "refId": "A" - } - ], - "title": "Trading Service Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of trading measurements over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "scheme", - "lineInterpolation": "smooth", - "lineWidth": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(trading_measurements_total[$__rate_interval])", - "legendFormat": "measurements/s", - "refId": "A" - } - ], - "title": "Trading Measurements Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Trading agent portfolio value trend", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "scheme", - "lineInterpolation": "smooth", - "lineWidth": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false - }, - "unit": "currencyUSD" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "min", - "max" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "trading_agent_portfolio_value_usd", - "legendFormat": "portfolio value", - "refId": "A" - } - ], - "title": "Portfolio Value Over Time", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 10, - "panels": [], - "title": "Trading Latency", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "End-to-end trading latency percentiles from trading_total_latency_seconds histogram", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "p99" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p95" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p50" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.5, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "Total Trading Latency (p50 / p95 / p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Order processing latency percentiles from trading_order_processing_seconds histogram", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "p99" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p95" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p50" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "Order Processing Latency (p50 / p95 / p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Risk check latency percentiles from trading_risk_check_seconds histogram", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "p99" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p95" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p50" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "Risk Check Latency (p50 / p95 / p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Market data fetch latency percentiles from trading_market_data_seconds histogram", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "p99" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p95" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p50" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.5, rate(trading_market_data_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(trading_market_data_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(trading_market_data_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99", - "refId": "C" - } - ], - "title": "Market Data Latency (p50 / p95 / p99)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 20, - "panels": [], - "title": "gRPC Trading Pipeline", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of gRPC calls handled per service (trading-service, trading-agent-service, broker-gateway)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 29 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (exported_service) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))", - "legendFormat": "{{exported_service}}", - "refId": "A" - } - ], - "title": "gRPC Request Rate by Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of non-OK gRPC responses across trading services", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 29 - }, - "id": 22, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "sum" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (exported_service, grpc_code) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval]))", - "legendFormat": "{{exported_service}} {{grpc_code}}", - "refId": "A" - } - ], - "title": "gRPC Error Rate by Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "99th percentile gRPC handling latency per trading service", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 29 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval])))", - "legendFormat": "{{exported_service}} p99", - "refId": "A" - } - ], - "title": "gRPC p99 Latency by Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total gRPC requests started across all trading services", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 37 - }, - "id": 24, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(grpc_server_started_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"})", - "instant": true, - "legendFormat": "total started", - "refId": "A" - } - ], - "title": "gRPC Total Requests Started", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ratio of non-OK to total handled gRPC requests across trading services", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 1, - "min": 0, - "noValue": "0%", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.05 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 37 - }, - "id": 25, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))", - "instant": true, - "legendFormat": "error ratio", - "refId": "A" - } - ], - "title": "gRPC Error Ratio", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total handled requests on broker-gateway (highest volume service)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 100 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 37 - }, - "id": 26, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(grpc_server_handled_total{exported_service=\"broker-gateway\"})", - "instant": true, - "legendFormat": "broker-gateway total", - "refId": "A" - } - ], - "title": "Broker Gateway Requests", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 30, - "panels": [], - "title": "Data Acquisition", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of currently active data acquisition feeds", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 42 - }, - "id": 31, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_data_acquisition_feeds_active", - "instant": true, - "legendFormat": "active feeds", - "refId": "A" - } - ], - "title": "Active Feeds", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of data acquisition errors over time", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 42 - }, - "id": 32, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(foxhunt_data_acquisition_errors_total[$__rate_interval])", - "legendFormat": "errors/s", - "refId": "A" - } - ], - "title": "Data Acquisition Error Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "How long the data acquisition service has been running", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "Down", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "green", - "value": 86400 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 42 - }, - "id": 33, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_data_acquisition_uptime_seconds", - "instant": true, - "legendFormat": "uptime", - "refId": "A" - } - ], - "title": "Data Acquisition Uptime", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 40, - "panels": [], - "title": "Backtesting", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total backtests completed", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 48 - }, - "id": 41, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "backtesting_backtests_completed_total", - "instant": true, - "legendFormat": "completed", - "refId": "A" - } - ], - "title": "Backtests Completed", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total backtests started", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 48 - }, - "id": 42, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "backtesting_backtests_started_total", - "instant": true, - "legendFormat": "started", - "refId": "A" - } - ], - "title": "Backtests Started", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ratio of completed to started backtests (1.0 = all finished)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 1, - "min": 0, - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 48 - }, - "id": 43, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "backtesting_backtests_completed_total / backtesting_backtests_started_total", - "instant": true, - "legendFormat": "completion ratio", - "refId": "A" - } - ], - "title": "Backtest Completion Ratio", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of backtesting errors over time", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 48 - }, - "id": 44, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(backtesting_errors_total[$__rate_interval])", - "legendFormat": "errors/s", - "refId": "A" - } - ], - "title": "Backtest Error Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of backtests started and completed, showing active backtest throughput", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false - }, - "unit": "ops" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "started" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "completed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 45, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "mean" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(backtesting_backtests_started_total[$__rate_interval])", - "legendFormat": "started", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(backtesting_backtests_completed_total[$__rate_interval])", - "legendFormat": "completed", - "refId": "B" - } - ], - "title": "Backtests Started vs Completed Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Estimated active backtests: started minus completed", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 53 - }, - "id": 46, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "backtesting_backtests_started_total - backtesting_backtests_completed_total", - "instant": true, - "legendFormat": "active", - "refId": "A" - } - ], - "title": "Active Backtests (est.)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "How long the backtesting service has been running", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "noValue": "Down", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "green", - "value": 86400 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 53 - }, - "id": 47, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "backtesting_service_uptime_seconds", - "instant": true, - "legendFormat": "uptime", - "refId": "A" - } - ], - "title": "Backtesting Uptime", - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [ - "foxhunt", - "trading", - "ml" - ], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "name": "interval", - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Trading & ML Intelligence", - "uid": "foxhunt-trading", - "weekStart": "", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json b/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json deleted file mode 100644 index df98ae908..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json +++ /dev/null @@ -1,3348 +0,0 @@ -{ - "annotations": { - "list": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "enable": true, - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=\"ERROR\"}", - "iconColor": "red", - "name": "Errors", - "tagKeys": "level", - "titleFormat": "Error" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "enable": true, - "expr": "changes(foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}[1m]) > 0", - "iconColor": "blue", - "name": "Checkpoints", - "titleFormat": "Checkpoint Saved" - } - ] - }, - "description": "Unified training pipeline monitor \u2014 target a running pod to see training curves, GPU health, eval metrics, hyperopt progress, container resources, and live logs in one place.", - "editable": true, - "fiscalYearStartMonth": 0, - "links": [ - { - "title": "CI Pipeline Logs", - "type": "link", - "url": "/d/foxhunt-ci-logs/foxhunt-ci-pipeline-logs" - }, - { - "title": "Training Logs", - "type": "link", - "url": "/d/foxhunt-training-logs/foxhunt-training-pipeline-logs" - }, - { - "title": "Training & CI/CD", - "type": "link", - "url": "/d/foxhunt-training/foxhunt-training-and-ci-cd" - }, - { - "title": "Operations Cockpit", - "type": "link", - "url": "/d/foxhunt-cockpit/foxhunt-operations-cockpit" - } - ], - "liveNow": true, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Job Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_training_current_epoch{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Epoch", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_training_step{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Step", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Loss", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Val Loss", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Sharpe", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.15 - }, - { - "color": "red", - "value": 0.3 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "MaxDD", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "GPU Util", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} / (DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} + DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}) * 100", - "legendFormat": "", - "refId": "A" - } - ], - "title": "VRAM Used", - "type": "gauge" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "CI Pipeline & Argo Workflows", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Currently running Argo workflows", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Active Workflows", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 6 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "argo_workflows_pod_pending_count", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Pending Pods", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 6 - }, - "id": 13, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "argo_workflows_total_count{status=\"Succeeded\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Completed", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 6 - }, - "id": 14, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "argo_workflows_error_count", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Errors", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Average workflow operation duration", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 6 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "rate(argo_workflows_operation_duration_seconds_sum[5m]) / rate(argo_workflows_operation_duration_seconds_count[5m])", - "legendFormat": "Avg Duration", - "refId": "A" - } - ], - "title": "Workflow Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Argo controller K8s API calls \u2014 high rates may indicate thrashing", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 6 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "rate(argo_workflows_k8s_request_total[5m])", - "legendFormat": "{{verb}} {{status_code}}", - "refId": "A" - } - ], - "title": "K8s API Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "CPU usage of selected CI/training pods", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 10 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"$pod\"}[5m])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CI Pod CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "RSS memory of selected CI/training pods", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 10 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "container_memory_rss{namespace=\"foxhunt\", pod=~\"$pod\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CI Pod Memory", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 19, - "panels": [], - "title": "CI Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "All logs from selected CI/training pods", - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 20, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"$search\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "CI Full Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "Rust compile output, test results, clippy warnings", - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 21, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cargo (build|test|check|clippy)|Compiling|Finished|running \\\\d+ test|test result|FAILED|passed|warning:)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Compile & Test Results", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "S3 binary uploads, kubectl deploys, image builds", - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 22, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(rclone|s3://|upload|download|kubectl|rollout|image|docker|kaniko|configmap)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Build Artifacts & Deploy", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "Data fetching, redis cache, dbn file operations", - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 23, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(redis|rclone|dbn|data|cache|download|decompress)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Redis Sidecar & Data", - "type": "logs" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 24, - "panels": [], - "title": "Training Curves", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Training and validation loss over time", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 37 - }, - "id": 25, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Train Loss", - "refId": "A" - }, - { - "expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Val Loss", - "refId": "B" - } - ], - "title": "Loss & Validation Loss", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Learning rate schedule", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 37 - }, - "id": 26, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_learning_rate{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "LR", - "refId": "A" - } - ], - "title": "Learning Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Gradient L2 norm \u2014 spikes indicate instability", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 37 - }, - "id": 27, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_gradient_norm{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Grad Norm", - "refId": "A" - } - ], - "title": "Gradient Norm", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 28, - "panels": [], - "title": "Trading Performance", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Risk-adjusted return metrics per epoch", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 45 - }, - "id": 29, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Sharpe", - "refId": "A" - }, - { - "expr": "foxhunt_training_epoch_sortino{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Sortino", - "refId": "B" - }, - { - "expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "MaxDD", - "refId": "C" - } - ], - "title": "Sharpe / Sortino / MaxDD", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 45 - }, - "id": 30, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_win_rate{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Win Rate", - "refId": "A" - }, - { - "expr": "foxhunt_training_epoch_profit_factor{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Profit Factor", - "refId": "B" - } - ], - "title": "Win Rate & Profit Factor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 45 - }, - "id": 31, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_total_return{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Total Return", - "refId": "A" - }, - { - "expr": "foxhunt_training_epoch_avg_return{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Avg Return", - "refId": "B" - } - ], - "title": "Total Return & Avg Return", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Action distribution per epoch \u2014 watch for collapse to single action", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 30, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "normal" - } - }, - "max": 1, - "min": 0, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 52 - }, - "id": 32, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_action_buy_pct{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Buy %", - "refId": "A" - }, - { - "expr": "foxhunt_training_epoch_action_hold_pct{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Hold %", - "refId": "B" - }, - { - "expr": "foxhunt_training_epoch_action_sell_pct{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Sell %", - "refId": "C" - } - ], - "title": "Action Distribution", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Higher entropy = more exploration. Collapse \u2192 policy degeneration", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 52 - }, - "id": 33, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_action_entropy{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Entropy", - "refId": "A" - }, - { - "expr": "foxhunt_training_action_diversity{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Diversity", - "refId": "B" - } - ], - "title": "Action Entropy & Diversity", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 52 - }, - "id": 34, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_total_trades{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Trades/Epoch", - "refId": "A" - } - ], - "title": "Total Trades", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 59 - }, - "id": 35, - "panels": [], - "title": "Evaluation Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "max": 1, - "min": 0, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 60 - }, - "id": 36, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_eval_accuracy{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Accuracy", - "refId": "A" - } - ], - "title": "Eval Accuracy", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Classification quality of directional predictions", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "max": 1, - "min": 0, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 60 - }, - "id": 37, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_eval_precision{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Precision", - "refId": "A" - }, - { - "expr": "foxhunt_training_eval_recall{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Recall", - "refId": "B" - }, - { - "expr": "foxhunt_training_eval_f1{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "F1", - "refId": "C" - } - ], - "title": "Precision / Recall / F1", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "DQN Q-value estimates \u2014 divergence from max indicates overestimation", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 60 - }, - "id": 38, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_q_value_mean{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Q Mean", - "refId": "A" - }, - { - "expr": "foxhunt_training_q_value_max{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Q Max", - "refId": "B" - } - ], - "title": "Q-Values (DQN)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ratio > 1 = Q-values overestimate true returns", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 60 - }, - "id": 39, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_q_overestimation_ratio{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Overestimation Ratio", - "refId": "A" - } - ], - "title": "Q-Overestimation & Replay Buffer", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 67 - }, - "id": 40, - "panels": [], - "title": "GPU & Hardware", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Overall GPU and graphics engine utilization", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 10, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "max": 100, - "min": 0, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 68 - }, - "id": 41, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "GPU %", - "refId": "A" - }, - { - "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100", - "legendFormat": "GR Engine %", - "refId": "B" - } - ], - "title": "GPU Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Tensor core and DRAM bandwidth utilization \u2014 key H100 saturation indicators", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 10, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "max": 100, - "min": 0, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 68 - }, - "id": 42, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100", - "legendFormat": "Tensor Cores %", - "refId": "A" - }, - { - "expr": "DCGM_FI_PROF_DRAM_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100", - "legendFormat": "DRAM Active %", - "refId": "B" - } - ], - "title": "Tensor Core Activity", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "GPU framebuffer memory usage", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 20, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "normal" - } - }, - "unit": "decmbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 68 - }, - "id": 43, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Used (MiB)", - "refId": "A" - }, - { - "expr": "DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Free (MiB)", - "refId": "B" - } - ], - "title": "VRAM (Framebuffer)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Thermal throttling starts at ~83\u00b0C on H100", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 68 - }, - "id": 44, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "GPU Temp", - "refId": "A" - }, - { - "expr": "DCGM_FI_DEV_MEMORY_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Memory Temp", - "refId": "B" - } - ], - "title": "GPU Temperature", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "GPU power draw \u2014 H100 TDP is 700W", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "watt" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 75 - }, - "id": 45, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Power (W)", - "refId": "A" - } - ], - "title": "Power Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "PCIe bandwidth \u2014 high RX during data loading, high TX during checkpoint saves", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 75 - }, - "id": 46, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "rate(DCGM_FI_PROF_PCIE_TX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])", - "legendFormat": "TX", - "refId": "A" - }, - { - "expr": "rate(DCGM_FI_PROF_PCIE_RX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])", - "legendFormat": "RX", - "refId": "B" - } - ], - "title": "PCIe Throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Clock throttling indicates thermal or power issues", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "clockhertz" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 75 - }, - "id": 47, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_SM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "SM Clock", - "refId": "A" - }, - { - "expr": "DCGM_FI_DEV_MEM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Mem Clock", - "refId": "B" - } - ], - "title": "SM & Memory Clock", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 10, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "max": 100, - "min": 0, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 75 - }, - "id": 48, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Mem Copy %", - "refId": "A" - } - ], - "title": "Memory Copy Utilization", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 82 - }, - "id": 49, - "panels": [], - "title": "Throughput & Performance", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Training throughput \u2014 higher is better", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 83 - }, - "id": 50, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_batches_per_second{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Batches/s", - "refId": "A" - } - ], - "title": "Batches / Second", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Wall-clock time per epoch", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 83 - }, - "id": 51, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_duration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Epoch Time", - "refId": "A" - } - ], - "title": "Epoch Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Per-step training iteration time", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 83 - }, - "id": 52, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_iteration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Iter Time", - "refId": "A" - } - ], - "title": "Iteration Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 83 - }, - "id": 53, - "options": { - "calculate": false, - "cellGap": 1, - "color": { - "mode": "scheme", - "scheme": "Oranges" - }, - "yAxis": { - "unit": "s" - } - }, - "targets": [ - { - "expr": "rate(foxhunt_training_data_load_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])", - "format": "heatmap", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Data Load Latency", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Checkpoint save count over time", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 90 - }, - "id": 54, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Total Saves", - "refId": "A" - } - ], - "title": "Checkpoint Saves", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Checkpoint file size", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 90 - }, - "id": 55, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_checkpoint_size_bytes{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Size (bytes)", - "refId": "A" - } - ], - "title": "Checkpoint Size & Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "DQN/PER replay buffer population", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 90 - }, - "id": 56, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_replay_buffer_size{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Buffer Size", - "refId": "A" - } - ], - "title": "Replay Buffer Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Parallel training workers", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 90 - }, - "id": 57, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_training_active_workers{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Workers", - "refId": "A" - } - ], - "title": "Active Workers", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 97 - }, - "id": 58, - "panels": [], - "title": "Hyperopt Trials", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 98 - }, - "id": 59, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_hyperopt_trial_current{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Current Trial", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 98 - }, - "id": 60, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_hyperopt_trial_total{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Trials", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 98 - }, - "id": 61, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Best Loss", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 98 - }, - "id": 62, - "options": { - "colorMode": "background", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "foxhunt_hyperopt_elapsed_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Elapsed", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Convergence curve \u2014 should decrease over trials", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 98 - }, - "id": 63, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Best Loss", - "refId": "A" - }, - { - "expr": "foxhunt_hyperopt_best_objective{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "Best Objective", - "refId": "B" - } - ], - "title": "Best Loss Over Trials", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 102 - }, - "id": 64, - "options": { - "calculate": false, - "cellGap": 1, - "color": { - "mode": "scheme", - "scheme": "Oranges" - }, - "yAxis": { - "unit": "s" - } - }, - "targets": [ - { - "expr": "rate(foxhunt_hyperopt_trial_duration_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])", - "format": "heatmap", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Trial Duration Distribution", - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 109 - }, - "id": 65, - "panels": [], - "title": "Container Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "CPU cores consumed by each container in the pod", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 110 - }, - "id": 66, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "rate(container_cpu_usage_seconds_total{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}[5m])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Container CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Resident set size \u2014 actual physical memory used", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 110 - }, - "id": 67, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "container_memory_rss{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Container Memory (RSS)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 110 - }, - "id": 68, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "rate(container_network_receive_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])", - "legendFormat": "RX {{interface}}", - "refId": "A" - }, - { - "expr": "rate(container_network_transmit_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])", - "legendFormat": "TX {{interface}}", - "refId": "B" - } - ], - "title": "Container Network I/O", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Non-zero restarts indicate OOM kills or crashes", - "fieldConfig": { - "defaults": { - "custom": { - "fillOpacity": 0, - "lineWidth": 1, - "spanNulls": true, - "stacking": { - "mode": "none" - } - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 110 - }, - "id": 69, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "expr": "kube_pod_container_status_restarts_total{pod=~\"$pod\", namespace=\"foxhunt\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Pod Restarts", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 117 - }, - "id": 70, - "panels": [], - "title": "Live Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "All logs from the selected pod, filtered by level and search text", - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 118 - }, - "id": 71, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"$level\"} |~ \"$search\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Full Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "Only ERROR and WARN level logs", - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 118 - }, - "id": 72, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"ERROR|WARN\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "Log lines mentioning training metrics \u2014 epoch completions, loss values, Sharpe ratios", - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 127 - }, - "id": 73, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(epoch|loss|sharpe|max_?dd|sortino|reward|return|accuracy|f1|precision|recall)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Training Progress (Epoch/Loss/Sharpe)", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "description": "GPU-related log lines \u2014 CUDA errors, OOM events, NCCL issues", - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 127 - }, - "id": 74, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cuda|gpu|vram|oom|out.of.memory|nccl|nvrtc|tensor.core|cublas)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "GPU / CUDA / OOM", - "type": "logs" - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [ - "foxhunt", - "training", - "gpu", - "loki", - "deep-dive" - ], - "templating": { - "list": [ - { - "current": {}, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "includeAll": true, - "label": "Pod", - "multi": true, - "name": "pod", - "query": "{namespace=\"foxhunt\", pod\\!~\"(grafana|prometheus|loki|promtail|${DS_TEMPO}|redis|postgres|mattermost|tailscale|gitlab|cilium|csi|konnectivity|nvidia-gpu|argo-workflows|questdb|web-gateway|trading-service).*\"}", - "refresh": 2, - "regex": "", - "sort": 2, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "includeAll": true, - "label": "Prometheus Job", - "multi": false, - "name": "prom_job", - "query": "label_values(up{namespace=\"foxhunt\"}, job)", - "refresh": 2, - "regex": "/(train|hyperopt|evaluate|pushgateway|training-pods|dcgm)/", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "includeAll": true, - "label": "Log Level", - "multi": true, - "name": "level", - "query": "ERROR,WARN,INFO,DEBUG,TRACE", - "type": "custom" - }, - { - "current": { - "text": "", - "value": "" - }, - "label": "Search", - "name": "search", - "type": "textbox" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timezone": "browser", - "title": "Training Deep-Dive", - "uid": "foxhunt-training-deep-dive", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json deleted file mode 100644 index 4e1832049..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json +++ /dev/null @@ -1,835 +0,0 @@ -{ - "description": "Loki-powered training log viewer \u2014 hyperopt trials, epoch progress, GPU status, errors", - "editable": true, - "graphTooltip": 2, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "title": "Training Overview", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"ERROR\"} [$__range]))", - "queryType": "instant", - "refId": "A" - } - ], - "title": "Errors", - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "yellow", - "mode": "fixed" - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"WARN\"} [$__range]))", - "queryType": "instant", - "refId": "A" - } - ], - "title": "Warnings", - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "noValue": "0" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__range]))", - "queryType": "instant", - "refId": "A" - } - ], - "title": "Total Log Lines", - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisLabel": "lines/interval", - "drawStyle": "bars", - "fillOpacity": 80, - "lineWidth": 0, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ERROR" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "WARN" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "INFO" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 5, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))", - "legendFormat": "{{level}}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Log Volume by Level", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "drawStyle": "bars", - "fillOpacity": 60, - "lineWidth": 0, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 6, - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "sum by (pod) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))", - "legendFormat": "{{pod}}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Log Volume by Pod", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 10, - "title": "Full Training Log Stream", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\", level=~\"$level\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Training Logs \u2014 $job", - "type": "logs" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 20, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 21, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(epoch|loss|reward|sharpe|accuracy|val_loss|train_loss|best_trial|trial|fold|window|learning_rate|lr=|MaxDD|win.?rate|drawdown)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Epoch / Loss / Reward / Sharpe", - "type": "logs" - } - ], - "title": "Epoch Progress & Loss", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 30, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 31, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"(hyperopt|train)-.*\"} |~ \"(?i)(trial|best|cost|param|learning_rate|hidden_dim|batch_size|buffer_size|gamma|epsilon|clip|PSO|iteration|objective|optimal)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Hyperopt Trial Results", - "type": "logs" - } - ], - "title": "Hyperopt Trials", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 40, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 41, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(GPU|CUDA|VRAM|cuda|gpu|vram|OOM|out.of.memory|nvrtc|tensor.core|BF16|bf16|SM|occupancy|kernel|device|nvidia)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "GPU / CUDA / VRAM Logs", - "type": "logs" - } - ], - "title": "GPU / CUDA / VRAM", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 50, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 51, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(walk.?forward|fold|backtest|window|train_start|train_end|val_start|val_end|test_start|test_end|embargo|purge|bars|features)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Walk-Forward / Fold / Backtest", - "type": "logs" - } - ], - "title": "Walk-Forward Windows", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 60, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 61, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked|OOM|NaN|inf|diverge)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "All Training Errors & Failures", - "type": "logs" - } - ], - "title": "Errors & Failures", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 70, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 71, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(loading|loaded|dbn|ofi|features|bars|MBP.?10|OHLCV|symbol|normalize|norm_stats|state_dim|rclone|S3|download)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "queryType": "range", - "refId": "A" - } - ], - "title": "Data Loading / OFI / DBN / Features", - "type": "logs" - } - ], - "title": "Data Loading & OFI", - "type": "row" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "foxhunt", - "training", - "loki", - "logs", - "gpu" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "All Training", - "value": "(train|hyperopt|compile-and-train)" - }, - "label": "Job Type", - "name": "job", - "options": [ - { - "selected": true, - "text": "All Training", - "value": "(train|hyperopt|compile-and-train)" - }, - { - "selected": false, - "text": "DQN", - "value": "train-dqn" - }, - { - "selected": false, - "text": "PPO", - "value": "train-ppo" - }, - { - "selected": false, - "text": "Supervised", - "value": "train-supervised" - }, - { - "selected": false, - "text": "Hyperopt", - "value": "hyperopt" - }, - { - "selected": false, - "text": "Compile+Train", - "value": "compile-and-train" - } - ], - "query": "All Training : (train|hyperopt|compile-and-train), DQN : train-dqn, PPO : train-ppo, Supervised : train-supervised, Hyperopt : hyperopt, Compile+Train : compile-and-train", - "type": "custom" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "includeAll": true, - "label": "Level", - "multi": true, - "name": "level", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "ERROR", - "value": "ERROR" - }, - { - "selected": false, - "text": "WARN", - "value": "WARN" - }, - { - "selected": false, - "text": "INFO", - "value": "INFO" - }, - { - "selected": false, - "text": "DEBUG", - "value": "DEBUG" - }, - { - "selected": false, - "text": "TRACE", - "value": "TRACE" - } - ], - "query": "ERROR,WARN,INFO,DEBUG,TRACE", - "type": "custom" - }, - { - "current": { - "selected": false, - "text": "", - "value": "" - }, - "label": "Search", - "name": "search", - "type": "textbox" - } - ] - }, - "time": { - "from": "now-12h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Training Pipeline Logs", - "uid": "foxhunt-training-logs", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json deleted file mode 100644 index 797583064..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ /dev/null @@ -1,4462 +0,0 @@ -{ - "description": "ML model training, hyperopt, GPU utilization, Argo CI pipelines, and full metrics coverage", - "editable": true, - "graphTooltip": 2, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Training Status", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(foxhunt_training_active_workers{job=\"training-pods\"}) or vector(0)", - "legendFormat": "Workers", - "refId": "A" - } - ], - "title": "Active Workers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current epoch and step within epoch", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 7, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}} / fold {{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_step{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "step {{model}}/{{fold}}", - "refId": "B" - } - ], - "title": "Current Epoch", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 3, - "y": 1 - }, - "id": 101, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "title": "Current Trial", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of improvement in best hyperopt objective (negative = improving for loss minimization)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "decimals": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 11, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "deriv(foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}[10m])", - "legendFormat": "\u0394 objective", - "refId": "A" - } - ], - "title": "Best Objective \u0394", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.3 - }, - { - "color": "green", - "value": 0.7 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 14, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"} / foxhunt_hyperopt_trial_total{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "title": "Hyperopt Trial Progress", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Errors" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 17, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "ml_training_jobs_started_total", - "legendFormat": "Jobs", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "ml_training_errors_total", - "legendFormat": "Errors", - "refId": "B" - } - ], - "title": "ML Jobs / Errors", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 99, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_elapsed_seconds{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "Elapsed", - "refId": "A" - } - ], - "title": "Hyperopt Elapsed", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 66, - "panels": [], - "title": "Run Summary", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.1 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 6 - }, - "id": 67, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", - "instant": false, - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Best Val Loss", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 3 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 6 - }, - "id": 68, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", - "instant": false, - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Best Sharpe", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.4 - }, - { - "color": "green", - "value": 0.55 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 6 - }, - "id": 69, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", - "instant": false, - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Best Win Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Objective", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "auto" - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 6 - }, - "id": 42, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "min" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "best {{model}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_trial_best_loss{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "trial loss {{model}}", - "refId": "B" - } - ], - "title": "Best Objective Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.1 - }, - { - "color": "red", - "value": 0.25 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 12 - }, - "id": 70, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", - "instant": false, - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Min Max Drawdown", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 12 - }, - "id": 71, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "instant": false, - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Total Epochs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 1.5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 12 - }, - "id": 72, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 20)", - "instant": false, - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Best Profit Factor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Duration", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 12 - }, - "id": 43, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.50, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", - "legendFormat": "p50 {{model}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", - "legendFormat": "p95 {{model}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", - "legendFormat": "p99 {{model}}", - "refId": "C" - } - ], - "title": "Trial Duration (p50/p95/p99)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 10, - "panels": [], - "title": "Training Curves", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "Loss", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never" - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "min" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "train {{model}}/{{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "val {{model}}/{{fold}}", - "refId": "B" - } - ], - "title": "Training & Validation Loss", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "LR", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "log": 10, - "type": "log" - }, - "showPoints": "never" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_learning_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Learning Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Norm", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 16, - "y": 19 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_gradient_norm{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Gradient Norm", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - } - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 19 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_batches_per_second{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "batch/s {{model}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_duration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "epoch dur {{model}}", - "refId": "B" - } - ], - "title": "Batches/sec & Epoch Duration", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 20, - "panels": [], - "title": "Training Quality", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Sharpe", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "auto" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 2 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 28 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "clamp_max(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 5)", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Epoch Sharpe Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Sortino", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "auto" - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 28 - }, - "id": 22, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "clamp_max(foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 10)", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Epoch Sortino Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Win Rate", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "auto" - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 28 - }, - "id": 23, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Win Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Drawdown", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "auto" - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 16, - "y": 28 - }, - "id": 24, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Max Drawdown", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "auto" - } - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.drawStyle", - "value": "bars" - }, - { - "id": "custom.fillOpacity", - "value": 30 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "foxhunt_training_epoch_profit_factor" - }, - "properties": [ - { - "id": "max", - "value": 20 - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 28 - }, - "id": 25, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "clamp_max(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 20)", - "legendFormat": "PF {{model}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_total_trades{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "trades {{model}}", - "refId": "B" - } - ], - "title": "Profit Factor & Total Trades", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 30, - "panels": [], - "title": "RL Diagnostics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Q-Value", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 36 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_q_value_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "mean {{model}}/{{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_q_value_max{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "max {{model}}/{{fold}}", - "refId": "B" - } - ], - "title": "Q-Value (Mean & Max)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-blues" - }, - "custom": { - "axisLabel": "Transitions", - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "scheme", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 6, - "y": 36 - }, - "id": 32, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_replay_buffer_size{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Replay Buffer Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Entropy", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "custom.thresholdsStyle": { - "mode": "line" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.1 - }, - { - "color": "green", - "value": 0.3 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 10, - "y": 36 - }, - "id": 33, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_action_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Action Entropy", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Fraction of 45 factored actions actually used by the policy.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Diversity", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 14, - "y": 36 - }, - "id": 34, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_action_diversity{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Action Diversity", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Pct", - "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 4, - "showPoints": "never", - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "buy.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "sell.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "hold.*" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 36 - }, - "id": 35, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_action_buy_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "buy {{model}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_action_sell_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "sell {{model}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_action_hold_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "hold {{model}}", - "refId": "C" - } - ], - "title": "Action Distribution (Buy/Sell/Hold %)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Q-max / |Q-mean| \u2014 spikes above 10 indicate DQN overestimation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 43 - }, - "id": 102, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_q_overestimation_ratio{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Q-Overestimation Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Policy entropy (exploration health) and KL divergence (policy drift)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 43 - }, - "id": 103, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_policy_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "entropy {{model}}/{{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_kl_divergence{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "KL {{model}}/{{fold}}", - "refId": "B" - } - ], - "title": "Policy Entropy & KL Divergence (PPO)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Mean advantage estimate \u2014 should hover near zero", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 43 - }, - "id": 104, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_advantage_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Advantage Mean (PPO)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 51 - }, - "id": 105, - "panels": [], - "title": "Training Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "NaN loss events, gradient explosions, and feature extraction failures", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 52 - }, - "id": 106, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(foxhunt_training_nan_detected_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", - "legendFormat": "NaN {{model}}/{{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(foxhunt_training_gradient_explosion_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", - "legendFormat": "grad explosion {{model}}/{{fold}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(foxhunt_training_feature_errors_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", - "legendFormat": "feature err {{model}}/{{fold}}", - "refId": "C" - } - ], - "title": "Training Anomalies", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Checkpoint save/failure rate", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 52 - }, - "id": 107, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(foxhunt_training_checkpoint_saves_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", - "legendFormat": "saves {{model}}/{{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(foxhunt_training_checkpoint_failures_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", - "legendFormat": "failures {{model}}/{{fold}}", - "refId": "B" - } - ], - "title": "Checkpoint Saves & Failures", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "p95 checkpoint save and data loading latency", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 52 - }, - "id": 108, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", - "legendFormat": "ckpt p95 {{model}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", - "legendFormat": "data load p95 {{model}}", - "refId": "B" - } - ], - "title": "Checkpoint & Data Load Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Size of last saved checkpoint", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 60 - }, - "id": 109, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Checkpoint Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total batches processed this run", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 60 - }, - "id": 110, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_batches_processed{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Batches Processed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Last epoch wall time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 60 - }, - "id": 111, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_iteration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Iteration Time", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 112, - "panels": [], - "title": "Epoch Returns", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Epoch total return (fractional)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 69 - }, - "id": 113, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_total_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Total Return per Epoch", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Average return per trade", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 69 - }, - "id": 114, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_avg_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ], - "title": "Avg Return per Trade", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 77 - }, - "id": 115, - "panels": [], - "title": "Supervised Model Eval", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Supervised eval accuracy and F1 score", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 78 - }, - "id": 116, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_accuracy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "accuracy {{model}}/{{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_f1{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "F1 {{model}}/{{fold}}", - "refId": "B" - } - ], - "title": "Accuracy & F1", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Supervised eval precision and recall", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 78 - }, - "id": 117, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_precision{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "precision {{model}}/{{fold}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_recall{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "recall {{model}}/{{fold}}", - "refId": "B" - } - ], - "title": "Precision & Recall", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 86 - }, - "id": 118, - "panels": [], - "title": "Hyperopt Details", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "1 = hyperopt active, 0 = training only", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 87 - }, - "id": 119, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_mode{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "title": "Hyperopt Mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current epoch within the active trial", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 4, - "y": 87 - }, - "id": 120, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_trial_epoch{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "title": "Hyperopt Trial Epoch", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Cumulative failed trial count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 3 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 14, - "y": 87 - }, - "id": 121, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_hyperopt_trials_failed_total{model=~\"$model\", job=~\"$source\"}", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "title": "Hyperopt Failed Trials", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 50, - "panels": [], - "title": "GPU Utilization", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "axisLabel": "Util %", - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "green", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 96 - }, - "id": 51, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_DEV_GPU_UTIL", - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "title": "GPU Utilization %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "MiB", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "decmbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "free.*" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 5 - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 10, - 5 - ], - "fill": "dash" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 96 - }, - "id": 52, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_DEV_FB_USED", - "legendFormat": "used GPU {{gpu}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_DEV_FB_FREE", - "legendFormat": "free GPU {{gpu}}", - "refId": "B" - } - ], - "title": "VRAM Used / Free (MiB)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Temp", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 85 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 96 - }, - "id": 53, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_DEV_GPU_TEMP", - "legendFormat": "GPU temp {{gpu}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_DEV_MEMORY_TEMP", - "legendFormat": "mem temp {{gpu}}", - "refId": "B" - } - ], - "title": "GPU / Memory Temperature", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Watts", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "watt" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 16, - "y": 96 - }, - "id": 54, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_DEV_POWER_USAGE", - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "title": "Power Usage (W)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Activity", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 96 - }, - "id": 55, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE", - "legendFormat": "tensor {{gpu}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_PROF_DRAM_ACTIVE", - "legendFormat": "DRAM {{gpu}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE", - "legendFormat": "engine {{gpu}}", - "refId": "C" - } - ], - "title": "Tensor Core & DRAM Activity", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 60, - "panels": [], - "title": "Argo CI/CD", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "Running" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Pending" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Failed|Error" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Succeeded" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 104 - }, - "id": 61, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "Running", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Pending\"}", - "legendFormat": "Pending", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Failed\"}", - "legendFormat": "Failed", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Succeeded\"}", - "legendFormat": "Succeeded", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Error\"}", - "legendFormat": "Error", - "refId": "E" - } - ], - "title": "Workflows by Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Pods", - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 6, - "y": 104 - }, - "id": 62, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_pods_gauge", - "legendFormat": "pods", - "refId": "A" - } - ], - "title": "Workflow Pods", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "custom": { - "axisLabel": "Errors/s", - "drawStyle": "bars", - "fillOpacity": 50, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 10, - "y": 104 - }, - "id": 63, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(argo_workflows_error_count[$__rate_interval])", - "legendFormat": "errors/s", - "refId": "A" - } - ], - "title": "Error Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Duration", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never" - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 14, - "y": 104 - }, - "id": 64, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95", - "refId": "B" - } - ], - "title": "Operation Duration (p50/p95)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Triggers/s", - "drawStyle": "bars", - "fillOpacity": 40, - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 4, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 19, - "y": 104 - }, - "id": 65, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(argo_workflows_workflowtemplate_triggered_total[$__rate_interval])", - "legendFormat": "triggers/s", - "refId": "A" - } - ], - "title": "Template Trigger Rate", - "type": "timeseries" - } - ], - "refresh": "15s", - "schemaVersion": 39, - "style": "dark", - "tags": [ - "foxhunt", - "training", - "gpu", - "ci" - ], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - }, - { - "current": { - "selected": true, - "text": "Live", - "value": "training-pods" - }, - "description": "Live = running training pods, CI History = completed CI runs (pushgateway), All = both", - "label": "Source", - "name": "source", - "options": [ - { - "selected": true, - "text": "Live", - "value": "training-pods" - }, - { - "selected": false, - "text": "CI History", - "value": ".+_baseline.*" - }, - { - "selected": false, - "text": "All", - "value": "training-pods|.+_baseline.*" - } - ], - "query": "Live : training-pods, CI History : .+_baseline.*, All : training-pods|.+_baseline.*", - "type": "custom" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "includeAll": true, - "label": "Model", - "multi": true, - "name": "model", - "query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)", - "refresh": 2, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "includeAll": true, - "label": "Fold", - "multi": true, - "name": "fold", - "query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)", - "refresh": 2, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Training & CI/CD", - "uid": "foxhunt-training", - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "type": "datasource", - "pluginId": "${DS_TEMPO}" - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh deleted file mode 100755 index 4193e2628..000000000 --- a/infra/k8s/monitoring/dashboards/import.sh +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Import all Grafana dashboards via API (Postgres-backed, no ConfigMaps). -# -# Dashboards use ${DS_PROMETHEUS}, ${DS_LOKI}, ${DS_TEMPO} variables. -# This script auto-detects datasource UIDs, resolves variables, and -# imports via the Grafana API. Dashboards are stored in Postgres. -# -# Required environment variables: -# GRAFANA_PASS - Password for admin user -# -# Optional overrides: -# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai) -# GRAFANA_USER - Grafana user (default: admin) -# GRAFANA_PROM_UID - Override Prometheus datasource UID -# GRAFANA_LOKI_UID - Override Loki datasource UID -# GRAFANA_TEMPO_UID - Override Tempo datasource UID -# -# Usage: -# GRAFANA_PASS="xxx" ./import.sh - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GRAFANA_USER="${GRAFANA_USER:-admin}" -GRAFANA_URL="${GRAFANA_URL:-https://grafana.fxhnt.ai}" -GRAFANA_URL="${GRAFANA_URL%/}" - -if [[ -z "${GRAFANA_PASS:-}" ]]; then - echo "ERROR: GRAFANA_PASS is not set" >&2 - exit 1 -fi - -# --- Datasource auto-detection --- - -detect_datasource_uid() { - local ds_type="$1" - curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/datasources" 2>/dev/null | \ - python3 -c " -import sys, json -for ds in json.load(sys.stdin): - if ds.get('type') == '${ds_type}': - print(ds['uid']) - break -" 2>/dev/null || true -} - -echo "Detecting datasource UIDs..." -PROM_UID="${GRAFANA_PROM_UID:-$(detect_datasource_uid prometheus)}" -LOKI_UID="${GRAFANA_LOKI_UID:-$(detect_datasource_uid loki)}" -TEMPO_UID="${GRAFANA_TEMPO_UID:-$(detect_datasource_uid tempo)}" - -echo " Prometheus: ${PROM_UID:-NOT FOUND}" -echo " Loki: ${LOKI_UID:-NOT FOUND}" -echo " Tempo: ${TEMPO_UID:-NOT FOUND}" - -if [[ -z "$PROM_UID" ]]; then - echo "ERROR: Could not detect Prometheus datasource. Set GRAFANA_PROM_UID." >&2 - exit 1 -fi - -# --- Resolve datasource variables --- - -resolve_dashboard() { - local file="$1" - python3 -c " -import sys, json - -with open('${file}') as f: - d = json.load(f) - -d.pop('__inputs', None) -d.pop('__requires', None) - -raw = json.dumps(d) -raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}') -raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}') -raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}') - -print(raw) -" -} - -# --- Ensure folders exist --- - -ensure_folder() { - local title="$1" - local uid - uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/folders" 2>/dev/null | \ - python3 -c " -import sys, json -for f in json.load(sys.stdin): - if f['title'] == '${title}': - print(f['uid']) - break -" 2>/dev/null || true) - - if [[ -z "$uid" ]]; then - uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" \ - -H "Content-Type: application/json" \ - -d "{\"title\": \"${title}\"}" \ - "${GRAFANA_URL}/api/folders" 2>/dev/null | \ - python3 -c "import sys,json; print(json.load(sys.stdin)['uid'])" 2>/dev/null) - echo " Created folder: ${title} (${uid})" - fi - echo "$uid" -} - -echo "" -echo "Ensuring folders..." -FOLDER_CI=$(ensure_folder "CI") -FOLDER_INFRA=$(ensure_folder "Infrastructure") -FOLDER_OPS=$(ensure_folder "Operations") -FOLDER_TRADING=$(ensure_folder "Trading") -FOLDER_TRAINING=$(ensure_folder "Training") -FOLDER_SERVICES=$(ensure_folder "Services") -FOLDER_PLATFORM=$(ensure_folder "Platform") - -# --- Import a single dashboard to a folder --- - -import_dashboard() { - local file="$1" - local folder_uid="$2" - local name - name="$(basename "$file" .json)" - - local resolved - resolved=$(resolve_dashboard "$file") - - local response - response=$(curl -s -w "\n%{http_code}" \ - -u "${GRAFANA_USER}:${GRAFANA_PASS}" \ - -H "Content-Type: application/json" \ - -d "{\"dashboard\": ${resolved}, \"folderUid\": \"${folder_uid}\", \"overwrite\": true}" \ - "${GRAFANA_URL}/api/dashboards/db") - - local http_code - http_code=$(echo "$response" | tail -1) - - if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then - echo " OK ${name}" - else - local body - body=$(echo "$response" | sed '$d') - echo " FAIL ${name} (${http_code}): ${body}" >&2 - return 1 - fi -} - -# --- Dashboard → Folder mapping --- - -declare -A DASHBOARD_FOLDERS=( - # Original dashboards - [foxhunt-cockpit]="${FOLDER_OPS}" - [foxhunt-trading]="${FOLDER_TRADING}" - [foxhunt-training]="${FOLDER_TRAINING}" - [foxhunt-infrastructure]="${FOLDER_INFRA}" - [foxhunt-observability]="${FOLDER_INFRA}" - [foxhunt-ci-logs]="${FOLDER_CI}" - [foxhunt-training-logs]="${FOLDER_TRAINING}" - [foxhunt-training-deep-dive]="${FOLDER_TRAINING}" - # Application services - [svc-api-gateway]="${FOLDER_SERVICES}" - [svc-trading-service]="${FOLDER_SERVICES}" - [svc-backtesting]="${FOLDER_SERVICES}" - [svc-data-acquisition]="${FOLDER_SERVICES}" - [svc-ml-training]="${FOLDER_SERVICES}" - [svc-trading-agent]="${FOLDER_SERVICES}" - [svc-web-gateway]="${FOLDER_SERVICES}" - # Platform services - [platform-prometheus]="${FOLDER_PLATFORM}" - [platform-grafana]="${FOLDER_PLATFORM}" - [platform-loki]="${FOLDER_PLATFORM}" - [platform-tempo]="${FOLDER_PLATFORM}" - [platform-redis]="${FOLDER_PLATFORM}" - [platform-postgres]="${FOLDER_PLATFORM}" - [platform-argo]="${FOLDER_PLATFORM}" - [platform-questdb]="${FOLDER_PLATFORM}" - [platform-costs]="${FOLDER_PLATFORM}" - [platform-scaleway]="${FOLDER_PLATFORM}" -) - -# --- Deploy all dashboards --- - -echo "" -echo "Importing dashboards..." - -failed=0 -total=0 - -for dashboard in "${SCRIPT_DIR}"/*.json; do - [[ -f "$dashboard" ]] || continue - name="$(basename "$dashboard" .json)" - folder_uid="${DASHBOARD_FOLDERS[$name]:-}" - - if [[ -z "$folder_uid" ]]; then - echo " SKIP ${name} (no folder mapping)" - continue - fi - - total=$((total + 1)) - if ! import_dashboard "$dashboard" "$folder_uid"; then - failed=$((failed + 1)) - fi -done - -echo "" -if [[ $failed -eq 0 ]]; then - echo "All ${total} dashboards deployed successfully." -else - echo "ERROR: ${failed}/${total} dashboard(s) failed." >&2 - exit 1 -fi diff --git a/infra/k8s/monitoring/dashboards/platform-argo.json b/infra/k8s/monitoring/dashboards/platform-argo.json deleted file mode 100644 index 4bf955d41..000000000 --- a/infra/k8s/monitoring/dashboards/platform-argo.json +++ /dev/null @@ -1,1632 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Argo Workflows Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Workflow Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Running\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Active Workflows", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 6 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "argo_workflows_gauge{status=\"Succeeded\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "title": "Succeeded", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 6 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "argo_workflows_gauge{status=\"Failed\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "title": "Failed", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 6 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "argo_workflows_pods_total_count{phase=\"Pending\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "title": "Pending Pods", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 6 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_workers_busy_count", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Workers Busy", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 6 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_queue_depth_gauge", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Queue Depth", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 17, - "panels": [], - "title": "Performance", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "Running", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_gauge{status=\"Pending\"}", - "legendFormat": "Pending", - "refId": "B" - } - ], - "title": "Workflow Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(argo_workflows_k8s_request_total[$interval])", - "legendFormat": "{{verb}} {{kind}}", - "refId": "A" - } - ], - "title": "K8s API Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "Operation Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 21, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(argo_workflows_queue_latency_bucket[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "Queue Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 22, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(argo_workflows_error_count[$interval])", - "legendFormat": "errors/s", - "refId": "A" - } - ], - "title": "Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 23, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_pods_total_count", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "title": "Pods Total", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 24, - "panels": [], - "title": "Go Runtime", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 29 - }, - "id": 25, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_process.runtime.go.goroutines", - "legendFormat": "goroutines", - "refId": "A" - } - ], - "title": "Goroutines", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 29 - }, - "id": 26, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_process.runtime.go.mem.heap_inuse", - "legendFormat": "in-use", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "argo_workflows_process.runtime.go.mem.heap_alloc", - "legendFormat": "alloc", - "refId": "B" - } - ], - "title": "Heap Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 29 - }, - "id": 27, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(argo_workflows_process.runtime.go.gc.pause_total_ns[$interval])", - "legendFormat": "ns/s", - "refId": "A" - } - ], - "title": "GC Pauses", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 28, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 38 - }, - "id": 29, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 38 - }, - "id": 30, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 38 - }, - "id": 31, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 46 - }, - "id": 32, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 33, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 57 - }, - "id": 34, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "argo", - "ci" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Argo Workflows", - "uid": "plat-argo" -} diff --git a/infra/k8s/monitoring/dashboards/platform-costs.json b/infra/k8s/monitoring/dashboards/platform-costs.json deleted file mode 100644 index 34769d505..000000000 --- a/infra/k8s/monitoring/dashboards/platform-costs.json +++ /dev/null @@ -1,1737 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 0, - "w": 24, - "h": 1 - }, - "id": 1, - "panels": [], - "title": "Cluster Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 0, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info)", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Total Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 3, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_pod_info{namespace=\"foxhunt\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Total Pods", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 6, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Running Pods", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 9, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_node_status_capacity{resource=\"cpu\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores (Total)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "x": 12, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_node_status_capacity{resource=\"memory\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (Total)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "x": 15, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Requested %", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "x": 18, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory Requested %", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 21, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_persistentvolumeclaim_info{namespace=\"foxhunt\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "PVCs", - "type": "stat" - }, - { - "id": 101, - "type": "row", - "title": "Actual Costs (Scaleway Billing API)", - "gridPos": { - "x": 0, - "y": 5, - "w": 24, - "h": 1 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 102, - "type": "stat", - "title": "Total MTD", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "scaleway_billing_total_eur{job=\"scaleway_billing\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "gridPos": { - "x": 0, - "y": 6, - "w": 4, - "h": 5 - }, - "fieldConfig": { - "defaults": { - "unit": "currencyEUR", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - } - }, - { - "id": 103, - "type": "stat", - "title": "Compute", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "scaleway_billing_consumption_eur{category=\"compute\",job=\"scaleway_billing\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "gridPos": { - "x": 4, - "y": 6, - "w": 4, - "h": 5 - }, - "fieldConfig": { - "defaults": { - "unit": "currencyEUR", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - }, - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - } - }, - { - "id": 104, - "type": "stat", - "title": "Storage", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "scaleway_billing_consumption_eur{category=\"storage\",job=\"scaleway_billing\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "gridPos": { - "x": 8, - "y": 6, - "w": 4, - "h": 5 - }, - "fieldConfig": { - "defaults": { - "unit": "currencyEUR", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - } - }, - { - "id": 105, - "type": "stat", - "title": "Network", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "scaleway_billing_consumption_eur{category=\"network\",job=\"scaleway_billing\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "gridPos": { - "x": 12, - "y": 6, - "w": 4, - "h": 5 - }, - "fieldConfig": { - "defaults": { - "unit": "currencyEUR", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - } - }, - { - "id": 106, - "type": "stat", - "title": "Registry", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "scaleway_billing_consumption_eur{category=\"containers\",job=\"scaleway_billing\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "range": false - } - ], - "gridPos": { - "x": 16, - "y": 6, - "w": 4, - "h": 5 - }, - "fieldConfig": { - "defaults": { - "unit": "currencyEUR", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - }, - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - } - }, - { - "id": 107, - "type": "barchart", - "title": "Cost Breakdown by Product", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sort_desc(scaleway_billing_product_eur{job=\"scaleway_billing\"})", - "legendFormat": "{{product}}", - "refId": "A", - "instant": true, - "range": false - } - ], - "gridPos": { - "x": 0, - "y": 16, - "w": 12, - "h": 10 - }, - "fieldConfig": { - "defaults": { - "unit": "currencyEUR", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "orientation": "horizontal", - "showValue": "always", - "xTickLabelRotation": 0, - "legend": { - "displayMode": "list", - "placement": "right" - } - } - }, - { - "id": 108, - "type": "piechart", - "title": "Cost by Category", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "scaleway_billing_consumption_eur{job=\"scaleway_billing\"}", - "legendFormat": "{{category}}", - "refId": "A", - "instant": true, - "range": false - } - ], - "gridPos": { - "x": 12, - "y": 16, - "w": 12, - "h": 10 - }, - "fieldConfig": { - "defaults": { - "unit": "currencyEUR", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "pieType": "donut", - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - } - } - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 26, - "w": 24, - "h": 1 - }, - "id": 10, - "panels": [], - "title": "Projected Costs (by node count)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 0, - "y": 27, - "w": 4, - "h": 5 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{node=~\".*platform.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Platform Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [] - }, - "gridPos": { - "x": 4, - "y": 27, - "w": 4, - "h": 5 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Platform Cost (\u20ac/hr)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [] - }, - "gridPos": { - "x": 8, - "y": 27, - "w": 4, - "h": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569 * 730", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Platform Cost (\u20ac/month)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 12, - "y": 27, - "w": 4, - "h": 5 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{node=~\".*h100.*\"}) or vector(0)", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "GPU Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "currencyEUR" - }, - "overrides": [] - }, - "gridPos": { - "x": 16, - "y": 27, - "w": 4, - "h": 5 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(count(kube_node_info{node=~\".*h100.*\"}) or vector(0)) * 5.08", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "GPU Cost (\u20ac/hr)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "x": 20, - "y": 27, - "w": 4, - "h": 5 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{node=~\".*compile.*\"}) or vector(0)", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CI CPU Nodes", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 32, - "w": 24, - "h": 1 - }, - "id": 17, - "panels": [], - "title": "Total Estimated Costs", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "normal" - } - }, - "unit": "currencyEUR" - }, - "overrides": [] - }, - "gridPos": { - "x": 0, - "y": 33, - "w": 12, - "h": 8 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569", - "legendFormat": "Platform", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(count(kube_node_info{node=~\".*h100.*\"}) or vector(0)) * 5.08", - "legendFormat": "GPU (H100)", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(count(kube_node_info{node=~\".*compile.*\"}) or vector(0)) * 0.569", - "legendFormat": "CI Compile", - "refId": "C" - } - ], - "title": "Hourly Compute Cost (\u20ac)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "normal" - } - }, - "unit": "currencyEUR" - }, - "overrides": [] - }, - "gridPos": { - "x": 12, - "y": 33, - "w": 12, - "h": 8 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569 * 730", - "legendFormat": "Platform", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(count(kube_node_info{node=~\".*h100.*\"}) or vector(0)) * 5.08 * 730", - "legendFormat": "GPU (H100)", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(count(kube_node_info{node=~\".*compile.*\"}) or vector(0)) * 0.569 * 730", - "legendFormat": "CI Compile", - "refId": "C" - } - ], - "title": "Projected Monthly Cost (\u20ac)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 41, - "w": 24, - "h": 1 - }, - "id": 20, - "panels": [], - "title": "Node Resource Usage", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "x": 0, - "y": 42, - "w": 12, - "h": 8 - }, - "id": 21, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "1 - avg by (node) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", - "legendFormat": "{{node}}", - "refId": "A" - } - ], - "title": "Node CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "x": 12, - "y": 42, - "w": 12, - "h": 8 - }, - "id": 22, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Node Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "x": 0, - "y": 50, - "w": 12, - "h": 8 - }, - "id": 23, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Node Disk Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "x": 12, - "y": 50, - "w": 12, - "h": 8 - }, - "id": 24, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(node_network_receive_bytes_total{device!=\"lo\"}[$interval])", - "legendFormat": "RX {{instance}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(node_network_transmit_bytes_total{device!=\"lo\"}[$interval])", - "legendFormat": "TX {{instance}}", - "refId": "B" - } - ], - "title": "Node Network", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 58, - "w": 24, - "h": 1 - }, - "id": 25, - "panels": [], - "title": "Namespace Resource Breakdown", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "x": 0, - "y": 59, - "w": 12, - "h": 8 - }, - "id": 26, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container!=\"\"}[$interval]))", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "CPU by Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "x": 12, - "y": 59, - "w": 12, - "h": 8 - }, - "id": 27, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", container!=\"\"})", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Memory by Service", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "costs", - "scaleway" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Cluster & Costs", - "uid": "plat-costs" -} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-grafana.json b/infra/k8s/monitoring/dashboards/platform-grafana.json deleted file mode 100644 index d69f1c22f..000000000 --- a/infra/k8s/monitoring/dashboards/platform-grafana.json +++ /dev/null @@ -1,802 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Grafana Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"grafana.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"grafana.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"grafana.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 16, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "grafana" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Grafana", - "uid": "plat-grafana" -} diff --git a/infra/k8s/monitoring/dashboards/platform-loki.json b/infra/k8s/monitoring/dashboards/platform-loki.json deleted file mode 100644 index 33f6dcc0b..000000000 --- a/infra/k8s/monitoring/dashboards/platform-loki.json +++ /dev/null @@ -1,1532 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Loki Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"loki.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"loki.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"loki.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Ingestion", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(loki_distributor_bytes_received_total[$interval])", - "legendFormat": "bytes/s", - "refId": "A" - } - ], - "title": "Log Ingestion Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "loki_ingester_streams_created_total - loki_ingester_streams_removed_total", - "legendFormat": "streams", - "refId": "A" - } - ], - "title": "Active Streams", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(loki_ingester_chunk_stored_bytes_total[$interval])", - "legendFormat": "bytes/s", - "refId": "A" - } - ], - "title": "Chunk Flushes", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Promtail", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"promtail.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 15 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"promtail.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 15 - }, - "id": 17, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 15 - }, - "id": 18, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"promtail.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 15 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 15 - }, - "id": 20, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 15 - }, - "id": 21, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"promtail.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 15 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"promtail.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 23, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(promtail_read_bytes_total[$interval])", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Promtail Read Bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 24, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "promtail_targets_active_total", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Promtail Targets", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 25, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 28 - }, - "id": 26, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 28 - }, - "id": 27, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 28 - }, - "id": 28, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 29, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 30, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 31, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "loki" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Loki & Promtail", - "uid": "plat-loki" -} diff --git a/infra/k8s/monitoring/dashboards/platform-postgres.json b/infra/k8s/monitoring/dashboards/platform-postgres.json deleted file mode 100644 index d8a639f7c..000000000 --- a/infra/k8s/monitoring/dashboards/platform-postgres.json +++ /dev/null @@ -1,1146 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "PostgreSQL Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"postgres.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"postgres.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"postgres.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Database Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "pg_stat_activity_count{namespace=\"foxhunt\", state=\"active\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Active Connections", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 6 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "pg_database_size_bytes{namespace=\"foxhunt\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "DB Size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 6 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "pg_stat_database_blks_hit{namespace=\"foxhunt\"} / (pg_stat_database_blks_hit{namespace=\"foxhunt\"} + pg_stat_database_blks_read{namespace=\"foxhunt\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Cache Hit Ratio", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 6 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(pg_stat_database_xact_commit{namespace=\"foxhunt\"}[5m])", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Transactions/s", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 6 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "pg_stat_database_deadlocks{namespace=\"foxhunt\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Deadlocks", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 6 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "pg_stat_database_temp_bytes{namespace=\"foxhunt\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Temp Files", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 17, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 21, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 22, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 23, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"} |~ \"(ERROR|WARNING|FATAL|PANIC):\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "postgres" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "PostgreSQL", - "uid": "plat-postgres" -} diff --git a/infra/k8s/monitoring/dashboards/platform-prometheus.json b/infra/k8s/monitoring/dashboards/platform-prometheus.json deleted file mode 100644 index 7a415ab1b..000000000 --- a/infra/k8s/monitoring/dashboards/platform-prometheus.json +++ /dev/null @@ -1,946 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Prometheus Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(up{job!=\"\"} == 1)", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Targets Up", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(up{job!=\"\"} == 0) or vector(0)", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Targets Down", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "prometheus_tsdb_head_series", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Head Series", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Ingestion Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "prometheus_tsdb_wal_storage_size_bytes", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "WAL Size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "prometheus_tsdb_head_chunks", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Head Chunks", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "prometheus_tsdb_compactions_total", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Compactions", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "prometheus_tsdb_storage_blocks_bytes", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Storage Size", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Scrape Performance", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "scrape_duration_seconds", - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Scrape Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(prometheus_tsdb_head_samples_appended_total[$interval])", - "legendFormat": "samples/s", - "refId": "A" - } - ], - "title": "Samples Ingested", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 17, - "panels": [], - "title": "Target Status", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 18, - "options": { - "showHeader": true, - "sortBy": [] - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "up", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Target Health", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 19, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 20, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 21, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "prometheus" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Prometheus", - "uid": "plat-prometheus" -} diff --git a/infra/k8s/monitoring/dashboards/platform-questdb.json b/infra/k8s/monitoring/dashboards/platform-questdb.json deleted file mode 100644 index 87e299857..000000000 --- a/infra/k8s/monitoring/dashboards/platform-questdb.json +++ /dev/null @@ -1,802 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "QuestDB Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"questdb.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"questdb.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"questdb.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 16, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "questdb" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "QuestDB", - "uid": "plat-questdb" -} diff --git a/infra/k8s/monitoring/dashboards/platform-redis.json b/infra/k8s/monitoring/dashboards/platform-redis.json deleted file mode 100644 index 4eae4ed15..000000000 --- a/infra/k8s/monitoring/dashboards/platform-redis.json +++ /dev/null @@ -1,1315 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Redis Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"redis.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"redis.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"redis.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Redis Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_connected_clients{namespace=\"foxhunt\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Connected Clients", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 6 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_memory_used_bytes{namespace=\"foxhunt\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Used Memory", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 6 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(redis_commands_processed_total{namespace=\"foxhunt\"}[5m])", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Ops/sec", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 6 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_keyspace_hits_total{namespace=\"foxhunt\"} / (redis_keyspace_hits_total{namespace=\"foxhunt\"} + redis_keyspace_misses_total{namespace=\"foxhunt\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Hit Ratio", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 6 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_db_keys{namespace=\"foxhunt\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Keys", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 6 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_evicted_keys_total{namespace=\"foxhunt\"}", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Evictions", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 17, - "panels": [], - "title": "Performance", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_memory_used_bytes{namespace=\"foxhunt\"}", - "legendFormat": "used", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_memory_max_bytes{namespace=\"foxhunt\"}", - "legendFormat": "max", - "refId": "B" - } - ], - "title": "Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(redis_commands_processed_total{namespace=\"foxhunt\"}[$interval])", - "legendFormat": "ops/s", - "refId": "A" - } - ], - "title": "Command Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "redis_connected_clients{namespace=\"foxhunt\"}", - "legendFormat": "clients", - "refId": "A" - } - ], - "title": "Connections", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 21, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 21 - }, - "id": 22, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 21 - }, - "id": 23, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 21 - }, - "id": 24, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 25, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 26, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 27, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"} |~ \"(# WARNING|# Error|ERR |FATAL)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "redis" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Redis", - "uid": "plat-redis" -} diff --git a/infra/k8s/monitoring/dashboards/platform-scaleway.json b/infra/k8s/monitoring/dashboards/platform-scaleway.json deleted file mode 100644 index d1ad6efde..000000000 --- a/infra/k8s/monitoring/dashboards/platform-scaleway.json +++ /dev/null @@ -1,2669 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 0, - "w": 24, - "h": 1 - }, - "id": 1, - "panels": [], - "title": "Scaleway Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Total Instances", - "gridPos": { - "x": 0, - "y": 1, - "w": 4, - "h": 4 - }, - "id": 2, - "targets": [ - { - "expr": "count(scaleway_instance_info)", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Running", - "gridPos": { - "x": 4, - "y": 1, - "w": 4, - "h": 4 - }, - "id": 3, - "targets": [ - { - "expr": "sum(scaleway_instance_running)", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "K8s Pools", - "gridPos": { - "x": 8, - "y": 1, - "w": 4, - "h": 4 - }, - "id": 4, - "targets": [ - { - "expr": "count(scaleway_pool_size)", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Block Volumes", - "gridPos": { - "x": 12, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 5, - "targets": [ - { - "expr": "scaleway_volumes_count{state=\"total\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Attached", - "gridPos": { - "x": 15, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 6, - "targets": [ - { - "expr": "scaleway_volumes_count{state=\"attached\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Detached (waste)", - "gridPos": { - "x": 18, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 7, - "targets": [ - { - "expr": "scaleway_volumes_count{state=\"detached\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 5 - }, - { - "color": "red", - "value": 15 - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "IPs", - "gridPos": { - "x": 21, - "y": 1, - "w": 3, - "h": 4 - }, - "id": 8, - "targets": [ - { - "expr": "scaleway_ips_total", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Total Storage", - "gridPos": { - "x": 0, - "y": 5, - "w": 6, - "h": 4 - }, - "id": 9, - "targets": [ - { - "expr": "scaleway_volumes_total_bytes", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "decbytes", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Attached Storage", - "gridPos": { - "x": 6, - "y": 5, - "w": 6, - "h": 4 - }, - "id": 10, - "targets": [ - { - "expr": "scaleway_volumes_attached_bytes", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Detached (wasted)", - "gridPos": { - "x": 12, - "y": 5, - "w": 6, - "h": 4 - }, - "id": 11, - "targets": [ - { - "expr": "scaleway_volumes_detached_bytes", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 200000000000.0 - }, - { - "color": "red", - "value": 500000000000.0 - } - ] - }, - "unit": "decbytes", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "MTD Cost", - "gridPos": { - "x": 18, - "y": 5, - "w": 6, - "h": 4 - }, - "id": 12, - "targets": [ - { - "expr": "scaleway_billing_total_eur", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 500 - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 9, - "w": 24, - "h": 1 - }, - "id": 13, - "panels": [], - "title": "Billing (Scaleway API \u2014 Month-to-Date)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Compute", - "gridPos": { - "x": 0, - "y": 10, - "w": 4, - "h": 4 - }, - "id": 14, - "targets": [ - { - "expr": "scaleway_billing_consumption_eur{category=\"compute\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#FF9830", - "value": null - } - ] - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Storage", - "gridPos": { - "x": 4, - "y": 10, - "w": 4, - "h": 4 - }, - "id": 15, - "targets": [ - { - "expr": "scaleway_billing_consumption_eur{category=\"storage\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#73BF69", - "value": null - } - ] - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Network", - "gridPos": { - "x": 8, - "y": 10, - "w": 4, - "h": 4 - }, - "id": 16, - "targets": [ - { - "expr": "scaleway_billing_consumption_eur{category=\"network\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#5794F2", - "value": null - } - ] - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Registry", - "gridPos": { - "x": 12, - "y": 10, - "w": 4, - "h": 4 - }, - "id": 17, - "targets": [ - { - "expr": "scaleway_billing_consumption_eur{category=\"container_registry\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#B877D9", - "value": null - } - ] - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Kubernetes", - "gridPos": { - "x": 16, - "y": 10, - "w": 4, - "h": 4 - }, - "id": 18, - "targets": [ - { - "expr": "scaleway_billing_consumption_eur{category=\"containers\"}", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#FF6E76", - "value": null - } - ] - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Other", - "gridPos": { - "x": 20, - "y": 10, - "w": 4, - "h": 4 - }, - "id": 19, - "targets": [ - { - "expr": "sum(scaleway_billing_consumption_eur) - sum(scaleway_billing_consumption_eur{category=~\"compute|storage|network|container_registry|containers\"})", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-yellow", - "value": null - } - ] - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "table", - "title": "Product Cost Breakdown", - "gridPos": { - "x": 0, - "y": 14, - "w": 12, - "h": 10 - }, - "id": 20, - "targets": [ - { - "expr": "scaleway_billing_product_eur", - "legendFormat": "{{category}} / {{product}}", - "refId": "A", - "instant": true, - "format": "table" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "unit", - "value": "currencyEUR" - }, - { - "id": "custom.width", - "value": 120 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "transformations": [ - { - "id": "sortBy", - "options": { - "sort": [ - { - "field": "Value", - "desc": true - } - ] - } - } - ], - "options": { - "showHeader": true, - "sortBy": [], - "cellHeight": "sm" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "piechart", - "title": "Cost by Category", - "gridPos": { - "x": 12, - "y": 14, - "w": 12, - "h": 10 - }, - "id": 21, - "targets": [ - { - "expr": "scaleway_billing_consumption_eur", - "legendFormat": "{{category}}", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "pieType": "donut", - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "tooltip": { - "mode": "single" - } - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "timeseries", - "title": "MTD Cost Over Time", - "gridPos": { - "x": 0, - "y": 24, - "w": 12, - "h": 7 - }, - "id": 22, - "targets": [ - { - "expr": "scaleway_billing_total_eur", - "legendFormat": "Total MTD", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "none" - } - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "timeseries", - "title": "Category Cost Over Time", - "gridPos": { - "x": 12, - "y": 24, - "w": 12, - "h": 7 - }, - "id": 23, - "targets": [ - { - "expr": "scaleway_billing_consumption_eur", - "legendFormat": "{{category}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 31, - "w": 24, - "h": 1 - }, - "id": 24, - "panels": [], - "title": "Compute Instances", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "table", - "title": "All Instances", - "gridPos": { - "x": 0, - "y": 32, - "w": 24, - "h": 8 - }, - "id": 25, - "targets": [ - { - "expr": "scaleway_instance_info", - "legendFormat": "", - "refId": "A", - "instant": true, - "format": "table" - }, - { - "expr": "scaleway_instance_running", - "legendFormat": "", - "refId": "B", - "instant": true, - "format": "table" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "state" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "running": { - "color": "green", - "text": "RUNNING" - }, - "stopped": { - "color": "red", - "text": "STOPPED" - }, - "stopping": { - "color": "orange", - "text": "STOPPING" - } - } - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Running" - }, - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "1": { - "text": "YES", - "color": "green" - }, - "0": { - "text": "NO", - "color": "red" - } - } - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "name" - }, - "properties": [ - { - "id": "displayName", - "value": "Instance Name" - }, - { - "id": "custom.width", - "value": 280 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "type" - }, - "properties": [ - { - "id": "displayName", - "value": "Instance Type" - }, - { - "id": "custom.width", - "value": 180 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "pool" - }, - "properties": [ - { - "id": "displayName", - "value": "K8s Pool" - }, - { - "id": "custom.width", - "value": 150 - } - ] - } - ] - }, - "transformations": [ - { - "id": "merge", - "options": {} - } - ], - "options": { - "showHeader": true, - "sortBy": [], - "cellHeight": "sm" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "bargauge", - "title": "Instances by Type", - "gridPos": { - "x": 0, - "y": 40, - "w": 24, - "h": 3 - }, - "id": 26, - "targets": [ - { - "expr": "scaleway_instances_total", - "legendFormat": "{{type}}", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "orientation": "horizontal", - "displayMode": "gradient", - "showUnfilled": true - } - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 44, - "w": 24, - "h": 1 - }, - "id": 27, - "panels": [], - "title": "Kubernetes Node Pools", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "table", - "title": "All Pools (Autoscaler Config)", - "gridPos": { - "x": 0, - "y": 45, - "w": 24, - "h": 8 - }, - "id": 28, - "targets": [ - { - "expr": "scaleway_pool_size", - "legendFormat": "", - "refId": "A", - "instant": true, - "format": "table" - }, - { - "expr": "scaleway_pool_min", - "legendFormat": "", - "refId": "B", - "instant": true, - "format": "table" - }, - { - "expr": "scaleway_pool_max", - "legendFormat": "", - "refId": "C", - "instant": true, - "format": "table" - }, - { - "expr": "scaleway_pool_autoscaling", - "legendFormat": "", - "refId": "D", - "instant": true, - "format": "table" - }, - { - "expr": "scaleway_pool_root_volume_bytes", - "legendFormat": "", - "refId": "E", - "instant": true, - "format": "table" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "pool" - }, - "properties": [ - { - "id": "displayName", - "value": "Pool Name" - }, - { - "id": "custom.width", - "value": 200 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "type" - }, - "properties": [ - { - "id": "displayName", - "value": "Node Type" - }, - { - "id": "custom.width", - "value": 180 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "status" - }, - "properties": [ - { - "id": "displayName", - "value": "Status" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "ready": { - "color": "green", - "text": "READY" - }, - "scaling": { - "color": "yellow", - "text": "SCALING" - }, - "warning": { - "color": "orange", - "text": "WARNING" - } - } - } - ] - }, - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "Current Size" - }, - { - "id": "custom.width", - "value": 110 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Min" - }, - { - "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "Max" - }, - { - "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #D" - }, - "properties": [ - { - "id": "displayName", - "value": "Autoscaling" - }, - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "1": { - "text": "ON", - "color": "green" - }, - "0": { - "text": "OFF", - "color": "red" - } - } - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "custom.width", - "value": 110 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #E" - }, - "properties": [ - { - "id": "displayName", - "value": "Root Volume" - }, - { - "id": "unit", - "value": "decbytes" - }, - { - "id": "custom.width", - "value": 120 - } - ] - } - ] - }, - "transformations": [ - { - "id": "merge", - "options": {} - } - ], - "options": { - "showHeader": true, - "sortBy": [], - "cellHeight": "sm" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "timeseries", - "title": "Pool Sizes Over Time", - "gridPos": { - "x": 0, - "y": 53, - "w": 24, - "h": 7 - }, - "id": 29, - "targets": [ - { - "expr": "scaleway_pool_size", - "legendFormat": "{{pool}} ({{type}})", - "refId": "A" - }, - { - "expr": "scaleway_pool_min", - "legendFormat": "{{pool}} min", - "refId": "B" - }, - { - "expr": "scaleway_pool_max", - "legendFormat": "{{pool}} max", - "refId": "C" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "none" - } - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 61, - "w": 24, - "h": 1 - }, - "id": 30, - "panels": [], - "title": "Block Storage", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "table", - "title": "All Block Volumes", - "gridPos": { - "x": 0, - "y": 62, - "w": 24, - "h": 12 - }, - "id": 31, - "targets": [ - { - "expr": "scaleway_volume_size_bytes", - "legendFormat": "", - "refId": "A", - "instant": true, - "format": "table" - }, - { - "expr": "scaleway_volume_attached", - "legendFormat": "", - "refId": "B", - "instant": true, - "format": "table" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "name" - }, - "properties": [ - { - "id": "displayName", - "value": "Volume Name" - }, - { - "id": "custom.width", - "value": 400 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "state" - }, - "properties": [ - { - "id": "displayName", - "value": "State" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "in_use": { - "color": "green", - "text": "IN USE" - }, - "available": { - "color": "red", - "text": "AVAILABLE (detached)" - } - } - } - ] - }, - { - "id": "custom.width", - "value": 200 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "Size" - }, - { - "id": "unit", - "value": "decbytes" - }, - { - "id": "custom.width", - "value": 120 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Attached" - }, - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "1": { - "text": "YES", - "color": "green" - }, - "0": { - "text": "NO \u2014 WASTED", - "color": "red" - } - } - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "custom.width", - "value": 140 - } - ] - } - ] - }, - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "sortBy", - "options": { - "sort": [ - { - "field": "Value #B", - "desc": false - } - ] - } - } - ], - "options": { - "showHeader": true, - "sortBy": [], - "cellHeight": "sm" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "timeseries", - "title": "Storage Over Time", - "gridPos": { - "x": 0, - "y": 74, - "w": 12, - "h": 7 - }, - "id": 32, - "targets": [ - { - "expr": "scaleway_volumes_total_bytes", - "legendFormat": "Total", - "refId": "A" - }, - { - "expr": "scaleway_volumes_attached_bytes", - "legendFormat": "Attached", - "refId": "B" - }, - { - "expr": "scaleway_volumes_detached_bytes", - "legendFormat": "Detached (waste)", - "refId": "C" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "none" - } - }, - "unit": "decbytes", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "gauge", - "title": "Storage Waste %", - "gridPos": { - "x": 12, - "y": 74, - "w": 6, - "h": 7 - }, - "id": 33, - "targets": [ - { - "expr": "scaleway_volumes_detached_bytes / scaleway_volumes_total_bytes", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.3 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Volumes Count", - "gridPos": { - "x": 18, - "y": 74, - "w": 6, - "h": 7 - }, - "id": 34, - "targets": [ - { - "expr": "scaleway_volumes_count", - "legendFormat": "{{state}}", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "auto" - } - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 82, - "w": 24, - "h": 1 - }, - "id": 35, - "panels": [], - "title": "Network & IPs", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "stat", - "title": "Total IPv4 IPs", - "gridPos": { - "x": 0, - "y": 83, - "w": 6, - "h": 4 - }, - "id": 36, - "targets": [ - { - "expr": "scaleway_ips_total", - "legendFormat": "", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "timeseries", - "title": "IP Count Over Time", - "gridPos": { - "x": 6, - "y": 83, - "w": 18, - "h": 4 - }, - "id": 37, - "targets": [ - { - "expr": "scaleway_ips_total", - "legendFormat": "IPv4 IPs", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "none" - } - }, - "unit": "short", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "collapsed": false, - "gridPos": { - "x": 0, - "y": 87, - "w": 24, - "h": 1 - }, - "id": 38, - "panels": [], - "title": "Projected Costs (Node-Based Estimates)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "timeseries", - "title": "Estimated Hourly Cost", - "gridPos": { - "x": 0, - "y": 88, - "w": 12, - "h": 7 - }, - "id": 39, - "targets": [ - { - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"platform\"}) * 0.569", - "legendFormat": "Platform (DEV1-L)", - "refId": "A" - }, - { - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gpu\"}) * 5.08", - "legendFormat": "GPU (H100)", - "refId": "B" - }, - { - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=~\"ci-.*\"}) * 0.569", - "legendFormat": "CI Nodes", - "refId": "C" - }, - { - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"infra\"}) * 0.142", - "legendFormat": "Infra (DEV1-M)", - "refId": "D" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "type": "timeseries", - "title": "Estimated Monthly Cost", - "gridPos": { - "x": 12, - "y": 88, - "w": 12, - "h": 7 - }, - "id": 40, - "targets": [ - { - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"platform\"}) * 0.569 * 730", - "legendFormat": "Platform", - "refId": "A" - }, - { - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gpu\"}) * 5.08 * 730", - "legendFormat": "GPU (H100)", - "refId": "B" - }, - { - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=~\"ci-.*\"}) * 0.569 * 730", - "legendFormat": "CI Nodes", - "refId": "C" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - }, - "unit": "currencyEUR", - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ], - "schemaVersion": 39, - "tags": [ - "platform", - "scaleway", - "costs", - "infrastructure" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Scaleway Infrastructure Cockpit", - "uid": "scaleway-cockpit", - "version": 1, - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - } - ] -} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-tempo.json b/infra/k8s/monitoring/dashboards/platform-tempo.json deleted file mode 100644 index c8d7c3db0..000000000 --- a/infra/k8s/monitoring/dashboards/platform-tempo.json +++ /dev/null @@ -1,802 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Tempo Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 16, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "platform", - "${DS_TEMPO}" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Tempo", - "uid": "plat-${DS_TEMPO}" -} diff --git a/infra/k8s/monitoring/dashboards/svc-api-gateway.json b/infra/k8s/monitoring/dashboards/svc-api-gateway.json deleted file mode 100644 index 27a4803b5..000000000 --- a/infra/k8s/monitoring/dashboards/svc-api-gateway.json +++ /dev/null @@ -1,963 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"api.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"api.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "gRPC Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(grpc_server_handled_total{grpc_service=~\"foxhunt.*\", pod=~\"api.*\"}[$interval])", - "legendFormat": "{{grpc_method}}", - "refId": "A" - } - ], - "title": "gRPC Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"api.*\"}[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "gRPC Latency (p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 15 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(grpc_server_handled_total{grpc_code!=\"OK\", pod=~\"api.*\"}[$interval])", - "legendFormat": "{{grpc_code}}", - "refId": "A" - } - ], - "title": "gRPC Errors", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 18, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 19, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 20, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "service", - "api" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "API Gateway", - "uid": "svc-api-gateway" -} diff --git a/infra/k8s/monitoring/dashboards/svc-backtesting.json b/infra/k8s/monitoring/dashboards/svc-backtesting.json deleted file mode 100644 index 129bd2228..000000000 --- a/infra/k8s/monitoring/dashboards/svc-backtesting.json +++ /dev/null @@ -1,1075 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Backtesting Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_backtest_duration_seconds{pod=~\"backtesting.*\"}", - "legendFormat": "{{strategy}}", - "refId": "A" - } - ], - "title": "Backtest Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_backtest_sharpe{pod=~\"backtesting.*\"}", - "legendFormat": "{{strategy}}", - "refId": "A" - } - ], - "title": "Backtest Sharpe", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 15 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_backtest_windows_total{pod=~\"backtesting.*\"}", - "legendFormat": "{{strategy}}", - "refId": "A" - } - ], - "title": "Walk-Forward Windows", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 18, - "panels": [], - "title": "gRPC", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(grpc_server_handled_total{pod=~\"backtesting.*\"}[$interval])", - "legendFormat": "{{grpc_method}}", - "refId": "A" - } - ], - "title": "gRPC Handled", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"backtesting.*\"}[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "gRPC Latency", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 21, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 22, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 23, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "service", - "backtesting" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Backtesting Service", - "uid": "svc-backtesting" -} diff --git a/infra/k8s/monitoring/dashboards/svc-data-acquisition.json b/infra/k8s/monitoring/dashboards/svc-data-acquisition.json deleted file mode 100644 index 58b9a3d8c..000000000 --- a/infra/k8s/monitoring/dashboards/svc-data-acquisition.json +++ /dev/null @@ -1,1031 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Data Feed Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_data_acquisition_feeds_active", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Active Feeds", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 6 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_data_acquisition_errors_total", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Errors", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 6 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_data_acquisition_uptime_seconds", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_data_acquisition_feeds_active", - "legendFormat": "Active Feeds", - "refId": "A" - } - ], - "title": "Feed Activity", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 15, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 19, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 20, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 21, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "service", - "data" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Data Acquisition", - "uid": "svc-data-acq" -} diff --git a/infra/k8s/monitoring/dashboards/svc-ml-training.json b/infra/k8s/monitoring/dashboards/svc-ml-training.json deleted file mode 100644 index 1d792ec13..000000000 --- a/infra/k8s/monitoring/dashboards/svc-ml-training.json +++ /dev/null @@ -1,1026 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"ml-training.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Job Management", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"training-.*\", phase=\"Running\"}", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Active Training Jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"training-.*\", phase=\"Succeeded\"}", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Job Completions", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 17, - "panels": [], - "title": "gRPC", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(grpc_server_handled_total{pod=~\"ml-training.*\"}[$interval])", - "legendFormat": "{{grpc_method}}", - "refId": "A" - } - ], - "title": "gRPC Handled", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"ml-training.*\"}[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "gRPC Latency", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 20, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 21, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 22, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "service", - "ml", - "training" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "ML Training Service", - "uid": "svc-ml-training" -} diff --git a/infra/k8s/monitoring/dashboards/svc-trading-agent.json b/infra/k8s/monitoring/dashboards/svc-trading-agent.json deleted file mode 100644 index 7e97104b7..000000000 --- a/infra/k8s/monitoring/dashboards/svc-trading-agent.json +++ /dev/null @@ -1,1075 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"trading-agent.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Agent Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_agent_ensemble_confidence{pod=~\"trading-agent.*\"}", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "title": "Ensemble Confidence", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(foxhunt_agent_signals_total{pod=~\"trading-agent.*\"}[$interval])", - "legendFormat": "{{signal_type}}", - "refId": "A" - } - ], - "title": "Signal Generation Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 15 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(foxhunt_agent_decision_seconds_bucket{pod=~\"trading-agent.*\"}[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "Decision Latency", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 18, - "panels": [], - "title": "gRPC", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(grpc_server_handled_total{pod=~\"trading-agent.*\"}[$interval])", - "legendFormat": "{{grpc_method}}", - "refId": "A" - } - ], - "title": "gRPC Handled", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"trading-agent.*\"}[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "gRPC Latency", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 21, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 22, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 23, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "service", - "agent" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Trading Agent", - "uid": "svc-trading-agent" -} diff --git a/infra/k8s/monitoring/dashboards/svc-trading-service.json b/infra/k8s/monitoring/dashboards/svc-trading-service.json deleted file mode 100644 index 768f1c409..000000000 --- a/infra/k8s/monitoring/dashboards/svc-trading-service.json +++ /dev/null @@ -1,1074 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"trading-service.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 14, - "panels": [], - "title": "Trading Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_trading_sharpe_ratio{pod=~\"trading-service.*\"}", - "legendFormat": "Sharpe", - "refId": "A" - } - ], - "title": "Ensemble Sharpe Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_trading_open_positions{pod=~\"trading-service.*\"}", - "legendFormat": "Open", - "refId": "A" - } - ], - "title": "Position Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 15 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(foxhunt_trading_orders_total{pod=~\"trading-service.*\"}[$interval])", - "legendFormat": "{{side}}", - "refId": "A" - } - ], - "title": "Order Rate", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 18, - "panels": [], - "title": "gRPC", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(grpc_server_handled_total{pod=~\"trading-service.*\"}[$interval])", - "legendFormat": "{{grpc_method}}", - "refId": "A" - } - ], - "title": "gRPC Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"trading-service.*\"}[$interval]))", - "legendFormat": "p99", - "refId": "A" - } - ], - "title": "gRPC Latency", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 21, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 22, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 23, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "service", - "trading" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Trading Service", - "uid": "svc-trading" -} diff --git a/infra/k8s/monitoring/dashboards/svc-web-gateway.json b/infra/k8s/monitoring/dashboards/svc-web-gateway.json deleted file mode 100644 index 0d4a926d0..000000000 --- a/infra/k8s/monitoring/dashboards/svc-web-gateway.json +++ /dev/null @@ -1,1022 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [], - "title": "Service Health", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}", - "instant": true, - "legendFormat": "Running", - "range": false, - "refId": "A" - } - ], - "title": "Pod Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"web-gateway.*\", resource=\"memory\"})", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"})", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Memory (bytes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 1 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network RX", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 1 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[5m]))", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Network TX", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [], - "title": "Gateway Metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 6 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_web_gateway_ws_connections_active", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Active WS Connections", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 6 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_web_gateway_uptime_seconds", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_web_gateway_ws_connections_active", - "legendFormat": "Active", - "refId": "A" - } - ], - "title": "WebSocket Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(foxhunt_web_gateway_http_requests_total{pod=~\"web-gateway.*\"}[$interval])", - "legendFormat": "{{method}} {{path}}", - "refId": "A" - } - ], - "title": "HTTP Request Rate", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 15, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}[$interval])", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}", - "legendFormat": "{{container}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "stacking": { - "mode": "none" - } - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[$interval])", - "legendFormat": "RX {{pod}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[$interval])", - "legendFormat": "TX {{pod}}", - "refId": "B" - } - ], - "title": "Network I/O", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 19, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 20, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Log Stream", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 21, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", - "refId": "A" - } - ], - "title": "Errors & Warnings", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "service", - "web" - ], - "templating": { - "list": [ - { - "auto": false, - "current": { - "text": "5m", - "value": "5m" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h", - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "title": "Web Gateway", - "uid": "svc-web-gateway" -} diff --git a/infra/k8s/monitoring/dcgm-exporter.yaml b/infra/k8s/monitoring/dcgm-exporter.yaml deleted file mode 100644 index 8e798e96b..000000000 --- a/infra/k8s/monitoring/dcgm-exporter.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# NVIDIA DCGM Exporter — GPU metrics for Cockpit -# Runs on all GPU node pools (ci-training, gpu-dev) -# Alloy auto-discovers and scrapes these pods via annotations ---- -apiVersion: v1 -kind: Namespace -metadata: - name: monitoring - labels: - app.kubernetes.io/part-of: foxhunt ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: dcgm-counters - namespace: monitoring -data: - counters.csv: | - # Basic GPU metrics (no DCP/profiling — avoids SYS_ADMIN requirement) - dcgm_sm_clock, gauge, SM clock frequency (in MHz). - dcgm_memory_clock, gauge, Memory clock frequency (in MHz). - dcgm_memory_temp, gauge, Memory temperature (in C). - dcgm_gpu_temp, gauge, GPU temperature (in C). - dcgm_power_usage, gauge, Power draw (in W). - dcgm_total_energy_consumption, counter, Total energy consumption since boot (in mJ). - dcgm_gpu_utilization, gauge, GPU utilization (in %). - dcgm_mem_copy_utilization, gauge, Memory utilization (in %). - dcgm_enc_utilization, gauge, Encoder utilization (in %). - dcgm_dec_utilization, gauge, Decoder utilization (in %). - dcgm_xid_errors, gauge, Value of the last XID error encountered. - dcgm_fb_free, gauge, Framebuffer memory free (in MiB). - dcgm_fb_used, gauge, Framebuffer memory used (in MiB). ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: dcgm-exporter - namespace: monitoring - labels: - app.kubernetes.io/name: dcgm-exporter - app.kubernetes.io/part-of: foxhunt -spec: - selector: - matchLabels: - app.kubernetes.io/name: dcgm-exporter - template: - metadata: - labels: - app.kubernetes.io/name: dcgm-exporter - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9400" - prometheus.io/path: "/metrics" - spec: - # Only schedule on nodes that have GPUs - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.present - operator: In - values: - - "true" - # NOTE: runtimeClassName: nvidia removed — Scaleway Kapsule GPU pools use - # the nvidia container runtime as the default containerd handler. The - # RuntimeClass CRD is only created by the full NVIDIA GPU Operator, not - # the device plugin alone, so requiring it prevents pod scheduling. - tolerations: - # Tolerate GPU taints so the exporter can run on GPU nodes - - operator: Exists - effect: NoSchedule - containers: - - name: dcgm-exporter - image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04 - ports: - - name: metrics - containerPort: 9400 - args: - - "-f" - - "/etc/dcgm-exporter/custom/counters.csv" - env: - - name: DCGM_EXPORTER_KUBERNETES - value: "true" - resources: - requests: - cpu: 50m - memory: 256Mi - limits: - cpu: 200m - memory: 1Gi - securityContext: - runAsNonRoot: false - runAsUser: 0 - volumeMounts: - - name: device-metrics - mountPath: /var/lib/dcgm - - name: custom-counters - mountPath: /etc/dcgm-exporter/custom - readOnly: true - volumes: - - name: device-metrics - emptyDir: {} - - name: custom-counters - configMap: - name: dcgm-counters ---- -apiVersion: v1 -kind: Service -metadata: - name: dcgm-exporter - namespace: monitoring - labels: - app.kubernetes.io/name: dcgm-exporter - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9400" -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: dcgm-exporter - ports: - - name: metrics - port: 9400 - targetPort: 9400 diff --git a/infra/k8s/monitoring/grafana-datasources.yaml b/infra/k8s/monitoring/grafana-datasources.yaml deleted file mode 100644 index 9210c5119..000000000 --- a/infra/k8s/monitoring/grafana-datasources.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Grafana datasource provisioning for Loki, Tempo, and QuestDB -# Apply via: kubectl apply -f grafana-datasources.yaml && kubectl rollout restart deploy/grafana -n foxhunt -apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-datasources-extra - namespace: foxhunt - labels: - app.kubernetes.io/name: grafana - grafana_datasource: "1" -data: - loki-tempo.yaml: | - apiVersion: 1 - datasources: - - name: Loki - type: loki - access: proxy - url: http://loki.foxhunt.svc.cluster.local:3100 - isDefault: false - jsonData: - derivedFields: - - datasourceUid: tempo - matcherRegex: '"trace_id":"(\w+)"' - name: TraceID - url: "$${__value.raw}" - - name: Tempo - type: tempo - uid: tempo - access: proxy - url: http://tempo.foxhunt.svc.cluster.local:3200 - isDefault: false - jsonData: - tracesToLogsV2: - datasourceUid: loki - filterByTraceID: true - nodeGraph: - enabled: true - serviceMap: - datasourceUid: prometheus - questdb.yaml: | - apiVersion: 1 - datasources: - - name: QuestDB - type: postgres - uid: questdb - access: proxy - url: questdb.foxhunt.svc.cluster.local:8812 - user: admin - database: qdb - isDefault: false - jsonData: - sslmode: disable - maxOpenConns: 5 - postgresVersion: 1200 - timescaledb: false - secureJsonData: - password: quest diff --git a/infra/k8s/monitoring/loki.yaml b/infra/k8s/monitoring/loki.yaml deleted file mode 100644 index 9fdbbef53..000000000 --- a/infra/k8s/monitoring/loki.yaml +++ /dev/null @@ -1,122 +0,0 @@ -# Grafana Loki — lightweight log aggregation (single-binary mode) -# Runs on gitlab node pool alongside Grafana and Prometheus -apiVersion: v1 -kind: ConfigMap -metadata: - name: loki-config - namespace: foxhunt - labels: - app.kubernetes.io/name: loki -data: - loki.yaml: | - auth_enabled: false - server: - http_listen_port: 3100 - grpc_listen_port: 9096 - common: - path_prefix: /loki - storage: - filesystem: - chunks_directory: /loki/chunks - rules_directory: /loki/rules - replication_factor: 1 - ring: - kvstore: - store: inmemory - schema_config: - configs: - - from: "2024-01-01" - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - limits_config: - retention_period: 168h # 7 days - max_query_series: 500 - compactor: - working_directory: /loki/compactor - compaction_interval: 10m - retention_enabled: true - delete_request_store: filesystem ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: loki - namespace: foxhunt - labels: - app.kubernetes.io/name: loki -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: loki - template: - metadata: - labels: - app.kubernetes.io/name: loki - spec: - nodeSelector: - k8s.scaleway.com/pool-name: infra - containers: - - name: loki - image: grafana/loki:3.4.2 - args: ["-config.file=/etc/loki/loki.yaml"] - ports: - - containerPort: 3100 - name: http - - containerPort: 9096 - name: grpc - volumeMounts: - - name: config - mountPath: /etc/loki - - name: data - mountPath: /loki - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi - readinessProbe: - httpGet: - path: /ready - port: 3100 - initialDelaySeconds: 10 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /ready - port: 3100 - initialDelaySeconds: 30 - periodSeconds: 15 - volumes: - - name: config - configMap: - name: loki-config - - name: data - emptyDir: - sizeLimit: 5Gi ---- -apiVersion: v1 -kind: Service -metadata: - name: loki - namespace: foxhunt - labels: - app.kubernetes.io/name: loki -spec: - selector: - app.kubernetes.io/name: loki - ports: - - port: 3100 - targetPort: 3100 - name: http - - port: 9096 - targetPort: 9096 - name: grpc diff --git a/infra/k8s/monitoring/prometheus-rules-broker.yaml b/infra/k8s/monitoring/prometheus-rules-broker.yaml deleted file mode 100644 index 354467b41..000000000 --- a/infra/k8s/monitoring/prometheus-rules-broker.yaml +++ /dev/null @@ -1,169 +0,0 @@ -# Broker Gateway alert rules — discovered automatically by Prometheus Operator -# Apply: kubectl apply -f prometheus-rules-broker.yaml -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: foxhunt-broker-alerts - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt - role: alert-rules -spec: - groups: - - name: broker_gateway_fix_session - interval: 30s - rules: - # FIX session disconnected >60s - - alert: BrokerGatewayFIXSessionDisconnected - expr: broker_gateway_fix_session_status == 0 - for: 60s - labels: - severity: critical - service: broker_gateway_service - component: fix_session - impact: trading_halted - annotations: - summary: "FIX session {{ $labels.session_id }} DISCONNECTED" - description: "FIX session down >60s. All order routing halted." - action: "Check FIX engine logs, verify network to AMP Futures gateway" - - # Order latency P95 > 100ms - - alert: BrokerGatewayHighOrderLatency - expr: | - histogram_quantile(0.95, - sum by(le, order_type) (rate(broker_gateway_order_latency_seconds_bucket[5m])) - ) * 1000 > 100 - for: 5m - labels: - severity: warning - service: broker_gateway_service - component: order_routing - impact: performance_degraded - annotations: - summary: "Order latency P95 > 100ms for {{ $labels.order_type }}" - - # Error rate > 5% - - alert: BrokerGatewayHighErrorRate - expr: | - ( - sum by(severity) (rate(broker_gateway_error_total[5m])) - / - (sum(rate(broker_gateway_orders_submitted_total[5m])) + 0.001) - ) > 0.05 - for: 5m - labels: - severity: warning - service: broker_gateway_service - component: error_handling - impact: reliability_degraded - annotations: - summary: "Error rate {{ $value | humanizePercentage }} ({{ $labels.severity }})" - - # No orders in 10m during market hours - - alert: BrokerGatewayNoOrderActivity - expr: | - (time() - broker_gateway_last_order_time) > 600 - and - (hour() >= 9 and hour() < 16) - for: 1m - labels: - severity: warning - service: broker_gateway_service - component: order_flow - impact: trading_inactive - annotations: - summary: "No order activity for 10+ minutes during market hours" - - # Position mismatch > $10K - - alert: BrokerGatewayPositionMismatch - expr: | - abs( - broker_gateway_position_value_usd - - - on(symbol, account_id) group_left() - trading_service_position_value_usd - ) > 10000 - for: 2m - labels: - severity: critical - service: broker_gateway_service - component: position_management - impact: data_integrity - annotations: - summary: "Position mismatch ${{ $value | humanize }} on {{ $labels.symbol }}" - action: "HALT trading on affected symbol, reconcile with broker" - - # FIX sequence number gap - - alert: BrokerGatewayFIXSequenceGap - expr: rate(broker_gateway_sequence_number_gap_total[5m]) > 0 - for: 1m - labels: - severity: warning - service: broker_gateway_service - component: fix_session - impact: message_loss - annotations: - summary: "FIX sequence gap on {{ $labels.session_id }} ({{ $value | humanize }} gaps/sec)" - - - name: broker_gateway_performance - interval: 60s - rules: - # High FIX heartbeat RTT - - alert: BrokerGatewayHighFIXHeartbeatRTT - expr: broker_gateway_fix_heartbeat_rtt_ms > 50 - for: 5m - labels: - severity: warning - service: broker_gateway_service - component: fix_session - impact: network_latency - annotations: - summary: "FIX heartbeat RTT {{ $value }}ms on {{ $labels.session_id }}" - - # High order rejection rate per symbol - - alert: BrokerGatewayHighOrderRejectionRate - expr: | - ( - sum by(symbol) (rate(broker_gateway_orders_rejected_total[5m])) - / - (sum by(symbol) (rate(broker_gateway_orders_submitted_total[5m])) + 0.001) - ) > 0.10 - for: 5m - labels: - severity: warning - service: broker_gateway_service - component: order_validation - impact: order_flow_disrupted - annotations: - summary: "Order rejection rate {{ $value | humanizePercentage }} for {{ $labels.symbol }}" - - # Slow database queries - - alert: BrokerGatewaySlowDatabaseQueries - expr: | - histogram_quantile(0.95, - sum by(le) (rate(database_query_duration_seconds_bucket{service="broker_gateway_service"}[5m])) - ) > 0.100 - for: 5m - labels: - severity: warning - service: broker_gateway_service - component: database - impact: performance_degraded - annotations: - summary: "Database query P95 > 100ms" - - # High margin usage - - alert: BrokerGatewayHighMarginUsage - expr: | - ( - broker_gateway_margin_used_usd - / (broker_gateway_cash_balance_usd + 0.001) - ) > 0.80 - for: 5m - labels: - severity: warning - service: broker_gateway_service - component: risk_management - impact: margin_pressure - annotations: - summary: "Margin usage {{ $value | humanizePercentage }} for {{ $labels.account_id }}" diff --git a/infra/k8s/monitoring/prometheus-rules-hft.yaml b/infra/k8s/monitoring/prometheus-rules-hft.yaml deleted file mode 100644 index a3cd5aa2c..000000000 --- a/infra/k8s/monitoring/prometheus-rules-hft.yaml +++ /dev/null @@ -1,328 +0,0 @@ -# HFT alert rules — discovered automatically by Prometheus Operator -# Apply: kubectl apply -f prometheus-rules-hft.yaml -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: foxhunt-hft-alerts - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt - role: alert-rules -spec: - groups: - # ═══════════════════════════════ - # CRITICAL HFT ALERTS - # ═══════════════════════════════ - - name: hft_critical - rules: - - alert: TradingServiceDown - expr: up{job="foxhunt-services", service="trading-service"} == 0 - for: 5s - labels: - severity: critical - component: trading_engine - impact: trading_halt - annotations: - summary: "Trading Service is DOWN" - description: "Trading Service has been down for >5s. All trading operations halted." - action: "Immediate intervention required" - - - alert: DataAcquisitionDown - expr: up{job="foxhunt-services", service="data-acquisition-service"} == 0 - for: 10s - labels: - severity: critical - component: market_data - impact: blind_trading - annotations: - summary: "Data Acquisition Service is DOWN" - description: "Market data feed down >10s. Trading without data is dangerous." - action: "Stop all trading immediately" - - - alert: BrokerGatewayDown - expr: up{job="foxhunt-services", service="broker-gateway-service"} == 0 - for: 5s - labels: - severity: critical - component: risk_management - impact: uncontrolled_risk - annotations: - summary: "Broker Gateway (risk controls) is DOWN" - description: "Broker gateway down >5s. Order routing and risk controls offline." - action: "Emergency trading halt" - - - alert: TradingLatencyHigh - expr: histogram_quantile(0.95, foxhunt_trading_latency_seconds_bucket{operation="order_placement"}) > 0.001 - for: 30s - labels: - severity: critical - component: trading_engine - impact: competitive_disadvantage - annotations: - summary: "Trading latency P95 > 1ms" - description: "95th percentile order placement latency is {{ $value }}s. HFT advantage compromised." - action: "Investigate performance bottlenecks" - - - alert: PositionLimitBreach - expr: foxhunt_position_size_usd / foxhunt_position_limit_usd > 0.95 - for: 0s - labels: - severity: critical - component: risk_management - impact: regulatory_breach - annotations: - summary: "Position limit nearly breached ({{ $value | humanizePercentage }})" - description: "Current position at {{ $value | humanizePercentage }} of limit." - action: "Reduce positions immediately" - - - alert: LossLimitBreach - expr: foxhunt_daily_pnl_usd < foxhunt_loss_limit_usd - for: 0s - labels: - severity: critical - component: risk_management - impact: financial_loss - annotations: - summary: "Daily loss limit breached" - description: "Daily P&L ${{ $value }} breaches loss limit. Trading must halt." - action: "Emergency trading halt" - - # ═══════════════════════════════ - # HIGH PRIORITY ALERTS - # ═══════════════════════════════ - - name: hft_high - rules: - - alert: DatabaseConnectionHigh - expr: foxhunt_database_connections_active / foxhunt_database_connections_max > 0.85 - for: 1m - labels: - severity: high - component: database - impact: performance_degradation - annotations: - summary: "Database connection pool at {{ $value | humanizePercentage }}" - - - alert: MemoryUsageHigh - expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90 - for: 2m - labels: - severity: high - component: system - impact: performance_degradation - annotations: - summary: "Memory usage {{ $value | humanizePercentage }} on {{ $labels.instance }}" - - - alert: CPUUsageHigh - expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 - for: 5m - labels: - severity: high - component: system - impact: performance_degradation - annotations: - summary: "CPU usage {{ $value | printf \"%.1f\" }}% on {{ $labels.instance }}" - - - alert: DiskUsageHigh - expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"})) > 0.85 - for: 5m - labels: - severity: high - component: system - impact: storage_failure - annotations: - summary: "Disk usage {{ $value | humanizePercentage }} on {{ $labels.instance }} {{ $labels.mountpoint }}" - - - alert: OrderRejectionRateHigh - expr: rate(foxhunt_orders_rejected_total[5m]) / rate(foxhunt_orders_total[5m]) > 0.05 - for: 2m - labels: - severity: high - component: trading_engine - impact: trading_inefficiency - annotations: - summary: "Order rejection rate {{ $value | humanizePercentage }}" - - - alert: MarketDataLag - expr: foxhunt_market_data_lag_seconds > 0.1 - for: 1m - labels: - severity: high - component: market_data - impact: stale_data - annotations: - summary: "Market data lag {{ $value }}s — trading on stale data" - - - alert: DrawdownHigh - expr: foxhunt_drawdown_percent > 0.05 - for: 15m - labels: - severity: high - component: risk_management - impact: financial_risk - annotations: - summary: "Portfolio drawdown {{ $value | humanizePercentage }}" - - # ═══════════════════════════════ - # MEDIUM / WARNING ALERTS - # ═══════════════════════════════ - - name: hft_medium - rules: - - alert: ServiceRestarted - expr: increase(process_start_time_seconds[10m]) > 0 - for: 0s - labels: - severity: medium - component: service - impact: disruption - annotations: - summary: "Service {{ $labels.job }} restarted on {{ $labels.instance }}" - - - alert: NetworkLatencyHigh - expr: foxhunt_network_latency_seconds > 0.01 - for: 5m - labels: - severity: medium - component: network - impact: performance_impact - annotations: - summary: "Network latency {{ $value }}s" - - - alert: BackupFailed - expr: time() - foxhunt_last_backup_timestamp > 86400 - for: 1h - labels: - severity: medium - component: backup - impact: data_risk - annotations: - summary: "Last backup {{ $value | humanizeDuration }} ago" - - - alert: SSLCertificateExpiringSoon - expr: (ssl_certificate_expiry_timestamp - time()) / 86400 < 30 - for: 1h - labels: - severity: medium - component: security - impact: service_disruption - annotations: - summary: "SSL cert expires in {{ $value }} days for {{ $labels.instance }}" - - # ═══════════════════════════════ - # BUSINESS LOGIC ALERTS - # ═══════════════════════════════ - - name: hft_business - rules: - - alert: TradingVolumeAnomalyHigh - expr: rate(foxhunt_trades_total[5m]) > (avg_over_time(rate(foxhunt_trades_total[5m])[1h:5m]) * 3) - for: 2m - labels: - severity: medium - component: trading_engine - impact: business_anomaly - annotations: - summary: "Trading volume 3x hourly average ({{ $value }} trades/sec)" - - - alert: TradingVolumeAnomalyLow - expr: rate(foxhunt_trades_total[5m]) < (avg_over_time(rate(foxhunt_trades_total[5m])[1h:5m]) * 0.1) - for: 10m - labels: - severity: medium - component: trading_engine - impact: business_anomaly - annotations: - summary: "Trading volume 10% of hourly average ({{ $value }} trades/sec)" - - - alert: PnLVolatilityHigh - expr: stddev_over_time(foxhunt_pnl_usd[1h]) > 10000 - for: 30m - labels: - severity: medium - component: risk_management - impact: financial_risk - annotations: - summary: "P&L std dev ${{ $value }} over 1h" - - # ═══════════════════════════════ - # INFRASTRUCTURE ALERTS - # ═══════════════════════════════ - - name: hft_infrastructure - rules: - - alert: RedisConnectionFailed - expr: redis_connected_clients{job="redis"} == 0 - for: 30s - labels: - severity: critical - component: redis - impact: cache_failure - annotations: - summary: "Redis has 0 connected clients" - - - alert: PostgreSQLDown - expr: pg_up{job="postgres"} == 0 - for: 30s - labels: - severity: critical - component: postgresql - impact: data_unavailable - annotations: - summary: "PostgreSQL is not responding" - - # ═══════════════════════════════ - # SECURITY ALERTS - # ═══════════════════════════════ - - name: hft_security - rules: - - alert: AuthenticationFailuresHigh - expr: rate(foxhunt_auth_failures_total[5m]) > 5 - for: 2m - labels: - severity: high - component: security - impact: security_breach - annotations: - summary: "{{ $value }} auth failures/sec" - - - alert: UnauthorizedAccessAttempts - expr: rate(foxhunt_unauthorized_requests_total[5m]) > 1 - for: 1m - labels: - severity: critical - component: security - impact: security_breach - annotations: - summary: "{{ $value }} unauthorized requests/sec from {{ $labels.source_ip }}" - - - alert: APIRateLimitExceeded - expr: rate(foxhunt_api_rate_limit_exceeded_total[5m]) > 0.1 - for: 5m - labels: - severity: medium - component: api - impact: service_degradation - annotations: - summary: "API rate limits exceeded {{ $value }}/sec" - - # ═══════════════════════════════ - # MONITORING HEALTH - # ═══════════════════════════════ - - name: hft_monitoring - rules: - - alert: PrometheusTargetDown - expr: up == 0 - for: 2m - labels: - severity: medium - component: monitoring - impact: observability_loss - annotations: - summary: "{{ $labels.job }} target {{ $labels.instance }} down >2m" - - - alert: HighAlertRate - expr: rate(prometheus_notifications_total[5m]) > 10 - for: 10m - labels: - severity: medium - component: monitoring - impact: alert_fatigue - annotations: - summary: "{{ $value }} alerts/sec — check for alert storm" diff --git a/infra/k8s/monitoring/prometheus-rules-storage.yaml b/infra/k8s/monitoring/prometheus-rules-storage.yaml deleted file mode 100644 index e5d618649..000000000 --- a/infra/k8s/monitoring/prometheus-rules-storage.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Storage alert rules — volume usage warnings before auto-expand kicks in -# Apply: kubectl apply -f prometheus-rules-storage.yaml -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: foxhunt-storage-alerts - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt - role: alert-rules -spec: - groups: - - name: storage_volume_usage - interval: 60s - rules: - # PVC usage > 75% — warning (auto-expand triggers at 85%) - - alert: PVCUsageHigh - expr: | - 100 * kubelet_volume_stats_used_bytes{namespace="foxhunt"} - / kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 75 - for: 5m - labels: - severity: warning - component: storage - annotations: - summary: "PVC {{ $labels.persistentvolumeclaim }} at {{ $value | printf \"%.0f\" }}% usage" - description: "Volume {{ $labels.persistentvolumeclaim }} is {{ $value | printf \"%.0f\" }}% full. Auto-expand triggers at 85%." - - # PVC usage > 90% — critical (auto-expand should have already acted) - - alert: PVCUsageCritical - expr: | - 100 * kubelet_volume_stats_used_bytes{namespace="foxhunt"} - / kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 90 - for: 2m - labels: - severity: critical - component: storage - annotations: - summary: "PVC {{ $labels.persistentvolumeclaim }} CRITICAL at {{ $value | printf \"%.0f\" }}%" - description: "Volume {{ $labels.persistentvolumeclaim }} is {{ $value | printf \"%.0f\" }}% full despite auto-expand. Investigate immediately." - action: "Check pvc-autoscaler CronJob logs. Manual expand: kubectl patch pvc {{ $labels.persistentvolumeclaim }} -n foxhunt --type merge -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"XXGi\"}}}}'" - - # PVC approaching max auto-expand limit (>80Gi of 100Gi max) - - alert: PVCApproachingMaxSize - expr: | - kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 80 * 1024 * 1024 * 1024 - and - 100 * kubelet_volume_stats_used_bytes{namespace="foxhunt"} - / kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 70 - for: 10m - labels: - severity: warning - component: storage - annotations: - summary: "PVC {{ $labels.persistentvolumeclaim }} is large and growing" - description: "Volume is >80Gi and >70% full. Auto-expand max is 100Gi. Consider manual intervention or data cleanup." diff --git a/infra/k8s/monitoring/prometheus-stack-values.yaml b/infra/k8s/monitoring/prometheus-stack-values.yaml deleted file mode 100644 index d589c786d..000000000 --- a/infra/k8s/monitoring/prometheus-stack-values.yaml +++ /dev/null @@ -1,400 +0,0 @@ -# kube-prometheus-stack Helm values — Foxhunt production monitoring -# -# Install: -# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -# helm repo update -# helm install prometheus-stack prometheus-community/kube-prometheus-stack \ -# --namespace foxhunt \ -# -f infra/k8s/monitoring/prometheus-stack-values.yaml -# -# Upgrade: -# helm upgrade prometheus-stack prometheus-community/kube-prometheus-stack \ -# --namespace foxhunt \ -# -f infra/k8s/monitoring/prometheus-stack-values.yaml -# -# Prerequisites: -# kubectl -n foxhunt create secret generic alertmanager-slack \ -# --from-literal=slack-webhook-url='https://hooks.slack.com/services/...' -# -# After install, delete old standalone resources: -# kubectl -n foxhunt delete deploy prometheus -# kubectl -n foxhunt delete svc prometheus -# kubectl -n foxhunt delete cm prometheus-config -# kubectl -n foxhunt delete pvc prometheus-data -# kubectl -n foxhunt delete sa prometheus -# kubectl delete clusterrole foxhunt-prometheus -# kubectl delete clusterrolebinding foxhunt-prometheus -# kubectl -n foxhunt delete deploy node-exporter -# kubectl -n foxhunt delete ds node-exporter # if DaemonSet -# kubectl -n foxhunt delete deploy kube-state-metrics - -# ───────────────────────────────────────────── -# Prometheus Operator -# ───────────────────────────────────────────── -prometheusOperator: - enabled: true - nodeSelector: - k8s.scaleway.com/pool-name: platform - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi - -# ───────────────────────────────────────────── -# Prometheus -# ───────────────────────────────────────────── -prometheus: - prometheusSpec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - retention: 14d - retentionSize: 8GB - replicas: 1 - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - storageSpec: - volumeClaimTemplate: - spec: - storageClassName: scw-bssd - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 10Gi - - # Discover ALL ServiceMonitors/PodMonitors/PrometheusRules in the cluster, - # not just those matching Helm release labels. - serviceMonitorSelectorNilUsesHelmValues: false - podMonitorSelectorNilUsesHelmValues: false - ruleSelectorNilUsesHelmValues: false - - # Scrape configs for targets that don't fit ServiceMonitor/PodMonitor CRDs - additionalScrapeConfigs: - # Argo Workflows controller (HTTPS with self-signed cert) - - job_name: argo-workflows - scheme: https - tls_config: - insecure_skip_verify: true - static_configs: - - targets: ['argo-workflows-workflow-controller-metrics.foxhunt.svc.cluster.local:9090'] - relabel_configs: - - target_label: service - replacement: argo-workflows - - # Training workflow pods (ephemeral Argo pods on port 9094, Running only) - - job_name: training-pods - kubernetes_sd_configs: - - role: pod - namespaces: - names: [foxhunt] - relabel_configs: - - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component] - action: keep - regex: training-workflow|compile-and-train - - source_labels: [__meta_kubernetes_pod_phase] - action: keep - regex: Running - - source_labels: [__meta_kubernetes_pod_ip] - target_label: __address__ - replacement: ${1}:9094 - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - source_labels: [__meta_kubernetes_pod_label_workflows_argoproj_io_workflow] - target_label: workflow - - # QuestDB (ML metrics store) - - job_name: questdb - metrics_path: /metrics - static_configs: - - targets: ['questdb.foxhunt.svc.cluster.local:9003'] - relabel_configs: - - target_label: service - replacement: questdb - - # Pushgateway (training job push metrics) - - job_name: pushgateway - honor_labels: true - static_configs: - - targets: ['pushgateway.foxhunt.svc.cluster.local:9091'] - - # Loki metrics - - job_name: loki - static_configs: - - targets: ['loki.foxhunt.svc.cluster.local:3100'] - - # Tempo metrics - - job_name: tempo - static_configs: - - targets: ['tempo.foxhunt.svc.cluster.local:3200'] - -# ───────────────────────────────────────────── -# AlertManager -# ───────────────────────────────────────────── -alertmanager: - alertmanagerSpec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - replicas: 1 - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - # Mount K8s Secrets into AlertManager at /etc/alertmanager/secrets// - secrets: - - alertmanager-mattermost - - config: - global: - # Mattermost incoming webhook (Slack-compatible API). - # Reads URL from K8s Secret: alertmanager-mattermost/webhook-url - # Create with: - # kubectl -n foxhunt create secret generic alertmanager-mattermost \ - # --from-literal=webhook-url='http://mattermost.foxhunt.svc.cluster.local:8065/hooks/' - slack_api_url_file: /etc/alertmanager/secrets/alertmanager-mattermost/webhook-url - - route: - receiver: default - group_by: ['alertname', 'severity', 'service'] - group_wait: 5s - group_interval: 30s - repeat_interval: 4h - - routes: - # CRITICAL trading/risk → immediate Slack + PagerDuty - - matchers: - - severity="critical" - - component=~"trading_engine|risk_management|trading|risk" - receiver: critical-trading - group_wait: 0s - group_interval: 5s - repeat_interval: 5m - continue: true - - # CRITICAL infrastructure → Slack critical channel - - matchers: - - severity="critical" - receiver: critical-infra - group_wait: 0s - group_interval: 10s - repeat_interval: 15m - - # HIGH priority → Slack alerts channel - - matchers: - - severity="high" - receiver: high-priority - group_wait: 10s - group_interval: 1m - repeat_interval: 30m - - # WARNING / MEDIUM → Slack digest - - matchers: - - severity=~"warning|WARNING|medium" - receiver: warning-alerts - group_wait: 30s - group_interval: 5m - repeat_interval: 4h - - receivers: - - name: default - slack_configs: - - channel: '#foxhunt-alerts' - title: '{{ .GroupLabels.alertname }}' - text: >- - {{ range .Alerts }}*{{ .Labels.severity | toUpper }}*: {{ .Annotations.summary }} - {{ end }} - send_resolved: true - - - name: critical-trading - slack_configs: - - channel: '#foxhunt-critical' - title: 'CRITICAL: {{ .GroupLabels.alertname }}' - text: >- - {{ range .Alerts }} - *Alert*: {{ .Annotations.summary }} - *Impact*: {{ .Annotations.impact }} - *Action*: {{ .Annotations.action }} - {{ end }} - color: 'danger' - send_resolved: true - - - name: critical-infra - slack_configs: - - channel: '#foxhunt-critical' - title: 'INFRA CRITICAL: {{ .GroupLabels.alertname }}' - text: >- - {{ range .Alerts }}{{ .Annotations.summary }} - {{ end }} - color: 'danger' - send_resolved: true - - - name: high-priority - slack_configs: - - channel: '#foxhunt-alerts' - title: 'HIGH: {{ .GroupLabels.alertname }}' - text: >- - {{ range .Alerts }}{{ .Annotations.summary }} - {{ end }} - color: 'warning' - send_resolved: true - - - name: warning-alerts - slack_configs: - - channel: '#foxhunt-alerts' - title: 'WARNING: {{ .GroupLabels.alertname }}' - text: >- - {{ range .Alerts }}{{ .Annotations.summary }} - {{ end }} - color: '#ff9500' - send_resolved: true - - inhibit_rules: - # Trading engine down → suppress all trading alerts - - source_matchers: - - alertname="TradingEngineDown" - target_matchers: - - component=~"trading_engine|trading" - equal: ['service'] - - # Node down → suppress service-level alerts on that node - - source_matchers: - - alertname=~"NodeDown|KubeNodeNotReady" - target_matchers: - - severity=~"high|warning|medium" - equal: ['instance'] - - # PostgreSQL down → suppress database alerts - - source_matchers: - - alertname="PostgreSQLDown" - target_matchers: - - component="database" - -# ───────────────────────────────────────────── -# ServiceMonitors -# ───────────────────────────────────────────── -additionalServiceMonitors: - # All foxhunt microservices (7 services, all expose named port "metrics") - - name: foxhunt-services - selector: - matchLabels: - app.kubernetes.io/part-of: foxhunt - namespaceSelector: - matchNames: [foxhunt] - endpoints: - - port: metrics - interval: 15s - - # DCGM Exporter (GPU metrics from kube-system) - - name: dcgm-exporter - selector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: [dcgm-exporter] - namespaceSelector: - matchNames: [kube-system, monitoring, foxhunt] - endpoints: - - port: metrics - interval: 15s - -# ───────────────────────────────────────────── -# Grafana (disabled — managed separately via grafana Helm chart) -# ───────────────────────────────────────────── -grafana: - enabled: false - -# ───────────────────────────────────────────── -# Node Exporter (replaces standalone node-exporter.yaml) -# ───────────────────────────────────────────── -prometheus-node-exporter: - tolerations: - - effect: NoSchedule - operator: Exists - resources: - requests: - cpu: 10m - memory: 16Mi - limits: - cpu: 50m - memory: 32Mi - -# ───────────────────────────────────────────── -# kube-state-metrics (replaces standalone kube-state-metrics.yaml) -# ───────────────────────────────────────────── -kube-state-metrics: - nodeSelector: - k8s.scaleway.com/pool-name: platform - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 50m - memory: 64Mi - -# ───────────────────────────────────────────── -# Default rules (Kubernetes cluster health alerts included out-of-box) -# ───────────────────────────────────────────── -defaultRules: - create: true - rules: - alertmanager: true - etcd: false # managed by Scaleway Kapsule - configReloaders: true - general: true - k8sContainerCpuUsageSecondsTotal: true - k8sContainerMemoryCache: true - k8sContainerMemoryRss: true - k8sContainerMemorySwap: true - k8sContainerMemoryWorkingSetBytes: true - k8sPodOwner: true - kubeApiserverAvailability: false # managed by Scaleway - kubeApiserverBurnrate: false - kubeApiserverHistogram: false - kubeApiserverSlos: false - kubeControllerManager: false # managed by Scaleway - kubelet: false # kubelet metrics not accessible on Kapsule - kubeProxy: false # managed by Scaleway - kubePrometheusGeneral: true - kubePrometheusNodeRecording: true - kubernetesAbsent: true - kubernetesApps: true - kubernetesResources: true - kubernetesStorage: true - kubernetesSystem: true - kubeScheduler: false # managed by Scaleway - kubeStateMetrics: true - network: true - node: true - nodeExporterAlerting: true - nodeExporterRecording: true - prometheus: true - prometheusOperator: true - -# ───────────────────────────────────────────── -# Disable components managed by Scaleway Kapsule -# ───────────────────────────────────────────── -kubeApiServer: - enabled: false -kubeControllerManager: - enabled: false -kubeScheduler: - enabled: false -kubeProxy: - enabled: false -kubelet: - enabled: false # kubelet metrics not accessible on Kapsule -kubeEtcd: - enabled: false diff --git a/infra/k8s/monitoring/prometheus-values.yaml b/infra/k8s/monitoring/prometheus-values.yaml deleted file mode 100644 index 8eac3e8f6..000000000 --- a/infra/k8s/monitoring/prometheus-values.yaml +++ /dev/null @@ -1,172 +0,0 @@ -# Prometheus override values for GitLab Helm chart -# Removes Cockpit remote_write, adds comprehensive cluster scrape configs -# -# Apply with: -# helm upgrade gitlab gitlab/gitlab -n foxhunt --reuse-values \ -# -f infra/k8s/monitoring/prometheus-values.yaml -prometheus: - server: - # Remove Cockpit remote_write — all metrics stay local - remoteWrite: [] - - # Extra scrape configs for full cluster observability - # NOTE: this is at prometheus.extraScrapeConfigs (chart root), NOT prometheus.server.extraScrapeConfigs - extraScrapeConfigs: | - # --- Host metrics from node-exporter --- - - job_name: node-exporter - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: [foxhunt] - relabel_configs: - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] - action: keep - regex: node-exporter - - source_labels: [__meta_kubernetes_endpoints_name] - action: keep - regex: node-exporter - - source_labels: [__meta_kubernetes_pod_node_name] - target_label: node - - # --- Kubernetes object state --- - - job_name: kube-state-metrics - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: [foxhunt] - relabel_configs: - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] - action: keep - regex: kube-state-metrics - - source_labels: [__meta_kubernetes_endpoint_port_name] - action: keep - regex: http - - # --- Kubelet metrics (container resource usage) --- - - job_name: kubelet - scheme: https - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - kubernetes_sd_configs: - - role: node - relabel_configs: - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/$1/proxy/metrics - - # --- cAdvisor (container CPU, memory, network, filesystem) --- - - job_name: cadvisor - scheme: https - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - kubernetes_sd_configs: - - role: node - relabel_configs: - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor - metric_relabel_configs: - - source_labels: [namespace] - action: keep - regex: (foxhunt|monitoring|kube-system) - - # --- GPU metrics from NVIDIA DCGM Exporter --- - - job_name: dcgm-exporter - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: [kube-system, monitoring] - relabel_configs: - - source_labels: [__meta_kubernetes_service_name] - action: keep - regex: .*dcgm.* - - source_labels: [__meta_kubernetes_pod_node_name] - target_label: node - - # --- Foxhunt services (app metrics ports) --- - - job_name: foxhunt-services - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: [foxhunt] - relabel_configs: - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of] - action: keep - regex: foxhunt - - source_labels: [__meta_kubernetes_endpoint_port_name] - action: keep - regex: metrics - - source_labels: [__meta_kubernetes_service_name] - target_label: service - - source_labels: [__meta_kubernetes_namespace] - target_label: namespace - - # --- General prometheus.io annotation-based scraping --- - - job_name: prometheus-annotated-pods - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: "true" - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] - action: replace - regex: (https?) - target_label: __scheme__ - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - regex: (.+) - target_label: __metrics_path__ - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - source_labels: [__meta_kubernetes_pod_node_name] - target_label: node - - # --- Loki metrics --- - - job_name: loki - static_configs: - - targets: ['loki.foxhunt.svc.cluster.local:3100'] - - # --- Tempo metrics --- - - job_name: tempo - static_configs: - - targets: ['tempo.foxhunt.svc.cluster.local:3200'] - - # --- Promtail metrics --- - - job_name: promtail - kubernetes_sd_configs: - - role: pod - namespaces: - names: [foxhunt] - relabel_configs: - - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] - action: keep - regex: promtail - - source_labels: [__meta_kubernetes_pod_ip] - target_label: __address__ - replacement: $1:9080 - - # --- Pushgateway (training job metrics) --- - - job_name: pushgateway - honor_labels: true - static_configs: - - targets: ['pushgateway.foxhunt.svc.cluster.local:9091'] diff --git a/infra/k8s/monitoring/promtail.yaml b/infra/k8s/monitoring/promtail.yaml deleted file mode 100644 index 16c4b354f..000000000 --- a/infra/k8s/monitoring/promtail.yaml +++ /dev/null @@ -1,265 +0,0 @@ -# Promtail — ships container logs to Loki (DaemonSet on all nodes) -apiVersion: v1 -kind: ServiceAccount -metadata: - name: promtail - namespace: foxhunt - labels: - app.kubernetes.io/name: promtail ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: promtail - labels: - app.kubernetes.io/name: promtail -rules: - - apiGroups: [""] - resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"] - verbs: ["get", "watch", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: promtail - labels: - app.kubernetes.io/name: promtail -subjects: - - kind: ServiceAccount - name: promtail - namespace: foxhunt -roleRef: - kind: ClusterRole - name: promtail - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: promtail-config - namespace: foxhunt - labels: - app.kubernetes.io/name: promtail -data: - promtail.yaml: | - server: - http_listen_port: 9080 - positions: - filename: /run/promtail/positions.yaml - clients: - - url: http://loki.foxhunt.svc.cluster.local:3100/loki/api/v1/push - scrape_configs: - # Foxhunt application pods (structured Rust logs) - - job_name: foxhunt-pods - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_namespace] - action: keep - regex: foxhunt - # Drop noisy infra pods (gitlab internals, prometheus, grafana) - - source_labels: [__meta_kubernetes_pod_label_app] - action: drop - regex: (gitlab|prometheus|grafana|promtail) - - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] - action: drop - regex: (promtail|node-exporter|kube-state-metrics) - # Map to containerd log path - - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name] - separator: / - target_label: __path__ - replacement: /var/log/pods/*$1/*.log - # Standard K8s metadata labels - - source_labels: [__meta_kubernetes_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - source_labels: [__meta_kubernetes_pod_container_name] - target_label: container - - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] - target_label: app - - source_labels: [__meta_kubernetes_pod_label_foxhunt_job__type] - target_label: job_type - - source_labels: [__meta_kubernetes_pod_label_foxhunt_model] - target_label: model - - source_labels: [__meta_kubernetes_pod_node_name] - target_label: node - pipeline_stages: - - json: - expressions: - level: level - message: fields.message - target: target - timestamp: timestamp - - labels: - level: - target: - - match: - selector: '{level=""}' - stages: - - regex: - expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR)' - - labels: - level: - - # Kapsule system pods (kube-system, kube-node-lease, kube-public) - - job_name: kapsule-system - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_namespace] - action: keep - regex: (kube-system|kube-node-lease|kube-public) - # Map to containerd log path - - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name] - separator: / - target_label: __path__ - replacement: /var/log/pods/*$1/*.log - - source_labels: [__meta_kubernetes_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - source_labels: [__meta_kubernetes_pod_container_name] - target_label: container - - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] - target_label: app - - source_labels: [__meta_kubernetes_pod_label_app] - target_label: kube_app - - source_labels: [__meta_kubernetes_pod_node_name] - target_label: node - pipeline_stages: - - match: - selector: '{namespace="kube-system"}' - stages: - - regex: - expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)' - - labels: - level: - - # GitLab CI runner logs (foxhunt namespace, runner pods) - - job_name: gitlab-runners - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_namespace] - action: keep - regex: foxhunt - - source_labels: [__meta_kubernetes_pod_label_app] - action: keep - regex: gitlab-runner - - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name] - separator: / - target_label: __path__ - replacement: /var/log/pods/*$1/*.log - - source_labels: [__meta_kubernetes_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - source_labels: [__meta_kubernetes_pod_container_name] - target_label: container - - replacement: gitlab-runner - target_label: app - - source_labels: [__meta_kubernetes_pod_node_name] - target_label: node - pipeline_stages: - - regex: - expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR|WARNING)' - - labels: - level: - - # Kubernetes node-level logs (kubelet, containerd) - - job_name: kapsule-node-logs - static_configs: - - targets: [localhost] - labels: - job: kapsule-nodes - __path__: /var/log/syslog - - targets: [localhost] - labels: - job: kapsule-nodes - __path__: /var/log/kern.log - - targets: [localhost] - labels: - job: kapsule-nodes - __path__: /var/log/kubelet.log - relabel_configs: - - source_labels: [__path__] - regex: .*/(.+)\.log - target_label: log_type - pipeline_stages: - - regex: - expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)' - - labels: - level: ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: promtail - namespace: foxhunt - labels: - app.kubernetes.io/name: promtail -spec: - selector: - matchLabels: - app.kubernetes.io/name: promtail - template: - metadata: - labels: - app.kubernetes.io/name: promtail - spec: - serviceAccountName: promtail - tolerations: - - effect: NoSchedule - operator: Exists - containers: - - name: promtail - image: grafana/promtail:3.4.2 - args: ["-config.file=/etc/promtail/promtail.yaml"] - ports: - - containerPort: 9080 - name: http - volumeMounts: - - name: config - mountPath: /etc/promtail - - name: varlog - mountPath: /var/log - readOnly: true - - name: containers - mountPath: /var/lib/docker/containers - readOnly: true - - name: pods - mountPath: /var/log/pods - readOnly: true - - name: run - mountPath: /run/promtail - env: - - name: HOSTNAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 256Mi - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - volumes: - - name: config - configMap: - name: promtail-config - - name: varlog - hostPath: - path: /var/log - - name: containers - hostPath: - path: /var/lib/docker/containers - - name: pods - hostPath: - path: /var/log/pods - - name: run - emptyDir: {} diff --git a/infra/k8s/monitoring/pushgateway.yaml b/infra/k8s/monitoring/pushgateway.yaml deleted file mode 100644 index 10082f25f..000000000 --- a/infra/k8s/monitoring/pushgateway.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pushgateway - namespace: foxhunt - labels: - app.kubernetes.io/name: pushgateway - app.kubernetes.io/part-of: monitoring -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: pushgateway - template: - metadata: - labels: - app.kubernetes.io/name: pushgateway - spec: - nodeSelector: - k8s.scaleway.com/pool-name: infra - containers: - - name: pushgateway - image: prom/pushgateway:v1.11.0 - args: - - "--persistence.interval=5m" - ports: - - containerPort: 9091 - name: http - resources: - requests: - cpu: 50m - memory: 32Mi - limits: - cpu: 200m - memory: 128Mi - livenessProbe: - httpGet: - path: /-/healthy - port: 9091 - initialDelaySeconds: 5 - readinessProbe: - httpGet: - path: /-/ready - port: 9091 - initialDelaySeconds: 5 ---- -apiVersion: v1 -kind: Service -metadata: - name: pushgateway - namespace: foxhunt - labels: - app.kubernetes.io/name: pushgateway -spec: - type: ClusterIP - ports: - - port: 9091 - targetPort: 9091 - name: http - selector: - app.kubernetes.io/name: pushgateway diff --git a/infra/k8s/monitoring/pvc-autoscaler.yaml b/infra/k8s/monitoring/pvc-autoscaler.yaml deleted file mode 100644 index 212e2d9e3..000000000 --- a/infra/k8s/monitoring/pvc-autoscaler.yaml +++ /dev/null @@ -1,250 +0,0 @@ -# PVC Auto-Expand — expands PVCs automatically when usage exceeds 85% -# Uses K8s API to list PVCs and exec df inside pods to check usage. -# Runs every 15 minutes. Apply: kubectl apply -f pvc-autoscaler.yaml ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: pvc-autoscaler - namespace: foxhunt - labels: - app.kubernetes.io/name: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: pvc-autoscaler - namespace: foxhunt - labels: - app.kubernetes.io/name: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt -rules: - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] - - apiGroups: ["apps"] - resources: ["deployments", "statefulsets"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: pvc-autoscaler-nodes - labels: - app.kubernetes.io/name: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] - - apiGroups: [""] - resources: ["nodes/proxy"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: pvc-autoscaler - namespace: foxhunt - labels: - app.kubernetes.io/name: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt -subjects: - - kind: ServiceAccount - name: pvc-autoscaler - namespace: foxhunt -roleRef: - kind: Role - name: pvc-autoscaler - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: pvc-autoscaler-nodes - labels: - app.kubernetes.io/name: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt -subjects: - - kind: ServiceAccount - name: pvc-autoscaler - namespace: foxhunt -roleRef: - kind: ClusterRole - name: pvc-autoscaler-nodes - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: pvc-autoscaler-script - namespace: foxhunt - labels: - app.kubernetes.io/name: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt -data: - autoscale.py: | - #!/usr/bin/env python3 - """PVC auto-expand via kubelet stats API (no Prometheus dependency).""" - import json, os, sys, ssl, urllib.request - from datetime import datetime, timezone - - THRESHOLD = int(os.environ.get("PVC_USAGE_THRESHOLD", "85")) - GROW_PERCENT = int(os.environ.get("PVC_GROW_PERCENT", "50")) - MAX_SIZE_GI = int(os.environ.get("PVC_MAX_SIZE_GI", "100")) - NAMESPACE = os.environ.get("NAMESPACE", "foxhunt") - - TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" - CA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - K8S_API = "https://kubernetes.default.svc" - - def k8s(path, method="GET", data=None): - token = open(TOKEN_PATH).read().strip() - ctx = ssl.create_default_context(cafile=CA_PATH) - body = json.dumps(data).encode() if data else None - req = urllib.request.Request(f"{K8S_API}{path}", data=body, method=method) - req.add_header("Authorization", f"Bearer {token}") - if data: - req.add_header("Content-Type", "application/merge-patch+json") - return json.loads(urllib.request.urlopen(req, context=ctx, timeout=30).read()) - - now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - print(f"=== PVC Auto-Expand {now} ===") - print(f"Threshold: {THRESHOLD}% | Grow: +{GROW_PERCENT}% | Max: {MAX_SIZE_GI}Gi") - sys.stdout.flush() - - # Step 1: Get all nodes and their kubelet stats via K8s proxy API - nodes = k8s("/api/v1/nodes") - pvc_usage = {} # pvc_name -> (used_bytes, capacity_bytes) - - for node in nodes["items"]: - node_name = node["metadata"]["name"] - try: - stats = k8s(f"/api/v1/nodes/{node_name}/proxy/stats/summary") - for pod in stats.get("pods", []): - if pod.get("podRef", {}).get("namespace") != NAMESPACE: - continue - for vol in pod.get("volume", []): - pvc_ref = vol.get("pvcRef") - if not pvc_ref: - continue - pvc_name = pvc_ref["name"] - used = vol.get("usedBytes", 0) - cap = vol.get("capacityBytes", 1) - pvc_usage[pvc_name] = (used, cap) - except Exception as e: - print(f" WARN: could not get stats from node {node_name}: {e}") - sys.stdout.flush() - - if not pvc_usage: - print(" No PVC stats found") - sys.exit(0) - - # Step 2: Check each PVC and expand if needed - expanded = 0 - for pvc_name, (used, cap) in sorted(pvc_usage.items()): - pct = 100 * used / cap if cap > 0 else 0 - used_gi = used / (1024**3) - cap_gi = cap / (1024**3) - print(f" {pvc_name}: {pct:.1f}% ({used_gi:.1f}G / {cap_gi:.1f}G)") - - if pct >= THRESHOLD: - print(f" >> {pvc_name} exceeds {THRESHOLD}%") - try: - pvc_obj = k8s(f"/api/v1/namespaces/{NAMESPACE}/persistentvolumeclaims/{pvc_name}") - sc = pvc_obj.get("spec", {}).get("storageClassName", "") - # Only expand if StorageClass supports it - if sc: - try: - sc_obj = k8s(f"/apis/storage.k8s.io/v1/storageclasses/{sc}") - if not sc_obj.get("allowVolumeExpansion"): - print(f" SKIP: StorageClass {sc} does not allow expansion") - continue - except Exception: - pass - - status_storage = pvc_obj.get("status", {}).get("capacity", {}).get("storage", "0Gi") - current_gi = int(status_storage.replace("Gi", "")) - spec_storage = pvc_obj["spec"]["resources"]["requests"]["storage"] - spec_gi = int(spec_storage.replace("Gi", "")) - - new_gi = current_gi + max(current_gi * GROW_PERCENT // 100, 1) - if new_gi > MAX_SIZE_GI: - print(f" SKIP: target {new_gi}Gi > max {MAX_SIZE_GI}Gi") - continue - if spec_gi >= new_gi: - print(f" SKIP: spec={spec_gi}Gi >= target={new_gi}Gi (resize pending)") - continue - - patch = {"spec": {"resources": {"requests": {"storage": f"{new_gi}Gi"}}}} - k8s(f"/api/v1/namespaces/{NAMESPACE}/persistentvolumeclaims/{pvc_name}", - method="PATCH", data=patch) - print(f" EXPANDED: {current_gi}Gi -> {new_gi}Gi") - expanded += 1 - except Exception as e: - print(f" FAILED: {e}", file=sys.stderr) - sys.stdout.flush() - - print(f"=== Done ({expanded} expanded) ===") ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: pvc-autoscaler - namespace: foxhunt - labels: - app.kubernetes.io/name: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt -spec: - schedule: "*/15 * * * *" - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 2 - activeDeadlineSeconds: 120 - template: - metadata: - labels: - app: pvc-autoscaler - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - serviceAccountName: pvc-autoscaler - restartPolicy: OnFailure - containers: - - name: autoscaler - image: python:3.12-alpine - command: ["python3", "-u", "/scripts/autoscale.py"] - env: - - name: PVC_USAGE_THRESHOLD - value: "85" - - name: PVC_GROW_PERCENT - value: "50" - - name: PVC_MAX_SIZE_GI - value: "100" - - name: NAMESPACE - value: "foxhunt" - volumeMounts: - - name: script - mountPath: /scripts - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - volumes: - - name: script - configMap: - name: pvc-autoscaler-script - items: - - key: autoscale.py - path: autoscale.py - defaultMode: 0755 diff --git a/infra/k8s/monitoring/scaleway-billing.yaml b/infra/k8s/monitoring/scaleway-billing.yaml deleted file mode 100644 index af7b1f975..000000000 --- a/infra/k8s/monitoring/scaleway-billing.yaml +++ /dev/null @@ -1,239 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: scaleway-billing - namespace: foxhunt -type: Opaque -stringData: - SCW_SECRET_KEY: "84fa878b-81c2-401a-8aef-22996d30e9ea" - SCW_ORG_ID: "a55098a3-2cf6-4f73-b011-fff5a299ed94" - SCW_PROJECT_ID: "c293eb98-228d-427d-9b16-f0941f3f2adb" - SCW_CLUSTER_ID: "34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: scaleway-billing-script - namespace: foxhunt -data: - scrape.py: | - #!/usr/bin/env python3 - """Scaleway infrastructure scraper → Prometheus pushgateway.""" - import json, os, sys, urllib.request, urllib.error, re - - def sanitize(s): - """Sanitize label values for Prometheus text format.""" - return str(s).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") - - PUSHGATEWAY = os.environ.get("PUSHGATEWAY_URL", "http://pushgateway.foxhunt.svc.cluster.local:9091") - SECRET = os.environ["SCW_SECRET_KEY"] - ORG = os.environ["SCW_ORG_ID"] - PROJECT = os.environ["SCW_PROJECT_ID"] - CLUSTER = os.environ["SCW_CLUSTER_ID"] - - SCW_BILLING = "https://api.scaleway.com/billing/v2beta1" - SCW_INSTANCE = "https://api.scaleway.com/instance/v1/zones/fr-par-2" - SCW_K8S = "https://api.scaleway.com/k8s/v1/regions/fr-par" - SCW_BLOCK = "https://api.scaleway.com/block/v1alpha1/zones/fr-par-2" - SCW_IPAM = "https://api.scaleway.com/ipam/v1/regions/fr-par" - - def fetch(url): - req = urllib.request.Request(url) - req.add_header("X-Auth-Token", SECRET) - try: - return json.loads(urllib.request.urlopen(req, timeout=30).read()) - except Exception as e: - print(f"WARN: {url}: {e}", file=sys.stderr) - return {} - - # ── Fetch all data (dynamic — auto-discovers any infra changes) ── - billing = fetch(f"{SCW_BILLING}/consumptions?organization_id={ORG}") - instances = fetch(f"{SCW_INSTANCE}/servers?project={PROJECT}&per_page=50") - pools = fetch(f"{SCW_K8S}/clusters/{CLUSTER}/pools") - volumes = fetch(f"{SCW_BLOCK}/volumes?project_id={PROJECT}&per_page=100") - ips = fetch(f"{SCW_IPAM}/ips?project_id={PROJECT}&is_ipv6=false&per_page=100") - - L = [] - - # ── Billing ── - total = 0 - cats = {} - products = {} - for c in billing.get("consumptions", []): - cat = c.get("category_name", "other").lower().replace(" ", "_").replace("&", "and") - product = c.get("product_name", "unknown").lower().replace(" ", "_").replace("-", "_") - sku = c.get("sku", "") - val = c.get("value", {}) - amount = int(val.get("units", 0) or 0) + int(val.get("nanos", 0) or 0) / 1e9 - total += amount - cats[cat] = cats.get(cat, 0) + amount - products[(cat, product, sku)] = products.get((cat, product, sku), 0) + amount - - L.append("# HELP scaleway_billing_total_eur Total current month billing in EUR") - L.append("# TYPE scaleway_billing_total_eur gauge") - L.append(f"scaleway_billing_total_eur {total:.4f}") - L.append("# HELP scaleway_billing_consumption_eur Billing by category in EUR") - L.append("# TYPE scaleway_billing_consumption_eur gauge") - for cat, amt in sorted(cats.items(), key=lambda x: -x[1]): - L.append(f'scaleway_billing_consumption_eur{{category="{sanitize(cat)}"}} {amt:.4f}') - L.append("# HELP scaleway_billing_product_eur Billing by product in EUR") - L.append("# TYPE scaleway_billing_product_eur gauge") - for (cat, product, sku), amt in sorted(products.items(), key=lambda x: -x[1]): - if amt >= 0.01: - L.append(f'scaleway_billing_product_eur{{category="{sanitize(cat)}",product="{sanitize(product)}",sku="{sanitize(sku)}"}} {amt:.4f}') - - # ── Instances (dynamic — auto-discovers new/removed instances) ── - L.append("# HELP scaleway_instance_info Instance metadata") - L.append("# TYPE scaleway_instance_info gauge") - L.append("# HELP scaleway_instance_running Instance running state (1=running)") - L.append("# TYPE scaleway_instance_running gauge") - type_counts = {} - for s in instances.get("servers", []): - name = s.get("name", "?") - itype = s.get("commercial_type", "?") - state = s.get("state", "?") - tags = dict(t.split("=", 1) for t in s.get("tags", []) if "=" in t) - pool = tags.get("pool-name", "unknown") - L.append(f'scaleway_instance_info{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}",state="{sanitize(state)}"}} 1') - L.append(f'scaleway_instance_running{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}"}} {1 if state == "running" else 0}') - type_counts[itype] = type_counts.get(itype, 0) + 1 - L.append("# HELP scaleway_instances_total Total instances by type") - L.append("# TYPE scaleway_instances_total gauge") - for t, c in type_counts.items(): - L.append(f'scaleway_instances_total{{type="{sanitize(t)}"}} {c}') - - # ── K8s Pools (dynamic — auto-discovers pool changes/autoscaler config) ── - L.append("# HELP scaleway_pool_size Current pool node count") - L.append("# TYPE scaleway_pool_size gauge") - L.append("# HELP scaleway_pool_min Minimum pool size") - L.append("# TYPE scaleway_pool_min gauge") - L.append("# HELP scaleway_pool_max Maximum pool size") - L.append("# TYPE scaleway_pool_max gauge") - L.append("# HELP scaleway_pool_autoscaling Pool autoscaling enabled") - L.append("# TYPE scaleway_pool_autoscaling gauge") - L.append("# HELP scaleway_pool_root_volume_bytes Pool root volume size") - L.append("# TYPE scaleway_pool_root_volume_bytes gauge") - for p in pools.get("pools", []): - name = p.get("name", "?") - ntype = p.get("node_type", "?") - status = p.get("status", "?") - lb = f'pool="{sanitize(name)}",type="{sanitize(ntype)}",status="{sanitize(status)}"' - L.append(f"scaleway_pool_size{{{lb}}} {p.get('size', 0)}") - L.append(f"scaleway_pool_min{{{lb}}} {p.get('min_size', 0)}") - L.append(f"scaleway_pool_max{{{lb}}} {p.get('max_size', 0)}") - L.append(f"scaleway_pool_autoscaling{{{lb}}} {1 if p.get('autoscaling') else 0}") - L.append(f"scaleway_pool_root_volume_bytes{{{lb}}} {p.get('root_volume_size', 0)}") - - # ── Block Storage (dynamic — auto-discovers volume additions/removals) ── - L.append("# HELP scaleway_volume_size_bytes Volume size in bytes") - L.append("# TYPE scaleway_volume_size_bytes gauge") - L.append("# HELP scaleway_volume_attached Volume attachment state (1=in_use, 0=available)") - L.append("# TYPE scaleway_volume_attached gauge") - total_bytes = attached_bytes = detached_bytes = 0 - vol_count = attached_count = detached_count = 0 - for v in volumes.get("volumes", []): - vid = v.get("id", "?")[:36] - name = v.get("name", "?")[:60] - size = v.get("size", 0) - state = v.get("status", "unknown") - attached = 1 if state == "in_use" else 0 - total_bytes += size; vol_count += 1 - if attached: - attached_bytes += size; attached_count += 1 - else: - detached_bytes += size; detached_count += 1 - L.append(f'scaleway_volume_size_bytes{{id="{sanitize(vid)}",name="{sanitize(name)}",state="{sanitize(state)}"}} {size}') - L.append(f'scaleway_volume_attached{{id="{sanitize(vid)}",name="{sanitize(name)}"}} {attached}') - L.append("# HELP scaleway_volumes_total_bytes Total block storage bytes") - L.append("# TYPE scaleway_volumes_total_bytes gauge") - L.append(f"scaleway_volumes_total_bytes {total_bytes}") - L.append("# HELP scaleway_volumes_attached_bytes Attached volume bytes") - L.append("# TYPE scaleway_volumes_attached_bytes gauge") - L.append(f"scaleway_volumes_attached_bytes {attached_bytes}") - L.append("# HELP scaleway_volumes_detached_bytes Detached (wasted) volume bytes") - L.append("# TYPE scaleway_volumes_detached_bytes gauge") - L.append(f"scaleway_volumes_detached_bytes {detached_bytes}") - L.append("# HELP scaleway_volumes_count Volume counts") - L.append("# TYPE scaleway_volumes_count gauge") - L.append(f'scaleway_volumes_count{{state="total"}} {vol_count}') - L.append(f'scaleway_volumes_count{{state="attached"}} {attached_count}') - L.append(f'scaleway_volumes_count{{state="detached"}} {detached_count}') - - # ── IPs (dynamic) ── - L.append("# HELP scaleway_ips_total Total allocated IPs") - L.append("# TYPE scaleway_ips_total gauge") - L.append(f'scaleway_ips_total {len(ips.get("ips", []))}') - - # ── Push to pushgateway ── - metrics = "\n".join(L) + "\n" - count = sum(1 for line in L if line.startswith("scaleway_")) - print(f"Generated {count} metrics ({len(metrics)} bytes)", file=sys.stderr) - - # Validate: no empty lines between metrics (pushgateway is strict) - metrics = re.sub(r'\n{2,}', '\n', metrics) - - url = f"{PUSHGATEWAY}/metrics/job/scaleway_billing" - req = urllib.request.Request(url, data=metrics.encode(), method="PUT") - req.add_header("Content-Type", "text/plain; version=0.0.4") - req.add_header("Connection", "close") - try: - resp = urllib.request.urlopen(req, timeout=30) - print(f"OK: pushed {count} metrics (HTTP {resp.status})") - except Exception as e: - print(f"WARN: PUT failed ({e}), trying POST...", file=sys.stderr) - req2 = urllib.request.Request(url, data=metrics.encode(), method="POST") - req2.add_header("Content-Type", "text/plain; version=0.0.4") - req2.add_header("Connection", "close") - try: - resp = urllib.request.urlopen(req2, timeout=30) - print(f"OK: pushed {count} metrics via POST (HTTP {resp.status})") - except Exception as e2: - print(f"ERROR: push failed: PUT={e}, POST={e2}", file=sys.stderr) - sys.exit(1) ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: scaleway-billing-scraper - namespace: foxhunt -spec: - schedule: "0 * * * *" - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 2 - activeDeadlineSeconds: 120 - template: - metadata: - labels: - app: scaleway-billing - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - restartPolicy: OnFailure - containers: - - name: scraper - image: python:3.12-alpine - command: ["python3", "/scripts/scrape.py"] - envFrom: - - secretRef: - name: scaleway-billing - volumeMounts: - - name: script - mountPath: /scripts - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - volumes: - - name: script - configMap: - name: scaleway-billing-script - items: - - key: scrape.py - path: scrape.py - defaultMode: 0755 diff --git a/infra/k8s/monitoring/service-monitors.yaml b/infra/k8s/monitoring/service-monitors.yaml deleted file mode 100644 index b520883ab..000000000 --- a/infra/k8s/monitoring/service-monitors.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# ServiceMonitors for Foxhunt services and GPU metrics -# Applied separately from the Helm chart (additionalServiceMonitors not rendered by chart) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: foxhunt-services - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - selector: - matchLabels: - app.kubernetes.io/part-of: foxhunt - namespaceSelector: - matchNames: [foxhunt] - endpoints: - - port: metrics - interval: 15s - # Override job label to "foxhunt-services" (matching old standalone config) - # so all existing dashboard queries continue to work - relabelings: - - targetLabel: job - replacement: foxhunt-services ---- -# DCGM Exporter (GPU metrics from nvidia-gpu-operator DaemonSet) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: dcgm-exporter - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - selector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: [dcgm-exporter] - namespaceSelector: - matchNames: [kube-system, monitoring, foxhunt] - endpoints: - - port: metrics - interval: 15s diff --git a/infra/k8s/monitoring/tempo.yaml b/infra/k8s/monitoring/tempo.yaml deleted file mode 100644 index aae720124..000000000 --- a/infra/k8s/monitoring/tempo.yaml +++ /dev/null @@ -1,139 +0,0 @@ -# Grafana Tempo — lightweight distributed tracing (single-binary mode) -# Receives OTLP traces from services, queryable via Grafana -apiVersion: v1 -kind: ConfigMap -metadata: - name: tempo-config - namespace: foxhunt - labels: - app.kubernetes.io/name: tempo -data: - tempo.yaml: | - server: - http_listen_port: 3200 - distributor: - receivers: - otlp: - protocols: - grpc: - endpoint: "0.0.0.0:4317" - http: - endpoint: "0.0.0.0:4318" - storage: - trace: - backend: local - local: - path: /var/tempo/traces - wal: - path: /var/tempo/wal - compactor: - compaction: - block_retention: 168h # 7 days - metrics_generator: - registry: - external_labels: - source: tempo - cluster: foxhunt - storage: - path: /var/tempo/generator/wal ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tempo - namespace: foxhunt - labels: - app.kubernetes.io/name: tempo -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: tempo - template: - metadata: - labels: - app.kubernetes.io/name: tempo - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - securityContext: - fsGroup: 10001 - runAsUser: 10001 - runAsNonRoot: true - containers: - - name: tempo - image: grafana/tempo:2.7.1 - args: ["-config.file=/etc/tempo/tempo.yaml"] - ports: - - containerPort: 3200 - name: http - - containerPort: 4317 - name: otlp-grpc - - containerPort: 4318 - name: otlp-http - volumeMounts: - - name: config - mountPath: /etc/tempo - - name: data - mountPath: /var/tempo - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - cpu: 500m - memory: 2Gi - readinessProbe: - httpGet: - path: /ready - port: 3200 - initialDelaySeconds: 10 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /ready - port: 3200 - initialDelaySeconds: 30 - periodSeconds: 15 - volumes: - - name: config - configMap: - name: tempo-config - - name: data - persistentVolumeClaim: - claimName: tempo-data ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: tempo-data - namespace: foxhunt -spec: - accessModes: [ReadWriteOnce] - storageClassName: scw-bssd - resources: - requests: - storage: 10Gi ---- -apiVersion: v1 -kind: Service -metadata: - name: tempo - namespace: foxhunt - labels: - app.kubernetes.io/name: tempo -spec: - selector: - app.kubernetes.io/name: tempo - ports: - - port: 3200 - targetPort: 3200 - name: http - - port: 4317 - targetPort: 4317 - name: otlp-grpc - - port: 4318 - targetPort: 4318 - name: otlp-http diff --git a/infra/k8s/namespace.yaml b/infra/k8s/namespace.yaml deleted file mode 100644 index 60ffb8750..000000000 --- a/infra/k8s/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/managed-by: kubectl diff --git a/infra/k8s/network-policies/alertmanager.yaml b/infra/k8s/network-policies/alertmanager.yaml deleted file mode 100644 index d3a984903..000000000 --- a/infra/k8s/network-policies/alertmanager.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# AlertManager network policy -# Ingress: Prometheus (alert delivery), Grafana (alert UI queries) -# Egress: Mattermost webhooks (cluster-internal), DNS -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: alertmanager - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: alertmanager - policyTypes: - - Ingress - - Egress - ingress: - # Prometheus pushes alerts and scrapes metrics (9093 API, 8080 config-reloader) - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: prometheus - ports: - - protocol: TCP - port: 9093 - - protocol: TCP - port: 8080 - # Grafana queries AlertManager API for alert state/silences - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: grafana - ports: - - protocol: TCP - port: 9093 - egress: - # DNS resolution - - to: - - namespaceSelector: {} - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - # Mattermost incoming webhook (cluster-internal) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost - ports: - - protocol: TCP - port: 8065 - # External HTTPS (PagerDuty, email, future integrations) - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - ports: - - protocol: TCP - port: 443 diff --git a/infra/k8s/network-policies/allow-dns.yaml b/infra/k8s/network-policies/allow-dns.yaml deleted file mode 100644 index 83f69d446..000000000 --- a/infra/k8s/network-policies/allow-dns.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Allow all foxhunt + CI pods to reach CoreDNS (kube-system) for name resolution. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-dns - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchExpressions: - - key: app.kubernetes.io/part-of - operator: In - values: [foxhunt, foxhunt-ci] - policyTypes: - - Egress - egress: - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 diff --git a/infra/k8s/network-policies/allow-monitoring.yaml b/infra/k8s/network-policies/allow-monitoring.yaml deleted file mode 100644 index 495ea8021..000000000 --- a/infra/k8s/network-policies/allow-monitoring.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Allow Prometheus to scrape metrics ports on all foxhunt + CI pods. -# Also allow all foxhunt + CI pods to push traces to Tempo. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-monitoring-scrape - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchExpressions: - - key: app.kubernetes.io/part-of - operator: In - values: ["foxhunt", "foxhunt-ci"] - policyTypes: - - Ingress - - Egress - ingress: - - from: - # External monitoring namespace (if any) - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: monitoring - # GitLab Prometheus in foxhunt namespace - - podSelector: - matchLabels: - app.kubernetes.io/name: prometheus - ports: - - protocol: TCP - port: 8080 - - protocol: TCP - port: 9003 - - protocol: TCP - port: 9091 - - protocol: TCP - port: 9092 - - protocol: TCP - port: 9093 - - protocol: TCP - port: 9094 - - protocol: TCP - port: 9095 - - protocol: TCP - port: 9096 - - protocol: TCP - port: 9097 - - protocol: TCP - port: 9098 - egress: - # Tempo OTLP gRPC - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: foxhunt - podSelector: - matchLabels: - app.kubernetes.io/name: tempo - ports: - - protocol: TCP - port: 4317 - # Kubernetes API (for kube-state-metrics list/watch) - - to: - - ipBlock: - cidr: 10.32.0.0/16 - ports: - - protocol: TCP - port: 443 - - protocol: TCP - port: 6443 - - to: - - ipBlock: - cidr: 172.16.0.0/16 - ports: - - protocol: TCP - port: 6443 diff --git a/infra/k8s/network-policies/api.yaml b/infra/k8s/network-policies/api.yaml deleted file mode 100644 index bb2c9f6e2..000000000 --- a/infra/k8s/network-policies/api.yaml +++ /dev/null @@ -1,110 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: api - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: api - policyTypes: - - Ingress - - Egress - ingress: - # Tailscale ingress proxy - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: tailscale-gitlab-proxy - ports: - - protocol: TCP - port: 50051 - egress: - # Postgres - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - # Redis - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: redis - ports: - - protocol: TCP - port: 6379 - # Downstream gRPC services - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: trading-service - ports: - - protocol: TCP - port: 50051 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: backtesting-service - ports: - - protocol: TCP - port: 50053 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: ml-training-service - ports: - - protocol: TCP - port: 50053 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: trading-agent-service - ports: - - protocol: TCP - port: 50055 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: broker-gateway - ports: - - protocol: TCP - port: 50056 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: data-acquisition-service - ports: - - protocol: TCP - port: 50057 - # Prometheus (metric scraping from monitoring handler) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: prometheus - ports: - - protocol: TCP - port: 9090 - # Minio (initContainer binary fetch) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 - # Kubernetes API (pod listing for Pods cockpit) - - to: - - ipBlock: - cidr: 10.32.0.0/12 - ports: - - protocol: TCP - port: 443 - - to: - - ipBlock: - cidr: 172.16.0.0/16 - ports: - - protocol: TCP - port: 6443 diff --git a/infra/k8s/network-policies/backtesting-service.yaml b/infra/k8s/network-policies/backtesting-service.yaml deleted file mode 100644 index a83e25186..000000000 --- a/infra/k8s/network-policies/backtesting-service.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: backtesting-service - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: backtesting-service - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: api - ports: - - protocol: TCP - port: 50053 - egress: - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: redis - ports: - - protocol: TCP - port: 6379 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 diff --git a/infra/k8s/network-policies/broker-gateway.yaml b/infra/k8s/network-policies/broker-gateway.yaml deleted file mode 100644 index 651012f8b..000000000 --- a/infra/k8s/network-policies/broker-gateway.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: broker-gateway - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: broker-gateway - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: api - - podSelector: - matchLabels: - app.kubernetes.io/name: trading-service - ports: - - protocol: TCP - port: 50056 - egress: - - to: - - podSelector: - matchLabels: - app: ib-gateway - ports: - - protocol: TCP - port: 4002 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: redis - ports: - - protocol: TCP - port: 6379 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 diff --git a/infra/k8s/network-policies/ci-egress.yaml b/infra/k8s/network-policies/ci-egress.yaml deleted file mode 100644 index 2fc012915..000000000 --- a/infra/k8s/network-policies/ci-egress.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# CI executor pods need broad egress: git clone (GitLab webservice), container -# registries, kubectl (K8s API), MinIO S3, and external tool downloads. -# These pods are ephemeral (minutes) — fine-grained egress adds little value. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: ci-egress - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/part-of: foxhunt-ci - policyTypes: - - Egress - egress: - - {} diff --git a/infra/k8s/network-policies/data-acquisition-service.yaml b/infra/k8s/network-policies/data-acquisition-service.yaml deleted file mode 100644 index 68b749ec8..000000000 --- a/infra/k8s/network-policies/data-acquisition-service.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: data-acquisition-service - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: data-acquisition-service - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: api - ports: - - protocol: TCP - port: 50057 - egress: - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: redis - ports: - - protocol: TCP - port: 6379 - # External HTTPS (Databento, Benzinga APIs) - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - ports: - - protocol: TCP - port: 443 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 diff --git a/infra/k8s/network-policies/data-download-job.yaml b/infra/k8s/network-policies/data-download-job.yaml deleted file mode 100644 index 2a9135a1d..000000000 --- a/infra/k8s/network-policies/data-download-job.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Egress policy for data-download jobs (MBP-10 download, etc.) -# Allows: MinIO (upload), Databento API (HTTPS download) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: data-download-job - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - foxhunt/job-type: data-download - policyTypes: - - Egress - egress: - # MinIO — binary fetch + data upload - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 - # Databento API (HTTPS) - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - - 100.64.0.0/10 - ports: - - protocol: TCP - port: 443 ---- -# Egress policy for data-sync jobs (MinIO → PVC sync) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: data-sync-job - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - foxhunt/job-type: data-sync - policyTypes: - - Egress - egress: - # MinIO — data sync source - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 diff --git a/infra/k8s/network-policies/default-deny.yaml b/infra/k8s/network-policies/default-deny.yaml deleted file mode 100644 index 566ffbe27..000000000 --- a/infra/k8s/network-policies/default-deny.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default deny all ingress and egress in foxhunt namespace. -# Every service must declare its own NetworkPolicy to communicate. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: default-deny-all - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/part-of: foxhunt - policyTypes: - - Ingress - - Egress diff --git a/infra/k8s/network-policies/gitlab-packages.yaml b/infra/k8s/network-policies/gitlab-packages.yaml deleted file mode 100644 index 2b65861c4..000000000 --- a/infra/k8s/network-policies/gitlab-packages.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Allow all foxhunt pods to fetch binaries from GitLab Generic Package Registry. -# initContainers in services and training jobs use curl to download release binaries -# from gitlab-webservice-default:8181 (GitLab API). -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: gitlab-packages - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/part-of: foxhunt - policyTypes: - - Egress - egress: - - to: - - podSelector: - matchLabels: - app: webservice - ports: - - protocol: TCP - port: 8181 diff --git a/infra/k8s/network-policies/ib-gateway.yaml b/infra/k8s/network-policies/ib-gateway.yaml deleted file mode 100644 index cbed5e445..000000000 --- a/infra/k8s/network-policies/ib-gateway.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: ib-gateway - namespace: foxhunt -spec: - podSelector: - matchLabels: - app: ib-gateway - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: trading-service - - podSelector: - matchLabels: - app.kubernetes.io/name: broker-gateway - - podSelector: - matchLabels: - app: multistrat-rebalancer - ports: - - protocol: TCP - port: 4002 - - protocol: TCP - port: 4004 - egress: - # IBKR external servers - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - ports: - - protocol: TCP - port: 443 - - protocol: TCP - port: 4000 - - protocol: TCP - port: 4001 diff --git a/infra/k8s/network-policies/infrastructure.yaml b/infra/k8s/network-policies/infrastructure.yaml deleted file mode 100644 index 48b252975..000000000 --- a/infra/k8s/network-policies/infrastructure.yaml +++ /dev/null @@ -1,127 +0,0 @@ -# Postgres: accepts connections from foxhunt app pods, GitLab, Grafana, and Gitea -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: postgres - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - policyTypes: - - Ingress - ingress: - - from: - - podSelector: - matchExpressions: - - key: app.kubernetes.io/part-of - operator: In - values: ["foxhunt", "argo-workflows"] - - podSelector: - matchLabels: - release: gitlab - - podSelector: - matchLabels: - app.kubernetes.io/name: grafana - - podSelector: # Gitea (Phase 2B) reuses the in-cluster postgres - matchLabels: - app.kubernetes.io/name: gitea - ports: - - protocol: TCP - port: 5432 ---- -# Redis: accepts connections from foxhunt app pods + GitLab (Helm release) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: redis - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: redis - policyTypes: - - Ingress - ingress: - - from: - - podSelector: - matchExpressions: - - key: app.kubernetes.io/part-of - operator: In - values: ["foxhunt"] - - podSelector: - matchLabels: - release: gitlab - ports: - - protocol: TCP - port: 6379 ---- -# Minio: accepts connections from foxhunt app pods, GitLab, and init jobs -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: minio - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: minio - policyTypes: - - Ingress - ingress: - - from: - - podSelector: - matchExpressions: - - key: app.kubernetes.io/part-of - operator: In - values: ["foxhunt", "foxhunt-ci", "argo-workflows"] - - podSelector: - matchLabels: - release: gitlab - - podSelector: - matchLabels: - app.kubernetes.io/name: minio-init - - podSelector: # tailscale proxy fronts minio.fxhnt.ai (S3 API) - matchLabels: - app.kubernetes.io/name: tailscale-gitlab-proxy - ports: - - protocol: TCP - port: 9000 ---- -# QuestDB: accepts ILP writes from trading-service + Prometheus metrics scrape -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: questdb - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: questdb - policyTypes: - - Ingress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: trading-service - - podSelector: - matchLabels: - foxhunt/job-type: training - ports: - - protocol: TCP - port: 9009 - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: prometheus - ports: - - protocol: TCP - port: 9003 - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: grafana - ports: - - protocol: TCP - port: 8812 diff --git a/infra/k8s/network-policies/maintenance-jobs.yaml b/infra/k8s/network-policies/maintenance-jobs.yaml deleted file mode 100644 index 893283ddb..000000000 --- a/infra/k8s/network-policies/maintenance-jobs.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# Network policies for maintenance CronJobs (postgres-backup, pvc-autoscaler) -# These jobs need egress to internal services but no internet access. -# Apply: kubectl apply -f maintenance-jobs.yaml ---- -# Postgres backup job: egress to postgres (5432) + minio (9000) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: postgres-backup - namespace: foxhunt -spec: - podSelector: - matchLabels: - app: postgres-backup - policyTypes: - - Egress - egress: - # DNS - - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - # Postgres - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - # MinIO (backup upload) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 ---- -# PVC autoscaler job: egress to K8s API + kubelet stats -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: pvc-autoscaler - namespace: foxhunt -spec: - podSelector: - matchLabels: - app: pvc-autoscaler - policyTypes: - - Egress - egress: - # DNS - - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - # Kubernetes API server (HTTPS) - - ports: - - protocol: TCP - port: 443 - # Kubelet stats API (port 10250) - - ports: - - protocol: TCP - port: 10250 diff --git a/infra/k8s/network-policies/mattermost.yaml b/infra/k8s/network-policies/mattermost.yaml deleted file mode 100644 index d63ac820f..000000000 --- a/infra/k8s/network-policies/mattermost.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: mattermost - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost - policyTypes: - - Ingress - - Egress - ingress: - # HTTP from Tailscale proxy + Argo workflow pods (webhook notifications) - - ports: - - port: 8065 - protocol: TCP - egress: - # DNS - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # PostgreSQL - - ports: - - port: 5432 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - # Stalwart SMTP (for email notifications, wired later) - - ports: - - port: 25 - protocol: TCP - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: stalwart diff --git a/infra/k8s/network-policies/ml-training-service.yaml b/infra/k8s/network-policies/ml-training-service.yaml deleted file mode 100644 index f629688a5..000000000 --- a/infra/k8s/network-policies/ml-training-service.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: ml-training-service - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: ml-training-service - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: api - ports: - - protocol: TCP - port: 50053 - egress: - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: redis - ports: - - protocol: TCP - port: 6379 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 - # Kubernetes API (for job creation — ml-training-service has RBAC for batch/jobs) - - to: - - ipBlock: - cidr: 0.0.0.0/0 - ports: - - protocol: TCP - port: 443 diff --git a/infra/k8s/network-policies/prometheus-cilium.yaml b/infra/k8s/network-policies/prometheus-cilium.yaml deleted file mode 100644 index 6b56bde69..000000000 --- a/infra/k8s/network-policies/prometheus-cilium.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# CiliumNetworkPolicy: allow Prometheus to reach hostNetwork pods (node-exporter). -# -# Vanilla K8s NetworkPolicy ipBlock rules don't match traffic to hostNetwork -# pods because Cilium assigns them the 'host'/'remote-node' identity, not 'world'. -# This CiliumNetworkPolicy uses toEntities to cover that gap. -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: prometheus-host-access - namespace: foxhunt -spec: - endpointSelector: - matchLabels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: foxhunt - egress: - - toEntities: - - host - - remote-node - toPorts: - - ports: - - port: "9100" - protocol: TCP diff --git a/infra/k8s/network-policies/prometheus.yaml b/infra/k8s/network-policies/prometheus.yaml deleted file mode 100644 index 5cbd159e4..000000000 --- a/infra/k8s/network-policies/prometheus.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# Prometheus egress: K8s API (service discovery), foxhunt services, node-exporter, -# kube-state-metrics, DCGM exporter, Argo workflow controller, pushgateway, -# Loki, Tempo, QuestDB, and AlertManager. -# Applies to operator-managed Prometheus (kube-prometheus-stack). -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: prometheus - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: prometheus - policyTypes: - - Ingress - - Egress - ingress: - # Grafana queries Prometheus on port 9090 - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: grafana - ports: - - protocol: TCP - port: 9090 - egress: - # DNS resolution (required for additionalScrapeConfigs with hostnames) - - to: - - namespaceSelector: {} - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - - # Kubernetes API server (required for kubernetes_sd_configs service discovery) - # ClusterIP 10.32.0.1:443 NATs to node IP 172.16.0.0/16:6443 - - to: - - ipBlock: - cidr: 10.32.0.0/16 - ports: - - protocol: TCP - port: 443 - - protocol: TCP - port: 6443 - - # Scrape foxhunt service metrics ports (9003, 9090-9098) - - to: - - podSelector: - matchExpressions: - - key: app.kubernetes.io/part-of - operator: In - values: ["foxhunt", "foxhunt-ci"] - ports: - - protocol: TCP - port: 9003 - - protocol: TCP - port: 9090 - - protocol: TCP - port: 9091 - - protocol: TCP - port: 9092 - - protocol: TCP - port: 9093 - - protocol: TCP - port: 9094 - - protocol: TCP - port: 9095 - - protocol: TCP - port: 9096 - - protocol: TCP - port: 9097 - - protocol: TCP - port: 9098 - - # kube-state-metrics (Helm sub-chart, port 8080) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: kube-state-metrics - ports: - - protocol: TCP - port: 8080 - - # Prometheus operator (container port 10250) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator - ports: - - protocol: TCP - port: 10250 - - # Alertmanager (port 9093 for API, port 8080 for config-reloader sidecar) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: alertmanager - ports: - - protocol: TCP - port: 9093 - - protocol: TCP - port: 8080 - - # Loki metrics (port 3100) and Tempo metrics (port 3200) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: loki - ports: - - protocol: TCP - port: 3100 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: tempo - ports: - - protocol: TCP - port: 3200 - - # QuestDB metrics - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: questdb - ports: - - protocol: TCP - port: 9003 - - # Node-exporter (hostNetwork), kubelet, and K8s API on node IPs - - to: - - ipBlock: - cidr: 172.16.0.0/16 - ports: - - protocol: TCP - port: 6443 - - protocol: TCP - port: 9100 - - protocol: TCP - port: 9400 - - protocol: TCP - port: 10250 # kubelet metrics (cAdvisor, resource usage) - - # kube-system: CoreDNS metrics, kube-state-metrics, node-exporter, DCGM - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system - ports: - - protocol: TCP - port: 8080 - - protocol: TCP - port: 8443 - - protocol: TCP - port: 9100 - - protocol: TCP - port: 9153 # CoreDNS metrics - - protocol: TCP - port: 9400 - - # monitoring namespace: DCGM exporter (GPU metrics) - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: monitoring - ports: - - protocol: TCP - port: 9400 - - # Pushgateway (training pods push metrics here) - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: pushgateway - ports: - - protocol: TCP - port: 9091 - - # Argo Workflows controller metrics - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: argo-workflows-workflow-controller - ports: - - protocol: TCP - port: 9090 diff --git a/infra/k8s/network-policies/stalwart.yaml b/infra/k8s/network-policies/stalwart.yaml deleted file mode 100644 index 01fdf2e4c..000000000 --- a/infra/k8s/network-policies/stalwart.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: stalwart - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: stalwart - policyTypes: - - Ingress - - Egress - ingress: - # SMTP from cluster (Mattermost internal relay) - - ports: - - port: 25 - protocol: TCP - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: mattermost - # IMAPS from Tailscale proxy - - ports: - - port: 993 - protocol: TCP - # HTTPS admin panel from Tailscale proxy - - ports: - - port: 443 - protocol: TCP - egress: - # DNS - - ports: - - port: 53 - protocol: UDP - - port: 53 - protocol: TCP - # HTTPS out — webadmin bundle + spam model downloads from GitHub - - ports: - - port: 443 - protocol: TCP - # No outbound SMTP — internal-only for now. - # To add SCW TEM relay later: - # - ports: - # - port: 465 - # protocol: TCP - # - port: 587 - # protocol: TCP diff --git a/infra/k8s/network-policies/trading-agent-service.yaml b/infra/k8s/network-policies/trading-agent-service.yaml deleted file mode 100644 index 78dcc9f19..000000000 --- a/infra/k8s/network-policies/trading-agent-service.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: trading-agent-service - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: trading-agent-service - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: api - ports: - - protocol: TCP - port: 50055 - egress: - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: trading-service - ports: - - protocol: TCP - port: 50051 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: redis - ports: - - protocol: TCP - port: 6379 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 diff --git a/infra/k8s/network-policies/trading-service.yaml b/infra/k8s/network-policies/trading-service.yaml deleted file mode 100644 index e8278e432..000000000 --- a/infra/k8s/network-policies/trading-service.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: trading-service - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: trading-service - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: api - - podSelector: - matchLabels: - app.kubernetes.io/name: trading-agent-service - ports: - - protocol: TCP - port: 50051 - egress: - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: postgres - ports: - - protocol: TCP - port: 5432 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: redis - ports: - - protocol: TCP - port: 6379 - - to: - - podSelector: - matchLabels: - app: ib-gateway - ports: - - protocol: TCP - port: 4002 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: questdb - ports: - - protocol: TCP - port: 9009 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 diff --git a/infra/k8s/network-policies/training-job.yaml b/infra/k8s/network-policies/training-job.yaml deleted file mode 100644 index 0fd0b6884..000000000 --- a/infra/k8s/network-policies/training-job.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Egress policy for training jobs (binary fetch, data sync, telemetry) -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: training-job - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -spec: - podSelector: - matchLabels: - foxhunt/job-type: training - policyTypes: - - Egress - egress: - # MinIO — binary fetch + data sync - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 - # Tempo — OTLP tracing - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: tempo - ports: - - protocol: TCP - port: 4317 - # Pushgateway — Prometheus metrics push - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: pushgateway - ports: - - protocol: TCP - port: 9091 - # QuestDB — ILP training metrics persistence - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: questdb - ports: - - protocol: TCP - port: 9009 diff --git a/infra/k8s/network-policies/web-dashboard.yaml b/infra/k8s/network-policies/web-dashboard.yaml deleted file mode 100644 index c5b92b02b..000000000 --- a/infra/k8s/network-policies/web-dashboard.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: web-dashboard - namespace: foxhunt -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: web-dashboard - policyTypes: - - Ingress - - Egress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: tailscale-gitlab-proxy - ports: - - protocol: TCP - port: 80 - egress: - # Init container fetches assets from MinIO - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 diff --git a/infra/k8s/secrets/databento-credentials.secret.example b/infra/k8s/secrets/databento-credentials.secret.example deleted file mode 100644 index d528eac9e..000000000 --- a/infra/k8s/secrets/databento-credentials.secret.example +++ /dev/null @@ -1,19 +0,0 @@ -# Databento API credentials for MBP-10 data download -# Copy this file and fill in your API key: -# cp databento-credentials.secret.example databento-credentials.yaml -# # Edit databento-credentials.yaml with your API key -# kubectl apply -f databento-credentials.yaml -# -# Or create imperatively: -# kubectl -n foxhunt create secret generic databento-credentials \ -# --from-literal=api-key=$DATABENTO_API_KEY -apiVersion: v1 -kind: Secret -metadata: - name: databento-credentials - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - api-key: REPLACE_WITH_YOUR_DATABENTO_API_KEY diff --git a/infra/k8s/secrets/foxhunt-secrets.yaml b/infra/k8s/secrets/foxhunt-secrets.yaml deleted file mode 100644 index d5b167e36..000000000 --- a/infra/k8s/secrets/foxhunt-secrets.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Database credentials — used by services that connect to PostgreSQL. -# Real values injected by scripts/deploy-secrets.sh from Scaleway Secrets Manager. -apiVersion: v1 -kind: Secret -metadata: - name: db-credentials - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - password: "REPLACE_IN_DEPLOY" ---- -# JWT signing secret — used by services that validate/issue JWTs. -apiVersion: v1 -kind: Secret -metadata: - name: jwt-secret - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - secret: "REPLACE_IN_DEPLOY" ---- -# Redis credentials — used by services with Redis connections. -apiVersion: v1 -kind: Secret -metadata: - name: redis-credentials - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - password: "REPLACE_IN_DEPLOY" ---- -# S3/Minio credentials for application-level access (model storage, data). -# NOT the same as minio-credentials (used by initContainers for binary fetch). -apiVersion: v1 -kind: Secret -metadata: - name: s3-credentials - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - access-key: "REPLACE_IN_DEPLOY" - secret-key: "REPLACE_IN_DEPLOY" ---- -# Registry pull secret for Scaleway Container Registry -# Create with: -# kubectl create secret docker-registry scw-registry \ -# --namespace foxhunt \ -# --docker-server=rg.fr-par.scw.cloud \ -# --docker-username=foxhunt \ -# --docker-password= diff --git a/infra/k8s/secrets/ibkr-credentials.yaml b/infra/k8s/secrets/ibkr-credentials.yaml deleted file mode 100644 index a14a136c0..000000000 --- a/infra/k8s/secrets/ibkr-credentials.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: ibkr-credentials - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - # These values are populated from Scaleway Secret Manager via CI/CD - # NEVER commit real credentials here - account-id: "REPLACE_FROM_SCW_SECRET" - username: "REPLACE_FROM_SCW_SECRET" - password: "REPLACE_FROM_SCW_SECRET" - vnc-password: "REPLACE_FROM_SCW_SECRET" diff --git a/infra/k8s/secrets/ml-training-tls.yaml b/infra/k8s/secrets/ml-training-tls.yaml deleted file mode 100644 index 6f8f26763..000000000 --- a/infra/k8s/secrets/ml-training-tls.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: ml-training-tls - namespace: foxhunt - labels: - app.kubernetes.io/part-of: foxhunt -type: kubernetes.io/tls -data: - # Base64-encoded TLS certificate and key - # Generate with: scripts/generate_dev_certs.sh - # Then: kubectl create secret tls ml-training-tls --cert=server.crt --key=server.key -n foxhunt - tls.crt: "REPLACE_WITH_BASE64_CERT" - tls.key: "REPLACE_WITH_BASE64_KEY" diff --git a/infra/k8s/services/api.yaml b/infra/k8s/services/api.yaml deleted file mode 100644 index 4f42f1b96..000000000 --- a/infra/k8s/services/api.yaml +++ /dev/null @@ -1,220 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: api - namespace: foxhunt - labels: - app.kubernetes.io/name: api - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment api --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: api - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9091" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: api - app.kubernetes.io/part-of: foxhunt - spec: - serviceAccountName: api - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - BINARY="api" - curl -fSL -o "/binaries/${BINARY}" \ - --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" - chmod +x "/binaries/${BINARY}" - echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" - env: - - name: GITLAB_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-deploy-token - key: token - - name: GITLAB_API - value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - name: FOXHUNT_RELEASE - value: "latest" - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: api - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: ["/binaries/api"] - ports: - - containerPort: 50051 - name: grpc - - containerPort: 9091 - name: metrics - env: - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - - name: REDIS_URL - value: "redis://redis:6379" - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: jwt-secret - key: secret - - name: JWT_ISSUER - value: foxhunt-api - - name: JWT_AUDIENCE - value: foxhunt-services - - name: TRADING_SERVICE_URL - value: "http://trading-service:50051" - - name: BACKTESTING_SERVICE_URL - value: "http://backtesting-service:50053" - - name: ML_TRAINING_SERVICE_URL - value: "http://ml-training-service:50053" - - name: TRADING_AGENT_SERVICE_URL - value: "http://trading-agent-service:50055" - - name: BROKER_GATEWAY_SERVICE_URL - value: "http://broker-gateway:50056" - - name: DATA_ACQUISITION_SERVICE_URL - value: "http://data-acquisition-service:50057" - - name: ML_SERVICE_URL - value: "http://trading-service:50051" - - name: PROMETHEUS_URL - value: "http://prometheus-stack-kube-prom-prometheus:9090" - - name: RUST_LOG - value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - - name: CORS_ORIGINS - value: "https://dashboard.fxhnt.ai,http://localhost:5173" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: tmp - mountPath: /tmp - readinessProbe: - exec: - command: - - grpc_health_probe - - -addr=localhost:50051 - initialDelaySeconds: 10 - periodSeconds: 10 - livenessProbe: - exec: - command: - - grpc_health_probe - - -addr=localhost:50051 - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - volumes: - - name: binaries - emptyDir: - sizeLimit: 200Mi - - name: tmp - emptyDir: - sizeLimit: 50Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: api - namespace: foxhunt - labels: - app.kubernetes.io/name: api - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: api - ports: - - port: 50051 - targetPort: 50051 - name: grpc - - port: 9091 - targetPort: 9091 - name: metrics ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: api - namespace: foxhunt - labels: - app.kubernetes.io/name: api - app.kubernetes.io/part-of: foxhunt ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: api-pod-reader - namespace: foxhunt -rules: - - apiGroups: [""] - resources: [pods] - verbs: [get, list, watch] - - apiGroups: [metrics.k8s.io] - resources: [pods] - verbs: [get, list] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: api-pod-reader - namespace: foxhunt -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: api-pod-reader -subjects: - - kind: ServiceAccount - name: api - namespace: foxhunt diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml deleted file mode 100644 index 30f6ca005..000000000 --- a/infra/k8s/services/backtesting-service.yaml +++ /dev/null @@ -1,165 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: backtesting-service - namespace: foxhunt - labels: - app.kubernetes.io/name: backtesting-service - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment backtesting-service --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: backtesting-service - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9093" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: backtesting-service - app.kubernetes.io/part-of: foxhunt - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - BINARY="backtesting-service" - curl -fSL -o "/binaries/${BINARY}" \ - --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" - chmod +x "/binaries/${BINARY}" - echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" - env: - - name: GITLAB_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-deploy-token - key: token - - name: GITLAB_API - value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - name: FOXHUNT_RELEASE - value: "latest" - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: backtesting-service - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: ["/binaries/backtesting-service"] - ports: - - containerPort: 50053 - name: grpc - - containerPort: 9093 - name: metrics - env: - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - - name: REDIS_URL - value: "redis://redis:6379" - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: jwt-secret - key: secret - - name: JWT_ISSUER - value: foxhunt-api - - name: JWT_AUDIENCE - value: foxhunt-services - - name: BENZINGA_API_KEY - value: "placeholder" - - name: RUST_LOG - value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: tmp - mountPath: /tmp - readinessProbe: - httpGet: - path: /health - port: 8082 - initialDelaySeconds: 15 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /health - port: 8082 - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - volumes: - - name: binaries - emptyDir: - sizeLimit: 200Mi - - name: tmp - emptyDir: - sizeLimit: 50Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: backtesting-service - namespace: foxhunt - labels: - app.kubernetes.io/name: backtesting-service - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: backtesting-service - ports: - - port: 50053 - targetPort: 50053 - name: grpc - - port: 9093 - targetPort: 9093 - name: metrics diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml deleted file mode 100644 index 9a0fc84a5..000000000 --- a/infra/k8s/services/broker-gateway.yaml +++ /dev/null @@ -1,176 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: broker-gateway - namespace: foxhunt - labels: - app.kubernetes.io/name: broker-gateway - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment broker-gateway --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: broker-gateway - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9096" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: broker-gateway - app.kubernetes.io/part-of: foxhunt - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - BINARY="broker-gateway" - curl -fSL -o "/binaries/${BINARY}" \ - --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" - chmod +x "/binaries/${BINARY}" - echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" - env: - - name: GITLAB_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-deploy-token - key: token - - name: GITLAB_API - value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - name: FOXHUNT_RELEASE - value: "latest" - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: broker-gateway - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: ["/binaries/broker-gateway"] - ports: - - containerPort: 50056 - name: grpc - - containerPort: 9096 - name: metrics - env: - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - - name: REDIS_URL - value: "redis://redis:6379" - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: jwt-secret - key: secret - - name: JWT_ISSUER - value: foxhunt-api - - name: JWT_AUDIENCE - value: foxhunt-services - - name: IBKR_HOST - value: "ib-gateway" - - name: IBKR_PORT - value: "4002" - - name: IBKR_CLIENT_ID - value: "2" - - name: IBKR_ACCOUNT_ID - valueFrom: - secretKeyRef: - name: ibkr-credentials - key: account-id - - name: RUST_LOG - value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: tmp - mountPath: /tmp - readinessProbe: - exec: - command: - - grpc_health_probe - - -addr=localhost:50056 - initialDelaySeconds: 10 - periodSeconds: 10 - livenessProbe: - exec: - command: - - grpc_health_probe - - -addr=localhost:50056 - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - volumes: - - name: binaries - emptyDir: - sizeLimit: 200Mi - - name: tmp - emptyDir: - sizeLimit: 50Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: broker-gateway - namespace: foxhunt - labels: - app.kubernetes.io/name: broker-gateway - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: broker-gateway - ports: - - port: 50056 - targetPort: 50056 - name: grpc - - port: 9096 - targetPort: 9096 - name: metrics diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml deleted file mode 100644 index d7f4d13da..000000000 --- a/infra/k8s/services/data-acquisition-service.yaml +++ /dev/null @@ -1,159 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-acquisition-service - namespace: foxhunt - labels: - app.kubernetes.io/name: data-acquisition-service - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment data-acquisition-service --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: data-acquisition-service - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9097" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: data-acquisition-service - app.kubernetes.io/part-of: foxhunt - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - BINARY="data-acquisition-service" - curl -fSL -o "/binaries/${BINARY}" \ - --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" - chmod +x "/binaries/${BINARY}" - echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" - env: - - name: GITLAB_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-deploy-token - key: token - - name: GITLAB_API - value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - name: FOXHUNT_RELEASE - value: "latest" - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: data-acquisition-service - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: ["/binaries/data-acquisition-service"] - ports: - - containerPort: 50057 - name: grpc - - containerPort: 8095 - name: health - - containerPort: 9097 - name: metrics - env: - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - - name: REDIS_URL - value: "redis://redis:6379" - - name: GRPC_PORT - value: "50057" - - name: RUST_LOG - value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: tmp - mountPath: /tmp - readinessProbe: - tcpSocket: - port: 50057 - initialDelaySeconds: 15 - periodSeconds: 10 - livenessProbe: - tcpSocket: - port: 50057 - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 1Gi - volumes: - - name: binaries - emptyDir: - sizeLimit: 200Mi - - name: tmp - emptyDir: - sizeLimit: 50Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: data-acquisition-service - namespace: foxhunt - labels: - app.kubernetes.io/name: data-acquisition-service - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: data-acquisition-service - ports: - - port: 50057 - targetPort: 50057 - name: grpc - - port: 8095 - targetPort: 8095 - name: health - - port: 9097 - targetPort: 9097 - name: metrics diff --git a/infra/k8s/services/ib-gateway.yaml b/infra/k8s/services/ib-gateway.yaml deleted file mode 100644 index a6e56c10b..000000000 --- a/infra/k8s/services/ib-gateway.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ib-gateway - namespace: foxhunt - labels: - app: ib-gateway - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment ib-gateway --replicas=1 - replicas: 0 - strategy: - type: Recreate - selector: - matchLabels: - app: ib-gateway - template: - metadata: - labels: - app: ib-gateway - app.kubernetes.io/part-of: foxhunt - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: ib-gateway - image: ghcr.io/gnzsnz/ib-gateway:stable - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - ports: - - name: tws-api - containerPort: 4002 - protocol: TCP - - name: tws-api-socat - containerPort: 4004 - protocol: TCP - - name: vnc - containerPort: 5900 - protocol: TCP - env: - - name: TWS_USERID - valueFrom: - secretKeyRef: - name: ibkr-credentials - key: username - - name: TWS_PASSWORD - valueFrom: - secretKeyRef: - name: ibkr-credentials - key: password - - name: TRADING_MODE - value: "paper" - - name: TWS_ACCEPT_INCOMING - value: "accept" - - name: READ_ONLY_API - value: "no" - - name: TWOFA_TIMEOUT_ACTION - value: "restart" - - name: EXISTING_SESSION_DETECTED_ACTION - value: "primaryoverride" - - name: VNC_SERVER_PASSWORD - valueFrom: - secretKeyRef: - name: ibkr-credentials - key: vnc-password - optional: true - resources: - requests: - memory: "1Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "1000m" - readinessProbe: - tcpSocket: - port: 4002 - initialDelaySeconds: 90 - periodSeconds: 10 - livenessProbe: - tcpSocket: - port: 4002 - initialDelaySeconds: 120 - periodSeconds: 30 - failureThreshold: 5 ---- -apiVersion: v1 -kind: Service -metadata: - name: ib-gateway - namespace: foxhunt - labels: - app: ib-gateway -spec: - selector: - app: ib-gateway - ports: - - name: tws-api - port: 4002 - targetPort: 4002 - - name: tws-api-socat - port: 4004 - targetPort: 4004 - - name: vnc - port: 5900 - targetPort: 5900 - type: ClusterIP diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml deleted file mode 100644 index 53a594310..000000000 --- a/infra/k8s/services/ml-training-service.yaml +++ /dev/null @@ -1,229 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ml-training-service - namespace: foxhunt - labels: - app.kubernetes.io/name: ml-training-service - app.kubernetes.io/part-of: foxhunt ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: ml-training-job-manager - namespace: foxhunt - labels: - app.kubernetes.io/name: ml-training-service - app.kubernetes.io/part-of: foxhunt -rules: - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create", "get", "list", "watch", "delete"] - - apiGroups: [""] - resources: ["pods", "pods/log"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: ml-training-job-manager - namespace: foxhunt - labels: - app.kubernetes.io/name: ml-training-service - app.kubernetes.io/part-of: foxhunt -subjects: - - kind: ServiceAccount - name: ml-training-service - namespace: foxhunt -roleRef: - kind: Role - name: ml-training-job-manager - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ml-training-service - namespace: foxhunt - labels: - app.kubernetes.io/name: ml-training-service - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Training runs via Argo workflows (scripts/argo-train.sh) — this HTTP service is not required. - # Restore with: kubectl -n foxhunt scale deployment ml-training-service --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: ml-training-service - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9094" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: ml-training-service - app.kubernetes.io/part-of: foxhunt - spec: - serviceAccountName: ml-training-service - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - BINARY="ml-training-service" - curl -fSL -o "/binaries/${BINARY}" \ - --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" - chmod +x "/binaries/${BINARY}" - echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" - env: - - name: GITLAB_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-deploy-token - key: token - - name: GITLAB_API - value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - name: FOXHUNT_RELEASE - value: "latest" - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: ml-training-service - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: ["/binaries/ml-training-service", "serve"] - ports: - - containerPort: 50053 - name: grpc - - containerPort: 9094 - name: metrics - env: - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - - name: REDIS_URL - value: "redis://redis:6379" - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: jwt-secret - key: secret - - name: JWT_ISSUER - value: foxhunt-api - - name: JWT_AUDIENCE - value: foxhunt-services - - name: S3_ENDPOINT - value: "http://minio.foxhunt.svc.cluster.local:9000" - - name: S3_BUCKET - value: foxhunt-models - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - - name: TRAINING_RUNTIME_IMAGE - value: "gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest" - - name: CALLBACK_ENDPOINT - value: "http://ml-training-service.foxhunt.svc.cluster.local:50053" - - name: RUST_LOG - value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: tls-certs - mountPath: /app/certs/ml_training_service - readOnly: true - - name: tmp - mountPath: /tmp - readinessProbe: - tcpSocket: - port: 50053 - initialDelaySeconds: 15 - periodSeconds: 10 - livenessProbe: - tcpSocket: - port: 50053 - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - volumes: - - name: binaries - emptyDir: - sizeLimit: 200Mi - - name: tls-certs - secret: - secretName: ml-training-tls - - name: tmp - emptyDir: - sizeLimit: 50Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: ml-training-service - namespace: foxhunt - labels: - app.kubernetes.io/name: ml-training-service - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: ml-training-service - ports: - - port: 50053 - targetPort: 50053 - name: grpc - - port: 9094 - targetPort: 9094 - name: metrics diff --git a/infra/k8s/services/multistrat-rebalancer.yaml b/infra/k8s/services/multistrat-rebalancer.yaml deleted file mode 100644 index fdc90f440..000000000 --- a/infra/k8s/services/multistrat-rebalancer.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# Multi-strat rebalancer — weekly CronJob running the adaptive book on IBKR (paper) via the -# in-cluster ib-gateway. Production deployment of scripts/surfer/multistrat_bot.py. -# -# Code is delivered via ConfigMap (no image build needed — lean). Create/refresh it with: -# kubectl create configmap multistrat-bot-code -n foxhunt \ -# --from-file=scripts/surfer/multistrat_bot.py \ -# --from-file=scripts/surfer/multistrat_paper.py \ -# --dry-run=client -o yaml | kubectl apply -f - -# -# SAFETY: MULTISTRAT_EXECUTE defaults to "false" (cluster dry-run). After validating a few dry-run -# CronJob logs, flip to "true" to place paper orders. Account is paper (ib-gateway TRADING_MODE=paper); -# the bot additionally refuses any non-DU (live) account unless MULTISTRAT_ALLOW_LIVE_CONFIRMED is set. ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: multistrat-state - namespace: foxhunt - labels: - app: multistrat-rebalancer -spec: - accessModes: ["ReadWriteOnce"] # sequential CronJob runs (Forbid concurrency) — RWO is fine - resources: - requests: - storage: 1Gi ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: multistrat-rebalancer - namespace: foxhunt -spec: - podSelector: - matchLabels: - app: multistrat-rebalancer - policyTypes: [Egress] - egress: - - to: [] # DNS - ports: - - {protocol: UDP, port: 53} - - {protocol: TCP, port: 53} - - to: # in-cluster ib-gateway (4002 API + 4004 socat bridge for pod-to-pod) - - podSelector: - matchLabels: - app: ib-gateway - ports: - - {protocol: TCP, port: 4002} - - {protocol: TCP, port: 4004} - - to: # internet 443: PyPI (pip) + Yahoo (prices), no internal ranges - - ipBlock: - cidr: 0.0.0.0/0 - except: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] - ports: - - {protocol: TCP, port: 443} ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: multistrat-rebalancer - namespace: foxhunt - labels: - app: multistrat-rebalancer - app.kubernetes.io/part-of: foxhunt -spec: - schedule: "35 14 * * 1-5" # weekdays 14:35 UTC (~1h after US open) — daily risk re-eval; hysteresis prevents churn - concurrencyPolicy: Forbid - startingDeadlineSeconds: 3600 - successfulJobsHistoryLimit: 5 - failedJobsHistoryLimit: 10 - jobTemplate: - spec: - backoffLimit: 1 - activeDeadlineSeconds: 600 - template: - metadata: - labels: - app: multistrat-rebalancer - spec: - restartPolicy: Never - nodeSelector: - k8s.scaleway.com/pool-name: platform - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - name: rebalancer - image: python:3.12-slim - command: ["/bin/sh", "-c"] - args: - - >- - pip install --quiet --no-cache-dir ib_async==2.1.0 numpy && - python /app/multistrat_bot.py run - env: - - {name: IB_HOST, value: "ib-gateway"} - - {name: IB_PORT, value: "4004"} # socat bridge: relays via localhost so IB Gateway trusts it (4002 = pod-IP, untrusted -> handshake timeout) - - {name: IB_CLIENT_ID, value: "11"} - - {name: MULTISTRAT_MAXLEV, value: "1.0"} - - {name: MULTISTRAT_HYST, value: "0.03"} - - {name: MULTISTRAT_REBALANCE_DAYS, value: "1"} # daily eval (weekday cron); hysteresis (3%) gates actual trades - - {name: MULTISTRAT_DD_HALT, value: "0.20"} - - {name: MULTISTRAT_MAX_ORDER, value: "0.30"} - - {name: MULTISTRAT_EXECUTE, value: "true"} # ARMED: places paper orders (account DU* paper-guarded; live needs MULTISTRAT_ALLOW_LIVE_CONFIRMED) - - {name: MULTISTRAT_STATE, value: "/data/state.json"} - - {name: MULTISTRAT_LOG, value: "/data/bot.log"} - - {name: PIP_DISABLE_PIP_VERSION_CHECK, value: "1"} - - {name: PYTHONUNBUFFERED, value: "1"} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - resources: - requests: {memory: "256Mi", cpu: "100m"} - limits: {memory: "512Mi", cpu: "500m"} - volumeMounts: - - {mountPath: /app, name: code, readOnly: true} - - {mountPath: /data, name: state} - volumes: - - name: code - configMap: - name: multistrat-bot-code - - name: state - persistentVolumeClaim: - claimName: multistrat-state diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml deleted file mode 100644 index 13db3d5b7..000000000 --- a/infra/k8s/services/trading-agent-service.yaml +++ /dev/null @@ -1,163 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trading-agent-service - namespace: foxhunt - labels: - app.kubernetes.io/name: trading-agent-service - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment trading-agent-service --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: trading-agent-service - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9095" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: trading-agent-service - app.kubernetes.io/part-of: foxhunt - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - BINARY="trading-agent-service" - curl -fSL -o "/binaries/${BINARY}" \ - --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" - chmod +x "/binaries/${BINARY}" - echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" - env: - - name: GITLAB_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-deploy-token - key: token - - name: GITLAB_API - value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - name: FOXHUNT_RELEASE - value: "latest" - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: trading-agent-service - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: ["/binaries/trading-agent-service"] - ports: - - containerPort: 50055 - name: grpc - - containerPort: 9095 - name: metrics - env: - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - - name: REDIS_URL - value: "redis://redis:6379" - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: jwt-secret - key: secret - - name: JWT_ISSUER - value: foxhunt-api - - name: JWT_AUDIENCE - value: foxhunt-services - - name: RUST_LOG - value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: tmp - mountPath: /tmp - readinessProbe: - httpGet: - path: /health - port: 8083 - initialDelaySeconds: 15 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /health - port: 8083 - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - volumes: - - name: binaries - emptyDir: - sizeLimit: 200Mi - - name: tmp - emptyDir: - sizeLimit: 50Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: trading-agent-service - namespace: foxhunt - labels: - app.kubernetes.io/name: trading-agent-service - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: trading-agent-service - ports: - - port: 50055 - targetPort: 50055 - name: grpc - - port: 9095 - targetPort: 9095 - name: metrics diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml deleted file mode 100644 index b126af6e4..000000000 --- a/infra/k8s/services/trading-service.yaml +++ /dev/null @@ -1,180 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trading-service - namespace: foxhunt - labels: - app.kubernetes.io/name: trading-service - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment trading-service --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: trading-service - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9092" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: trading-service - app.kubernetes.io/part-of: foxhunt - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - imagePullSecrets: - - name: gitlab-registry - nodeSelector: - k8s.scaleway.com/pool-name: platform - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - BINARY="trading-service" - curl -fSL -o "/binaries/${BINARY}" \ - --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" - chmod +x "/binaries/${BINARY}" - echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" - env: - - name: GITLAB_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-deploy-token - key: token - - name: GITLAB_API - value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - name: FOXHUNT_RELEASE - value: "latest" - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: trading-service - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: ["/binaries/trading-service"] - ports: - - containerPort: 50051 - name: grpc - - containerPort: 9092 - name: metrics - env: - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: DATABASE_URL - value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - - name: REDIS_URL - value: "redis://redis:6379" - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: jwt-secret - key: secret - - name: JWT_ISSUER - value: foxhunt-api - - name: JWT_AUDIENCE - value: foxhunt-services - - name: QUESTDB_ILP_HOST - value: "questdb:9009" - - name: IBKR_HOST - value: "ib-gateway" - - name: IBKR_PORT - value: "4002" - - name: IBKR_CLIENT_ID - value: "1" - - name: IBKR_ACCOUNT_ID - valueFrom: - secretKeyRef: - name: ibkr-credentials - key: account-id - - name: GRPC_PORT - value: "50051" - - name: RUST_LOG - value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: tmp - mountPath: /tmp - readinessProbe: - exec: - command: - - grpc_health_probe - - -addr=localhost:50051 - initialDelaySeconds: 10 - periodSeconds: 10 - livenessProbe: - exec: - command: - - grpc_health_probe - - -addr=localhost:50051 - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - volumes: - - name: binaries - emptyDir: - sizeLimit: 200Mi - - name: tmp - emptyDir: - sizeLimit: 50Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: trading-service - namespace: foxhunt - labels: - app.kubernetes.io/name: trading-service - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: trading-service - ports: - - port: 50051 - targetPort: 50051 - name: grpc - - port: 9092 - targetPort: 9092 - name: metrics diff --git a/infra/k8s/services/web-dashboard.yaml b/infra/k8s/services/web-dashboard.yaml deleted file mode 100644 index 27e87cca8..000000000 --- a/infra/k8s/services/web-dashboard.yaml +++ /dev/null @@ -1,150 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: web-dashboard - namespace: foxhunt - labels: - app.kubernetes.io/name: web-dashboard - app.kubernetes.io/part-of: foxhunt -spec: - # Dev-mode: paused while working on model development. - # Restore with: kubectl -n foxhunt scale deployment web-dashboard --replicas=1 - replicas: 0 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - app.kubernetes.io/name: web-dashboard - template: - metadata: - labels: - app.kubernetes.io/name: web-dashboard - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - imagePullSecrets: - - name: gitlab-registry - initContainers: - - name: fetch-assets - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - rclone copy \ - ":s3:foxhunt-binaries/web-dashboard/" \ - "/usr/share/nginx/html/" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - echo "Fetched web-dashboard assets ($(find /usr/share/nginx/html -type f | wc -l) files)" - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: html - mountPath: /usr/share/nginx/html - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: nginx - image: nginx:alpine - ports: - - containerPort: 80 - name: http - volumeMounts: - - name: html - mountPath: /usr/share/nginx/html - readOnly: true - - name: nginx-conf - mountPath: /etc/nginx/conf.d - readOnly: true - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - livenessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 5 - periodSeconds: 30 - readinessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 2 - periodSeconds: 10 - volumes: - - name: html - emptyDir: {} - - name: nginx-conf - configMap: - name: web-dashboard-nginx ---- -apiVersion: v1 -kind: Service -metadata: - name: web-dashboard - namespace: foxhunt - labels: - app.kubernetes.io/name: web-dashboard - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: web-dashboard - ports: - - port: 80 - targetPort: 80 - name: http ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: web-dashboard-nginx - namespace: foxhunt -data: - default.conf: | - server { - listen 80; - root /usr/share/nginx/html; - index index.html; - - # SPA: route all non-file requests to index.html - location / { - try_files $uri $uri/ /index.html; - } - - # Cache static assets aggressively - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff2?)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # No cache for index.html (so deploys take effect immediately) - location = /index.html { - add_header Cache-Control "no-cache"; - } - } diff --git a/infra/k8s/stalwart/stalwart.yaml b/infra/k8s/stalwart/stalwart.yaml deleted file mode 100644 index c2e4e9b49..000000000 --- a/infra/k8s/stalwart/stalwart.yaml +++ /dev/null @@ -1,163 +0,0 @@ -# Internal-only mail server. No public MX, no LoadBalancer. -# SMTP: cluster-internal (Mattermost relay) -# IMAP: via Tailscale proxy (port 993) -# HTTPS admin: via Tailscale proxy (port 443 → mail.fxhnt.ai) -# Outbound: not configured (add SCW TEM relay later if needed) ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: stalwart-data - namespace: foxhunt - labels: - app.kubernetes.io/name: stalwart - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: [ReadWriteOnce] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: stalwart-config - namespace: foxhunt - labels: - app.kubernetes.io/name: stalwart - app.kubernetes.io/part-of: foxhunt -data: - config.toml: | - [server] - hostname = "mail.fxhnt.ai" - - [server.listener.smtp] - bind = "[::]:25" - protocol = "smtp" - - [server.listener.imaptls] - bind = "[::]:993" - protocol = "imap" - tls.implicit = true - - [server.listener.https] - bind = "[::]:443" - protocol = "http" - tls.implicit = true - - [storage] - data = "rocksdb" - blob = "rocksdb" - fts = "rocksdb" - lookup = "rocksdb" - directory = "internal" - - [store.rocksdb] - type = "rocksdb" - path = "/opt/stalwart/data" - - [directory.internal] - type = "internal" - store = "rocksdb" - - [authentication.fallback-admin] - user = "admin" - secret = "%{env:ADMIN_SECRET}%" - - [certificate.default] - cert = "%{file:/opt/stalwart/tls/tls.crt}%" - private-key = "%{file:/opt/stalwart/tls/tls.key}%" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: stalwart - namespace: foxhunt - labels: - app.kubernetes.io/name: stalwart - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: stalwart - strategy: - type: Recreate # RWO PVC - template: - metadata: - labels: - app.kubernetes.io/name: stalwart - app.kubernetes.io/part-of: foxhunt - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: stalwart - image: stalwartlabs/stalwart:latest - env: - - name: ADMIN_SECRET - valueFrom: - secretKeyRef: - name: stalwart-admin - key: password - ports: - - containerPort: 25 - name: smtp - - containerPort: 993 - name: imaps - - containerPort: 443 - name: https - volumeMounts: - - name: stalwart-data - mountPath: /opt/stalwart/data - - name: stalwart-config - mountPath: /opt/stalwart/etc/config.toml - subPath: config.toml - - name: stalwart-tls - mountPath: /opt/stalwart/tls - readOnly: true - readinessProbe: - tcpSocket: - port: 25 - initialDelaySeconds: 10 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - volumes: - - name: stalwart-data - persistentVolumeClaim: - claimName: stalwart-data - - name: stalwart-config - configMap: - name: stalwart-config - - name: stalwart-tls - secret: - secretName: stalwart-tls ---- -apiVersion: v1 -kind: Service -metadata: - name: stalwart - namespace: foxhunt - labels: - app.kubernetes.io/name: stalwart - app.kubernetes.io/part-of: foxhunt -spec: - selector: - app.kubernetes.io/name: stalwart - ports: - - port: 25 - targetPort: 25 - name: smtp - - port: 993 - targetPort: 993 - name: imaps - - port: 443 - targetPort: 443 - name: https diff --git a/infra/k8s/storage/dev-home-pvc.yaml b/infra/k8s/storage/dev-home-pvc.yaml deleted file mode 100644 index 35c8a6fac..000000000 --- a/infra/k8s/storage/dev-home-pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: dev-home-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: dev-home - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: devpod -spec: - storageClassName: scw-bssd - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi diff --git a/infra/k8s/storage/sccache-pvc.yaml b/infra/k8s/storage/sccache-pvc.yaml deleted file mode 100644 index b24f9ed7a..000000000 --- a/infra/k8s/storage/sccache-pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: sccache-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: sccache - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: ci -spec: - storageClassName: scw-bssd - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi diff --git a/infra/k8s/storage/training-data-pv.yaml b/infra/k8s/storage/training-data-pv.yaml deleted file mode 100644 index ef4bd1b42..000000000 --- a/infra/k8s/storage/training-data-pv.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: training-data-pv - labels: - app.kubernetes.io/name: training-data - app.kubernetes.io/part-of: foxhunt -spec: - capacity: - storage: 100Gi - accessModes: - - ReadOnlyMany - csi: - driver: csi.scaleway.com - volumeHandle: "fr-par-2/aca3c18b-89c0-4920-af67-d79ec9173430" - storageClassName: "" - persistentVolumeReclaimPolicy: Retain ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: training-data-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: training-data - app.kubernetes.io/part-of: foxhunt -spec: - storageClassName: "" - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Gi - volumeName: training-data-pv diff --git a/infra/k8s/tailscale/deployment.yaml b/infra/k8s/tailscale/deployment.yaml deleted file mode 100644 index 4dce2060c..000000000 --- a/infra/k8s/tailscale/deployment.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tailscale-subnet-router - namespace: tailscale - labels: - app.kubernetes.io/name: tailscale-subnet-router - app.kubernetes.io/part-of: foxhunt -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: tailscale-subnet-router - template: - metadata: - labels: - app.kubernetes.io/name: tailscale-subnet-router - spec: - nodeSelector: - k8s.scaleway.com/pool-name: infra - serviceAccountName: tailscale - containers: - - name: tailscale - image: ghcr.io/tailscale/tailscale:latest - env: - - name: TS_AUTHKEY - valueFrom: - secretKeyRef: - name: tailscale-auth - key: TS_AUTHKEY - - name: TS_KUBE_SECRET - value: tailscale-state - - name: TS_USERSPACE - value: "false" - - name: TS_ROUTES - value: "10.42.0.0/16" - - name: TS_HOSTNAME - value: foxhunt-kapsule - - name: TS_ACCEPT_DNS - value: "false" - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi diff --git a/infra/k8s/tailscale/namespace.yaml b/infra/k8s/tailscale/namespace.yaml deleted file mode 100644 index dc31e9da5..000000000 --- a/infra/k8s/tailscale/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: tailscale - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/managed-by: kubectl diff --git a/infra/k8s/tailscale/rbac.yaml b/infra/k8s/tailscale/rbac.yaml deleted file mode 100644 index f2811afd4..000000000 --- a/infra/k8s/tailscale/rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: tailscale - namespace: tailscale - labels: - app.kubernetes.io/name: tailscale - app.kubernetes.io/part-of: foxhunt ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: tailscale - namespace: tailscale - labels: - app.kubernetes.io/name: tailscale - app.kubernetes.io/part-of: foxhunt -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "get", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: tailscale - namespace: tailscale - labels: - app.kubernetes.io/name: tailscale - app.kubernetes.io/part-of: foxhunt -subjects: - - kind: ServiceAccount - name: tailscale - namespace: tailscale -roleRef: - kind: Role - name: tailscale - apiGroup: rbac.authorization.k8s.io diff --git a/infra/k8s/tailscale/secret.yaml.example b/infra/k8s/tailscale/secret.yaml.example deleted file mode 100644 index f80d99333..000000000 --- a/infra/k8s/tailscale/secret.yaml.example +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: tailscale-auth - namespace: tailscale - labels: - app.kubernetes.io/name: tailscale - app.kubernetes.io/part-of: foxhunt -type: Opaque -stringData: - TS_AUTHKEY: "tskey-auth-REPLACE_ME" diff --git a/infra/k8s/training/data-sync-job.yaml b/infra/k8s/training/data-sync-job.yaml deleted file mode 100644 index c569fb8bb..000000000 --- a/infra/k8s/training/data-sync-job.yaml +++ /dev/null @@ -1,113 +0,0 @@ -# Sync training data from MinIO → training-data PVC. -# Runs rclone sync (only copies new/changed files). -# -# Usage: -# kubectl apply -f infra/k8s/training/data-sync-job.yaml -# -# First run: full sync (~50GB, ~10 min). -# Subsequent runs: delta only (seconds). -# -# Data layout on PVC: -# /data/futures-baseline/ — OHLCV-1m bars (ES.FUT, NQ.FUT, etc.) -# /data/futures-baseline-mbp10/ — MBP-10 order book (ES.FUT) -apiVersion: batch/v1 -kind: Job -metadata: - generateName: data-sync- - namespace: foxhunt - labels: - app.kubernetes.io/name: data-sync - app.kubernetes.io/part-of: foxhunt - foxhunt/job-type: data-sync -spec: - backoffLimit: 2 - activeDeadlineSeconds: 7200 # 2 hours - ttlSecondsAfterFinished: 300 - template: - metadata: - labels: - app.kubernetes.io/name: data-sync - app.kubernetes.io/part-of: foxhunt - foxhunt/job-type: data-sync - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - imagePullSecrets: - - name: gitlab-registry - restartPolicy: Never - containers: - - name: sync - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - - echo "=== Training Data Sync: MinIO → PVC ===" - echo "Syncing from MinIO buckets to /data/..." - - # Sync OHLCV data - echo "" - echo "--- OHLCV (futures-baseline) ---" - rclone sync \ - ":s3:foxhunt-training-data/futures-baseline" \ - /data/futures-baseline \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --progress \ - --stats-one-line \ - -v - echo "OHLCV sync complete." - - # Sync MBP-10 data - echo "" - echo "--- MBP-10 (futures-baseline-mbp10) ---" - rclone sync \ - ":s3:foxhunt-training-data/futures-baseline-mbp10" \ - /data/futures-baseline-mbp10 \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --progress \ - --stats-one-line \ - -v - echo "MBP-10 sync complete." - - echo "" - echo "=== Sync Summary ===" - echo "OHLCV:" - du -sh /data/futures-baseline/ 2>/dev/null || echo " (empty)" - echo "MBP-10:" - du -sh /data/futures-baseline-mbp10/ 2>/dev/null || echo " (empty)" - echo "Total:" - du -sh /data/ - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: training-data - mountPath: /data - resources: - requests: - cpu: "1" - memory: 512Mi - limits: - cpu: "2" - memory: 1Gi - volumes: - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc diff --git a/infra/k8s/training/data-upload-job.yaml b/infra/k8s/training/data-upload-job.yaml deleted file mode 100644 index 68107f259..000000000 --- a/infra/k8s/training/data-upload-job.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Temporary pod for uploading training data to PVC. -# Usage: -# 1. kubectl apply -f infra/k8s/training/training-data-pvc.yaml -# 2. kubectl apply -f infra/k8s/training/data-upload-job.yaml -# 3. kubectl cp test_data/futures-baseline foxhunt/data-upload:/data/futures-baseline -# 4. kubectl cp test_data/futures-baseline-l2 foxhunt/data-upload:/data/futures-baseline-l2 -# 5. kubectl delete pod data-upload -n foxhunt -apiVersion: v1 -kind: Pod -metadata: - name: data-upload - namespace: foxhunt -spec: - containers: - - name: upload - image: busybox:1.36 - command: ["sleep", "3600"] - volumeMounts: - - name: training-data - mountPath: /data - volumes: - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - restartPolicy: Never diff --git a/infra/k8s/training/download-mbp10-job.yaml b/infra/k8s/training/download-mbp10-job.yaml deleted file mode 100644 index 667783a62..000000000 --- a/infra/k8s/training/download-mbp10-job.yaml +++ /dev/null @@ -1,164 +0,0 @@ -# Download MBP-10 order book data for ES.FUT from Databento → MinIO. -# -# Prerequisites: -# 1. Compile + upload the binary: -# cargo build -p ml --example download_baseline --release -# rclone copyto target/release/examples/download_baseline \ -# :s3:foxhunt-binaries/tools/download_baseline \ -# --s3-provider=Minio --s3-endpoint= \ -# --s3-access-key-id= --s3-secret-access-key= \ -# --s3-no-check-bucket -# -# 2. Create Databento API key secret: -# kubectl -n foxhunt create secret generic databento-credentials \ -# --from-literal=api-key=$DATABENTO_API_KEY -# -# 3. Apply: -# kubectl apply -f infra/k8s/training/download-mbp10-job.yaml -# -# The job downloads ES.FUT MBP-10 data (730 days, quarterly chunks) and -# uploads each file to s3:foxhunt-training-data/futures-baseline-mbp10/. -apiVersion: batch/v1 -kind: Job -metadata: - name: download-mbp10 - namespace: foxhunt - labels: - app.kubernetes.io/name: download-mbp10 - app.kubernetes.io/part-of: foxhunt - foxhunt/job-type: data-download -spec: - backoffLimit: 2 - activeDeadlineSeconds: 14400 # 4 hours - ttlSecondsAfterFinished: 3600 - template: - metadata: - labels: - app.kubernetes.io/name: download-mbp10 - app.kubernetes.io/part-of: foxhunt - foxhunt/job-type: data-download - spec: - nodeSelector: - k8s.scaleway.com/pool-name: platform - imagePullSecrets: - - name: gitlab-registry - restartPolicy: Never - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - rclone copyto \ - :s3:foxhunt-binaries/tools/download_baseline \ - /binaries/download_baseline \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/download_baseline - echo "Fetched download_baseline ($(stat -c%s /binaries/download_baseline) bytes)" - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: downloader - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - - # Write universe config inline (avoids ConfigMap for a one-off job) - mkdir -p /tmp/config - cat > /tmp/config/universe-es-mbp10.toml <<'TOML' - [universe] - name = "es-mbp10" - description = "ES.FUT MBP-10 order book data for OFI features" - date_range_start = "2024-03-01" - date_range_end = "2026-02-22" - bar_size = "tick" - databento_dataset = "GLBX.MDP3" - databento_schema = "mbp-10" - - [[symbols]] - symbol = "ES.FUT" - exchange = "GLBX.MDP3" - asset_class = "Future" - TOML - - echo "=== ES.FUT MBP-10 Download Job ===" - echo "Downloading from Databento and uploading to MinIO..." - - /binaries/download_baseline \ - --universe-config /tmp/config/universe-es-mbp10.toml \ - --output-dir /tmp/mbp10-download \ - --rclone-dest "s3:foxhunt-training-data/futures-baseline-mbp10" \ - --parallel 3 \ - --yes - - echo "=== Download complete ===" - echo "Files in MinIO: foxhunt-training-data/futures-baseline-mbp10/ES.FUT/" - env: - - name: DATABENTO_API_KEY - valueFrom: - secretKeyRef: - name: databento-credentials - key: api-key - # rclone env-var config (used by download_baseline --rclone-dest) - - name: RCLONE_S3_PROVIDER - value: Minio - - name: RCLONE_S3_ENDPOINT - value: http://minio.foxhunt.svc.cluster.local:9000 - - name: RCLONE_S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: RCLONE_S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - - name: RCLONE_S3_NO_CHECK_BUCKET - value: "true" - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: scratch - mountPath: /tmp - resources: - requests: - cpu: "1" - memory: 1Gi - limits: - cpu: "2" - memory: 4Gi - volumes: - - name: binaries - emptyDir: - sizeLimit: 100Mi - - name: scratch - emptyDir: - sizeLimit: 100Gi # MBP-10: ~5GB/quarter × 9 quarters + headroom diff --git a/infra/k8s/training/download-trades-job.yaml b/infra/k8s/training/download-trades-job.yaml deleted file mode 100644 index 25c2abc98..000000000 --- a/infra/k8s/training/download-trades-job.yaml +++ /dev/null @@ -1,177 +0,0 @@ -# Download ES.FUT trade data (Schema::Trades) from Databento → PVC. -# -# Writes directly to the training-data PVC, then backs up to MinIO. -# Trade data is used by VPIN and Kyle's Lambda OFI features. -# -# Prerequisites: -# 1. Compile + upload the binary (same as download-mbp10-job): -# cargo build -p ml --example download_baseline --release -# rclone copyto target/release/examples/download_baseline \ -# :s3:foxhunt-binaries/tools/download_baseline \ -# --s3-provider=Minio --s3-endpoint= \ -# --s3-access-key-id= --s3-secret-access-key= \ -# --s3-no-check-bucket -# -# 2. Create Databento API key secret (if not already done): -# kubectl -n foxhunt create secret generic databento-credentials \ -# --from-literal=api-key=$DATABENTO_API_KEY -# -# 3. Apply: -# kubectl apply -f infra/k8s/training/download-trades-job.yaml -# -# NOTE: training-data-pvc is RWO — ensure no training job is running. -apiVersion: batch/v1 -kind: Job -metadata: - name: download-trades - namespace: foxhunt - labels: - app.kubernetes.io/name: download-trades - app.kubernetes.io/part-of: foxhunt - foxhunt/job-type: data-download -spec: - backoffLimit: 2 - activeDeadlineSeconds: 14400 # 4 hours - ttlSecondsAfterFinished: 3600 - template: - metadata: - labels: - app.kubernetes.io/name: download-trades - app.kubernetes.io/part-of: foxhunt - foxhunt/job-type: data-download - spec: - nodeSelector: - k8s.scaleway.com/pool-name: ci-compile-cpu - tolerations: - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - imagePullSecrets: - - name: gitlab-registry - restartPolicy: Never - initContainers: - - name: fetch-binary - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - rclone copyto \ - :s3:foxhunt-binaries/tools/download_baseline \ - /binaries/download_baseline \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/download_baseline - echo "Fetched download_baseline ($(stat -c%s /binaries/download_baseline) bytes)" - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: binaries - mountPath: /binaries - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 128Mi - containers: - - name: downloader - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - - # Write universe config for trades schema - mkdir -p /tmp/config - cat > /tmp/config/universe-es-trades.toml <<'TOML' - [universe] - name = "es-trades" - date_range_start = "2024-03-01" - date_range_end = "2026-02-22" - databento_dataset = "GLBX.MDP3" - databento_schema = "trades" - - [[symbols]] - symbol = "ES.FUT" - exchange = "GLBX.MDP3" - asset_class = "Future" - TOML - - echo "=== ES.FUT Trades Download Job ===" - echo "Downloading from Databento directly to PVC..." - - /binaries/download_baseline \ - --universe-config /tmp/config/universe-es-trades.toml \ - --output-dir /data/futures-baseline-trades \ - --parallel 6 \ - --yes - - echo "" - echo "=== Download complete ===" - du -sh /data/futures-baseline-trades/ - ls -lhS /data/futures-baseline-trades/ES.FUT/ - - # Backup to MinIO - RCLONE_FLAGS="--s3-provider=Minio --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=${MINIO_ACCESS_KEY} --s3-secret-access-key=${MINIO_SECRET_KEY} --s3-no-check-bucket" - echo "" - echo "=== Backing up to MinIO ===" - rclone sync \ - /data/futures-baseline-trades \ - ":s3:foxhunt-training-data/futures-baseline-trades" \ - $RCLONE_FLAGS --stats-one-line -v - echo "MinIO backup complete." - env: - - name: DATABENTO_API_KEY - valueFrom: - secretKeyRef: - name: databento-credentials - key: api-key - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: binaries - mountPath: /binaries - readOnly: true - - name: training-data - mountPath: /data - - name: scratch - mountPath: /tmp - resources: - requests: - cpu: "2" - memory: 4Gi - limits: - cpu: "4" - memory: 16Gi - volumes: - - name: binaries - emptyDir: - sizeLimit: 100Mi - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - - name: scratch - emptyDir: - sizeLimit: 1Gi diff --git a/infra/k8s/training/test-data-pvc.yaml b/infra/k8s/training/test-data-pvc.yaml deleted file mode 100644 index 5deb184a0..000000000 --- a/infra/k8s/training/test-data-pvc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# infra/k8s/training/test-data-pvc.yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: test-data-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: test-data - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: [ReadWriteOnce] - storageClassName: scw-bssd - resources: - requests: - storage: 50Gi diff --git a/infra/k8s/training/training-data-pvc.yaml b/infra/k8s/training/training-data-pvc.yaml deleted file mode 100644 index a489e7218..000000000 --- a/infra/k8s/training/training-data-pvc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: training-data-pvc - namespace: foxhunt - labels: - app.kubernetes.io/name: training-data - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 200Gi - storageClassName: scw-bssd diff --git a/infra/live/production/block-storage/terragrunt.hcl b/infra/live/production/block-storage/terragrunt.hcl deleted file mode 100644 index 84ede5bdf..000000000 --- a/infra/live/production/block-storage/terragrunt.hcl +++ /dev/null @@ -1,11 +0,0 @@ -include "root" { - path = find_in_parent_folders("root.hcl") -} - -terraform { - source = "../../../modules/block-storage" -} - -inputs = { - size_in_gb = 100 -} diff --git a/infra/live/production/dns/terragrunt.hcl b/infra/live/production/dns/terragrunt.hcl deleted file mode 100644 index 26fb9efbb..000000000 --- a/infra/live/production/dns/terragrunt.hcl +++ /dev/null @@ -1,14 +0,0 @@ -include "root" { - path = find_in_parent_folders("root.hcl") -} - -terraform { - source = "../../../modules/dns" -} - -inputs = { - dns_zone = "fxhnt.ai" - # Tailscale IP of the foxhunt-gitlab proxy pod (live node foxhunt-gitlab; the old 100.90.76.85 - # was a stale prior IP — its duplicate A records across *.fxhnt.ai were removed 2026-06-21). - git_ip = "100.95.225.27" -} diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl deleted file mode 100644 index 4b75ac7cb..000000000 --- a/infra/live/production/kapsule/terragrunt.hcl +++ /dev/null @@ -1,52 +0,0 @@ -include "root" { - path = find_in_parent_folders("root.hcl") -} - -terraform { - source = "../../../modules/kapsule" -} - -inputs = { - cluster_name = "foxhunt" - k8s_version = "1.34" - - # Platform pool — runs everything: app services, databases, GitLab, monitoring - platform_type = "DEV1-L" - platform_max_size = 3 - - # CI build pool — right-sized for the Python fxhnt cockpit/deploy builds (kaniko + pip, ~2-6 vCPU). - # Was POP2-HC-32C-64G (32 vCPU/64GB) for the now-dormant Rust cargo compile + precompute_features; - # downsized 2026-06-21 to POP2-4C-16G (in stock in fr-par-2; the 32C type hit a zone stock-out that - # jammed the autoscaler — "Resource POP2-HC-32C-64G is out of stock"). Far cheaper, scale-to-zero. - enable_ci_compile_cpu_pool = true - ci_compile_cpu_type = "POP2-4C-16G" - ci_compile_cpu_max_size = 4 - - # High-memory CPU pool (POP2-HM-32C-256G — 32 vCPU, 256GB RAM). - # Reserved for the rare full-rebuild fxcache path that can't fit on - # the 64GB ci-compile-cpu pool (9-quarter precompute_features peaks - # ~50-60GB during the post-OFI alpha_trades conversion). - # min_size=0 + size=0 initial so this pool costs nothing when idle. - enable_ci_compile_cpu_hm_pool = false # decommissioned 2026-06-21 — Rust precompute_features retired - ci_compile_cpu_hm_type = "POP2-HM-32C-256G" - ci_compile_cpu_hm_max_size = 1 - - # L40S training pool (48GB VRAM, CUDA CC 89) - enable_ci_training_l40s_pool = false # decommissioned 2026-06-21 — Rust ML training retired - ci_training_l40s_type = "L40S-1-48G" - ci_training_l40s_max_size = 1 - - # H100 training pool (80GB VRAM, CUDA CC 90) - enable_ci_training_h100_pool = false # decommissioned 2026-06-21 — Rust RL training retired - ci_training_h100_type = "H100-1-80G" - ci_training_h100_max_size = 1 - - # H100x2 and H100-SXM pools removed 2026-05-16: - # - SXM: Scaleway account quota is 0/0 (cp_servers_type_H100_SXM_2_80G). - # The pool was stuck trying to provision and jammed the autoscaler. - # - H100x2: more expensive than current workloads justify; the single-GPU - # H100 pool covers all training paths. - - # NAT gateway handles egress — no public IPs on nodes - public_ip_disabled = true -} diff --git a/infra/live/production/public-gateway/terragrunt.hcl b/infra/live/production/public-gateway/terragrunt.hcl deleted file mode 100644 index f6b8629e6..000000000 --- a/infra/live/production/public-gateway/terragrunt.hcl +++ /dev/null @@ -1,17 +0,0 @@ -include "root" { - path = find_in_parent_folders("root.hcl") -} - -terraform { - source = "../../../modules/public-gateway" -} - -dependency "kapsule" { - config_path = "../kapsule" -} - -inputs = { - private_network_id = dependency.kapsule.outputs.private_network_id - gateway_type = "VPC-GW-S" - bastion_enabled = true # matches the live gateway (carried over from the old kapsule gw config) -} diff --git a/infra/live/production/root.hcl b/infra/live/production/root.hcl deleted file mode 100644 index e78d6ccb2..000000000 --- a/infra/live/production/root.hcl +++ /dev/null @@ -1,70 +0,0 @@ -# Root Terragrunt config for foxhunt production -# All child modules inherit provider, backend, and common inputs from here. - -locals { - region = "fr-par" - zone = "fr-par-2" - project_id = get_env("SCW_DEFAULT_PROJECT_ID") -} - -# Remote state in Scaleway Object Storage (bucket foxhunt-tfstate, fr-par) — migrated off GitLab 2026-06-21. -# The s3 backend reads AWS_* env vars (set them to the Scaleway access/secret keys); SCW_* drives the provider. -remote_state { - backend = "s3" - generate = { - path = "backend.tf" - if_exists = "overwrite" - } - config = { - bucket = "foxhunt-tfstate" - key = "${path_relative_to_include()}/terraform.tfstate" - region = "fr-par" - - endpoints = { - s3 = "https://s3.fr-par.scw.cloud" - } - - # Scaleway S3-compat: skip AWS-specific preflight calls - skip_credentials_validation = true - skip_region_validation = true - skip_requesting_account_id = true - skip_metadata_api_check = true - - # OpenTofu-native lock (no DynamoDB) - use_lockfile = true - } -} - -# Generate provider config for all child modules -generate "provider" { - path = "provider.tf" - if_exists = "overwrite" - contents = <<-EOF - terraform { - required_version = ">= 1.8.0" - required_providers { - scaleway = { - source = "scaleway/scaleway" - version = "~> 2.46" - } - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - } - } - - provider "scaleway" { - region = "${local.region}" - zone = "${local.zone}" - project_id = "${local.project_id}" - } - EOF -} - -# Common inputs available to all child modules -inputs = { - region = local.region - zone = local.zone - project_id = local.project_id -} diff --git a/infra/modules/block-storage/main.tf b/infra/modules/block-storage/main.tf deleted file mode 100644 index b0307ff28..000000000 --- a/infra/modules/block-storage/main.tf +++ /dev/null @@ -1,6 +0,0 @@ -resource "scaleway_block_volume" "training_data" { - name = var.volume_name - iops = 5000 - size_in_gb = var.size_in_gb - zone = var.zone -} diff --git a/infra/modules/block-storage/outputs.tf b/infra/modules/block-storage/outputs.tf deleted file mode 100644 index 3a4df0c95..000000000 --- a/infra/modules/block-storage/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "volume_id" { - description = "ID of the block storage volume" - value = scaleway_block_volume.training_data.id -} - -output "volume_name" { - description = "Name of the block storage volume" - value = scaleway_block_volume.training_data.name -} diff --git a/infra/modules/block-storage/variables.tf b/infra/modules/block-storage/variables.tf deleted file mode 100644 index 0d5395e64..000000000 --- a/infra/modules/block-storage/variables.tf +++ /dev/null @@ -1,16 +0,0 @@ -variable "zone" { - description = "Scaleway zone for the block storage volume" - type = string -} - -variable "volume_name" { - description = "Name of the block storage volume" - type = string - default = "foxhunt-training-data" -} - -variable "size_in_gb" { - description = "Size of the volume in GB" - type = number - default = 100 -} diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf deleted file mode 100644 index 421b644d4..000000000 --- a/infra/modules/dns/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -# DNS records for fxhnt.ai zone (Scaleway DNS) - -resource "scaleway_domain_record" "git" { - dns_zone = var.dns_zone - name = "git" - type = "A" - data = var.git_ip - ttl = 300 -} - -resource "scaleway_domain_record" "grafana" { - dns_zone = var.dns_zone - name = "grafana" - type = "A" - data = var.git_ip - ttl = 300 -} - -resource "scaleway_domain_record" "dashboard" { - dns_zone = var.dns_zone - name = "dashboard" - type = "A" - data = var.git_ip - ttl = 300 -} - -resource "scaleway_domain_record" "api" { - dns_zone = var.dns_zone - name = "api" - type = "A" - data = var.git_ip - ttl = 300 -} - -resource "scaleway_domain_record" "minio" { - dns_zone = var.dns_zone - name = "minio" - type = "A" - data = var.git_ip - ttl = 300 -} - -resource "scaleway_domain_record" "chat" { - dns_zone = var.dns_zone - name = "chat" - type = "A" - data = var.git_ip - ttl = 300 -} - -resource "scaleway_domain_record" "mail" { - dns_zone = var.dns_zone - name = "mail" - type = "A" - data = var.git_ip # Tailscale proxy — IMAP + admin only, no public MX - ttl = 300 -} - diff --git a/infra/modules/dns/outputs.tf b/infra/modules/dns/outputs.tf deleted file mode 100644 index 0cf18939d..000000000 --- a/infra/modules/dns/outputs.tf +++ /dev/null @@ -1,24 +0,0 @@ -output "git_fqdn" { - description = "FQDN for the GitLab instance" - value = "${scaleway_domain_record.git.name}.${scaleway_domain_record.git.dns_zone}" -} - -output "grafana_fqdn" { - description = "FQDN for the Grafana instance" - value = "${scaleway_domain_record.grafana.name}.${scaleway_domain_record.grafana.dns_zone}" -} - -output "api_fqdn" { - description = "FQDN for the API Gateway (gRPC)" - value = "${scaleway_domain_record.api.name}.${scaleway_domain_record.api.dns_zone}" -} - -output "chat_fqdn" { - description = "FQDN for the Mattermost chat" - value = "${scaleway_domain_record.chat.name}.${scaleway_domain_record.chat.dns_zone}" -} - -output "mail_fqdn" { - description = "FQDN for the Stalwart mail server" - value = "${scaleway_domain_record.mail.name}.${scaleway_domain_record.mail.dns_zone}" -} diff --git a/infra/modules/dns/variables.tf b/infra/modules/dns/variables.tf deleted file mode 100644 index fcb240a95..000000000 --- a/infra/modules/dns/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "dns_zone" { - description = "DNS zone to manage records in" - type = string - default = "fxhnt.ai" -} - -variable "git_ip" { - description = "IP address for git.fxhnt.ai (Tailscale subnet router or ClusterIP)" - type = string -} diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf deleted file mode 100644 index 436f45e25..000000000 --- a/infra/modules/kapsule/main.tf +++ /dev/null @@ -1,138 +0,0 @@ -resource "scaleway_vpc_private_network" "foxhunt" { - name = "${var.cluster_name}-pn" - region = var.region -} - -# NAT gateway (foxhunt-gw) is owned by the dedicated `public-gateway` module (IPAM mode + -# dedicated IP), which depends on this module's private_network_id output. The legacy -# gateway/dhcp/gateway_network resources that used to live here were a pre-refactor duplicate -# (never in this module's state) and were removed 2026-06-21 to stop `terragrunt apply` here -# from trying to create a second gateway. See reference_ci_pool_and_tfstate / incident_dns_deadlock. - -resource "scaleway_k8s_cluster" "foxhunt" { - name = var.cluster_name - version = var.k8s_version - cni = "cilium" - region = var.region - delete_additional_resources = true - private_network_id = scaleway_vpc_private_network.foxhunt.id - - auto_upgrade { - enable = true - maintenance_window_start_hour = 4 - maintenance_window_day = "sunday" - } - - autoscaler_config { - disable_scale_down = false - scale_down_delay_after_add = "10m" - scale_down_unneeded_time = "10m" - estimator = "binpacking" - ignore_daemonsets_utilization = true - } -} - -# Platform pool — runs everything: app services, databases, GitLab, monitoring -resource "scaleway_k8s_pool" "platform" { - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "platform" - node_type = var.platform_type - size = 1 - min_size = 1 - max_size = var.platform_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} - -# L40S training pool (48GB VRAM, CUDA CC 89 — hyperopt + supervised training) -resource "scaleway_k8s_pool" "ci_training_l40s" { - count = var.enable_ci_training_l40s_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-training-l40s" - node_type = var.ci_training_l40s_type - size = 1 - min_size = 0 - max_size = var.ci_training_l40s_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} - -# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt) -resource "scaleway_k8s_pool" "ci_training_h100" { - count = var.enable_ci_training_h100_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-training-h100" - node_type = var.ci_training_h100_type - size = 1 - min_size = 0 # Scale to zero when idle. Cold start: ~15 min NVIDIA driver install. - max_size = var.ci_training_h100_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} - -# NOTE: ci-training-h100x2 and ci-training-h100-sxm pools removed -# 2026-05-16 — Scaleway account doesn't have quota for the SXM variant -# (cp_servers_type_H100_SXM_2_80G 0/0), and the H100x2 variant is -# more expensive than our actual usage justifies. Keeping the regular -# single-GPU H100 pool (ci_training_h100) + the L40S pool covers -# every current training workload. - -# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build) -resource "scaleway_k8s_pool" "ci_compile_cpu" { - count = var.enable_ci_compile_cpu_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-compile-cpu" - node_type = var.ci_compile_cpu_type - size = 1 - min_size = 0 - max_size = var.ci_compile_cpu_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} - -# High-memory CPU compile pool (POP2-HM — same CPU count as ci-compile-cpu -# but 4× the RAM; reserved for rare full-rebuild precompute_features runs -# that can't fit on the standard 64GB nodes). min_size=0 + size=0 initial -# so the pool costs nothing when idle — autoscaler provisions one node -# only when a pod targets `nodeSelector: ci-compile-cpu-hm`. -resource "scaleway_k8s_pool" "ci_compile_cpu_hm" { - count = var.enable_ci_compile_cpu_hm_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-compile-cpu-hm" - node_type = var.ci_compile_cpu_hm_type - size = 0 - min_size = 0 - max_size = var.ci_compile_cpu_hm_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} diff --git a/infra/modules/kapsule/outputs.tf b/infra/modules/kapsule/outputs.tf deleted file mode 100644 index f4e760b14..000000000 --- a/infra/modules/kapsule/outputs.tf +++ /dev/null @@ -1,40 +0,0 @@ -output "cluster_id" { - description = "ID of the Kapsule cluster" - value = scaleway_k8s_cluster.foxhunt.id -} - -output "kubeconfig" { - description = "Kubeconfig file content for the cluster" - value = scaleway_k8s_cluster.foxhunt.kubeconfig[0].config_file - sensitive = true -} - -output "apiserver_url" { - description = "URL of the Kubernetes API server" - value = scaleway_k8s_cluster.foxhunt.apiserver_url -} - -output "platform_pool_id" { - description = "ID of the platform node pool" - value = scaleway_k8s_pool.platform.id -} - -output "ci_compile_cpu_pool_id" { - description = "ID of the CPU compile node pool" - value = var.enable_ci_compile_cpu_pool ? scaleway_k8s_pool.ci_compile_cpu[0].id : "" -} - -output "ci_training_l40s_pool_id" { - description = "ID of the L40S training node pool" - value = var.enable_ci_training_l40s_pool ? scaleway_k8s_pool.ci_training_l40s[0].id : "" -} - -output "ci_training_h100_pool_id" { - description = "ID of the H100 training node pool" - value = var.enable_ci_training_h100_pool ? scaleway_k8s_pool.ci_training_h100[0].id : "" -} - -output "private_network_id" { - description = "ID of the VPC private network the cluster is attached to" - value = scaleway_vpc_private_network.foxhunt.id -} diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf deleted file mode 100644 index 4f097476a..000000000 --- a/infra/modules/kapsule/variables.tf +++ /dev/null @@ -1,107 +0,0 @@ -variable "region" { - description = "Scaleway region for the Kapsule cluster" - type = string -} - -variable "cluster_name" { - description = "Name of the Kubernetes cluster" - type = string -} - -variable "k8s_version" { - description = "Kubernetes version for the Kapsule cluster" - type = string -} - -variable "platform_type" { - description = "Instance type for the platform node pool" - type = string -} - -variable "platform_max_size" { - description = "Maximum number of nodes in the platform pool" - type = number -} - -variable "enable_ci_training_l40s_pool" { - description = "Create L40S GPU node pool for training + hyperopt" - type = bool - default = false -} - -variable "ci_training_l40s_type" { - description = "Instance type for the L40S training pool" - type = string - default = "L40S-1-48G" -} - -variable "ci_training_l40s_max_size" { - description = "Maximum number of nodes in the L40S training pool" - type = number - default = 1 -} - -variable "enable_ci_training_h100_pool" { - description = "Create H100 GPU node pool for training + hyperopt" - type = bool - default = false -} - -variable "ci_training_h100_type" { - description = "Instance type for the H100 training pool" - type = string - default = "H100-1-80G" -} - -variable "ci_training_h100_max_size" { - description = "Maximum number of nodes in the H100 training pool" - type = number - default = 1 -} - -variable "enable_ci_compile_cpu_pool" { - description = "Create high-CPU node pool for CI compilation" - type = bool - default = false -} - -variable "ci_compile_cpu_type" { - description = "Instance type for the CPU compile pool" - type = string -} - -variable "ci_compile_cpu_max_size" { - description = "Maximum number of nodes in the CPU compile pool" - type = number - default = 1 -} - -# ─── High-memory CPU compile pool (rare full-rebuild path) ────────────── -# Separate pool with min_size=0 autoscaling — pays nothing when idle, only -# provisions a large-RAM node when `ensure-fxcache` requests this pool -# (9-quarter precompute_features needs ~50-60GB peak which won't fit on -# the standard 64GB ci-compile-cpu pool with kernel + page-cache overhead). - -variable "enable_ci_compile_cpu_hm_pool" { - description = "Create a high-memory CPU pool for rare full-rebuild fxcache runs" - type = bool - default = false -} - -variable "ci_compile_cpu_hm_type" { - description = "Instance type for the high-memory CPU pool (e.g. POP2-HM-32C-256G)" - type = string - default = "POP2-HM-32C-256G" -} - -variable "ci_compile_cpu_hm_max_size" { - description = "Maximum number of nodes in the high-memory CPU pool" - type = number - default = 1 -} - -variable "public_ip_disabled" { - description = "Disable public IPs on all node pools (requires Public Gateway for egress)" - type = bool - default = false -} diff --git a/infra/modules/public-gateway/main.tf b/infra/modules/public-gateway/main.tf deleted file mode 100644 index be2793967..000000000 --- a/infra/modules/public-gateway/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "scaleway_vpc_public_gateway_ip" "foxhunt" { - zone = var.zone -} - -resource "scaleway_vpc_public_gateway" "foxhunt" { - name = "foxhunt-gw" - type = var.gateway_type - zone = var.zone - bastion_enabled = var.bastion_enabled - ip_id = scaleway_vpc_public_gateway_ip.foxhunt.id -} - -resource "scaleway_vpc_gateway_network" "foxhunt" { - gateway_id = scaleway_vpc_public_gateway.foxhunt.id - private_network_id = var.private_network_id - enable_masquerade = true - - ipam_config { - push_default_route = true - } -} diff --git a/infra/modules/public-gateway/outputs.tf b/infra/modules/public-gateway/outputs.tf deleted file mode 100644 index 72064357e..000000000 --- a/infra/modules/public-gateway/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "gateway_id" { - description = "ID of the Public Gateway" - value = scaleway_vpc_public_gateway.foxhunt.id -} - -output "gateway_ip" { - description = "Public IP address of the gateway" - value = scaleway_vpc_public_gateway_ip.foxhunt.address -} diff --git a/infra/modules/public-gateway/variables.tf b/infra/modules/public-gateway/variables.tf deleted file mode 100644 index 07f2fc5c9..000000000 --- a/infra/modules/public-gateway/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -variable "region" { - description = "Scaleway region" - type = string -} - -variable "zone" { - description = "Scaleway zone for the gateway" - type = string -} - -variable "project_id" { - description = "Scaleway project ID" - type = string -} - -variable "private_network_id" { - description = "ID of the existing VPC private network to attach the gateway to" - type = string -} - -variable "gateway_type" { - description = "Public Gateway instance type (VPC-GW-S, VPC-GW-M)" - type = string - default = "VPC-GW-S" -} - -variable "bastion_enabled" { - description = "Enable SSH bastion on the gateway" - type = bool - default = false -} diff --git a/infra/scripts/generate-minio-tls.sh b/infra/scripts/generate-minio-tls.sh deleted file mode 100755 index b7f2d98fa..000000000 --- a/infra/scripts/generate-minio-tls.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# --------------------------------------------------------------------------- -# generate-minio-tls.sh — One-time: generate CA + server cert for MinIO TLS -# --------------------------------------------------------------------------- -# Creates: -# ConfigMap/minio-ca-cert — CA public cert (mounted everywhere) -# Secret/minio-tls — server cert+key (mounted only by MinIO) -# -# SANs cover all K8s DNS forms for the minio Service in foxhunt namespace. -# Run from any machine with openssl + kubectl access to the cluster. -# --------------------------------------------------------------------------- - -NAMESPACE="${NAMESPACE:-foxhunt}" -CA_DAYS=3650 # 10 years -SERVER_DAYS=730 # 2 years -WORK_DIR="$(mktemp -d)" -trap 'rm -rf "$WORK_DIR"' EXIT - -echo "=== Generating MinIO TLS certs (workdir: $WORK_DIR) ===" - -# --- CA (self-signed, 10 years) ------------------------------------------- -openssl genrsa -out "$WORK_DIR/ca.key" 4096 -openssl req -new -x509 \ - -key "$WORK_DIR/ca.key" \ - -out "$WORK_DIR/ca.crt" \ - -days "$CA_DAYS" \ - -subj "/CN=foxhunt-minio-ca/O=foxhunt" - -# --- Server cert (signed by CA, 2 years) ---------------------------------- -cat > "$WORK_DIR/server.cnf" <