gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(infra): relocate platform IaC to fxhnt (Phase 2D)

Browse Source 
Platform terraform (live+modules) + k8s manifests moved to fxhnt (now the IaC home).
Rust-only remnants kept: docker build images, gpu-taint.yaml, Rust scripts. foxhunt
being archived; history retained. Live platform IaC now in the fxhnt repo.
This commit is contained in:
jgrusewski
2026-06-21 18:21:34 +02:00
parent ffc83da643
commit bf9a6d20ee
144 changed files with 0 additions and 49382 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
infra/k8s/argo/archive-rbac.yaml
View File

@@ -1,34 +0,0 @@
# RBAC for Argo workflow archive — grants controller and server access
# to the PostgreSQL archive secret and MinIO credentials.
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: argo-archive-secrets
namespace: foxhunt
labels:
app.kubernetes.io/part-of: argo-workflows
rules:
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["argo-postgres-config", "minio-credentials"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: argo-archive-secrets
namespace: foxhunt
labels:
app.kubernetes.io/part-of: argo-workflows
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: argo-archive-secrets
subjects:
- kind: ServiceAccount
name: argo-workflows-workflow-controller
namespace: foxhunt
- kind: ServiceAccount
name: argo-workflows-server
namespace: foxhunt

518
infra/k8s/argo/argo-workflow-netpol.yaml
View File

@@ -1,518 +0,0 @@
# Base egress for ALL Argo workflow pods: DNS, K8s API, MinIO (logging), Mattermost (notifications).
# Applies to every pod with app.kubernetes.io/part-of: foxhunt.
# Component-specific policies below add extra rules (GitLab SSH, external HTTPS, etc.).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: argo-base-egress
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/part-of: foxhunt
policyTypes:
- Egress
egress:
# DNS
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Kubernetes API — Argo executor reports task results
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
# MinIO — log archival (Argo wait sidecar)
- ports:
- port: 9000
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
# Mattermost — exit handler webhook notifications
- ports:
- port: 8065
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: argo-training-workflow
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/component: training-workflow
policyTypes:
- Egress
egress:
# DNS — required for hostname resolution (minio, gitlab, etc.)
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Kubernetes API — Argo executor reports task results
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
# HTTPS (external registries, etc.)
- ports:
- port: 443
protocol: TCP
# GitLab SSH — git clone
- ports:
- port: 2222
protocol: TCP
to:
- ipBlock:
cidr: 100.90.76.85/32
- podSelector:
matchLabels:
app: gitlab-shell
# GitLab API — binary fetch from packages
- ports:
- port: 8181
protocol: TCP
to:
- podSelector:
matchLabels:
app: webservice
# MinIO — log archival + result upload
- ports:
- port: 9000
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
# Pushgateway — Prometheus metrics
- ports:
- port: 9091
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: pushgateway
# Tempo — OTLP traces
- ports:
- port: 4317
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: tempo
# Mattermost — exit handler webhook notifications
- ports:
- port: 8065
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
# Container registry — image pulls
- ports:
- port: 5000
protocol: TCP
to:
- podSelector:
matchLabels:
app: registry
---
# Compile workflow pods: need GitLab SSH, MinIO, k8s API, external HTTPS (cargo deps)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: argo-compile-workflow
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/component: compile
policyTypes:
- Egress
egress:
# DNS — resolve external hostnames (crates.io, github.com)
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Kubernetes API — Argo executor
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
# External HTTPS — cargo fetches crates.io, github.com (candle git dep)
- ports:
- port: 443
protocol: TCP
# GitLab SSH — git clone (Tailscale IP + cluster service)
- ports:
- port: 2222
protocol: TCP
to:
- ipBlock:
cidr: 100.90.76.85/32
- podSelector:
matchLabels:
app: gitlab-shell
# MinIO — binary upload
- ports:
- port: 9000
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
# Mattermost — exit handler webhook notifications
- ports:
- port: 8065
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
---
# Compile-and-deploy workflow pods: need GitLab HTTP (tagging/releases), MinIO,
# k8s API (deploy rollouts + Argo executor), DNS, GitLab SSH, external HTTPS.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: argo-compile-and-deploy-workflow
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/component: compile-and-deploy
policyTypes:
- Egress
egress:
# DNS
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Kubernetes API — Argo executor + kubectl rollout
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
# External HTTPS — cargo fetches, kubectl download
- ports:
- port: 443
protocol: TCP
# GitLab SSH — git clone
- ports:
- port: 2222
protocol: TCP
to:
- ipBlock:
cidr: 100.90.76.85/32
- podSelector:
matchLabels:
app: gitlab-shell
# GitLab webservice — CalVer tagging, releases, package uploads
- ports:
- port: 8181
protocol: TCP
to:
- podSelector:
matchLabels:
app: webservice
# MinIO — binary upload + log archival
- ports:
- port: 9000
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
# Mattermost — exit handler webhook notifications
- ports:
- port: 8065
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
---
# Compile-and-train workflow pods: compile step needs GitLab SSH + external HTTPS + DNS,
# training steps need GitLab HTTP (package fetch/upload), MinIO, Tempo, k8s API.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: argo-train-workflow
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/component: train
policyTypes:
- Egress
egress:
# DNS
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Kubernetes API — Argo executor
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
# External HTTPS — cargo fetches crates.io, github.com (candle git dep)
- ports:
- port: 443
protocol: TCP
# GitLab SSH — git clone
- ports:
- port: 2222
protocol: TCP
to:
- ipBlock:
cidr: 100.90.76.85/32
- podSelector:
matchLabels:
app: gitlab-shell
# GitLab webservice — package upload/download
- ports:
- port: 8181
protocol: TCP
to:
- podSelector:
matchLabels:
app: webservice
# MinIO — binary upload
- ports:
- port: 9000
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
# Tempo — OTLP traces
- ports:
- port: 4317
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: tempo
# Mattermost — exit handler webhook notifications
- ports:
- port: 8065
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
---
# CI pipeline pods (detect-changes + compile-service inline templates)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: argo-ci-pipeline
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/component: ci-pipeline
policyTypes:
- Egress
egress:
# DNS — resolve external hostnames
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Kubernetes API — Argo executor
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
# External HTTP/HTTPS — Kaniko base image pulls, apt-get, cargo fetches
- ports:
- port: 80
protocol: TCP
- port: 443
protocol: TCP
# GitLab SSH — git clone (compile-service uses ssh://git@git.fxhnt.ai)
- ports:
- port: 2222
protocol: TCP
to:
- ipBlock:
cidr: 100.90.76.85/32
- podSelector:
matchLabels:
app: gitlab-shell
# MinIO — binary upload from compile-service
- ports:
- port: 9000
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
# GitLab registry — Kaniko image push
- ports:
- port: 5000
protocol: TCP
to:
- podSelector:
matchLabels:
app: registry
# GitLab webservice — JWT auth for registry tokens
- ports:
- port: 8181
protocol: TCP
to:
- podSelector:
matchLabels:
app: webservice
# Mattermost — exit handler webhook notifications
- ports:
- port: 8065
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
---
# ── sanitizer-test + nsys-test + smoke-test workflows: same egress as ci-pipeline ──
# (git fetch over SSH, MinIO log archive + nsys-rep upload, registry image pull,
# external HTTPS for mc download)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: argo-sanitizer-nsys-workflow
namespace: foxhunt
spec:
podSelector:
matchExpressions:
- key: app.kubernetes.io/component
operator: In
values:
- sanitizer-test
- nsys-test
- smoke-test
policyTypes:
- Egress
egress:
# DNS
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# Kubernetes API
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
# External HTTPS — mc binary download from dl.min.io
- ports:
- port: 80
protocol: TCP
- port: 443
protocol: TCP
# GitLab SSH — git fetch
- ports:
- port: 2222
protocol: TCP
to:
- ipBlock:
cidr: 100.90.76.85/32
- podSelector:
matchLabels:
app: gitlab-shell
# MinIO — log archive + nsys profile upload
- ports:
- port: 9000
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
# GitLab registry — ci-builder image pull
- ports:
- port: 5000
protocol: TCP
to:
- podSelector:
matchLabels:
app: registry
# GitLab webservice — JWT auth for registry tokens
- ports:
- port: 8181
protocol: TCP
to:
- podSelector:
matchLabels:
app: webservice

47
infra/k8s/argo/ci-deploy-rbac.yaml
View File

@@ -1,47 +0,0 @@
# RBAC for CI pipeline — deploy step + self-apply Argo templates
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ci-deploy
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
rules:
# apps: roll + apply Deployments. watch is required by `kubectl rollout status`.
- apiGroups: ["apps"]
resources: [deployments]
verbs: [get, list, watch, patch]
- apiGroups: ["argoproj.io"]
resources: [workflowtemplates, eventsources, sensors, eventbus]
verbs: [get, list, create, update, patch]
# core: services/configmaps, plus serviceaccounts (tailscale-dashboard, dagster) and the forward-track PVC
# — added so the fxhnt-cockpit deploy can `kubectl apply` its full manifest set without a partial-apply failure.
- apiGroups: [""]
resources: [services, configmaps, serviceaccounts, persistentvolumeclaims]
verbs: [get, list, create, update, patch]
- apiGroups: ["networking.k8s.io"]
resources: [networkpolicies]
verbs: [get, list, create, update, patch]
# batch: the fxhnt-forward CronJob
- apiGroups: ["batch"]
resources: [cronjobs]
verbs: [get, list, create, update, patch]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: [roles, rolebindings]
verbs: [get, list, create, update, patch, bind, escalate]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ci-deploy
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ci-deploy
subjects:
- kind: ServiceAccount
name: argo-workflow
namespace: foxhunt

135
infra/k8s/argo/databento-download-template.yaml
View File

@@ -1,135 +0,0 @@
# Databento data download via streaming API (get_range).
#
# Parameterized Argo WorkflowTemplate — replaces the hardcoded K8s Job.
# Binary pre-uploaded to /data/bin/download_baseline on training-data-pvc.
# Universe configs loaded from configmap databento-universe-all.
#
# Usage:
# argo submit -n foxhunt --from=wftmpl/databento-download
# argo submit -n foxhunt --from=wftmpl/databento-download -p schema=trades -p output-dir=/data/futures-baseline-trades
# ./scripts/argo-download.sh mbp-10 --parallel 9 --pool ci-compile-cpu
#
# Databento rate limits: 100 concurrent connections per IP, 100 req/s.
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
name: databento-download
namespace: foxhunt
labels:
app.kubernetes.io/name: databento-download
app.kubernetes.io/part-of: foxhunt
spec:
entrypoint: download
serviceAccountName: argo-workflow
podMetadata:
labels:
app.kubernetes.io/part-of: foxhunt
app.kubernetes.io/component: databento-download
securityContext:
fsGroup: 1000
ttlStrategy:
secondsAfterCompletion: 3600
activeDeadlineSeconds: 86400 # 24 hours
arguments:
parameters:
- name: schema
value: mbp-10
description: "Databento schema: ohlcv-1m, ohlcv-1s, trades, mbp-10"
- name: output-dir
value: /data/futures-baseline-mbp10
description: "Output directory on training-data PVC"
- name: parallel
value: "9"
description: "Number of parallel quarterly downloads (max ~100 per Databento rate limit)"
- name: node-pool
value: ci-compile-cpu
description: "K8s node pool: ci-compile-cpu (32c/64G) or platform (4c/8G)"
volumes:
- name: training-data
persistentVolumeClaim:
claimName: training-data-pvc
- name: universe-configs
configMap:
name: databento-universe-all
templates:
- name: download
inputs:
parameters:
- name: schema
- name: output-dir
- name: parallel
- name: node-pool
nodeSelector:
k8s.scaleway.com/pool-name: "{{inputs.parameters.node-pool}}"
topology.kubernetes.io/zone: fr-par-2
container:
image: ubuntu:24.04
command: ["/bin/bash", "-c"]
env:
- name: RUST_LOG
value: info
- name: DATABENTO_API_KEY
valueFrom:
secretKeyRef:
name: databento-credentials
key: api-key
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: "2"
memory: 4Gi
volumeMounts:
- name: training-data
mountPath: /data
- name: universe-configs
mountPath: /configs
args:
- |
set -e
apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq ca-certificates > /dev/null 2>&1
BINARY=/data/bin/download_baseline
chmod +x "$BINARY"
SCHEMA="{{inputs.parameters.schema}}"
OUTPUT_DIR="{{inputs.parameters.output-dir}}"
PARALLEL="{{inputs.parameters.parallel}}"
# Map schema name to configmap key
CONFIG="/configs/${SCHEMA}.toml"
if [ ! -f "$CONFIG" ]; then
echo "ERROR: Config not found: $CONFIG"
echo "Available configs:"
ls /configs/
exit 1
fi
mkdir -p "$OUTPUT_DIR"
echo "=== Databento Download ==="
echo "Schema: $SCHEMA"
echo "Config: $CONFIG"
echo "Output: $OUTPUT_DIR"
echo "Parallel: $PARALLEL"
echo "Node: $(hostname), CPUs: $(nproc), RAM: $(free -g | awk '/Mem:/{print $2}')G"
echo ""
$BINARY \
--universe-config "$CONFIG" \
--output-dir "$OUTPUT_DIR" \
--parallel "$PARALLEL" \
--yes
echo ""
echo "=== Download complete ==="
echo "Files:"
find "$OUTPUT_DIR" -name '*.dbn.zst' -exec ls -lh {} \;
echo ""
du -sh "$OUTPUT_DIR"
df -h /data

25
infra/k8s/argo/dns-bootstrap-policy.yaml
View File

@@ -1,25 +0,0 @@
# DNS Bootstrap Prevention — Prevent CoreDNS chicken-and-egg deadlock
#
# ROOT CAUSE (2026-03-18 incident):
# 1. Rolling replacement of ALL platform nodes simultaneously
# 2. New nodes boot with /etc/resolv.conf → 10.32.0.10 (kube-dns only)
# 3. CoreDNS pods need to pull images → need DNS → need CoreDNS
# 4. Cilium BPF socket LB returns EPERM when kube-dns has no endpoints
# 5. Complete cluster DNS deadlock — nothing can pull images
#
# PREVENTION:
# 1. VPC DefaultRoutePropagation MUST be enabled (API, not Terraform yet)
# 2. NEVER replace ALL nodes at once — one at a time, wait for CoreDNS
# 3. Do NOT create CiliumClusterwideNetworkPolicy with egress rules
# (enables default deny, makes the problem worse)
# 4. bpf-lb-sock MUST remain "true" — disabling breaks ALL service routing
# 5. Keep bastion enabled on VPC gateway for emergency SSH access
#
# RECOVERY PROCEDURE:
# 1. SSH via bastion: ssh -o ProxyCommand="ssh -W %h:%p -p 61000 bastion@<gw-public-ip>" root@<node-private-ip>
# 2. On the node: /usr/local/bin/crictl pull <coredns-image>
# 3. kubectl delete pods -l k8s-app=kube-dns --force
# 4. CoreDNS starts from cached image → DNS recovers → cascade
#
# DATA PROTECTION:
# All PVs set to Retain reclaim policy — data survives cluster deletion

10
infra/k8s/argo/events/eventbus.yaml
View File

@@ -1,10 +0,0 @@
apiVersion: argoproj.io/v1alpha1
kind: EventBus
metadata:
name: default
namespace: foxhunt
spec:
nats:
native:
replicas: 3
auth: token

16
infra/k8s/argo/events/eventsource-svc.yaml
View File

@@ -1,16 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: gitlab-push-eventsource-svc
namespace: foxhunt
labels:
app.kubernetes.io/name: gitlab-push-eventsource
app.kubernetes.io/part-of: foxhunt
spec:
type: ClusterIP
ports:
- port: 12000
targetPort: 12000
protocol: TCP
selector:
eventsource-name: gitlab-push

56
infra/k8s/argo/events/gitea-deploy-sensor.yaml
View File

@@ -1,56 +0,0 @@
# On push to fxhnt main/master in Gitea, submit the fxhnt-cockpit build+deploy workflow (Phase 2B).
# Gitea push payload: { "ref": "refs/heads/main", "after": "<new-sha>", ... }
apiVersion: argoproj.io/v1alpha1
kind: Sensor
metadata:
name: gitea-deploy
namespace: foxhunt
labels:
app.kubernetes.io/name: gitea-deploy
app.kubernetes.io/part-of: foxhunt
spec:
template:
# pod intentionally NOT part-of:foxhunt (avoids default-deny-all; matches working sensors)
serviceAccountName: argo-workflow
nodeSelector:
k8s.scaleway.com/pool-name: platform
dependencies:
- name: gitea-push-dep
eventSourceName: gitea-push
eventName: push
filters:
data:
- path: body.ref
type: string
value:
- "refs/heads/main"
- "refs/heads/master"
triggers:
- template:
name: deploy-cockpit
conditions: gitea-push-dep
argoWorkflow:
operation: submit
source:
resource:
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: fxhnt-cockpit-
namespace: foxhunt
spec:
serviceAccountName: argo-workflow
podGC:
strategy: OnPodCompletion
ttlStrategy:
secondsAfterCompletion: 3600
workflowTemplateRef:
name: fxhnt-cockpit
arguments:
parameters:
- name: commit-sha
parameters:
- src:
dependencyName: gitea-push-dep
dataKey: body.after
dest: spec.arguments.parameters.0.value

28
infra/k8s/argo/events/gitea-push-eventsource.yaml
View File

@@ -1,28 +0,0 @@
# Gitea push webhook → Argo Events. Replaces the old gitlab-push eventsource (Phase 2B).
# Gitea posts to gitea-push-eventsource-svc:12000/push on push to the fxhnt repo.
apiVersion: argoproj.io/v1alpha1
kind: EventSource
metadata:
name: gitea-push
namespace: foxhunt
labels:
app.kubernetes.io/name: gitea-push
app.kubernetes.io/part-of: foxhunt
spec:
template:
# NOTE: pod is intentionally NOT part-of:foxhunt — that label triggers default-deny-all. The working
# workflow-trigger/gitlab-push eventsource pods have no part-of, so they stay default-allow.
metadata:
labels:
app.kubernetes.io/name: gitea-push
nodeSelector:
k8s.scaleway.com/pool-name: platform
service:
ports:
- port: 12000
targetPort: 12000
webhook:
push:
port: "12000"
endpoint: /push
method: POST

29
infra/k8s/argo/events/rbac.yaml
View File

@@ -1,29 +0,0 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: argo-sensor-submit
namespace: foxhunt
rules:
- apiGroups: ["argoproj.io"]
resources: ["workflows"]
verbs: ["create", "get", "list", "watch"]
- apiGroups: ["argoproj.io"]
resources: ["workflowtemplates"]
verbs: ["get", "list"]
- apiGroups: ["argoproj.io"]
resources: ["workflowtaskresults"]
verbs: ["create", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: argo-sensor-submit
namespace: foxhunt
subjects:
- kind: ServiceAccount
name: argo-workflow
namespace: foxhunt
roleRef:
kind: Role
name: argo-sensor-submit
apiGroup: rbac.authorization.k8s.io

51
infra/k8s/argo/events/workflow-trigger-eventsource.yaml
View File

@@ -1,51 +0,0 @@
# Manual workflow trigger EventSource.
#
# Webhook endpoints for triggering compile/deploy, training, and image builds.
# Each endpoint accepts a JSON POST body with workflow-specific parameters.
#
# Endpoints (all on port 12001):
# POST /compile-deploy — {"commit_sha": "HEAD"}
# POST /train/dqn — {"binary_tag": "latest"}
# POST /train/ppo — {"binary_tag": "latest"}
# POST /train/supervised — {"model": "tft", "binary_tag": "latest"}
# POST /build-image — {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}
#
# Example:
# curl -X POST http://workflow-trigger-eventsource-svc.foxhunt:12001/build-image \
# -H 'Content-Type: application/json' \
# -d '{"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}'
---
apiVersion: argoproj.io/v1alpha1
kind: EventSource
metadata:
name: workflow-trigger
namespace: foxhunt
labels:
app.kubernetes.io/name: workflow-trigger
app.kubernetes.io/part-of: foxhunt
spec:
eventBusName: default
template:
nodeSelector:
k8s.scaleway.com/pool-name: platform
webhook:
compile-deploy:
port: "12001"
endpoint: /compile-deploy
method: POST
train-dqn:
port: "12001"
endpoint: /train/dqn
method: POST
train-ppo:
port: "12001"
endpoint: /train/ppo
method: POST
train-supervised:
port: "12001"
endpoint: /train/supervised
method: POST
build-image:
port: "12001"
endpoint: /build-image
method: POST

217
infra/k8s/argo/events/workflow-trigger-sensor.yaml
View File

@@ -1,217 +0,0 @@
# Sensor for manual workflow triggers.
#
# Listens on workflow-trigger EventSource and submits the appropriate workflow
# when a webhook endpoint is hit. Each trigger maps JSON body fields to
# workflow parameters.
#
# Trigger reference:
# compile-deploy — POST /compile-deploy {"commit_sha": "..."}
# train-dqn — POST /train/dqn {"binary_tag": "..."}
# train-ppo — POST /train/ppo {"binary_tag": "..."}
# train-supervised — POST /train/supervised {"model": "...", "binary_tag": "..."}
# build-image — POST /build-image {"dockerfile": "...", "image_name": "...", "commit_sha": "..."}
---
apiVersion: argoproj.io/v1alpha1
kind: Sensor
metadata:
name: workflow-trigger
namespace: foxhunt
labels:
app.kubernetes.io/name: workflow-trigger
app.kubernetes.io/part-of: foxhunt
spec:
template:
serviceAccountName: argo-workflow
nodeSelector:
k8s.scaleway.com/pool-name: platform
eventBusName: default
dependencies:
- name: compile-deploy-dep
eventSourceName: workflow-trigger
eventName: compile-deploy
- name: train-dqn-dep
eventSourceName: workflow-trigger
eventName: train-dqn
- name: train-ppo-dep
eventSourceName: workflow-trigger
eventName: train-ppo
- name: train-supervised-dep
eventSourceName: workflow-trigger
eventName: train-supervised
- name: build-image-dep
eventSourceName: workflow-trigger
eventName: build-image
triggers:
# ── Compile + Deploy ──
# POST /compile-deploy {"commit_sha": "abc1234"}
# Omit service_packages to build all 7 services (default).
- template:
name: compile-deploy
conditions: compile-deploy-dep
argoWorkflow:
operation: submit
source:
resource:
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: compile-deploy-
namespace: foxhunt
spec:
serviceAccountName: argo-workflow
podGC:
strategy: OnPodCompletion
ttlStrategy:
secondsAfterCompletion: 3600
workflowTemplateRef:
name: compile-and-deploy
arguments:
parameters:
- name: commit-sha
parameters:
- src:
dependencyName: compile-deploy-dep
dataKey: body.commit_sha
dest: spec.arguments.parameters.0.value
# ── Train DQN ──
# POST /train/dqn {"binary_tag": "latest"}
- template:
name: train-dqn
conditions: train-dqn-dep
argoWorkflow:
operation: submit
source:
resource:
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: train-dqn-
namespace: foxhunt
spec:
serviceAccountName: argo-workflow
podGC:
strategy: OnPodCompletion
ttlStrategy:
secondsAfterCompletion: 3600
workflowTemplateRef:
name: train-dqn
arguments:
parameters:
- name: binary-tag
parameters:
- src:
dependencyName: train-dqn-dep
dataKey: body.binary_tag
dest: spec.arguments.parameters.0.value
# ── Train PPO ──
# POST /train/ppo {"binary_tag": "latest"}
- template:
name: train-ppo
conditions: train-ppo-dep
argoWorkflow:
operation: submit
source:
resource:
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: train-ppo-
namespace: foxhunt
spec:
serviceAccountName: argo-workflow
podGC:
strategy: OnPodCompletion
ttlStrategy:
secondsAfterCompletion: 3600
workflowTemplateRef:
name: train-ppo
arguments:
parameters:
- name: binary-tag
parameters:
- src:
dependencyName: train-ppo-dep
dataKey: body.binary_tag
dest: spec.arguments.parameters.0.value
# ── Train Supervised ──
# POST /train/supervised {"model": "tft", "binary_tag": "latest"}
# model: one of tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion
- template:
name: train-supervised
conditions: train-supervised-dep
argoWorkflow:
operation: submit
source:
resource:
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: train-supervised-
namespace: foxhunt
spec:
serviceAccountName: argo-workflow
podGC:
strategy: OnPodCompletion
ttlStrategy:
secondsAfterCompletion: 3600
workflowTemplateRef:
name: train-supervised
arguments:
parameters:
- name: model
- name: binary-tag
parameters:
- src:
dependencyName: train-supervised-dep
dataKey: body.model
dest: spec.arguments.parameters.0.value
- src:
dependencyName: train-supervised-dep
dataKey: body.binary_tag
dest: spec.arguments.parameters.1.value
# ── Build Image ──
# POST /build-image {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}
- template:
name: build-image
conditions: build-image-dep
argoWorkflow:
operation: submit
source:
resource:
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: build-image-
namespace: foxhunt
spec:
serviceAccountName: argo-workflow
podGC:
strategy: OnPodCompletion
ttlStrategy:
secondsAfterCompletion: 3600
workflowTemplateRef:
name: build-ci-image
arguments:
parameters:
- name: commit-sha
- name: dockerfile
- name: image-name
parameters:
- src:
dependencyName: build-image-dep
dataKey: body.commit_sha
dest: spec.arguments.parameters.0.value
- src:
dependencyName: build-image-dep
dataKey: body.dockerfile
dest: spec.arguments.parameters.1.value
- src:
dependencyName: build-image-dep
dataKey: body.image_name
dest: spec.arguments.parameters.2.value

16
infra/k8s/argo/events/workflow-trigger-svc.yaml
View File

@@ -1,16 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: workflow-trigger-eventsource-svc
namespace: foxhunt
labels:
app.kubernetes.io/name: workflow-trigger
app.kubernetes.io/part-of: foxhunt
spec:
type: ClusterIP
ports:
- port: 12001
targetPort: 12001
protocol: TCP
selector:
eventsource-name: workflow-trigger

18
infra/k8s/argo/feature-cache-pvc.yaml
View File

@@ -1,18 +0,0 @@
# Dedicated PVC for precomputed .fxcache files.
# Survives workflow completions (unlike workspace volumeClaimTemplate).
# Precompute writes here, all training steps read from here.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: feature-cache-pvc
namespace: foxhunt
labels:
app.kubernetes.io/name: feature-cache
app.kubernetes.io/part-of: foxhunt
spec:
accessModes:
- ReadWriteOnce
storageClassName: scw-bssd-retain
resources:
requests:
storage: 10Gi

8
infra/k8s/argo/kustomization.yaml
View File

@@ -1,8 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
# Rust train/CI/cache templates removed 2026-06-21 (decommission-rust-infra).
# Remaining: platform RBAC + network policy used by the active fxhnt-cockpit deploys.
resources:
- argo-workflow-netpol.yaml
- ci-deploy-rbac.yaml
- archive-rbac.yaml

11
infra/k8s/argo/notification-webhook.yaml
View File

@@ -1,11 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: notification-webhook
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
# Mattermost incoming webhook URL — update after Mattermost is deployed
webhook-url: "http://mattermost.foxhunt.svc.cluster.local:8065/hooks/PLACEHOLDER"

80
infra/k8s/argo/values.yaml
View File

@@ -1,80 +0,0 @@
# Argo Workflows Helm values — foxhunt cluster
# Chart: argo/argo-workflows v1.0.2 (Argo v4.0.2)
# Install: helm upgrade argo-workflows argo/argo-workflows -n foxhunt -f infra/k8s/argo/values.yaml
controller:
nodeSelector:
k8s.scaleway.com/pool-name: platform
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 200m
memory: 256Mi
# Only watch foxhunt namespace (not cluster-wide)
workflowNamespaces:
- foxhunt
# Workflow archive — persists workflow state + logs to PostgreSQL after pod GC
persistence:
archive: true
postgresql:
host: postgres.foxhunt.svc.cluster.local
port: 5432
database: argo_archive
tableName: argo_workflows
userNameSecret:
name: argo-postgres-config
key: username
passwordSecret:
name: argo-postgres-config
key: password
ssl: false
sslMode: disable
server:
nodeSelector:
k8s.scaleway.com/pool-name: platform
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 100m
memory: 128Mi
ingress:
enabled: false
extraArgs:
- --auth-mode=server
- --secure=false
# Default artifact repository — MinIO (in-cluster)
artifactRepository:
archiveLogs: true
s3:
endpoint: minio.foxhunt.svc.cluster.local:9000
insecure: true
bucket: argo-logs
accessKeySecret:
name: minio-credentials
key: access-key
secretKeySecret:
name: minio-credentials
key: secret-key
executor:
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
workflow:
serviceAccount:
create: true
name: argo-workflow
crds:
install: true

72
infra/k8s/cert-manager/fxhnt-acme.yaml
View File

@@ -1,72 +0,0 @@
# ACME (Let's Encrypt) wildcard cert for *.fxhnt.ai via cert-manager + Scaleway DNS-01.
# Replaces the prior MANUAL, unrenewed LE cert that expired 2026-05-26 (the tailscale-proxy
# fell back to the GitLab self-signed cert). DNS-01 is required: the services are tailnet-only
# (*.fxhnt.ai -> 100.x Tailscale CGNAT) so HTTP-01 is unreachable, and wildcards need DNS-01.
#
# Solver creds: cert-manager/scaleway-dns-credentials (SCW_ACCESS_KEY/SCW_SECRET_KEY, copied from
# foxhunt/scaleway-credentials). Webhook: scaleway-certmanager-webhook (helm, cert-manager ns).
# The issued secret gitlab-tls-cert is mounted by infra/k8s/gitlab/tailscale-proxy.yaml.
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: letsencrypt-staging
spec:
acme:
email: jeroen@grusewski.nl
server: https://acme-staging-v02.api.letsencrypt.org/directory
privateKeySecretRef:
name: letsencrypt-staging-account-key
solvers:
- dns01:
webhook:
groupName: acme.scaleway.com
solverName: scaleway
config:
accessKeySecretRef:
name: scaleway-dns-credentials
key: SCW_ACCESS_KEY
secretKeySecretRef:
name: scaleway-dns-credentials
key: SCW_SECRET_KEY
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: letsencrypt-prod
spec:
acme:
email: jeroen@grusewski.nl
server: https://acme-v02.api.letsencrypt.org/directory
privateKeySecretRef:
name: letsencrypt-prod-account-key
solvers:
- dns01:
webhook:
groupName: acme.scaleway.com
solverName: scaleway
config:
accessKeySecretRef:
name: scaleway-dns-credentials
key: SCW_ACCESS_KEY
secretKeySecretRef:
name: scaleway-dns-credentials
key: SCW_SECRET_KEY
---
# Wildcard cert for the tailscale-proxy. issuerRef flips staging->prod after staging validates;
# secretName flips to gitlab-tls-cert (the secret the proxy mounts) for the prod issuance.
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: fxhnt-wildcard
namespace: foxhunt
spec:
secretName: gitlab-tls-cert # takes over the secret the tailscale-proxy mounts
privateKey:
rotationPolicy: Always # old manual key had a mismatching algorithm; regenerate on issue/renew
dnsNames:
- fxhnt.ai
- "*.fxhnt.ai"
issuerRef:
name: letsencrypt-prod
kind: ClusterIssuer

249
infra/k8s/databases/postgres-backup.yaml
View File

@@ -1,249 +0,0 @@
# Postgres backup — daily pg_dump to MinIO S3
# Uses initContainer (postgres:16-alpine) for pg_dump, then main container
# (python:3.12-alpine, already cached) uploads to MinIO via S3 API.
# Retains 7 daily + 4 weekly backups.
# Apply: kubectl apply -f postgres-backup.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name: postgres-backup-script
namespace: foxhunt
labels:
app.kubernetes.io/name: postgres-backup
app.kubernetes.io/part-of: foxhunt
data:
upload.py: |
#!/usr/bin/env python3
"""Upload pg_dump backup to MinIO, prune old backups."""
import hashlib, hmac, os, sys, glob, urllib.request, urllib.error
from datetime import datetime, timezone
ENDPOINT = os.environ["MINIO_ENDPOINT"] # http://minio.foxhunt...:9000
ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"]
SECRET_KEY = os.environ["MINIO_SECRET_KEY"]
BUCKET = "foxhunt-backups"
BACKUP_DIR = "/backup"
def s3_sign(method, path, headers, payload_hash):
"""AWS Signature V4 for MinIO."""
now = datetime.now(timezone.utc)
date_stamp = now.strftime("%Y%m%d")
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
region = "us-east-1"
service = "s3"
scope = f"{date_stamp}/{region}/{service}/aws4_request"
headers["x-amz-date"] = amz_date
headers["x-amz-content-sha256"] = payload_hash
signed_headers = ";".join(sorted(headers.keys()))
canonical_headers = "".join(f"{k}:{headers[k]}\n" for k in sorted(headers.keys()))
canonical_request = f"{method}\n{path}\n\n{canonical_headers}\n{signed_headers}\n{payload_hash}"
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{hashlib.sha256(canonical_request.encode()).hexdigest()}"
def sign(key, msg):
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
signing_key = sign(sign(sign(sign(
f"AWS4{SECRET_KEY}".encode(), date_stamp), region), service), "aws4_request")
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
headers["Authorization"] = (
f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/{scope}, "
f"SignedHeaders={signed_headers}, Signature={signature}"
)
return headers
def s3_put(key, data):
path = f"/{BUCKET}/{key}"
url = f"{ENDPOINT}{path}"
payload_hash = hashlib.sha256(data).hexdigest()
host = ENDPOINT.split("//")[1]
headers = {"host": host, "content-length": str(len(data))}
headers = s3_sign("PUT", path, headers, payload_hash)
req = urllib.request.Request(url, data=data, method="PUT", headers=headers)
urllib.request.urlopen(req, timeout=120)
def s3_list(prefix):
path = f"/{BUCKET}?prefix={prefix}&list-type=2"
url = f"{ENDPOINT}{path}"
host = ENDPOINT.split("//")[1]
headers = {"host": host}
headers = s3_sign("GET", f"/{BUCKET}", headers, hashlib.sha256(b"").hexdigest())
req = urllib.request.Request(f"{url}", method="GET", headers=headers)
try:
resp = urllib.request.urlopen(req, timeout=30).read().decode()
import re
return sorted(re.findall(r"<Key>([^<]+)</Key>", resp))
except Exception:
return []
def s3_delete(key):
path = f"/{BUCKET}/{key}"
url = f"{ENDPOINT}{path}"
host = ENDPOINT.split("//")[1]
headers = {"host": host}
headers = s3_sign("DELETE", path, headers, hashlib.sha256(b"").hexdigest())
req = urllib.request.Request(url, method="DELETE", headers=headers)
urllib.request.urlopen(req, timeout=30)
def s3_ensure_bucket():
path = f"/{BUCKET}"
url = f"{ENDPOINT}{path}"
host = ENDPOINT.split("//")[1]
headers = {"host": host}
headers = s3_sign("PUT", path, headers, hashlib.sha256(b"").hexdigest())
req = urllib.request.Request(url, method="PUT", headers=headers)
try:
urllib.request.urlopen(req, timeout=30)
except urllib.error.HTTPError as e:
if e.code not in (409, 200): # 409 = bucket exists
raise
# Find the backup file
files = glob.glob(f"{BACKUP_DIR}/*.sql.gz")
if not files:
print("ERROR: No backup file found in /backup/")
sys.exit(1)
backup_file = files[0]
filename = os.path.basename(backup_file)
size_mb = os.path.getsize(backup_file) / (1024 * 1024)
print(f"Uploading {filename} ({size_mb:.1f} MB)")
s3_ensure_bucket()
with open(backup_file, "rb") as f:
data = f.read()
# Upload daily
s3_put(f"daily/{filename}", data)
print(f" Uploaded daily/{filename}")
# On Sundays, also upload weekly
if datetime.now(timezone.utc).isoweekday() == 7:
s3_put(f"weekly/{filename}", data)
print(f" Uploaded weekly/{filename}")
# Prune old dailies (keep 7)
dailies = s3_list("daily/")
if len(dailies) > 7:
for old in dailies[:len(dailies) - 7]:
s3_delete(old)
print(f" Pruned {old}")
# Prune old weeklies (keep 4)
weeklies = s3_list("weekly/")
if len(weeklies) > 4:
for old in weeklies[:len(weeklies) - 4]:
s3_delete(old)
print(f" Pruned {old}")
print("=== Backup upload complete ===")
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: postgres-backup
namespace: foxhunt
labels:
app.kubernetes.io/name: postgres-backup
app.kubernetes.io/part-of: foxhunt
spec:
schedule: "30 2 * * *"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
activeDeadlineSeconds: 600
template:
metadata:
labels:
app: postgres-backup
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
restartPolicy: OnFailure
# Step 1: pg_dump in initContainer → shared /backup volume
initContainers:
- name: pg-dump
image: postgres:16-alpine
command:
- sh
- -c
- |
TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)
echo "=== Postgres Backup ${TIMESTAMP} ==="
PGPASSWORD="${POSTGRES_PASSWORD}" pg_dumpall \
-h "${POSTGRES_HOST}" \
-p 5432 \
-U "${POSTGRES_USER}" \
--clean --if-exists \
| gzip > "/backup/foxhunt-${TIMESTAMP}.sql.gz"
ls -lh /backup/
echo "Dump complete"
env:
- name: POSTGRES_HOST
value: "postgres.foxhunt.svc.cluster.local"
- name: POSTGRES_USER
value: "foxhunt"
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: db-credentials
key: password
volumeMounts:
- name: backup
mountPath: /backup
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
# Step 2: Upload to MinIO in main container (python cached on nodes)
containers:
- name: upload
image: python:3.12-alpine
command: ["python3", "-u", "/scripts/upload.py"]
env:
- name: MINIO_ENDPOINT
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-credentials
key: access-key
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-credentials
key: secret-key
volumeMounts:
- name: backup
mountPath: /backup
- name: script
mountPath: /scripts
resources:
requests:
cpu: 10m
memory: 64Mi
limits:
cpu: 100m
memory: 128Mi
volumes:
- name: backup
emptyDir:
sizeLimit: 2Gi
- name: script
configMap:
name: postgres-backup-script
items:
- key: upload.py
path: upload.py
defaultMode: 0755

96
infra/k8s/databases/postgres.yaml
View File

@@ -1,96 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: postgres-pvc
namespace: foxhunt
labels:
app.kubernetes.io/name: postgres
app.kubernetes.io/part-of: foxhunt
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres
namespace: foxhunt
labels:
app.kubernetes.io/name: postgres
app.kubernetes.io/part-of: foxhunt
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: postgres
strategy:
type: Recreate
template:
metadata:
labels:
app.kubernetes.io/name: postgres
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: infra
containers:
- name: postgres
image: timescale/timescaledb:latest-pg16
ports:
- containerPort: 5432
name: postgres
args:
- -c
- max_connections=100
env:
- name: POSTGRES_DB
value: foxhunt
- name: POSTGRES_USER
value: foxhunt
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: db-credentials
key: password
volumeMounts:
- name: postgres-data
mountPath: /var/lib/postgresql/data
subPath: pgdata
readinessProbe:
exec:
command:
- pg_isready
- -U
- foxhunt
initialDelaySeconds: 10
periodSeconds: 10
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
volumes:
- name: postgres-data
persistentVolumeClaim:
claimName: postgres-pvc
---
apiVersion: v1
kind: Service
metadata:
name: postgres
namespace: foxhunt
labels:
app.kubernetes.io/name: postgres
app.kubernetes.io/part-of: foxhunt
spec:
selector:
app.kubernetes.io/name: postgres
ports:
- port: 5432
targetPort: 5432
name: postgres

95
infra/k8s/databases/questdb.yaml
View File

@@ -1,95 +0,0 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: questdb-pvc
namespace: foxhunt
labels:
app.kubernetes.io/name: questdb
app.kubernetes.io/part-of: foxhunt
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: questdb
namespace: foxhunt
labels:
app.kubernetes.io/name: questdb
app.kubernetes.io/part-of: foxhunt
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: questdb
strategy:
type: Recreate
template:
metadata:
labels:
app.kubernetes.io/name: questdb
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: questdb
image: questdb/questdb:8.2.3
ports:
- containerPort: 9009
name: ilp
- containerPort: 8812
name: pg
- containerPort: 9003
name: metrics
env:
- name: QDB_PG_ENABLED
value: "true"
- name: QDB_METRICS_ENABLED
value: "TRUE"
volumeMounts:
- name: questdb-data
mountPath: /var/lib/questdb
readinessProbe:
httpGet:
path: /exec?query=SELECT%201
port: 9003
initialDelaySeconds: 10
periodSeconds: 10
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
volumes:
- name: questdb-data
persistentVolumeClaim:
claimName: questdb-pvc
---
apiVersion: v1
kind: Service
metadata:
name: questdb
namespace: foxhunt
labels:
app.kubernetes.io/name: questdb
app.kubernetes.io/part-of: foxhunt
spec:
selector:
app.kubernetes.io/name: questdb
ports:
- port: 9009
targetPort: 9009
name: ilp
- port: 8812
targetPort: 8812
name: pg
- port: 9003
targetPort: 9003
name: metrics

63
infra/k8s/databases/redis.yaml
View File

@@ -1,63 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: foxhunt
labels:
app.kubernetes.io/name: redis
app.kubernetes.io/part-of: foxhunt
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: redis
template:
metadata:
labels:
app.kubernetes.io/name: redis
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: infra
containers:
- name: redis
image: redis:7-alpine
command:
- redis-server
- --maxmemory
- 512mb
- --maxmemory-policy
- allkeys-lru
ports:
- containerPort: 6379
name: redis
readinessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 5
periodSeconds: 10
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 250m
memory: 640Mi
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: foxhunt
labels:
app.kubernetes.io/name: redis
app.kubernetes.io/part-of: foxhunt
spec:
selector:
app.kubernetes.io/name: redis
ports:
- port: 6379
targetPort: 6379
name: redis

182
infra/k8s/dns/node-local-dns.yaml
View File

@@ -1,182 +0,0 @@
# Node Bootstrap DaemonSet for Kapsule
#
# Two problems on Kapsule nodes:
# 1. Nodes have nameserver 127.0.53.53 which can't resolve .svc.cluster.local
# 2. Containerd defaults to HTTPS for non-localhost registries
#
# Fixes:
# 1. Point /etc/resolv.conf to kube-dns ClusterIP (10.32.0.10)
# 2. Create /etc/containerd/certs.d/ config for HTTP GitLab registry
# 3. On GPU nodes (containerd v3 with conf.d/), add registry drop-in
#
# Init container sets up containerd registry config (one-time).
# Main container manages resolv.conf (watches for Kapsule resets).
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: node-bootstrap
namespace: kube-system
labels:
app.kubernetes.io/name: node-bootstrap
spec:
selector:
matchLabels:
app.kubernetes.io/name: node-bootstrap
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
app.kubernetes.io/name: node-bootstrap
spec:
hostNetwork: true
dnsPolicy: Default
priorityClassName: system-node-critical
tolerations:
- operator: Exists
hostPID: true
initContainers:
# Configure containerd to use HTTP for the GitLab registry.
# On GPU nodes (NVIDIA overlay), also creates a v3-compatible conf.d drop-in
# and restarts containerd so it picks up the new config.
- name: setup-registry
image: alpine:3.19
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
set -e
NEEDS_RESTART=false
# 1. Create per-host containerd registry config (HTTP for GitLab)
REGISTRY_DIR="/host-etc/containerd/certs.d/gitlab-registry.foxhunt.svc.cluster.local:5000"
mkdir -p "$REGISTRY_DIR"
DESIRED='server = "http://gitlab-registry.foxhunt.svc.cluster.local:5000"
[host."http://gitlab-registry.foxhunt.svc.cluster.local:5000"]
capabilities = ["pull", "resolve"]'
DESIRED=$(printf '%s' "$DESIRED" | sed 's/^ //')
if [ ! -f "$REGISTRY_DIR/hosts.toml" ] || [ "$(cat "$REGISTRY_DIR/hosts.toml")" != "$DESIRED" ]; then
printf '%s\n' "$DESIRED" > "$REGISTRY_DIR/hosts.toml"
echo "Created hosts.toml for HTTP gitlab-registry"
NEEDS_RESTART=true
else
echo "hosts.toml already up to date"
fi
# 2. Ensure conf.d/ exists and always write the v3 registry drop-in.
# containerd 2.x may ignore the v2-style grpc.v1.cri.registry.config_path
# even when it's set in config.toml. The v1 images drop-in is the reliable path.
# Create conf.d/ if missing (base config already has imports = ["conf.d/*.toml"]).
CONF_D="/host-etc/containerd/conf.d"
mkdir -p "$CONF_D"
if [ -d "$CONF_D" ]; then
DROPIN='version = 3
[plugins."io.containerd.cri.v1.images".registry]
config_path = "/etc/containerd/certs.d"'
DROPIN=$(printf '%s' "$DROPIN" | sed 's/^ //')
if [ ! -f "$CONF_D/50-registry.toml" ] || [ "$(cat "$CONF_D/50-registry.toml")" != "$DROPIN" ]; then
printf '%s\n' "$DROPIN" > "$CONF_D/50-registry.toml"
echo "GPU node: created v3 registry config drop-in"
NEEDS_RESTART=true
else
echo "GPU node: v3 drop-in already up to date"
fi
fi
# 3. Restart containerd only if we changed config files.
# Uses /proc/1/root chroot to access host systemctl (requires hostPID + privileged).
# On a new node this will kill the DaemonSet pod; kubelet recreates it,
# second run finds files unchanged (NEEDS_RESTART=false), proceeds normally.
if [ "$NEEDS_RESTART" = "true" ]; then
echo "Restarting containerd to pick up config changes..."
chroot /proc/1/root systemctl restart containerd || \
echo "WARNING: containerd restart failed (node may need manual restart)"
echo "containerd restart issued"
fi
resources:
requests:
cpu: 10m
memory: 8Mi
limits:
cpu: 50m
memory: 16Mi
volumeMounts:
- name: host-etc
mountPath: /host-etc
containers:
- name: resolv-manager
image: busybox:1.37
command: ["/bin/sh", "-c"]
args:
- |
set -e
RESOLV="/host-etc/resolv.conf"
BACKUP="/host-etc/resolv.conf.pre-kube-dns"
KUBE_DNS="10.32.0.10"
restore() {
if [ -f "$BACKUP" ]; then
cp "$BACKUP" "$RESOLV"
echo "$(date): Restored original resolv.conf"
fi
exit 0
}
trap restore TERM INT
# Save original
if [ ! -f "$BACKUP" ]; then
cp "$RESOLV" "$BACKUP"
echo "Saved original: $(cat "$BACKUP")"
fi
apply() {
# CRITICAL: keep VPC upstream resolver as FALLBACK after kube-dns.
# If resolv.conf ONLY has 10.32.0.10, node DNS is dead when CoreDNS
# is down — containerd can't pull images, CoreDNS can't start,
# entire cluster deadlocks (incident 2026-03-18, 2026-04-12).
# Scaleway VPC gateway pushes 172.16.0.13 as DNS via DHCP.
UPSTREAM=""
if [ -f "$BACKUP" ]; then
UPSTREAM=$(grep "^nameserver" "$BACKUP" | grep -v "127.0.53.53\|$KUBE_DNS" | head -1 | awk '{print $2}')
fi
if [ -z "$UPSTREAM" ]; then
UPSTREAM="169.254.169.254" # Scaleway metadata DNS fallback
fi
printf 'nameserver %s\nnameserver %s\nsearch foxhunt.svc.cluster.local svc.cluster.local cluster.local\noptions ndots:5 timeout:2 attempts:3\n' "$KUBE_DNS" "$UPSTREAM" > "$RESOLV"
echo "$(date): resolv.conf → $KUBE_DNS + $UPSTREAM (fallback)"
}
# Initial apply
apply
# Watch for resets
while true; do
sleep 30 &
wait $!
if ! grep -q "$KUBE_DNS" "$RESOLV" 2>/dev/null; then
echo "$(date): resolv.conf was reset, re-applying"
apply
fi
done
resources:
requests:
cpu: 5m
memory: 8Mi
limits:
cpu: 10m
memory: 16Mi
volumeMounts:
- name: host-etc
mountPath: /host-etc
terminationGracePeriodSeconds: 10
volumes:
- name: host-etc
hostPath:
path: /etc
type: Directory

40
infra/k8s/gitea/clusterip-services.yaml
View File

@@ -1,40 +0,0 @@
# The gitea chart's gitea-http/gitea-ssh services are HEADLESS (clusterIP: None), so they resolve to the
# pod IP (100.64.x.x). That range OVERLAPS the Tailscale CGNAT range (100.64.0.0/10), so the tailscale
# sidecar in the tailscale-gitlab-proxy pod swallows traffic to the pod IP. These normal ClusterIP services
# give Gitea a service IP in the service CIDR (10.32.x.x, outside the tailscale range) — the proxy nginx +
# socat target these instead. (GitLab worked because its webservice svc was a normal ClusterIP.)
apiVersion: v1
kind: Service
metadata:
name: gitea-web
namespace: foxhunt
labels:
app.kubernetes.io/name: gitea
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: gitea
app.kubernetes.io/instance: gitea
ports:
- name: http
port: 3000
targetPort: 3000
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: gitea-sshd
namespace: foxhunt
labels:
app.kubernetes.io/name: gitea
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: gitea
app.kubernetes.io/instance: gitea
ports:
- name: ssh
port: 22
targetPort: 2222 # rootless gitea listens on 2222 internally (can't bind privileged 22)
protocol: TCP

58
infra/k8s/gitea/networkpolicy.yaml
View File

@@ -1,58 +0,0 @@
# Gitea NetworkPolicy (foxhunt ns has default-deny-all). Egress: postgres + DNS + in-cluster webhook
# target + general HTTPS. Ingress: from platform/argo pods + the tailscale proxy → :3000 (web) and :22 (ssh).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: gitea
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: gitea
policyTypes:
- Ingress
- Egress
ingress:
# web (3000) + ssh (22) from platform + argo-workflow pods (cockpit build clones over HTTP)
- from:
- podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: ["foxhunt", "argo-workflows"]
ports:
- { port: 3000, protocol: TCP }
- { port: 2222, protocol: TCP }
# web + ssh from the tailscale proxy (nginx + socat) that fronts git.fxhnt.ai
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: tailscale-gitlab-proxy
ports:
- { port: 3000, protocol: TCP }
- { port: 2222, protocol: TCP }
egress:
# PostgreSQL
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- { port: 5432, protocol: TCP }
# DNS
- ports:
- { port: 53, protocol: UDP }
- { port: 53, protocol: TCP }
# Argo Events webhook target (gitea push hook → gitea-push eventsource :12000). The eventsource pod is
# not part-of:foxhunt (stays default-allow), so allow egress to any in-namespace pod on 12000.
- to:
- podSelector: {}
ports:
- { port: 12000, protocol: TCP }
# general HTTPS (avatars, external clones) — external only, exclude cluster CIDRs
- to:
- ipBlock:
cidr: 0.0.0.0/0
except: ["10.32.0.0/16", "172.16.0.0/16"]
ports:
- { port: 443, protocol: TCP }

73
infra/k8s/gitea/values.yaml
View File

@@ -1,73 +0,0 @@
# Gitea — lightweight git host replacing GitLab (Phase 2B). External Postgres (existing in-cluster
# `postgres`), no bundled DB/redis/memcached, Actions off (Argo does CI). Internal-only until cutover.
replicaCount: 1
image:
rootless: true
# RWO PVC (sbs-default-retain) → only one pod can mount it. RollingUpdate deadlocks (new pod can't
# attach while old holds it). Recreate terminates old before starting new.
strategy:
type: Recreate
# Disable all bundled subcharts — reuse the existing in-cluster postgres, no redis/memcached.
postgresql:
enabled: false
postgresql-ha:
enabled: false
redis-cluster:
enabled: false
redis:
enabled: false
# chart 12.x replaced redis with valkey — disable both, gitea uses embedded queue/cache/session
valkey-cluster:
enabled: false
valkey:
enabled: false
persistence:
enabled: true
size: 5Gi
storageClass: sbs-default-retain
resources:
requests: { cpu: 100m, memory: 256Mi }
limits: { cpu: "1", memory: 2Gi } # 2Gi headroom: indexing the 876MB foxhunt pack OOMed at 512Mi
service:
http: { type: ClusterIP, port: 3000 }
ssh: { type: ClusterIP, port: 22 }
gitea:
admin:
existingSecret: gitea-admin
config:
actions:
ENABLED: "false" # Argo does CI/CD; Gitea Actions off (subchart removed in chart 12.x)
server:
ROOT_URL: https://git.fxhnt.ai/
DOMAIN: git.fxhnt.ai
SSH_DOMAIN: git.fxhnt.ai
SSH_PORT: "22" # clean git@git.fxhnt.ai clone URLs; :2222 retired (Task 5).
DISABLE_SSH: "false"
database:
DB_TYPE: postgres
HOST: postgres.foxhunt.svc.cluster.local:5432
NAME: gitea
USER: gitea
service:
DISABLE_REGISTRATION: "true"
webhook:
# allow delivery to in-cluster targets (Argo Events eventsource on the 10.32.x service CIDR);
# default blocks private hosts. Argo auto-deploy webhook needs this.
ALLOWED_HOST_LIST: "private"
# No redis/memcached — use embedded adapters
cache:
ADAPTER: memory
session:
PROVIDER: db
queue:
TYPE: level
additionalConfigFromEnvs:
- name: GITEA__database__PASSWD
valueFrom:
secretKeyRef: { name: gitea-db, key: password }

120
infra/k8s/gitlab/grafana-values.yaml
View File

@@ -1,120 +0,0 @@
# Grafana — Foxhunt monitoring dashboard
# Accessed via Tailscale proxy on port 3000
replicas: 1
nodeSelector:
k8s.scaleway.com/pool-name: platform
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 200m
memory: 256Mi
persistence:
enabled: false
admin:
existingSecret: grafana-admin
userKey: admin-user
passwordKey: admin-password
envFromSecrets:
- name: db-credentials
grafana.ini:
server:
root_url: https://grafana.fxhnt.ai
unified_alerting:
enabled: true
alerting:
enabled: false
database:
type: postgres
host: postgres:5432
name: grafana
user: foxhunt
password: $__env{password}
ssl_mode: disable
datasources:
datasources.yaml:
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
url: http://prometheus-stack-kube-prom-prometheus.foxhunt.svc.cluster.local:9090
access: proxy
isDefault: true
- name: Alertmanager
type: alertmanager
uid: alertmanager
access: proxy
url: http://prometheus-stack-kube-prom-alertmanager.foxhunt.svc.cluster.local:9093
isDefault: false
jsonData:
implementation: prometheus
handleGrafanaManagedAlerts: true
- name: Loki
type: loki
access: proxy
url: http://loki.foxhunt.svc.cluster.local:3100
isDefault: false
jsonData:
derivedFields:
- datasourceUid: tempo
matcherRegex: '"trace_id":"(\w+)"'
name: TraceID
url: "$${__value.raw}"
- name: Tempo
type: tempo
uid: tempo
access: proxy
url: http://tempo.foxhunt.svc.cluster.local:3200
isDefault: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
nodeGraph:
enabled: true
serviceMap:
datasourceUid: prometheus
sidecar:
dashboards:
enabled: true
folder: /tmp/dashboards
folderAnnotation: grafana_folder
label: grafana_dashboard
labelValue: "1"
searchNamespace: foxhunt
provider:
foldersFromFilesStructure: true
datasources:
enabled: false
label: grafana_datasource
labelValue: ""
dashboardProviders:
dashboardproviders.yaml:
apiVersion: 1
providers:
- name: default
orgId: 1
folder: Foxhunt
type: file
disableDeletion: false
editable: true
options:
path: /var/lib/grafana/dashboards/default
dashboards:
default:
node-exporter:
gnetId: 1860
revision: 37
datasource: Prometheus

325
infra/k8s/gitlab/tailscale-proxy.yaml
View File

@@ -1,325 +0,0 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: tailscale-gitlab
namespace: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: tailscale-gitlab
namespace: foxhunt
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "get", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: tailscale-gitlab
namespace: foxhunt
subjects:
- kind: ServiceAccount
name: tailscale-gitlab
namespace: foxhunt
roleRef:
kind: Role
name: tailscale-gitlab
apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tailscale-gitlab-proxy
namespace: foxhunt
labels:
app.kubernetes.io/name: tailscale-gitlab-proxy
app.kubernetes.io/part-of: foxhunt
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: tailscale-gitlab-proxy
template:
metadata:
labels:
app.kubernetes.io/name: tailscale-gitlab-proxy
spec:
serviceAccountName: tailscale-gitlab
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: nginx
image: nginx:alpine
ports:
- containerPort: 80
- containerPort: 443
volumeMounts:
- name: nginx-conf
mountPath: /etc/nginx/conf.d
- name: tls-certs
mountPath: /etc/nginx/certs
readOnly: true
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
- name: ssh-proxy
image: alpine/socat:latest
args:
- "TCP-LISTEN:22,fork,reuseaddr,sndbuf=1048576,rcvbuf=1048576"
- "TCP:gitea-sshd.foxhunt.svc.cluster.local:22,sndbuf=1048576,rcvbuf=1048576"
ports:
- containerPort: 22
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
- name: imap-proxy
image: alpine/socat:latest
args:
- "TCP-LISTEN:993,fork,reuseaddr"
- "TCP:stalwart.foxhunt.svc.cluster.local:993"
ports:
- containerPort: 993
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
cpu: 50m
memory: 32Mi
- name: tailscale
image: ghcr.io/tailscale/tailscale:latest
env:
- name: TS_AUTHKEY
valueFrom:
secretKeyRef:
name: tailscale-auth
key: TS_AUTHKEY
- name: TS_KUBE_SECRET
value: tailscale-gitlab-state
- name: TS_USERSPACE
value: "false"
- name: TS_HOSTNAME
value: "foxhunt-gitlab"
- name: TS_ACCEPT_DNS
value: "false"
- name: TS_EXTRA_ARGS
value: ""
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumes:
- name: nginx-conf
configMap:
name: tailscale-gitlab-nginx
- name: tls-certs
secret:
secretName: gitlab-tls-cert
---
apiVersion: v1
kind: ConfigMap
metadata:
name: tailscale-gitlab-nginx
namespace: foxhunt
data:
default.conf: |
# HTTP → HTTPS redirect for all subdomains
server {
listen 80;
server_name *.fxhnt.ai;
return 301 https://$host$request_uri;
}
# Gitea — git.fxhnt.ai (replaced GitLab, Phase 2B cutover)
server {
listen 443 ssl;
server_name git.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 0;
location / {
proxy_pass http://gitea-web.foxhunt.svc.cluster.local:3000;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_buffering off;
proxy_request_buffering off;
proxy_http_version 1.1;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_connect_timeout 300s;
}
}
# Grafana — grafana.fxhnt.ai
server {
listen 443 ssl;
server_name grafana.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
location / {
proxy_pass http://grafana.foxhunt.svc.cluster.local:80;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
# fxhnt cockpit — dashboard.fxhnt.ai
# Proxies to the cockpit's Tailscale node (peer-to-peer over the tailnet) rather than the cluster Service:
# the pod CIDR 100.64.0.0/15 overlaps Tailscale CGNAT, so this kernel-mode proxy can't reach platform-pool
# pods via ClusterIP, but it CAN reach another tailnet node. IP is stable while the cockpit's TS state
# secret (fxhnt-dashboard-ts-state) persists. (Was web-dashboard.foxhunt.svc — replaced by the fxhnt cockpit.)
server {
listen 443 ssl;
server_name dashboard.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
location / {
proxy_pass http://100.81.150.18:80; # fxhnt-dashboard tailnet node (TCP:80 -> cockpit :8080)
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
# API Gateway (gRPC-Web + gRPC) — api.fxhnt.ai
# Browser dashboard uses gRPC-web (HTTP/1.1 POST), so proxy_pass not grpc_pass
server {
listen 443 ssl;
http2 on;
server_name api.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
location / {
proxy_pass http://api.foxhunt.svc.cluster.local:50051;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_http_version 1.1;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_connect_timeout 30s;
}
}
# Stalwart Admin — mail.fxhnt.ai
server {
listen 443 ssl;
server_name mail.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
location / {
proxy_pass https://stalwart.foxhunt.svc.cluster.local:443;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_http_version 1.1;
proxy_ssl_verify off;
}
}
# Mattermost — chat.fxhnt.ai
server {
listen 443 ssl;
server_name chat.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 50m;
location / {
proxy_pass http://mattermost.foxhunt.svc.cluster.local:8065;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
}
}
# MinIO S3 API — minio.fxhnt.ai
server {
listen 443 ssl;
server_name minio.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 0;
location / {
proxy_pass http://minio.foxhunt.svc.cluster.local:9000;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_buffering off;
proxy_request_buffering off;
proxy_http_version 1.1;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_connect_timeout 300s;
# MinIO uses self-signed TLS internally
proxy_ssl_verify off;
}
}

34
infra/k8s/jobs/migrate.yaml
View File

@@ -1,34 +0,0 @@
apiVersion: batch/v1
kind: Job
metadata:
name: foxhunt-migrate
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
app.kubernetes.io/component: migration
spec:
backoffLimit: 3
template:
spec:
restartPolicy: Never
imagePullSecrets:
- name: gitlab-registry
containers:
- name: migrate
image: rg.fr-par.scw.cloud/foxhunt/api:latest
command: ["sqlx", "migrate", "run"]
env:
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: db-credentials
key: password
- name: DATABASE_URL
value: "postgresql://foxhunt:$(DB_PASSWORD)@postgres:5432/foxhunt"
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "256Mi"
cpu: "200m"

121
infra/k8s/mattermost/mattermost.yaml
View File

@@ -1,121 +0,0 @@
# Mattermost Team Edition — self-hosted Slack alternative for foxhunt.
# Runs on platform nodes. PostgreSQL is the shared foxhunt instance.
# Access: https://chat.fxhnt.ai (Tailscale only)
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mattermost-data
namespace: foxhunt
labels:
app.kubernetes.io/name: mattermost
app.kubernetes.io/part-of: foxhunt
spec:
accessModes: [ReadWriteOnce]
storageClassName: scw-bssd
resources:
requests:
storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: mattermost
namespace: foxhunt
labels:
app.kubernetes.io/name: mattermost
app.kubernetes.io/part-of: foxhunt
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: mattermost
strategy:
type: Recreate # RWO PVC
template:
metadata:
labels:
app.kubernetes.io/name: mattermost
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: mattermost
image: mattermost/mattermost-team-edition:10.5
ports:
- containerPort: 8065
name: http
env:
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: db-credentials
key: password
- name: MM_SQLSETTINGS_DRIVERNAME
value: postgres
- name: MM_SQLSETTINGS_DATASOURCE
value: "postgres://foxhunt:$(DB_PASSWORD)@postgres.foxhunt.svc.cluster.local:5432/mattermost?sslmode=disable"
- name: MM_SERVICESETTINGS_SITEURL
value: "https://chat.fxhnt.ai"
- name: MM_SERVICESETTINGS_LISTENADDRESS
value: ":8065"
- name: MM_SERVICESETTINGS_ENABLEINCOMINGWEBHOOKS
value: "true"
- name: MM_SERVICESETTINGS_ENABLEPOSTUSERNAMEOVERRIDE
value: "true"
- name: MM_SERVICESETTINGS_ENABLEPOSTICONOVERRIDE
value: "true"
- name: MM_LOGSETTINGS_ENABLECONSOLE
value: "true"
- name: MM_LOGSETTINGS_CONSOLELEVEL
value: INFO
- name: MM_LOGSETTINGS_ENABLEFILE
value: "false"
- name: MM_FILESETTINGS_DRIVERNAME
value: local
- name: MM_FILESETTINGS_DIRECTORY
value: /mattermost/data
# Email via Stalwart — uncomment after Stalwart is deployed:
# - name: MM_EMAILSETTINGS_SMTPSERVER
# value: stalwart.foxhunt.svc.cluster.local
# - name: MM_EMAILSETTINGS_SMTPPORT
# value: "25"
# - name: MM_EMAILSETTINGS_SENDEMAILNOTIFICATIONS
# value: "true"
volumeMounts:
- name: mattermost-data
mountPath: /mattermost/data
readinessProbe:
httpGet:
path: /api/v4/system/ping
port: 8065
initialDelaySeconds: 15
periodSeconds: 10
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
volumes:
- name: mattermost-data
persistentVolumeClaim:
claimName: mattermost-data
---
apiVersion: v1
kind: Service
metadata:
name: mattermost
namespace: foxhunt
labels:
app.kubernetes.io/name: mattermost
app.kubernetes.io/part-of: foxhunt
spec:
selector:
app.kubernetes.io/name: mattermost
ports:
- port: 8065
targetPort: 8065
name: http

4
infra/k8s/minio/kustomization.yaml
View File

@@ -1,4 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- minio.yaml

188
infra/k8s/minio/minio.yaml
View File

@@ -1,188 +0,0 @@
# MinIO — in-cluster S3 for compiled binaries and trained models
# Replaces Scaleway public S3 to keep proprietary code/IP in-cluster
# Plain HTTP only — internal cluster traffic, no TLS needed
apiVersion: v1
kind: Secret
metadata:
name: minio-credentials
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
root-user: foxhunt-minio
root-password: CHANGE_ME_BEFORE_DEPLOY
# rclone-compatible keys (same values, different key names for initContainers)
access-key: foxhunt-minio
secret-key: CHANGE_ME_BEFORE_DEPLOY
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: minio-data-new
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
accessModes:
- ReadWriteOnce
storageClassName: scw-bssd
resources:
requests:
storage: 150Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
replicas: 1
strategy:
type: Recreate # single-node MinIO — no rolling update
selector:
matchLabels:
app.kubernetes.io/name: minio
template:
metadata:
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: minio
image: minio/minio:latest
args: ["server", "/data", "--console-address", ":9001"]
ports:
- containerPort: 9000
name: api
- containerPort: 9001
name: console
env:
- name: MINIO_ROOT_USER
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-user
- name: MINIO_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-password
volumeMounts:
- name: data
mountPath: /data
readinessProbe:
httpGet:
path: /minio/health/live
port: 9000
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /minio/health/live
port: 9000
initialDelaySeconds: 10
periodSeconds: 15
failureThreshold: 5
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: "1"
memory: 2Gi
volumes:
- name: data
persistentVolumeClaim:
claimName: minio-data-new
---
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
selector:
app.kubernetes.io/name: minio
ports:
- port: 9000
targetPort: 9000
name: api
- port: 9001
targetPort: 9001
name: console
---
# Create buckets on first deploy
apiVersion: batch/v1
kind: Job
metadata:
name: minio-init-buckets
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
backoffLimit: 5
ttlSecondsAfterFinished: 300
template:
metadata:
labels:
app.kubernetes.io/name: minio-init
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
restartPolicy: OnFailure
containers:
- name: mc
image: minio/mc:latest
command: ["/bin/sh", "-c"]
args:
- |
set -e
echo "Waiting for MinIO to be ready..."
until mc alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do
echo "MinIO not ready, retrying in 3s..."
sleep 3
done
echo "Creating buckets..."
mc mb --ignore-existing foxhunt/foxhunt-binaries
mc mb --ignore-existing foxhunt/foxhunt-models
mc mb --ignore-existing foxhunt/foxhunt-training-results
mc mb --ignore-existing foxhunt/argo-logs
mc mb --ignore-existing foxhunt/foxhunt-sccache
mc mb --ignore-existing foxhunt/foxhunt-training-data
# Plan 5 Task 3 (A.4.1): nsys profile artefacts bucket.
# Stores .nsys-rep files keyed by commit SHA under
# profiles/<short-sha>/profile-seed*-fold*-*.nsys-rep.
mc mb --ignore-existing foxhunt/foxhunt-training-artifacts
echo "Buckets created:"
mc ls foxhunt/
env:
- name: MINIO_ROOT_USER
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-user
- name: MINIO_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-password
resources:
requests:
cpu: 50m
memory: 32Mi
limits:
cpu: 200m
memory: 64Mi

104
infra/k8s/monitoring/APPLY-ORDER.md
View File

@@ -1,104 +0,0 @@
# Cockpit + NAT Gateway — Apply Order
## Prerequisites
- `scw` CLI authenticated
- `terragrunt` installed
- `kubectl` access to foxhunt cluster
- `helm` installed
## Phase 1: Cockpit (metrics backend)
1. Apply Cockpit Terraform module:
```bash
cd infra/live/production/cockpit
terragrunt apply
```
2. Note the outputs:
```bash
terragrunt output metrics_push_url
terragrunt output -raw push_token
terragrunt output grafana_url
terragrunt output -raw grafana_password
```
3. Create the K8s secret for the push token:
```bash
kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
kubectl -n monitoring create secret generic cockpit-push-token \
--from-literal=token=$(cd infra/live/production/cockpit && terragrunt output -raw push_token)
```
4. Update placeholders in `infra/k8s/monitoring/alloy-values.yaml`:
- Replace `COCKPIT_METRICS_PUSH_URL` with the `metrics_push_url` output
5. Deploy DCGM Exporter:
```bash
kubectl apply -f infra/k8s/monitoring/dcgm-exporter.yaml
```
6. Deploy Grafana Alloy (k8s-monitoring):
```bash
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
helm install k8s-monitoring grafana/k8s-monitoring \
--namespace monitoring \
-f infra/k8s/monitoring/alloy-values.yaml
```
7. Verify metrics flow:
- Open Cockpit Grafana (URL from step 2)
- Login with foxhunt-admin / (password from step 2)
- Check "Kubernetes / Compute Resources / Cluster" dashboard
## Phase 2: GitLab Prometheus remote_write
1. Update placeholders in `infra/k8s/gitlab/values.yaml`:
- Replace `COCKPIT_METRICS_PUSH_URL` and `COCKPIT_PUSH_TOKEN`
2. Upgrade GitLab Helm release:
```bash
helm upgrade gitlab gitlab/gitlab \
--namespace foxhunt \
-f infra/k8s/gitlab/values.yaml
```
## Phase 3: NAT Gateway
1. Apply Public Gateway module:
```bash
cd infra/live/production/public-gateway
terragrunt apply
```
This creates the gateway and attaches it to the private network.
Existing nodes still have public IPs — nothing breaks yet.
2. Apply Kapsule pool changes (disables public IPs):
```bash
cd infra/live/production/kapsule
terragrunt apply
```
**WARNING**: This may recreate nodes. For zero-downtime:
3. Zero-downtime node migration for services pool:
```bash
# Temporarily allow 2 nodes
scw k8s pool update <services-pool-id> max-size=2 size=2
# Wait for new node (private IP) to be Ready
kubectl get nodes -w
# Drain old node
kubectl drain <old-node> --ignore-daemonsets --delete-emptydir-data
# Scale back
scw k8s pool update <services-pool-id> max-size=1 size=1
```
4. Repeat for gitlab pool (if not autoscaling, same process).
5. Autoscaling pools (ci-training, gpu-dev):
- No action needed — next scale-up automatically uses private IP.
6. Verify Tailscale still works:
```bash
tailscale status # should show foxhunt-kapsule and foxhunt-gitlab nodes
curl -k https://git.fxhnt.ai # should still resolve via Tailscale
```

88
infra/k8s/monitoring/alloy-values.yaml
View File

@@ -1,88 +0,0 @@
# Grafana k8s-monitoring Helm chart values (v1.x format)
# Chart: grafana/k8s-monitoring --version 1.6.50
# Deploys: Alloy (DaemonSet) + node-exporter + kube-state-metrics
#
# Install:
# helm repo add grafana https://grafana.github.io/helm-charts
# PUSH_TOKEN=$(cd infra/live/production/cockpit && terragrunt output -raw push_token)
# helm install k8s-monitoring grafana/k8s-monitoring \
# --namespace monitoring --version 1.6.50 \
# -f infra/k8s/monitoring/alloy-values.yaml \
# --set "externalServices.prometheus.extraHeaders.X-Token=${PUSH_TOKEN}"
cluster:
name: foxhunt
externalServices:
prometheus:
host: "https://d8cd4c5d-6d3b-4258-9f92-2ad0eaa36ba0.metrics.cockpit.fr-par.scw.cloud"
writeEndpoint: "/api/v1/push"
authMode: none
# X-Token header passed via --set to keep secret out of git
# Loki host required by chart validation even with logs disabled
loki:
host: "https://noop.localhost"
authMode: none
metrics:
enabled: true
cost:
enabled: false
node-exporter:
enabled: true
kube-state-metrics:
enabled: true
kubelet:
enabled: true
cadvisor:
enabled: true
apiserver:
enabled: true
# Alloy DaemonSet — runs on every node
alloy-metrics:
alloy:
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 200m
memory: 256Mi
# node-exporter — host metrics (CPU, memory, disk, network)
prometheus-node-exporter:
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
# kube-state-metrics — K8s object state
kube-state-metrics:
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 100m
memory: 128Mi
# Scheduled on gitlab node (GP1-XS 16GB) — services (DEV1-M 4GB) too small
# Disable components we don't need
logs:
enabled: false
cluster_events:
enabled: false
traces:
enabled: false
opencost:
enabled: false
# Disable log/event collectors
alloy-logs:
enabled: false
alloy-events:
enabled: false

809
infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json
View File

@@ -1,809 +0,0 @@
{
"description": "Loki-powered CI pipeline log viewer \u2014 test gate, clippy, builds, errors",
"editable": true,
"graphTooltip": 2,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"title": "Pipeline Overview",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "red",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"ERROR\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Errors",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "yellow",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 1
},
"id": 3,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"WARN\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Warnings",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"noValue": "0"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 1
},
"id": 4,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Total Log Lines",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisLabel": "lines/interval",
"drawStyle": "bars",
"fillOpacity": 80,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "ERROR"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "WARN"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "INFO"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "DEBUG"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 1
},
"id": 5,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))",
"legendFormat": "{{level}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Level",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"drawStyle": "bars",
"fillOpacity": 60,
"lineWidth": 0,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 5
},
"id": 6,
"options": {
"legend": {
"displayMode": "list",
"placement": "right"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (container) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))",
"legendFormat": "{{container}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Container",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"id": 10,
"title": "Full Pipeline Log Stream",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 14,
"w": 24,
"x": 0,
"y": 10
},
"id": 11,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=~\"$level\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}",
"queryType": "range",
"refId": "A"
}
],
"title": "Pipeline Logs \u2014 $pipeline",
"type": "logs"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 24
},
"id": 20,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 25
},
"id": 21,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"main\"} |~ \"(clippy|cargo test|warning\\\\[|error\\\\[|FAILED|test result|running \\\\d+ test|failures:)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Clippy + Test Output",
"type": "logs"
}
],
"title": "Test Gate (clippy + cargo test)",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 25
},
"id": 30,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 26
},
"id": 31,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"redis\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Redis Sidecar Logs",
"type": "logs"
}
],
"title": "Redis Sidecar",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 26
},
"id": 40,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"id": 41,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "All Errors & Failures",
"type": "logs"
}
],
"title": "Errors & Failures",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"id": 50,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 28
},
"id": 51,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy|compile-and-train)-.*\", container=\"main\"} |~ \"(Compiling|Finished|warning|error|Downloading|Building|cargo build|rclone)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Compile Logs",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 28
},
"id": 52,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy)-.*\", container=\"main\"} |~ \"(kubectl|apply|rollout|deploy|configured|unchanged|created|image|kaniko)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Deploy & Argo Template Logs",
"type": "logs"
}
],
"title": "Build & Deploy",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 28
},
"id": 60,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 29
},
"id": 61,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"gpu-test-.*\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "GPU Test Logs (DQN/PPO/TFT + Supervised)",
"type": "logs"
}
],
"title": "GPU Test Pipeline",
"type": "row"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"foxhunt",
"ci",
"loki",
"logs"
],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "ci-pipeline",
"value": "ci-pipeline"
},
"label": "Pipeline",
"name": "pipeline",
"options": [],
"query": "ci-pipeline,compile-and-deploy,compile-and-train,gpu-test,gpu-diag,gpu-walltime,build-ci-image,build-training-runtime,evaluate",
"type": "custom"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"includeAll": true,
"label": "Level",
"multi": true,
"name": "level",
"options": [
{
"selected": true,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "ERROR",
"value": "ERROR"
},
{
"selected": false,
"text": "WARN",
"value": "WARN"
},
{
"selected": false,
"text": "INFO",
"value": "INFO"
},
{
"selected": false,
"text": "DEBUG",
"value": "DEBUG"
},
{
"selected": false,
"text": "TRACE",
"value": "TRACE"
}
],
"query": "ERROR,WARN,INFO,DEBUG,TRACE",
"type": "custom"
},
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"label": "Search",
"name": "search",
"type": "textbox"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "CI Pipeline Logs",
"uid": "foxhunt-ci-logs",
"__inputs": [
{
"name": "DS_PROMETHEUS",
"type": "datasource",
"pluginId": "prometheus"
},
{
"name": "DS_LOKI",
"type": "datasource",
"pluginId": "loki"
},
{
"name": "DS_TEMPO",
"type": "datasource",
"pluginId": "${DS_TEMPO}"
}
]
}

2762
infra/k8s/monitoring/dashboards/foxhunt-cockpit.json
View File

File diff suppressed because it is too large Load Diff

2206
infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json
View File

File diff suppressed because it is too large Load Diff

1621
infra/k8s/monitoring/dashboards/foxhunt-observability.json
View File

File diff suppressed because it is too large Load Diff

2167
infra/k8s/monitoring/dashboards/foxhunt-trading.json
View File

File diff suppressed because it is too large Load Diff

3348
infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json
View File

File diff suppressed because it is too large Load Diff

835
infra/k8s/monitoring/dashboards/foxhunt-training-logs.json
View File

@@ -1,835 +0,0 @@
{
"description": "Loki-powered training log viewer \u2014 hyperopt trials, epoch progress, GPU status, errors",
"editable": true,
"graphTooltip": 2,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"title": "Training Overview",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "red",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"ERROR\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Errors",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "yellow",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 1
},
"id": 3,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"WARN\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Warnings",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"noValue": "0"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 1
},
"id": 4,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Total Log Lines",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisLabel": "lines/interval",
"drawStyle": "bars",
"fillOpacity": 80,
"lineWidth": 0,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "ERROR"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "WARN"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "INFO"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 1
},
"id": 5,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))",
"legendFormat": "{{level}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Level",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"drawStyle": "bars",
"fillOpacity": 60,
"lineWidth": 0,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 5
},
"id": 6,
"options": {
"legend": {
"displayMode": "list",
"placement": "right"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (pod) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))",
"legendFormat": "{{pod}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Pod",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"id": 10,
"title": "Full Training Log Stream",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 14,
"w": 24,
"x": 0,
"y": 10
},
"id": 11,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\", level=~\"$level\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}",
"queryType": "range",
"refId": "A"
}
],
"title": "Training Logs \u2014 $job",
"type": "logs"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 24
},
"id": 20,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 25
},
"id": 21,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(epoch|loss|reward|sharpe|accuracy|val_loss|train_loss|best_trial|trial|fold|window|learning_rate|lr=|MaxDD|win.?rate|drawdown)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Epoch / Loss / Reward / Sharpe",
"type": "logs"
}
],
"title": "Epoch Progress & Loss",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 25
},
"id": 30,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 26
},
"id": 31,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"(hyperopt|train)-.*\"} |~ \"(?i)(trial|best|cost|param|learning_rate|hidden_dim|batch_size|buffer_size|gamma|epsilon|clip|PSO|iteration|objective|optimal)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Hyperopt Trial Results",
"type": "logs"
}
],
"title": "Hyperopt Trials",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 26
},
"id": 40,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"id": 41,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(GPU|CUDA|VRAM|cuda|gpu|vram|OOM|out.of.memory|nvrtc|tensor.core|BF16|bf16|SM|occupancy|kernel|device|nvidia)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "GPU / CUDA / VRAM Logs",
"type": "logs"
}
],
"title": "GPU / CUDA / VRAM",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"id": 50,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 28
},
"id": 51,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(walk.?forward|fold|backtest|window|train_start|train_end|val_start|val_end|test_start|test_end|embargo|purge|bars|features)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Walk-Forward / Fold / Backtest",
"type": "logs"
}
],
"title": "Walk-Forward Windows",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 28
},
"id": 60,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 29
},
"id": 61,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked|OOM|NaN|inf|diverge)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "All Training Errors & Failures",
"type": "logs"
}
],
"title": "Errors & Failures",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 29
},
"id": 70,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 30
},
"id": 71,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(loading|loaded|dbn|ofi|features|bars|MBP.?10|OHLCV|symbol|normalize|norm_stats|state_dim|rclone|S3|download)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Data Loading / OFI / DBN / Features",
"type": "logs"
}
],
"title": "Data Loading & OFI",
"type": "row"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"foxhunt",
"training",
"loki",
"logs",
"gpu"
],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "All Training",
"value": "(train|hyperopt|compile-and-train)"
},
"label": "Job Type",
"name": "job",
"options": [
{
"selected": true,
"text": "All Training",
"value": "(train|hyperopt|compile-and-train)"
},
{
"selected": false,
"text": "DQN",
"value": "train-dqn"
},
{
"selected": false,
"text": "PPO",
"value": "train-ppo"
},
{
"selected": false,
"text": "Supervised",
"value": "train-supervised"
},
{
"selected": false,
"text": "Hyperopt",
"value": "hyperopt"
},
{
"selected": false,
"text": "Compile+Train",
"value": "compile-and-train"
}
],
"query": "All Training : (train|hyperopt|compile-and-train), DQN : train-dqn, PPO : train-ppo, Supervised : train-supervised, Hyperopt : hyperopt, Compile+Train : compile-and-train",
"type": "custom"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"includeAll": true,
"label": "Level",
"multi": true,
"name": "level",
"options": [
{
"selected": true,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "ERROR",
"value": "ERROR"
},
{
"selected": false,
"text": "WARN",
"value": "WARN"
},
{
"selected": false,
"text": "INFO",
"value": "INFO"
},
{
"selected": false,
"text": "DEBUG",
"value": "DEBUG"
},
{
"selected": false,
"text": "TRACE",
"value": "TRACE"
}
],
"query": "ERROR,WARN,INFO,DEBUG,TRACE",
"type": "custom"
},
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"label": "Search",
"name": "search",
"type": "textbox"
}
]
},
"time": {
"from": "now-12h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Training Pipeline Logs",
"uid": "foxhunt-training-logs",
"__inputs": [
{
"name": "DS_PROMETHEUS",
"type": "datasource",
"pluginId": "prometheus"
},
{
"name": "DS_LOKI",
"type": "datasource",
"pluginId": "loki"
},
{
"name": "DS_TEMPO",
"type": "datasource",
"pluginId": "${DS_TEMPO}"
}
]
}

4462
infra/k8s/monitoring/dashboards/foxhunt-training.json
View File

File diff suppressed because it is too large Load Diff

212
infra/k8s/monitoring/dashboards/import.sh
View File

@@ -1,212 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Import all Grafana dashboards via API (Postgres-backed, no ConfigMaps).
#
# Dashboards use ${DS_PROMETHEUS}, ${DS_LOKI}, ${DS_TEMPO} variables.
# This script auto-detects datasource UIDs, resolves variables, and
# imports via the Grafana API. Dashboards are stored in Postgres.
#
# Required environment variables:
# GRAFANA_PASS - Password for admin user
#
# Optional overrides:
# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai)
# GRAFANA_USER - Grafana user (default: admin)
# GRAFANA_PROM_UID - Override Prometheus datasource UID
# GRAFANA_LOKI_UID - Override Loki datasource UID
# GRAFANA_TEMPO_UID - Override Tempo datasource UID
#
# Usage:
# GRAFANA_PASS="xxx" ./import.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GRAFANA_USER="${GRAFANA_USER:-admin}"
GRAFANA_URL="${GRAFANA_URL:-https://grafana.fxhnt.ai}"
GRAFANA_URL="${GRAFANA_URL%/}"
if [[ -z "${GRAFANA_PASS:-}" ]]; then
echo "ERROR: GRAFANA_PASS is not set" >&2
exit 1
fi
# --- Datasource auto-detection ---
detect_datasource_uid() {
local ds_type="$1"
curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/datasources" 2>/dev/null | \
python3 -c "
import sys, json
for ds in json.load(sys.stdin):
if ds.get('type') == '${ds_type}':
print(ds['uid'])
break
" 2>/dev/null || true
}
echo "Detecting datasource UIDs..."
PROM_UID="${GRAFANA_PROM_UID:-$(detect_datasource_uid prometheus)}"
LOKI_UID="${GRAFANA_LOKI_UID:-$(detect_datasource_uid loki)}"
TEMPO_UID="${GRAFANA_TEMPO_UID:-$(detect_datasource_uid tempo)}"
echo " Prometheus: ${PROM_UID:-NOT FOUND}"
echo " Loki: ${LOKI_UID:-NOT FOUND}"
echo " Tempo: ${TEMPO_UID:-NOT FOUND}"
if [[ -z "$PROM_UID" ]]; then
echo "ERROR: Could not detect Prometheus datasource. Set GRAFANA_PROM_UID." >&2
exit 1
fi
# --- Resolve datasource variables ---
resolve_dashboard() {
local file="$1"
python3 -c "
import sys, json
with open('${file}') as f:
d = json.load(f)
d.pop('__inputs', None)
d.pop('__requires', None)
raw = json.dumps(d)
raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}')
raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}')
raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}')
print(raw)
"
}
# --- Ensure folders exist ---
ensure_folder() {
local title="$1"
local uid
uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/folders" 2>/dev/null | \
python3 -c "
import sys, json
for f in json.load(sys.stdin):
if f['title'] == '${title}':
print(f['uid'])
break
" 2>/dev/null || true)
if [[ -z "$uid" ]]; then
uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" \
-H "Content-Type: application/json" \
-d "{\"title\": \"${title}\"}" \
"${GRAFANA_URL}/api/folders" 2>/dev/null | \
python3 -c "import sys,json; print(json.load(sys.stdin)['uid'])" 2>/dev/null)
echo " Created folder: ${title} (${uid})"
fi
echo "$uid"
}
echo ""
echo "Ensuring folders..."
FOLDER_CI=$(ensure_folder "CI")
FOLDER_INFRA=$(ensure_folder "Infrastructure")
FOLDER_OPS=$(ensure_folder "Operations")
FOLDER_TRADING=$(ensure_folder "Trading")
FOLDER_TRAINING=$(ensure_folder "Training")
FOLDER_SERVICES=$(ensure_folder "Services")
FOLDER_PLATFORM=$(ensure_folder "Platform")
# --- Import a single dashboard to a folder ---
import_dashboard() {
local file="$1"
local folder_uid="$2"
local name
name="$(basename "$file" .json)"
local resolved
resolved=$(resolve_dashboard "$file")
local response
response=$(curl -s -w "\n%{http_code}" \
-u "${GRAFANA_USER}:${GRAFANA_PASS}" \
-H "Content-Type: application/json" \
-d "{\"dashboard\": ${resolved}, \"folderUid\": \"${folder_uid}\", \"overwrite\": true}" \
"${GRAFANA_URL}/api/dashboards/db")
local http_code
http_code=$(echo "$response" | tail -1)
if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then
echo " OK ${name}"
else
local body
body=$(echo "$response" | sed '$d')
echo " FAIL ${name} (${http_code}): ${body}" >&2
return 1
fi
}
# --- Dashboard → Folder mapping ---
declare -A DASHBOARD_FOLDERS=(
# Original dashboards
[foxhunt-cockpit]="${FOLDER_OPS}"
[foxhunt-trading]="${FOLDER_TRADING}"
[foxhunt-training]="${FOLDER_TRAINING}"
[foxhunt-infrastructure]="${FOLDER_INFRA}"
[foxhunt-observability]="${FOLDER_INFRA}"
[foxhunt-ci-logs]="${FOLDER_CI}"
[foxhunt-training-logs]="${FOLDER_TRAINING}"
[foxhunt-training-deep-dive]="${FOLDER_TRAINING}"
# Application services
[svc-api-gateway]="${FOLDER_SERVICES}"
[svc-trading-service]="${FOLDER_SERVICES}"
[svc-backtesting]="${FOLDER_SERVICES}"
[svc-data-acquisition]="${FOLDER_SERVICES}"
[svc-ml-training]="${FOLDER_SERVICES}"
[svc-trading-agent]="${FOLDER_SERVICES}"
[svc-web-gateway]="${FOLDER_SERVICES}"
# Platform services
[platform-prometheus]="${FOLDER_PLATFORM}"
[platform-grafana]="${FOLDER_PLATFORM}"
[platform-loki]="${FOLDER_PLATFORM}"
[platform-tempo]="${FOLDER_PLATFORM}"
[platform-redis]="${FOLDER_PLATFORM}"
[platform-postgres]="${FOLDER_PLATFORM}"
[platform-argo]="${FOLDER_PLATFORM}"
[platform-questdb]="${FOLDER_PLATFORM}"
[platform-costs]="${FOLDER_PLATFORM}"
[platform-scaleway]="${FOLDER_PLATFORM}"
)
# --- Deploy all dashboards ---
echo ""
echo "Importing dashboards..."
failed=0
total=0
for dashboard in "${SCRIPT_DIR}"/*.json; do
[[ -f "$dashboard" ]] || continue
name="$(basename "$dashboard" .json)"
folder_uid="${DASHBOARD_FOLDERS[$name]:-}"
if [[ -z "$folder_uid" ]]; then
echo " SKIP ${name} (no folder mapping)"
continue
fi
total=$((total + 1))
if ! import_dashboard "$dashboard" "$folder_uid"; then
failed=$((failed + 1))
fi
done
echo ""
if [[ $failed -eq 0 ]]; then
echo "All ${total} dashboards deployed successfully."
else
echo "ERROR: ${failed}/${total} dashboard(s) failed." >&2
exit 1
fi

1632
infra/k8s/monitoring/dashboards/platform-argo.json
View File

File diff suppressed because it is too large Load Diff

1737
infra/k8s/monitoring/dashboards/platform-costs.json
View File

File diff suppressed because it is too large Load Diff

802
infra/k8s/monitoring/dashboards/platform-grafana.json
View File

@@ -1,802 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Grafana Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"grafana.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"grafana.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"grafana.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 25
},
"id": 16,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"grafana"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "Grafana",
"uid": "plat-grafana"
}

1532
infra/k8s/monitoring/dashboards/platform-loki.json
View File

File diff suppressed because it is too large Load Diff

1146
infra/k8s/monitoring/dashboards/platform-postgres.json
View File

File diff suppressed because it is too large Load Diff

946
infra/k8s/monitoring/dashboards/platform-prometheus.json
View File

@@ -1,946 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Prometheus Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "count(up{job!=\"\"} == 1)",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Targets Up",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "count(up{job!=\"\"} == 0) or vector(0)",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Targets Down",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_head_series",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Head Series",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Ingestion Rate",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_wal_storage_size_bytes",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "WAL Size",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_head_chunks",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Head Chunks",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_compactions_total",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Compactions",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_storage_blocks_bytes",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Storage Size",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Scrape Performance",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "scrape_duration_seconds",
"legendFormat": "{{job}}",
"refId": "A"
}
],
"title": "Scrape Duration",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 15
},
"id": 16,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(prometheus_tsdb_head_samples_appended_total[$interval])",
"legendFormat": "samples/s",
"refId": "A"
}
],
"title": "Samples Ingested",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 17,
"panels": [],
"title": "Target Status",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 24
},
"id": 18,
"options": {
"showHeader": true,
"sortBy": []
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "up",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Target Health",
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 32
},
"id": 19,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 33
},
"id": 20,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 43
},
"id": 21,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"prometheus"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "Prometheus",
"uid": "plat-prometheus"
}

802
infra/k8s/monitoring/dashboards/platform-questdb.json
View File

@@ -1,802 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "QuestDB Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"questdb.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"questdb.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"questdb.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 25
},
"id": 16,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"questdb"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "QuestDB",
"uid": "plat-questdb"
}

1315
infra/k8s/monitoring/dashboards/platform-redis.json
View File

File diff suppressed because it is too large Load Diff

2669
infra/k8s/monitoring/dashboards/platform-scaleway.json
View File

File diff suppressed because it is too large Load Diff

802
infra/k8s/monitoring/dashboards/platform-tempo.json
View File

@@ -1,802 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Tempo Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 25
},
"id": 16,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"${DS_TEMPO}"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "Tempo",
"uid": "plat-${DS_TEMPO}"
}

963
infra/k8s/monitoring/dashboards/svc-api-gateway.json
View File

@@ -1,963 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Service Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"api.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"api.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "gRPC Metrics",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(grpc_server_handled_total{grpc_service=~\"foxhunt.*\", pod=~\"api.*\"}[$interval])",
"legendFormat": "{{grpc_method}}",
"refId": "A"
}
],
"title": "gRPC Request Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 15
},
"id": 16,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"api.*\"}[$interval]))",
"legendFormat": "p99",
"refId": "A"
}
],
"title": "gRPC Latency (p99)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 15
},
"id": 17,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(grpc_server_handled_total{grpc_code!=\"OK\", pod=~\"api.*\"}[$interval])",
"legendFormat": "{{grpc_code}}",
"refId": "A"
}
],
"title": "gRPC Errors",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 18,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 24
},
"id": 19,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 34
},
"id": 20,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"service",
"api"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "API Gateway",
"uid": "svc-api-gateway"
}

1075
infra/k8s/monitoring/dashboards/svc-backtesting.json
View File

File diff suppressed because it is too large Load Diff

1031
infra/k8s/monitoring/dashboards/svc-data-acquisition.json
View File

File diff suppressed because it is too large Load Diff

1026
infra/k8s/monitoring/dashboards/svc-ml-training.json
View File

File diff suppressed because it is too large Load Diff

1075
infra/k8s/monitoring/dashboards/svc-trading-agent.json
View File

File diff suppressed because it is too large Load Diff

1074
infra/k8s/monitoring/dashboards/svc-trading-service.json
View File

File diff suppressed because it is too large Load Diff

1022
infra/k8s/monitoring/dashboards/svc-web-gateway.json
View File

File diff suppressed because it is too large Load Diff

125
infra/k8s/monitoring/dcgm-exporter.yaml
View File

@@ -1,125 +0,0 @@
# NVIDIA DCGM Exporter — GPU metrics for Cockpit
# Runs on all GPU node pools (ci-training, gpu-dev)
# Alloy auto-discovers and scrapes these pods via annotations
---
apiVersion: v1
kind: Namespace
metadata:
name: monitoring
labels:
app.kubernetes.io/part-of: foxhunt
---
apiVersion: v1
kind: ConfigMap
metadata:
name: dcgm-counters
namespace: monitoring
data:
counters.csv: |
# Basic GPU metrics (no DCP/profiling — avoids SYS_ADMIN requirement)
dcgm_sm_clock, gauge, SM clock frequency (in MHz).
dcgm_memory_clock, gauge, Memory clock frequency (in MHz).
dcgm_memory_temp, gauge, Memory temperature (in C).
dcgm_gpu_temp, gauge, GPU temperature (in C).
dcgm_power_usage, gauge, Power draw (in W).
dcgm_total_energy_consumption, counter, Total energy consumption since boot (in mJ).
dcgm_gpu_utilization, gauge, GPU utilization (in %).
dcgm_mem_copy_utilization, gauge, Memory utilization (in %).
dcgm_enc_utilization, gauge, Encoder utilization (in %).
dcgm_dec_utilization, gauge, Decoder utilization (in %).
dcgm_xid_errors, gauge, Value of the last XID error encountered.
dcgm_fb_free, gauge, Framebuffer memory free (in MiB).
dcgm_fb_used, gauge, Framebuffer memory used (in MiB).
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: dcgm-exporter
namespace: monitoring
labels:
app.kubernetes.io/name: dcgm-exporter
app.kubernetes.io/part-of: foxhunt
spec:
selector:
matchLabels:
app.kubernetes.io/name: dcgm-exporter
template:
metadata:
labels:
app.kubernetes.io/name: dcgm-exporter
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9400"
prometheus.io/path: "/metrics"
spec:
# Only schedule on nodes that have GPUs
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: nvidia.com/gpu.present
operator: In
values:
- "true"
# NOTE: runtimeClassName: nvidia removed — Scaleway Kapsule GPU pools use
# the nvidia container runtime as the default containerd handler. The
# RuntimeClass CRD is only created by the full NVIDIA GPU Operator, not
# the device plugin alone, so requiring it prevents pod scheduling.
tolerations:
# Tolerate GPU taints so the exporter can run on GPU nodes
- operator: Exists
effect: NoSchedule
containers:
- name: dcgm-exporter
image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
ports:
- name: metrics
containerPort: 9400
args:
- "-f"
- "/etc/dcgm-exporter/custom/counters.csv"
env:
- name: DCGM_EXPORTER_KUBERNETES
value: "true"
resources:
requests:
cpu: 50m
memory: 256Mi
limits:
cpu: 200m
memory: 1Gi
securityContext:
runAsNonRoot: false
runAsUser: 0
volumeMounts:
- name: device-metrics
mountPath: /var/lib/dcgm
- name: custom-counters
mountPath: /etc/dcgm-exporter/custom
readOnly: true
volumes:
- name: device-metrics
emptyDir: {}
- name: custom-counters
configMap:
name: dcgm-counters
---
apiVersion: v1
kind: Service
metadata:
name: dcgm-exporter
namespace: monitoring
labels:
app.kubernetes.io/name: dcgm-exporter
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9400"
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: dcgm-exporter
ports:
- name: metrics
port: 9400
targetPort: 9400

57
infra/k8s/monitoring/grafana-datasources.yaml
View File

@@ -1,57 +0,0 @@
# Grafana datasource provisioning for Loki, Tempo, and QuestDB
# Apply via: kubectl apply -f grafana-datasources.yaml && kubectl rollout restart deploy/grafana -n foxhunt
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-datasources-extra
namespace: foxhunt
labels:
app.kubernetes.io/name: grafana
grafana_datasource: "1"
data:
loki-tempo.yaml: |
apiVersion: 1
datasources:
- name: Loki
type: loki
access: proxy
url: http://loki.foxhunt.svc.cluster.local:3100
isDefault: false
jsonData:
derivedFields:
- datasourceUid: tempo
matcherRegex: '"trace_id":"(\w+)"'
name: TraceID
url: "$${__value.raw}"
- name: Tempo
type: tempo
uid: tempo
access: proxy
url: http://tempo.foxhunt.svc.cluster.local:3200
isDefault: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
nodeGraph:
enabled: true
serviceMap:
datasourceUid: prometheus
questdb.yaml: |
apiVersion: 1
datasources:
- name: QuestDB
type: postgres
uid: questdb
access: proxy
url: questdb.foxhunt.svc.cluster.local:8812
user: admin
database: qdb
isDefault: false
jsonData:
sslmode: disable
maxOpenConns: 5
postgresVersion: 1200
timescaledb: false
secureJsonData:
password: quest

122
infra/k8s/monitoring/loki.yaml
View File

@@ -1,122 +0,0 @@
# Grafana Loki — lightweight log aggregation (single-binary mode)
# Runs on gitlab node pool alongside Grafana and Prometheus
apiVersion: v1
kind: ConfigMap
metadata:
name: loki-config
namespace: foxhunt
labels:
app.kubernetes.io/name: loki
data:
loki.yaml: |
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
retention_period: 168h # 7 days
max_query_series: 500
compactor:
working_directory: /loki/compactor
compaction_interval: 10m
retention_enabled: true
delete_request_store: filesystem
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: loki
namespace: foxhunt
labels:
app.kubernetes.io/name: loki
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: loki
template:
metadata:
labels:
app.kubernetes.io/name: loki
spec:
nodeSelector:
k8s.scaleway.com/pool-name: infra
containers:
- name: loki
image: grafana/loki:3.4.2
args: ["-config.file=/etc/loki/loki.yaml"]
ports:
- containerPort: 3100
name: http
- containerPort: 9096
name: grpc
volumeMounts:
- name: config
mountPath: /etc/loki
- name: data
mountPath: /loki
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
readinessProbe:
httpGet:
path: /ready
port: 3100
initialDelaySeconds: 10
periodSeconds: 10
livenessProbe:
httpGet:
path: /ready
port: 3100
initialDelaySeconds: 30
periodSeconds: 15
volumes:
- name: config
configMap:
name: loki-config
- name: data
emptyDir:
sizeLimit: 5Gi
---
apiVersion: v1
kind: Service
metadata:
name: loki
namespace: foxhunt
labels:
app.kubernetes.io/name: loki
spec:
selector:
app.kubernetes.io/name: loki
ports:
- port: 3100
targetPort: 3100
name: http
- port: 9096
targetPort: 9096
name: grpc

169
infra/k8s/monitoring/prometheus-rules-broker.yaml
View File

@@ -1,169 +0,0 @@
# Broker Gateway alert rules — discovered automatically by Prometheus Operator
# Apply: kubectl apply -f prometheus-rules-broker.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: foxhunt-broker-alerts
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
role: alert-rules
spec:
groups:
- name: broker_gateway_fix_session
interval: 30s
rules:
# FIX session disconnected >60s
- alert: BrokerGatewayFIXSessionDisconnected
expr: broker_gateway_fix_session_status == 0
for: 60s
labels:
severity: critical
service: broker_gateway_service
component: fix_session
impact: trading_halted
annotations:
summary: "FIX session {{ $labels.session_id }} DISCONNECTED"
description: "FIX session down >60s. All order routing halted."
action: "Check FIX engine logs, verify network to AMP Futures gateway"
# Order latency P95 > 100ms
- alert: BrokerGatewayHighOrderLatency
expr: |
histogram_quantile(0.95,
sum by(le, order_type) (rate(broker_gateway_order_latency_seconds_bucket[5m]))
) * 1000 > 100
for: 5m
labels:
severity: warning
service: broker_gateway_service
component: order_routing
impact: performance_degraded
annotations:
summary: "Order latency P95 > 100ms for {{ $labels.order_type }}"
# Error rate > 5%
- alert: BrokerGatewayHighErrorRate
expr: |
(
sum by(severity) (rate(broker_gateway_error_total[5m]))
/
(sum(rate(broker_gateway_orders_submitted_total[5m])) + 0.001)
) > 0.05
for: 5m
labels:
severity: warning
service: broker_gateway_service
component: error_handling
impact: reliability_degraded
annotations:
summary: "Error rate {{ $value | humanizePercentage }} ({{ $labels.severity }})"
# No orders in 10m during market hours
- alert: BrokerGatewayNoOrderActivity
expr: |
(time() - broker_gateway_last_order_time) > 600
and
(hour() >= 9 and hour() < 16)
for: 1m
labels:
severity: warning
service: broker_gateway_service
component: order_flow
impact: trading_inactive
annotations:
summary: "No order activity for 10+ minutes during market hours"
# Position mismatch > $10K
- alert: BrokerGatewayPositionMismatch
expr: |
abs(
broker_gateway_position_value_usd
-
on(symbol, account_id) group_left()
trading_service_position_value_usd
) > 10000
for: 2m
labels:
severity: critical
service: broker_gateway_service
component: position_management
impact: data_integrity
annotations:
summary: "Position mismatch ${{ $value | humanize }} on {{ $labels.symbol }}"
action: "HALT trading on affected symbol, reconcile with broker"
# FIX sequence number gap
- alert: BrokerGatewayFIXSequenceGap
expr: rate(broker_gateway_sequence_number_gap_total[5m]) > 0
for: 1m
labels:
severity: warning
service: broker_gateway_service
component: fix_session
impact: message_loss
annotations:
summary: "FIX sequence gap on {{ $labels.session_id }} ({{ $value | humanize }} gaps/sec)"
- name: broker_gateway_performance
interval: 60s
rules:
# High FIX heartbeat RTT
- alert: BrokerGatewayHighFIXHeartbeatRTT
expr: broker_gateway_fix_heartbeat_rtt_ms > 50
for: 5m
labels:
severity: warning
service: broker_gateway_service
component: fix_session
impact: network_latency
annotations:
summary: "FIX heartbeat RTT {{ $value }}ms on {{ $labels.session_id }}"
# High order rejection rate per symbol
- alert: BrokerGatewayHighOrderRejectionRate
expr: |
(
sum by(symbol) (rate(broker_gateway_orders_rejected_total[5m]))
/
(sum by(symbol) (rate(broker_gateway_orders_submitted_total[5m])) + 0.001)
) > 0.10
for: 5m
labels:
severity: warning
service: broker_gateway_service
component: order_validation
impact: order_flow_disrupted
annotations:
summary: "Order rejection rate {{ $value | humanizePercentage }} for {{ $labels.symbol }}"
# Slow database queries
- alert: BrokerGatewaySlowDatabaseQueries
expr: |
histogram_quantile(0.95,
sum by(le) (rate(database_query_duration_seconds_bucket{service="broker_gateway_service"}[5m]))
) > 0.100
for: 5m
labels:
severity: warning
service: broker_gateway_service
component: database
impact: performance_degraded
annotations:
summary: "Database query P95 > 100ms"
# High margin usage
- alert: BrokerGatewayHighMarginUsage
expr: |
(
broker_gateway_margin_used_usd
/ (broker_gateway_cash_balance_usd + 0.001)
) > 0.80
for: 5m
labels:
severity: warning
service: broker_gateway_service
component: risk_management
impact: margin_pressure
annotations:
summary: "Margin usage {{ $value | humanizePercentage }} for {{ $labels.account_id }}"

328
infra/k8s/monitoring/prometheus-rules-hft.yaml
View File

@@ -1,328 +0,0 @@
# HFT alert rules — discovered automatically by Prometheus Operator
# Apply: kubectl apply -f prometheus-rules-hft.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: foxhunt-hft-alerts
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
role: alert-rules
spec:
groups:
# ═══════════════════════════════
# CRITICAL HFT ALERTS
# ═══════════════════════════════
- name: hft_critical
rules:
- alert: TradingServiceDown
expr: up{job="foxhunt-services", service="trading-service"} == 0
for: 5s
labels:
severity: critical
component: trading_engine
impact: trading_halt
annotations:
summary: "Trading Service is DOWN"
description: "Trading Service has been down for >5s. All trading operations halted."
action: "Immediate intervention required"
- alert: DataAcquisitionDown
expr: up{job="foxhunt-services", service="data-acquisition-service"} == 0
for: 10s
labels:
severity: critical
component: market_data
impact: blind_trading
annotations:
summary: "Data Acquisition Service is DOWN"
description: "Market data feed down >10s. Trading without data is dangerous."
action: "Stop all trading immediately"
- alert: BrokerGatewayDown
expr: up{job="foxhunt-services", service="broker-gateway-service"} == 0
for: 5s
labels:
severity: critical
component: risk_management
impact: uncontrolled_risk
annotations:
summary: "Broker Gateway (risk controls) is DOWN"
description: "Broker gateway down >5s. Order routing and risk controls offline."
action: "Emergency trading halt"
- alert: TradingLatencyHigh
expr: histogram_quantile(0.95, foxhunt_trading_latency_seconds_bucket{operation="order_placement"}) > 0.001
for: 30s
labels:
severity: critical
component: trading_engine
impact: competitive_disadvantage
annotations:
summary: "Trading latency P95 > 1ms"
description: "95th percentile order placement latency is {{ $value }}s. HFT advantage compromised."
action: "Investigate performance bottlenecks"
- alert: PositionLimitBreach
expr: foxhunt_position_size_usd / foxhunt_position_limit_usd > 0.95
for: 0s
labels:
severity: critical
component: risk_management
impact: regulatory_breach
annotations:
summary: "Position limit nearly breached ({{ $value | humanizePercentage }})"
description: "Current position at {{ $value | humanizePercentage }} of limit."
action: "Reduce positions immediately"
- alert: LossLimitBreach
expr: foxhunt_daily_pnl_usd < foxhunt_loss_limit_usd
for: 0s
labels:
severity: critical
component: risk_management
impact: financial_loss
annotations:
summary: "Daily loss limit breached"
description: "Daily P&L ${{ $value }} breaches loss limit. Trading must halt."
action: "Emergency trading halt"
# ═══════════════════════════════
# HIGH PRIORITY ALERTS
# ═══════════════════════════════
- name: hft_high
rules:
- alert: DatabaseConnectionHigh
expr: foxhunt_database_connections_active / foxhunt_database_connections_max > 0.85
for: 1m
labels:
severity: high
component: database
impact: performance_degradation
annotations:
summary: "Database connection pool at {{ $value | humanizePercentage }}"
- alert: MemoryUsageHigh
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90
for: 2m
labels:
severity: high
component: system
impact: performance_degradation
annotations:
summary: "Memory usage {{ $value | humanizePercentage }} on {{ $labels.instance }}"
- alert: CPUUsageHigh
expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 5m
labels:
severity: high
component: system
impact: performance_degradation
annotations:
summary: "CPU usage {{ $value | printf \"%.1f\" }}% on {{ $labels.instance }}"
- alert: DiskUsageHigh
expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"})) > 0.85
for: 5m
labels:
severity: high
component: system
impact: storage_failure
annotations:
summary: "Disk usage {{ $value | humanizePercentage }} on {{ $labels.instance }} {{ $labels.mountpoint }}"
- alert: OrderRejectionRateHigh
expr: rate(foxhunt_orders_rejected_total[5m]) / rate(foxhunt_orders_total[5m]) > 0.05
for: 2m
labels:
severity: high
component: trading_engine
impact: trading_inefficiency
annotations:
summary: "Order rejection rate {{ $value | humanizePercentage }}"
- alert: MarketDataLag
expr: foxhunt_market_data_lag_seconds > 0.1
for: 1m
labels:
severity: high
component: market_data
impact: stale_data
annotations:
summary: "Market data lag {{ $value }}s — trading on stale data"
- alert: DrawdownHigh
expr: foxhunt_drawdown_percent > 0.05
for: 15m
labels:
severity: high
component: risk_management
impact: financial_risk
annotations:
summary: "Portfolio drawdown {{ $value | humanizePercentage }}"
# ═══════════════════════════════
# MEDIUM / WARNING ALERTS
# ═══════════════════════════════
- name: hft_medium
rules:
- alert: ServiceRestarted
expr: increase(process_start_time_seconds[10m]) > 0
for: 0s
labels:
severity: medium
component: service
impact: disruption
annotations:
summary: "Service {{ $labels.job }} restarted on {{ $labels.instance }}"
- alert: NetworkLatencyHigh
expr: foxhunt_network_latency_seconds > 0.01
for: 5m
labels:
severity: medium
component: network
impact: performance_impact
annotations:
summary: "Network latency {{ $value }}s"
- alert: BackupFailed
expr: time() - foxhunt_last_backup_timestamp > 86400
for: 1h
labels:
severity: medium
component: backup
impact: data_risk
annotations:
summary: "Last backup {{ $value | humanizeDuration }} ago"
- alert: SSLCertificateExpiringSoon
expr: (ssl_certificate_expiry_timestamp - time()) / 86400 < 30
for: 1h
labels:
severity: medium
component: security
impact: service_disruption
annotations:
summary: "SSL cert expires in {{ $value }} days for {{ $labels.instance }}"
# ═══════════════════════════════
# BUSINESS LOGIC ALERTS
# ═══════════════════════════════
- name: hft_business
rules:
- alert: TradingVolumeAnomalyHigh
expr: rate(foxhunt_trades_total[5m]) > (avg_over_time(rate(foxhunt_trades_total[5m])[1h:5m]) * 3)
for: 2m
labels:
severity: medium
component: trading_engine
impact: business_anomaly
annotations:
summary: "Trading volume 3x hourly average ({{ $value }} trades/sec)"
- alert: TradingVolumeAnomalyLow
expr: rate(foxhunt_trades_total[5m]) < (avg_over_time(rate(foxhunt_trades_total[5m])[1h:5m]) * 0.1)
for: 10m
labels:
severity: medium
component: trading_engine
impact: business_anomaly
annotations:
summary: "Trading volume 10% of hourly average ({{ $value }} trades/sec)"
- alert: PnLVolatilityHigh
expr: stddev_over_time(foxhunt_pnl_usd[1h]) > 10000
for: 30m
labels:
severity: medium
component: risk_management
impact: financial_risk
annotations:
summary: "P&L std dev ${{ $value }} over 1h"
# ═══════════════════════════════
# INFRASTRUCTURE ALERTS
# ═══════════════════════════════
- name: hft_infrastructure
rules:
- alert: RedisConnectionFailed
expr: redis_connected_clients{job="redis"} == 0
for: 30s
labels:
severity: critical
component: redis
impact: cache_failure
annotations:
summary: "Redis has 0 connected clients"
- alert: PostgreSQLDown
expr: pg_up{job="postgres"} == 0
for: 30s
labels:
severity: critical
component: postgresql
impact: data_unavailable
annotations:
summary: "PostgreSQL is not responding"
# ═══════════════════════════════
# SECURITY ALERTS
# ═══════════════════════════════
- name: hft_security
rules:
- alert: AuthenticationFailuresHigh
expr: rate(foxhunt_auth_failures_total[5m]) > 5
for: 2m
labels:
severity: high
component: security
impact: security_breach
annotations:
summary: "{{ $value }} auth failures/sec"
- alert: UnauthorizedAccessAttempts
expr: rate(foxhunt_unauthorized_requests_total[5m]) > 1
for: 1m
labels:
severity: critical
component: security
impact: security_breach
annotations:
summary: "{{ $value }} unauthorized requests/sec from {{ $labels.source_ip }}"
- alert: APIRateLimitExceeded
expr: rate(foxhunt_api_rate_limit_exceeded_total[5m]) > 0.1
for: 5m
labels:
severity: medium
component: api
impact: service_degradation
annotations:
summary: "API rate limits exceeded {{ $value }}/sec"
# ═══════════════════════════════
# MONITORING HEALTH
# ═══════════════════════════════
- name: hft_monitoring
rules:
- alert: PrometheusTargetDown
expr: up == 0
for: 2m
labels:
severity: medium
component: monitoring
impact: observability_loss
annotations:
summary: "{{ $labels.job }} target {{ $labels.instance }} down >2m"
- alert: HighAlertRate
expr: rate(prometheus_notifications_total[5m]) > 10
for: 10m
labels:
severity: medium
component: monitoring
impact: alert_fatigue
annotations:
summary: "{{ $value }} alerts/sec — check for alert storm"

56
infra/k8s/monitoring/prometheus-rules-storage.yaml
View File

@@ -1,56 +0,0 @@
# Storage alert rules — volume usage warnings before auto-expand kicks in
# Apply: kubectl apply -f prometheus-rules-storage.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: foxhunt-storage-alerts
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
role: alert-rules
spec:
groups:
- name: storage_volume_usage
interval: 60s
rules:
# PVC usage > 75% — warning (auto-expand triggers at 85%)
- alert: PVCUsageHigh
expr: |
100 * kubelet_volume_stats_used_bytes{namespace="foxhunt"}
/ kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 75
for: 5m
labels:
severity: warning
component: storage
annotations:
summary: "PVC {{ $labels.persistentvolumeclaim }} at {{ $value | printf \"%.0f\" }}% usage"
description: "Volume {{ $labels.persistentvolumeclaim }} is {{ $value | printf \"%.0f\" }}% full. Auto-expand triggers at 85%."
# PVC usage > 90% — critical (auto-expand should have already acted)
- alert: PVCUsageCritical
expr: |
100 * kubelet_volume_stats_used_bytes{namespace="foxhunt"}
/ kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 90
for: 2m
labels:
severity: critical
component: storage
annotations:
summary: "PVC {{ $labels.persistentvolumeclaim }} CRITICAL at {{ $value | printf \"%.0f\" }}%"
description: "Volume {{ $labels.persistentvolumeclaim }} is {{ $value | printf \"%.0f\" }}% full despite auto-expand. Investigate immediately."
action: "Check pvc-autoscaler CronJob logs. Manual expand: kubectl patch pvc {{ $labels.persistentvolumeclaim }} -n foxhunt --type merge -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"XXGi\"}}}}'"
# PVC approaching max auto-expand limit (>80Gi of 100Gi max)
- alert: PVCApproachingMaxSize
expr: |
kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 80 * 1024 * 1024 * 1024
and
100 * kubelet_volume_stats_used_bytes{namespace="foxhunt"}
/ kubelet_volume_stats_capacity_bytes{namespace="foxhunt"} > 70
for: 10m
labels:
severity: warning
component: storage
annotations:
summary: "PVC {{ $labels.persistentvolumeclaim }} is large and growing"
description: "Volume is >80Gi and >70% full. Auto-expand max is 100Gi. Consider manual intervention or data cleanup."

400
infra/k8s/monitoring/prometheus-stack-values.yaml
View File

@@ -1,400 +0,0 @@
# kube-prometheus-stack Helm values — Foxhunt production monitoring
#
# Install:
# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
# helm repo update
# helm install prometheus-stack prometheus-community/kube-prometheus-stack \
# --namespace foxhunt \
# -f infra/k8s/monitoring/prometheus-stack-values.yaml
#
# Upgrade:
# helm upgrade prometheus-stack prometheus-community/kube-prometheus-stack \
# --namespace foxhunt \
# -f infra/k8s/monitoring/prometheus-stack-values.yaml
#
# Prerequisites:
# kubectl -n foxhunt create secret generic alertmanager-slack \
# --from-literal=slack-webhook-url='https://hooks.slack.com/services/...'
#
# After install, delete old standalone resources:
# kubectl -n foxhunt delete deploy prometheus
# kubectl -n foxhunt delete svc prometheus
# kubectl -n foxhunt delete cm prometheus-config
# kubectl -n foxhunt delete pvc prometheus-data
# kubectl -n foxhunt delete sa prometheus
# kubectl delete clusterrole foxhunt-prometheus
# kubectl delete clusterrolebinding foxhunt-prometheus
# kubectl -n foxhunt delete deploy node-exporter
# kubectl -n foxhunt delete ds node-exporter # if DaemonSet
# kubectl -n foxhunt delete deploy kube-state-metrics
# ─────────────────────────────────────────────
# Prometheus Operator
# ─────────────────────────────────────────────
prometheusOperator:
enabled: true
nodeSelector:
k8s.scaleway.com/pool-name: platform
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
# ─────────────────────────────────────────────
# Prometheus
# ─────────────────────────────────────────────
prometheus:
prometheusSpec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
podMetadata:
labels:
app.kubernetes.io/part-of: foxhunt
retention: 14d
retentionSize: 8GB
replicas: 1
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
storageSpec:
volumeClaimTemplate:
spec:
storageClassName: scw-bssd
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 10Gi
# Discover ALL ServiceMonitors/PodMonitors/PrometheusRules in the cluster,
# not just those matching Helm release labels.
serviceMonitorSelectorNilUsesHelmValues: false
podMonitorSelectorNilUsesHelmValues: false
ruleSelectorNilUsesHelmValues: false
# Scrape configs for targets that don't fit ServiceMonitor/PodMonitor CRDs
additionalScrapeConfigs:
# Argo Workflows controller (HTTPS with self-signed cert)
- job_name: argo-workflows
scheme: https
tls_config:
insecure_skip_verify: true
static_configs:
- targets: ['argo-workflows-workflow-controller-metrics.foxhunt.svc.cluster.local:9090']
relabel_configs:
- target_label: service
replacement: argo-workflows
# Training workflow pods (ephemeral Argo pods on port 9094, Running only)
- job_name: training-pods
kubernetes_sd_configs:
- role: pod
namespaces:
names: [foxhunt]
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
action: keep
regex: training-workflow|compile-and-train
- source_labels: [__meta_kubernetes_pod_phase]
action: keep
regex: Running
- source_labels: [__meta_kubernetes_pod_ip]
target_label: __address__
replacement: ${1}:9094
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_label_workflows_argoproj_io_workflow]
target_label: workflow
# QuestDB (ML metrics store)
- job_name: questdb
metrics_path: /metrics
static_configs:
- targets: ['questdb.foxhunt.svc.cluster.local:9003']
relabel_configs:
- target_label: service
replacement: questdb
# Pushgateway (training job push metrics)
- job_name: pushgateway
honor_labels: true
static_configs:
- targets: ['pushgateway.foxhunt.svc.cluster.local:9091']
# Loki metrics
- job_name: loki
static_configs:
- targets: ['loki.foxhunt.svc.cluster.local:3100']
# Tempo metrics
- job_name: tempo
static_configs:
- targets: ['tempo.foxhunt.svc.cluster.local:3200']
# ─────────────────────────────────────────────
# AlertManager
# ─────────────────────────────────────────────
alertmanager:
alertmanagerSpec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
replicas: 1
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
# Mount K8s Secrets into AlertManager at /etc/alertmanager/secrets/<name>/
secrets:
- alertmanager-mattermost
config:
global:
# Mattermost incoming webhook (Slack-compatible API).
# Reads URL from K8s Secret: alertmanager-mattermost/webhook-url
# Create with:
# kubectl -n foxhunt create secret generic alertmanager-mattermost \
# --from-literal=webhook-url='http://mattermost.foxhunt.svc.cluster.local:8065/hooks/<HOOK_ID>'
slack_api_url_file: /etc/alertmanager/secrets/alertmanager-mattermost/webhook-url
route:
receiver: default
group_by: ['alertname', 'severity', 'service']
group_wait: 5s
group_interval: 30s
repeat_interval: 4h
routes:
# CRITICAL trading/risk → immediate Slack + PagerDuty
- matchers:
- severity="critical"
- component=~"trading_engine|risk_management|trading|risk"
receiver: critical-trading
group_wait: 0s
group_interval: 5s
repeat_interval: 5m
continue: true
# CRITICAL infrastructure → Slack critical channel
- matchers:
- severity="critical"
receiver: critical-infra
group_wait: 0s
group_interval: 10s
repeat_interval: 15m
# HIGH priority → Slack alerts channel
- matchers:
- severity="high"
receiver: high-priority
group_wait: 10s
group_interval: 1m
repeat_interval: 30m
# WARNING / MEDIUM → Slack digest
- matchers:
- severity=~"warning|WARNING|medium"
receiver: warning-alerts
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receivers:
- name: default
slack_configs:
- channel: '#foxhunt-alerts'
title: '{{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}*{{ .Labels.severity | toUpper }}*: {{ .Annotations.summary }}
{{ end }}
send_resolved: true
- name: critical-trading
slack_configs:
- channel: '#foxhunt-critical'
title: 'CRITICAL: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}
*Alert*: {{ .Annotations.summary }}
*Impact*: {{ .Annotations.impact }}
*Action*: {{ .Annotations.action }}
{{ end }}
color: 'danger'
send_resolved: true
- name: critical-infra
slack_configs:
- channel: '#foxhunt-critical'
title: 'INFRA CRITICAL: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}{{ .Annotations.summary }}
{{ end }}
color: 'danger'
send_resolved: true
- name: high-priority
slack_configs:
- channel: '#foxhunt-alerts'
title: 'HIGH: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}{{ .Annotations.summary }}
{{ end }}
color: 'warning'
send_resolved: true
- name: warning-alerts
slack_configs:
- channel: '#foxhunt-alerts'
title: 'WARNING: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}{{ .Annotations.summary }}
{{ end }}
color: '#ff9500'
send_resolved: true
inhibit_rules:
# Trading engine down → suppress all trading alerts
- source_matchers:
- alertname="TradingEngineDown"
target_matchers:
- component=~"trading_engine|trading"
equal: ['service']
# Node down → suppress service-level alerts on that node
- source_matchers:
- alertname=~"NodeDown|KubeNodeNotReady"
target_matchers:
- severity=~"high|warning|medium"
equal: ['instance']
# PostgreSQL down → suppress database alerts
- source_matchers:
- alertname="PostgreSQLDown"
target_matchers:
- component="database"
# ─────────────────────────────────────────────
# ServiceMonitors
# ─────────────────────────────────────────────
additionalServiceMonitors:
# All foxhunt microservices (7 services, all expose named port "metrics")
- name: foxhunt-services
selector:
matchLabels:
app.kubernetes.io/part-of: foxhunt
namespaceSelector:
matchNames: [foxhunt]
endpoints:
- port: metrics
interval: 15s
# DCGM Exporter (GPU metrics from kube-system)
- name: dcgm-exporter
selector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values: [dcgm-exporter]
namespaceSelector:
matchNames: [kube-system, monitoring, foxhunt]
endpoints:
- port: metrics
interval: 15s
# ─────────────────────────────────────────────
# Grafana (disabled — managed separately via grafana Helm chart)
# ─────────────────────────────────────────────
grafana:
enabled: false
# ─────────────────────────────────────────────
# Node Exporter (replaces standalone node-exporter.yaml)
# ─────────────────────────────────────────────
prometheus-node-exporter:
tolerations:
- effect: NoSchedule
operator: Exists
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
cpu: 50m
memory: 32Mi
# ─────────────────────────────────────────────
# kube-state-metrics (replaces standalone kube-state-metrics.yaml)
# ─────────────────────────────────────────────
kube-state-metrics:
nodeSelector:
k8s.scaleway.com/pool-name: platform
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 50m
memory: 64Mi
# ─────────────────────────────────────────────
# Default rules (Kubernetes cluster health alerts included out-of-box)
# ─────────────────────────────────────────────
defaultRules:
create: true
rules:
alertmanager: true
etcd: false # managed by Scaleway Kapsule
configReloaders: true
general: true
k8sContainerCpuUsageSecondsTotal: true
k8sContainerMemoryCache: true
k8sContainerMemoryRss: true
k8sContainerMemorySwap: true
k8sContainerMemoryWorkingSetBytes: true
k8sPodOwner: true
kubeApiserverAvailability: false # managed by Scaleway
kubeApiserverBurnrate: false
kubeApiserverHistogram: false
kubeApiserverSlos: false
kubeControllerManager: false # managed by Scaleway
kubelet: false # kubelet metrics not accessible on Kapsule
kubeProxy: false # managed by Scaleway
kubePrometheusGeneral: true
kubePrometheusNodeRecording: true
kubernetesAbsent: true
kubernetesApps: true
kubernetesResources: true
kubernetesStorage: true
kubernetesSystem: true
kubeScheduler: false # managed by Scaleway
kubeStateMetrics: true
network: true
node: true
nodeExporterAlerting: true
nodeExporterRecording: true
prometheus: true
prometheusOperator: true
# ─────────────────────────────────────────────
# Disable components managed by Scaleway Kapsule
# ─────────────────────────────────────────────
kubeApiServer:
enabled: false
kubeControllerManager:
enabled: false
kubeScheduler:
enabled: false
kubeProxy:
enabled: false
kubelet:
enabled: false # kubelet metrics not accessible on Kapsule
kubeEtcd:
enabled: false

172
infra/k8s/monitoring/prometheus-values.yaml
View File

@@ -1,172 +0,0 @@
# Prometheus override values for GitLab Helm chart
# Removes Cockpit remote_write, adds comprehensive cluster scrape configs
#
# Apply with:
# helm upgrade gitlab gitlab/gitlab -n foxhunt --reuse-values \
# -f infra/k8s/monitoring/prometheus-values.yaml
prometheus:
server:
# Remove Cockpit remote_write — all metrics stay local
remoteWrite: []
# Extra scrape configs for full cluster observability
# NOTE: this is at prometheus.extraScrapeConfigs (chart root), NOT prometheus.server.extraScrapeConfigs
extraScrapeConfigs: |
# --- Host metrics from node-exporter ---
- job_name: node-exporter
kubernetes_sd_configs:
- role: endpoints
namespaces:
names: [foxhunt]
relabel_configs:
- source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
action: keep
regex: node-exporter
- source_labels: [__meta_kubernetes_endpoints_name]
action: keep
regex: node-exporter
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
# --- Kubernetes object state ---
- job_name: kube-state-metrics
kubernetes_sd_configs:
- role: endpoints
namespaces:
names: [foxhunt]
relabel_configs:
- source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
action: keep
regex: kube-state-metrics
- source_labels: [__meta_kubernetes_endpoint_port_name]
action: keep
regex: http
# --- Kubelet metrics (container resource usage) ---
- job_name: kubelet
scheme: https
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
kubernetes_sd_configs:
- role: node
relabel_configs:
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$1/proxy/metrics
# --- cAdvisor (container CPU, memory, network, filesystem) ---
- job_name: cadvisor
scheme: https
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
kubernetes_sd_configs:
- role: node
relabel_configs:
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
metric_relabel_configs:
- source_labels: [namespace]
action: keep
regex: (foxhunt|monitoring|kube-system)
# --- GPU metrics from NVIDIA DCGM Exporter ---
- job_name: dcgm-exporter
kubernetes_sd_configs:
- role: endpoints
namespaces:
names: [kube-system, monitoring]
relabel_configs:
- source_labels: [__meta_kubernetes_service_name]
action: keep
regex: .*dcgm.*
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
# --- Foxhunt services (app metrics ports) ---
- job_name: foxhunt-services
kubernetes_sd_configs:
- role: endpoints
namespaces:
names: [foxhunt]
relabel_configs:
- source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of]
action: keep
regex: foxhunt
- source_labels: [__meta_kubernetes_endpoint_port_name]
action: keep
regex: metrics
- source_labels: [__meta_kubernetes_service_name]
target_label: service
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
# --- General prometheus.io annotation-based scraping ---
- job_name: prometheus-annotated-pods
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: "true"
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
action: replace
regex: (https?)
target_label: __scheme__
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
regex: (.+)
target_label: __metrics_path__
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
# --- Loki metrics ---
- job_name: loki
static_configs:
- targets: ['loki.foxhunt.svc.cluster.local:3100']
# --- Tempo metrics ---
- job_name: tempo
static_configs:
- targets: ['tempo.foxhunt.svc.cluster.local:3200']
# --- Promtail metrics ---
- job_name: promtail
kubernetes_sd_configs:
- role: pod
namespaces:
names: [foxhunt]
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
action: keep
regex: promtail
- source_labels: [__meta_kubernetes_pod_ip]
target_label: __address__
replacement: $1:9080
# --- Pushgateway (training job metrics) ---
- job_name: pushgateway
honor_labels: true
static_configs:
- targets: ['pushgateway.foxhunt.svc.cluster.local:9091']

265
infra/k8s/monitoring/promtail.yaml
View File

@@ -1,265 +0,0 @@
# Promtail — ships container logs to Loki (DaemonSet on all nodes)
apiVersion: v1
kind: ServiceAccount
metadata:
name: promtail
namespace: foxhunt
labels:
app.kubernetes.io/name: promtail
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: promtail
labels:
app.kubernetes.io/name: promtail
rules:
- apiGroups: [""]
resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"]
verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: promtail
labels:
app.kubernetes.io/name: promtail
subjects:
- kind: ServiceAccount
name: promtail
namespace: foxhunt
roleRef:
kind: ClusterRole
name: promtail
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
name: promtail-config
namespace: foxhunt
labels:
app.kubernetes.io/name: promtail
data:
promtail.yaml: |
server:
http_listen_port: 9080
positions:
filename: /run/promtail/positions.yaml
clients:
- url: http://loki.foxhunt.svc.cluster.local:3100/loki/api/v1/push
scrape_configs:
# Foxhunt application pods (structured Rust logs)
- job_name: foxhunt-pods
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_namespace]
action: keep
regex: foxhunt
# Drop noisy infra pods (gitlab internals, prometheus, grafana)
- source_labels: [__meta_kubernetes_pod_label_app]
action: drop
regex: (gitlab|prometheus|grafana|promtail)
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
action: drop
regex: (promtail|node-exporter|kube-state-metrics)
# Map to containerd log path
- source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
separator: /
target_label: __path__
replacement: /var/log/pods/*$1/*.log
# Standard K8s metadata labels
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_container_name]
target_label: container
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
target_label: app
- source_labels: [__meta_kubernetes_pod_label_foxhunt_job__type]
target_label: job_type
- source_labels: [__meta_kubernetes_pod_label_foxhunt_model]
target_label: model
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
pipeline_stages:
- json:
expressions:
level: level
message: fields.message
target: target
timestamp: timestamp
- labels:
level:
target:
- match:
selector: '{level=""}'
stages:
- regex:
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR)'
- labels:
level:
# Kapsule system pods (kube-system, kube-node-lease, kube-public)
- job_name: kapsule-system
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_namespace]
action: keep
regex: (kube-system|kube-node-lease|kube-public)
# Map to containerd log path
- source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
separator: /
target_label: __path__
replacement: /var/log/pods/*$1/*.log
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_container_name]
target_label: container
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
target_label: app
- source_labels: [__meta_kubernetes_pod_label_app]
target_label: kube_app
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
pipeline_stages:
- match:
selector: '{namespace="kube-system"}'
stages:
- regex:
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)'
- labels:
level:
# GitLab CI runner logs (foxhunt namespace, runner pods)
- job_name: gitlab-runners
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_namespace]
action: keep
regex: foxhunt
- source_labels: [__meta_kubernetes_pod_label_app]
action: keep
regex: gitlab-runner
- source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
separator: /
target_label: __path__
replacement: /var/log/pods/*$1/*.log
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_container_name]
target_label: container
- replacement: gitlab-runner
target_label: app
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
pipeline_stages:
- regex:
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|WARNING)'
- labels:
level:
# Kubernetes node-level logs (kubelet, containerd)
- job_name: kapsule-node-logs
static_configs:
- targets: [localhost]
labels:
job: kapsule-nodes
__path__: /var/log/syslog
- targets: [localhost]
labels:
job: kapsule-nodes
__path__: /var/log/kern.log
- targets: [localhost]
labels:
job: kapsule-nodes
__path__: /var/log/kubelet.log
relabel_configs:
- source_labels: [__path__]
regex: .*/(.+)\.log
target_label: log_type
pipeline_stages:
- regex:
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)'
- labels:
level:
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: promtail
namespace: foxhunt
labels:
app.kubernetes.io/name: promtail
spec:
selector:
matchLabels:
app.kubernetes.io/name: promtail
template:
metadata:
labels:
app.kubernetes.io/name: promtail
spec:
serviceAccountName: promtail
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: promtail
image: grafana/promtail:3.4.2
args: ["-config.file=/etc/promtail/promtail.yaml"]
ports:
- containerPort: 9080
name: http
volumeMounts:
- name: config
mountPath: /etc/promtail
- name: varlog
mountPath: /var/log
readOnly: true
- name: containers
mountPath: /var/lib/docker/containers
readOnly: true
- name: pods
mountPath: /var/log/pods
readOnly: true
- name: run
mountPath: /run/promtail
env:
- name: HOSTNAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
volumes:
- name: config
configMap:
name: promtail-config
- name: varlog
hostPath:
path: /var/log
- name: containers
hostPath:
path: /var/lib/docker/containers
- name: pods
hostPath:
path: /var/log/pods
- name: run
emptyDir: {}

61
infra/k8s/monitoring/pushgateway.yaml
View File

@@ -1,61 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: pushgateway
namespace: foxhunt
labels:
app.kubernetes.io/name: pushgateway
app.kubernetes.io/part-of: monitoring
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: pushgateway
template:
metadata:
labels:
app.kubernetes.io/name: pushgateway
spec:
nodeSelector:
k8s.scaleway.com/pool-name: infra
containers:
- name: pushgateway
image: prom/pushgateway:v1.11.0
args:
- "--persistence.interval=5m"
ports:
- containerPort: 9091
name: http
resources:
requests:
cpu: 50m
memory: 32Mi
limits:
cpu: 200m
memory: 128Mi
livenessProbe:
httpGet:
path: /-/healthy
port: 9091
initialDelaySeconds: 5
readinessProbe:
httpGet:
path: /-/ready
port: 9091
initialDelaySeconds: 5
---
apiVersion: v1
kind: Service
metadata:
name: pushgateway
namespace: foxhunt
labels:
app.kubernetes.io/name: pushgateway
spec:
type: ClusterIP
ports:
- port: 9091
targetPort: 9091
name: http
selector:
app.kubernetes.io/name: pushgateway

250
infra/k8s/monitoring/pvc-autoscaler.yaml
View File

@@ -1,250 +0,0 @@
# PVC Auto-Expand — expands PVCs automatically when usage exceeds 85%
# Uses K8s API to list PVCs and exec df inside pods to check usage.
# Runs every 15 minutes. Apply: kubectl apply -f pvc-autoscaler.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: pvc-autoscaler
namespace: foxhunt
labels:
app.kubernetes.io/name: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: pvc-autoscaler
namespace: foxhunt
labels:
app.kubernetes.io/name: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
rules:
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "patch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups: ["apps"]
resources: ["deployments", "statefulsets"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: pvc-autoscaler-nodes
labels:
app.kubernetes.io/name: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["list"]
- apiGroups: [""]
resources: ["nodes/proxy"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: pvc-autoscaler
namespace: foxhunt
labels:
app.kubernetes.io/name: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
subjects:
- kind: ServiceAccount
name: pvc-autoscaler
namespace: foxhunt
roleRef:
kind: Role
name: pvc-autoscaler
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: pvc-autoscaler-nodes
labels:
app.kubernetes.io/name: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
subjects:
- kind: ServiceAccount
name: pvc-autoscaler
namespace: foxhunt
roleRef:
kind: ClusterRole
name: pvc-autoscaler-nodes
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
name: pvc-autoscaler-script
namespace: foxhunt
labels:
app.kubernetes.io/name: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
data:
autoscale.py: |
#!/usr/bin/env python3
"""PVC auto-expand via kubelet stats API (no Prometheus dependency)."""
import json, os, sys, ssl, urllib.request
from datetime import datetime, timezone
THRESHOLD = int(os.environ.get("PVC_USAGE_THRESHOLD", "85"))
GROW_PERCENT = int(os.environ.get("PVC_GROW_PERCENT", "50"))
MAX_SIZE_GI = int(os.environ.get("PVC_MAX_SIZE_GI", "100"))
NAMESPACE = os.environ.get("NAMESPACE", "foxhunt")
TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token"
CA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
K8S_API = "https://kubernetes.default.svc"
def k8s(path, method="GET", data=None):
token = open(TOKEN_PATH).read().strip()
ctx = ssl.create_default_context(cafile=CA_PATH)
body = json.dumps(data).encode() if data else None
req = urllib.request.Request(f"{K8S_API}{path}", data=body, method=method)
req.add_header("Authorization", f"Bearer {token}")
if data:
req.add_header("Content-Type", "application/merge-patch+json")
return json.loads(urllib.request.urlopen(req, context=ctx, timeout=30).read())
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
print(f"=== PVC Auto-Expand {now} ===")
print(f"Threshold: {THRESHOLD}% | Grow: +{GROW_PERCENT}% | Max: {MAX_SIZE_GI}Gi")
sys.stdout.flush()
# Step 1: Get all nodes and their kubelet stats via K8s proxy API
nodes = k8s("/api/v1/nodes")
pvc_usage = {} # pvc_name -> (used_bytes, capacity_bytes)
for node in nodes["items"]:
node_name = node["metadata"]["name"]
try:
stats = k8s(f"/api/v1/nodes/{node_name}/proxy/stats/summary")
for pod in stats.get("pods", []):
if pod.get("podRef", {}).get("namespace") != NAMESPACE:
continue
for vol in pod.get("volume", []):
pvc_ref = vol.get("pvcRef")
if not pvc_ref:
continue
pvc_name = pvc_ref["name"]
used = vol.get("usedBytes", 0)
cap = vol.get("capacityBytes", 1)
pvc_usage[pvc_name] = (used, cap)
except Exception as e:
print(f" WARN: could not get stats from node {node_name}: {e}")
sys.stdout.flush()
if not pvc_usage:
print(" No PVC stats found")
sys.exit(0)
# Step 2: Check each PVC and expand if needed
expanded = 0
for pvc_name, (used, cap) in sorted(pvc_usage.items()):
pct = 100 * used / cap if cap > 0 else 0
used_gi = used / (1024**3)
cap_gi = cap / (1024**3)
print(f" {pvc_name}: {pct:.1f}% ({used_gi:.1f}G / {cap_gi:.1f}G)")
if pct >= THRESHOLD:
print(f" >> {pvc_name} exceeds {THRESHOLD}%")
try:
pvc_obj = k8s(f"/api/v1/namespaces/{NAMESPACE}/persistentvolumeclaims/{pvc_name}")
sc = pvc_obj.get("spec", {}).get("storageClassName", "")
# Only expand if StorageClass supports it
if sc:
try:
sc_obj = k8s(f"/apis/storage.k8s.io/v1/storageclasses/{sc}")
if not sc_obj.get("allowVolumeExpansion"):
print(f" SKIP: StorageClass {sc} does not allow expansion")
continue
except Exception:
pass
status_storage = pvc_obj.get("status", {}).get("capacity", {}).get("storage", "0Gi")
current_gi = int(status_storage.replace("Gi", ""))
spec_storage = pvc_obj["spec"]["resources"]["requests"]["storage"]
spec_gi = int(spec_storage.replace("Gi", ""))
new_gi = current_gi + max(current_gi * GROW_PERCENT // 100, 1)
if new_gi > MAX_SIZE_GI:
print(f" SKIP: target {new_gi}Gi > max {MAX_SIZE_GI}Gi")
continue
if spec_gi >= new_gi:
print(f" SKIP: spec={spec_gi}Gi >= target={new_gi}Gi (resize pending)")
continue
patch = {"spec": {"resources": {"requests": {"storage": f"{new_gi}Gi"}}}}
k8s(f"/api/v1/namespaces/{NAMESPACE}/persistentvolumeclaims/{pvc_name}",
method="PATCH", data=patch)
print(f" EXPANDED: {current_gi}Gi -> {new_gi}Gi")
expanded += 1
except Exception as e:
print(f" FAILED: {e}", file=sys.stderr)
sys.stdout.flush()
print(f"=== Done ({expanded} expanded) ===")
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: pvc-autoscaler
namespace: foxhunt
labels:
app.kubernetes.io/name: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
spec:
schedule: "*/15 * * * *"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
activeDeadlineSeconds: 120
template:
metadata:
labels:
app: pvc-autoscaler
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
serviceAccountName: pvc-autoscaler
restartPolicy: OnFailure
containers:
- name: autoscaler
image: python:3.12-alpine
command: ["python3", "-u", "/scripts/autoscale.py"]
env:
- name: PVC_USAGE_THRESHOLD
value: "85"
- name: PVC_GROW_PERCENT
value: "50"
- name: PVC_MAX_SIZE_GI
value: "100"
- name: NAMESPACE
value: "foxhunt"
volumeMounts:
- name: script
mountPath: /scripts
resources:
requests:
cpu: 10m
memory: 64Mi
limits:
cpu: 100m
memory: 128Mi
volumes:
- name: script
configMap:
name: pvc-autoscaler-script
items:
- key: autoscale.py
path: autoscale.py
defaultMode: 0755

239
infra/k8s/monitoring/scaleway-billing.yaml
View File

@@ -1,239 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: scaleway-billing
namespace: foxhunt
type: Opaque
stringData:
SCW_SECRET_KEY: "84fa878b-81c2-401a-8aef-22996d30e9ea"
SCW_ORG_ID: "a55098a3-2cf6-4f73-b011-fff5a299ed94"
SCW_PROJECT_ID: "c293eb98-228d-427d-9b16-f0941f3f2adb"
SCW_CLUSTER_ID: "34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: scaleway-billing-script
namespace: foxhunt
data:
scrape.py: |
#!/usr/bin/env python3
"""Scaleway infrastructure scraper → Prometheus pushgateway."""
import json, os, sys, urllib.request, urllib.error, re
def sanitize(s):
"""Sanitize label values for Prometheus text format."""
return str(s).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
PUSHGATEWAY = os.environ.get("PUSHGATEWAY_URL", "http://pushgateway.foxhunt.svc.cluster.local:9091")
SECRET = os.environ["SCW_SECRET_KEY"]
ORG = os.environ["SCW_ORG_ID"]
PROJECT = os.environ["SCW_PROJECT_ID"]
CLUSTER = os.environ["SCW_CLUSTER_ID"]
SCW_BILLING = "https://api.scaleway.com/billing/v2beta1"
SCW_INSTANCE = "https://api.scaleway.com/instance/v1/zones/fr-par-2"
SCW_K8S = "https://api.scaleway.com/k8s/v1/regions/fr-par"
SCW_BLOCK = "https://api.scaleway.com/block/v1alpha1/zones/fr-par-2"
SCW_IPAM = "https://api.scaleway.com/ipam/v1/regions/fr-par"
def fetch(url):
req = urllib.request.Request(url)
req.add_header("X-Auth-Token", SECRET)
try:
return json.loads(urllib.request.urlopen(req, timeout=30).read())
except Exception as e:
print(f"WARN: {url}: {e}", file=sys.stderr)
return {}
# ── Fetch all data (dynamic — auto-discovers any infra changes) ──
billing = fetch(f"{SCW_BILLING}/consumptions?organization_id={ORG}")
instances = fetch(f"{SCW_INSTANCE}/servers?project={PROJECT}&per_page=50")
pools = fetch(f"{SCW_K8S}/clusters/{CLUSTER}/pools")
volumes = fetch(f"{SCW_BLOCK}/volumes?project_id={PROJECT}&per_page=100")
ips = fetch(f"{SCW_IPAM}/ips?project_id={PROJECT}&is_ipv6=false&per_page=100")
L = []
# ── Billing ──
total = 0
cats = {}
products = {}
for c in billing.get("consumptions", []):
cat = c.get("category_name", "other").lower().replace(" ", "_").replace("&", "and")
product = c.get("product_name", "unknown").lower().replace(" ", "_").replace("-", "_")
sku = c.get("sku", "")
val = c.get("value", {})
amount = int(val.get("units", 0) or 0) + int(val.get("nanos", 0) or 0) / 1e9
total += amount
cats[cat] = cats.get(cat, 0) + amount
products[(cat, product, sku)] = products.get((cat, product, sku), 0) + amount
L.append("# HELP scaleway_billing_total_eur Total current month billing in EUR")
L.append("# TYPE scaleway_billing_total_eur gauge")
L.append(f"scaleway_billing_total_eur {total:.4f}")
L.append("# HELP scaleway_billing_consumption_eur Billing by category in EUR")
L.append("# TYPE scaleway_billing_consumption_eur gauge")
for cat, amt in sorted(cats.items(), key=lambda x: -x[1]):
L.append(f'scaleway_billing_consumption_eur{{category="{sanitize(cat)}"}} {amt:.4f}')
L.append("# HELP scaleway_billing_product_eur Billing by product in EUR")
L.append("# TYPE scaleway_billing_product_eur gauge")
for (cat, product, sku), amt in sorted(products.items(), key=lambda x: -x[1]):
if amt >= 0.01:
L.append(f'scaleway_billing_product_eur{{category="{sanitize(cat)}",product="{sanitize(product)}",sku="{sanitize(sku)}"}} {amt:.4f}')
# ── Instances (dynamic — auto-discovers new/removed instances) ──
L.append("# HELP scaleway_instance_info Instance metadata")
L.append("# TYPE scaleway_instance_info gauge")
L.append("# HELP scaleway_instance_running Instance running state (1=running)")
L.append("# TYPE scaleway_instance_running gauge")
type_counts = {}
for s in instances.get("servers", []):
name = s.get("name", "?")
itype = s.get("commercial_type", "?")
state = s.get("state", "?")
tags = dict(t.split("=", 1) for t in s.get("tags", []) if "=" in t)
pool = tags.get("pool-name", "unknown")
L.append(f'scaleway_instance_info{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}",state="{sanitize(state)}"}} 1')
L.append(f'scaleway_instance_running{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}"}} {1 if state == "running" else 0}')
type_counts[itype] = type_counts.get(itype, 0) + 1
L.append("# HELP scaleway_instances_total Total instances by type")
L.append("# TYPE scaleway_instances_total gauge")
for t, c in type_counts.items():
L.append(f'scaleway_instances_total{{type="{sanitize(t)}"}} {c}')
# ── K8s Pools (dynamic — auto-discovers pool changes/autoscaler config) ──
L.append("# HELP scaleway_pool_size Current pool node count")
L.append("# TYPE scaleway_pool_size gauge")
L.append("# HELP scaleway_pool_min Minimum pool size")
L.append("# TYPE scaleway_pool_min gauge")
L.append("# HELP scaleway_pool_max Maximum pool size")
L.append("# TYPE scaleway_pool_max gauge")
L.append("# HELP scaleway_pool_autoscaling Pool autoscaling enabled")
L.append("# TYPE scaleway_pool_autoscaling gauge")
L.append("# HELP scaleway_pool_root_volume_bytes Pool root volume size")
L.append("# TYPE scaleway_pool_root_volume_bytes gauge")
for p in pools.get("pools", []):
name = p.get("name", "?")
ntype = p.get("node_type", "?")
status = p.get("status", "?")
lb = f'pool="{sanitize(name)}",type="{sanitize(ntype)}",status="{sanitize(status)}"'
L.append(f"scaleway_pool_size{{{lb}}} {p.get('size', 0)}")
L.append(f"scaleway_pool_min{{{lb}}} {p.get('min_size', 0)}")
L.append(f"scaleway_pool_max{{{lb}}} {p.get('max_size', 0)}")
L.append(f"scaleway_pool_autoscaling{{{lb}}} {1 if p.get('autoscaling') else 0}")
L.append(f"scaleway_pool_root_volume_bytes{{{lb}}} {p.get('root_volume_size', 0)}")
# ── Block Storage (dynamic — auto-discovers volume additions/removals) ──
L.append("# HELP scaleway_volume_size_bytes Volume size in bytes")
L.append("# TYPE scaleway_volume_size_bytes gauge")
L.append("# HELP scaleway_volume_attached Volume attachment state (1=in_use, 0=available)")
L.append("# TYPE scaleway_volume_attached gauge")
total_bytes = attached_bytes = detached_bytes = 0
vol_count = attached_count = detached_count = 0
for v in volumes.get("volumes", []):
vid = v.get("id", "?")[:36]
name = v.get("name", "?")[:60]
size = v.get("size", 0)
state = v.get("status", "unknown")
attached = 1 if state == "in_use" else 0
total_bytes += size; vol_count += 1
if attached:
attached_bytes += size; attached_count += 1
else:
detached_bytes += size; detached_count += 1
L.append(f'scaleway_volume_size_bytes{{id="{sanitize(vid)}",name="{sanitize(name)}",state="{sanitize(state)}"}} {size}')
L.append(f'scaleway_volume_attached{{id="{sanitize(vid)}",name="{sanitize(name)}"}} {attached}')
L.append("# HELP scaleway_volumes_total_bytes Total block storage bytes")
L.append("# TYPE scaleway_volumes_total_bytes gauge")
L.append(f"scaleway_volumes_total_bytes {total_bytes}")
L.append("# HELP scaleway_volumes_attached_bytes Attached volume bytes")
L.append("# TYPE scaleway_volumes_attached_bytes gauge")
L.append(f"scaleway_volumes_attached_bytes {attached_bytes}")
L.append("# HELP scaleway_volumes_detached_bytes Detached (wasted) volume bytes")
L.append("# TYPE scaleway_volumes_detached_bytes gauge")
L.append(f"scaleway_volumes_detached_bytes {detached_bytes}")
L.append("# HELP scaleway_volumes_count Volume counts")
L.append("# TYPE scaleway_volumes_count gauge")
L.append(f'scaleway_volumes_count{{state="total"}} {vol_count}')
L.append(f'scaleway_volumes_count{{state="attached"}} {attached_count}')
L.append(f'scaleway_volumes_count{{state="detached"}} {detached_count}')
# ── IPs (dynamic) ──
L.append("# HELP scaleway_ips_total Total allocated IPs")
L.append("# TYPE scaleway_ips_total gauge")
L.append(f'scaleway_ips_total {len(ips.get("ips", []))}')
# ── Push to pushgateway ──
metrics = "\n".join(L) + "\n"
count = sum(1 for line in L if line.startswith("scaleway_"))
print(f"Generated {count} metrics ({len(metrics)} bytes)", file=sys.stderr)
# Validate: no empty lines between metrics (pushgateway is strict)
metrics = re.sub(r'\n{2,}', '\n', metrics)
url = f"{PUSHGATEWAY}/metrics/job/scaleway_billing"
req = urllib.request.Request(url, data=metrics.encode(), method="PUT")
req.add_header("Content-Type", "text/plain; version=0.0.4")
req.add_header("Connection", "close")
try:
resp = urllib.request.urlopen(req, timeout=30)
print(f"OK: pushed {count} metrics (HTTP {resp.status})")
except Exception as e:
print(f"WARN: PUT failed ({e}), trying POST...", file=sys.stderr)
req2 = urllib.request.Request(url, data=metrics.encode(), method="POST")
req2.add_header("Content-Type", "text/plain; version=0.0.4")
req2.add_header("Connection", "close")
try:
resp = urllib.request.urlopen(req2, timeout=30)
print(f"OK: pushed {count} metrics via POST (HTTP {resp.status})")
except Exception as e2:
print(f"ERROR: push failed: PUT={e}, POST={e2}", file=sys.stderr)
sys.exit(1)
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: scaleway-billing-scraper
namespace: foxhunt
spec:
schedule: "0 * * * *"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
activeDeadlineSeconds: 120
template:
metadata:
labels:
app: scaleway-billing
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
restartPolicy: OnFailure
containers:
- name: scraper
image: python:3.12-alpine
command: ["python3", "/scripts/scrape.py"]
envFrom:
- secretRef:
name: scaleway-billing
volumeMounts:
- name: script
mountPath: /scripts
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
volumes:
- name: script
configMap:
name: scaleway-billing-script
items:
- key: scrape.py
path: scrape.py
defaultMode: 0755

43
infra/k8s/monitoring/service-monitors.yaml
View File

@@ -1,43 +0,0 @@
# ServiceMonitors for Foxhunt services and GPU metrics
# Applied separately from the Helm chart (additionalServiceMonitors not rendered by chart)
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: foxhunt-services
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
selector:
matchLabels:
app.kubernetes.io/part-of: foxhunt
namespaceSelector:
matchNames: [foxhunt]
endpoints:
- port: metrics
interval: 15s
# Override job label to "foxhunt-services" (matching old standalone config)
# so all existing dashboard queries continue to work
relabelings:
- targetLabel: job
replacement: foxhunt-services
---
# DCGM Exporter (GPU metrics from nvidia-gpu-operator DaemonSet)
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: dcgm-exporter
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
selector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values: [dcgm-exporter]
namespaceSelector:
matchNames: [kube-system, monitoring, foxhunt]
endpoints:
- port: metrics
interval: 15s

139
infra/k8s/monitoring/tempo.yaml
View File

@@ -1,139 +0,0 @@
# Grafana Tempo — lightweight distributed tracing (single-binary mode)
# Receives OTLP traces from services, queryable via Grafana
apiVersion: v1
kind: ConfigMap
metadata:
name: tempo-config
namespace: foxhunt
labels:
app.kubernetes.io/name: tempo
data:
tempo.yaml: |
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: "0.0.0.0:4317"
http:
endpoint: "0.0.0.0:4318"
storage:
trace:
backend: local
local:
path: /var/tempo/traces
wal:
path: /var/tempo/wal
compactor:
compaction:
block_retention: 168h # 7 days
metrics_generator:
registry:
external_labels:
source: tempo
cluster: foxhunt
storage:
path: /var/tempo/generator/wal
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tempo
namespace: foxhunt
labels:
app.kubernetes.io/name: tempo
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: tempo
template:
metadata:
labels:
app.kubernetes.io/name: tempo
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
securityContext:
fsGroup: 10001
runAsUser: 10001
runAsNonRoot: true
containers:
- name: tempo
image: grafana/tempo:2.7.1
args: ["-config.file=/etc/tempo/tempo.yaml"]
ports:
- containerPort: 3200
name: http
- containerPort: 4317
name: otlp-grpc
- containerPort: 4318
name: otlp-http
volumeMounts:
- name: config
mountPath: /etc/tempo
- name: data
mountPath: /var/tempo
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: 500m
memory: 2Gi
readinessProbe:
httpGet:
path: /ready
port: 3200
initialDelaySeconds: 10
periodSeconds: 10
livenessProbe:
httpGet:
path: /ready
port: 3200
initialDelaySeconds: 30
periodSeconds: 15
volumes:
- name: config
configMap:
name: tempo-config
- name: data
persistentVolumeClaim:
claimName: tempo-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: tempo-data
namespace: foxhunt
spec:
accessModes: [ReadWriteOnce]
storageClassName: scw-bssd
resources:
requests:
storage: 10Gi
---
apiVersion: v1
kind: Service
metadata:
name: tempo
namespace: foxhunt
labels:
app.kubernetes.io/name: tempo
spec:
selector:
app.kubernetes.io/name: tempo
ports:
- port: 3200
targetPort: 3200
name: http
- port: 4317
targetPort: 4317
name: otlp-grpc
- port: 4318
targetPort: 4318
name: otlp-http

7
infra/k8s/namespace.yaml
View File

@@ -1,7 +0,0 @@
apiVersion: v1
kind: Namespace
metadata:
name: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
app.kubernetes.io/managed-by: kubectl

64
infra/k8s/network-policies/alertmanager.yaml
View File

@@ -1,64 +0,0 @@
# AlertManager network policy
# Ingress: Prometheus (alert delivery), Grafana (alert UI queries)
# Egress: Mattermost webhooks (cluster-internal), DNS
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: alertmanager
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: alertmanager
policyTypes:
- Ingress
- Egress
ingress:
# Prometheus pushes alerts and scrapes metrics (9093 API, 8080 config-reloader)
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- protocol: TCP
port: 9093
- protocol: TCP
port: 8080
# Grafana queries AlertManager API for alert state/silences
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: grafana
ports:
- protocol: TCP
port: 9093
egress:
# DNS resolution
- to:
- namespaceSelector: {}
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
# Mattermost incoming webhook (cluster-internal)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
ports:
- protocol: TCP
port: 8065
# External HTTPS (PagerDuty, email, future integrations)
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 172.16.0.0/12
- 192.168.0.0/16
ports:
- protocol: TCP
port: 443

26
infra/k8s/network-policies/allow-dns.yaml
View File

@@ -1,26 +0,0 @@
# Allow all foxhunt + CI pods to reach CoreDNS (kube-system) for name resolution.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-dns
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: [foxhunt, foxhunt-ci]
policyTypes:
- Egress
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53

76
infra/k8s/network-policies/allow-monitoring.yaml
View File

@@ -1,76 +0,0 @@
# Allow Prometheus to scrape metrics ports on all foxhunt + CI pods.
# Also allow all foxhunt + CI pods to push traces to Tempo.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-monitoring-scrape
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: ["foxhunt", "foxhunt-ci"]
policyTypes:
- Ingress
- Egress
ingress:
- from:
# External monitoring namespace (if any)
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: monitoring
# GitLab Prometheus in foxhunt namespace
- podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- protocol: TCP
port: 8080
- protocol: TCP
port: 9003
- protocol: TCP
port: 9091
- protocol: TCP
port: 9092
- protocol: TCP
port: 9093
- protocol: TCP
port: 9094
- protocol: TCP
port: 9095
- protocol: TCP
port: 9096
- protocol: TCP
port: 9097
- protocol: TCP
port: 9098
egress:
# Tempo OTLP gRPC
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: foxhunt
podSelector:
matchLabels:
app.kubernetes.io/name: tempo
ports:
- protocol: TCP
port: 4317
# Kubernetes API (for kube-state-metrics list/watch)
- to:
- ipBlock:
cidr: 10.32.0.0/16
ports:
- protocol: TCP
port: 443
- protocol: TCP
port: 6443
- to:
- ipBlock:
cidr: 172.16.0.0/16
ports:
- protocol: TCP
port: 6443

110
infra/k8s/network-policies/api.yaml
View File

@@ -1,110 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: api
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: api
policyTypes:
- Ingress
- Egress
ingress:
# Tailscale ingress proxy
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: tailscale-gitlab-proxy
ports:
- protocol: TCP
port: 50051
egress:
# Postgres
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
# Redis
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
# Downstream gRPC services
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: trading-service
ports:
- protocol: TCP
port: 50051
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: backtesting-service
ports:
- protocol: TCP
port: 50053
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: ml-training-service
ports:
- protocol: TCP
port: 50053
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: trading-agent-service
ports:
- protocol: TCP
port: 50055
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: broker-gateway
ports:
- protocol: TCP
port: 50056
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: data-acquisition-service
ports:
- protocol: TCP
port: 50057
# Prometheus (metric scraping from monitoring handler)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- protocol: TCP
port: 9090
# Minio (initContainer binary fetch)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000
# Kubernetes API (pod listing for Pods cockpit)
- to:
- ipBlock:
cidr: 10.32.0.0/12
ports:
- protocol: TCP
port: 443
- to:
- ipBlock:
cidr: 172.16.0.0/16
ports:
- protocol: TCP
port: 6443

42
infra/k8s/network-policies/backtesting-service.yaml
View File

@@ -1,42 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: backtesting-service
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: backtesting-service
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: api
ports:
- protocol: TCP
port: 50053
egress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000

52
infra/k8s/network-policies/broker-gateway.yaml
View File

@@ -1,52 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: broker-gateway
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: broker-gateway
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: api
- podSelector:
matchLabels:
app.kubernetes.io/name: trading-service
ports:
- protocol: TCP
port: 50056
egress:
- to:
- podSelector:
matchLabels:
app: ib-gateway
ports:
- protocol: TCP
port: 4002
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000

18
infra/k8s/network-policies/ci-egress.yaml
View File

@@ -1,18 +0,0 @@
# CI executor pods need broad egress: git clone (GitLab webservice), container
# registries, kubectl (K8s API), MinIO S3, and external tool downloads.
# These pods are ephemeral (minutes) — fine-grained egress adds little value.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ci-egress
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/part-of: foxhunt-ci
policyTypes:
- Egress
egress:
- {}

53
infra/k8s/network-policies/data-acquisition-service.yaml
View File

@@ -1,53 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: data-acquisition-service
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: data-acquisition-service
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: api
ports:
- protocol: TCP
port: 50057
egress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
# External HTTPS (Databento, Benzinga APIs)
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 172.16.0.0/12
- 192.168.0.0/16
ports:
- protocol: TCP
port: 443
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000

60
infra/k8s/network-policies/data-download-job.yaml
View File

@@ -1,60 +0,0 @@
# Egress policy for data-download jobs (MBP-10 download, etc.)
# Allows: MinIO (upload), Databento API (HTTPS download)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: data-download-job
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
foxhunt/job-type: data-download
policyTypes:
- Egress
egress:
# MinIO — binary fetch + data upload
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000
# Databento API (HTTPS)
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 172.16.0.0/12
- 192.168.0.0/16
- 100.64.0.0/10
ports:
- protocol: TCP
port: 443
---
# Egress policy for data-sync jobs (MinIO → PVC sync)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: data-sync-job
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
foxhunt/job-type: data-sync
policyTypes:
- Egress
egress:
# MinIO — data sync source
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000

16
infra/k8s/network-policies/default-deny.yaml
View File

@@ -1,16 +0,0 @@
# Default deny all ingress and egress in foxhunt namespace.
# Every service must declare its own NetworkPolicy to communicate.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny-all
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/part-of: foxhunt
policyTypes:
- Ingress
- Egress

24
infra/k8s/network-policies/gitlab-packages.yaml
View File

@@ -1,24 +0,0 @@
# Allow all foxhunt pods to fetch binaries from GitLab Generic Package Registry.
# initContainers in services and training jobs use curl to download release binaries
# from gitlab-webservice-default:8181 (GitLab API).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: gitlab-packages
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/part-of: foxhunt
policyTypes:
- Egress
egress:
- to:
- podSelector:
matchLabels:
app: webservice
ports:
- protocol: TCP
port: 8181

44
infra/k8s/network-policies/ib-gateway.yaml
View File

@@ -1,44 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ib-gateway
namespace: foxhunt
spec:
podSelector:
matchLabels:
app: ib-gateway
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: trading-service
- podSelector:
matchLabels:
app.kubernetes.io/name: broker-gateway
- podSelector:
matchLabels:
app: multistrat-rebalancer
ports:
- protocol: TCP
port: 4002
- protocol: TCP
port: 4004
egress:
# IBKR external servers
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 172.16.0.0/12
- 192.168.0.0/16
ports:
- protocol: TCP
port: 443
- protocol: TCP
port: 4000
- protocol: TCP
port: 4001

127
infra/k8s/network-policies/infrastructure.yaml
View File

@@ -1,127 +0,0 @@
# Postgres: accepts connections from foxhunt app pods, GitLab, Grafana, and Gitea
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: postgres
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: postgres
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: ["foxhunt", "argo-workflows"]
- podSelector:
matchLabels:
release: gitlab
- podSelector:
matchLabels:
app.kubernetes.io/name: grafana
- podSelector: # Gitea (Phase 2B) reuses the in-cluster postgres
matchLabels:
app.kubernetes.io/name: gitea
ports:
- protocol: TCP
port: 5432
---
# Redis: accepts connections from foxhunt app pods + GitLab (Helm release)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: redis
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: redis
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: ["foxhunt"]
- podSelector:
matchLabels:
release: gitlab
ports:
- protocol: TCP
port: 6379
---
# Minio: accepts connections from foxhunt app pods, GitLab, and init jobs
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: minio
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: minio
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: ["foxhunt", "foxhunt-ci", "argo-workflows"]
- podSelector:
matchLabels:
release: gitlab
- podSelector:
matchLabels:
app.kubernetes.io/name: minio-init
- podSelector: # tailscale proxy fronts minio.fxhnt.ai (S3 API)
matchLabels:
app.kubernetes.io/name: tailscale-gitlab-proxy
ports:
- protocol: TCP
port: 9000
---
# QuestDB: accepts ILP writes from trading-service + Prometheus metrics scrape
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: questdb
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: questdb
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: trading-service
- podSelector:
matchLabels:
foxhunt/job-type: training
ports:
- protocol: TCP
port: 9009
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- protocol: TCP
port: 9003
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: grafana
ports:
- protocol: TCP
port: 8812

67
infra/k8s/network-policies/maintenance-jobs.yaml
View File

@@ -1,67 +0,0 @@
# Network policies for maintenance CronJobs (postgres-backup, pvc-autoscaler)
# These jobs need egress to internal services but no internet access.
# Apply: kubectl apply -f maintenance-jobs.yaml
---
# Postgres backup job: egress to postgres (5432) + minio (9000)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: postgres-backup
namespace: foxhunt
spec:
podSelector:
matchLabels:
app: postgres-backup
policyTypes:
- Egress
egress:
# DNS
- ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
# Postgres
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
# MinIO (backup upload)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000
---
# PVC autoscaler job: egress to K8s API + kubelet stats
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: pvc-autoscaler
namespace: foxhunt
spec:
podSelector:
matchLabels:
app: pvc-autoscaler
policyTypes:
- Egress
egress:
# DNS
- ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
# Kubernetes API server (HTTPS)
- ports:
- protocol: TCP
port: 443
# Kubelet stats API (port 10250)
- ports:
- protocol: TCP
port: 10250

42
infra/k8s/network-policies/mattermost.yaml
View File

@@ -1,42 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: mattermost
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
policyTypes:
- Ingress
- Egress
ingress:
# HTTP from Tailscale proxy + Argo workflow pods (webhook notifications)
- ports:
- port: 8065
protocol: TCP
egress:
# DNS
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# PostgreSQL
- ports:
- port: 5432
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
# Stalwart SMTP (for email notifications, wired later)
- ports:
- port: 25
protocol: TCP
to:
- podSelector:
matchLabels:
app.kubernetes.io/name: stalwart

49
infra/k8s/network-policies/ml-training-service.yaml
View File

@@ -1,49 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ml-training-service
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ml-training-service
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: api
ports:
- protocol: TCP
port: 50053
egress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000
# Kubernetes API (for job creation — ml-training-service has RBAC for batch/jobs)
- to:
- ipBlock:
cidr: 0.0.0.0/0
ports:
- protocol: TCP
port: 443

23
infra/k8s/network-policies/prometheus-cilium.yaml
View File

@@ -1,23 +0,0 @@
# CiliumNetworkPolicy: allow Prometheus to reach hostNetwork pods (node-exporter).
#
# Vanilla K8s NetworkPolicy ipBlock rules don't match traffic to hostNetwork
# pods because Cilium assigns them the 'host'/'remote-node' identity, not 'world'.
# This CiliumNetworkPolicy uses toEntities to cover that gap.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: prometheus-host-access
namespace: foxhunt
spec:
endpointSelector:
matchLabels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: foxhunt
egress:
- toEntities:
- host
- remote-node
toPorts:
- ports:
- port: "9100"
protocol: TCP

188
infra/k8s/network-policies/prometheus.yaml
View File

@@ -1,188 +0,0 @@
# Prometheus egress: K8s API (service discovery), foxhunt services, node-exporter,
# kube-state-metrics, DCGM exporter, Argo workflow controller, pushgateway,
# Loki, Tempo, QuestDB, and AlertManager.
# Applies to operator-managed Prometheus (kube-prometheus-stack).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: prometheus
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
policyTypes:
- Ingress
- Egress
ingress:
# Grafana queries Prometheus on port 9090
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: grafana
ports:
- protocol: TCP
port: 9090
egress:
# DNS resolution (required for additionalScrapeConfigs with hostnames)
- to:
- namespaceSelector: {}
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
# Kubernetes API server (required for kubernetes_sd_configs service discovery)
# ClusterIP 10.32.0.1:443 NATs to node IP 172.16.0.0/16:6443
- to:
- ipBlock:
cidr: 10.32.0.0/16
ports:
- protocol: TCP
port: 443
- protocol: TCP
port: 6443
# Scrape foxhunt service metrics ports (9003, 9090-9098)
- to:
- podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: ["foxhunt", "foxhunt-ci"]
ports:
- protocol: TCP
port: 9003
- protocol: TCP
port: 9090
- protocol: TCP
port: 9091
- protocol: TCP
port: 9092
- protocol: TCP
port: 9093
- protocol: TCP
port: 9094
- protocol: TCP
port: 9095
- protocol: TCP
port: 9096
- protocol: TCP
port: 9097
- protocol: TCP
port: 9098
# kube-state-metrics (Helm sub-chart, port 8080)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: kube-state-metrics
ports:
- protocol: TCP
port: 8080
# Prometheus operator (container port 10250)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator
ports:
- protocol: TCP
port: 10250
# Alertmanager (port 9093 for API, port 8080 for config-reloader sidecar)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: alertmanager
ports:
- protocol: TCP
port: 9093
- protocol: TCP
port: 8080
# Loki metrics (port 3100) and Tempo metrics (port 3200)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: loki
ports:
- protocol: TCP
port: 3100
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: tempo
ports:
- protocol: TCP
port: 3200
# QuestDB metrics
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: questdb
ports:
- protocol: TCP
port: 9003
# Node-exporter (hostNetwork), kubelet, and K8s API on node IPs
- to:
- ipBlock:
cidr: 172.16.0.0/16
ports:
- protocol: TCP
port: 6443
- protocol: TCP
port: 9100
- protocol: TCP
port: 9400
- protocol: TCP
port: 10250 # kubelet metrics (cAdvisor, resource usage)
# kube-system: CoreDNS metrics, kube-state-metrics, node-exporter, DCGM
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: TCP
port: 8080
- protocol: TCP
port: 8443
- protocol: TCP
port: 9100
- protocol: TCP
port: 9153 # CoreDNS metrics
- protocol: TCP
port: 9400
# monitoring namespace: DCGM exporter (GPU metrics)
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: monitoring
ports:
- protocol: TCP
port: 9400
# Pushgateway (training pods push metrics here)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: pushgateway
ports:
- protocol: TCP
port: 9091
# Argo Workflows controller metrics
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: argo-workflows-workflow-controller
ports:
- protocol: TCP
port: 9090

49
infra/k8s/network-policies/stalwart.yaml
View File

@@ -1,49 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: stalwart
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: stalwart
policyTypes:
- Ingress
- Egress
ingress:
# SMTP from cluster (Mattermost internal relay)
- ports:
- port: 25
protocol: TCP
from:
- podSelector:
matchLabels:
app.kubernetes.io/name: mattermost
# IMAPS from Tailscale proxy
- ports:
- port: 993
protocol: TCP
# HTTPS admin panel from Tailscale proxy
- ports:
- port: 443
protocol: TCP
egress:
# DNS
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
# HTTPS out — webadmin bundle + spam model downloads from GitHub
- ports:
- port: 443
protocol: TCP
# No outbound SMTP — internal-only for now.
# To add SCW TEM relay later:
# - ports:
# - port: 465
# protocol: TCP
# - port: 587
# protocol: TCP

49
infra/k8s/network-policies/trading-agent-service.yaml
View File

@@ -1,49 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: trading-agent-service
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: trading-agent-service
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: api
ports:
- protocol: TCP
port: 50055
egress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: trading-service
ports:
- protocol: TCP
port: 50051
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000

59
infra/k8s/network-policies/trading-service.yaml
View File

@@ -1,59 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: trading-service
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: trading-service
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: api
- podSelector:
matchLabels:
app.kubernetes.io/name: trading-agent-service
ports:
- protocol: TCP
port: 50051
egress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
- to:
- podSelector:
matchLabels:
app: ib-gateway
ports:
- protocol: TCP
port: 4002
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: questdb
ports:
- protocol: TCP
port: 9009
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000

47
infra/k8s/network-policies/training-job.yaml
View File

@@ -1,47 +0,0 @@
# Egress policy for training jobs (binary fetch, data sync, telemetry)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: training-job
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
foxhunt/job-type: training
policyTypes:
- Egress
egress:
# MinIO — binary fetch + data sync
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000
# Tempo — OTLP tracing
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: tempo
ports:
- protocol: TCP
port: 4317
# Pushgateway — Prometheus metrics push
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: pushgateway
ports:
- protocol: TCP
port: 9091
# QuestDB — ILP training metrics persistence
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: questdb
ports:
- protocol: TCP
port: 9009

29
infra/k8s/network-policies/web-dashboard.yaml
View File

@@ -1,29 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: web-dashboard
namespace: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: web-dashboard
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: tailscale-gitlab-proxy
ports:
- protocol: TCP
port: 80
egress:
# Init container fetches assets from MinIO
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: minio
ports:
- protocol: TCP
port: 9000

19
infra/k8s/secrets/databento-credentials.secret.example
View File

@@ -1,19 +0,0 @@
# Databento API credentials for MBP-10 data download
# Copy this file and fill in your API key:
# cp databento-credentials.secret.example databento-credentials.yaml
# # Edit databento-credentials.yaml with your API key
# kubectl apply -f databento-credentials.yaml
#
# Or create imperatively:
# kubectl -n foxhunt create secret generic databento-credentials \
# --from-literal=api-key=$DATABENTO_API_KEY
apiVersion: v1
kind: Secret
metadata:
name: databento-credentials
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
api-key: REPLACE_WITH_YOUR_DATABENTO_API_KEY

58
infra/k8s/secrets/foxhunt-secrets.yaml
View File

@@ -1,58 +0,0 @@
# Database credentials — used by services that connect to PostgreSQL.
# Real values injected by scripts/deploy-secrets.sh from Scaleway Secrets Manager.
apiVersion: v1
kind: Secret
metadata:
name: db-credentials
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
password: "REPLACE_IN_DEPLOY"
---
# JWT signing secret — used by services that validate/issue JWTs.
apiVersion: v1
kind: Secret
metadata:
name: jwt-secret
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
secret: "REPLACE_IN_DEPLOY"
---
# Redis credentials — used by services with Redis connections.
apiVersion: v1
kind: Secret
metadata:
name: redis-credentials
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
password: "REPLACE_IN_DEPLOY"
---
# S3/Minio credentials for application-level access (model storage, data).
# NOT the same as minio-credentials (used by initContainers for binary fetch).
apiVersion: v1
kind: Secret
metadata:
name: s3-credentials
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
access-key: "REPLACE_IN_DEPLOY"
secret-key: "REPLACE_IN_DEPLOY"
---
# Registry pull secret for Scaleway Container Registry
# Create with:
# kubectl create secret docker-registry scw-registry \
# --namespace foxhunt \
# --docker-server=rg.fr-par.scw.cloud \
# --docker-username=foxhunt \
# --docker-password=<SCW_SECRET_KEY>

Some files were not shown because too many files have changed in this diff Show More