From 2a3c107b31cb4b453e3321d2388c7e7c5d385d23 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 24 Feb 2026 14:27:32 +0100 Subject: [PATCH] infra: add K8s manifests for full paper trading stack - Tailscale subnet router (zero public access) - PostgreSQL, Redis, QuestDB (PVCs on always-on node) - 7 service deployments (trading, api-gw, broker-gw, ml, backtest, agent, web-gw) - GPU training Job template + idle reaper CronJob - Block Storage PV/PVC for shared training data Co-Authored-By: Claude Opus 4.6 --- infra/k8s/databases/postgres.yaml | 91 ++++++++++++++++++ infra/k8s/databases/questdb.yaml | 92 +++++++++++++++++++ infra/k8s/databases/redis.yaml | 62 +++++++++++++ infra/k8s/services/api-gateway.yaml | 92 +++++++++++++++++++ infra/k8s/services/backtesting-service.yaml | 83 +++++++++++++++++ infra/k8s/services/broker-gateway.yaml | 84 +++++++++++++++++ infra/k8s/services/ml-training-service.yaml | 87 ++++++++++++++++++ infra/k8s/services/trading-agent-service.yaml | 83 +++++++++++++++++ infra/k8s/services/trading-service.yaml | 88 ++++++++++++++++++ infra/k8s/services/web-gateway.yaml | 77 ++++++++++++++++ infra/k8s/storage/training-data-pv.yaml | 32 +++++++ infra/k8s/tailscale/deployment.yaml | 52 +++++++++++ infra/k8s/tailscale/namespace.yaml | 7 ++ infra/k8s/tailscale/rbac.yaml | 38 ++++++++ infra/k8s/tailscale/secret.yaml.example | 11 +++ infra/k8s/training/idle-reaper.yaml | 75 +++++++++++++++ infra/k8s/training/job-template.yaml | 63 +++++++++++++ 17 files changed, 1117 insertions(+) create mode 100644 infra/k8s/databases/postgres.yaml create mode 100644 infra/k8s/databases/questdb.yaml create mode 100644 infra/k8s/databases/redis.yaml create mode 100644 infra/k8s/services/api-gateway.yaml create mode 100644 infra/k8s/services/backtesting-service.yaml create mode 100644 infra/k8s/services/broker-gateway.yaml create mode 100644 infra/k8s/services/ml-training-service.yaml create mode 100644 infra/k8s/services/trading-agent-service.yaml create mode 100644 infra/k8s/services/trading-service.yaml create mode 100644 infra/k8s/services/web-gateway.yaml create mode 100644 infra/k8s/storage/training-data-pv.yaml create mode 100644 infra/k8s/tailscale/deployment.yaml create mode 100644 infra/k8s/tailscale/namespace.yaml create mode 100644 infra/k8s/tailscale/rbac.yaml create mode 100644 infra/k8s/tailscale/secret.yaml.example create mode 100644 infra/k8s/training/idle-reaper.yaml create mode 100644 infra/k8s/training/job-template.yaml diff --git a/infra/k8s/databases/postgres.yaml b/infra/k8s/databases/postgres.yaml new file mode 100644 index 000000000..cddfd531a --- /dev/null +++ b/infra/k8s/databases/postgres.yaml @@ -0,0 +1,91 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc + namespace: foxhunt + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: foxhunt +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: foxhunt + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: postgres + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + containers: + - name: postgres + image: timescale/timescaledb:latest-pg16 + ports: + - containerPort: 5432 + name: postgres + env: + - name: POSTGRES_DB + value: foxhunt + - name: POSTGRES_USER + value: foxhunt + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: DATABASE_PASSWORD + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + readinessProbe: + exec: + command: + - pg_isready + - -U + - foxhunt + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: foxhunt + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: postgres + ports: + - port: 5432 + targetPort: 5432 + name: postgres diff --git a/infra/k8s/databases/questdb.yaml b/infra/k8s/databases/questdb.yaml new file mode 100644 index 000000000..93ece0eaf --- /dev/null +++ b/infra/k8s/databases/questdb.yaml @@ -0,0 +1,92 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: questdb-pvc + namespace: foxhunt + labels: + app.kubernetes.io/name: questdb + app.kubernetes.io/part-of: foxhunt +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: questdb + namespace: foxhunt + labels: + app.kubernetes.io/name: questdb + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: questdb + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: questdb + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + containers: + - name: questdb + image: questdb/questdb:8.2.3 + ports: + - containerPort: 9009 + name: ilp + - containerPort: 8812 + name: pg + - containerPort: 9003 + name: http + env: + - name: QDB_PG_ENABLED + value: "true" + volumeMounts: + - name: questdb-data + mountPath: /var/lib/questdb + readinessProbe: + httpGet: + path: /exec?query=SELECT%201 + port: 9003 + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + volumes: + - name: questdb-data + persistentVolumeClaim: + claimName: questdb-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: questdb + namespace: foxhunt + labels: + app.kubernetes.io/name: questdb + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: questdb + ports: + - port: 9009 + targetPort: 9009 + name: ilp + - port: 8812 + targetPort: 8812 + name: pg + - port: 9003 + targetPort: 9003 + name: http diff --git a/infra/k8s/databases/redis.yaml b/infra/k8s/databases/redis.yaml new file mode 100644 index 000000000..dc4c02e5b --- /dev/null +++ b/infra/k8s/databases/redis.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: foxhunt + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: redis + template: + metadata: + labels: + app.kubernetes.io/name: redis + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + containers: + - name: redis + image: redis:7-alpine + command: + - redis-server + - --maxmemory + - 512mb + - --maxmemory-policy + - allkeys-lru + ports: + - containerPort: 6379 + name: redis + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 250m + memory: 640Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: foxhunt + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: redis + ports: + - port: 6379 + targetPort: 6379 + name: redis diff --git a/infra/k8s/services/api-gateway.yaml b/infra/k8s/services/api-gateway.yaml new file mode 100644 index 000000000..99254cdd4 --- /dev/null +++ b/infra/k8s/services/api-gateway.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api-gateway + namespace: foxhunt + labels: + app.kubernetes.io/name: api-gateway + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: api-gateway + template: + metadata: + labels: + app.kubernetes.io/name: api-gateway + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + imagePullSecrets: + - name: scw-registry + containers: + - name: api-gateway + image: rg.nl-ams.scw.cloud/foxhunt/api_gateway:latest + ports: + - containerPort: 50050 + name: grpc + - containerPort: 9091 + name: metrics + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: DATABASE_PASSWORD + - name: DATABASE_URL + value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" + - name: REDIS_URL + value: "redis://redis:6379" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: JWT_SECRET + - name: JWT_ISSUER + value: foxhunt-api-gateway + - name: JWT_AUDIENCE + value: foxhunt-services + - name: TRADING_SERVICE_URL + value: "http://trading-service:50051" + - name: BACKTESTING_SERVICE_URL + value: "http://backtesting-service:50053" + - name: ML_TRAINING_SERVICE_URL + value: "http://ml-training-service:50053" + - name: TRADING_AGENT_SERVICE_URL + value: "http://trading-agent-service:50055" + - name: RUST_LOG + value: info + readinessProbe: + exec: + command: + - grpc_health_probe + - -addr=localhost:50050 + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: api-gateway + namespace: foxhunt + labels: + app.kubernetes.io/name: api-gateway + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: api-gateway + ports: + - port: 50050 + targetPort: 50050 + name: grpc + - port: 9091 + targetPort: 9091 + name: metrics diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml new file mode 100644 index 000000000..f648a1faa --- /dev/null +++ b/infra/k8s/services/backtesting-service.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backtesting-service + namespace: foxhunt + labels: + app.kubernetes.io/name: backtesting-service + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: backtesting-service + template: + metadata: + labels: + app.kubernetes.io/name: backtesting-service + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + imagePullSecrets: + - name: scw-registry + containers: + - name: backtesting-service + image: rg.nl-ams.scw.cloud/foxhunt/backtesting_service:latest + ports: + - containerPort: 50053 + name: grpc + - containerPort: 9093 + name: metrics + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: DATABASE_PASSWORD + - name: DATABASE_URL + value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" + - name: REDIS_URL + value: "redis://redis:6379" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: JWT_SECRET + - name: JWT_ISSUER + value: foxhunt-api-gateway + - name: JWT_AUDIENCE + value: foxhunt-services + - name: RUST_LOG + value: info + readinessProbe: + httpGet: + path: /health + port: 8082 + initialDelaySeconds: 15 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: backtesting-service + namespace: foxhunt + labels: + app.kubernetes.io/name: backtesting-service + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: backtesting-service + ports: + - port: 50053 + targetPort: 50053 + name: grpc + - port: 9093 + targetPort: 9093 + name: metrics diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml new file mode 100644 index 000000000..f95efb350 --- /dev/null +++ b/infra/k8s/services/broker-gateway.yaml @@ -0,0 +1,84 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: broker-gateway + namespace: foxhunt + labels: + app.kubernetes.io/name: broker-gateway + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: broker-gateway + template: + metadata: + labels: + app.kubernetes.io/name: broker-gateway + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + imagePullSecrets: + - name: scw-registry + containers: + - name: broker-gateway + image: rg.nl-ams.scw.cloud/foxhunt/broker_gateway_service:latest + ports: + - containerPort: 50056 + name: grpc + - containerPort: 9096 + name: metrics + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: DATABASE_PASSWORD + - name: DATABASE_URL + value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" + - name: REDIS_URL + value: "redis://redis:6379" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: JWT_SECRET + - name: JWT_ISSUER + value: foxhunt-api-gateway + - name: JWT_AUDIENCE + value: foxhunt-services + - name: RUST_LOG + value: info + readinessProbe: + exec: + command: + - grpc_health_probe + - -addr=localhost:50056 + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: broker-gateway + namespace: foxhunt + labels: + app.kubernetes.io/name: broker-gateway + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: broker-gateway + ports: + - port: 50056 + targetPort: 50056 + name: grpc + - port: 9096 + targetPort: 9096 + name: metrics diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml new file mode 100644 index 000000000..e94026bcf --- /dev/null +++ b/infra/k8s/services/ml-training-service.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-training-service + namespace: foxhunt + labels: + app.kubernetes.io/name: ml-training-service + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ml-training-service + template: + metadata: + labels: + app.kubernetes.io/name: ml-training-service + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + imagePullSecrets: + - name: scw-registry + containers: + - name: ml-training-service + image: rg.nl-ams.scw.cloud/foxhunt/ml_training_service:latest + ports: + - containerPort: 50053 + name: grpc + - containerPort: 9094 + name: metrics + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: DATABASE_PASSWORD + - name: DATABASE_URL + value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" + - name: REDIS_URL + value: "redis://redis:6379" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: JWT_SECRET + - name: JWT_ISSUER + value: foxhunt-api-gateway + - name: JWT_AUDIENCE + value: foxhunt-services + - name: S3_ENDPOINT + value: "https://s3.nl-ams.scw.cloud" + - name: S3_BUCKET + value: foxhunt-artifacts + - name: RUST_LOG + value: info + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: ml-training-service + namespace: foxhunt + labels: + app.kubernetes.io/name: ml-training-service + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: ml-training-service + ports: + - port: 50053 + targetPort: 50053 + name: grpc + - port: 9094 + targetPort: 9094 + name: metrics diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml new file mode 100644 index 000000000..6053d5ed9 --- /dev/null +++ b/infra/k8s/services/trading-agent-service.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-agent-service + namespace: foxhunt + labels: + app.kubernetes.io/name: trading-agent-service + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: trading-agent-service + template: + metadata: + labels: + app.kubernetes.io/name: trading-agent-service + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + imagePullSecrets: + - name: scw-registry + containers: + - name: trading-agent-service + image: rg.nl-ams.scw.cloud/foxhunt/trading_agent_service:latest + ports: + - containerPort: 50055 + name: grpc + - containerPort: 9095 + name: metrics + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: DATABASE_PASSWORD + - name: DATABASE_URL + value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" + - name: REDIS_URL + value: "redis://redis:6379" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: JWT_SECRET + - name: JWT_ISSUER + value: foxhunt-api-gateway + - name: JWT_AUDIENCE + value: foxhunt-services + - name: RUST_LOG + value: info + readinessProbe: + httpGet: + path: /health + port: 8083 + initialDelaySeconds: 15 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: trading-agent-service + namespace: foxhunt + labels: + app.kubernetes.io/name: trading-agent-service + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: trading-agent-service + ports: + - port: 50055 + targetPort: 50055 + name: grpc + - port: 9095 + targetPort: 9095 + name: metrics diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml new file mode 100644 index 000000000..c54191834 --- /dev/null +++ b/infra/k8s/services/trading-service.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-service + namespace: foxhunt + labels: + app.kubernetes.io/name: trading-service + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: trading-service + template: + metadata: + labels: + app.kubernetes.io/name: trading-service + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + imagePullSecrets: + - name: scw-registry + containers: + - name: trading-service + image: rg.nl-ams.scw.cloud/foxhunt/trading_service:latest + ports: + - containerPort: 50051 + name: grpc + - containerPort: 9092 + name: metrics + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: DATABASE_PASSWORD + - name: DATABASE_URL + value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" + - name: REDIS_URL + value: "redis://redis:6379" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: JWT_SECRET + - name: JWT_ISSUER + value: foxhunt-api-gateway + - name: JWT_AUDIENCE + value: foxhunt-services + - name: QUESTDB_ILP_HOST + value: "questdb:9009" + - name: GRPC_PORT + value: "50051" + - name: RUST_LOG + value: info + readinessProbe: + exec: + command: + - grpc_health_probe + - -addr=localhost:50051 + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: trading-service + namespace: foxhunt + labels: + app.kubernetes.io/name: trading-service + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: trading-service + ports: + - port: 50051 + targetPort: 50051 + name: grpc + - port: 9092 + targetPort: 9092 + name: metrics diff --git a/infra/k8s/services/web-gateway.yaml b/infra/k8s/services/web-gateway.yaml new file mode 100644 index 000000000..e2cde7865 --- /dev/null +++ b/infra/k8s/services/web-gateway.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web-gateway + namespace: foxhunt + labels: + app.kubernetes.io/name: web-gateway + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: web-gateway + template: + metadata: + labels: + app.kubernetes.io/name: web-gateway + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + imagePullSecrets: + - name: scw-registry + containers: + - name: web-gateway + image: rg.nl-ams.scw.cloud/foxhunt/web-gateway:latest + ports: + - containerPort: 3000 + name: http + env: + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: JWT_SECRET + - name: TRADING_SERVICE_URL + value: "http://trading-service:50051" + - name: BACKTESTING_SERVICE_URL + value: "http://backtesting-service:50053" + - name: ML_TRAINING_SERVICE_URL + value: "http://ml-training-service:50053" + - name: TRADING_AGENT_SERVICE_URL + value: "http://trading-agent-service:50055" + - name: BROKER_GATEWAY_SERVICE_URL + value: "http://broker-gateway:50056" + - name: API_GATEWAY_URL + value: "http://api-gateway:50050" + - name: RUST_LOG + value: info + readinessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: web-gateway + namespace: foxhunt + labels: + app.kubernetes.io/name: web-gateway + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: web-gateway + ports: + - port: 3000 + targetPort: 3000 + name: http diff --git a/infra/k8s/storage/training-data-pv.yaml b/infra/k8s/storage/training-data-pv.yaml new file mode 100644 index 000000000..31eccbea0 --- /dev/null +++ b/infra/k8s/storage/training-data-pv.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: training-data-pv + labels: + app.kubernetes.io/name: training-data + app.kubernetes.io/part-of: foxhunt +spec: + capacity: + storage: 100Gi + accessModes: + - ReadOnlyMany + csi: + driver: csi.scaleway.com + volumeHandle: "VOLUME_ID" + persistentVolumeReclaimPolicy: Retain +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: training-data-pvc + namespace: foxhunt + labels: + app.kubernetes.io/name: training-data + app.kubernetes.io/part-of: foxhunt +spec: + accessModes: + - ReadOnlyMany + resources: + requests: + storage: 100Gi + volumeName: training-data-pv diff --git a/infra/k8s/tailscale/deployment.yaml b/infra/k8s/tailscale/deployment.yaml new file mode 100644 index 000000000..8dde9832c --- /dev/null +++ b/infra/k8s/tailscale/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tailscale-subnet-router + namespace: tailscale + labels: + app.kubernetes.io/name: tailscale-subnet-router + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tailscale-subnet-router + template: + metadata: + labels: + app.kubernetes.io/name: tailscale-subnet-router + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + serviceAccountName: tailscale + containers: + - name: tailscale + image: ghcr.io/tailscale/tailscale:latest + env: + - name: TS_AUTHKEY + valueFrom: + secretKeyRef: + name: tailscale-auth + key: TS_AUTHKEY + - name: TS_KUBE_SECRET + value: tailscale-state + - name: TS_USERSPACE + value: "false" + - name: TS_ROUTES + value: "10.42.0.0/16" + - name: TS_HOSTNAME + value: foxhunt-kapsule + - name: TS_ACCEPT_DNS + value: "false" + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi diff --git a/infra/k8s/tailscale/namespace.yaml b/infra/k8s/tailscale/namespace.yaml new file mode 100644 index 000000000..dc31e9da5 --- /dev/null +++ b/infra/k8s/tailscale/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tailscale + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/managed-by: kubectl diff --git a/infra/k8s/tailscale/rbac.yaml b/infra/k8s/tailscale/rbac.yaml new file mode 100644 index 000000000..f2811afd4 --- /dev/null +++ b/infra/k8s/tailscale/rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tailscale + namespace: tailscale + labels: + app.kubernetes.io/name: tailscale + app.kubernetes.io/part-of: foxhunt +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tailscale + namespace: tailscale + labels: + app.kubernetes.io/name: tailscale + app.kubernetes.io/part-of: foxhunt +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tailscale + namespace: tailscale + labels: + app.kubernetes.io/name: tailscale + app.kubernetes.io/part-of: foxhunt +subjects: + - kind: ServiceAccount + name: tailscale + namespace: tailscale +roleRef: + kind: Role + name: tailscale + apiGroup: rbac.authorization.k8s.io diff --git a/infra/k8s/tailscale/secret.yaml.example b/infra/k8s/tailscale/secret.yaml.example new file mode 100644 index 000000000..f80d99333 --- /dev/null +++ b/infra/k8s/tailscale/secret.yaml.example @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: tailscale-auth + namespace: tailscale + labels: + app.kubernetes.io/name: tailscale + app.kubernetes.io/part-of: foxhunt +type: Opaque +stringData: + TS_AUTHKEY: "tskey-auth-REPLACE_ME" diff --git a/infra/k8s/training/idle-reaper.yaml b/infra/k8s/training/idle-reaper.yaml new file mode 100644 index 000000000..9a56e5732 --- /dev/null +++ b/infra/k8s/training/idle-reaper.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gpu-reaper + namespace: foxhunt + labels: + app.kubernetes.io/name: gpu-reaper + app.kubernetes.io/part-of: foxhunt +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gpu-reaper + namespace: foxhunt + labels: + app.kubernetes.io/name: gpu-reaper + app.kubernetes.io/part-of: foxhunt +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gpu-reaper + namespace: foxhunt + labels: + app.kubernetes.io/name: gpu-reaper + app.kubernetes.io/part-of: foxhunt +subjects: + - kind: ServiceAccount + name: gpu-reaper + namespace: foxhunt +roleRef: + kind: Role + name: gpu-reaper + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: gpu-idle-reaper + namespace: foxhunt + labels: + app.kubernetes.io/name: gpu-idle-reaper + app.kubernetes.io/part-of: foxhunt +spec: + schedule: "*/5 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + serviceAccountName: gpu-reaper + restartPolicy: Never + containers: + - name: reaper + image: bitnami/kubectl:latest + command: + - /bin/sh + - -c + - | + ACTIVE=$(kubectl get jobs -n foxhunt -l foxhunt/job-type=training \ + --field-selector=status.active=1 --no-headers 2>/dev/null | wc -l) + echo "Active training jobs: $ACTIVE" + if [ "$ACTIVE" -eq 0 ]; then + echo "No active training jobs. GPU pool can be scaled down." + else + echo "Training jobs still running. GPU pool stays up." + fi diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml new file mode 100644 index 000000000..26f4a7d07 --- /dev/null +++ b/infra/k8s/training/job-template.yaml @@ -0,0 +1,63 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: training-MODEL-TIMESTAMP + namespace: foxhunt + labels: + app.kubernetes.io/name: training + app.kubernetes.io/part-of: foxhunt + foxhunt/job-type: training +spec: + backoffLimit: 1 + activeDeadlineSeconds: 3600 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app.kubernetes.io/name: training + foxhunt/job-type: training + spec: + nodeSelector: + k8s.scaleway.com/pool-name: gpu + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + imagePullSecrets: + - name: scw-registry + restartPolicy: Never + containers: + - name: training + image: rg.nl-ams.scw.cloud/foxhunt/training:latest + command: ["./train"] + args: + - "--model=dqn" + - "--symbol=ES.FUT" + - "--data-dir=/data" + - "--output-dir=/output" + env: + - name: RUST_LOG + value: info + - name: SQLX_OFFLINE + value: "true" + volumeMounts: + - name: training-data + mountPath: /data + readOnly: true + - name: output + mountPath: /output + resources: + requests: + nvidia.com/gpu: "1" + cpu: "4" + memory: 16Gi + limits: + nvidia.com/gpu: "1" + cpu: "8" + memory: 32Gi + volumes: + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + - name: output + emptyDir: {}