diff --git a/.gitignore b/.gitignore index b044a8572..4cb572844 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,12 @@ coordination/orchestration/* *.sqlite-wal claude-flow +# Terragrunt cache and generated files +.terragrunt-cache/ +**/.terragrunt-cache/ +.terraform.lock.hcl +**/.terraform.lock.hcl + # Data cache (downloaded market data, not checked in) data/cache/* # Exception: futures baseline training data is checked in diff --git a/common/src/database.rs b/common/src/database.rs index 806cf03b7..d04f86da6 100644 --- a/common/src/database.rs +++ b/common/src/database.rs @@ -96,8 +96,8 @@ impl Default for PoolConfig { Self { max_connections: 50, min_connections: 10, - connect_timeout_ms: 100, - acquire_timeout_ms: 50, + connect_timeout_ms: 5000, + acquire_timeout_ms: 5000, max_lifetime_seconds: 3600, idle_timeout_seconds: 300, } @@ -126,9 +126,9 @@ impl From for LocalDatabaseConfig { max_connections: config.max_connections, // 20% of max, min 2. Uses integer division intentionally for simplicity. min_connections: (config.max_connections / 5).max(2), - connect_timeout_ms: u64::try_from(config.connect_timeout.as_millis().min(100)) - .unwrap_or(100), // Convert to ms, cap at 100ms for HFT - acquire_timeout_ms: 50, // Fast acquire for HFT + connect_timeout_ms: u64::try_from(config.connect_timeout.as_millis()) + .unwrap_or(5000), // Pool connect timeout (not query timeout) + acquire_timeout_ms: 5000, // Pool acquire timeout (not query timeout) max_lifetime_seconds: 3600, // 1 hour default idle_timeout_seconds: 300, // 5 minutes default }, @@ -156,7 +156,7 @@ impl From for LocalDatabaseConfig { // 25% of max, min 2. Uses integer division intentionally for simplicity. min_connections: (max_conn / 4).max(2), connect_timeout_ms: config.acquire_timeout_ms.unwrap_or(1000), // Use acquire timeout as connection timeout - acquire_timeout_ms: 100, // Less strict for backtesting + acquire_timeout_ms: 5000, // Pool acquire timeout (not query timeout) max_lifetime_seconds: 3600, // 1 hour default idle_timeout_seconds: 600, // 10 minutes for backtesting }, diff --git a/infra/docker/Dockerfile.service b/infra/docker/Dockerfile.service index 02407a222..437a95398 100644 --- a/infra/docker/Dockerfile.service +++ b/infra/docker/Dockerfile.service @@ -12,7 +12,7 @@ ARG SERVICE ARG SCCACHE_BUCKET="" ARG AWS_ACCESS_KEY_ID="" ARG AWS_SECRET_ACCESS_KEY="" -ARG AWS_DEFAULT_REGION="nl-ams" +ARG AWS_DEFAULT_REGION="fr-par" ARG SCCACHE_ENDPOINT="" RUN test -n "${SERVICE}" || (echo "ERROR: SERVICE build-arg is required" && exit 1) @@ -67,7 +67,6 @@ COPY database ./database COPY config ./config COPY web-gateway ./web-gateway COPY ctrader-openapi ./ctrader-openapi -COPY foxhunt-deploy ./foxhunt-deploy COPY services ./services COPY tests ./tests diff --git a/infra/docker/Dockerfile.training b/infra/docker/Dockerfile.training index 3fab62c87..daa13f591 100644 --- a/infra/docker/Dockerfile.training +++ b/infra/docker/Dockerfile.training @@ -50,7 +50,6 @@ COPY database ./database COPY config ./config COPY web-gateway ./web-gateway COPY ctrader-openapi ./ctrader-openapi -COPY foxhunt-deploy ./foxhunt-deploy COPY services ./services COPY tests ./tests diff --git a/infra/docker/Dockerfile.web-gateway b/infra/docker/Dockerfile.web-gateway index 58dad6e87..06df7f216 100644 --- a/infra/docker/Dockerfile.web-gateway +++ b/infra/docker/Dockerfile.web-gateway @@ -53,7 +53,6 @@ COPY database ./database COPY config ./config COPY web-gateway ./web-gateway COPY ctrader-openapi ./ctrader-openapi -COPY foxhunt-deploy ./foxhunt-deploy COPY services ./services COPY tests ./tests @@ -88,6 +87,6 @@ EXPOSE 3000 USER foxhunt HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD curl -f http://localhost:3000/api/health || exit 1 + CMD curl -f http://localhost:3000/health || exit 1 ENTRYPOINT ["./web-gateway"] diff --git a/infra/k8s/databases/postgres.yaml b/infra/k8s/databases/postgres.yaml index cddfd531a..1d5fd56e3 100644 --- a/infra/k8s/databases/postgres.yaml +++ b/infra/k8s/databases/postgres.yaml @@ -41,6 +41,9 @@ spec: ports: - containerPort: 5432 name: postgres + args: + - -c + - max_connections=100 env: - name: POSTGRES_DB value: foxhunt @@ -50,10 +53,11 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: DATABASE_PASSWORD + key: db-password volumeMounts: - name: postgres-data mountPath: /var/lib/postgresql/data + subPath: pgdata readinessProbe: exec: command: diff --git a/infra/k8s/gpu-taint.yaml b/infra/k8s/gpu-taint.yaml new file mode 100644 index 000000000..71b047d4c --- /dev/null +++ b/infra/k8s/gpu-taint.yaml @@ -0,0 +1,90 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gpu-taint-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gpu-taint-controller +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gpu-taint-controller +subjects: + - kind: ServiceAccount + name: gpu-taint-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: gpu-taint-controller + apiGroup: rbac.authorization.k8s.io +--- +# DaemonSet that runs on GPU nodes and taints them on startup. +# This ensures new GPU nodes auto-scaled by the cluster autoscaler +# get the nvidia.com/gpu taint, preventing non-GPU workloads from +# scheduling on expensive GPU instances. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gpu-taint-controller + namespace: kube-system + labels: + app.kubernetes.io/name: gpu-taint-controller +spec: + selector: + matchLabels: + app.kubernetes.io/name: gpu-taint-controller + template: + metadata: + labels: + app.kubernetes.io/name: gpu-taint-controller + spec: + serviceAccountName: gpu-taint-controller + # Only schedule on nodes that have GPU pool labels + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: k8s.scaleway.com/pool-name + operator: In + values: + - gpu + - gpu-training + - gpu-inference + # Tolerate the taint we're about to set (otherwise the DaemonSet + # gets evicted by its own taint) + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: taint-setter + image: bitnami/kubectl:latest + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - /bin/sh + - -c + - | + echo "Applying GPU taint to node ${NODE_NAME}" + kubectl taint nodes "${NODE_NAME}" nvidia.com/gpu=true:NoSchedule --overwrite || true + echo "Taint applied. Sleeping." + while true; do sleep 86400; done + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi diff --git a/infra/k8s/services/api-gateway.yaml b/infra/k8s/services/api-gateway.yaml index 99254cdd4..a485ab015 100644 --- a/infra/k8s/services/api-gateway.yaml +++ b/infra/k8s/services/api-gateway.yaml @@ -16,15 +16,13 @@ spec: labels: app.kubernetes.io/name: api-gateway spec: - nodeSelector: - k8s.scaleway.com/pool-name: always-on imagePullSecrets: - name: scw-registry containers: - name: api-gateway - image: rg.nl-ams.scw.cloud/foxhunt/api_gateway:latest + image: rg.fr-par.scw.cloud/foxhunt/api_gateway:latest ports: - - containerPort: 50050 + - containerPort: 50051 name: grpc - containerPort: 9091 name: metrics @@ -33,7 +31,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: DATABASE_PASSWORD + key: db-password - name: DATABASE_URL value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - name: REDIS_URL @@ -42,7 +40,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: JWT_SECRET + key: jwt-secret - name: JWT_ISSUER value: foxhunt-api-gateway - name: JWT_AUDIENCE @@ -61,7 +59,7 @@ spec: exec: command: - grpc_health_probe - - -addr=localhost:50050 + - -addr=localhost:50051 initialDelaySeconds: 10 periodSeconds: 10 resources: @@ -84,8 +82,8 @@ spec: selector: app.kubernetes.io/name: api-gateway ports: - - port: 50050 - targetPort: 50050 + - port: 50051 + targetPort: 50051 name: grpc - port: 9091 targetPort: 9091 diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index f648a1faa..3ed146620 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -16,13 +16,11 @@ spec: labels: app.kubernetes.io/name: backtesting-service spec: - nodeSelector: - k8s.scaleway.com/pool-name: always-on imagePullSecrets: - name: scw-registry containers: - name: backtesting-service - image: rg.nl-ams.scw.cloud/foxhunt/backtesting_service:latest + image: rg.fr-par.scw.cloud/foxhunt/backtesting_service:latest ports: - containerPort: 50053 name: grpc @@ -33,7 +31,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: DATABASE_PASSWORD + key: db-password - name: DATABASE_URL value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - name: REDIS_URL @@ -42,13 +40,18 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: JWT_SECRET + key: jwt-secret - name: JWT_ISSUER value: foxhunt-api-gateway - name: JWT_AUDIENCE value: foxhunt-services + - name: BENZINGA_API_KEY + value: "placeholder" - name: RUST_LOG value: info + volumeMounts: + - name: logs + mountPath: /app/logs readinessProbe: httpGet: path: /health @@ -62,6 +65,9 @@ spec: limits: cpu: 500m memory: 512Mi + volumes: + - name: logs + emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index f95efb350..c5bbea331 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -16,13 +16,11 @@ spec: labels: app.kubernetes.io/name: broker-gateway spec: - nodeSelector: - k8s.scaleway.com/pool-name: always-on imagePullSecrets: - name: scw-registry containers: - name: broker-gateway - image: rg.nl-ams.scw.cloud/foxhunt/broker_gateway_service:latest + image: rg.fr-par.scw.cloud/foxhunt/broker_gateway_service:latest ports: - containerPort: 50056 name: grpc @@ -33,7 +31,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: DATABASE_PASSWORD + key: db-password - name: DATABASE_URL value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - name: REDIS_URL @@ -42,7 +40,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: JWT_SECRET + key: jwt-secret - name: JWT_ISSUER value: foxhunt-api-gateway - name: JWT_AUDIENCE diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index e94026bcf..74541355d 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -16,13 +16,11 @@ spec: labels: app.kubernetes.io/name: ml-training-service spec: - nodeSelector: - k8s.scaleway.com/pool-name: always-on imagePullSecrets: - name: scw-registry containers: - name: ml-training-service - image: rg.nl-ams.scw.cloud/foxhunt/ml_training_service:latest + image: rg.fr-par.scw.cloud/foxhunt/ml_training_service:latest ports: - containerPort: 50053 name: grpc @@ -33,7 +31,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: DATABASE_PASSWORD + key: db-password - name: DATABASE_URL value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - name: REDIS_URL @@ -42,17 +40,24 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: JWT_SECRET + key: jwt-secret - name: JWT_ISSUER value: foxhunt-api-gateway - name: JWT_AUDIENCE value: foxhunt-services - name: S3_ENDPOINT - value: "https://s3.nl-ams.scw.cloud" + value: "https://s3.fr-par.scw.cloud" - name: S3_BUCKET value: foxhunt-artifacts - name: RUST_LOG value: info + command: ["./ml_training_service", "serve"] + volumeMounts: + - name: tls-certs + mountPath: /app/certs/ml_training_service + readOnly: true + - name: logs + mountPath: /app/logs readinessProbe: httpGet: path: /health @@ -66,6 +71,12 @@ spec: limits: cpu: 500m memory: 512Mi + volumes: + - name: tls-certs + secret: + secretName: ml-training-tls + - name: logs + emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 6053d5ed9..6a0421a33 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -16,13 +16,11 @@ spec: labels: app.kubernetes.io/name: trading-agent-service spec: - nodeSelector: - k8s.scaleway.com/pool-name: always-on imagePullSecrets: - name: scw-registry containers: - name: trading-agent-service - image: rg.nl-ams.scw.cloud/foxhunt/trading_agent_service:latest + image: rg.fr-par.scw.cloud/foxhunt/trading_agent_service:latest ports: - containerPort: 50055 name: grpc @@ -33,7 +31,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: DATABASE_PASSWORD + key: db-password - name: DATABASE_URL value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - name: REDIS_URL @@ -42,7 +40,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: JWT_SECRET + key: jwt-secret - name: JWT_ISSUER value: foxhunt-api-gateway - name: JWT_AUDIENCE diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index c54191834..6bbf3f6e4 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -16,13 +16,11 @@ spec: labels: app.kubernetes.io/name: trading-service spec: - nodeSelector: - k8s.scaleway.com/pool-name: always-on imagePullSecrets: - name: scw-registry containers: - name: trading-service - image: rg.nl-ams.scw.cloud/foxhunt/trading_service:latest + image: rg.fr-par.scw.cloud/foxhunt/trading_service:latest ports: - containerPort: 50051 name: grpc @@ -33,7 +31,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: DATABASE_PASSWORD + key: db-password - name: DATABASE_URL value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt" - name: REDIS_URL @@ -42,7 +40,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: JWT_SECRET + key: jwt-secret - name: JWT_ISSUER value: foxhunt-api-gateway - name: JWT_AUDIENCE @@ -53,6 +51,9 @@ spec: value: "50051" - name: RUST_LOG value: info + volumeMounts: + - name: logs + mountPath: /app/logs readinessProbe: exec: command: @@ -67,6 +68,9 @@ spec: limits: cpu: 500m memory: 512Mi + volumes: + - name: logs + emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/infra/k8s/services/web-gateway.yaml b/infra/k8s/services/web-gateway.yaml index e2cde7865..a12a40698 100644 --- a/infra/k8s/services/web-gateway.yaml +++ b/infra/k8s/services/web-gateway.yaml @@ -16,13 +16,11 @@ spec: labels: app.kubernetes.io/name: web-gateway spec: - nodeSelector: - k8s.scaleway.com/pool-name: always-on imagePullSecrets: - name: scw-registry containers: - name: web-gateway - image: rg.nl-ams.scw.cloud/foxhunt/web-gateway:latest + image: rg.fr-par.scw.cloud/foxhunt/web-gateway:latest ports: - containerPort: 3000 name: http @@ -31,7 +29,7 @@ spec: valueFrom: secretKeyRef: name: foxhunt-secrets - key: JWT_SECRET + key: jwt-secret - name: TRADING_SERVICE_URL value: "http://trading-service:50051" - name: BACKTESTING_SERVICE_URL @@ -48,7 +46,7 @@ spec: value: info readinessProbe: httpGet: - path: /api/health + path: /health port: 3000 initialDelaySeconds: 10 periodSeconds: 10 diff --git a/infra/k8s/storage/training-data-pv.yaml b/infra/k8s/storage/training-data-pv.yaml index 31eccbea0..ef4bd1b42 100644 --- a/infra/k8s/storage/training-data-pv.yaml +++ b/infra/k8s/storage/training-data-pv.yaml @@ -12,7 +12,8 @@ spec: - ReadOnlyMany csi: driver: csi.scaleway.com - volumeHandle: "VOLUME_ID" + volumeHandle: "fr-par-2/aca3c18b-89c0-4920-af67-d79ec9173430" + storageClassName: "" persistentVolumeReclaimPolicy: Retain --- apiVersion: v1 @@ -24,6 +25,7 @@ metadata: app.kubernetes.io/name: training-data app.kubernetes.io/part-of: foxhunt spec: + storageClassName: "" accessModes: - ReadOnlyMany resources: diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index 26f4a7d07..4cd49e36c 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -28,7 +28,7 @@ spec: restartPolicy: Never containers: - name: training - image: rg.nl-ams.scw.cloud/foxhunt/training:latest + image: rg.fr-par.scw.cloud/foxhunt/training:latest command: ["./train"] args: - "--model=dqn" diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index ed2848615..81a34817c 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -7,8 +7,16 @@ terraform { } inputs = { - cluster_name = "foxhunt" - k8s_version = "1.30" - gpu_type = "H100-1-80G" - gpu_max_size = 2 + cluster_name = "foxhunt" + k8s_version = "1.34" + + # GPU training pool (H100 for 10-model ensemble training) + enable_gpu_training_pool = true + gpu_training_type = "H100-1-80G" + gpu_training_max_size = 1 + + # GPU inference pool (L4 for cost-effective trading inference) + enable_gpu_inference_pool = true + gpu_inference_type = "L4-1-24G" + gpu_inference_max_size = 1 } diff --git a/infra/live/production/terragrunt.hcl b/infra/live/production/terragrunt.hcl index 6181b6ac1..9dede28b5 100644 --- a/infra/live/production/terragrunt.hcl +++ b/infra/live/production/terragrunt.hcl @@ -2,8 +2,8 @@ # All child modules inherit provider, backend, and common inputs from here. locals { - region = "nl-ams" - zone = "nl-ams-1" + region = "fr-par" + zone = "fr-par-2" project_id = get_env("SCW_DEFAULT_PROJECT_ID") } @@ -52,8 +52,9 @@ generate "provider" { } provider "scaleway" { - region = "${local.region}" - zone = "${local.zone}" + region = "${local.region}" + zone = "${local.zone}" + project_id = "${local.project_id}" } EOF } diff --git a/infra/modules/block-storage/main.tf b/infra/modules/block-storage/main.tf index 6e3ff09f2..b0307ff28 100644 --- a/infra/modules/block-storage/main.tf +++ b/infra/modules/block-storage/main.tf @@ -1,6 +1,6 @@ -resource "scaleway_instance_volume" "training_data" { +resource "scaleway_block_volume" "training_data" { name = var.volume_name - type = "b_ssd" + iops = 5000 size_in_gb = var.size_in_gb zone = var.zone } diff --git a/infra/modules/block-storage/outputs.tf b/infra/modules/block-storage/outputs.tf index 8bd51f440..3a4df0c95 100644 --- a/infra/modules/block-storage/outputs.tf +++ b/infra/modules/block-storage/outputs.tf @@ -1,9 +1,9 @@ output "volume_id" { description = "ID of the block storage volume" - value = scaleway_instance_volume.training_data.id + value = scaleway_block_volume.training_data.id } output "volume_name" { description = "Name of the block storage volume" - value = scaleway_instance_volume.training_data.name + value = scaleway_block_volume.training_data.name } diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index cde417de0..41eb01555 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -1,8 +1,15 @@ +resource "scaleway_vpc_private_network" "foxhunt" { + name = "${var.cluster_name}-pn" + region = var.region +} + resource "scaleway_k8s_cluster" "foxhunt" { - name = var.cluster_name - version = var.k8s_version - cni = "cilium" - region = var.region + name = var.cluster_name + version = var.k8s_version + cni = "cilium" + region = var.region + delete_additional_resources = true + private_network_id = scaleway_vpc_private_network.foxhunt.id auto_upgrade { enable = true @@ -35,22 +42,50 @@ resource "scaleway_k8s_pool" "ci" { cluster_id = scaleway_k8s_cluster.foxhunt.id name = "ci" node_type = var.ci_type - size = 0 + size = 1 min_size = 0 max_size = var.ci_max_size autoscaling = true autohealing = true region = var.region + + lifecycle { + ignore_changes = [size] + } } -resource "scaleway_k8s_pool" "gpu" { +# GPU pool for ML training (H100 — large VRAM for ensemble training) +resource "scaleway_k8s_pool" "gpu_training" { + count = var.enable_gpu_training_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "gpu" - node_type = var.gpu_type - size = 0 + name = "gpu-training" + node_type = var.gpu_training_type + size = 1 min_size = 0 - max_size = var.gpu_max_size + max_size = var.gpu_training_max_size autoscaling = true autohealing = true region = var.region + + lifecycle { + ignore_changes = [size] + } +} + +# GPU pool for inference during trading (L4 — cost-effective for forward passes) +resource "scaleway_k8s_pool" "gpu_inference" { + count = var.enable_gpu_inference_pool ? 1 : 0 + cluster_id = scaleway_k8s_cluster.foxhunt.id + name = "gpu-inference" + node_type = var.gpu_inference_type + size = 1 + min_size = 0 + max_size = var.gpu_inference_max_size + autoscaling = true + autohealing = true + region = var.region + + lifecycle { + ignore_changes = [size] + } } diff --git a/infra/modules/kapsule/outputs.tf b/infra/modules/kapsule/outputs.tf index a42ba383d..1710d1404 100644 --- a/infra/modules/kapsule/outputs.tf +++ b/infra/modules/kapsule/outputs.tf @@ -24,7 +24,12 @@ output "ci_pool_id" { value = scaleway_k8s_pool.ci.id } -output "gpu_pool_id" { - description = "ID of the GPU node pool" - value = scaleway_k8s_pool.gpu.id +output "gpu_training_pool_id" { + description = "ID of the GPU training node pool" + value = var.enable_gpu_training_pool ? scaleway_k8s_pool.gpu_training[0].id : "" +} + +output "gpu_inference_pool_id" { + description = "ID of the GPU inference node pool" + value = var.enable_gpu_inference_pool ? scaleway_k8s_pool.gpu_inference[0].id : "" } diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index da4604d7f..631ed2a58 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -12,7 +12,7 @@ variable "cluster_name" { variable "k8s_version" { description = "Kubernetes version for the Kapsule cluster" type = string - default = "1.30" + default = "1.34" } variable "always_on_type" { @@ -33,14 +33,38 @@ variable "ci_max_size" { default = 1 } -variable "gpu_type" { - description = "Instance type for the GPU node pool" +variable "enable_gpu_training_pool" { + description = "Create GPU node pool for ML training (H100)" + type = bool + default = false +} + +variable "gpu_training_type" { + description = "Instance type for the GPU training node pool" type = string default = "H100-1-80G" } -variable "gpu_max_size" { - description = "Maximum number of nodes in the GPU pool" +variable "gpu_training_max_size" { + description = "Maximum number of nodes in the GPU training pool" type = number - default = 2 + default = 1 +} + +variable "enable_gpu_inference_pool" { + description = "Create GPU node pool for inference (L4)" + type = bool + default = false +} + +variable "gpu_inference_type" { + description = "Instance type for the GPU inference node pool" + type = string + default = "L4-1-24G" +} + +variable "gpu_inference_max_size" { + description = "Maximum number of nodes in the GPU inference pool" + type = number + default = 1 } diff --git a/infra/scripts/build-and-push.sh b/infra/scripts/build-and-push.sh new file mode 100755 index 000000000..b511ad86d --- /dev/null +++ b/infra/scripts/build-and-push.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# build-and-push.sh - Build and push service images to Scaleway registry +# --------------------------------------------------------------------------- +# Builds all Foxhunt service Docker images and pushes to the Scaleway +# Container Registry. Supports parallel builds, selective service builds, +# and sccache for faster compilation. +# +# Usage: +# ./infra/scripts/build-and-push.sh # Build & push all +# ./infra/scripts/build-and-push.sh --no-push # Build only +# ./infra/scripts/build-and-push.sh --services "api_gateway trading_service" +# ./infra/scripts/build-and-push.sh --parallel 3 # 3 concurrent builds +# ./infra/scripts/build-and-push.sh --dry-run # Show what would run +# --------------------------------------------------------------------------- + +REGISTRY="rg.fr-par.scw.cloud" +NAMESPACE="foxhunt" +GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") +TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") + +# Services built with Dockerfile.service (--build-arg SERVICE=) +STANDARD_SERVICES=( + api_gateway + trading_service + trading_agent_service + ml_training_service + backtesting_service + broker_gateway_service +) + +# Defaults +DO_PUSH=true +DRY_RUN=false +PARALLEL=1 +SELECTED_SERVICES=() +SKIP_STANDARD=false +SKIP_WEBGW=false +SKIP_TRAINING=false +TAG_LATEST=true + +usage() { + cat </dev/null; then + GIT_COMMIT="${GIT_COMMIT}-dirty" +fi + +TAG_COMMIT="${GIT_COMMIT}" +TAG_TS="${TIMESTAMP}" + +echo "=== Foxhunt Docker Build ===" +echo "Registry: ${REGISTRY}/${NAMESPACE}" +echo "Commit: ${TAG_COMMIT}" +echo "Timestamp: ${TAG_TS}" +echo "Parallel: ${PARALLEL}" +echo "" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +PASS=0 +FAIL=0 +FAILURES=() + +build_and_tag() { + local image_name="$1" + local dockerfile="$2" + shift 2 + local extra_args=("$@") + + local full="${REGISTRY}/${NAMESPACE}/${image_name}" + + local tags=("-t" "${full}:${TAG_COMMIT}" "-t" "${full}:${TAG_TS}") + if [[ "$TAG_LATEST" == "true" ]]; then + tags+=("-t" "${full}:latest") + fi + + local cmd=(docker build "${tags[@]}" -f "$dockerfile" "${extra_args[@]}" .) + + if [[ "$DRY_RUN" == "true" ]]; then + echo " [DRY] ${cmd[*]}" + return 0 + fi + + echo " Building ${image_name}..." + local start + start=$(date +%s) + + if DOCKER_BUILDKIT=1 "${cmd[@]}" 2>&1 | tail -5; then + local elapsed=$(( $(date +%s) - start )) + echo " [OK] ${image_name} (${elapsed}s)" + (( PASS++ )) + else + echo " [FAIL] ${image_name}" + (( FAIL++ )) + FAILURES+=("${image_name}") + return 1 + fi +} + +push_image() { + local image_name="$1" + local full="${REGISTRY}/${NAMESPACE}/${image_name}" + + for tag in "${TAG_COMMIT}" "${TAG_TS}" $( [[ "$TAG_LATEST" == "true" ]] && echo "latest" ); do + if [[ "$DRY_RUN" == "true" ]]; then + echo " [DRY] docker push ${full}:${tag}" + else + echo " Pushing ${full}:${tag}" + docker push "${full}:${tag}" + fi + done +} + +# --------------------------------------------------------------------------- +# Ensure registry login +# --------------------------------------------------------------------------- + +if [[ "$DRY_RUN" != "true" && "$DO_PUSH" == "true" ]]; then + echo "--- Registry Login ---" + if ! docker login "${REGISTRY}" -u nologin --password-stdin <<< "$(scw registry login -o json 2>/dev/null | python3 -c 'import sys,json; print(json.load(sys.stdin).get("password",""))' 2>/dev/null)" 2>/dev/null; then + echo " Trying scw registry login..." + scw registry login + fi + echo "" +fi + +# --------------------------------------------------------------------------- +# Build standard services (Dockerfile.service) +# --------------------------------------------------------------------------- + +echo "--- Standard Services ---" + +if [[ "$PARALLEL" -gt 1 ]]; then + # Parallel builds using background jobs + PIDS=() + for svc in "${SERVICES[@]}"; do + ( + build_and_tag "$svc" "infra/docker/Dockerfile.service" --build-arg "SERVICE=${svc}" + ) & + PIDS+=($!) + # Throttle to PARALLEL concurrent jobs + while [[ $(jobs -r | wc -l) -ge $PARALLEL ]]; do + sleep 1 + done + done + # Wait for all builds + for pid in "${PIDS[@]}"; do + wait "$pid" || (( FAIL++ )) + done +else + for svc in "${SERVICES[@]}"; do + build_and_tag "$svc" "infra/docker/Dockerfile.service" --build-arg "SERVICE=${svc}" || true + done +fi + +# --------------------------------------------------------------------------- +# Build web-gateway (Dockerfile.web-gateway) +# --------------------------------------------------------------------------- + +if [[ "$SKIP_WEBGW" != "true" ]]; then + echo "" + echo "--- Web Gateway ---" + build_and_tag "web-gateway" "infra/docker/Dockerfile.web-gateway" || true +fi + +# --------------------------------------------------------------------------- +# Build training image (Dockerfile.training) +# --------------------------------------------------------------------------- + +if [[ "$SKIP_TRAINING" != "true" ]]; then + echo "" + echo "--- Training Image ---" + build_and_tag "training" "infra/docker/Dockerfile.training" || true +fi + +# --------------------------------------------------------------------------- +# Push +# --------------------------------------------------------------------------- + +if [[ "$DO_PUSH" == "true" ]]; then + echo "" + echo "--- Pushing Images ---" + + for svc in "${SERVICES[@]}"; do + push_image "$svc" + done + + if [[ "$SKIP_WEBGW" != "true" ]]; then + push_image "web-gateway" + fi + + if [[ "$SKIP_TRAINING" != "true" ]]; then + push_image "training" + fi +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +echo "" +echo "======================================================================" +printf " Results: %d built, %d failed\n" "$PASS" "$FAIL" +echo "======================================================================" + +if [[ $FAIL -gt 0 ]]; then + echo "" + echo " Failed:" + for f in "${FAILURES[@]}"; do + echo " - ${f}" + done + echo "" + exit 1 +fi + +echo "" +echo " All images built successfully." +if [[ "$DO_PUSH" == "true" && "$DRY_RUN" != "true" ]]; then + echo " Pushed to: ${REGISTRY}/${NAMESPACE}/" +fi +exit 0 diff --git a/infra/scripts/harden.sh b/infra/scripts/harden.sh new file mode 100755 index 000000000..2e038a5f7 --- /dev/null +++ b/infra/scripts/harden.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# harden.sh - Post-provisioning security hardening for Foxhunt infra +# --------------------------------------------------------------------------- +# Run after terragrunt apply to lock down API access, verify bucket ACLs, +# and validate no services are publicly exposed. +# --------------------------------------------------------------------------- + +CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:?Set SCW_KAPSULE_CLUSTER_ID}" +SCW_REGION="${SCW_REGION:-fr-par}" +ADMIN_IPV4="${ADMIN_IPV4:-}" +ADMIN_IPV6="${ADMIN_IPV6:-}" + +echo "=== Foxhunt Security Hardening ===" +echo "Cluster: ${CLUSTER_ID}" +echo "Region: ${SCW_REGION}" +echo "" + +# --- 1. K8s API ACL (no TF resource, must use CLI) ----------------------- +echo "--- K8s API ACL ---" + +ACLS="acls.0.ip=100.64.0.0/10 acls.0.description=tailscale-cgnat" +IDX=1 + +if [[ -n "$ADMIN_IPV4" ]]; then + ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV4}/32 acls.${IDX}.description=admin-v4" + (( IDX++ )) +fi + +if [[ -n "$ADMIN_IPV6" ]]; then + ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV6}/128 acls.${IDX}.description=admin-v6" + (( IDX++ )) +fi + +# shellcheck disable=SC2086 +scw k8s acl set "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" $ACLS +echo " K8s API restricted to Tailscale + admin IPs" + +# --- 2. Verify no LoadBalancer/NodePort services ------------------------- +echo "" +echo "--- Exposed Services Check ---" +EXPOSED=$(KUBECONFIG="${KUBECONFIG}" kubectl get svc -A -o json 2>/dev/null \ + | python3 -c " +import sys, json +data = json.load(sys.stdin) +for svc in data['items']: + t = svc['spec']['type'] + if t in ('LoadBalancer', 'NodePort'): + print(f\" {svc['metadata']['namespace']}/{svc['metadata']['name']}: {t}\") +" 2>/dev/null || true) + +if [[ -z "$EXPOSED" ]]; then + echo " No LoadBalancer or NodePort services found" +else + echo " WARNING: Exposed services found:" + echo "$EXPOSED" +fi + +# --- 3. Verify no Ingress resources -------------------------------------- +INGRESS=$(KUBECONFIG="${KUBECONFIG}" kubectl get ingress -A --no-headers 2>/dev/null || true) +if [[ -z "$INGRESS" ]]; then + echo " No Ingress resources found" +else + echo " WARNING: Ingress resources found:" + echo "$INGRESS" +fi + +# --- 4. Verify container registry is private ---------------------------- +echo "" +echo "--- Container Registry ---" +REG_PUBLIC=$(scw registry namespace list "region=${SCW_REGION}" -o json 2>/dev/null \ + | python3 -c " +import sys, json +for ns in json.load(sys.stdin): + if ns.get('name') == 'foxhunt': + print('public' if ns.get('is_public') else 'private') +" 2>/dev/null || echo "unknown") +echo " foxhunt registry: ${REG_PUBLIC}" + +# --- 5. Verify S3 buckets deny anonymous access ------------------------- +echo "" +echo "--- S3 Bucket Anonymous Access ---" +for bucket_url in \ + "https://foxhunt-artifacts.s3.fr-par.scw.cloud/" \ + "https://foxhunt-tfstate.s3.nl-ams.scw.cloud/"; do + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$bucket_url" 2>/dev/null || echo "000") + BUCKET=$(echo "$bucket_url" | sed 's|https://\(.*\)\.s3\..*|\1|') + if [[ "$HTTP_CODE" == "403" ]]; then + echo " ${BUCKET}: denied (HTTP ${HTTP_CODE})" + else + echo " WARNING: ${BUCKET}: HTTP ${HTTP_CODE} (expected 403)" + fi +done + +# --- 6. Scale CI/GPU pools to 0 if idle --------------------------------- +echo "" +echo "--- Pool Scale-Down ---" +for pool_name in ci gpu; do + POOL_ID=$(scw k8s pool list "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" -o json 2>/dev/null \ + | python3 -c " +import sys, json +for p in json.load(sys.stdin): + if p.get('name') == '${pool_name}': + print(p['id']) +" 2>/dev/null || true) + + if [[ -n "$POOL_ID" ]]; then + POOL_SIZE=$(scw k8s pool get "$POOL_ID" "region=${SCW_REGION}" -o json 2>/dev/null \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('size',0))" 2>/dev/null || echo "?") + echo " ${pool_name} pool: size=${POOL_SIZE}" + if [[ "$POOL_SIZE" != "0" ]]; then + KUBECONFIG="${KUBECONFIG}" kubectl get pods -A --field-selector="spec.nodeName" -o json 2>/dev/null \ + | python3 -c " +import sys, json +pods = json.load(sys.stdin)['items'] +pool_pods = [p for p in pods if '${pool_name}' in p.get('spec',{}).get('nodeName','')] +user_pods = [p for p in pool_pods if p['metadata']['namespace'] not in ('kube-system','cilium-secrets')] +if not user_pods: + print(' No user pods — safe to scale down') +else: + print(f' {len(user_pods)} user pods running — keeping nodes') +" 2>/dev/null || true + fi + fi +done + +echo "" +echo "=== Hardening complete ===" diff --git a/infra/scripts/smoke-test.sh b/infra/scripts/smoke-test.sh index a638d6446..98f3efbde 100755 --- a/infra/scripts/smoke-test.sh +++ b/infra/scripts/smoke-test.sh @@ -12,7 +12,7 @@ NAMESPACE="foxhunt" TAILSCALE_NS="tailscale" KAPSULE_NODE_NAME="foxhunt-kapsule" CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:-}" -SCW_REGION="${SCW_REGION:-nl-ams}" +SCW_REGION="${SCW_REGION:-fr-par}" GRPC_SERVICES=( trading-engine