gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'infra: provision Kapsule cluster with services and GPU pool split' (#1) from worktree-infra-provisioning into main

Browse Source
This commit is contained in:
foxhunt-admin
2026-02-24 16:43:38 +00:00
parent 4cc37057e1 5380bd9186
commit 04f3db65ab
26 changed files with 664 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@@ -148,6 +148,12 @@ coordination/orchestration/*
*.sqlite-wal
claude-flow
# Terragrunt cache and generated files
.terragrunt-cache/
**/.terragrunt-cache/
.terraform.lock.hcl
**/.terraform.lock.hcl
# Data cache (downloaded market data, not checked in)
data/cache/*
# Exception: futures baseline training data is checked in

12
common/src/database.rs
View File

@@ -96,8 +96,8 @@ impl Default for PoolConfig {
Self {
max_connections: 50,
min_connections: 10,
connect_timeout_ms: 100,
acquire_timeout_ms: 50,
connect_timeout_ms: 5000,
acquire_timeout_ms: 5000,
max_lifetime_seconds: 3600,
idle_timeout_seconds: 300,
}
@@ -126,9 +126,9 @@ impl From<DatabaseConfig> for LocalDatabaseConfig {
max_connections: config.max_connections,
// 20% of max, min 2. Uses integer division intentionally for simplicity.
min_connections: (config.max_connections / 5).max(2),
connect_timeout_ms: u64::try_from(config.connect_timeout.as_millis().min(100))
.unwrap_or(100), // Convert to ms, cap at 100ms for HFT
acquire_timeout_ms: 50, // Fast acquire for HFT
connect_timeout_ms: u64::try_from(config.connect_timeout.as_millis())
.unwrap_or(5000), // Pool connect timeout (not query timeout)
acquire_timeout_ms: 5000, // Pool acquire timeout (not query timeout)
max_lifetime_seconds: 3600, // 1 hour default
idle_timeout_seconds: 300, // 5 minutes default
},
@@ -156,7 +156,7 @@ impl From<BacktestingDatabaseConfig> for LocalDatabaseConfig {
// 25% of max, min 2. Uses integer division intentionally for simplicity.
min_connections: (max_conn / 4).max(2),
connect_timeout_ms: config.acquire_timeout_ms.unwrap_or(1000), // Use acquire timeout as connection timeout
acquire_timeout_ms: 100, // Less strict for backtesting
acquire_timeout_ms: 5000, // Pool acquire timeout (not query timeout)
max_lifetime_seconds: 3600, // 1 hour default
idle_timeout_seconds: 600, // 10 minutes for backtesting
},

3
infra/docker/Dockerfile.service
View File

@@ -12,7 +12,7 @@ ARG SERVICE
ARG SCCACHE_BUCKET=""
ARG AWS_ACCESS_KEY_ID=""
ARG AWS_SECRET_ACCESS_KEY=""
ARG AWS_DEFAULT_REGION="nl-ams"
ARG AWS_DEFAULT_REGION="fr-par"
ARG SCCACHE_ENDPOINT=""
RUN test -n "${SERVICE}" || (echo "ERROR: SERVICE build-arg is required" && exit 1)
@@ -67,7 +67,6 @@ COPY database ./database
COPY config ./config
COPY web-gateway ./web-gateway
COPY ctrader-openapi ./ctrader-openapi
COPY foxhunt-deploy ./foxhunt-deploy
COPY services ./services
COPY tests ./tests

1
infra/docker/Dockerfile.training
View File

@@ -50,7 +50,6 @@ COPY database ./database
COPY config ./config
COPY web-gateway ./web-gateway
COPY ctrader-openapi ./ctrader-openapi
COPY foxhunt-deploy ./foxhunt-deploy
COPY services ./services
COPY tests ./tests

3
infra/docker/Dockerfile.web-gateway
View File

@@ -53,7 +53,6 @@ COPY database ./database
COPY config ./config
COPY web-gateway ./web-gateway
COPY ctrader-openapi ./ctrader-openapi
COPY foxhunt-deploy ./foxhunt-deploy
COPY services ./services
COPY tests ./tests
@@ -88,6 +87,6 @@ EXPOSE 3000
USER foxhunt
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD curl -f http://localhost:3000/api/health || exit 1
CMD curl -f http://localhost:3000/health || exit 1
ENTRYPOINT ["./web-gateway"]

6
infra/k8s/databases/postgres.yaml
View File

@@ -41,6 +41,9 @@ spec:
ports:
- containerPort: 5432
name: postgres
args:
- -c
- max_connections=100
env:
- name: POSTGRES_DB
value: foxhunt
@@ -50,10 +53,11 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: DATABASE_PASSWORD
key: db-password
volumeMounts:
- name: postgres-data
mountPath: /var/lib/postgresql/data
subPath: pgdata
readinessProbe:
exec:
command:

90
infra/k8s/gpu-taint.yaml Normal file
View File

@@ -0,0 +1,90 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: gpu-taint-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: gpu-taint-controller
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: gpu-taint-controller
subjects:
- kind: ServiceAccount
name: gpu-taint-controller
namespace: kube-system
roleRef:
kind: ClusterRole
name: gpu-taint-controller
apiGroup: rbac.authorization.k8s.io
---
# DaemonSet that runs on GPU nodes and taints them on startup.
# This ensures new GPU nodes auto-scaled by the cluster autoscaler
# get the nvidia.com/gpu taint, preventing non-GPU workloads from
# scheduling on expensive GPU instances.
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: gpu-taint-controller
namespace: kube-system
labels:
app.kubernetes.io/name: gpu-taint-controller
spec:
selector:
matchLabels:
app.kubernetes.io/name: gpu-taint-controller
template:
metadata:
labels:
app.kubernetes.io/name: gpu-taint-controller
spec:
serviceAccountName: gpu-taint-controller
# Only schedule on nodes that have GPU pool labels
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: k8s.scaleway.com/pool-name
operator: In
values:
- gpu
- gpu-training
- gpu-inference
# Tolerate the taint we're about to set (otherwise the DaemonSet
# gets evicted by its own taint)
tolerations:
- key: nvidia.com/gpu
operator: Exists
effect: NoSchedule
containers:
- name: taint-setter
image: bitnami/kubectl:latest
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
command:
- /bin/sh
- -c
- |
echo "Applying GPU taint to node ${NODE_NAME}"
kubectl taint nodes "${NODE_NAME}" nvidia.com/gpu=true:NoSchedule --overwrite || true
echo "Taint applied. Sleeping."
while true; do sleep 86400; done
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
cpu: 50m
memory: 32Mi

16
infra/k8s/services/api-gateway.yaml
View File

@@ -16,15 +16,13 @@ spec:
labels:
app.kubernetes.io/name: api-gateway
spec:
nodeSelector:
k8s.scaleway.com/pool-name: always-on
imagePullSecrets:
- name: scw-registry
containers:
- name: api-gateway
image: rg.nl-ams.scw.cloud/foxhunt/api_gateway:latest
image: rg.fr-par.scw.cloud/foxhunt/api_gateway:latest
ports:
- containerPort: 50050
- containerPort: 50051
name: grpc
- containerPort: 9091
name: metrics
@@ -33,7 +31,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: DATABASE_PASSWORD
key: db-password
- name: DATABASE_URL
value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
- name: REDIS_URL
@@ -42,7 +40,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: JWT_SECRET
key: jwt-secret
- name: JWT_ISSUER
value: foxhunt-api-gateway
- name: JWT_AUDIENCE
@@ -61,7 +59,7 @@ spec:
exec:
command:
- grpc_health_probe
- -addr=localhost:50050
- -addr=localhost:50051
initialDelaySeconds: 10
periodSeconds: 10
resources:
@@ -84,8 +82,8 @@ spec:
selector:
app.kubernetes.io/name: api-gateway
ports:
- port: 50050
targetPort: 50050
- port: 50051
targetPort: 50051
name: grpc
- port: 9091
targetPort: 9091

16
infra/k8s/services/backtesting-service.yaml
View File

@@ -16,13 +16,11 @@ spec:
labels:
app.kubernetes.io/name: backtesting-service
spec:
nodeSelector:
k8s.scaleway.com/pool-name: always-on
imagePullSecrets:
- name: scw-registry
containers:
- name: backtesting-service
image: rg.nl-ams.scw.cloud/foxhunt/backtesting_service:latest
image: rg.fr-par.scw.cloud/foxhunt/backtesting_service:latest
ports:
- containerPort: 50053
name: grpc
@@ -33,7 +31,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: DATABASE_PASSWORD
key: db-password
- name: DATABASE_URL
value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
- name: REDIS_URL
@@ -42,13 +40,18 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: JWT_SECRET
key: jwt-secret
- name: JWT_ISSUER
value: foxhunt-api-gateway
- name: JWT_AUDIENCE
value: foxhunt-services
- name: BENZINGA_API_KEY
value: "placeholder"
- name: RUST_LOG
value: info
volumeMounts:
- name: logs
mountPath: /app/logs
readinessProbe:
httpGet:
path: /health
@@ -62,6 +65,9 @@ spec:
limits:
cpu: 500m
memory: 512Mi
volumes:
- name: logs
emptyDir: {}
---
apiVersion: v1
kind: Service

8
infra/k8s/services/broker-gateway.yaml
View File

@@ -16,13 +16,11 @@ spec:
labels:
app.kubernetes.io/name: broker-gateway
spec:
nodeSelector:
k8s.scaleway.com/pool-name: always-on
imagePullSecrets:
- name: scw-registry
containers:
- name: broker-gateway
image: rg.nl-ams.scw.cloud/foxhunt/broker_gateway_service:latest
image: rg.fr-par.scw.cloud/foxhunt/broker_gateway_service:latest
ports:
- containerPort: 50056
name: grpc
@@ -33,7 +31,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: DATABASE_PASSWORD
key: db-password
- name: DATABASE_URL
value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
- name: REDIS_URL
@@ -42,7 +40,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: JWT_SECRET
key: jwt-secret
- name: JWT_ISSUER
value: foxhunt-api-gateway
- name: JWT_AUDIENCE

23
infra/k8s/services/ml-training-service.yaml
View File

@@ -16,13 +16,11 @@ spec:
labels:
app.kubernetes.io/name: ml-training-service
spec:
nodeSelector:
k8s.scaleway.com/pool-name: always-on
imagePullSecrets:
- name: scw-registry
containers:
- name: ml-training-service
image: rg.nl-ams.scw.cloud/foxhunt/ml_training_service:latest
image: rg.fr-par.scw.cloud/foxhunt/ml_training_service:latest
ports:
- containerPort: 50053
name: grpc
@@ -33,7 +31,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: DATABASE_PASSWORD
key: db-password
- name: DATABASE_URL
value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
- name: REDIS_URL
@@ -42,17 +40,24 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: JWT_SECRET
key: jwt-secret
- name: JWT_ISSUER
value: foxhunt-api-gateway
- name: JWT_AUDIENCE
value: foxhunt-services
- name: S3_ENDPOINT
value: "https://s3.nl-ams.scw.cloud"
value: "https://s3.fr-par.scw.cloud"
- name: S3_BUCKET
value: foxhunt-artifacts
- name: RUST_LOG
value: info
command: ["./ml_training_service", "serve"]
volumeMounts:
- name: tls-certs
mountPath: /app/certs/ml_training_service
readOnly: true
- name: logs
mountPath: /app/logs
readinessProbe:
httpGet:
path: /health
@@ -66,6 +71,12 @@ spec:
limits:
cpu: 500m
memory: 512Mi
volumes:
- name: tls-certs
secret:
secretName: ml-training-tls
- name: logs
emptyDir: {}
---
apiVersion: v1
kind: Service

8
infra/k8s/services/trading-agent-service.yaml
View File

@@ -16,13 +16,11 @@ spec:
labels:
app.kubernetes.io/name: trading-agent-service
spec:
nodeSelector:
k8s.scaleway.com/pool-name: always-on
imagePullSecrets:
- name: scw-registry
containers:
- name: trading-agent-service
image: rg.nl-ams.scw.cloud/foxhunt/trading_agent_service:latest
image: rg.fr-par.scw.cloud/foxhunt/trading_agent_service:latest
ports:
- containerPort: 50055
name: grpc
@@ -33,7 +31,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: DATABASE_PASSWORD
key: db-password
- name: DATABASE_URL
value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
- name: REDIS_URL
@@ -42,7 +40,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: JWT_SECRET
key: jwt-secret
- name: JWT_ISSUER
value: foxhunt-api-gateway
- name: JWT_AUDIENCE

14
infra/k8s/services/trading-service.yaml
View File

@@ -16,13 +16,11 @@ spec:
labels:
app.kubernetes.io/name: trading-service
spec:
nodeSelector:
k8s.scaleway.com/pool-name: always-on
imagePullSecrets:
- name: scw-registry
containers:
- name: trading-service
image: rg.nl-ams.scw.cloud/foxhunt/trading_service:latest
image: rg.fr-par.scw.cloud/foxhunt/trading_service:latest
ports:
- containerPort: 50051
name: grpc
@@ -33,7 +31,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: DATABASE_PASSWORD
key: db-password
- name: DATABASE_URL
value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
- name: REDIS_URL
@@ -42,7 +40,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: JWT_SECRET
key: jwt-secret
- name: JWT_ISSUER
value: foxhunt-api-gateway
- name: JWT_AUDIENCE
@@ -53,6 +51,9 @@ spec:
value: "50051"
- name: RUST_LOG
value: info
volumeMounts:
- name: logs
mountPath: /app/logs
readinessProbe:
exec:
command:
@@ -67,6 +68,9 @@ spec:
limits:
cpu: 500m
memory: 512Mi
volumes:
- name: logs
emptyDir: {}
---
apiVersion: v1
kind: Service

8
infra/k8s/services/web-gateway.yaml
View File

@@ -16,13 +16,11 @@ spec:
labels:
app.kubernetes.io/name: web-gateway
spec:
nodeSelector:
k8s.scaleway.com/pool-name: always-on
imagePullSecrets:
- name: scw-registry
containers:
- name: web-gateway
image: rg.nl-ams.scw.cloud/foxhunt/web-gateway:latest
image: rg.fr-par.scw.cloud/foxhunt/web-gateway:latest
ports:
- containerPort: 3000
name: http
@@ -31,7 +29,7 @@ spec:
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: JWT_SECRET
key: jwt-secret
- name: TRADING_SERVICE_URL
value: "http://trading-service:50051"
- name: BACKTESTING_SERVICE_URL
@@ -48,7 +46,7 @@ spec:
value: info
readinessProbe:
httpGet:
path: /api/health
path: /health
port: 3000
initialDelaySeconds: 10
periodSeconds: 10

4
infra/k8s/storage/training-data-pv.yaml
View File

@@ -12,7 +12,8 @@ spec:
- ReadOnlyMany
csi:
driver: csi.scaleway.com
volumeHandle: "VOLUME_ID"
volumeHandle: "fr-par-2/aca3c18b-89c0-4920-af67-d79ec9173430"
storageClassName: ""
persistentVolumeReclaimPolicy: Retain
---
apiVersion: v1
@@ -24,6 +25,7 @@ metadata:
app.kubernetes.io/name: training-data
app.kubernetes.io/part-of: foxhunt
spec:
storageClassName: ""
accessModes:
- ReadOnlyMany
resources:

2
infra/k8s/training/job-template.yaml
View File

@@ -28,7 +28,7 @@ spec:
restartPolicy: Never
containers:
- name: training
image: rg.nl-ams.scw.cloud/foxhunt/training:latest
image: rg.fr-par.scw.cloud/foxhunt/training:latest
command: ["./train"]
args:
- "--model=dqn"

16
infra/live/production/kapsule/terragrunt.hcl
View File

@@ -7,8 +7,16 @@ terraform {
}
inputs = {
cluster_name = "foxhunt"
k8s_version = "1.30"
gpu_type = "H100-1-80G"
gpu_max_size = 2
cluster_name = "foxhunt"
k8s_version = "1.34"
# GPU training pool (H100 for 10-model ensemble training)
enable_gpu_training_pool = true
gpu_training_type = "H100-1-80G"
gpu_training_max_size = 1
# GPU inference pool (L4 for cost-effective trading inference)
enable_gpu_inference_pool = true
gpu_inference_type = "L4-1-24G"
gpu_inference_max_size = 1
}

9
infra/live/production/terragrunt.hcl
View File

@@ -2,8 +2,8 @@
# All child modules inherit provider, backend, and common inputs from here.
locals {
region = "nl-ams"
zone = "nl-ams-1"
region = "fr-par"
zone = "fr-par-2"
project_id = get_env("SCW_DEFAULT_PROJECT_ID")
}
@@ -52,8 +52,9 @@ generate "provider" {
}
provider "scaleway" {
region = "${local.region}"
zone = "${local.zone}"
region = "${local.region}"
zone = "${local.zone}"
project_id = "${local.project_id}"
}
EOF
}

4
infra/modules/block-storage/main.tf
View File

@@ -1,6 +1,6 @@
resource "scaleway_instance_volume" "training_data" {
resource "scaleway_block_volume" "training_data" {
name = var.volume_name
type = "b_ssd"
iops = 5000
size_in_gb = var.size_in_gb
zone = var.zone
}

4
infra/modules/block-storage/outputs.tf
View File

@@ -1,9 +1,9 @@
output "volume_id" {
description = "ID of the block storage volume"
value = scaleway_instance_volume.training_data.id
value = scaleway_block_volume.training_data.id
}
output "volume_name" {
description = "Name of the block storage volume"
value = scaleway_instance_volume.training_data.name
value = scaleway_block_volume.training_data.name
}

55
infra/modules/kapsule/main.tf
View File

@@ -1,8 +1,15 @@
resource "scaleway_vpc_private_network" "foxhunt" {
name = "${var.cluster_name}-pn"
region = var.region
}
resource "scaleway_k8s_cluster" "foxhunt" {
name = var.cluster_name
version = var.k8s_version
cni = "cilium"
region = var.region
name = var.cluster_name
version = var.k8s_version
cni = "cilium"
region = var.region
delete_additional_resources = true
private_network_id = scaleway_vpc_private_network.foxhunt.id
auto_upgrade {
enable = true
@@ -35,22 +42,50 @@ resource "scaleway_k8s_pool" "ci" {
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "ci"
node_type = var.ci_type
size = 0
size = 1
min_size = 0
max_size = var.ci_max_size
autoscaling = true
autohealing = true
region = var.region
lifecycle {
ignore_changes = [size]
}
}
resource "scaleway_k8s_pool" "gpu" {
# GPU pool for ML training (H100 — large VRAM for ensemble training)
resource "scaleway_k8s_pool" "gpu_training" {
count = var.enable_gpu_training_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "gpu"
node_type = var.gpu_type
size = 0
name = "gpu-training"
node_type = var.gpu_training_type
size = 1
min_size = 0
max_size = var.gpu_max_size
max_size = var.gpu_training_max_size
autoscaling = true
autohealing = true
region = var.region
lifecycle {
ignore_changes = [size]
}
}
# GPU pool for inference during trading (L4 — cost-effective for forward passes)
resource "scaleway_k8s_pool" "gpu_inference" {
count = var.enable_gpu_inference_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "gpu-inference"
node_type = var.gpu_inference_type
size = 1
min_size = 0
max_size = var.gpu_inference_max_size
autoscaling = true
autohealing = true
region = var.region
lifecycle {
ignore_changes = [size]
}
}

11
infra/modules/kapsule/outputs.tf
View File

@@ -24,7 +24,12 @@ output "ci_pool_id" {
value = scaleway_k8s_pool.ci.id
}
output "gpu_pool_id" {
description = "ID of the GPU node pool"
value = scaleway_k8s_pool.gpu.id
output "gpu_training_pool_id" {
description = "ID of the GPU training node pool"
value = var.enable_gpu_training_pool ? scaleway_k8s_pool.gpu_training[0].id : ""
}
output "gpu_inference_pool_id" {
description = "ID of the GPU inference node pool"
value = var.enable_gpu_inference_pool ? scaleway_k8s_pool.gpu_inference[0].id : ""
}

36
infra/modules/kapsule/variables.tf
View File

@@ -12,7 +12,7 @@ variable "cluster_name" {
variable "k8s_version" {
description = "Kubernetes version for the Kapsule cluster"
type = string
default = "1.30"
default = "1.34"
}
variable "always_on_type" {
@@ -33,14 +33,38 @@ variable "ci_max_size" {
default = 1
}
variable "gpu_type" {
description = "Instance type for the GPU node pool"
variable "enable_gpu_training_pool" {
description = "Create GPU node pool for ML training (H100)"
type = bool
default = false
}
variable "gpu_training_type" {
description = "Instance type for the GPU training node pool"
type = string
default = "H100-1-80G"
}
variable "gpu_max_size" {
description = "Maximum number of nodes in the GPU pool"
variable "gpu_training_max_size" {
description = "Maximum number of nodes in the GPU training pool"
type = number
default = 2
default = 1
}
variable "enable_gpu_inference_pool" {
description = "Create GPU node pool for inference (L4)"
type = bool
default = false
}
variable "gpu_inference_type" {
description = "Instance type for the GPU inference node pool"
type = string
default = "L4-1-24G"
}
variable "gpu_inference_max_size" {
description = "Maximum number of nodes in the GPU inference pool"
type = number
default = 1
}

263
infra/scripts/build-and-push.sh Executable file
View File

@@ -0,0 +1,263 @@
#!/usr/bin/env bash
set -euo pipefail
# ---------------------------------------------------------------------------
# build-and-push.sh - Build and push service images to Scaleway registry
# ---------------------------------------------------------------------------
# Builds all Foxhunt service Docker images and pushes to the Scaleway
# Container Registry. Supports parallel builds, selective service builds,
# and sccache for faster compilation.
#
# Usage:
# ./infra/scripts/build-and-push.sh # Build & push all
# ./infra/scripts/build-and-push.sh --no-push # Build only
# ./infra/scripts/build-and-push.sh --services "api_gateway trading_service"
# ./infra/scripts/build-and-push.sh --parallel 3 # 3 concurrent builds
# ./infra/scripts/build-and-push.sh --dry-run # Show what would run
# ---------------------------------------------------------------------------
REGISTRY="rg.fr-par.scw.cloud"
NAMESPACE="foxhunt"
GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S")
# Services built with Dockerfile.service (--build-arg SERVICE=<name>)
STANDARD_SERVICES=(
api_gateway
trading_service
trading_agent_service
ml_training_service
backtesting_service
broker_gateway_service
)
# Defaults
DO_PUSH=true
DRY_RUN=false
PARALLEL=1
SELECTED_SERVICES=()
SKIP_STANDARD=false
SKIP_WEBGW=false
SKIP_TRAINING=false
TAG_LATEST=true
usage() {
cat <<EOF
Usage: $(basename "$0") [OPTIONS]
Options:
--no-push Build only, don't push to registry
--dry-run Show what would be built
--services "s1 s2" Build only these standard services
--skip-webgw Skip web-gateway build
--skip-training Skip training image build
--parallel N Run N builds concurrently (default: 1)
--no-latest Don't tag images as :latest
-h, --help Show this help
EOF
}
while [[ $# -gt 0 ]]; do
case $1 in
--no-push) DO_PUSH=false; shift ;;
--dry-run) DRY_RUN=true; shift ;;
--services) IFS=' ' read -ra SELECTED_SERVICES <<< "$2"; shift 2 ;;
--skip-webgw) SKIP_WEBGW=true; shift ;;
--skip-training) SKIP_TRAINING=true; shift ;;
--parallel) PARALLEL="$2"; shift 2 ;;
--no-latest) TAG_LATEST=false; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "Unknown option: $1"; usage; exit 1 ;;
esac
done
# If --services was given, use only those; otherwise use all standard services
if [[ ${#SELECTED_SERVICES[@]} -gt 0 ]]; then
SERVICES=("${SELECTED_SERVICES[@]}")
SKIP_WEBGW=true
SKIP_TRAINING=true
else
SERVICES=("${STANDARD_SERVICES[@]}")
fi
# Dirty check
if ! git diff-index --quiet HEAD -- 2>/dev/null; then
GIT_COMMIT="${GIT_COMMIT}-dirty"
fi
TAG_COMMIT="${GIT_COMMIT}"
TAG_TS="${TIMESTAMP}"
echo "=== Foxhunt Docker Build ==="
echo "Registry: ${REGISTRY}/${NAMESPACE}"
echo "Commit: ${TAG_COMMIT}"
echo "Timestamp: ${TAG_TS}"
echo "Parallel: ${PARALLEL}"
echo ""
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
PASS=0
FAIL=0
FAILURES=()
build_and_tag() {
local image_name="$1"
local dockerfile="$2"
shift 2
local extra_args=("$@")
local full="${REGISTRY}/${NAMESPACE}/${image_name}"
local tags=("-t" "${full}:${TAG_COMMIT}" "-t" "${full}:${TAG_TS}")
if [[ "$TAG_LATEST" == "true" ]]; then
tags+=("-t" "${full}:latest")
fi
local cmd=(docker build "${tags[@]}" -f "$dockerfile" "${extra_args[@]}" .)
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY] ${cmd[*]}"
return 0
fi
echo " Building ${image_name}..."
local start
start=$(date +%s)
if DOCKER_BUILDKIT=1 "${cmd[@]}" 2>&1 | tail -5; then
local elapsed=$(( $(date +%s) - start ))
echo " [OK] ${image_name} (${elapsed}s)"
(( PASS++ ))
else
echo " [FAIL] ${image_name}"
(( FAIL++ ))
FAILURES+=("${image_name}")
return 1
fi
}
push_image() {
local image_name="$1"
local full="${REGISTRY}/${NAMESPACE}/${image_name}"
for tag in "${TAG_COMMIT}" "${TAG_TS}" $( [[ "$TAG_LATEST" == "true" ]] && echo "latest" ); do
if [[ "$DRY_RUN" == "true" ]]; then
echo " [DRY] docker push ${full}:${tag}"
else
echo " Pushing ${full}:${tag}"
docker push "${full}:${tag}"
fi
done
}
# ---------------------------------------------------------------------------
# Ensure registry login
# ---------------------------------------------------------------------------
if [[ "$DRY_RUN" != "true" && "$DO_PUSH" == "true" ]]; then
echo "--- Registry Login ---"
if ! docker login "${REGISTRY}" -u nologin --password-stdin <<< "$(scw registry login -o json 2>/dev/null | python3 -c 'import sys,json; print(json.load(sys.stdin).get("password",""))' 2>/dev/null)" 2>/dev/null; then
echo " Trying scw registry login..."
scw registry login
fi
echo ""
fi
# ---------------------------------------------------------------------------
# Build standard services (Dockerfile.service)
# ---------------------------------------------------------------------------
echo "--- Standard Services ---"
if [[ "$PARALLEL" -gt 1 ]]; then
# Parallel builds using background jobs
PIDS=()
for svc in "${SERVICES[@]}"; do
(
build_and_tag "$svc" "infra/docker/Dockerfile.service" --build-arg "SERVICE=${svc}"
) &
PIDS+=($!)
# Throttle to PARALLEL concurrent jobs
while [[ $(jobs -r | wc -l) -ge $PARALLEL ]]; do
sleep 1
done
done
# Wait for all builds
for pid in "${PIDS[@]}"; do
wait "$pid" || (( FAIL++ ))
done
else
for svc in "${SERVICES[@]}"; do
build_and_tag "$svc" "infra/docker/Dockerfile.service" --build-arg "SERVICE=${svc}" || true
done
fi
# ---------------------------------------------------------------------------
# Build web-gateway (Dockerfile.web-gateway)
# ---------------------------------------------------------------------------
if [[ "$SKIP_WEBGW" != "true" ]]; then
echo ""
echo "--- Web Gateway ---"
build_and_tag "web-gateway" "infra/docker/Dockerfile.web-gateway" || true
fi
# ---------------------------------------------------------------------------
# Build training image (Dockerfile.training)
# ---------------------------------------------------------------------------
if [[ "$SKIP_TRAINING" != "true" ]]; then
echo ""
echo "--- Training Image ---"
build_and_tag "training" "infra/docker/Dockerfile.training" || true
fi
# ---------------------------------------------------------------------------
# Push
# ---------------------------------------------------------------------------
if [[ "$DO_PUSH" == "true" ]]; then
echo ""
echo "--- Pushing Images ---"
for svc in "${SERVICES[@]}"; do
push_image "$svc"
done
if [[ "$SKIP_WEBGW" != "true" ]]; then
push_image "web-gateway"
fi
if [[ "$SKIP_TRAINING" != "true" ]]; then
push_image "training"
fi
fi
# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
echo ""
echo "======================================================================"
printf " Results: %d built, %d failed\n" "$PASS" "$FAIL"
echo "======================================================================"
if [[ $FAIL -gt 0 ]]; then
echo ""
echo " Failed:"
for f in "${FAILURES[@]}"; do
echo " - ${f}"
done
echo ""
exit 1
fi
echo ""
echo " All images built successfully."
if [[ "$DO_PUSH" == "true" && "$DRY_RUN" != "true" ]]; then
echo " Pushed to: ${REGISTRY}/${NAMESPACE}/"
fi
exit 0

130
infra/scripts/harden.sh Executable file
View File

@@ -0,0 +1,130 @@
#!/usr/bin/env bash
set -euo pipefail
# ---------------------------------------------------------------------------
# harden.sh - Post-provisioning security hardening for Foxhunt infra
# ---------------------------------------------------------------------------
# Run after terragrunt apply to lock down API access, verify bucket ACLs,
# and validate no services are publicly exposed.
# ---------------------------------------------------------------------------
CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:?Set SCW_KAPSULE_CLUSTER_ID}"
SCW_REGION="${SCW_REGION:-fr-par}"
ADMIN_IPV4="${ADMIN_IPV4:-}"
ADMIN_IPV6="${ADMIN_IPV6:-}"
echo "=== Foxhunt Security Hardening ==="
echo "Cluster: ${CLUSTER_ID}"
echo "Region: ${SCW_REGION}"
echo ""
# --- 1. K8s API ACL (no TF resource, must use CLI) -----------------------
echo "--- K8s API ACL ---"
ACLS="acls.0.ip=100.64.0.0/10 acls.0.description=tailscale-cgnat"
IDX=1
if [[ -n "$ADMIN_IPV4" ]]; then
ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV4}/32 acls.${IDX}.description=admin-v4"
(( IDX++ ))
fi
if [[ -n "$ADMIN_IPV6" ]]; then
ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV6}/128 acls.${IDX}.description=admin-v6"
(( IDX++ ))
fi
# shellcheck disable=SC2086
scw k8s acl set "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" $ACLS
echo " K8s API restricted to Tailscale + admin IPs"
# --- 2. Verify no LoadBalancer/NodePort services -------------------------
echo ""
echo "--- Exposed Services Check ---"
EXPOSED=$(KUBECONFIG="${KUBECONFIG}" kubectl get svc -A -o json 2>/dev/null \
| python3 -c "
import sys, json
data = json.load(sys.stdin)
for svc in data['items']:
t = svc['spec']['type']
if t in ('LoadBalancer', 'NodePort'):
print(f\" {svc['metadata']['namespace']}/{svc['metadata']['name']}: {t}\")
" 2>/dev/null || true)
if [[ -z "$EXPOSED" ]]; then
echo " No LoadBalancer or NodePort services found"
else
echo " WARNING: Exposed services found:"
echo "$EXPOSED"
fi
# --- 3. Verify no Ingress resources --------------------------------------
INGRESS=$(KUBECONFIG="${KUBECONFIG}" kubectl get ingress -A --no-headers 2>/dev/null || true)
if [[ -z "$INGRESS" ]]; then
echo " No Ingress resources found"
else
echo " WARNING: Ingress resources found:"
echo "$INGRESS"
fi
# --- 4. Verify container registry is private ----------------------------
echo ""
echo "--- Container Registry ---"
REG_PUBLIC=$(scw registry namespace list "region=${SCW_REGION}" -o json 2>/dev/null \
| python3 -c "
import sys, json
for ns in json.load(sys.stdin):
if ns.get('name') == 'foxhunt':
print('public' if ns.get('is_public') else 'private')
" 2>/dev/null || echo "unknown")
echo " foxhunt registry: ${REG_PUBLIC}"
# --- 5. Verify S3 buckets deny anonymous access -------------------------
echo ""
echo "--- S3 Bucket Anonymous Access ---"
for bucket_url in \
"https://foxhunt-artifacts.s3.fr-par.scw.cloud/" \
"https://foxhunt-tfstate.s3.nl-ams.scw.cloud/"; do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$bucket_url" 2>/dev/null || echo "000")
BUCKET=$(echo "$bucket_url" | sed 's|https://\(.*\)\.s3\..*|\1|')
if [[ "$HTTP_CODE" == "403" ]]; then
echo " ${BUCKET}: denied (HTTP ${HTTP_CODE})"
else
echo " WARNING: ${BUCKET}: HTTP ${HTTP_CODE} (expected 403)"
fi
done
# --- 6. Scale CI/GPU pools to 0 if idle ---------------------------------
echo ""
echo "--- Pool Scale-Down ---"
for pool_name in ci gpu; do
POOL_ID=$(scw k8s pool list "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" -o json 2>/dev/null \
| python3 -c "
import sys, json
for p in json.load(sys.stdin):
if p.get('name') == '${pool_name}':
print(p['id'])
" 2>/dev/null || true)
if [[ -n "$POOL_ID" ]]; then
POOL_SIZE=$(scw k8s pool get "$POOL_ID" "region=${SCW_REGION}" -o json 2>/dev/null \
| python3 -c "import sys,json; print(json.load(sys.stdin).get('size',0))" 2>/dev/null || echo "?")
echo " ${pool_name} pool: size=${POOL_SIZE}"
if [[ "$POOL_SIZE" != "0" ]]; then
KUBECONFIG="${KUBECONFIG}" kubectl get pods -A --field-selector="spec.nodeName" -o json 2>/dev/null \
| python3 -c "
import sys, json
pods = json.load(sys.stdin)['items']
pool_pods = [p for p in pods if '${pool_name}' in p.get('spec',{}).get('nodeName','')]
user_pods = [p for p in pool_pods if p['metadata']['namespace'] not in ('kube-system','cilium-secrets')]
if not user_pods:
print(' No user pods — safe to scale down')
else:
print(f' {len(user_pods)} user pods running — keeping nodes')
" 2>/dev/null || true
fi
fi
done
echo ""
echo "=== Hardening complete ==="

2
infra/scripts/smoke-test.sh
View File

@@ -12,7 +12,7 @@ NAMESPACE="foxhunt"
TAILSCALE_NS="tailscale"
KAPSULE_NODE_NAME="foxhunt-kapsule"
CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:-}"
SCW_REGION="${SCW_REGION:-nl-ams}"
SCW_REGION="${SCW_REGION:-fr-par}"
GRPC_SERVICES=(
trading-engine