diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 856f4fac8..abeb6117b 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -15,7 +15,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.SCW_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.SCW_SECRET_KEY }} RUSTC_WRAPPER: /usr/local/bin/sccache - REGISTRY: rg.nl-ams.scw.cloud/foxhunt + REGISTRY: rg.fr-par.scw.cloud/foxhunt jobs: # --------------------------------------------------------------------------- @@ -91,7 +91,7 @@ jobs: - name: Login to Scaleway Container Registry run: | - echo "${{ secrets.SCW_SECRET_KEY }}" | docker login rg.nl-ams.scw.cloud -u nologin --password-stdin + echo "${{ secrets.SCW_SECRET_KEY }}" | docker login rg.fr-par.scw.cloud -u nologin --password-stdin - name: Build ${{ matrix.service }} run: | @@ -123,7 +123,7 @@ jobs: - name: Login to Scaleway Container Registry run: | - echo "${{ secrets.SCW_SECRET_KEY }}" | docker login rg.nl-ams.scw.cloud -u nologin --password-stdin + echo "${{ secrets.SCW_SECRET_KEY }}" | docker login rg.fr-par.scw.cloud -u nologin --password-stdin - name: Build web-gateway run: | @@ -150,11 +150,15 @@ jobs: - name: Login to Scaleway Container Registry run: | - echo "${{ secrets.SCW_SECRET_KEY }}" | docker login rg.nl-ams.scw.cloud -u nologin --password-stdin + echo "${{ secrets.SCW_SECRET_KEY }}" | docker login rg.fr-par.scw.cloud -u nologin --password-stdin - name: Build training image run: | docker build \ + --build-arg SCCACHE_BUCKET=${{ secrets.SCCACHE_BUCKET }} \ + --build-arg AWS_ACCESS_KEY_ID=${{ secrets.SCW_ACCESS_KEY }} \ + --build-arg AWS_SECRET_ACCESS_KEY=${{ secrets.SCW_SECRET_KEY }} \ + --build-arg SCCACHE_ENDPOINT=${{ secrets.SCCACHE_ENDPOINT }} \ -f infra/docker/Dockerfile.training \ -t ${{ env.REGISTRY }}/training:${{ github.sha }} \ -t ${{ env.REGISTRY }}/training:latest \ diff --git a/docs/plans/2026-02-24-h100-smoketest-design.md b/docs/plans/2026-02-24-h100-smoketest-design.md new file mode 100644 index 000000000..284d4c477 --- /dev/null +++ b/docs/plans/2026-02-24-h100-smoketest-design.md @@ -0,0 +1,125 @@ +# H100 GPU Smoketest Design + +**Date:** 2026-02-24 +**Goal:** Validate DQN and PPO training pipeline end-to-end on 730 days of real futures data using Scaleway H100 GPU. + +## Scope — What We're Validating + +| Question | How we answer it | +|---|---| +| Does CUDA training work on H100? | Builds compile with `CUDA_COMPUTE_CAP=90`, GPU is detected | +| Do models converge on real data? | Loss decreases across epochs, doesn't diverge | +| Are walk-forward results realistic? | Sharpe, drawdown, win rate on out-of-sample folds | +| Does the full pipeline hold together? | train → checkpoint → evaluate → report, no crashes | + +## Out of Scope + +- Hyperopt tuning (run after validation) +- L2 data / TLOB (separate step, needs download first) +- Ensemble training (validate individual models first) +- Production deployment + +## Architecture + +### Storage Strategy (Hybrid) + +- **PVC (block storage, 10Gi)** — input data. Pre-staged with 53MB OHLCV. Instant on H100 mount. Pre-populated by cheap CPU pods for future large datasets (L2). +- **S3 (object storage)** — output sink. Checkpoints, logs, evaluation reports. Mounted locally via `rclone mount` for instant review. +- **H100 never downloads data.** Data is ready on PVC when it spins up. + +``` +┌─ DATA FLOW ───────────────────────────────────────────────────┐ +│ │ +│ LOCAL MACHINE SCALEWAY (fr-par) │ +│ ────────────── ───────────────── │ +│ │ +│ rclone mount ◄──────────────► S3 bucket (foxhunt-artifacts) │ +│ ~/foxhunt-results/ ├── runs/ │ +│ └── 2026-02-24-smoketest/ │ └── / │ +│ ├── dqn/ │ ├── dqn/ │ +│ ├── ppo/ │ ├── ppo/ │ +│ └── eval/ │ └── eval/ │ +│ │ │ +│ │ PVC (block storage) │ +│ │ ┌────────────────┐ │ +│ kubectl cp (one-time) ──────────►│ │ /futures-baseline/ │ │ +│ 53MB OHLCV data │ └────────────────┘ │ +│ │ │ │ +│ │ read-only mount │ +│ │ ▼ │ +│ │ ┌──────────┐ │ +│ │ │ H100 │ │ +│ │ │ train │──► S3 │ +│ │ └──────────┘ │ +└────────────────────────────────────────────────────────────────┘ +``` + +### Build Pipeline + +Docker training image built by Gitea CI on push to main. No local builds. + +``` +git push to main + │ + ▼ +Gitea CI (CI pool runner) + │ + └── build-training (Dockerfile.training) + │ - CUDA 12.4.1, compute cap 90 (H100 native) + │ - sccache for fast rebuilds + │ - rclone in runtime image for S3 output sync + │ + ▼ + rg.fr-par.scw.cloud/foxhunt/training:latest +``` + +### Training Parameters + +| Parameter | DQN | PPO | +|---|---|---| +| Batch size | 2048 | 2048 | +| Epochs | 50 | 50 | +| Walk-forward windows | 12mo train / 3mo val / 3mo test | same | +| Folds | 4 (730 days of data) | 4 | +| Max steps/epoch | unlimited (H100 handles full dataset) | unlimited | +| Estimated time | ~15-30 min | ~20-40 min | +| Estimated cost | ~EUR 2-3 | ~EUR 2-3 | + +### Job Flow + +1. `train.sh quick-test --model dqn` or `single-model --model dqn` +2. K8s Job: H100 pod starts, mounts PVC read-only at `/data` +3. Runs `train_baseline --model dqn --epochs 50 --batch-size 2048 --data-dir /data/futures-baseline --output-dir /output` +4. On exit: `rclone sync /output/ scw:foxhunt-artifacts/runs//dqn/` +5. Repeat for PPO +6. Run `evaluate_baseline` Job with both model checkpoints +7. Review locally via `rclone mount` + +### Future: L2 Data Pipeline + +After smoketest validates the core system: + +- CPU Job (DEV1-M, always-on pool) runs `download_l2_data` → writes directly to PVC +- PVC resized to 100Gi for L2 data (~20GB MBP-10) +- H100 Job mounts PVC and trains TLOB immediately — zero download wait +- New Databento downloads always happen on SCW, never local + +## Infrastructure + +- **Region**: fr-par (all resources) +- **Registry**: `rg.fr-par.scw.cloud/foxhunt` +- **Cluster**: Kapsule 1.34 +- **GPU pool**: H100-1-80G, autoscale 0→1 +- **Block storage**: `scw-bssd`, 10Gi PVC (expandable to 100Gi) +- **Object storage**: `foxhunt-artifacts` S3 bucket + +## Cost Estimate + +| Item | Cost | +|---|---| +| DQN training (~30 min H100) | ~EUR 2 | +| PPO training (~40 min H100) | ~EUR 3 | +| Evaluation (~5 min H100) | ~EUR 0.50 | +| S3 storage (< 1GB) | ~EUR 0.01/mo | +| PVC storage (10Gi) | ~EUR 0.80/mo | +| **Total smoketest** | **~EUR 5-6** | diff --git a/docs/plans/2026-02-24-h100-smoketest-plan.md b/docs/plans/2026-02-24-h100-smoketest-plan.md new file mode 100644 index 000000000..bcf6c3be1 --- /dev/null +++ b/docs/plans/2026-02-24-h100-smoketest-plan.md @@ -0,0 +1,884 @@ +# H100 GPU Smoketest Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Validate DQN and PPO training end-to-end on 730 days of real futures data using Scaleway H100 GPU, with results accessible locally via S3 mount. + +**Architecture:** PVC (block storage) for pre-staged input data, S3 (object storage) for output with local rclone mount. Docker training image built via Gitea CI. No Rust code changes — shell wrapper handles S3 sync. + +**Tech Stack:** Terragrunt/Scaleway, K8s Jobs, Docker (CUDA 12.4.1), rclone, Scaleway Object Storage (S3-compatible) + +**Design doc:** `docs/plans/2026-02-24-h100-smoketest-design.md` + +--- + +## Task 1: Fix Container Registry Region (nl-ams → fr-par) + +The CI pipeline pushes images to `rg.nl-ams.scw.cloud/foxhunt` but all infra is in `fr-par`. Fix all references to use `fr-par`. + +**Files:** +- Modify: `.gitea/workflows/ci.yaml:18` (REGISTRY env var) +- Modify: `.gitea/workflows/ci.yaml:94` (docker login in build-images) +- Modify: `.gitea/workflows/ci.yaml:127` (docker login in build-web-gateway) +- Modify: `.gitea/workflows/ci.yaml:153` (docker login in build-training) + +**Step 1: Update the REGISTRY env var** + +In `.gitea/workflows/ci.yaml`, change line 18: + +```yaml +# OLD + REGISTRY: rg.nl-ams.scw.cloud/foxhunt + +# NEW + REGISTRY: rg.fr-par.scw.cloud/foxhunt +``` + +**Step 2: Update all docker login commands** + +All three `docker login` steps reference `rg.nl-ams.scw.cloud`. Change each to: + +```yaml +echo "${{ secrets.SCW_SECRET_KEY }}" | docker login rg.fr-par.scw.cloud -u nologin --password-stdin +``` + +Locations: lines 94, 127, 153. + +**Step 3: Verify no other nl-ams references remain** + +Run: `grep -n 'nl-ams' .gitea/workflows/ci.yaml` +Expected: No matches. + +Note: The tfstate bucket (`foxhunt-tfstate`) is intentionally in nl-ams — that's fine, it's just state storage. Don't change it. + +**Step 4: Commit** + +```bash +git add .gitea/workflows/ci.yaml +git commit -m "fix(ci): change container registry region from nl-ams to fr-par" +``` + +--- + +## Task 2: Add sccache to Training Image Build + +The service image builds use sccache but the training image build doesn't, causing cold ~30 min builds. Add sccache build-args to match the service builds. + +**Files:** +- Modify: `.gitea/workflows/ci.yaml:155-161` (build-training job) +- Modify: `infra/docker/Dockerfile.training:25-26` (install sccache in builder stage) + +**Step 1: Add sccache to Dockerfile.training builder stage** + +In `infra/docker/Dockerfile.training`, after the Rust install (line 26), add sccache install and config: + +```dockerfile +# Install Rust stable toolchain +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install sccache for build caching +ARG SCCACHE_BUCKET="" +ARG AWS_ACCESS_KEY_ID="" +ARG AWS_SECRET_ACCESS_KEY="" +ARG SCCACHE_ENDPOINT="" +RUN if [ -n "$SCCACHE_BUCKET" ]; then \ + curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \ + | tar xz --strip-components=1 -C /usr/local/bin sccache-v0.8.1-x86_64-unknown-linux-musl/sccache \ + && chmod +x /usr/local/bin/sccache; \ + fi +ENV RUSTC_WRAPPER=${SCCACHE_BUCKET:+/usr/local/bin/sccache} +ENV SCCACHE_BUCKET=${SCCACHE_BUCKET} +ENV SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} +ENV SCCACHE_S3_USE_SSL=true +ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} +ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} +``` + +Note: When `SCCACHE_BUCKET` is empty (local builds), `RUSTC_WRAPPER` is empty and sccache is not installed — no-op fallback. + +**Step 2: Add build-args to CI training build step** + +In `.gitea/workflows/ci.yaml`, update the `Build training image` step (line 155): + +```yaml + - name: Build training image + run: | + docker build \ + --build-arg SCCACHE_BUCKET=${{ secrets.SCCACHE_BUCKET }} \ + --build-arg AWS_ACCESS_KEY_ID=${{ secrets.SCW_ACCESS_KEY }} \ + --build-arg AWS_SECRET_ACCESS_KEY=${{ secrets.SCW_SECRET_KEY }} \ + --build-arg SCCACHE_ENDPOINT=${{ secrets.SCCACHE_ENDPOINT }} \ + -f infra/docker/Dockerfile.training \ + -t ${{ env.REGISTRY }}/training:${{ github.sha }} \ + -t ${{ env.REGISTRY }}/training:latest \ + . +``` + +**Step 3: Commit** + +```bash +git add infra/docker/Dockerfile.training .gitea/workflows/ci.yaml +git commit -m "perf(ci): add sccache to training image build for faster rebuilds" +``` + +--- + +## Task 3: Add rclone to Training Docker Image + +Add rclone binary to the runtime stage so training Jobs can sync output to S3 on completion. + +**Files:** +- Modify: `infra/docker/Dockerfile.training:89-92` (runtime stage apt install) + +**Step 1: Add rclone to runtime apt install** + +In `infra/docker/Dockerfile.training`, replace the runtime apt-get block (lines 89-92): + +```dockerfile +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl3 \ + curl \ + unzip \ + && curl -fsSL https://downloads.rclone.org/current/rclone-current-linux-amd64.zip -o /tmp/rclone.zip \ + && unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \ + && chmod +x /usr/local/bin/rclone \ + && rm /tmp/rclone.zip \ + && apt-get purge -y unzip \ + && rm -rf /var/lib/apt/lists/* +``` + +**Step 2: Verify rclone is available** + +This will be validated when the image is built by CI. You can also test locally: + +```bash +docker build -f infra/docker/Dockerfile.training --target builder -t test . 2>&1 | tail -1 +# (only tests the syntax parses — full build happens in CI) +``` + +**Step 3: Commit** + +```bash +git add infra/docker/Dockerfile.training +git commit -m "feat(training): add rclone to runtime image for S3 output sync" +``` + +--- + +## Task 4: Add S3 Output Sync to train.sh + +Modify `train.sh` to wrap the training command with rclone S3 upload on completion. The Job needs S3 credentials and a run ID. + +**Files:** +- Modify: `infra/scripts/train.sh` +- Create: `infra/k8s/training/s3-credentials-secret.yaml` + +**Step 1: Create the S3 credentials Secret manifest** + +Create `infra/k8s/training/s3-credentials-secret.yaml`: + +```yaml +# S3 credentials for rclone output sync. +# Apply with actual values: +# kubectl -n foxhunt create secret generic s3-credentials \ +# --from-literal=access-key= \ +# --from-literal=secret-key= +# +# This file is a reference template — do NOT commit real credentials. +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials + namespace: foxhunt + labels: + app.kubernetes.io/name: s3-credentials + app.kubernetes.io/part-of: foxhunt +type: Opaque +stringData: + access-key: REPLACE_ME + secret-key: REPLACE_ME +``` + +**Step 2: Update train.sh defaults and add S3 config** + +In `infra/scripts/train.sh`, add after the existing defaults (after line 28): + +```bash +S3_BUCKET="foxhunt-artifacts" +S3_ENDPOINT="https://s3.fr-par.scw.cloud" +RUN_ID="$(date +%Y%m%d-%H%M%S)" +``` + +**Step 3: Add --output-dir and --data-dir to build_args()** + +Replace the `build_args()` function (lines 117-130): + +```bash +build_args() { + local model="$1" + local binary="${MODEL_BINARY[$model]}" + local args=("$binary") + + args+=("--symbol" "$SYMBOL") + args+=("--max-steps-per-epoch" "$MAX_STEPS") + args+=("--data-dir" "$DATA_DIR") + args+=("--output-dir" "/output") + + if [[ "$PRESET" == "hyperopt" ]]; then + args+=("--trials" "$TRIALS") + fi + + echo "${args[*]}" +} +``` + +**Step 4: Update generate_job_manifest() with S3 sync and credentials** + +Replace the `generate_job_manifest()` function (lines 133-196) entirely: + +```bash +generate_job_manifest() { + local model="$1" + local job_name="train-${model}-${TIMESTAMP}" + local train_args + train_args="$(build_args "$model")" + + cat < /tmp/rclone/rclone.conf < /tmp/rclone/rclone.conf < | grep training +``` + +Or check the Scaleway console Container Registry page. + +--- + +## Task 7: Create K8s Secrets and Stage Data + +Set up the namespace, secrets, PVC, and upload the 53MB OHLCV dataset. + +**Step 1: Ensure foxhunt namespace exists** + +```bash +kubectl create namespace foxhunt --dry-run=client -o yaml | kubectl apply -f - +``` + +**Step 2: Create SCW registry pull secret** + +```bash +kubectl -n foxhunt create secret docker-registry scw-registry \ + --docker-server=rg.fr-par.scw.cloud \ + --docker-username=nologin \ + --docker-password= +``` + +**Step 3: Create S3 credentials secret** + +```bash +kubectl -n foxhunt create secret generic s3-credentials \ + --from-literal=access-key= \ + --from-literal=secret-key= +``` + +**Step 4: Apply PVC** + +```bash +kubectl apply -f infra/k8s/training/training-data-pvc.yaml +``` + +Verify: +```bash +kubectl -n foxhunt get pvc training-data-pvc +``` +Expected: STATUS = `Bound` (or `Pending` until a pod mounts it). + +**Step 5: Upload OHLCV data via upload pod** + +```bash +kubectl apply -f infra/k8s/training/data-upload-job.yaml +kubectl -n foxhunt wait --for=condition=ready pod/data-upload --timeout=60s +kubectl cp data/cache/futures-baseline foxhunt/data-upload:/data/futures-baseline +``` + +**Step 6: Verify data on PVC** + +```bash +kubectl -n foxhunt exec data-upload -- ls -la /data/futures-baseline/ +kubectl -n foxhunt exec data-upload -- du -sh /data/futures-baseline/ +``` + +Expected: 36 `.dbn.zst` files, ~53MB total. + +**Step 7: Clean up upload pod** + +```bash +kubectl -n foxhunt delete pod data-upload +``` + +--- + +## Task 8: Set Up Local rclone Mount + +Configure rclone locally to mount the S3 bucket for browsing training output. + +**Step 1: Install rclone (if not already installed)** + +```bash +# Ubuntu/Debian +sudo apt install rclone +# or +curl https://rclone.org/install.sh | sudo bash +``` + +**Step 2: Configure Scaleway S3 remote** + +```bash +rclone config +``` + +Or create the config directly: + +```bash +mkdir -p ~/.config/rclone +cat >> ~/.config/rclone/rclone.conf <<'EOF' +[scw] +type = s3 +provider = Scaleway +access_key_id = +secret_access_key = +endpoint = https://s3.fr-par.scw.cloud +region = fr-par +acl = private +EOF +``` + +**Step 3: Test connectivity** + +```bash +rclone ls scw:foxhunt-artifacts/ --max-depth 1 +``` + +Expected: Lists bucket contents (may be empty). + +**Step 4: Create mount point and mount** + +```bash +mkdir -p ~/foxhunt-results +rclone mount scw:foxhunt-artifacts/runs/ ~/foxhunt-results/ \ + --read-only \ + --vfs-cache-mode full \ + --daemon +``` + +Verify: +```bash +ls ~/foxhunt-results/ +``` + +To unmount later: +```bash +fusermount -u ~/foxhunt-results +``` + +--- + +## Task 9: Run DQN Smoketest + +Submit the DQN training Job to the H100. + +**Step 1: Verify the training image is available** + +Wait for Task 6 CI build to complete. Then: + +```bash +kubectl -n foxhunt run image-test --rm -it \ + --image=rg.fr-par.scw.cloud/foxhunt/training:latest \ + --overrides='{"spec":{"imagePullSecrets":[{"name":"scw-registry"}]}}' \ + -- train_baseline --help +``` + +Expected: Shows `train_baseline` help text with CLI options. + +**Step 2: Submit DQN quick-test first (sanity check)** + +```bash +./infra/scripts/train.sh quick-test --model dqn +``` + +This runs with `--max-steps=500` to validate the pipeline works before committing to a full run. + +**Step 3: Monitor the quick-test** + +```bash +kubectl -n foxhunt get pods -l foxhunt/job-type=training --watch +# In another terminal: +kubectl -n foxhunt logs -f job/train-dqn- +``` + +Expected: GPU detected, training starts, completes in ~2-5 min, output synced to S3. + +**Step 4: Verify output on S3** + +```bash +rclone ls scw:foxhunt-artifacts/runs/ --max-depth 3 +# Or via local mount: +ls ~/foxhunt-results/ +``` + +Expected: `/dqn/` directory with checkpoint files. + +**Step 5: Submit full DQN training** + +```bash +./infra/scripts/train.sh single-model --model dqn --max-steps 0 --timeout 7200 +``` + +Note: `--max-steps 0` means unlimited (full dataset). `--timeout 7200` = 2 hour deadline. + +**Step 6: Monitor and verify** + +```bash +kubectl -n foxhunt logs -f job/train-dqn- +``` + +Expected: 4 walk-forward folds, ~50 epochs each, loss decreasing. Output: `dqn_fold{0..3}_best.safetensors`, `norm_stats_fold{0..3}.json`. + +--- + +## Task 10: Run PPO Smoketest + +Submit PPO training Job after DQN completes (to avoid two H100 nodes). + +**Step 1: Submit PPO training** + +```bash +./infra/scripts/train.sh single-model --model ppo --max-steps 0 --timeout 7200 +``` + +**Step 2: Monitor** + +```bash +kubectl -n foxhunt logs -f job/train-ppo- +``` + +Expected: PPO actor/critic training across 4 folds. Note: PPO value loss may fluctuate (this is normal — unlike DQN, value estimates shift as policy changes). + +**Step 3: Verify output** + +```bash +ls ~/foxhunt-results//ppo/ +``` + +Expected: `ppo_fold{0..3}_actor.safetensors`, `ppo_fold{0..3}_critic.safetensors`, `ppo_fold{0..3}_meta.json`. + +--- + +## Task 11: Run Walk-Forward Evaluation + +Evaluate both trained models on out-of-sample test folds. + +**Step 1: Submit evaluation** + +Use the run ID from the training jobs: + +```bash +./infra/scripts/train.sh evaluate --run-id +``` + +**Step 2: Monitor** + +```bash +kubectl -n foxhunt logs -f job/eval-- +``` + +Expected: Loads DQN and PPO checkpoints, runs inference on each test fold, computes metrics. + +**Step 3: Review results locally** + +```bash +cat ~/foxhunt-results//eval/evaluation_report.json | python3 -m json.tool +``` + +Key metrics to look for: +- **Sharpe ratio**: > 0.5 is promising, > 1.0 is good, > 2.0 is excellent +- **Max drawdown**: < 20% is acceptable for a smoketest +- **Win rate**: > 50% indicates the model learned something +- **Profit factor**: > 1.0 means profitable + +**Step 4: Compare DQN vs PPO** + +Review per-fold metrics to understand: +- Which model performs better out-of-sample? +- Are results consistent across folds or do they vary? +- Does either model show signs of overfitting (great train, poor test)? + +--- + +## Task 12: Document Results and Clean Up + +**Step 1: Save results summary** + +After reviewing `evaluation_report.json`, note findings. Do NOT commit model artifacts to git. + +**Step 2: Clean up K8s Jobs** + +```bash +kubectl -n foxhunt delete jobs -l foxhunt/job-type=training +kubectl -n foxhunt delete jobs -l foxhunt/job-type=evaluate +``` + +**Step 3: Verify H100 scales down** + +The Kapsule autoscaler + idle-reaper should scale the GPU pool to 0 after 10 minutes of no jobs: + +```bash +kubectl get nodes -l k8s.scaleway.com/pool-name=gpu-training +``` + +Expected: 0 nodes (or node in `NotReady` → removed state). + +If the node doesn't scale down automatically: + +```bash +# Check the autoscaler +kubectl -n kube-system logs -l app=cluster-autoscaler --tail=20 +``` + +**Step 4: Keep the PVC and S3 data** + +The PVC with OHLCV data stays — it's ready for future runs. S3 results persist for review. Monthly cost: ~EUR 0.80 (PVC) + ~EUR 0.01 (S3). + +--- + +## Quick Reference + +### Submit training +```bash +./infra/scripts/train.sh single-model --model dqn --timeout 7200 +./infra/scripts/train.sh single-model --model ppo --timeout 7200 +``` + +### Submit evaluation +```bash +./infra/scripts/train.sh evaluate --run-id +``` + +### Monitor +```bash +kubectl -n foxhunt get pods -l foxhunt/job-type=training --watch +kubectl -n foxhunt logs -f job/ +``` + +### Review results locally +```bash +ls ~/foxhunt-results/ +cat ~/foxhunt-results//eval/evaluation_report.json +``` + +### Emergency: kill running Job +```bash +kubectl -n foxhunt delete job +``` diff --git a/infra/docker/Dockerfile.training b/infra/docker/Dockerfile.training index e8814d2f7..6231636f4 100644 --- a/infra/docker/Dockerfile.training +++ b/infra/docker/Dockerfile.training @@ -25,6 +25,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable ENV PATH="/root/.cargo/bin:${PATH}" +# Install sccache for build caching (no-op when SCCACHE_BUCKET is empty) +ARG SCCACHE_BUCKET="" +ARG AWS_ACCESS_KEY_ID="" +ARG AWS_SECRET_ACCESS_KEY="" +ARG SCCACHE_ENDPOINT="" +RUN if [ -n "$SCCACHE_BUCKET" ]; then \ + curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \ + | tar xz --strip-components=1 -C /usr/local/bin sccache-v0.8.1-x86_64-unknown-linux-musl/sccache \ + && chmod +x /usr/local/bin/sccache; \ + fi +ENV RUSTC_WRAPPER=${SCCACHE_BUCKET:+/usr/local/bin/sccache} +ENV SCCACHE_BUCKET=${SCCACHE_BUCKET} +ENV SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} +ENV SCCACHE_S3_USE_SSL=true +ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} +ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + WORKDIR /build # Copy workspace manifests first for layer caching @@ -90,6 +107,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ libssl3 \ curl \ + unzip \ + && curl -fsSL https://downloads.rclone.org/current/rclone-current-linux-amd64.zip -o /tmp/rclone.zip \ + && unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \ + && chmod +x /usr/local/bin/rclone \ + && rm /tmp/rclone.zip \ + && apt-get purge -y unzip \ && rm -rf /var/lib/apt/lists/* RUN groupadd -g 1000 foxhunt \ diff --git a/infra/k8s/training/s3-credentials.secret.example b/infra/k8s/training/s3-credentials.secret.example new file mode 100644 index 000000000..afad5339c --- /dev/null +++ b/infra/k8s/training/s3-credentials.secret.example @@ -0,0 +1,17 @@ +# S3 credentials for rclone output sync. +# Apply with actual values (do NOT use this file directly): +# kubectl -n foxhunt create secret generic s3-credentials \ +# --from-literal=access-key= \ +# --from-literal=secret-key= +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials + namespace: foxhunt + labels: + app.kubernetes.io/name: s3-credentials + app.kubernetes.io/part-of: foxhunt +type: Opaque +stringData: + access-key: REPLACE_ME + secret-key: REPLACE_ME diff --git a/infra/scripts/train.sh b/infra/scripts/train.sh index 292f7bd25..c2dcd79da 100755 --- a/infra/scripts/train.sh +++ b/infra/scripts/train.sh @@ -26,6 +26,9 @@ IMAGE="rg.fr-par.scw.cloud/foxhunt/training:latest" MODEL="" PRESET="" TIMESTAMP="$(date +%s)" +S3_BUCKET="foxhunt-artifacts" +S3_ENDPOINT="https://s3.fr-par.scw.cloud" +RUN_ID="$(date +%Y%m%d-%H%M%S)" ALL_MODELS=(dqn ppo tft mamba2 tggn tlob liquid kan xlstm diffusion) @@ -43,6 +46,8 @@ declare -A MODEL_BINARY=( [diffusion]=train_diffusion_dbn ) +EVAL_BINARY="evaluate_baseline" + # --- Helpers -------------------------------------------------------------- usage() { cat <<'USAGE' @@ -53,6 +58,7 @@ Presets: single-model Full training run (requires --model) full-ensemble Trains all 10 models sequentially (dqn ppo tft mamba2 tggn tlob liquid kan xlstm diffusion) hyperopt Hyperparameter search (requires --model, uses --trials) + evaluate Evaluate trained models (requires --run-id from prior training run) Options: --model MODEL Model name: dqn | ppo | tft | mamba2 | tggn | tlob | liquid | kan | xlstm | diffusion @@ -84,6 +90,7 @@ while [[ $# -gt 0 ]]; do --trials) TRIALS="$2"; shift 2 ;; --max-steps) MAX_STEPS="$2"; shift 2 ;; --timeout) TIMEOUT="$2"; shift 2 ;; + --run-id) RUN_ID="$2"; shift 2 ;; -h|--help) usage ;; *) die "Unknown option: $1" ;; esac @@ -108,8 +115,11 @@ case "$PRESET" in [[ -z "$MODEL" ]] && die "hyperopt preset requires --model" validate_model "$MODEL" ;; + evaluate) + [[ -z "$RUN_ID" ]] && die "evaluate preset requires --run-id (from a previous training run)" + ;; *) - die "Unknown preset '${PRESET}'. Valid: quick-test, single-model, full-ensemble, hyperopt" + die "Unknown preset '${PRESET}'. Valid: quick-test, single-model, full-ensemble, hyperopt, evaluate" ;; esac @@ -121,6 +131,8 @@ build_args() { args+=("--symbol" "$SYMBOL") args+=("--max-steps-per-epoch" "$MAX_STEPS") + args+=("--data-dir" "$DATA_DIR") + args+=("--output-dir" "/output") if [[ "$PRESET" == "hyperopt" ]]; then args+=("--trials" "$TRIALS") @@ -146,9 +158,11 @@ metadata: foxhunt/job-type: training foxhunt/model: ${model} foxhunt/preset: ${PRESET} + foxhunt/run-id: "${RUN_ID}" spec: activeDeadlineSeconds: ${TIMEOUT} backoffLimit: 0 + ttlSecondsAfterFinished: 600 template: metadata: labels: @@ -170,7 +184,27 @@ spec: image: ${IMAGE} command: ["/bin/sh", "-c"] args: - - "${train_args}" + - | + set -e + echo "=== Training: ${model} (run: ${RUN_ID}) ===" + nvidia-smi || echo "WARNING: nvidia-smi not available" + echo "=== Starting training ===" + ${train_args} + echo "=== Training complete, syncing output to S3 ===" + RCLONE_CONFIG_SCW_TYPE=s3 \\ + RCLONE_CONFIG_SCW_PROVIDER=Scaleway \\ + RCLONE_CONFIG_SCW_ACCESS_KEY_ID=\$(cat /secrets/access-key) \\ + RCLONE_CONFIG_SCW_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ + RCLONE_CONFIG_SCW_ENDPOINT=${S3_ENDPOINT} \\ + RCLONE_CONFIG_SCW_REGION=fr-par \\ + RCLONE_CONFIG_SCW_ACL=private \\ + rclone sync /output/ "scw:${S3_BUCKET}/runs/${RUN_ID}/${model}/" -v + echo "=== Output synced to s3://${S3_BUCKET}/runs/${RUN_ID}/${model}/ ===" + env: + - name: RUST_LOG + value: info + - name: SQLX_OFFLINE + value: "true" resources: requests: nvidia.com/gpu: "1" @@ -186,12 +220,115 @@ spec: readOnly: true - name: output mountPath: /output + - name: s3-credentials + mountPath: /secrets + readOnly: true volumes: - name: training-data persistentVolumeClaim: claimName: training-data-pvc - name: output emptyDir: {} + - name: s3-credentials + secret: + secretName: s3-credentials +EOF +} + +generate_eval_manifest() { + local job_name="eval-${RUN_ID}-${TIMESTAMP}" + + cat </dev/null || true done ;; + evaluate) + echo "--- Submitting evaluation job for run ${RUN_ID}" + generate_eval_manifest | kubectl apply -f - + submitted_jobs+=("eval-${RUN_ID}-${TIMESTAMP}") + ;; esac echo ""