diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index be48e3b5a..623dc648b 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -210,6 +210,22 @@ async fn main() -> Result<()> { let t0 = Instant::now(); + // ── Early exit if cache already exists for this data ───────────────────── + let hex_key_early = ml::feature_cache::calculate_dbn_cache_key_full( + &data_dir, + mbp10_dir.as_deref(), + trades_dir.as_deref(), + ).context("Failed to compute cache key")?; + let early_check_path = output_dir.join(format!("{hex_key_early}.fxcache")); + if early_check_path.exists() { + let size_mb = std::fs::metadata(&early_check_path) + .map(|m| m.len() as f64 / 1_048_576.0) + .unwrap_or(0.0); + println!("Cache already exists: {} ({:.1} MB) — skipping extraction", + early_check_path.display(), size_mb); + return Ok(()); + } + // ── Step 1: Load OHLCV bars from DBN files ────────────────────────────── info!("Loading OHLCV bars from {} (symbol={})...", data_dir.display(), opts.symbol); let mut dbn_files = collect_dbn_files_filtered(&data_dir, Some(&opts.symbol)); diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml index bd7b358c1..5086cd84f 100644 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ b/infra/k8s/argo/compile-and-train-template.yaml @@ -115,9 +115,12 @@ spec: parameters: - name: tag value: "{{tasks.compile-training.outputs.parameters.tag}}" + - name: precompute + template: precompute + dependencies: [compile-training] - name: hyperopt template: hyperopt - dependencies: [fetch-binary, gpu-warmup] + dependencies: [fetch-binary, gpu-warmup, precompute] when: "{{workflow.parameters.hyperopt-trials}} != 0" - name: train-best template: train-best @@ -173,6 +176,8 @@ spec: readOnly: true - name: cargo-target-cuda mountPath: /cargo-target + - name: training-data + mountPath: /data args: - | set -e @@ -275,9 +280,52 @@ spec: "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}" done + # Copy binaries to PVC so precompute and other workflows can use them + mkdir -p /data/bin + cp "$BUILD/bin/training/"* /data/bin/ + echo "=== Binaries deployed to PVC /data/bin/ ===" + ls -lh /data/bin/ + echo "=== Compile + upload done (${TAG}) ===" echo "$TAG" > /tmp/tag + # ── precompute: generate fxcache if data changed (skips if cache exists) ── + - name: precompute + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: ubuntu:24.04 + command: ["/bin/bash", "-c"] + env: + - name: RUST_LOG + value: info + args: + - | + set -e + BINARY=/data/bin/precompute_features + chmod +x "$BINARY" + $BINARY \ + --data-dir /data/futures-baseline \ + --mbp10-data-dir /data/futures-baseline-mbp10 \ + --trades-data-dir /data/futures-baseline-trades \ + --output-dir /data/feature-cache \ + --symbol ES.FUT \ + --yes + volumeMounts: + - name: training-data + mountPath: /data + resources: + requests: + cpu: "4" + memory: 16Gi + limits: + cpu: "28" + memory: 56Gi + # ── gpu-warmup: trigger GPU autoscale during compile ── - name: gpu-warmup nodeSelector: diff --git a/infra/k8s/argo/precompute-features-template.yaml b/infra/k8s/argo/precompute-features-template.yaml index 0dc6ee4ec..64d09aaa0 100644 --- a/infra/k8s/argo/precompute-features-template.yaml +++ b/infra/k8s/argo/precompute-features-template.yaml @@ -83,13 +83,12 @@ spec: args: - | set -e - # Fetch latest binary from GitLab Package Registry + # Binary deployed to PVC by compile-and-train workflow BINARY=/data/bin/precompute_features - GITLAB="http://gitlab.foxhunt.svc.cluster.local:8181" - echo "Downloading precompute_features from GitLab packages..." - curl -f -o "$BINARY" \ - "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/latest/precompute_features" \ - || echo "WARN: GitLab download failed, using existing binary on PVC" + if [ ! -f "$BINARY" ]; then + echo "ERROR: $BINARY not found on PVC. Run compile-and-train first." + exit 1 + fi chmod +x "$BINARY" SYMBOL="{{inputs.parameters.symbol}}"