diff --git a/crates/ml/build.rs b/crates/ml/build.rs index 000ea0538..f3ad295d6 100644 --- a/crates/ml/build.rs +++ b/crates/ml/build.rs @@ -281,9 +281,24 @@ fn try_compile_kernel( fn emit_feature_schema_hash() { // Source files whose bytes determine the on-disk schema. // Touching ANY of these files implies "fxcache regen on next deploy". + // + // 2026-04-27: added `examples/precompute_features.rs` after diagnosing the + // L40S `label_scale=5420` (raw-price magnitude) vs smoke `label_scale=0.05` + // (z-score) divergence. Root cause: the z-score normalisation step landed + // in `precompute_features.rs:625-630` on 2026-04-03 but that file was not + // tracked here, so the hash didn't change. PVC fxcaches written by an + // earlier build (no normalisation) still validated against the current + // binary, silently feeding raw features through column 0 of next_states + // and leaking future-bar prices into the policy via the aux head's + // shared trunk → impossible Sharpe (141 at epoch 0, 0.32% max DD over + // 214k bars). Adding the precompute pipeline to the hash forces every + // fxcache to be regenerated when its writer changes — including any + // future tweaks to feature normalisation, target ordering, or per-feature + // post-processing. let schema_sources: &[&str] = &[ "src/features/extraction.rs", "src/fxcache.rs", + "examples/precompute_features.rs", "../ml-core/src/state_layout.rs", ]; diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 8a368becc..f84ccb48a 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,28 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +fxcache schema-hash gap fix (2026-04-27): `build.rs::emit_feature_schema_hash` +hashes only `src/features/extraction.rs`, `src/fxcache.rs`, and +`../ml-core/src/state_layout.rs`. The z-score normalization step lives in +`examples/precompute_features.rs:625-631` (added 2026-04-03 in commit +`9f7c14978f`) and was NOT covered by the hash, so a fxcache written by an +older precompute build (raw features, no normalization) silently passed +today's `validate()` because every other field matched. The PVC fxcache +on the L40S Argo path was such a stale artefact: feature column 0 stored +RAW CLOSE PRICES (~$5180 ES futures level) instead of z-normalized log- +returns. The aux next-bar head reads `next_states[:, 0]` as its label +(`gpu_dqn_trainer.rs:7758-7789`); with raw-price labels, the EMA +`label_scale` reached 5420 vs smoke's 0.05. The shared trunk learned +to predict next-bar prices, leaking future info into the policy → +`train-f8h6q` epoch-0 reported impossible Sharpe=141.99 with 0.32% +max-drawdown over 214k bars. Fix: add `examples/precompute_features.rs` +to `schema_sources`. New hash invalidates the stale PVC cache; Argo's +`ensure-fxcache` regenerate-on-failure branch +(`infra/k8s/argo/train-template.yaml:372-383`) auto-regens with current +normalized precompute. Generalises beyond this incident: any future +change to feature normalization, target ordering, or precompute +post-processing is now hash-tracked. + mag_stats wr_h/wr_f attribution fix (2026-04-27): `experience_kernels.cu` line 1916 binned `action_mag_per_sample` by `actual_mag_core` at every step, including trade-close events. `unified_env_step_core` forces