Compare commits
3 Commits
main
...
ml-alpha-d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
251b871294 | ||
|
|
b7bbb700f8 | ||
|
|
84caa99b65 |
@@ -1145,12 +1145,25 @@ fn main() -> Result<()> {
|
||||
// the train-end policy's OOS performance. True pure-eval (forward
|
||||
// only, no backward) is a follow-up architectural change.
|
||||
if cli.n_eval_steps > 0 && !eval_files.is_empty() {
|
||||
let head_before_eval = sim
|
||||
.read_total_trade_count()
|
||||
.context("read trade count pre-eval")?;
|
||||
// dd049d9a4 baseline: NO reset_session_state (that method came in
|
||||
// a later Phase 4-era commit and is intentionally excluded from
|
||||
// this experiment per "test dd049d9a4 baseline with eval-math fix only").
|
||||
let head_before_per_b = sim
|
||||
.read_per_backtest_trade_counts()
|
||||
.context("snapshot per-backtest trade-count pre-eval")?;
|
||||
let head_before_min = head_before_per_b.iter().min().copied().unwrap_or(0);
|
||||
let head_before_max = head_before_per_b.iter().max().copied().unwrap_or(0);
|
||||
let head_before_sum: u64 =
|
||||
head_before_per_b.iter().map(|&h| h as u64).sum();
|
||||
eprintln!(
|
||||
"── eval phase: {} steps on {} held-out files (trade-record checkpoint head={}) ──",
|
||||
cli.n_eval_steps, eval_files.len(), head_before_eval
|
||||
"── eval phase: {} steps on {} held-out files; pre-eval head per b_size={} \
|
||||
accounts: min={} max={} sum={} ──",
|
||||
cli.n_eval_steps,
|
||||
eval_files.len(),
|
||||
head_before_per_b.len(),
|
||||
head_before_min,
|
||||
head_before_max,
|
||||
head_before_sum
|
||||
);
|
||||
|
||||
let eval_loader_cfg = MultiHorizonLoaderConfig {
|
||||
@@ -1198,22 +1211,82 @@ fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
// Drain trade records, slice to eval-only, compute summary.
|
||||
let all_records = sim
|
||||
.read_trade_records(0)
|
||||
.context("read trade records post-eval")?;
|
||||
let head_before_usize = head_before_eval as usize;
|
||||
let eval_records: Vec<_> = if all_records.len() > head_before_usize {
|
||||
all_records[head_before_usize..].to_vec()
|
||||
} else {
|
||||
// Trade log wrapped past TRADE_LOG_CAP — use what we have.
|
||||
// For smoke (b_size=1, ≤200 trades) this branch never fires.
|
||||
// Aggregate eval-phase trades across ALL b_size accounts, with
|
||||
// correct per-account head_before slicing. Replaces the previous
|
||||
// single-account read of backtest 0 + aggregate-vs-per-account
|
||||
// scale mismatch (see spec
|
||||
// docs/superpowers/specs/2026-05-31-eval-summary-trade-aggregation-design.md
|
||||
// for the root-cause analysis).
|
||||
let all_per_b = sim
|
||||
.read_trade_records_all()
|
||||
.context("read all backtests' trade records post-eval")?;
|
||||
let head_after_per_b = sim
|
||||
.read_per_backtest_trade_counts()
|
||||
.context("snapshot per-backtest trade-count post-eval")?;
|
||||
|
||||
let cap_u32 = ml_backtesting::lob::TRADE_LOG_CAP as u32;
|
||||
let mut eval_records: Vec<ml_backtesting::order::TradeRecord> = Vec::new();
|
||||
let mut n_eval_trades_seen: u64 = 0;
|
||||
let mut n_eval_trades_dropped: u64 = 0;
|
||||
let mut n_pre_eval_wrapped: u64 = 0;
|
||||
|
||||
for (b, records) in all_per_b.iter().enumerate() {
|
||||
let head_before = head_before_per_b[b];
|
||||
let head_after = head_after_per_b[b];
|
||||
let eval_count_total = head_after.saturating_sub(head_before);
|
||||
n_eval_trades_seen += eval_count_total as u64;
|
||||
|
||||
// The ring's contents cover cumulative-stream indices
|
||||
// [max(0, head_after - cap), head_after).
|
||||
let ring_start_stream_idx = head_after.saturating_sub(cap_u32);
|
||||
|
||||
if head_before < ring_start_stream_idx {
|
||||
// Some pre-eval trades had wrapped out before eval started —
|
||||
// diagnostic only, doesn't affect eval slicing.
|
||||
n_pre_eval_wrapped += (ring_start_stream_idx - head_before) as u64;
|
||||
}
|
||||
|
||||
if eval_count_total > cap_u32 {
|
||||
// Eval-phase trades wrapped (lost). Should be 0 with
|
||||
// TRADE_LOG_CAP=4096 at typical cluster scale.
|
||||
n_eval_trades_dropped += (eval_count_total - cap_u32) as u64;
|
||||
}
|
||||
|
||||
// Slice the ring contents to eval-only.
|
||||
// ring index of first eval trade = head_before − ring_start_stream_idx
|
||||
// (clamped at 0 if all pre-eval already wrapped out).
|
||||
let eval_start_in_ring =
|
||||
head_before.saturating_sub(ring_start_stream_idx) as usize;
|
||||
if eval_start_in_ring < records.len() {
|
||||
eval_records.extend_from_slice(&records[eval_start_in_ring..]);
|
||||
}
|
||||
}
|
||||
|
||||
if n_eval_trades_dropped > 0 {
|
||||
eprintln!(
|
||||
"warning: trade log wrapped — head_before={} but only {} records readable",
|
||||
head_before_usize, all_records.len()
|
||||
"warning: {} eval trades wrapped out across {} accounts \
|
||||
(TRADE_LOG_CAP={}); summary based on {} captured eval trades",
|
||||
n_eval_trades_dropped,
|
||||
all_per_b.len(),
|
||||
cap_u32,
|
||||
eval_records.len()
|
||||
);
|
||||
all_records
|
||||
};
|
||||
}
|
||||
if n_pre_eval_wrapped > 0 {
|
||||
eprintln!(
|
||||
"info: {} pre-eval trades had already wrapped before eval phase \
|
||||
(no effect on eval summary)",
|
||||
n_pre_eval_wrapped
|
||||
);
|
||||
}
|
||||
eprintln!(
|
||||
"eval phase trade accounting: n_eval_trades_seen={} n_captured={} \
|
||||
n_dropped={} b_size={}",
|
||||
n_eval_trades_seen,
|
||||
eval_records.len(),
|
||||
n_eval_trades_dropped,
|
||||
all_per_b.len()
|
||||
);
|
||||
|
||||
// Synthesise a pnl_curve from per-trade cumulative PnL for
|
||||
// compute_summary's max_drawdown calc.
|
||||
@@ -1227,19 +1300,42 @@ fn main() -> Result<()> {
|
||||
ml_backtesting::artifacts::compute_summary(&eval_records, &pnl_curve);
|
||||
|
||||
eprintln!(
|
||||
"eval summary: n_trades={} pnl_usd={:.2} pf={:.3} sharpe_ann={:.3} max_dd_usd={:.2} win_rate={:.3}",
|
||||
"eval summary: n_trades={} pnl_usd={:.2} pf={:.3} sharpe_ann={:.3} \
|
||||
max_dd_usd={:.2} win_rate={:.3} | seen={} dropped={} b={}",
|
||||
eval_summary.n_trades,
|
||||
eval_summary.total_pnl_usd,
|
||||
eval_summary.profit_factor,
|
||||
eval_summary.sharpe_ann,
|
||||
eval_summary.max_drawdown_usd,
|
||||
eval_summary.win_rate,
|
||||
n_eval_trades_seen,
|
||||
n_eval_trades_dropped,
|
||||
all_per_b.len(),
|
||||
);
|
||||
|
||||
// Write eval_summary.json with the existing compute_summary fields
|
||||
// PLUS four new aggregation-aware fields:
|
||||
// n_eval_trades_seen — true total cumulative dones across b_size
|
||||
// n_eval_trades_dropped — eval trades lost to ring wrap (0 at typical scale)
|
||||
// n_pre_eval_trades_wrapped — pre-eval trades wrapped before eval (diagnostic)
|
||||
// b_size — context for downstream interpretation
|
||||
let aggregated_summary = serde_json::json!({
|
||||
"n_trades": eval_summary.n_trades,
|
||||
"total_pnl_usd": eval_summary.total_pnl_usd,
|
||||
"profit_factor": eval_summary.profit_factor,
|
||||
"sharpe_ann": eval_summary.sharpe_ann,
|
||||
"max_drawdown_usd": eval_summary.max_drawdown_usd,
|
||||
"win_rate": eval_summary.win_rate,
|
||||
"n_eval_trades_seen": n_eval_trades_seen,
|
||||
"n_eval_trades_dropped": n_eval_trades_dropped,
|
||||
"n_pre_eval_trades_wrapped": n_pre_eval_wrapped,
|
||||
"b_size": cli.n_backtests,
|
||||
});
|
||||
|
||||
let eval_summary_path = cli.out.join("eval_summary.json");
|
||||
let f = std::fs::File::create(&eval_summary_path)
|
||||
.with_context(|| format!("create {}", eval_summary_path.display()))?;
|
||||
serde_json::to_writer_pretty(f, &eval_summary)
|
||||
serde_json::to_writer_pretty(f, &aggregated_summary)
|
||||
.with_context(|| format!("write {}", eval_summary_path.display()))?;
|
||||
eprintln!("eval summary written: {}", eval_summary_path.display());
|
||||
}
|
||||
|
||||
@@ -11,9 +11,21 @@ fn main() -> Result<(), String> {
|
||||
|
||||
// Per pearl_build_rs_rerun_if_env_changed.md: pair every env::var
|
||||
// with rerun-if-env-changed.
|
||||
// Default sm_86 covers RTX 3050 (local dev). Production Argo workflows
|
||||
// override via FOXHUNT_CUDA_ARCH=sm_89 (L40S Ada) or sm_90 (H100).
|
||||
let arch = std::env::var("FOXHUNT_CUDA_ARCH").unwrap_or_else(|_| "sm_86".into());
|
||||
//
|
||||
// Arch detection (mirrors crates/ml-alpha/build.rs:detect_arch — the
|
||||
// original "mirror" docstring above was aspirational, B-9 cluster
|
||||
// alpha-rl-mbg2n on H100 caught the gap: ml-alpha picked sm_90 via
|
||||
// CUDA_COMPUTE_CAP but ml-backtesting fell through to default sm_86
|
||||
// → no kernel image error at LobSimCuda::new). Priority:
|
||||
// 1. CUDA_COMPUTE_CAP (numeric, e.g. "90") — set by alpha-rl-template
|
||||
// from nvidia-smi inside the compile pod
|
||||
// 2. FOXHUNT_CUDA_ARCH (sm_-prefixed, e.g. "sm_90") — set by
|
||||
// lob-backtest-sweep-template
|
||||
// 3. nvidia-smi --query-gpu=compute_cap at build time
|
||||
// 4. Default sm_86 (RTX 3050 Ti local dev)
|
||||
let arch = detect_arch();
|
||||
eprintln!(" ml-backtesting: compiling kernels for {arch}");
|
||||
println!("cargo:rerun-if-env-changed=CUDA_COMPUTE_CAP");
|
||||
println!("cargo:rerun-if-env-changed=FOXHUNT_CUDA_ARCH");
|
||||
println!("cargo:rerun-if-env-changed=CUDA_PATH");
|
||||
println!("cargo:rerun-if-env-changed=NVCC");
|
||||
@@ -60,3 +72,38 @@ fn main() -> Result<(), String> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the `sm_<NN>` arch string, honoring (in order):
|
||||
/// `CUDA_COMPUTE_CAP` numeric env → `FOXHUNT_CUDA_ARCH` sm_-prefixed env
|
||||
/// → `nvidia-smi --query-gpu=compute_cap` → default `sm_86`.
|
||||
fn detect_arch() -> String {
|
||||
// 1. Numeric CUDA_COMPUTE_CAP (e.g. "90") from alpha-rl-template.
|
||||
if let Ok(cap) = std::env::var("CUDA_COMPUTE_CAP") {
|
||||
let trimmed = cap.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return format!("sm_{trimmed}");
|
||||
}
|
||||
}
|
||||
// 2. sm_-prefixed FOXHUNT_CUDA_ARCH (e.g. "sm_90") from lob-backtest-sweep.
|
||||
if let Ok(arch) = std::env::var("FOXHUNT_CUDA_ARCH") {
|
||||
let trimmed = arch.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
}
|
||||
// 3. Query the GPU on this machine (e.g. "9.0" → "90" → "sm_90").
|
||||
if let Ok(output) = Command::new("nvidia-smi")
|
||||
.args(["--query-gpu=compute_cap", "--format=csv,noheader"])
|
||||
.output()
|
||||
{
|
||||
if output.status.success() {
|
||||
let s = String::from_utf8_lossy(&output.stdout);
|
||||
let cap = s.trim().replace('.', "");
|
||||
if !cap.is_empty() {
|
||||
return format!("sm_{cap}");
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. Default — RTX 3050 Ti local dev.
|
||||
"sm_86".to_string()
|
||||
}
|
||||
|
||||
@@ -16,8 +16,13 @@ pub const STOP_SLOT_BYTES: usize = 32;
|
||||
/// Bytes per Orders struct (limits[32] + stops[16]).
|
||||
pub const ORDERS_BYTES: usize = MAX_LIMITS * LIMIT_SLOT_BYTES + MAX_STOPS * STOP_SLOT_BYTES;
|
||||
/// Max closed-trade records buffered per backtest before the host
|
||||
/// must drain. Sized for a single fixture day at ~30s decision cadence.
|
||||
pub const TRADE_LOG_CAP: usize = 1024;
|
||||
/// must drain. Bumped from 1024 → 4096 on 2026-05-31 to prevent
|
||||
/// eval-phase ring wrap at cluster scale (b=1024 × 5000 eval steps
|
||||
/// produces ~342 dones/account mean, ~1000 peak; 4096 gives 4×
|
||||
/// headroom). Memory cost at b=1024: 1024 × 4096 × 40 B = 167 MB
|
||||
/// device (was 41 MB) — comfortable on L40S 48 GB / H100 80 GB.
|
||||
/// See spec `docs/superpowers/specs/2026-05-31-eval-summary-trade-aggregation-design.md`.
|
||||
pub const TRADE_LOG_CAP: usize = 4096;
|
||||
/// Bytes per TradeRecord (must match crates/ml-backtesting/src/order.rs).
|
||||
pub const TRADE_RECORD_BYTES: usize = 40;
|
||||
/// Bytes per OpenTradeState (kernel-side tracking of currently-open entry).
|
||||
|
||||
@@ -1515,6 +1515,132 @@ impl LobSimCuda {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Read per-backtest cumulative trade counters. Returns a `Vec<u32>`
|
||||
/// of length `n_backtests` where entry `i` is the total number of
|
||||
/// closed-trade records ever pushed to backtest `i`'s ring buffer
|
||||
/// (may exceed `TRADE_LOG_CAP` if the ring wrapped).
|
||||
///
|
||||
/// Used by `alpha_rl_train` to snapshot the eval-phase boundary per
|
||||
/// account: `head_before_per_b[i] = read_per_backtest_trade_counts()[i]`
|
||||
/// at eval start; post-eval `head_after_per_b[i] - head_before_per_b[i]`
|
||||
/// gives the true eval-phase trade count for backtest `i`.
|
||||
///
|
||||
/// Uses mapped-pinned staging per `feedback_no_htod_htoh_only_mapped_pinned`:
|
||||
/// allocate a `MappedRecordBuffer<u32>`, DtoD copy from
|
||||
/// `trade_log_head_d` into its dev_ptr, sync, read via host_ptr.
|
||||
///
|
||||
/// Replaces the (`head_before_eval = read_total_trade_count()`,
|
||||
/// `all_records = read_trade_records(0)`) pattern that conflated
|
||||
/// aggregate counts with per-account ring contents and produced
|
||||
/// meaningless eval_summary metrics — see spec
|
||||
/// `docs/superpowers/specs/2026-05-31-eval-summary-trade-aggregation-design.md`.
|
||||
pub fn read_per_backtest_trade_counts(&self) -> Result<Vec<u32>> {
|
||||
use ml_alpha::pinned_mem::MappedRecordBuffer;
|
||||
use ml_alpha::trainer::raw_launch::raw_memcpy_dtod_async;
|
||||
let raw_stream = self.stream.cu_stream();
|
||||
let staging: MappedRecordBuffer<u32> = unsafe { MappedRecordBuffer::new(self.n_backtests) }
|
||||
.map_err(|e| anyhow::anyhow!("alloc trade_head staging: {e}"))?;
|
||||
unsafe {
|
||||
raw_memcpy_dtod_async(
|
||||
staging.dev_ptr,
|
||||
self.trade_log_head_d.raw_ptr(),
|
||||
self.n_backtests * std::mem::size_of::<u32>(),
|
||||
raw_stream,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("DtoD trade_log_head: {e:?}"))?;
|
||||
}
|
||||
self.stream
|
||||
.synchronize()
|
||||
.context("sync after trade_log_head DtoD")?;
|
||||
let mut out = Vec::with_capacity(self.n_backtests);
|
||||
for i in 0..self.n_backtests {
|
||||
// SAFETY: mapped-pinned host_ptr is valid for [0, len); sync
|
||||
// above ensures the DtoD writes are visible to host.
|
||||
unsafe {
|
||||
out.push(std::ptr::read_volatile(staging.host_ptr.add(i)));
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Read all backtests' trade record rings. Returns
|
||||
/// `Vec<Vec<TradeRecord>>` of length `n_backtests`; entry `i` is the
|
||||
/// most recent up-to-`TRADE_LOG_CAP` records for backtest `i` (oldest
|
||||
/// dropped on wrap).
|
||||
///
|
||||
/// Uses mapped-pinned staging for both the per-backtest head counters
|
||||
/// and the trade-log payload (per `feedback_no_htod_htoh_only_mapped_pinned`).
|
||||
/// At cluster scale (b=1024, cap=4096) the payload staging buffer
|
||||
/// is 167 MB — allocated once per call (end-of-run). Acceptable
|
||||
/// because this is invoked once per fold-eval, not in the hot path.
|
||||
///
|
||||
/// Callers compose this with
|
||||
/// [`read_per_backtest_trade_counts`](Self::read_per_backtest_trade_counts)
|
||||
/// to identify each account's eval-phase-only slice.
|
||||
pub fn read_trade_records_all(
|
||||
&self,
|
||||
) -> Result<Vec<Vec<crate::order::TradeRecord>>> {
|
||||
use ml_alpha::pinned_mem::MappedRecordBuffer;
|
||||
use ml_alpha::trainer::raw_launch::raw_memcpy_dtod_async;
|
||||
let raw_stream = self.stream.cu_stream();
|
||||
let rec_bytes = crate::lob::TRADE_RECORD_BYTES;
|
||||
let cap = crate::lob::TRADE_LOG_CAP;
|
||||
let payload_len_u8 = self.n_backtests * cap * rec_bytes;
|
||||
|
||||
let head_staging: MappedRecordBuffer<u32> =
|
||||
unsafe { MappedRecordBuffer::new(self.n_backtests) }
|
||||
.map_err(|e| anyhow::anyhow!("alloc head staging: {e}"))?;
|
||||
let payload_staging: MappedRecordBuffer<u8> =
|
||||
unsafe { MappedRecordBuffer::new(payload_len_u8) }
|
||||
.map_err(|e| anyhow::anyhow!("alloc payload staging ({payload_len_u8} B): {e}"))?;
|
||||
|
||||
unsafe {
|
||||
raw_memcpy_dtod_async(
|
||||
head_staging.dev_ptr,
|
||||
self.trade_log_head_d.raw_ptr(),
|
||||
self.n_backtests * std::mem::size_of::<u32>(),
|
||||
raw_stream,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("DtoD trade_log_head: {e:?}"))?;
|
||||
raw_memcpy_dtod_async(
|
||||
payload_staging.dev_ptr,
|
||||
self.trade_log_d.raw_ptr(),
|
||||
payload_len_u8,
|
||||
raw_stream,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("DtoD trade_log payload: {e:?}"))?;
|
||||
}
|
||||
self.stream
|
||||
.synchronize()
|
||||
.context("sync after trade_log DtoD")?;
|
||||
|
||||
let mut out = Vec::with_capacity(self.n_backtests);
|
||||
for b in 0..self.n_backtests {
|
||||
// SAFETY: mapped-pinned host_ptr is valid + post-sync the
|
||||
// device-side DtoD writes are visible to the host.
|
||||
let head = unsafe { std::ptr::read_volatile(head_staging.host_ptr.add(b)) } as usize;
|
||||
let n_to_read = head.min(cap);
|
||||
let off = b * cap * rec_bytes;
|
||||
let mut records = Vec::with_capacity(n_to_read);
|
||||
for i in 0..n_to_read {
|
||||
let rec_off = off + i * rec_bytes;
|
||||
// SAFETY: bounds-checked above (i < n_to_read ≤ cap; rec_off + rec_bytes ≤ payload_len_u8).
|
||||
let bytes: [u8; crate::lob::TRADE_RECORD_BYTES] = unsafe {
|
||||
let mut buf = [0u8; crate::lob::TRADE_RECORD_BYTES];
|
||||
for k in 0..rec_bytes {
|
||||
buf[k] = std::ptr::read_volatile(payload_staging.host_ptr.add(rec_off + k));
|
||||
}
|
||||
buf
|
||||
};
|
||||
let r: crate::order::TradeRecord =
|
||||
bytemuck::pod_read_unaligned(&bytes);
|
||||
records.push(r);
|
||||
}
|
||||
out.push(records);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Read back the Pos state for a specific backtest.
|
||||
pub fn read_pos(&self, backtest_idx: usize) -> Result<PosFlat> {
|
||||
anyhow::ensure!(
|
||||
|
||||
Reference in New Issue
Block a user