gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(test): use single symbol (ES.FUT) in smoke tests for CI speed

Browse Source 
Loading all 4 symbols × 9 quarters (5.5GB) just for a smoke test
is wasteful. One symbol's data (~600K bars) is sufficient to validate
the full pipeline: data loading → feature extraction → GPU upload →
training → checkpoint.

Smoke tests should take seconds, not minutes of I/O.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-19 20:42:42 +01:00
parent 10b86b4319
commit 8139911c3d
1 changed files with 15 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
crates/ml/tests/smoke_test_real_data.rs
View File

@@ -138,7 +138,8 @@ fn try_data(relative: &str) -> Option<PathBuf> {
#[test]
fn smoke_ohlcv_parse_and_extract_features() {
let Some(ohlcv_dir) = try_data("ohlcv") else { return };
// Single symbol for CI speed
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return };
// Load OHLCV bars from .dbn.zst
let bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(&ohlcv_dir)
@@ -515,7 +516,8 @@ mod gpu_smoke {
#[test]
fn smoke_gpu_real_ohlcv_forward() -> Result<(), anyhow::Error> {
let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) };
// Use single symbol for CI speed — full dataset is tested via nightly
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) };
let Some((_device, stream)) = try_cuda() else { return Ok(()) };
let (market_buf, target_buf, num_bars) =
@@ -596,7 +598,7 @@ mod gpu_smoke {
#[test]
fn smoke_gpu_real_ohlcv_plus_ofi_forward() -> Result<(), anyhow::Error> {
if !has_enough_vram(2048) { return Ok(()) }
let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) };
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) };
let Some(mbp10_dir) = try_data("mbp10/ES.FUT") else { return Ok(()) };
let Some((_device, stream)) = try_cuda() else { return Ok(()) };
@@ -668,7 +670,7 @@ mod gpu_smoke {
fn smoke_gpu_real_data_noisy_distributional() -> Result<(), anyhow::Error> {
use ml::dqn::branching::{BranchingConfig, BranchingDuelingQNetwork};
let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) };
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) };
let Some((_device, stream)) = try_cuda() else { return Ok(()) };
let (market_buf, target_buf, num_bars) =
@@ -779,7 +781,15 @@ async fn smoke_e2e_dqn_training_loop() {
return;
}
let Some(ohlcv_dir) = try_data("ohlcv") else { return };
let data_dir = ohlcv_dir.to_string_lossy().to_string();
// Use single symbol subdir for CI speed — loading all 4 symbols × 9 quarters
// takes minutes just for I/O. One symbol (ES.FUT, ~600K bars) is enough to
// validate the full pipeline: data loading → GPU upload → training → checkpoint.
let es_dir = ohlcv_dir.join("ES.FUT");
let data_dir = if es_dir.exists() {
es_dir.to_string_lossy().to_string()
} else {
ohlcv_dir.to_string_lossy().to_string()
};
// Configure for a fast smoke run: few epochs, small batch, low warmup
let mut hyperparams = DQNHyperparameters::conservative();