gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

revert: remove data subset hacks — need proper approach

Browse Source 
Reverts 4 commits (8139911c, 282f3aff, b3aca96d, dad61e4e) that
tried to workaround slow CI by limiting data subsets and cleaning
caches. The real issue is debug-mode .dbn.zst parsing taking minutes.

Kept: --test-threads=1 fix (root cause), hard CUDA error, action
range fixes, #[ignore] for heavy tests.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-19 21:55:46 +01:00
parent dad61e4e7b
commit 9f5b88c81d
4 changed files with 10 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crates/ml/src/trainers/dqn/config.rs
View File

@@ -1119,10 +1119,6 @@ pub struct DQNHyperparameters {
/// Production: leave at 0 for full-dataset training.
pub max_training_steps_per_epoch: usize,
/// Maximum number of bars to load from training data.
/// 0 = unlimited (load all). CI/smoke: set to 2000-5000 for fast I/O.
pub max_bars: usize,
/// Hidden dimension base for GPU-dynamic network sizing.
/// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4].
pub hidden_dim_base: Option<usize>,
@@ -1384,7 +1380,6 @@ impl DQNHyperparameters {
gpu_timesteps_per_episode: 500, // Default: 500 timesteps per episode
avg_spread: 0.0001, // Default: 1bp (ES/NQ futures)
max_training_steps_per_epoch: 0, // Default: unlimited (full dataset training)
max_bars: 0, // Default: unlimited (load all bars)
// GPU-dynamic network sizing — None means auto-detect from hardware.
// H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds.

7
crates/ml/src/trainers/dqn/data_loading.rs
View File

@@ -634,13 +634,6 @@ impl DQNTrainer {
);
all_ohlcv_bars.extend(file_bars);
// CI/smoke: cap total bars for fast I/O
if self.hyperparams.max_bars > 0 && all_ohlcv_bars.len() >= self.hyperparams.max_bars {
all_ohlcv_bars.truncate(self.hyperparams.max_bars);
info!("max_bars={}: truncated to {} bars", self.hyperparams.max_bars, all_ohlcv_bars.len());
break;
}
}
if all_ohlcv_bars.is_empty() {

37
crates/ml/tests/smoke_test_real_data.rs
View File

@@ -138,13 +138,11 @@ fn try_data(relative: &str) -> Option<PathBuf> {
#[test]
fn smoke_ohlcv_parse_and_extract_features() {
// Single symbol for CI speed
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return };
let Some(ohlcv_dir) = try_data("ohlcv") else { return };
// Load OHLCV bars from .dbn.zst (capped for CI speed)
let mut bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(&ohlcv_dir)
// Load OHLCV bars from .dbn.zst
let bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(&ohlcv_dir)
.expect("Failed to load OHLCV bars from DBN");
if bars.len() > 2000 { bars.truncate(2000); }
info!(count = bars.len(), dir = %ohlcv_dir.display(), "Loaded OHLCV bars");
assert!(bars.len() > 100, "Expected >100 bars, got {}", bars.len());
@@ -430,14 +428,10 @@ mod gpu_smoke {
mbp10_dir: Option<&Path>,
stream: &Arc<CudaStream>,
) -> Result<(CudaSlice<f32>, CudaSlice<f32>, usize), anyhow::Error> {
// 1. Load OHLCV bars (capped for CI speed)
const MAX_BARS_CI: usize = 2000;
let mut bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(ohlcv_dir)
// 1. Load OHLCV bars
let bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(ohlcv_dir)
.expect("Failed to load OHLCV bars");
if bars.len() > MAX_BARS_CI {
bars.truncate(MAX_BARS_CI);
}
info!(count = bars.len(), "Loaded OHLCV bars (capped to {})", MAX_BARS_CI);
info!(count = bars.len(), "Loaded OHLCV bars");
// 2. Extract 42-dim features
let features = ml::features::extract_ml_features(&bars)
@@ -521,8 +515,7 @@ mod gpu_smoke {
#[test]
fn smoke_gpu_real_ohlcv_forward() -> Result<(), anyhow::Error> {
// Use single symbol for CI speed — full dataset is tested via nightly
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) };
let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) };
let Some((_device, stream)) = try_cuda() else { return Ok(()) };
let (market_buf, target_buf, num_bars) =
@@ -603,7 +596,7 @@ mod gpu_smoke {
#[test]
fn smoke_gpu_real_ohlcv_plus_ofi_forward() -> Result<(), anyhow::Error> {
if !has_enough_vram(2048) { return Ok(()) }
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) };
let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) };
let Some(mbp10_dir) = try_data("mbp10/ES.FUT") else { return Ok(()) };
let Some((_device, stream)) = try_cuda() else { return Ok(()) };
@@ -675,7 +668,7 @@ mod gpu_smoke {
fn smoke_gpu_real_data_noisy_distributional() -> Result<(), anyhow::Error> {
use ml::dqn::branching::{BranchingConfig, BranchingDuelingQNetwork};
let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) };
let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) };
let Some((_device, stream)) = try_cuda() else { return Ok(()) };
let (market_buf, target_buf, num_bars) =
@@ -786,15 +779,7 @@ async fn smoke_e2e_dqn_training_loop() {
return;
}
let Some(ohlcv_dir) = try_data("ohlcv") else { return };
// Use single symbol subdir for CI speed — loading all 4 symbols × 9 quarters
// takes minutes just for I/O. One symbol (ES.FUT, ~600K bars) is enough to
// validate the full pipeline: data loading → GPU upload → training → checkpoint.
let es_dir = ohlcv_dir.join("ES.FUT");
let data_dir = if es_dir.exists() {
es_dir.to_string_lossy().to_string()
} else {
ohlcv_dir.to_string_lossy().to_string()
};
let data_dir = ohlcv_dir.to_string_lossy().to_string();
// Configure for a fast smoke run: few epochs, small batch, low warmup
let mut hyperparams = DQNHyperparameters::conservative();
@@ -817,8 +802,6 @@ async fn smoke_e2e_dqn_training_loop() {
// 64 steps x batch_size 32 = 2048 gradient updates — sufficient to validate
// finite loss, gradient flow, and action diversity without 400s/epoch overhead.
hyperparams.max_training_steps_per_epoch = 64;
// CI: load only 2000 bars (not 600K) — validates pipeline, not data coverage.
hyperparams.max_bars = 2000;
let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir");
let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer");

3
infra/k8s/argo/gpu-test-pipeline-template.yaml
View File

@@ -208,9 +208,6 @@ spec:
export PATH="${CARGO_HOME}/bin:${PATH}"
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
# --- Clean stale test binaries (PVC incremental cache) ---
cargo clean -p ml -p ml-core -p ml-dqn -p ml-ppo 2>/dev/null || true
# --- PTX cache invalidation ---
# Purge stale cached PTX if any CUDA kernel source changed since last run.
bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache"