diff --git a/crates/ml/src/walk_forward.rs b/crates/ml/src/walk_forward.rs index fbbfb7ca8..4aecc9757 100644 --- a/crates/ml/src/walk_forward.rs +++ b/crates/ml/src/walk_forward.rs @@ -72,6 +72,234 @@ pub struct WalkForwardWindow { pub val_end: NaiveDate, /// End date of the test period (exclusive boundary). pub test_end: NaiveDate, + /// Difficulty score for curriculum learning (mean ADX over training bars). + /// Low ADX (<20) = easy trending market, high ADX fluctuation = harder. + /// Computed via [`compute_difficulty`] after window generation. + pub difficulty_score: f64, +} + +/// Curriculum phase for difficulty-based walk-forward window filtering. +/// +/// Controls which windows are included in training based on their ADX-derived +/// difficulty score. Progression: Easy -> Mixed -> Full mirrors the standard +/// curriculum learning approach of starting with simple examples. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DifficultyPhase { + /// Only windows with mean ADX > 30 (clear trending markets, easier to learn). + Easy, + /// All windows included, but trending windows (ADX > 30) are weighted 2x + /// via duplication in the returned list. + Mixed, + /// All windows with equal weight (full training, no filtering). + Full, +} + +impl DifficultyPhase { + /// Create a `DifficultyPhase` from a numeric index. + /// + /// * `0` => `Easy` + /// * `1` => `Mixed` + /// * `2` (or any other value) => `Full` + pub fn from_index(idx: usize) -> Self { + match idx { + 0 => Self::Easy, + 1 => Self::Mixed, + _ => Self::Full, + } + } +} + +/// ADX threshold used to classify a window as "trending" (easy) for curriculum learning. +const ADX_TRENDING_THRESHOLD: f64 = 30.0; + +/// Compute a difficulty score for a slice of OHLCV bars based on mean ADX(14). +/// +/// ADX (Average Directional Index) measures trend strength on a 0-100 scale. +/// High ADX (>30) indicates a clear trend which is easier for RL agents to exploit. +/// Low or fluctuating ADX indicates choppy, range-bound markets that are harder. +/// +/// Returns the mean ADX value over the bar slice. If the slice has fewer than +/// 15 bars (minimum for ADX(14) calculation), returns 0.0. +/// +/// # Algorithm +/// +/// 1. Compute True Range (TR) for each bar +/// 2. Compute +DM / -DM directional movement +/// 3. Smooth TR, +DM, -DM with 14-period Wilder EMA +/// 4. Compute +DI / -DI and DX +/// 5. Smooth DX with 14-period Wilder EMA to get ADX +/// 6. Return mean ADX over valid bars +pub fn compute_difficulty(bars: &[OHLCVBar]) -> f64 { + let period: usize = 14; + // Need at least period + 1 bars for one ADX value + if bars.len() < period.saturating_add(1) { + return 0.0; + } + + // Step 1-2: Compute True Range and Directional Movement + let mut tr_values = Vec::with_capacity(bars.len().saturating_sub(1)); + let mut plus_dm_values = Vec::with_capacity(bars.len().saturating_sub(1)); + let mut minus_dm_values = Vec::with_capacity(bars.len().saturating_sub(1)); + + for i in 1..bars.len() { + let prev = match bars.get(i.wrapping_sub(1)) { + Some(b) => b, + None => continue, + }; + let curr = match bars.get(i) { + Some(b) => b, + None => continue, + }; + + // True Range = max(H-L, |H-prev_close|, |L-prev_close|) + let hl = curr.high - curr.low; + let hpc = (curr.high - prev.close).abs(); + let lpc = (curr.low - prev.close).abs(); + let tr = hl.max(hpc).max(lpc); + tr_values.push(tr); + + // +DM = max(H - prev_H, 0) if > max(prev_L - L, 0), else 0 + let up_move = curr.high - prev.high; + let down_move = prev.low - curr.low; + + let plus_dm = if up_move > down_move && up_move > 0.0 { + up_move + } else { + 0.0 + }; + let minus_dm = if down_move > up_move && down_move > 0.0 { + down_move + } else { + 0.0 + }; + + plus_dm_values.push(plus_dm); + minus_dm_values.push(minus_dm); + } + + if tr_values.len() < period { + return 0.0; + } + + // Step 3: Wilder smoothing (initial = sum of first `period` values, then EMA) + let mut smoothed_tr: f64 = tr_values.iter().take(period).sum(); + let mut smoothed_plus_dm: f64 = plus_dm_values.iter().take(period).sum(); + let mut smoothed_minus_dm: f64 = minus_dm_values.iter().take(period).sum(); + + let period_f64 = period as f64; + let mut dx_values = Vec::with_capacity(tr_values.len().saturating_sub(period)); + + // First DX from initial smoothed values + if smoothed_tr.abs() > 1e-12 { + let plus_di = 100.0 * smoothed_plus_dm / smoothed_tr; + let minus_di = 100.0 * smoothed_minus_dm / smoothed_tr; + let di_sum = plus_di + minus_di; + if di_sum.abs() > 1e-12 { + let dx = 100.0 * (plus_di - minus_di).abs() / di_sum; + dx_values.push(dx); + } + } + + // Subsequent smoothed values using Wilder's method + for i in period..tr_values.len() { + let tr_val = match tr_values.get(i) { + Some(&v) => v, + None => continue, + }; + let pdm_val = match plus_dm_values.get(i) { + Some(&v) => v, + None => continue, + }; + let mdm_val = match minus_dm_values.get(i) { + Some(&v) => v, + None => continue, + }; + + smoothed_tr = smoothed_tr - (smoothed_tr / period_f64) + tr_val; + smoothed_plus_dm = smoothed_plus_dm - (smoothed_plus_dm / period_f64) + pdm_val; + smoothed_minus_dm = smoothed_minus_dm - (smoothed_minus_dm / period_f64) + mdm_val; + + if smoothed_tr.abs() > 1e-12 { + let plus_di = 100.0 * smoothed_plus_dm / smoothed_tr; + let minus_di = 100.0 * smoothed_minus_dm / smoothed_tr; + let di_sum = plus_di + minus_di; + if di_sum.abs() > 1e-12 { + let dx = 100.0 * (plus_di - minus_di).abs() / di_sum; + dx_values.push(dx); + } + } + } + + if dx_values.len() < period { + return 0.0; + } + + // Step 5: Smooth DX with Wilder EMA to get ADX + let mut adx: f64 = dx_values.iter().take(period).sum::() / period_f64; + let mut adx_values = vec![adx]; + + for i in period..dx_values.len() { + let dx_val = match dx_values.get(i) { + Some(&v) => v, + None => continue, + }; + adx = (adx * (period_f64 - 1.0) + dx_val) / period_f64; + adx_values.push(adx); + } + + // Step 6: Return mean ADX + if adx_values.is_empty() { + return 0.0; + } + let sum: f64 = adx_values.iter().sum(); + sum / adx_values.len() as f64 +} + +/// Filter walk-forward windows by curriculum difficulty phase. +/// +/// This function applies difficulty-based filtering to a set of walk-forward +/// windows, enabling curriculum learning by controlling which market conditions +/// the agent trains on. +/// +/// # Phases +/// +/// * [`DifficultyPhase::Easy`] — Only windows with ADX > 30 (trending, easier markets). +/// * [`DifficultyPhase::Mixed`] — All windows, but trending windows are duplicated for 2x weight. +/// * [`DifficultyPhase::Full`] — All windows with equal weight (no filtering). +/// +/// # Arguments +/// +/// * `windows` — Walk-forward windows with pre-computed `difficulty_score`. +/// * `phase` — The curriculum phase to apply. +pub fn filter_by_difficulty( + windows: &[WalkForwardWindow], + phase: DifficultyPhase, +) -> Vec { + match phase { + DifficultyPhase::Easy => { + // Only trending windows (ADX > 30) + windows + .iter() + .filter(|w| w.difficulty_score > ADX_TRENDING_THRESHOLD) + .cloned() + .collect() + } + DifficultyPhase::Mixed => { + // All windows, trending ones duplicated for 2x weight + let mut result = Vec::with_capacity(windows.len().saturating_mul(2)); + for w in windows { + result.push(w.clone()); + if w.difficulty_score > ADX_TRENDING_THRESHOLD { + result.push(w.clone()); + } + } + result + } + DifficultyPhase::Full => { + // All windows, equal weight + windows.to_vec() + } + } } /// Minimum number of bars required in each split to form a valid window. @@ -188,6 +416,7 @@ pub fn generate_walk_forward_windows( continue; } + let difficulty_score = compute_difficulty(&train_bars); windows.push(WalkForwardWindow { fold, train: train_bars, @@ -196,6 +425,7 @@ pub fn generate_walk_forward_windows( train_end, val_end, test_end, + difficulty_score, }); fold = fold.saturating_add(1);