foxhunt/ml/tests/dqn_penalty_effectiveness_test.rs

//! DQN Penalty Effectiveness Tests
//!
//! Tests verifying the dynamic HOLD penalty is STRONG ENOUGH to prevent extreme bias,
//! not just mathematically correct. Addresses the problem where Wave 3 tests verified
//! penalty CALCULATION was correct but didn't verify penalty EFFECTIVENESS, resulting
//! in 99.3% HOLD bias in production hyperopt despite "passing" tests.
//!
//! **Context**: The HOLD reward mechanism uses a binary system based on price movement:
//! - Low volatility (< movement_threshold): +0.002 reward (encourages holding in flat markets)
//! - High volatility (≥ movement_threshold): -0.001 penalty (discourages holding in trending markets)
//!
//! **Problem**: The -0.001 penalty is too small compared to typical reward magnitudes (±1.0),
//! making it ineffective at preventing 99.3% HOLD bias. This test suite verifies penalties
//! are actually effective at influencing learning, not just mathematically correct.

use ml::dqn::agent::{TradingAction, TradingState};
use ml::dqn::reward::{RewardConfig, RewardFunction};
use rust_decimal::Decimal;

/// Helper to create a test state with specific close price
fn create_state_with_price(close_price: f32) -> TradingState {
    TradingState {
        price_features: vec![close_price, close_price, close_price, close_price], // OHLC
        technical_indicators: vec![0.5, 0.5, 0.5, 0.5],
        market_features: vec![0.001, 100.0, 0.0, 0.0], // spread, volume, etc.
        portfolio_features: vec![1.0, 0.0, 0.0, 0.0], // normalized portfolio value, position, etc.
    }
}

/// Test 1: Current HOLD penalty magnitude is too small
///
/// **Problem**: The -0.001 penalty for high volatility is 1000× smaller than
/// typical BUY/SELL rewards (±1.0), making it effectively invisible to the optimizer.
///
/// **Critical Finding**: This test DOCUMENTS THE BUG - it should FAIL with current
/// implementation, proving the penalty is inadequate.
#[test]
#[should_panic(expected = "Penalty is too small")]
fn test_current_penalty_is_too_small_to_be_effective() {
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // High volatility scenario (5% move, above 2% threshold)
    let current_state = create_state_with_price(100.0);
    let next_state = create_state_with_price(105.0);

    let hold_penalty = reward_fn
        .calculate_reward(TradingAction::Hold, &current_state, &next_state, &recent_actions)
        .unwrap();

    // Current implementation gives -0.001
    let expected_penalty = Decimal::try_from(-0.001).unwrap();
    assert_eq!(
        hold_penalty, expected_penalty,
        "Expected penalty {}, got {}",
        expected_penalty, hold_penalty
    );

    // Typical BUY/SELL reward magnitude (based on calculate_reward in dqn.rs: price_change / 10.0)
    // For 5% move ≈ 0.5 reward
    let typical_reward = Decimal::try_from(0.5).unwrap();

    // Calculate penalty-to-reward ratio
    let ratio = hold_penalty.abs() / typical_reward;
    // Current: 0.001 / 0.5 = 0.002 = 0.2%

    // CRITICAL: Penalty should be at least 10% of typical reward to be effective
    let min_effective_ratio = Decimal::try_from(0.1).unwrap(); // 10%

    assert!(
        ratio >= min_effective_ratio,
        "Penalty is too small! Ratio: {:.1}% (need ≥10%). \
         Current penalty ({}) is {} smaller than typical reward ({}). \
         This explains 99.3% HOLD bias in hyperopt.",
        ratio * Decimal::from(100),
        hold_penalty,
        typical_reward / hold_penalty.abs(),
        typical_reward
    );
}

/// Test 2: Penalty should be significant compared to reward magnitude
///
/// **Success Criteria**: For a penalty to effectively prevent 99% HOLD bias,
/// it must be at least 10-30% of typical reward magnitude. Current -0.001
/// is only 0.2% (50× too small).
#[test]
fn test_penalty_should_be_significant_fraction_of_reward() {
    let recent_actions = vec![]; // No action history for unit test
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Test scenario: 5% price increase (above 2% threshold)
    let current_state = create_state_with_price(100.0);
    let next_state = create_state_with_price(105.0);

    let hold_penalty = reward_fn
        .calculate_reward(TradingAction::Hold, &current_state, &next_state, &recent_actions)
        .unwrap();

    // Expected current value: -0.001
    let expected_current = Decimal::try_from(-0.001).unwrap();
    assert_eq!(hold_penalty, expected_current, "Verify current implementation");

    // Typical reward magnitude for price-based rewards
    // From dqn.rs: (price_change / 10.0).clamp(-1.0, 1.0)
    // 5 point move / 10.0 = 0.5 reward
    let typical_reward = Decimal::try_from(0.5).unwrap();

    // Current ratio
    let current_ratio = hold_penalty.abs() / typical_reward;
    // 0.001 / 0.5 = 0.002 = 0.2%

    println!(
        "Current penalty-to-reward ratio: {:.2}%",
        current_ratio * Decimal::from(100)
    );
    println!(
        "Penalty magnitude: {} vs typical reward: {}",
        hold_penalty.abs(),
        typical_reward
    );
    println!(
        "Penalty is {:.0}× too small to be effective",
        (Decimal::try_from(0.1).unwrap() / current_ratio)
    );

    // Document the required fix
    let required_penalty = typical_reward * Decimal::try_from(0.2).unwrap(); // 20% of reward
    println!(
        "Required penalty for effectiveness: ~{} (current: {})",
        required_penalty, hold_penalty
    );
    println!(
        "Needs to be {:.0}× stronger",
        required_penalty / hold_penalty.abs()
    );

    // This test documents the problem but doesn't enforce a fix (yet)
    // When the penalty is strengthened, update this test to enforce the minimum
}

/// Test 3: Binary reward system creates weak signal
///
/// **Problem**: The binary +0.002/-0.001 system has only 0.003 total range,
/// which is 333× smaller than the ±1.0 range of prlet recent_actions = vec![]; // No action history for unit test
ice-based rewards.
#[test]
fn test_binary_reward_range_too_small() {
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Low volatility: +0.002 reward
    let state_low_vol = create_state_with_price(100.0);
    let next_low_vol = create_state_with_price(100.5); // 0.5% (below 2% threshold)

    let reward_low_vol = reward_fn
        .calculate_reward(TradingAction::Hold, &state_low_vol, &next_low_vol, &recent_actions)
        .unwrap();

    // High volatility: -0.001 penalty
    let mut reward_fn2 = RewardFunction::new(RewardConfig::default());
    let state_high_vol = create_state_with_price(100.0);
    let next_high_vol = create_state_with_price(105.0); // 5% (above 2% threshold)

    let reward_high_vol = reward_fn2
        .calculate_reward(TradingAction::Hold, &state_high_vol, &next_high_vol, &recent_actions)
        .unwrap();

    // Expected current values
    let expected_low = Decimal::try_from(0.002).unwrap();
    let expected_high = Decimal::try_from(-0.001).unwrap();

    assert_eq!(reward_low_vol, expected_low, "Verify low volatility reward");
    assert_eq!(reward_high_vol, expected_high, "Verify high volatility penalty");

    // Total range of HOLD reward system
    let hold_range = (reward_low_vol - reward_high_vol).abs();
    // 0.002 - (-0.001) = 0.003

    println!("HOLD reward range: {}", hold_range);

    // Typical BUY/SELL reward range: [-1.0, 1.0] = 2.0
    let typical_range = Decimal::try_from(2.0).unwrap();

    println!("Typical reward range: {}", typical_range);

    let range_ratio = hold_range / typical_range;
    // 0.003 / 2.0 = 0.0015 = 0.15%

    println!(
        "HOLD range is {:.2}% of typical range",
        range_ratio * Decimal::from(100)
    );
    println!(
        "HOLD signal is {:.0}× weaker than price signal",
        typical_range / hold_range
    );

    // Document that HOLD signal is drowned out by price signal
    assert!(
        hold_range < typical_range,
        "HOLD signal should be much weaker than price signal (current implementation)"
    );
}

/// Test 4: Penalty ineffective at extreme bias (99% HOLD)
///
/// **Scenario**: With 99% HOLD bias, the agent is essentially always choosing HOLD.
/// The -0let recent_actions = vec![]; // No action history for unit test
.001 penalty for high volatility is not strong enough to break this pattern.
#[test]
fn test_penalty_ineffective_at_extreme_bias() {
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Simulate extreme trending market (10% move)
    let current_state = create_state_with_price(100.0);
    let next_state = create_state_with_price(110.0);

    // HOLD penalty in this scenario
    let hold_penalty = reward_fn
        .calculate_reward(TradingAction::Hold, &current_state, &next_state, &recent_actions)
        .unwrap();

    // Expected: -0.001 (binary penalty for high volatility)
    let expected = Decimal::try_from(-0.001).unwrap();
    assert_eq!(hold_penalty, expected);

    // But a correct BUY action would get:
    // (10.0 / 10.0).clamp(-1.0, 1.0) = 1.0 reward
    let correct_action_reward = Decimal::try_from(1.0).unwrap();

    // Opportunity cost of HOLD
    let opportunity_cost = correct_action_reward - hold_penalty;
    // 1.0 - (-0.001) = 1.001

    println!("HOLD penalty: {}", hold_penalty);
    println!("Correct action reward: {}", correct_action_reward);
    println!("Opportunity cost: {}", opportunity_cost);

    // The penalty (-0.001) is only 0.1% of the opportunity cost (1.001)
    let penalty_fraction = hold_penalty.abs() / opportunity_cost;
    println!(
        "Penalty is only {:.2}% of opportunity cost",
        penalty_fraction * Decimal::from(100)
    );

    // For penalty to be effective, it should be at least 10% of opportunity cost
    let min_effective_fraction = Decimal::try_from(0.1).unwrap();
    println!(
        "Penalty needs to be {:.0}× stronger",
        min_effective_fraction / penalty_fraction
    );
}

/// Test 5: Recommended penalty strength
///
/// **Proposal**: To prevent 99% HOLD bias, the penalty should be:
/// - At least 0.1 (10% of max reward) for let recent_actions = vec![]; // No action history for unit test
extreme movements
/// - Scale with movement magnitude for gradual correction
///
/// This test documents what an effective penalty would look like.
#[test]
fn test_recommended_penalty_strength() {
    // Current implementation
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Extreme volatility scenario (10% move)
    let current_state = create_state_with_price(100.0);
    let next_state = create_state_with_price(110.0);

    let current_penalty = reward_fn
        .calculate_reward(TradingAction::Hold, &current_state, &next_state, &recent_actions)
        .unwrap();

    // Current: -0.001
    assert_eq!(
        current_penalty,
        Decimal::try_from(-0.001).unwrap(),
        "Verify current implementation"
    );

    // Recommended penalty calculation (not implemented yet):
    // For 10% move (8% over 2% threshold):
    // penalty = -0.001 (base) - 0.01 * excess
    // penalty = -0.001 - 0.01 * 0.08 = -0.001 - 0.0008 = -0.0018
    //
    // OR with stronger multiplier (10×):
    // penalty = -0.001 - 0.1 * 0.08 = -0.001 - 0.008 = -0.009
    //
    // OR with even stronger multiplier (100×):
    // penalty = -0.001 - 1.0 * 0.08 = -0.001 - 0.08 = -0.081

    let recommended_penalty_weak = Decimal::try_from(-0.0018).unwrap();
    let recommended_penalty_medium = Decimal::try_from(-0.009).unwrap();
    let recommended_penalty_strong = Decimal::try_from(-0.081).unwrap();

    println!("Current penalty: {}", current_penalty);
    println!("Recommended (weak, 1× multiplier): {}", recommended_penalty_weak);
    println!(
        "Recommended (medium, 10× multiplier): {}",
        recommended_penalty_medium
    );
    println!(
        "Recommended (strong, 100× multiplier): {}",
        recommended_penalty_strong
    );

    // Calculate improvement factors
    let improvement_weak = recommended_penalty_weak.abs() / current_penalty.abs();
    let improvement_medium = recommended_penalty_medium.abs() / current_penalty.abs();
    let improvement_strong = recommended_penalty_strong.abs() / current_penalty.abs();

    println!("\nImprovement factors:");
    println!("Weak: {:.1}×", improvement_weak);
    println!("Medium: {:.1}×", improvement_medium);
    println!("Strong: {:.1}×", improvement_strong);

    // For 10% penalty-to-reward ratio (minimum effective):
    // Max reward = 1.0, so penalty should be ~0.1
    // For 10% move, penalty should be ~0.1
    // Current is 0.001 → needs 100× multiplier

    println!("\nTo achieve 10% penalty-to-reward ratio:");
    println!("Need penalty ~0.1 for 10% move");
    println!("Current: {}", current_penalty.abs());
    println!(
        "Multiplier needed: ~{:.0}×",
        Decimal::try_from(0.1).unwrap() / current_penalty.abs()
    );
}