foxhunt/ml/tests/bug16_reward_integration_test.rs

//! Bug #16 Reward Integration Tests
//!
//! Comprehensive test suite to verify Bug #16 fix enables non-zero P&L rewards
//!
//! # Bug #16 Context
//! Before fix: portfolio_features were frozen at [1.0, 0.0, 0.0001], causing rewards
//! to always be ~0.0 because P&L calculations had no actual portfolio state changes.
//!
//! After fix: PortfolioTracker populates portfolio_features with actual values:
//! - portfolio_features[0]: portfolio_value (normalized, changes with P&L)
//! - portfolio_features[1]: position (±1.0 for active trades, 0.0 for flat)
//! - portfolio_features[2]: spread (actual bid-ask spread)
//!
//! # Test Strategy (Test-Driven Development)
//! Each test should:
//! - FAIL before fix (rewards ≈ 0.0 due to frozen portfolio_features)
//! - PASS after fix (rewards reflect actual P&L from portfolio state changes)
//!
//! # Test Coverage
//! 1. test_reward_uses_actual_portfolio_value: Profitable long trade (price up 1%)
//! 2. test_reward_uses_actual_position: Profitable short trade (price down 1%)
//! 3. test_reward_zero_when_flat: No exposure = no P&L (price moves 5%)
//! 4. test_reward_negative_on_loss: Losing long trade (price down 2%)

#![allow(unused_crate_dependencies)]

use anyhow::Result;

use ml::dqn::action_space::{ExposureLevel, FactoredAction, OrderType, Urgency};
use ml::dqn::reward::{RewardConfig, RewardFunction};
use ml::dqn::TradingState;

// ============================================================================
// Helper Functions - Create Test States
// ============================================================================

/// Create a TradingState with specific portfolio features
///
/// # Arguments
/// * `price` - Current price (used in price_features)
/// * `portfolio_value` - Portfolio value (normalized, 1.0 = initial capital)
/// * `position` - Position size (signed: +Long, -Short, 0=Flat)
/// * `spread` - Bid-ask spread
///
/// # Structure
/// 128-dim state: 4 price + 121 technical + 3 portfolio
fn create_state(price: f32, portfolio_value: f32, position: f32, spread: f32) -> TradingState {
    TradingState {
        price_features: vec![price, price, price, price], // OHLC (all same for simplicity)
        technical_indicators: vec![0.5; 121],             // 121 technical indicators
        market_features: vec![],                          // Included in technical indicators
        portfolio_features: vec![portfolio_value, position, spread],
    }
}

/// Create a FLAT state (no position, price movement doesn't affect portfolio)
fn create_flat_state(price: f32, portfolio_value: f32) -> TradingState {
    create_state(price, portfolio_value, 0.0, 0.0001)
}

/// Create a LONG state (position = +0.5 to avoid risk penalty at threshold 0.8)
fn create_long_state(price: f32, portfolio_value: f32) -> TradingState {
    create_state(price, portfolio_value, 0.5, 0.0001)
}

/// Create a SHORT state (position = -0.5 to avoid risk penalty at threshold 0.8)
fn create_short_state(price: f32, portfolio_value: f32) -> TradingState {
    create_state(price, portfolio_value, -0.5, 0.0001)
}

/// Create a BUY FactoredAction (Long100 + Market + Normal)
fn create_buy_action() -> FactoredAction {
    FactoredAction::new(ExposureLevel::Long100, OrderType::Market, Urgency::Normal)
}

/// Create a SELL FactoredAction (Short100 + Market + Normal)
fn create_sell_action() -> FactoredAction {
    FactoredAction::new(ExposureLevel::Short100, OrderType::Market, Urgency::Normal)
}

/// Create a balanced action history to avoid diversity penalty
/// Returns 100 actions with balanced distribution (33% BUY, 33% SELL, 34% HOLD)
fn create_balanced_action_history() -> Vec<FactoredAction> {
    let mut actions = Vec::with_capacity(100);

    // Add 33 BUY actions
    for _ in 0..33 {
        actions.push(create_buy_action());
    }

    // Add 33 SELL actions
    for _ in 0..33 {
        actions.push(create_sell_action());
    }

    // Add 34 HOLD actions
    for _ in 0..34 {
        actions.push(FactoredAction::new(ExposureLevel::Flat, OrderType::Market, Urgency::Normal));
    }

    actions
}

// ============================================================================
// Test 1: Profitable Long Trade (BUY action, price increases 1%)
// ============================================================================

#[test]
fn test_reward_uses_actual_portfolio_value() -> Result<()> {
    // Setup: RewardFunction with default config
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Scenario: Execute BUY action (position +1.0), price increases 1%
    let buy_action = create_buy_action();

    // Current state: Portfolio value = 1.0 (initial capital), Position = +1.0 (long)
    // Price = 100.0
    let current_state = create_long_state(100.0, 1.0);

    // Next state: Price increases 1% to 101.0
    // Portfolio value increases by 1% (1.0 → 1.01) due to long position
    let next_state = create_long_state(101.0, 1.01);

    // Calculate reward (use balanced action history to avoid diversity penalty)
    let recent_actions = create_balanced_action_history();
    let reward = reward_fn.calculate_reward(
        buy_action,
        &current_state,
        &next_state,
        &recent_actions,
    )?;

    // Convert Decimal reward to f64 for comparison
    let reward_f64: f64 = reward.try_into().unwrap_or(0.0);

    // ASSERTION: Reward should be positive (profitable trade)
    // Before Bug #16 fix: reward ≈ 0.0 (portfolio_value frozen at 1.0 → 1.0)
    // After Bug #16 fix: reward > 0.0 (portfolio_value changes 1.0 → 1.01)
    assert!(
        reward_f64 > 0.0,
        "Expected positive reward for profitable long trade (price +1%), got {:.6}",
        reward_f64
    );

    // Additional check: Reward should be approximately 0.01 (1% gain, clamped to [-1, 1])
    // Allow for small numerical differences and reward component adjustments
    assert!(
        reward_f64 >= 0.005,
        "Expected reward >= 0.005 for 1% gain, got {:.6}",
        reward_f64
    );

    println!(
        "✅ Test 1 PASSED: Profitable long trade reward = {:.6} (expected > 0.0)",
        reward_f64
    );

    Ok(())
}

// ============================================================================
// Test 2: Profitable Short Trade (SELL action, price decreases 1%)
// ============================================================================

#[test]
fn test_reward_uses_actual_position() -> Result<()> {
    // Setup: RewardFunction with default config
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Scenario: Execute SELL action (position -1.0), price decreases 1%
    let sell_action = create_sell_action();

    // Current state: Portfolio value = 1.0, Position = -1.0 (short)
    // Price = 100.0
    let current_state = create_short_state(100.0, 1.0);

    // Next state: Price decreases 1% to 99.0
    // Portfolio value increases by 1% (1.0 → 1.01) due to profitable short
    let next_state = create_short_state(99.0, 1.01);

    // Calculate reward (use balanced action history to avoid diversity penalty)
    let recent_actions = create_balanced_action_history();
    let reward = reward_fn.calculate_reward(
        sell_action,
        &current_state,
        &next_state,
        &recent_actions,
    )?;

    let reward_f64: f64 = reward.try_into().unwrap_or(0.0);

    // ASSERTION: Reward should be positive (profitable short trade)
    // Before Bug #16 fix: reward ≈ 0.0 (portfolio_value frozen)
    // After Bug #16 fix: reward > 0.0 (short profits from price decline)
    assert!(
        reward_f64 > 0.0,
        "Expected positive reward for profitable short trade (price -1%), got {:.6}",
        reward_f64
    );

    assert!(
        reward_f64 >= 0.005,
        "Expected reward >= 0.005 for 1% short gain, got {:.6}",
        reward_f64
    );

    println!(
        "✅ Test 2 PASSED: Profitable short trade reward = {:.6} (expected > 0.0)",
        reward_f64
    );

    Ok(())
}

// ============================================================================
// Test 3: Flat Position (No exposure, price moves 5%)
// ============================================================================

#[test]
fn test_reward_zero_when_flat() -> Result<()> {
    // Setup: RewardFunction with default config
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Scenario: Position = 0.0 (flat), price moves 5% (either direction)
    // Since no position, P&L should be ~0.0
    let hold_action = FactoredAction::new(ExposureLevel::Flat, OrderType::Market, Urgency::Normal);

    // Current state: Portfolio value = 1.0, Position = 0.0 (flat)
    // Price = 100.0
    let current_state = create_flat_state(100.0, 1.0);

    // Next state: Price moves 5% to 105.0
    // Portfolio value unchanged (1.0 → 1.0) since no position
    let next_state = create_flat_state(105.0, 1.0);

    // Calculate reward (use balanced action history to avoid diversity penalty)
    let recent_actions = create_balanced_action_history();
    let reward = reward_fn.calculate_reward(
        hold_action,
        &current_state,
        &next_state,
        &recent_actions,
    )?;

    let reward_f64: f64 = reward.try_into().unwrap_or(0.0);

    // ASSERTION: Reward should be approximately 0.0 (no exposure = no P&L)
    // Before Bug #16 fix: reward ≈ 0.0 (coincidentally correct for this case)
    // After Bug #16 fix: reward ≈ 0.0 (portfolio_value unchanged: 1.0 → 1.0)
    //
    // Note: Reward may be slightly negative due to hold_penalty_weight (0.01 default)
    // if price movement exceeds movement_threshold (0.01 = 1%)
    // Allow for small hold penalty but confirm no P&L component
    assert!(
        reward_f64.abs() <= 0.02,
        "Expected reward ≈ 0.0 for flat position (no exposure), got {:.6}. May include small hold penalty.",
        reward_f64
    );

    println!(
        "✅ Test 3 PASSED: Flat position reward = {:.6} (expected ≈ 0.0, ±hold penalty)",
        reward_f64
    );

    Ok(())
}

// ============================================================================
// Test 4: Losing Long Trade (BUY action, price decreases 2%)
// ============================================================================

#[test]
fn test_reward_negative_on_loss() -> Result<()> {
    // Setup: RewardFunction with default config
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Scenario: Execute BUY action (position +1.0), price decreases 2%
    let buy_action = create_buy_action();

    // Current state: Portfolio value = 1.0, Position = +1.0 (long)
    // Price = 100.0
    let current_state = create_long_state(100.0, 1.0);

    // Next state: Price decreases 2% to 98.0
    // Portfolio value decreases by 2% (1.0 → 0.98) due to losing long position
    let next_state = create_long_state(98.0, 0.98);

    // Calculate reward (use balanced action history to avoid diversity penalty)
    let recent_actions = create_balanced_action_history();
    let reward = reward_fn.calculate_reward(
        buy_action,
        &current_state,
        &next_state,
        &recent_actions,
    )?;

    let reward_f64: f64 = reward.try_into().unwrap_or(0.0);

    // ASSERTION: Reward should be negative (losing trade)
    // Before Bug #16 fix: reward ≈ 0.0 (portfolio_value frozen)
    // After Bug #16 fix: reward < 0.0 (portfolio_value decreases 1.0 → 0.98)
    assert!(
        reward_f64 < 0.0,
        "Expected negative reward for losing long trade (price -2%), got {:.6}",
        reward_f64
    );

    assert!(
        reward_f64 <= -0.01,
        "Expected reward <= -0.01 for 2% loss, got {:.6}",
        reward_f64
    );

    println!(
        "✅ Test 4 PASSED: Losing long trade reward = {:.6} (expected < 0.0)",
        reward_f64
    );

    Ok(())
}

// ============================================================================
// Test 5: Edge Case - Large Price Movement (10% gain)
// ============================================================================

#[test]
fn test_reward_clamped_on_large_gain() -> Result<()> {
    // Setup: RewardFunction with default config
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Scenario: Large price movement (10% gain), reward should be clamped to [-1, 1]
    let buy_action = create_buy_action();

    // Current state: Portfolio value = 1.0, Position = +1.0 (long)
    let current_state = create_long_state(100.0, 1.0);

    // Next state: Price increases 10% to 110.0
    // Portfolio value increases by 10% (1.0 → 1.10)
    let next_state = create_long_state(110.0, 1.10);

    // Calculate reward (use balanced action history to avoid diversity penalty)
    let recent_actions = create_balanced_action_history();
    let reward = reward_fn.calculate_reward(
        buy_action,
        &current_state,
        &next_state,
        &recent_actions,
    )?;

    let reward_f64: f64 = reward.try_into().unwrap_or(0.0);

    // ASSERTION: Reward should be positive but clamped to max 1.0
    assert!(
        reward_f64 > 0.0,
        "Expected positive reward for large gain, got {:.6}",
        reward_f64
    );

    assert!(
        reward_f64 <= 1.0,
        "Expected reward clamped to 1.0 max, got {:.6}",
        reward_f64
    );

    println!(
        "✅ Test 5 PASSED: Large gain reward = {:.6} (clamped to max 1.0)",
        reward_f64
    );

    Ok(())
}

// ============================================================================
// Test 6: Edge Case - Portfolio Value Validation
// ============================================================================

#[test]
fn test_reward_handles_missing_portfolio_features() -> Result<()> {
    // Setup: RewardFunction with default config
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Scenario: Create state with insufficient portfolio_features (edge case)
    // This simulates the bug #16 condition before fix
    let buy_action = create_buy_action();

    // Current state: Empty portfolio_features (simulates old buggy behavior)
    let current_state = TradingState {
        price_features: vec![100.0, 100.0, 100.0, 100.0],
        technical_indicators: vec![0.5; 121],
        market_features: vec![],
        portfolio_features: vec![], // EMPTY - simulates bug #16
    };

    // Next state: Also empty portfolio_features
    let next_state = TradingState {
        price_features: vec![101.0, 101.0, 101.0, 101.0],
        technical_indicators: vec![0.5; 121],
        market_features: vec![],
        portfolio_features: vec![], // EMPTY
    };

    // Calculate reward - should handle gracefully (log warning, use defaults)
    let recent_actions = vec![buy_action];
    let result = reward_fn.calculate_reward(
        buy_action,
        &current_state,
        &next_state,
        &recent_actions,
    );

    // ASSERTION: Should not panic, should return Ok(reward)
    assert!(
        result.is_ok(),
        "Expected graceful handling of empty portfolio_features, got error: {:?}",
        result
    );

    let reward = result.unwrap();
    let reward_f64: f64 = reward.try_into().unwrap_or(0.0);

    // Reward should be close to 0.0 (defaults to 1.0 → 1.0 portfolio value)
    assert!(
        reward_f64.abs() <= 0.1,
        "Expected reward ≈ 0.0 for empty portfolio_features (defaults applied), got {:.6}",
        reward_f64
    );

    println!(
        "✅ Test 6 PASSED: Gracefully handled empty portfolio_features, reward = {:.6}",
        reward_f64
    );

    Ok(())
}

// ============================================================================
// Test 7: Integration Test - Multiple Trades Sequence
// ============================================================================

#[test]
fn test_reward_multiple_trades_sequence() -> Result<()> {
    // Setup: RewardFunction with default config
    let config = RewardConfig::default();
    let mut reward_fn = RewardFunction::new(config);

    // Scenario: Simulate a sequence of 3 trades:
    // 1. BUY at 100.0, price → 102.0 (2% gain)
    // 2. HOLD at 102.0, price → 101.0 (1% drawdown)
    // 3. SELL at 101.0, price → 99.0 (2% gain on short)

    let mut total_reward = 0.0;

    // Start with balanced action history to avoid diversity penalty
    let mut recent_actions = create_balanced_action_history();

    // Trade 1: BUY (100.0 → 102.0, 2% gain)
    let buy_action = create_buy_action();
    let state1_current = create_long_state(100.0, 1.0);
    let state1_next = create_long_state(102.0, 1.02); // 2% portfolio gain
    let reward1 = reward_fn.calculate_reward(
        buy_action,
        &state1_current,
        &state1_next,
        &recent_actions,
    )?;
    let reward1_f64: f64 = reward1.try_into().unwrap_or(0.0);
    total_reward += reward1_f64;

    assert!(
        reward1_f64 > 0.0,
        "Trade 1 (BUY, 2% gain) should have positive reward, got {:.6}",
        reward1_f64
    );

    // Trade 2: HOLD (102.0 → 101.0, 1% drawdown)
    // Note: HOLD action but still holding the long position from Trade 1
    let hold_action = FactoredAction::new(ExposureLevel::Flat, OrderType::Market, Urgency::Normal);
    let state2_current = create_long_state(102.0, 1.02);
    let state2_next = create_long_state(101.0, 1.01); // 1% portfolio loss

    recent_actions.push(hold_action);
    let reward2 = reward_fn.calculate_reward(
        hold_action,
        &state2_current,
        &state2_next,
        &recent_actions,
    )?;
    let reward2_f64: f64 = reward2.try_into().unwrap_or(0.0);
    total_reward += reward2_f64;

    // Reward2 should be small (hold penalty or small loss)
    // Not strictly positive or negative - depends on hold_penalty vs P&L
    println!("Trade 2 (HOLD, 1% drawdown) reward = {:.6}", reward2_f64);

    // Trade 3: SELL (101.0 → 99.0, 2% gain on short)
    let sell_action = create_sell_action();
    let state3_current = create_short_state(101.0, 1.01);
    let state3_next = create_short_state(99.0, 1.03); // 2% portfolio gain from short

    recent_actions.push(sell_action);
    let reward3 = reward_fn.calculate_reward(
        sell_action,
        &state3_current,
        &state3_next,
        &recent_actions,
    )?;
    let reward3_f64: f64 = reward3.try_into().unwrap_or(0.0);
    total_reward += reward3_f64;

    assert!(
        reward3_f64 > 0.0,
        "Trade 3 (SELL, 2% gain) should have positive reward, got {:.6}",
        reward3_f64
    );

    // ASSERTION: Total reward should be positive (2 winning trades + 1 hold/small loss)
    assert!(
        total_reward > 0.0,
        "Expected total reward > 0.0 for profitable trade sequence, got {:.6}",
        total_reward
    );

    println!(
        "✅ Test 7 PASSED: Multiple trades sequence total reward = {:.6} (expected > 0.0)",
        total_reward
    );
    println!("  - Trade 1 (BUY +2%): {:.6}", reward1_f64);
    println!("  - Trade 2 (HOLD -1%): {:.6}", reward2_f64);
    println!("  - Trade 3 (SELL +2%): {:.6}", reward3_f64);

    Ok(())
}