foxhunt/ml/examples/train_continuous_ppo_parquet.rs

//! Continuous PPO Training Example with Parquet Data
//!
//! Trains a Continuous PPO model with Gaussian policies on market data from Parquet files:
//! - Real OHLCV data + 51-dimensional features
//! - Continuous position sizing in [-1.0, 1.0] range
//! - PnL-based rewards with transaction costs
//! - GAE advantages on real price trajectories
//! - Dual learning rates (policy/value)
//!
//! # Usage
//!
//! ```bash
//! # Train with default parameters (50 epochs, conservative exploration)
//! cargo run -p ml --example train_continuous_ppo_parquet --release --features cuda -- \
//!   --parquet-file test_data/ES_FUT_180d.parquet
//!
//! # Custom parameters with narrow action bounds
//! cargo run -p ml --example train_continuous_ppo_parquet --release --features cuda -- \
//!   --parquet-file test_data/ES_FUT_180d.parquet \
//!   --epochs 100 \
//!   --policy-lr 0.000001 \
//!   --value-lr 0.001 \
//!   --action-min -0.5 \
//!   --action-max 0.5
//!
//! # High exploration mode
//! cargo run -p ml --example train_continuous_ppo_parquet --release --features cuda -- \
//!   --parquet-file test_data/NQ_FUT_180d.parquet \
//!   --init-log-std 0.0
//! ```

use anyhow::{Context, Result};
use clap::Parser;
use std::fs::File;
use std::path::PathBuf;
use tracing::{info, warn};
use tracing_subscriber::FmtSubscriber;

use arrow::array::{Array, Float64Array, PrimitiveArray, UInt64Array};
use arrow::datatypes::TimestampNanosecondType;
use arrow::record_batch::RecordBatch;
use candle_core::Device;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;

use ml::features::extraction::{extract_ml_features, OHLCVBar};
use ml::ppo::continuous_ppo::{
    ContinuousPPO, ContinuousPPOConfig, ContinuousTrajectory,
    ContinuousTrajectoryBatch, ContinuousTrajectoryStep,
};
use ml::ppo::flow_policy::FlowPolicyConfig;
use ml::ppo::gae::GAEConfig;
use ml::evaluation::engine::{Action, EvaluationEngine};
use ml::evaluation::metrics::{PerformanceMetrics, OHLCVBarF32 as MetricsOHLCVBar};

/// Results from dual-phase backtesting (exploration vs exploitation)
#[derive(Debug, Clone)]
pub struct DualPhaseBacktestResults {
    /// Metrics from exploration phase (epochs 0 to burn_in_epochs-1)
    pub exploration_metrics: PerformanceMetrics,
    /// Metrics from exploitation phase (epochs burn_in_epochs to total_epochs-1)
    pub exploitation_metrics: PerformanceMetrics,
    /// Number of burn-in epochs used
    pub burn_in_epochs: usize,
    /// Total number of epochs
    pub total_epochs: usize,
}

/// Train Continuous PPO model on Parquet market data
#[derive(Debug, Parser)]
#[command(
    name = "train_continuous_ppo_parquet",
    about = "Train Continuous PPO model on Parquet market data"
)]
struct Opts {
    /// Path to Parquet file with market data
    #[arg(long)]
    parquet_file: String,

    /// Number of training epochs
    #[arg(long, default_value = "50")]
    epochs: usize,

    /// Policy (actor) learning rate
    #[arg(long, default_value = "0.000001")]
    policy_lr: f64,

    /// Value (critic) learning rate (reduced from 0.001 to prevent gradient explosion)
    #[arg(long, default_value = "0.0001")]
    value_lr: f64,

    /// Minimum action bound (position size)
    #[arg(long, default_value = "-1.0")]
    action_min: f32,

    /// Maximum action bound (position size)
    #[arg(long, default_value = "1.0")]
    action_max: f32,

    /// Initial log standard deviation (exploration level)
    #[arg(long, default_value = "-1.0")]
    init_log_std: f32,

    /// Checkpoint directory
    #[arg(long, default_value = "checkpoints/continuous_ppo")]
    checkpoint_dir: String,

    /// Checkpoint save interval (epochs)
    #[arg(long, default_value = "10")]
    checkpoint_interval: usize,

    /// Verbose logging
    #[arg(short, long)]
    verbose: bool,

    /// Number of burn-in epochs for dual-phase backtesting
    /// Epochs 0 to burn_in_epochs-1 are "exploration" phase
    /// Epochs burn_in_epochs to total are "exploitation" phase
    #[arg(long, default_value = "50")]
    burn_in_epochs: usize,
}

#[tokio::main]
async fn main() -> Result<()> {
    // Parse CLI options
    let opts = Opts::parse();

    // Setup logging
    let level = if opts.verbose {
        tracing::Level::DEBUG
    } else {
        tracing::Level::INFO
    };

    let subscriber = FmtSubscriber::builder().with_max_level(level).finish();
    tracing::subscriber::set_global_default(subscriber)
        .context("Failed to set tracing subscriber")?;

    info!("🚀 Starting Continuous PPO Training with Parquet Data");
    info!("Configuration:");
    info!("  • Parquet file: {}", opts.parquet_file);
    info!("  • Epochs: {}", opts.epochs);
    info!("  • Policy learning rate: {}", opts.policy_lr);
    info!("  • Value learning rate: {}", opts.value_lr);
    info!(
        "  • Action bounds: [{:.2}, {:.2}]",
        opts.action_min, opts.action_max
    );
    info!("  • Initial log std: {:.2}", opts.init_log_std);
    info!("  • GPU: CUDA if available (auto-fallback to CPU)");
    info!("  • Checkpoint directory: {}", opts.checkpoint_dir);
    info!(
        "  • Checkpoint interval: {} epochs",
        opts.checkpoint_interval
    );

    // Create checkpoint directory
    let checkpoint_path = PathBuf::from(&opts.checkpoint_dir);
    if !checkpoint_path.exists() {
        std::fs::create_dir_all(&checkpoint_path)
            .context("Failed to create checkpoint directory")?;
        info!("✅ Created checkpoint directory: {}", opts.checkpoint_dir);
    }

    // Load market data from Parquet file
    info!("\n📊 Loading market data from Parquet file...");
    let bars = load_parquet_data(&opts.parquet_file)
        .await
        .context("Failed to load Parquet data")?;

    info!("✅ Loaded {} OHLCV bars", bars.len());

    // Extract 51-dimensional feature vectors (43 base + 8 OFI)
    info!("\n🏗️  Extracting 51-dimensional feature vectors...");
    let feature_vectors =
        extract_ml_features(&bars).context("Failed to extract 51-dimensional features")?;

    info!(
        "✅ Extracted {} feature vectors (dim=51, warmup bars skipped=50)",
        feature_vectors.len()
    );

    // Convert FeatureVector ([f64; 51]) to Vec<Vec<f32>> for PPO trainer
    let state_dim = 51;
    let market_data: Vec<Vec<f32>> = feature_vectors
        .iter()
        .map(|fv| fv.iter().map(|&v| v as f32).collect())
        .collect();

    // Validate state dimensions
    if let Some(first_state) = market_data.first() {
        if first_state.len() != state_dim {
            return Err(anyhow::anyhow!(
                "State dimension mismatch: expected {}, got {}",
                state_dim,
                first_state.len()
            ));
        }
    }

    info!(
        "✅ Feature extraction complete: {} samples",
        market_data.len()
    );

    // Configure Flow-Based Policy for Continuous PPO
    let policy_config = FlowPolicyConfig {
        state_dim,
        action_dim: 1,
        context_dim: 128,
        num_layers: 4,
        scale_clamp: 5.0,
    };

    let config = ContinuousPPOConfig {
        state_dim,
        policy_config,
        value_hidden_dims: vec![512, 384, 256, 128, 64],
        policy_learning_rate: opts.policy_lr,
        value_learning_rate: opts.value_lr,
        clip_epsilon: 0.2,
        value_loss_coeff: 0.5,
        entropy_coeff: 0.01,
        gae_config: GAEConfig {
            gamma: 0.99,
            lambda: 0.95,
            normalize_advantages: true,
        },
        batch_size: 2048,
        mini_batch_size: 64,
        num_epochs: 10,
        max_grad_norm: 0.5,
    };

    // Create Continuous PPO agent
    let device = Device::cuda_if_available(0).unwrap_or(Device::Cpu);
    info!("Using device: {:?}", device);

    let mut agent = ContinuousPPO::new(config.clone())
        .context("Failed to create Continuous PPO agent")?;

    info!("✅ Continuous PPO agent initialized (state_dim={})", state_dim);

    // Training loop
    info!("\n🏋️  Starting training...\n");
    let start_time = std::time::Instant::now();

    let transaction_cost_bps = 0.05; // 0.05% transaction cost
    let hold_penalty = 0.0001; // Small penalty for holding positions

    for epoch in 0..opts.epochs {
        // Collect trajectories
        let trajectories = collect_trajectories(
            &agent,
            &market_data,
            transaction_cost_bps,
            hold_penalty,
        )?;

        // Compute GAE advantages
        let mut batch = prepare_batch(trajectories, &config)?;

        // Update agent
        let (policy_loss, value_loss) = agent
            .update(&mut batch)
            .context("Failed to update agent")?;

        // Compute metrics
        let mean_reward = batch.advantages.iter().sum::<f32>() / batch.advantages.len() as f32;
        let sharpe_ratio = compute_sharpe_ratio(&batch.advantages);

        info!(
            "📊 Epoch {}/{}: policy_loss={:.4}, value_loss={:.4}, mean_reward={:.4}, sharpe={:.4}",
            epoch + 1,
            opts.epochs,
            policy_loss,
            value_loss,
            mean_reward,
            sharpe_ratio
        );

        // Save checkpoint
        if (epoch + 1) % opts.checkpoint_interval == 0 {
            let actor_path = checkpoint_path.join(format!("actor_epoch_{}.safetensors", epoch + 1));
            let critic_path =
                checkpoint_path.join(format!("critic_epoch_{}.safetensors", epoch + 1));

            agent.actor.vars().save(&actor_path).with_context(|| {
                format!("Failed to save actor checkpoint: {:?}", actor_path)
            })?;
            agent.critic.vars().save(&critic_path).with_context(|| {
                format!("Failed to save critic checkpoint: {:?}", critic_path)
            })?;

            info!(
                "💾 Checkpoint saved at epoch {} (actor: {:?}, critic: {:?})",
                epoch + 1,
                actor_path,
                critic_path
            );
        }
    }

    let training_duration = start_time.elapsed();

    // Print final training metrics
    info!("\n✅ Training completed successfully!");
    info!(
        "  • Training time: {:.1}s ({:.1} min)",
        training_duration.as_secs_f64(),
        training_duration.as_secs_f64() / 60.0
    );
    info!("  • Training steps: {}", agent.get_training_steps());

    // Run dual-phase backtest to separate exploration from exploitation
    info!("\n🔍 Running dual-phase backtest (exploration vs exploitation)...");
    let backtest_start = std::time::Instant::now();

    let dual_results = backtest_trained_agent_dual_phase(
        &agent,
        &market_data,
        &bars,
        opts.burn_in_epochs,
        opts.epochs,
    )?;

    let backtest_duration = backtest_start.elapsed();

    info!("\n📊 Dual-Phase Backtest Results:");
    info!("  • Backtest time: {:.2}s", backtest_duration.as_secs_f64());
    info!("  • Burn-in epochs: {} / {}", dual_results.burn_in_epochs, dual_results.total_epochs);

    info!("\n--- Exploration Phase (Epochs 0-{}) ---", dual_results.burn_in_epochs.saturating_sub(1));
    info!("  • Total trades: {}", dual_results.exploration_metrics.total_trades);
    info!("  • Sharpe ratio: {:.4}", dual_results.exploration_metrics.sharpe_ratio);
    info!("  • Win rate: {:.2}%", dual_results.exploration_metrics.win_rate);
    info!("  • Max drawdown: {:.2}%", dual_results.exploration_metrics.max_drawdown_pct);
    info!("  • Total return: {:.2}%", dual_results.exploration_metrics.total_return_pct);
    info!("  • Average trade PnL: {:.4}", dual_results.exploration_metrics.avg_trade_pnl);
    info!("  • Final equity: ${:.2}", dual_results.exploration_metrics.final_equity);

    info!("\n--- Exploitation Phase (Epochs {}-{}) ---",
          dual_results.burn_in_epochs,
          dual_results.total_epochs.saturating_sub(1));
    info!("  • Total trades: {}", dual_results.exploitation_metrics.total_trades);
    info!("  • Sharpe ratio: {:.4}", dual_results.exploitation_metrics.sharpe_ratio);
    info!("  • Win rate: {:.2}%", dual_results.exploitation_metrics.win_rate);
    info!("  • Max drawdown: {:.2}%", dual_results.exploitation_metrics.max_drawdown_pct);
    info!("  • Total return: {:.2}%", dual_results.exploitation_metrics.total_return_pct);
    info!("  • Average trade PnL: {:.4}", dual_results.exploitation_metrics.avg_trade_pnl);
    info!("  • Final equity: ${:.2}", dual_results.exploitation_metrics.final_equity);

    // Calculate improvement (avoid division by zero)
    let sharpe_improvement = if dual_results.exploration_metrics.sharpe_ratio.abs() > 1e-6 {
        ((dual_results.exploitation_metrics.sharpe_ratio - dual_results.exploration_metrics.sharpe_ratio)
         / dual_results.exploration_metrics.sharpe_ratio.abs()) * 100.0
    } else if dual_results.exploitation_metrics.sharpe_ratio.abs() > 1e-6 {
        f64::INFINITY
    } else {
        0.0
    };

    info!("\n--- Performance Improvement ---");
    if sharpe_improvement.is_infinite() {
        info!("  • Sharpe improvement: +INF% (exploration Sharpe near zero)");
    } else {
        info!("  • Sharpe improvement: {:.2}%", sharpe_improvement);
    }

    // Save final checkpoint
    let final_actor_path = checkpoint_path.join(format!("actor_epoch_{}.safetensors", opts.epochs));
    let final_critic_path =
        checkpoint_path.join(format!("critic_epoch_{}.safetensors", opts.epochs));

    agent
        .actor
        .vars()
        .save(&final_actor_path)
        .with_context(|| format!("Failed to save final actor: {:?}", final_actor_path))?;
    agent
        .critic
        .vars()
        .save(&final_critic_path)
        .with_context(|| format!("Failed to save final critic: {:?}", final_critic_path))?;

    info!(
        "\n💾 Final checkpoint saved to: {:?}, {:?}",
        final_actor_path, final_critic_path
    );
    info!("\n🎉 Continuous PPO training complete with Parquet data!");

    Ok(())
}

/// Collect continuous trajectories from market data
fn collect_trajectories(
    agent: &ContinuousPPO,
    market_data: &[Vec<f32>],
    transaction_cost_bps: f32,
    hold_penalty: f32,
) -> Result<Vec<ContinuousTrajectory>> {
    let mut trajectories = Vec::new();
    let mut current_trajectory = ContinuousTrajectory::new();

    let mut position: f32 = 0.0; // Current position size
    let max_steps = market_data.len().min(2048);

    // Diagnostic tracking
    let mut total_rewards = 0.0f32;
    let mut non_zero_rewards = 0usize;
    let mut position_samples = Vec::new();

    for step_idx in 0..max_steps {
        let state = &market_data[step_idx];

        // Get action from agent
        let (action, log_prob, value) = agent
            .act_with_log_prob(state)
            .context("Failed to select action")?;

        let new_position = action.position_size();

        // Compute reward using current position as old_position
        // (position holds the previous step's action, which is correct for PnL calculation)
        let log_return = state[state.len() - 1]; // Last feature is log return
        let reward = compute_reward(
            new_position,
            position,  // old_position from previous step
            log_return,
            transaction_cost_bps,
            hold_penalty,
        );

        let done = step_idx == max_steps - 1;

        // Add step to trajectory
        let traj_step = ContinuousTrajectoryStep::new(
            state.clone(),
            action,
            log_prob,
            reward,
            value,
            done,
        );
        current_trajectory.add_step(traj_step);

        // Update position for next step
        position = new_position;

        // Track diagnostics (sample every 100 steps)
        if step_idx % 100 == 0 {
            position_samples.push(new_position);
        }
        total_rewards += reward;
        if reward.abs() > 1e-6 {
            non_zero_rewards += 1;
        }

        // Start new trajectory every 1024 steps or at episode end
        if current_trajectory.len() >= 1024 || done {
            trajectories.push(current_trajectory);
            current_trajectory = ContinuousTrajectory::new();
            position = 0.0; // Reset position for new trajectory
        }
    }

    // Add remaining trajectory if not empty
    if !current_trajectory.is_empty() {
        trajectories.push(current_trajectory);
    }

    // Log diagnostic summary
    let avg_position = if !position_samples.is_empty() {
        position_samples.iter().sum::<f32>() / position_samples.len() as f32
    } else {
        0.0
    };

    info!(
        "Trajectory collection: {} steps, avg_reward={:.6}, non_zero_rewards={}/{}, avg_position={:.4}",
        max_steps, total_rewards / max_steps as f32, non_zero_rewards, max_steps, avg_position
    );

    // Log first few position samples for debugging
    if !position_samples.is_empty() {
        let sample_slice = &position_samples[..position_samples.len().min(5)];
        info!("Position samples (first 5): {:?}", sample_slice);
    }

    Ok(trajectories)
}

/// Compute reward for continuous position sizing
fn compute_reward(
    new_position: f32,
    old_position: f32,
    log_return: f32,
    transaction_cost_bps: f32,
    hold_penalty: f32,
) -> f32 {
    // PnL from position and market movement (scaled to reasonable range)
    let pnl = old_position * log_return * 1000.0;

    // Transaction cost (proportional to position change)
    let position_change = (new_position - old_position).abs();
    let transaction_cost = position_change * transaction_cost_bps;

    // Hold penalty (small penalty for non-zero positions to encourage active trading)
    let hold_cost = new_position.abs() * hold_penalty;

    // Total reward
    pnl - transaction_cost - hold_cost
}

/// Prepare batch with GAE advantages
fn prepare_batch(
    trajectories: Vec<ContinuousTrajectory>,
    config: &ContinuousPPOConfig,
) -> Result<ContinuousTrajectoryBatch> {
    let gamma = config.gae_config.gamma;
    let lambda = config.gae_config.lambda;

    // Compute GAE advantages for each trajectory
    let mut all_advantages = Vec::new();
    let mut all_returns = Vec::new();

    for trajectory in &trajectories {
        let steps = trajectory.steps();

        // Extract rewards, values, and dones
        let rewards: Vec<f32> = steps.iter().map(|s| s.reward).collect();
        let values: Vec<f32> = steps.iter().map(|s| s.value).collect();
        let dones: Vec<bool> = steps.iter().map(|s| s.done).collect();

        // Compute GAE advantages
        let advantages = compute_gae_advantages(&rewards, &values, &dones, gamma, lambda);

        // Compute returns
        let returns = compute_returns(&rewards, gamma);

        all_advantages.extend(advantages);
        all_returns.extend(returns);
    }

    // Create batch from trajectories
    let batch = ContinuousTrajectoryBatch::from_trajectories(
        trajectories,
        all_advantages,
        all_returns,
    );

    Ok(batch)
}

/// Compute GAE advantages
fn compute_gae_advantages(
    rewards: &[f32],
    values: &[f32],
    dones: &[bool],
    gamma: f32,
    lambda: f32,
) -> Vec<f32> {
    let n = rewards.len();
    let mut advantages = vec![0.0; n];
    let mut gae = 0.0;

    for t in (0..n).rev() {
        let reward = rewards[t];
        let value = values[t];
        let next_value = if t + 1 < n { values[t + 1] } else { 0.0 };
        let done = dones[t];

        let mask = if done { 0.0 } else { 1.0 };
        let delta = reward + gamma * next_value * mask - value;
        gae = delta + gamma * lambda * mask * gae;

        advantages[t] = gae;
    }

    advantages
}

/// Compute discounted returns
fn compute_returns(rewards: &[f32], gamma: f32) -> Vec<f32> {
    let n = rewards.len();
    let mut returns = vec![0.0; n];
    let mut cumulative = 0.0;

    for t in (0..n).rev() {
        cumulative = rewards[t] + gamma * cumulative;
        returns[t] = cumulative;
    }

    returns
}

/// Compute Sharpe ratio from rewards/advantages
fn compute_sharpe_ratio(values: &[f32]) -> f32 {
    if values.is_empty() {
        return 0.0;
    }

    let mean = values.iter().sum::<f32>() / values.len() as f32;
    let variance = values
        .iter()
        .map(|v| (v - mean).powi(2))
        .sum::<f32>()
        / values.len() as f32;

    let std = (variance + 1e-8).sqrt();

    mean / std
}

/// Load OHLCV data from Parquet file (Databento schema)
async fn load_parquet_data(parquet_path: &str) -> Result<Vec<OHLCVBar>> {
    info!("Loading Parquet file: {}", parquet_path);

    // Open Parquet file
    let file = File::open(parquet_path)
        .with_context(|| format!("Failed to open Parquet file: {}", parquet_path))?;

    // Create Parquet reader
    let builder = ParquetRecordBatchReaderBuilder::try_new(file)
        .with_context(|| "Failed to create Parquet reader")?;

    let reader = builder
        .build()
        .with_context(|| "Failed to build Parquet reader")?;

    // Read all batches
    let mut all_ohlcv_bars = Vec::new();

    for batch_result in reader {
        let batch: RecordBatch = batch_result.with_context(|| "Failed to read record batch")?;

        // Extract columns from Databento Parquet schema:
        // Column 3: open, Column 4: high, Column 5: low, Column 6: close
        // Column 7: volume, Column 9: ts_event (Timestamp(Nanosecond, Some("UTC")))
        let timestamps = batch
            .column(9)
            .as_any()
            .downcast_ref::<PrimitiveArray<TimestampNanosecondType>>()
            .ok_or_else(|| {
                anyhow::anyhow!(
                    "Failed to downcast timestamp column. Expected Timestamp(Nanosecond), got: {:?}",
                    batch.column(9).data_type()
                )
            })?;

        let opens = batch
            .column(3)
            .as_any()
            .downcast_ref::<Float64Array>()
            .ok_or_else(|| anyhow::anyhow!("Failed to downcast open column"))?;

        let highs = batch
            .column(4)
            .as_any()
            .downcast_ref::<Float64Array>()
            .ok_or_else(|| anyhow::anyhow!("Failed to downcast high column"))?;

        let lows = batch
            .column(5)
            .as_any()
            .downcast_ref::<Float64Array>()
            .ok_or_else(|| anyhow::anyhow!("Failed to downcast low column"))?;

        let closes = batch
            .column(6)
            .as_any()
            .downcast_ref::<Float64Array>()
            .ok_or_else(|| anyhow::anyhow!("Failed to downcast close column"))?;

        let volumes = batch
            .column(7)
            .as_any()
            .downcast_ref::<UInt64Array>()
            .ok_or_else(|| anyhow::anyhow!("Failed to downcast volume column"))?;

        // Convert to OHLCVBar structs
        for i in 0..batch.num_rows() {
            let timestamp_ns = timestamps.value(i);

            // Convert nanoseconds to DateTime<Utc>
            let timestamp = chrono::DateTime::from_timestamp(
                (timestamp_ns / 1_000_000_000) as i64,
                (timestamp_ns % 1_000_000_000) as u32,
            )
            .unwrap_or_else(|| chrono::Utc::now());

            let bar = OHLCVBar {
                timestamp,
                open: opens.value(i),
                high: highs.value(i),
                low: lows.value(i),
                close: closes.value(i),
                volume: volumes.value(i) as f64,
            };

            all_ohlcv_bars.push(bar);
        }
    }

    info!("✅ Loaded {} OHLCV bars from Parquet", all_ohlcv_bars.len());

    Ok(all_ohlcv_bars)
}

/// Run backtest on trained PPO agent to compute actual trading performance
#[allow(dead_code)]
fn backtest_trained_agent(
    agent: &ContinuousPPO,
    market_data: &[Vec<f32>],
    bars: &[OHLCVBar],
) -> Result<PerformanceMetrics> {
    // Create evaluation engine with $10K initial capital
    let mut engine = EvaluationEngine::new(10000.0);

    // Skip warmup bars (50 bars used for feature extraction)
    let warmup_bars = 50;
    let mut metrics_bars = Vec::new();

    // Run backtest using trained agent (greedy, no exploration)
    for (step_idx, state) in market_data.iter().enumerate() {
        // Get corresponding OHLCV bar (accounting for warmup)
        let bar_idx = step_idx + warmup_bars;
        if bar_idx >= bars.len() {
            warn!("Bar index {} exceeds available bars ({})", bar_idx, bars.len());
            break;
        }
        let bar = &bars[bar_idx];

        // Get action from agent (greedy policy - use mean without sampling)
        let (action, _value) = agent
            .act(state)
            .context("Failed to select action during backtest")?;

        // Convert continuous position size to discrete trading action
        // Position size in [-1.0, 1.0]: negative = short, positive = long, near-zero = hold
        let trading_action = continuous_to_discrete_action(action.position_size());

        // Convert to OHLCVBar for metrics (same timestamp, prices from original bar)
        let metrics_bar = MetricsOHLCVBar {
            timestamp: bar.timestamp.timestamp(),
            open: bar.open as f32,
            high: bar.high as f32,
            low: bar.low as f32,
            close: bar.close as f32,
            volume: bar.volume as f32,
        };

        // Process bar in evaluation engine
        engine.process_bar(step_idx, &metrics_bar, trading_action);
        metrics_bars.push(metrics_bar);
    }

    // Close any open position at end of backtest
    if let Some(last_bar) = metrics_bars.last() {
        engine.close_position(metrics_bars.len() - 1, last_bar);
    }

    // Calculate performance metrics from actual trades
    let metrics = PerformanceMetrics::from_trades(
        &engine.trades,
        engine.initial_capital,
        &metrics_bars,
    );

    Ok(metrics)
}

/// Run dual-phase backtest to separate exploration from exploitation performance
///
/// # Arguments
/// * `agent` - Trained Continuous PPO agent
/// * `market_data` - Feature vectors (51-dim)
/// * `bars` - OHLCV bars (for timestamping)
/// * `burn_in_epochs` - Number of epochs to treat as "exploration" phase
/// * `total_epochs` - Total number of training epochs
///
/// # Returns
/// Dual-phase backtest results with separate metrics for each phase
fn backtest_trained_agent_dual_phase(
    agent: &ContinuousPPO,
    market_data: &[Vec<f32>],
    bars: &[OHLCVBar],
    burn_in_epochs: usize,
    total_epochs: usize,
) -> Result<DualPhaseBacktestResults> {
    // Create separate engines for each phase
    let mut exploration_engine = EvaluationEngine::new(10000.0);
    let mut exploitation_engine = EvaluationEngine::new(10000.0);

    // Skip warmup bars (50 bars used for feature extraction)
    let warmup_bars = 50;
    let mut exploration_bars = Vec::new();
    let mut exploitation_bars = Vec::new();

    // Calculate steps per epoch (approximate)
    let total_steps = market_data.len();
    let steps_per_epoch = 1024; // From trajectory collection
    let actual_epochs = (total_steps + steps_per_epoch - 1) / steps_per_epoch;

    info!(
        "Dual-phase backtest: burn_in={}, total_epochs={}, actual_epochs={}, total_steps={}",
        burn_in_epochs, total_epochs, actual_epochs, total_steps
    );

    // Run backtest through all market data
    for (step_idx, state) in market_data.iter().enumerate() {
        // Calculate current epoch
        let current_epoch = step_idx / steps_per_epoch;

        // Get corresponding OHLCV bar (accounting for warmup)
        let bar_idx = step_idx + warmup_bars;
        if bar_idx >= bars.len() {
            warn!("Bar index {} exceeds available bars ({})", bar_idx, bars.len());
            break;
        }
        let bar = &bars[bar_idx];

        // Get action from agent (greedy policy - use mean without sampling)
        let (action, _value) = agent
            .act(state)
            .context("Failed to select action during backtest")?;

        // Convert continuous position size to discrete trading action
        let trading_action = continuous_to_discrete_action(action.position_size());

        // Convert to OHLCVBar for metrics
        let metrics_bar = MetricsOHLCVBar {
            timestamp: bar.timestamp.timestamp(),
            open: bar.open as f32,
            high: bar.high as f32,
            low: bar.low as f32,
            close: bar.close as f32,
            volume: bar.volume as f32,
        };

        // Route to appropriate engine based on current epoch
        if current_epoch < burn_in_epochs {
            // Exploration phase
            exploration_engine.process_bar(step_idx, &metrics_bar, trading_action);
            exploration_bars.push(metrics_bar);
        } else {
            // Exploitation phase
            exploitation_engine.process_bar(step_idx, &metrics_bar, trading_action);
            exploitation_bars.push(metrics_bar);
        }
    }

    // Close any open positions in both engines
    if let Some(last_bar) = exploration_bars.last() {
        let last_idx = exploration_bars.len() - 1;
        exploration_engine.close_position(last_idx, last_bar);
    }

    if let Some(last_bar) = exploitation_bars.last() {
        let last_idx = exploitation_bars.len() - 1;
        exploitation_engine.close_position(last_idx, last_bar);
    }

    // Calculate metrics for each phase
    let exploration_metrics = if burn_in_epochs > 0 && !exploration_engine.trades.is_empty() {
        PerformanceMetrics::from_trades(
            &exploration_engine.trades,
            exploration_engine.initial_capital,
            &exploration_bars,
        )
    } else {
        PerformanceMetrics::default()
    };

    let exploitation_metrics = if actual_epochs > burn_in_epochs && !exploitation_engine.trades.is_empty() {
        PerformanceMetrics::from_trades(
            &exploitation_engine.trades,
            exploitation_engine.initial_capital,
            &exploitation_bars,
        )
    } else {
        PerformanceMetrics::default()
    };

    info!(
        "Phase distribution: exploration_trades={}, exploitation_trades={}",
        exploration_engine.trades.len(),
        exploitation_engine.trades.len()
    );

    Ok(DualPhaseBacktestResults {
        exploration_metrics,
        exploitation_metrics,
        burn_in_epochs,
        total_epochs,
    })
}

/// Convert continuous position size to discrete trading action
///
/// # Arguments
/// * `position_size` - Continuous position size in [-1.0, 1.0]
///
/// # Returns
/// Discrete action (Buy, Hold, Sell)
///
/// # Logic
/// - position_size > 0.3: Buy (strong long signal)
/// - position_size < -0.3: Sell (strong short signal)
/// - otherwise: Hold (weak signal or neutral)
fn continuous_to_discrete_action(position_size: f32) -> Action {
    const BUY_THRESHOLD: f32 = 0.3;
    const SELL_THRESHOLD: f32 = -0.3;

    if position_size > BUY_THRESHOLD {
        Action::Buy
    } else if position_size < SELL_THRESHOLD {
        Action::Sell
    } else {
        Action::Hold
    }
}