foxhunt/ml/examples/hyperopt_continuous_ppo_demo.rs

//! Continuous PPO Hyperparameter Optimization Demo
//!
//! This example demonstrates the continuous PPO hyperparameter optimization adapter
//! using the generic egobox optimization framework.
//!
//! # Usage
//!
//! ```bash
//! # Run with default settings (5 trials, 10 epochs)
//! cargo run -p ml --example hyperopt_continuous_ppo_demo --release --features cuda -- \
//!     --parquet-file test_data/ES_FUT_180d.parquet
//!
//! # Custom trials and epochs
//! cargo run -p ml --example hyperopt_continuous_ppo_demo --release --features cuda -- \
//!     --parquet-file test_data/ES_FUT_180d.parquet \
//!     --trials 30 \
//!     --epochs 50
//! ```
//!
//! # Expected Output
//!
//! - Real continuous PPO training with market data
//! - Varying Sharpe ratios across trials
//! - Convergence visible (best Sharpe improves)
//! - Logs showing actual PPO training steps
//! - GPU utilization (if CUDA available)

use anyhow::Result;
use clap::Parser;
use std::path::PathBuf;
use tracing::{info, Level};

use ml::hyperopt::adapters::continuous_ppo::{ContinuousPPOParams, ContinuousPPOTrainer};
use ml::hyperopt::paths::{generate_run_id, TrainingPaths};
use ml::hyperopt::traits::ParameterSpace;
use ml::hyperopt::EgoboxOptimizer;

/// CLI arguments
#[derive(Parser, Debug)]
#[command(
    name = "hyperopt_continuous_ppo_demo",
    about = "Continuous PPO hyperparameter optimization demonstration"
)]
struct Args {
    /// Path to Parquet file with OHLCV data
    #[arg(long, help = "Path to Parquet file with OHLCV data")]
    parquet_file: String,

    /// Number of optimization trials
    #[arg(long, default_value = "5", help = "Number of optimization trials")]
    trials: usize,

    /// Epochs per trial
    #[arg(long, default_value = "10", help = "Training epochs per trial")]
    epochs: usize,

    /// Base directory for training outputs
    #[arg(
        long,
        default_value = "/tmp/ml_training",
        help = "Base directory for training outputs"
    )]
    base_dir: PathBuf,

    /// Run ID (auto-generated if not provided)
    #[arg(long, help = "Unique run ID (YYYYMMDD_HHMMSS_type if not provided)")]
    run_id: Option<String>,

    /// Run type for auto-generated run ID
    #[arg(
        long,
        default_value = "hyperopt_continuous_ppo",
        help = "Run type for auto-generated run ID"
    )]
    run_type: String,
}

fn main() -> Result<()> {
    // Initialize tracing
    tracing_subscriber::fmt()
        .with_max_level(Level::INFO)
        .with_target(false)
        .with_thread_ids(false)
        .init();

    info!("╔═══════════════════════════════════════════════════════════╗");
    info!("║   Continuous PPO Hyperparameter Optimization Demo        ║");
    info!("╚═══════════════════════════════════════════════════════════╝");
    info!("");

    // Parse arguments
    let args = Args::parse();

    info!("Configuration:");
    info!("  Trials: {}", args.trials);
    info!("  Epochs per trial: {}", args.epochs);
    info!("  Parquet file: {}", args.parquet_file);
    info!("");

    // Validate Parquet file exists
    let parquet_path = std::path::Path::new(&args.parquet_file);
    if !parquet_path.exists() {
        anyhow::bail!("Parquet file not found: {}", args.parquet_file);
    }

    // Generate run ID if not provided
    let run_id = args
        .run_id
        .unwrap_or_else(|| generate_run_id(&args.run_type));

    // Create training paths
    let training_paths = TrainingPaths::new(&args.base_dir, "continuous_ppo", &run_id);

    // Create all directories
    training_paths
        .create_all()
        .map_err(|e| anyhow::anyhow!("Failed to create training directories: {}", e))?;

    info!("Training Paths:");
    info!("  Base directory: {:?}", args.base_dir);
    info!("  Run ID: {}", run_id);
    info!("  Run directory: {:?}", training_paths.run_dir());
    info!("  Checkpoints: {:?}", training_paths.checkpoints_dir());
    info!("");

    // Create continuous PPO trainer with training paths
    let trainer = ContinuousPPOTrainer::new(&args.parquet_file, args.epochs)?
        .with_training_paths(training_paths);

    info!("Parameter Space:");
    let names = ContinuousPPOParams::param_names();
    let cont_bounds = ContinuousPPOParams::continuous_bounds();
    let int_bounds = ContinuousPPOParams::integer_bounds();
    let cat_choices = ContinuousPPOParams::categorical_choices();

    info!("  Continuous Parameters:");
    for (i, name) in names.iter().take(cont_bounds.len()).enumerate() {
        let (min, max) = cont_bounds[i];
        info!("    {}: [{:.6}, {:.6}]", name, min, max);
    }

    info!("  Integer Parameters:");
    for (i, name) in names
        .iter()
        .skip(cont_bounds.len())
        .take(int_bounds.len())
        .enumerate()
    {
        let (min, max) = int_bounds[i];
        info!("    {}: [{}, {}]", name, min, max);
    }

    info!("  Categorical Parameters:");
    for (i, name) in names
        .iter()
        .skip(cont_bounds.len() + int_bounds.len())
        .take(cat_choices.len())
        .enumerate()
    {
        let choices = &cat_choices[i];
        info!("    {}: {:?}", name, choices);
    }
    info!("");

    // Create optimizer
    let optimizer = EgoboxOptimizer::with_trials(args.trials, 3);

    info!("Starting optimization...");
    info!("");

    // Run optimization
    let result = optimizer.optimize(trainer)?;

    info!("");
    info!("╔═══════════════════════════════════════════════════════════╗");
    info!("║   Optimization Complete                                   ║");
    info!("╚═══════════════════════════════════════════════════════════╝");
    info!("");
    info!("Best Parameters:");
    info!("  Policy LR: {:.6}", result.best_params.policy_lr);
    info!("  Value LR: {:.6}", result.best_params.value_lr);
    info!(
        "  Action bounds: [{:.2}, {:.2}]",
        result.best_params.action_min, result.best_params.action_max
    );
    info!("  Init log std: {:.2}", result.best_params.init_log_std);
    info!("  Learnable std: {}", result.best_params.learnable_std);
    info!("  Clip epsilon: {:.3}", result.best_params.clip_epsilon);
    info!("  Entropy coeff: {:.6}", result.best_params.entropy_coeff);
    info!("  GAE lambda: {:.3}", result.best_params.gae_lambda);
    info!("  Gamma: {:.3}", result.best_params.gamma);
    info!("  Batch size: {}", result.best_params.batch_size);
    info!("  Num epochs: {}", result.best_params.num_epochs);
    info!("");
    info!(
        "Best Objective (Sharpe ratio): {:.6}",
        -result.best_objective
    ); // Negated (optimizer minimizes)
    info!("Total Evaluations: {}", result.all_trials.len());
    info!("");

    // Print trial history for convergence analysis
    info!("Trial History:");
    info!("┌───────┬──────────────────┬──────────────────┬──────────────────┐");
    info!("│ Trial │   Policy LR      │    Value LR      │  Sharpe Ratio    │");
    info!("├───────┼──────────────────┼──────────────────┼──────────────────┤");

    for trial in &result.all_trials {
        info!(
            "│ {:5} │ {:16.6} │ {:16.6} │ {:16.6} │",
            trial.trial_num,
            trial.params.policy_lr,
            trial.params.value_lr,
            -trial.objective // Negate to show actual Sharpe
        );
    }

    info!("└───────┴──────────────────┴──────────────────┴──────────────────┘");
    info!("");

    // Compute convergence metrics
    if result.all_trials.len() >= 2 {
        let first_sharpe = -result.all_trials[0].objective;
        let best_sharpe = -result.best_objective;
        let improvement = ((best_sharpe - first_sharpe) / first_sharpe.abs().max(1.0)) * 100.0;

        info!("Convergence Analysis:");
        info!("  First Trial Sharpe: {:.6}", first_sharpe);
        info!("  Best Trial Sharpe: {:.6}", best_sharpe);
        info!("  Improvement: {:.2}%", improvement);
        info!("");

        // Compute variance in Sharpe values
        let sharpe_values: Vec<f64> = result.all_trials.iter().map(|e| -e.objective).collect();
        let mean_sharpe: f64 = sharpe_values.iter().sum::<f64>() / sharpe_values.len() as f64;
        let variance: f64 = sharpe_values
            .iter()
            .map(|s| (s - mean_sharpe).powi(2))
            .sum::<f64>()
            / sharpe_values.len() as f64;
        let std_dev = variance.sqrt();
        let coeff_var = (std_dev / mean_sharpe.abs().max(0.001)) * 100.0;

        info!("Sharpe Variance Analysis:");
        info!("  Mean Sharpe: {:.6}", mean_sharpe);
        info!("  Std Dev: {:.6}", std_dev);
        info!("  Coefficient of Variation: {:.2}%", coeff_var);
        info!("");

        if coeff_var < 5.0 {
            info!(
                "⚠️  WARNING: Low Sharpe variance ({:.2}%) suggests mock metrics",
                coeff_var
            );
        } else {
            info!(
                "✓ Sharpe variance ({:.2}%) confirms real training",
                coeff_var
            );
        }
    }

    info!("✓ Continuous PPO hyperparameter optimization demo complete");

    Ok(())
}