foxhunt/ml/examples/hyperopt_ppo_demo.rs

//! PPO Hyperparameter Optimization Demo
//!
//! This example demonstrates the PPO hyperparameter optimization adapter
//! using the generic egobox optimization framework.
//!
//! # Usage
//!
//! ```bash
//! # Run with default settings (3 trials, 1000 episodes)
//! cargo run -p ml --example hyperopt_ppo_demo --release --features cuda
//!
//! # Custom trials and episodes
//! cargo run -p ml --example hyperopt_ppo_demo --release --features cuda -- \
//!     --trials 5 \
//!     --episodes 500
//! ```
//!
//! # Expected Output
//!
//! - Real PPO training with synthetic trajectories
//! - Varying loss values across trials (not hardcoded)
//! - Convergence visible (best metric improves)
//! - Logs showing actual PPO training steps
//! - GPU utilization (if CUDA available)

use anyhow::Result;
use clap::Parser;
use std::path::PathBuf;
use tracing::{info, Level};

use ml::hyperopt::adapters::ppo::{PPOParams, PPOTrainer};
use ml::hyperopt::paths::{generate_run_id, TrainingPaths};
use ml::hyperopt::traits::ParameterSpace;
use ml::hyperopt::EgoboxOptimizer;

/// CLI arguments
#[derive(Parser, Debug)]
#[command(
    name = "hyperopt_ppo_demo",
    about = "PPO hyperparameter optimization demonstration"
)]
struct Args {
    /// Number of optimization trials
    #[arg(long, default_value = "3", help = "Number of optimization trials")]
    trials: usize,

    /// Episodes per trial
    #[arg(long, default_value = "1000", help = "Training episodes per trial")]
    episodes: usize,

    /// Base directory for training outputs
    #[arg(
        long,
        default_value = "/tmp/ml_training",
        help = "Base directory for training outputs"
    )]
    base_dir: PathBuf,

    /// Run ID (auto-generated if not provided)
    #[arg(long, help = "Unique run ID (YYYYMMDD_HHMMSS_type if not provided)")]
    run_id: Option<String>,

    /// Run type for auto-generated run ID
    #[arg(
        long,
        default_value = "hyperopt",
        help = "Run type for auto-generated run ID"
    )]
    run_type: String,
}

fn main() -> Result<()> {
    // Initialize tracing
    tracing_subscriber::fmt()
        .with_max_level(Level::INFO)
        .with_target(false)
        .with_thread_ids(false)
        .init();

    info!("╔═══════════════════════════════════════════════════════════╗");
    info!("║   PPO Hyperparameter Optimization Demo                   ║");
    info!("╚═══════════════════════════════════════════════════════════╝");
    info!("");

    // Parse arguments
    let args = Args::parse();

    info!("Configuration:");
    info!("  Trials: {}", args.trials);
    info!("  Episodes per trial: {}", args.episodes);
    info!("");

    // Generate run ID if not provided
    let run_id = args
        .run_id
        .unwrap_or_else(|| generate_run_id(&args.run_type));

    // Create training paths
    let training_paths = TrainingPaths::new(&args.base_dir, "ppo", &run_id);

    // Create all directories
    training_paths
        .create_all()
        .map_err(|e| anyhow::anyhow!("Failed to create training directories: {}", e))?;

    info!("Training Paths:");
    info!("  Base directory: {:?}", args.base_dir);
    info!("  Run ID: {}", run_id);
    info!("  Run directory: {:?}", training_paths.run_dir());
    info!("  Checkpoints: {:?}", training_paths.checkpoints_dir());
    info!("");

    // Create PPO trainer with training paths
    let trainer = PPOTrainer::new(args.episodes)?.with_training_paths(training_paths);

    info!("Parameter Space:");
    let names = PPOParams::param_names();
    let bounds = PPOParams::continuous_bounds();
    for (name, (min, max)) in names.iter().zip(bounds.iter()) {
        info!("  {}: [{:.6}, {:.6}]", name, min, max);
    }
    info!("");

    // Create optimizer
    let optimizer = EgoboxOptimizer::with_trials(args.trials, 3);

    info!("Starting optimization...");
    info!("");

    // Run optimization
    let result = optimizer.optimize(trainer)?;

    info!("");
    info!("╔═══════════════════════════════════════════════════════════╗");
    info!("║   Optimization Complete                                   ║");
    info!("╚═══════════════════════════════════════════════════════════╝");
    info!("");
    info!("Best Parameters:");
    info!("  Policy LR: {:.6}", result.best_params.policy_learning_rate);
    info!("  Value LR: {:.6}", result.best_params.value_learning_rate);
    info!("  Clip epsilon: {:.3}", result.best_params.clip_epsilon);
    info!("  Value loss coeff: {:.3}", result.best_params.value_loss_coeff);
    info!("  Entropy coeff: {:.6}", result.best_params.entropy_coeff);
    info!("");
    info!("Best Objective (combined loss): {:.6}", result.best_objective);
    info!("Total Evaluations: {}", result.all_trials.len());
    info!("");

    // Print trial history for convergence analysis
    info!("Trial History:");
    info!("┌───────┬──────────────────┬──────────────────┬──────────────────┐");
    info!("│ Trial │   Policy LR      │    Value LR      │  Combined Loss   │");
    info!("├───────┼──────────────────┼──────────────────┼──────────────────┤");

    for trial in &result.all_trials {
        info!(
            "│ {:5} │ {:16.6} │ {:16.6} │ {:16.6} │",
            trial.trial_num,
            trial.params.policy_learning_rate,
            trial.params.value_learning_rate,
            trial.objective
        );
    }

    info!("└───────┴──────────────────┴──────────────────┴──────────────────┘");
    info!("");

    // Compute convergence metrics
    if result.all_trials.len() >= 2 {
        let first_loss = result.all_trials[0].objective;
        let best_loss = result.best_objective;
        let improvement = ((first_loss - best_loss) / first_loss) * 100.0;

        info!("Convergence Analysis:");
        info!("  First Trial Loss: {:.6}", first_loss);
        info!("  Best Trial Loss: {:.6}", best_loss);
        info!("  Improvement: {:.2}%", improvement);
        info!("");

        // Compute variance in loss values
        let mean_loss: f64 = result.all_trials.iter().map(|e| e.objective).sum::<f64>()
            / result.all_trials.len() as f64;
        let variance: f64 = result
            .all_trials
            .iter()
            .map(|e| (e.objective - mean_loss).powi(2))
            .sum::<f64>()
            / result.all_trials.len() as f64;
        let std_dev = variance.sqrt();
        let coeff_var = (std_dev / mean_loss) * 100.0;

        info!("Loss Variance Analysis:");
        info!("  Mean Loss: {:.6}", mean_loss);
        info!("  Std Dev: {:.6}", std_dev);
        info!("  Coefficient of Variation: {:.2}%", coeff_var);
        info!("");

        if coeff_var < 5.0 {
            info!("⚠️  WARNING: Low loss variance ({:.2}%) suggests mock metrics", coeff_var);
        } else {
            info!("✓ Loss variance ({:.2}%) confirms real training", coeff_var);
        }
    }

    info!("✓ PPO hyperparameter optimization demo complete");

    Ok(())
}