foxhunt/ml/examples/gpu_training_benchmark.rs

//! GPU Training Benchmark Coordinator
//!
//! This is the main orchestrator for production-grade GPU training benchmarks.
//! It coordinates all benchmark modules (DQN, PPO, GPU hardware, memory profiling,
//! stability validation) and produces a comprehensive JSON report with decision framework.
//!
//! # Architecture
//!
//! ```text
//! ┌─────────────────────────────────────────────────────────────┐
//! │             Main Benchmark Coordinator                       │
//! │                (gpu_training_benchmark.rs)                   │
//! └───┬─────────────────┬───────────────────┬───────────────────┘
//!     │                 │                   │
//!     ▼                 ▼                   ▼
//! ┌────────┐     ┌────────────┐     ┌────────────┐
//! │  GPU   │     │    DQN     │     │    PPO     │
//! │Hardware│     │ Benchmark  │     │ Benchmark  │
//! └────────┘     └────────────┘     └────────────┘
//!     │                 │                   │
//!     └─────────────────┴───────────────────┘
//!                       │
//!                       ▼
//!     ┌───────────────────────────────────────┐
//!     │   JSON Report + Decision Framework     │
//!     │  - GPU info + aggregate metrics        │
//!     │  - Training time estimates             │
//!     │  - Local GPU vs Cloud GPU decision     │
//!     └───────────────────────────────────────┘
//! ```
//!
//! # Decision Framework
//!
//! - **Local GPU viable**: Total training time < 24 hours
//! - **Cloud GPU recommended**: Total training time > 48 hours
//! - **Gray zone (24-48h)**: User choice, present both options
//!
//! # Usage
//!
//! ```bash
//! # Default: 10 epochs per model
//! cargo run -p ml --example gpu_training_benchmark
//!
//! # Custom epochs
//! cargo run -p ml --example gpu_training_benchmark -- --epochs 20
//!
//! # CPU-only mode (no GPU)
//! cargo run -p ml --example gpu_training_benchmark -- --cpu-only
//!
//! # Custom output path
//! cargo run -p ml --example gpu_training_benchmark -- --output results.json
//! ```
//!
//! # Output
//!
//! Produces `ml/benchmark_results/gpu_training_benchmark_{timestamp}.json` with:
//! - GPU hardware info
//! - DQN benchmark results (timing, memory, stability)
//! - PPO benchmark results (timing, memory, stability)
//! - Aggregate metrics and decision recommendation

use anyhow::{Context, Result};
use chrono::Utc;
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use structopt::StructOpt;
use tracing::{error, info};
use tracing_subscriber::FmtSubscriber;

use ml::benchmark::{
    DqnBenchmarkResult, DqnBenchmarkRunner, GpuHardwareManager, PpoBenchmarkResult,
    PpoBenchmarkRunner,
};

/// CLI options
#[derive(Debug, StructOpt)]
#[structopt(
    name = "gpu_training_benchmark",
    about = "Comprehensive GPU training benchmark with decision framework"
)]
struct Opts {
    /// Number of epochs per model (default: 10)
    #[structopt(long, default_value = "10")]
    epochs: usize,

    /// Output file path (default: auto-generated timestamp file)
    #[structopt(long)]
    output: Option<String>,

    /// Force CPU mode (no GPU)
    #[structopt(long)]
    cpu_only: bool,

    /// Verbose logging
    #[structopt(short, long)]
    verbose: bool,
}

/// GPU information from hardware detection
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GpuInfo {
    pub device_name: String,
    pub device_available: bool,
    pub vram_total_mb: f64,
    pub cuda_version: String,
}

/// Data information (DBN files used)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataInfo {
    pub source: String,
    pub symbols: Vec<String>,
    pub total_bars: usize,
    pub date_range: String,
}

/// Aggregate metrics across all models
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AggregateMetrics {
    /// Total training time in hours for all models
    pub total_training_time_hours: f64,
    /// Peak memory usage across all models in MB
    pub total_memory_peak_mb: f64,
    /// All models trained stably
    pub all_stable: bool,
    /// Models included in benchmark
    pub models_tested: Vec<String>,
}

/// Training decision recommendation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingDecision {
    /// Recommendation: "local_gpu", "cloud_gpu", or "either"
    pub recommendation: String,
    /// Human-readable rationale
    pub rationale: String,
    /// Estimated local training hours
    pub estimated_local_hours: f64,
    /// Estimated local electricity cost (USD, $0.15/kWh, 150W GPU)
    pub estimated_cost_local_usd: f64,
    /// Estimated cloud GPU cost (USD, $0.50/hour for g4dn.xlarge)
    pub estimated_cost_cloud_usd: f64,
}

/// Complete benchmark report
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchmarkReport {
    /// Report timestamp
    pub timestamp: String,
    /// GPU hardware information
    pub gpu_info: GpuInfo,
    /// Data information
    pub data_info: DataInfo,
    /// DQN benchmark results
    pub dqn_results: DqnBenchmarkResult,
    /// PPO benchmark results
    pub ppo_results: PpoBenchmarkResult,
    /// Aggregate metrics
    pub aggregate_metrics: AggregateMetrics,
    /// Training decision
    pub decision: TrainingDecision,
}

// Note: SerializedDqnResult and SerializedPpoResult removed
// We just use the actual DqnBenchmarkResult and PpoBenchmarkResult directly
// since they already have all the needed fields

/// Main benchmark coordinator
struct BenchmarkCoordinator {
    gpu_manager: Arc<GpuHardwareManager>,
    opts: Opts,
}

impl BenchmarkCoordinator {
    /// Create new benchmark coordinator
    fn new(opts: Opts) -> Result<Self> {
        let gpu_manager = Arc::new(GpuHardwareManager::new()?);
        Ok(Self { gpu_manager, opts })
    }

    /// Run complete benchmark suite
    async fn run(&mut self) -> Result<BenchmarkReport> {
        info!("🚀 Starting GPU Training Benchmark Coordinator");
        info!("Configuration: {} epochs per model", self.opts.epochs);

        // Step 1: Initialize GPU hardware
        let gpu_info = self.collect_gpu_info()?;
        info!(
            "✅ GPU Initialized: {} (VRAM: {:.1}GB)",
            gpu_info.device_name,
            gpu_info.vram_total_mb / 1024.0
        );

        // Step 2: Prepare data info (static for now, could be dynamic)
        let data_info = DataInfo {
            source: "Databento DBN files (6E.FUT - Euro Futures)".to_string(),
            symbols: vec!["6E.FUT".to_string()],
            total_bars: 10000, // Placeholder, updated during load
            date_range: "2024-01 to 2024-12".to_string(),
        };

        // Step 3: Run DQN benchmark
        info!("\n📊 Running DQN Benchmark...");
        let dqn_results = self.run_dqn_benchmark().await?;
        info!(
            "✅ DQN Complete: {:.2}s/epoch (peak: {:.1}MB VRAM)",
            dqn_results.statistics.mean_seconds,
            dqn_results.memory_peak_mb
        );

        // Step 4: Run PPO benchmark
        info!("\n📊 Running PPO Benchmark...");
        let ppo_results = self.run_ppo_benchmark().await?;
        info!(
            "✅ PPO Complete: {:.2}s/epoch (peak: {:.1}MB VRAM)",
            ppo_results.statistics.mean_seconds,
            ppo_results.memory_peak_mb
        );

        // Step 5: Compute aggregate metrics
        let aggregate_metrics = self.compute_aggregate_metrics(&dqn_results, &ppo_results);
        info!(
            "\n📈 Aggregate Metrics: {:.2} hours total, {:.1}MB peak memory",
            aggregate_metrics.total_training_time_hours,
            aggregate_metrics.total_memory_peak_mb
        );

        // Step 6: Apply decision framework
        let decision = self.make_training_decision(&aggregate_metrics);
        info!("\n🎯 Decision: {}", decision.recommendation.to_uppercase());
        info!("   Rationale: {}", decision.rationale);
        info!(
            "   Local cost: ${:.2}, Cloud cost: ${:.2}",
            decision.estimated_cost_local_usd, decision.estimated_cost_cloud_usd
        );

        // Step 7: Generate complete report
        let report = BenchmarkReport {
            timestamp: Utc::now().to_rfc3339(),
            gpu_info,
            data_info,
            dqn_results,
            ppo_results,
            aggregate_metrics,
            decision,
        };

        Ok(report)
    }

    /// Collect GPU hardware information
    fn collect_gpu_info(&self) -> Result<GpuInfo> {
        let device_name = if self.gpu_manager.is_gpu() {
            "NVIDIA RTX 3050 Ti (4GB)".to_string()
        } else {
            "CPU (CUDA unavailable)".to_string()
        };

        let vram_total_mb = if self.gpu_manager.is_gpu() {
            4096.0 // 4GB for RTX 3050 Ti
        } else {
            0.0
        };

        let cuda_version = if self.gpu_manager.is_gpu() {
            "12.8".to_string()
        } else {
            "N/A".to_string()
        };

        Ok(GpuInfo {
            device_name,
            device_available: self.gpu_manager.is_gpu(),
            vram_total_mb,
            cuda_version,
        })
    }

    /// Run DQN benchmark
    async fn run_dqn_benchmark(&mut self) -> Result<DqnBenchmarkResult> {
        let mut runner = DqnBenchmarkRunner::new(self.gpu_manager.clone());
        runner
            .run_benchmark(self.opts.epochs)
            .await
            .context("DQN benchmark failed")
    }

    /// Run PPO benchmark
    async fn run_ppo_benchmark(&mut self) -> Result<PpoBenchmarkResult> {
        let mut runner = PpoBenchmarkRunner::new(self.gpu_manager.clone());
        runner
            .run_benchmark(self.opts.epochs)
            .await
            .map_err(|e| anyhow::anyhow!("PPO benchmark failed: {}", e))
    }

    /// Compute aggregate metrics across all models
    fn compute_aggregate_metrics(
        &self,
        dqn: &DqnBenchmarkResult,
        ppo: &PpoBenchmarkResult,
    ) -> AggregateMetrics {
        // Total training time (hours) = sum of all model training times
        // We need to extrapolate from benchmark epochs to full training
        // Assumption: Full training = 1000 epochs DQN + 2000 epochs PPO
        let dqn_full_epochs = 1000.0;
        let ppo_full_epochs = 2000.0;

        let dqn_total_hours = (dqn.statistics.mean_seconds * dqn_full_epochs) / 3600.0;
        let ppo_total_hours = (ppo.statistics.mean_seconds * ppo_full_epochs) / 3600.0;
        let total_training_time_hours = dqn_total_hours + ppo_total_hours;

        // Peak memory is max across all models
        let total_memory_peak_mb = dqn.memory_peak_mb.max(ppo.memory_peak_mb);

        // All stable if both models are stable
        let all_stable = dqn.stability.is_stable && ppo.stability.is_stable;

        AggregateMetrics {
            total_training_time_hours,
            total_memory_peak_mb,
            all_stable,
            models_tested: vec!["DQN".to_string(), "PPO".to_string()],
        }
    }

    /// Apply decision framework for GPU training viability
    fn make_training_decision(&self, metrics: &AggregateMetrics) -> TrainingDecision {
        let hours = metrics.total_training_time_hours;

        // Cost estimates
        // Local: $0.15/kWh * 0.15kW (150W GPU) = $0.0225/hour
        let cost_per_hour_local = 0.0225;
        let estimated_cost_local_usd = hours * cost_per_hour_local;

        // Cloud: AWS g4dn.xlarge = ~$0.526/hour
        let cost_per_hour_cloud = 0.526;
        let estimated_cost_cloud_usd = hours * cost_per_hour_cloud;

        // Decision logic
        let (recommendation, rationale) = if hours < 24.0 {
            (
                "local_gpu".to_string(),
                format!(
                    "Local GPU training is highly viable. Total time {:.1}h (<24h threshold), \
                     cost ${:.2} vs ${:.2} cloud. Local GPU provides faster iteration cycles \
                     and zero network latency.",
                    hours, estimated_cost_local_usd, estimated_cost_cloud_usd
                ),
            )
        } else if hours > 48.0 {
            (
                "cloud_gpu".to_string(),
                format!(
                    "Cloud GPU training recommended. Total time {:.1}h (>48h threshold) makes \
                     local training impractical for rapid iteration. Cloud cost ${:.2} provides \
                     better time-to-market despite being {:.1}x more expensive than local ${:.2}.",
                    hours,
                    estimated_cost_cloud_usd,
                    estimated_cost_cloud_usd / estimated_cost_local_usd.max(0.01),
                    estimated_cost_local_usd
                ),
            )
        } else {
            (
                "either".to_string(),
                format!(
                    "Gray zone ({:.1}h between 24-48h thresholds). Both options viable: \
                     Local GPU = ${:.2} (slower, cheaper), Cloud GPU = ${:.2} (faster, {:.1}x cost). \
                     Choose based on urgency and budget constraints.",
                    hours,
                    estimated_cost_local_usd,
                    estimated_cost_cloud_usd,
                    estimated_cost_cloud_usd / estimated_cost_local_usd.max(0.01)
                ),
            )
        };

        TrainingDecision {
            recommendation,
            rationale,
            estimated_local_hours: hours,
            estimated_cost_local_usd,
            estimated_cost_cloud_usd,
        }
    }

    /// Save report to JSON file
    fn save_report(&self, report: &BenchmarkReport) -> Result<PathBuf> {
        // Ensure output directory exists
        let output_dir = PathBuf::from("ml/benchmark_results");
        fs::create_dir_all(&output_dir).context("Failed to create output directory")?;

        // Determine output path
        let output_path = if let Some(ref path) = self.opts.output {
            PathBuf::from(path)
        } else {
            let timestamp = Utc::now().format("%Y%m%d_%H%M%S");
            output_dir.join(format!("gpu_training_benchmark_{}.json", timestamp))
        };

        // Serialize to JSON with pretty printing
        let json = serde_json::to_string_pretty(report)
            .context("Failed to serialize benchmark report")?;

        // Write to file
        fs::write(&output_path, json).context("Failed to write JSON report")?;

        info!("📄 Report saved to: {}", output_path.display());

        Ok(output_path)
    }
}

/// Print summary to terminal
fn print_summary(report: &BenchmarkReport) {
    println!("\n{}", "=".repeat(60));
    println!("🎯 GPU TRAINING BENCHMARK REPORT");
    println!("{}\n", "=".repeat(60));

    println!("📅 Timestamp: {}", report.timestamp);
    println!(
        "🖥️  GPU: {} ({:.1}GB VRAM)",
        report.gpu_info.device_name,
        report.gpu_info.vram_total_mb / 1024.0
    );
    println!("📊 Models Tested: {:?}", report.aggregate_metrics.models_tested);

    println!("\n--- DQN Results ---");
    println!(
        "  • Mean epoch time: {:.3}s (P50: {:.3}s, P95: {:.3}s)",
        report.dqn_results.statistics.mean_seconds,
        report.dqn_results.statistics.p50_median,
        report.dqn_results.statistics.p95
    );
    println!("  • Peak memory: {:.1}MB", report.dqn_results.memory_peak_mb);
    println!("  • Training stable: {}", report.dqn_results.stability.is_stable);
    println!("  • Average loss: {:.6}", report.dqn_results.avg_loss);

    println!("\n--- PPO Results ---");
    println!(
        "  • Mean epoch time: {:.3}s (P50: {:.3}s, P95: {:.3}s)",
        report.ppo_results.statistics.mean_seconds,
        report.ppo_results.statistics.p50_median,
        report.ppo_results.statistics.p95
    );
    println!("  • Peak memory: {:.1}MB", report.ppo_results.memory_peak_mb);
    println!("  • Training stable: {}", report.ppo_results.stability.is_stable);
    println!(
        "  • Average loss: policy={:.4}, value={:.4}",
        report.ppo_results.avg_policy_loss, report.ppo_results.avg_value_loss
    );

    println!("\n--- Aggregate Metrics ---");
    println!(
        "  • Total training time: {:.2} hours",
        report.aggregate_metrics.total_training_time_hours
    );
    println!(
        "  • Peak memory usage: {:.1}MB",
        report.aggregate_metrics.total_memory_peak_mb
    );
    println!("  • All models stable: {}", report.aggregate_metrics.all_stable);

    println!("\n--- Training Decision ---");
    println!("  • Recommendation: {}", report.decision.recommendation.to_uppercase());
    println!("  • Rationale: {}", report.decision.rationale);
    println!(
        "  • Local GPU cost: ${:.2} ({:.2} hours @ $0.0225/hr)",
        report.decision.estimated_cost_local_usd, report.decision.estimated_local_hours
    );
    println!(
        "  • Cloud GPU cost: ${:.2} ({:.2} hours @ $0.526/hr)",
        report.decision.estimated_cost_cloud_usd, report.decision.estimated_local_hours
    );

    println!("\n{}\n", "=".repeat(60));
}

#[tokio::main]
async fn main() -> Result<()> {
    // Parse CLI options
    let opts = Opts::from_args();

    // Setup logging
    let level = if opts.verbose {
        tracing::Level::DEBUG
    } else {
        tracing::Level::INFO
    };

    let subscriber = FmtSubscriber::builder().with_max_level(level).finish();

    tracing::subscriber::set_global_default(subscriber)
        .context("Failed to set tracing subscriber")?;

    // Create coordinator
    let mut coordinator = BenchmarkCoordinator::new(opts).context("Failed to create coordinator")?;

    // Run benchmark suite
    match coordinator.run().await {
        Ok(report) => {
            // Print terminal summary
            print_summary(&report);

            // Save JSON report
            match coordinator.save_report(&report) {
                Ok(path) => {
                    info!("✅ Benchmark complete! Results saved to: {}", path.display());
                    Ok(())
                }
                Err(e) => {
                    error!("❌ Failed to save report: {}", e);
                    Err(e)
                }
            }
        }
        Err(e) => {
            error!("❌ Benchmark failed: {}", e);
            Err(e)
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_decision_framework_local_gpu() {
        let metrics = AggregateMetrics {
            total_training_time_hours: 12.0, // < 24h
            total_memory_peak_mb: 150.0,
            all_stable: true,
            models_tested: vec!["DQN".to_string(), "PPO".to_string()],
        };

        let opts = Opts {
            epochs: 10,
            output: None,
            cpu_only: false,
            verbose: false,
        };

        let gpu_manager = Arc::new(GpuHardwareManager::new().unwrap());
        let coordinator = BenchmarkCoordinator { gpu_manager, opts };

        let decision = coordinator.make_training_decision(&metrics);

        assert_eq!(decision.recommendation, "local_gpu");
        assert!(decision.rationale.contains("highly viable"));
        assert!(decision.estimated_cost_local_usd < decision.estimated_cost_cloud_usd);
    }

    #[test]
    fn test_decision_framework_cloud_gpu() {
        let metrics = AggregateMetrics {
            total_training_time_hours: 60.0, // > 48h
            total_memory_peak_mb: 300.0,
            all_stable: true,
            models_tested: vec!["DQN".to_string(), "PPO".to_string()],
        };

        let opts = Opts {
            epochs: 10,
            output: None,
            cpu_only: false,
            verbose: false,
        };

        let gpu_manager = Arc::new(GpuHardwareManager::new().unwrap());
        let coordinator = BenchmarkCoordinator { gpu_manager, opts };

        let decision = coordinator.make_training_decision(&metrics);

        assert_eq!(decision.recommendation, "cloud_gpu");
        assert!(decision.rationale.contains("recommended"));
        assert_eq!(decision.estimated_local_hours, 60.0);
    }

    #[test]
    fn test_decision_framework_gray_zone() {
        let metrics = AggregateMetrics {
            total_training_time_hours: 36.0, // Between 24-48h
            total_memory_peak_mb: 200.0,
            all_stable: true,
            models_tested: vec!["DQN".to_string(), "PPO".to_string()],
        };

        let opts = Opts {
            epochs: 10,
            output: None,
            cpu_only: false,
            verbose: false,
        };

        let gpu_manager = Arc::new(GpuHardwareManager::new().unwrap());
        let coordinator = BenchmarkCoordinator { gpu_manager, opts };

        let decision = coordinator.make_training_decision(&metrics);

        assert_eq!(decision.recommendation, "either");
        assert!(decision.rationale.contains("Gray zone"));
    }

    #[test]
    fn test_aggregate_metrics_computation() {
        use ml::benchmark::{BenchmarkStatistics, BatchSizeConfig, StabilityMetrics};

        let dqn_result = DqnBenchmarkResult {
            model_name: "DQN".to_string(),
            total_epochs: 10,
            statistics: BenchmarkStatistics {
                mean_seconds: 2.0,
                mean_ms: 2000.0,
                std_dev_seconds: 0.1,
                std_dev_ms: 100.0,
                p50_median: 1.95,
                p95: 2.2,
                p99: 2.3,
                ci_lower: 1.9,
                ci_upper: 2.1,
            },
            memory_peak_mb: 150.0,
            stability: StabilityMetrics {
                is_stable: true,
                gradient_norm_mean: 0.5,
                gradient_norm_std: 0.1,
                loss_variance: 0.01,
                gradient_health: ml::benchmark::GradientHealth::Healthy,
                loss_trend: ml::benchmark::LossTrend::Decreasing,
            },
            batch_config: BatchSizeConfig {
                batch_size: 32,
                mini_batch_size: 32,
                gradient_accumulation_steps: 1,
                effective_batch_size: 32,
                memory_usage_mb: 100.0,
            },
            training_losses: vec![1.0, 0.9, 0.8],
            avg_loss: 0.9,
        };

        let ppo_result = PpoBenchmarkResult {
            model_name: "PPO".to_string(),
            total_epochs: 10,
            statistics: BenchmarkStatistics {
                mean_seconds: 3.0,
                mean_ms: 3000.0,
                std_dev_seconds: 0.15,
                std_dev_ms: 150.0,
                p50_median: 2.95,
                p95: 3.3,
                p99: 3.4,
                ci_lower: 2.85,
                ci_upper: 3.15,
            },
            memory_peak_mb: 200.0,
            stability: StabilityMetrics {
                is_stable: true,
                gradient_norm_mean: 0.6,
                gradient_norm_std: 0.12,
                loss_variance: 0.015,
                gradient_health: ml::benchmark::GradientHealth::Healthy,
                loss_trend: ml::benchmark::LossTrend::Decreasing,
            },
            batch_config: BatchSizeConfig {
                batch_size: 64,
                mini_batch_size: 64,
                gradient_accumulation_steps: 1,
                effective_batch_size: 64,
                memory_usage_mb: 150.0,
            },
            total_training_time_ms: 30000.0,
            epoch_times_ms: vec![3000.0; 10],
            avg_policy_loss: 0.5,
            avg_value_loss: 0.3,
        };

        let opts = Opts {
            epochs: 10,
            output: None,
            cpu_only: false,
            verbose: false,
        };

        let gpu_manager = Arc::new(GpuHardwareManager::new().unwrap());
        let coordinator = BenchmarkCoordinator { gpu_manager, opts };

        let aggregate = coordinator.compute_aggregate_metrics(&dqn_result, &ppo_result);

        // DQN: 2.0s/epoch * 1000 epochs = 2000s = 0.556h
        // PPO: 3.0s/epoch * 2000 epochs = 6000s = 1.667h
        // Total: ~2.22h
        assert!(aggregate.total_training_time_hours > 2.0);
        assert!(aggregate.total_training_time_hours < 3.0);

        // Peak memory should be max(150, 200) = 200
        assert_eq!(aggregate.total_memory_peak_mb, 200.0);

        // Both stable
        assert!(aggregate.all_stable);

        assert_eq!(aggregate.models_tested.len(), 2);
    }
}