# Comprehensive PPO Hyperparameter Tuning Configuration
# Mission: Optimize PPO for better explained variance and Sharpe ratio
# Target: Improve on Epoch 380 baseline (expl_var=0.4469)

# Global tuning settings
global:
  optimization_direction: maximize  # maximize combined objective
  pruning_enabled: true
  median_pruner:
    n_startup_trials: 5      # No pruning for first 5 trials (establish baseline)
    n_warmup_steps: 10       # Wait 10 epochs before starting to prune
    interval_steps: 5        # Check for pruning every 5 epochs
  sampler: TPE  # Tree-structured Parzen Estimator (best for continuous spaces)

# Combined objective function
objective:
  type: composite
  # Combined metric: 0.7 * sharpe_ratio + 0.3 * explained_variance
  formula: "0.7 * sharpe + 0.3 * explained_var"
  metrics:
    - sharpe_ratio        # Primary metric: risk-adjusted returns
    - explained_variance  # Secondary metric: value network accuracy

# Early stopping configuration
early_stopping:
  enabled: true
  max_epochs_per_trial: 50  # Stop trials at epoch 50 for efficiency
  min_improvement_threshold: 0.02  # 2% minimum improvement to continue
  patience_epochs: 10  # Wait 10 epochs for improvement before stopping

# Validation dataset
validation:
  symbols:
    - "6E.FUT"  # Euro FX Futures (primary, 1,661 bars)
    - "ZN.FUT"  # Treasury futures (28,935 bars)
    - "ES.FUT"  # E-mini S&P 500 (1,674 bars)
    - "NQ.FUT"  # Nasdaq futures
  split_ratio: 0.8  # 80% train, 20% validation
  cross_validation: false  # Too expensive for 50 trials

# PPO-specific search space (based on user requirements)
models:
  PPO:
    # Core hyperparameters
    learning_rate:
      type: float
      values: [0.0001, 0.0003, 0.001]  # Categorical choices
      description: "Learning rate for both actor and critic"

    batch_size:
      type: categorical
      choices: [32, 64, 128, 256]
      description: "Training batch size (GPU validated up to 230)"

    gamma:
      type: categorical
      choices: [0.95, 0.99]
      description: "Discount factor for future rewards"

    gae_lambda:
      type: categorical
      choices: [0.9, 0.95, 0.98]
      description: "GAE lambda for advantage estimation"

    clip_epsilon:
      type: categorical
      choices: [0.1, 0.2, 0.3]
      description: "PPO clipping parameter (0.2 is standard)"

    entropy_coef:
      type: categorical
      choices: [0.001, 0.01, 0.1]
      description: "Entropy coefficient for exploration"

    # Network architecture (fixed for consistency)
    policy_hidden_dims:
      type: fixed
      value: [128, 64]
      description: "Policy network architecture"

    value_hidden_dims:
      type: fixed
      value: [128, 64]
      description: "Value network architecture"

    # Training configuration
    epochs:
      type: fixed
      value: 50  # Early stopping at epoch 50
      description: "Max epochs per trial (early stopping)"

    rollout_steps:
      type: fixed
      value: 2048
      description: "Steps per rollout (balance data/compute)"

    minibatch_size:
      type: fixed
      value: 64
      description: "Mini-batch size for PPO updates"

    num_ppo_epochs:
      type: fixed
      value: 10
      description: "Number of PPO update epochs per rollout"

    # Fixed parameters (from Agent 32 fix)
    value_loss_coef:
      type: fixed
      value: 1.0
      description: "Value function coefficient (prioritize value learning)"

    max_grad_norm:
      type: fixed
      value: 0.5
      description: "Gradient clipping threshold"

# Hardware configuration
hardware:
  use_gpu: true
  gpu_device: 0
  gpu_memory_limit: 4096  # 4GB VRAM (RTX 3050 Ti)
  batch_size_limit: 230   # GPU-validated max batch size

# Output configuration
output:
  checkpoint_frequency: 10  # Save checkpoint every 10 epochs
  log_frequency: 1          # Log metrics every epoch
  save_best_only: true      # Only save checkpoints with improved objective
  output_dir: "ml/trained_models/tuning/ppo_comprehensive"
  study_name: "ppo_comprehensive_tuning_50trials"

# Baseline comparison (from Agent 79 analysis)
baseline:
  model: "Epoch 380"
  explained_variance: 0.4469
  sharpe_ratio: null  # To be measured during tuning
  checkpoint_path: "ml/trained_models/production/ppo_real_data/ppo_actor_epoch_380.safetensors"
  description: "Best checkpoint from 500-epoch production training"

# Experiment tracking
experiment:
  name: "PPO Comprehensive Hyperparameter Tuning"
  description: "50-trial Optuna optimization with early stopping at epoch 50"
  tags:
    - "ppo"
    - "hyperparameter-tuning"
    - "optuna"
    - "multi-symbol-validation"
    - "early-stopping"
  notes: |
    Mission: Improve PPO performance beyond Epoch 380 baseline.
    Search space: 6 hyperparameters (3^1 * 4^1 * 2^4 = 192 combinations)
    Objective: 0.7 * sharpe_ratio + 0.3 * explained_variance
    Expected duration: 8-12 hours (50 trials * 10-15 min per trial)
    GPU: RTX 3050 Ti (CUDA 12.1)
    Validation: All 4 symbols (6E, ZN, ES, NQ)

# Performance expectations
performance:
  expected_trial_duration: "10-15 minutes"
  total_estimated_duration: "8-12 hours"
  trials_per_hour: 4-6
  early_stopping_savings: "30-40% time reduction"
  target_improvement: ">5% over baseline (combined objective)"

# Risk management
risk:
  max_loss_threshold: -0.5  # Stop trial if loss explodes
  nan_detection: true       # Auto-fail on NaN/Inf values
  policy_collapse_detection: true  # Monitor KL divergence
  value_network_divergence_check: true  # Monitor explained variance

# Post-tuning analysis
analysis:
  generate_plots: true
  plot_types:
    - "optimization_history"
    - "param_importances"
    - "parallel_coordinate"
    - "contour_plot"
  compare_with_baseline: true
  generate_report: true
  report_format: "markdown"