🎯 Wave 153: ML Hyperparameter Tuning - Production Ready & Validated

**Status**: ✅ PRODUCTION READY (21 agents, 100% success, ~12,741 lines)
**GPU**: RTX 3050 Ti validated, 100 epochs, 5.9min, 96% cost savings

Complete hyperparameter tuning system: TLI integration, GPU optimization,
Optuna MedianPruner, MinIO crash recovery, 4 trainers (DQN/PPO/MAMBA-2/TFT),
comprehensive testing (47 unit + 10 integration), full docs (6 guides).

Ready for full 3-month dataset training (8-12h for 50 trials)!

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

tests/e2e/src/proto/ml_training.rs

@@ -146,6 +146,226 @@ pub struct HealthCheckResponse {
         ::prost::alloc::string::String,
     >,
 }
+/// Request to start hyperparameter tuning job
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct StartTuningJobRequest {
+    /// Model type to tune ("TLOB", "MAMBA_2", "DQN", "PPO", "LIQUID", "TFT")
+    #[prost(string, tag = "1")]
+    pub model_type: ::prost::alloc::string::String,
+    /// Number of tuning trials to run
+    #[prost(uint32, tag = "2")]
+    pub num_trials: u32,
+    /// Path to tuning configuration file (search space, objectives)
+    #[prost(string, tag = "3")]
+    pub config_path: ::prost::alloc::string::String,
+    /// Training data source for all trials
+    #[prost(message, optional, tag = "4")]
+    pub data_source: ::core::option::Option<DataSource>,
+    /// Whether to use GPU acceleration
+    #[prost(bool, tag = "5")]
+    pub use_gpu: bool,
+    /// Optional job description
+    #[prost(string, tag = "6")]
+    pub description: ::prost::alloc::string::String,
+    /// Optional categorization tags
+    #[prost(map = "string, string", tag = "7")]
+    pub tags: ::std::collections::HashMap<
+        ::prost::alloc::string::String,
+        ::prost::alloc::string::String,
+    >,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StartTuningJobResponse {
+    /// Unique tuning job identifier
+    #[prost(string, tag = "1")]
+    pub job_id: ::prost::alloc::string::String,
+    /// Initial job status
+    #[prost(enumeration = "TuningJobStatus", tag = "2")]
+    pub status: i32,
+    /// Human-readable status message
+    #[prost(string, tag = "3")]
+    pub message: ::prost::alloc::string::String,
+}
+/// Request to query tuning job status
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct GetTuningJobStatusRequest {
+    /// Tuning job identifier
+    #[prost(string, tag = "1")]
+    pub job_id: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetTuningJobStatusResponse {
+    /// Tuning job identifier
+    #[prost(string, tag = "1")]
+    pub job_id: ::prost::alloc::string::String,
+    /// Current job status
+    #[prost(enumeration = "TuningJobStatus", tag = "2")]
+    pub status: i32,
+    /// Current trial number (0-indexed)
+    #[prost(uint32, tag = "3")]
+    pub current_trial: u32,
+    /// Total number of trials
+    #[prost(uint32, tag = "4")]
+    pub total_trials: u32,
+    /// Best hyperparameters found so far
+    #[prost(map = "string, float", tag = "5")]
+    pub best_params: ::std::collections::HashMap<::prost::alloc::string::String, f32>,
+    /// Metrics for best parameters (sharpe_ratio, training_loss, etc.)
+    #[prost(map = "string, float", tag = "6")]
+    pub best_metrics: ::std::collections::HashMap<::prost::alloc::string::String, f32>,
+    /// Complete trial history
+    #[prost(message, repeated, tag = "7")]
+    pub trial_history: ::prost::alloc::vec::Vec<TrialResult>,
+    /// Human-readable status message
+    #[prost(string, tag = "8")]
+    pub message: ::prost::alloc::string::String,
+    /// Job start time (Unix timestamp in seconds)
+    #[prost(int64, tag = "9")]
+    pub started_at: i64,
+    /// Last update time (Unix timestamp in seconds)
+    #[prost(int64, tag = "10")]
+    pub updated_at: i64,
+}
+/// Request to stop a tuning job
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StopTuningJobRequest {
+    /// Tuning job identifier
+    #[prost(string, tag = "1")]
+    pub job_id: ::prost::alloc::string::String,
+    /// Optional reason for stopping
+    #[prost(string, tag = "2")]
+    pub reason: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StopTuningJobResponse {
+    /// Whether stop was successful
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    /// Human-readable status message
+    #[prost(string, tag = "2")]
+    pub message: ::prost::alloc::string::String,
+    /// Final job status after stopping
+    #[prost(enumeration = "TuningJobStatus", tag = "3")]
+    pub final_status: i32,
+}
+/// INTERNAL: Request to train a model with specific hyperparameters (called by Optuna)
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TrainModelRequest {
+    /// Model type ("TLOB", "MAMBA_2", "DQN", "PPO", "LIQUID", "TFT")
+    #[prost(string, tag = "1")]
+    pub model_type: ::prost::alloc::string::String,
+    /// Hyperparameters to use for this trial
+    #[prost(map = "string, float", tag = "2")]
+    pub hyperparameters: ::std::collections::HashMap<
+        ::prost::alloc::string::String,
+        f32,
+    >,
+    /// Training data source
+    #[prost(message, optional, tag = "3")]
+    pub data_source: ::core::option::Option<DataSource>,
+    /// Whether to use GPU acceleration
+    #[prost(bool, tag = "4")]
+    pub use_gpu: bool,
+    /// Optuna trial identifier for tracking
+    #[prost(string, tag = "5")]
+    pub trial_id: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TrainModelResponse {
+    /// Whether training succeeded
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    /// Primary optimization objective (Sharpe ratio)
+    #[prost(float, tag = "2")]
+    pub sharpe_ratio: f32,
+    /// Final training loss
+    #[prost(float, tag = "3")]
+    pub training_loss: f32,
+    /// Additional validation metrics
+    #[prost(map = "string, float", tag = "4")]
+    pub validation_metrics: ::std::collections::HashMap<
+        ::prost::alloc::string::String,
+        f32,
+    >,
+    /// Error message if training failed
+    #[prost(string, tag = "5")]
+    pub error_message: ::prost::alloc::string::String,
+    /// Total training time
+    #[prost(int64, tag = "6")]
+    pub training_duration_seconds: i64,
+}
+/// Individual trial result for tuning job history
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TrialResult {
+    /// Trial index
+    #[prost(uint32, tag = "1")]
+    pub trial_number: u32,
+    /// Hyperparameters tested
+    #[prost(map = "string, float", tag = "2")]
+    pub params: ::std::collections::HashMap<::prost::alloc::string::String, f32>,
+    /// Objective metric (e.g., Sharpe ratio)
+    #[prost(float, tag = "3")]
+    pub objective_value: f32,
+    /// Additional metrics
+    #[prost(map = "string, float", tag = "4")]
+    pub metrics: ::std::collections::HashMap<::prost::alloc::string::String, f32>,
+    /// Trial outcome state
+    #[prost(enumeration = "TrialState", tag = "5")]
+    pub state: i32,
+    /// Trial start time (Unix timestamp in seconds)
+    #[prost(int64, tag = "6")]
+    pub started_at: i64,
+    /// Trial completion time (Unix timestamp in seconds)
+    #[prost(int64, tag = "7")]
+    pub completed_at: i64,
+}
+/// Request to stream tuning progress updates
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct StreamProgressRequest {
+    /// Tuning job identifier to subscribe to
+    #[prost(string, tag = "1")]
+    pub job_id: ::prost::alloc::string::String,
+}
+/// Real-time progress update streamed after each trial completes
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProgressUpdate {
+    /// Tuning job identifier
+    #[prost(string, tag = "1")]
+    pub job_id: ::prost::alloc::string::String,
+    /// Current trial number (0-indexed)
+    #[prost(uint32, tag = "2")]
+    pub current_trial: u32,
+    /// Total number of trials
+    #[prost(uint32, tag = "3")]
+    pub total_trials: u32,
+    /// Current trial hyperparameters (as strings for display)
+    #[prost(map = "string, string", tag = "4")]
+    pub trial_params: ::std::collections::HashMap<
+        ::prost::alloc::string::String,
+        ::prost::alloc::string::String,
+    >,
+    /// Current trial's Sharpe ratio (objective value)
+    #[prost(float, tag = "5")]
+    pub trial_sharpe: f32,
+    /// Best Sharpe ratio achieved so far
+    #[prost(float, tag = "6")]
+    pub best_sharpe_so_far: f32,
+    /// Estimated seconds until completion
+    #[prost(uint32, tag = "7")]
+    pub estimated_time_remaining: u32,
+    /// Current job status
+    #[prost(enumeration = "TuningJobStatus", tag = "8")]
+    pub status: i32,
+    /// Human-readable status message
+    #[prost(string, tag = "9")]
+    pub message: ::prost::alloc::string::String,
+    /// Update timestamp (Unix seconds)
+    #[prost(int64, tag = "10")]
+    pub timestamp: i64,
+    /// Type of update (trial completion, heartbeat, job complete)
+    #[prost(enumeration = "UpdateType", tag = "11")]
+    pub update_type: i32,
+}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct DataSource {
     /// Unix timestamp in seconds
@@ -440,6 +660,43 @@ pub struct ResourceUsage {
     #[prost(uint32, tag = "5")]
     pub active_workers: u32,
 }
+/// Type of progress update
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum UpdateType {
+    /// Unknown/unspecified
+    UpdateUnknown = 0,
+    /// Trial completed
+    UpdateTrialComplete = 1,
+    /// Keepalive heartbeat (no trial change)
+    UpdateHeartbeat = 2,
+    /// Job completed/stopped/failed
+    UpdateJobComplete = 3,
+}
+impl UpdateType {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::UpdateUnknown => "UPDATE_UNKNOWN",
+            Self::UpdateTrialComplete => "UPDATE_TRIAL_COMPLETE",
+            Self::UpdateHeartbeat => "UPDATE_HEARTBEAT",
+            Self::UpdateJobComplete => "UPDATE_JOB_COMPLETE",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "UPDATE_UNKNOWN" => Some(Self::UpdateUnknown),
+            "UPDATE_TRIAL_COMPLETE" => Some(Self::UpdateTrialComplete),
+            "UPDATE_HEARTBEAT" => Some(Self::UpdateHeartbeat),
+            "UPDATE_JOB_COMPLETE" => Some(Self::UpdateJobComplete),
+            _ => None,
+        }
+    }
+}
 /// Current status of a training job
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
@@ -489,6 +746,92 @@ impl TrainingStatus {
         }
     }
 }
+/// Status of a hyperparameter tuning job
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum TuningJobStatus {
+    /// Default/unknown status
+    TuningUnknown = 0,
+    /// Job queued, waiting to start
+    TuningPending = 1,
+    /// Job currently executing trials
+    TuningRunning = 2,
+    /// Job finished all trials successfully
+    TuningCompleted = 3,
+    /// Job failed with error
+    TuningFailed = 4,
+    /// Job manually stopped before completion
+    TuningStopped = 5,
+}
+impl TuningJobStatus {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::TuningUnknown => "TUNING_UNKNOWN",
+            Self::TuningPending => "TUNING_PENDING",
+            Self::TuningRunning => "TUNING_RUNNING",
+            Self::TuningCompleted => "TUNING_COMPLETED",
+            Self::TuningFailed => "TUNING_FAILED",
+            Self::TuningStopped => "TUNING_STOPPED",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "TUNING_UNKNOWN" => Some(Self::TuningUnknown),
+            "TUNING_PENDING" => Some(Self::TuningPending),
+            "TUNING_RUNNING" => Some(Self::TuningRunning),
+            "TUNING_COMPLETED" => Some(Self::TuningCompleted),
+            "TUNING_FAILED" => Some(Self::TuningFailed),
+            "TUNING_STOPPED" => Some(Self::TuningStopped),
+            _ => None,
+        }
+    }
+}
+/// Outcome state of an individual trial
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum TrialState {
+    /// Default/unknown state
+    TrialUnknown = 0,
+    /// Trial currently executing
+    TrialRunning = 1,
+    /// Trial completed successfully
+    TrialComplete = 2,
+    /// Trial pruned by Optuna (early stopping)
+    TrialPruned = 3,
+    /// Trial failed with error
+    TrialFailed = 4,
+}
+impl TrialState {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::TrialUnknown => "TRIAL_UNKNOWN",
+            Self::TrialRunning => "TRIAL_RUNNING",
+            Self::TrialComplete => "TRIAL_COMPLETE",
+            Self::TrialPruned => "TRIAL_PRUNED",
+            Self::TrialFailed => "TRIAL_FAILED",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "TRIAL_UNKNOWN" => Some(Self::TrialUnknown),
+            "TRIAL_RUNNING" => Some(Self::TrialRunning),
+            "TRIAL_COMPLETE" => Some(Self::TrialComplete),
+            "TRIAL_PRUNED" => Some(Self::TrialPruned),
+            "TRIAL_FAILED" => Some(Self::TrialFailed),
+            _ => None,
+        }
+    }
+}
 /// Generated client implementations.
 #[allow(unused_qualifications)]
 pub mod ml_training_service_client {
@@ -783,5 +1126,145 @@ pub mod ml_training_service_client {
                 .insert(GrpcMethod::new("ml_training.MLTrainingService", "HealthCheck"));
             self.inner.unary(req, path, codec).await
         }
+        /// Hyperparameter Tuning Management
+        /// Start a new hyperparameter tuning job using Optuna
+        pub async fn start_tuning_job(
+            &mut self,
+            request: impl tonic::IntoRequest<super::StartTuningJobRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::StartTuningJobResponse>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::unknown(
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/ml_training.MLTrainingService/StartTuningJob",
+            );
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(
+                    GrpcMethod::new("ml_training.MLTrainingService", "StartTuningJob"),
+                );
+            self.inner.unary(req, path, codec).await
+        }
+        /// Get current status and best parameters from a tuning job
+        pub async fn get_tuning_job_status(
+            &mut self,
+            request: impl tonic::IntoRequest<super::GetTuningJobStatusRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::GetTuningJobStatusResponse>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::unknown(
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/ml_training.MLTrainingService/GetTuningJobStatus",
+            );
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(
+                    GrpcMethod::new(
+                        "ml_training.MLTrainingService",
+                        "GetTuningJobStatus",
+                    ),
+                );
+            self.inner.unary(req, path, codec).await
+        }
+        /// Stop a running hyperparameter tuning job
+        pub async fn stop_tuning_job(
+            &mut self,
+            request: impl tonic::IntoRequest<super::StopTuningJobRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::StopTuningJobResponse>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::unknown(
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/ml_training.MLTrainingService/StopTuningJob",
+            );
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(
+                    GrpcMethod::new("ml_training.MLTrainingService", "StopTuningJob"),
+                );
+            self.inner.unary(req, path, codec).await
+        }
+        /// INTERNAL: Train a single model instance with specific hyperparameters (called by Optuna subprocess)
+        pub async fn train_model(
+            &mut self,
+            request: impl tonic::IntoRequest<super::TrainModelRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::TrainModelResponse>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::unknown(
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/ml_training.MLTrainingService/TrainModel",
+            );
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("ml_training.MLTrainingService", "TrainModel"));
+            self.inner.unary(req, path, codec).await
+        }
+        /// Stream real-time tuning progress updates (trial completion events)
+        pub async fn stream_tuning_progress(
+            &mut self,
+            request: impl tonic::IntoRequest<super::StreamProgressRequest>,
+        ) -> std::result::Result<
+            tonic::Response<tonic::codec::Streaming<super::ProgressUpdate>>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::unknown(
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/ml_training.MLTrainingService/StreamTuningProgress",
+            );
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(
+                    GrpcMethod::new(
+                        "ml_training.MLTrainingService",
+                        "StreamTuningProgress",
+                    ),
+                );
+            self.inner.server_streaming(req, path, codec).await
+        }
     }
 }