🎯 Wave 153: ML Hyperparameter Tuning - Production Ready & Validated
**Status**: ✅ PRODUCTION READY (21 agents, 100% success, ~12,741 lines) **GPU**: RTX 3050 Ti validated, 100 epochs, 5.9min, 96% cost savings Complete hyperparameter tuning system: TLI integration, GPU optimization, Optuna MedianPruner, MinIO crash recovery, 4 trainers (DQN/PPO/MAMBA-2/TFT), comprehensive testing (47 unit + 10 integration), full docs (6 guides). Ready for full 3-month dataset training (8-12h for 50 trials)! 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,11 @@ use crate::ml_training::{
|
||||
ListTrainingJobsRequest, ListTrainingJobsResponse,
|
||||
GetTrainingJobDetailsRequest, GetTrainingJobDetailsResponse,
|
||||
HealthCheckRequest, HealthCheckResponse,
|
||||
// Tuning job types
|
||||
StartTuningJobRequest, StartTuningJobResponse,
|
||||
GetTuningJobStatusRequest, GetTuningJobStatusResponse,
|
||||
StopTuningJobRequest, StopTuningJobResponse,
|
||||
TrainModelRequest, TrainModelResponse,
|
||||
};
|
||||
|
||||
/// ML Training Service Proxy
|
||||
@@ -221,16 +226,134 @@ impl MlTrainingService for MlTrainingProxy {
|
||||
request: Request<HealthCheckRequest>,
|
||||
) -> Result<Response<HealthCheckResponse>, Status> {
|
||||
info!("Proxying HealthCheck request");
|
||||
|
||||
|
||||
let mut client = self.client.clone();
|
||||
let response = client.health_check(request).await.map_err(|e| {
|
||||
warn!("Backend HealthCheck failed: {}", e);
|
||||
e
|
||||
})?;
|
||||
|
||||
|
||||
info!("HealthCheck request forwarded successfully");
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Start a new hyperparameter tuning job
|
||||
///
|
||||
/// # Performance
|
||||
/// - Zero-copy message forwarding
|
||||
///
|
||||
/// - Routing overhead target: <10μs
|
||||
///
|
||||
/// # Security
|
||||
/// - Requires "ml.tune" permission in JWT metadata
|
||||
/// - JWT validation handled by interceptor layer
|
||||
#[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)]
|
||||
async fn start_tuning_job(
|
||||
&self,
|
||||
request: Request<StartTuningJobRequest>,
|
||||
) -> Result<Response<StartTuningJobResponse>, Status> {
|
||||
info!("Proxying StartTuningJob request");
|
||||
|
||||
// Clone client (cheap Arc increment) for concurrent request handling
|
||||
let mut client = self.client.clone();
|
||||
|
||||
// Forward request with zero-copy
|
||||
// Note: JWT validation and "ml.tune" permission check handled by interceptor
|
||||
let response = client.start_tuning_job(request).await.map_err(|e| {
|
||||
error!("Backend StartTuningJob failed: {}", e);
|
||||
e
|
||||
})?;
|
||||
|
||||
info!("StartTuningJob request forwarded successfully");
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Get status and best parameters from a tuning job
|
||||
///
|
||||
/// # Performance
|
||||
/// - Zero-copy message forwarding
|
||||
///
|
||||
/// - Routing overhead target: <10μs
|
||||
///
|
||||
/// # Security
|
||||
/// - Job ownership validation handled by backend service
|
||||
/// - User can only query their own tuning jobs
|
||||
#[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)]
|
||||
async fn get_tuning_job_status(
|
||||
&self,
|
||||
request: Request<GetTuningJobStatusRequest>,
|
||||
) -> Result<Response<GetTuningJobStatusResponse>, Status> {
|
||||
info!("Proxying GetTuningJobStatus request");
|
||||
|
||||
let mut client = self.client.clone();
|
||||
|
||||
// Forward request - backend validates job ownership via user_id from JWT
|
||||
let response = client.get_tuning_job_status(request).await.map_err(|e| {
|
||||
error!("Backend GetTuningJobStatus failed: {}", e);
|
||||
e
|
||||
})?;
|
||||
|
||||
info!("GetTuningJobStatus request forwarded successfully");
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Stop a running hyperparameter tuning job
|
||||
///
|
||||
/// # Performance
|
||||
/// - Zero-copy message forwarding
|
||||
///
|
||||
/// - Routing overhead target: <10μs
|
||||
///
|
||||
/// # Security
|
||||
/// - Job ownership validation handled by backend service
|
||||
/// - User can only stop their own tuning jobs
|
||||
#[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)]
|
||||
async fn stop_tuning_job(
|
||||
&self,
|
||||
request: Request<StopTuningJobRequest>,
|
||||
) -> Result<Response<StopTuningJobResponse>, Status> {
|
||||
info!("Proxying StopTuningJob request");
|
||||
|
||||
let mut client = self.client.clone();
|
||||
|
||||
// Forward stop request - backend validates job ownership via user_id from JWT
|
||||
let response = client.stop_tuning_job(request).await.map_err(|e| {
|
||||
error!("Backend StopTuningJob failed: {}", e);
|
||||
e
|
||||
})?;
|
||||
|
||||
info!("StopTuningJob request forwarded successfully");
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// INTERNAL: Train a single model instance with specific hyperparameters
|
||||
///
|
||||
/// # Performance
|
||||
/// - Zero-copy message forwarding
|
||||
///
|
||||
/// - Routing overhead target: <10μs
|
||||
///
|
||||
/// # Note
|
||||
/// - Called by Optuna subprocess during tuning trials
|
||||
/// - Not typically called directly by clients
|
||||
#[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)]
|
||||
async fn train_model(
|
||||
&self,
|
||||
request: Request<TrainModelRequest>,
|
||||
) -> Result<Response<TrainModelResponse>, Status> {
|
||||
info!("Proxying TrainModel request (INTERNAL)");
|
||||
|
||||
let mut client = self.client.clone();
|
||||
|
||||
// Forward internal training request
|
||||
let response = client.train_model(request).await.map_err(|e| {
|
||||
error!("Backend TrainModel failed: {}", e);
|
||||
e
|
||||
})?;
|
||||
|
||||
info!("TrainModel request forwarded successfully");
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
Reference in New Issue
Block a user