Extend ml_training_service to dispatch GPU training jobs as K8s batch/v1 Jobs, collect results via a Rust sidecar uploader, and support model promotion with operator approval via fxt CLI. - K8s dispatcher creates Jobs on gpu-training pool with native sidecar - training_uploader crate: watches DONE/FAILED marker, uploads to S3, reports completion via ReportJobCompletion gRPC - PromotionManager compares metrics, queues better models for approval - 4 new proto RPCs: ReportJobCompletion, ListPendingPromotions, ApprovePromotion, RejectPromotion - fxt commands: train start, model list/approve/reject - Training binaries write DONE/FAILED markers + metrics.json - Dockerfile, K8s job template, and CI pipeline updated - StartTraining gracefully falls back to in-process when outside K8s - 27 new tests (16 service + 11 promotion), 141 total service tests pass Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
124 lines
3.7 KiB
Rust
124 lines
3.7 KiB
Rust
#![allow(missing_docs)] // Internal implementation details
|
|
// ML training service domain lints: generated protobuf code and training orchestration
|
|
#![allow(clippy::mixed_attributes_style)] // Generated protobuf code uses mixed attributes
|
|
#![allow(clippy::manual_clamp)] // Explicit min/max preferred in training parameters
|
|
#![allow(clippy::too_many_arguments)] // Training functions need many hyperparameters
|
|
#![allow(clippy::type_complexity)] // Complex types in async service handlers
|
|
#![allow(clippy::wildcard_in_or_patterns)] // Wildcard patterns in job queue matching
|
|
#![allow(clippy::redundant_pattern_matching)] // Explicit pattern matching preferred
|
|
#![allow(clippy::while_let_loop)] // Explicit loop with break for service processing
|
|
#![allow(clippy::doc_lazy_continuation)] // Doc formatting acceptable
|
|
//! ML Training Service Library
|
|
//!
|
|
//! This library provides the core functionality for the ML Training Service,
|
|
//! including training orchestration, job management, and gRPC API implementation.
|
|
#![deny(unsafe_code)]
|
|
#![deny(clippy::unwrap_used, clippy::expect_used)]
|
|
|
|
pub mod asset_parser;
|
|
pub mod batch_tuning_manager;
|
|
pub mod checkpoint_manager;
|
|
pub mod data_config;
|
|
pub mod data_loader;
|
|
pub mod data_file_discovery;
|
|
pub mod database;
|
|
pub mod dbn_data_loader;
|
|
pub mod deployment_pipeline;
|
|
pub mod encryption;
|
|
pub mod ensemble_training_coordinator;
|
|
pub mod gpu_config;
|
|
pub mod gpu_resource_manager;
|
|
pub mod grpc;
|
|
pub mod grpc_tuning_handlers;
|
|
pub mod job_tracker;
|
|
pub mod k8s_dispatcher;
|
|
pub mod job_queue;
|
|
pub mod job_spawner;
|
|
pub mod monitoring;
|
|
pub mod optuna_persistence;
|
|
pub mod orchestrator;
|
|
pub mod promotion_manager;
|
|
pub mod schema_types;
|
|
pub mod service;
|
|
pub mod simple_metrics;
|
|
pub mod storage;
|
|
pub mod technical_indicators;
|
|
pub mod training_metrics;
|
|
pub mod trial_executor;
|
|
pub mod tuning_manager;
|
|
pub mod validation_pipeline;
|
|
|
|
// TLI BacktestingService proto (client only -- for validation pipeline)
|
|
pub mod backtesting_proto {
|
|
tonic::include_proto!("foxhunt.tli");
|
|
}
|
|
|
|
// Re-export proto module for test access
|
|
pub use service::proto;
|
|
|
|
/// Error types for the ML training service
|
|
pub mod errors {
|
|
use thiserror::Error;
|
|
|
|
/// Training service errors
|
|
#[derive(Error, Debug)]
|
|
pub enum TrainingServiceError {
|
|
/// Configuration error
|
|
#[error("Configuration error: {message}")]
|
|
Configuration { message: String },
|
|
|
|
/// Database error
|
|
#[error("Database error: {message}")]
|
|
Database { message: String },
|
|
|
|
/// Storage error
|
|
#[error("Storage error: {message}")]
|
|
Storage { message: String },
|
|
|
|
/// Training error
|
|
#[error("Training error: {message}")]
|
|
Training { message: String },
|
|
|
|
/// Resource allocation error
|
|
#[error("Resource allocation error: {message}")]
|
|
Resource { message: String },
|
|
|
|
/// Invalid request error
|
|
#[error("Invalid request: {message}")]
|
|
InvalidRequest { message: String },
|
|
|
|
/// Job not found error
|
|
#[error("Job not found: {job_id}")]
|
|
JobNotFound { job_id: String },
|
|
|
|
/// Internal service error
|
|
#[error("Internal error: {message}")]
|
|
Internal { message: String },
|
|
}
|
|
|
|
/// Result type for training service operations
|
|
pub type Result<T> = std::result::Result<T, TrainingServiceError>;
|
|
}
|
|
|
|
/// Version information
|
|
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
|
|
|
/// Service metadata
|
|
pub const SERVICE_NAME: &str = "ml_training_service";
|
|
|
|
#[cfg(test)]
|
|
#[allow(clippy::unwrap_used, clippy::expect_used)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_version() {
|
|
assert!(!VERSION.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_service_name() {
|
|
assert_eq!(SERVICE_NAME, "ml_training_service");
|
|
}
|
|
}
|