Files
foxhunt/services/ml_training_service/src/lib.rs
jgrusewski 0f9d756caa feat: on-demand training dispatch via K8s Jobs with sidecar uploader
Extend ml_training_service to dispatch GPU training jobs as K8s batch/v1
Jobs, collect results via a Rust sidecar uploader, and support model
promotion with operator approval via fxt CLI.

- K8s dispatcher creates Jobs on gpu-training pool with native sidecar
- training_uploader crate: watches DONE/FAILED marker, uploads to S3,
  reports completion via ReportJobCompletion gRPC
- PromotionManager compares metrics, queues better models for approval
- 4 new proto RPCs: ReportJobCompletion, ListPendingPromotions,
  ApprovePromotion, RejectPromotion
- fxt commands: train start, model list/approve/reject
- Training binaries write DONE/FAILED markers + metrics.json
- Dockerfile, K8s job template, and CI pipeline updated
- StartTraining gracefully falls back to in-process when outside K8s
- 27 new tests (16 service + 11 promotion), 141 total service tests pass

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 12:43:17 +01:00

124 lines
3.7 KiB
Rust

#![allow(missing_docs)] // Internal implementation details
// ML training service domain lints: generated protobuf code and training orchestration
#![allow(clippy::mixed_attributes_style)] // Generated protobuf code uses mixed attributes
#![allow(clippy::manual_clamp)] // Explicit min/max preferred in training parameters
#![allow(clippy::too_many_arguments)] // Training functions need many hyperparameters
#![allow(clippy::type_complexity)] // Complex types in async service handlers
#![allow(clippy::wildcard_in_or_patterns)] // Wildcard patterns in job queue matching
#![allow(clippy::redundant_pattern_matching)] // Explicit pattern matching preferred
#![allow(clippy::while_let_loop)] // Explicit loop with break for service processing
#![allow(clippy::doc_lazy_continuation)] // Doc formatting acceptable
//! ML Training Service Library
//!
//! This library provides the core functionality for the ML Training Service,
//! including training orchestration, job management, and gRPC API implementation.
#![deny(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used)]
pub mod asset_parser;
pub mod batch_tuning_manager;
pub mod checkpoint_manager;
pub mod data_config;
pub mod data_loader;
pub mod data_file_discovery;
pub mod database;
pub mod dbn_data_loader;
pub mod deployment_pipeline;
pub mod encryption;
pub mod ensemble_training_coordinator;
pub mod gpu_config;
pub mod gpu_resource_manager;
pub mod grpc;
pub mod grpc_tuning_handlers;
pub mod job_tracker;
pub mod k8s_dispatcher;
pub mod job_queue;
pub mod job_spawner;
pub mod monitoring;
pub mod optuna_persistence;
pub mod orchestrator;
pub mod promotion_manager;
pub mod schema_types;
pub mod service;
pub mod simple_metrics;
pub mod storage;
pub mod technical_indicators;
pub mod training_metrics;
pub mod trial_executor;
pub mod tuning_manager;
pub mod validation_pipeline;
// TLI BacktestingService proto (client only -- for validation pipeline)
pub mod backtesting_proto {
tonic::include_proto!("foxhunt.tli");
}
// Re-export proto module for test access
pub use service::proto;
/// Error types for the ML training service
pub mod errors {
use thiserror::Error;
/// Training service errors
#[derive(Error, Debug)]
pub enum TrainingServiceError {
/// Configuration error
#[error("Configuration error: {message}")]
Configuration { message: String },
/// Database error
#[error("Database error: {message}")]
Database { message: String },
/// Storage error
#[error("Storage error: {message}")]
Storage { message: String },
/// Training error
#[error("Training error: {message}")]
Training { message: String },
/// Resource allocation error
#[error("Resource allocation error: {message}")]
Resource { message: String },
/// Invalid request error
#[error("Invalid request: {message}")]
InvalidRequest { message: String },
/// Job not found error
#[error("Job not found: {job_id}")]
JobNotFound { job_id: String },
/// Internal service error
#[error("Internal error: {message}")]
Internal { message: String },
}
/// Result type for training service operations
pub type Result<T> = std::result::Result<T, TrainingServiceError>;
}
/// Version information
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Service metadata
pub const SERVICE_NAME: &str = "ml_training_service";
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
#[test]
fn test_version() {
assert!(!VERSION.is_empty());
}
#[test]
fn test_service_name() {
assert_eq!(SERVICE_NAME, "ml_training_service");
}
}