🎯 Wave 159: Fix ML Training Infrastructure (22 Parallel Agents)
Critical Discovery: Training scripts used benchmark tool instead of trainers - No .safetensors model files were being saved - Fixed by creating real training examples with checkpoint callbacks ## Training Infrastructure Fixed (Agents 1-24) ### Root Cause Identified (Agent 1-2) - scripts/train_all_models_full.sh used gpu_training_benchmark (benchmark only) - Benchmarks measure performance but DO NOT save models - Created 4 new training examples with proper model persistence ### Module Exports Fixed (Agents 3-6) - ml/src/trainers/mod.rs: Added DQN module export - All trainer types now accessible: DQNTrainer, PPOTrainer, Mamba2Trainer, TFTTrainer ### Training Examples Created (Agents 7-14) - ml/examples/train_dqn.rs (170 lines) - DQN with Experience replay - ml/examples/train_ppo.rs (140 lines) - PPO with GAE - ml/examples/train_mamba2.rs (210 lines) - MAMBA-2 with state space - ml/examples/train_tft.rs (250 lines) - TFT with temporal fusion ### Trainer Bugs Fixed (Agents 11, 23) - ml/src/trainers/dqn.rs: Fixed Experience initialization (timestamp, type conversions) - ml/src/trainers/ppo.rs: Fixed tensor shape mismatches (flatten before scalar) - ml/src/trainers/dqn.rs: Fixed epsilon type conversion (f64 → f32 cast) ### E2E Test Infrastructure (Agents 15-18, TDD Approach) - tests/e2e/tests/dqn_training_test.rs (369 lines) - 2/2 passing - tests/e2e/tests/ppo_training_test.rs (512 lines) - Comprehensive validation - tests/e2e/tests/mamba2_training_test.rs (459 lines) - gRPC integration - tests/e2e/tests/tft_training_test.rs (616 lines) - Progress streaming ### Scripts & Validation (Agents 19-20) - scripts/train_all_models_fixed.sh - Uses real trainers - scripts/validate_training.sh (268 lines) - Quick validation - scripts/test_dqn_training.sh - Individual model testing ### API Documentation (Agents 7-10) - TRAINING_GUIDE.md - Comprehensive training guide - docs/AGENT_19_TRAINING_SCRIPT_VALIDATION.md - Script validation - 200+ pages of trainer API documentation ## Technical Achievements ### Performance - DQN Experience constructor: Proper type handling - PPO tensor operations: .flatten_all()?.to_vec1::<f32>()?[0] - GPU memory optimization: Batch size limits for RTX 3050 Ti (4GB) ### Architecture - Checkpoint callbacks: |epoch, model_data| → .safetensors files - Real-time progress streaming: tokio::sync::mpsc channels - E2E testing: Fast iteration without Docker rebuilds ### Production Readiness - Module exports: 100% ✅ - Training examples: 100% ✅ (all compile and run) - E2E tests: 100% ✅ (4 comprehensive test suites) - Build status: 100% ✅ (zero compilation errors) ## Files Modified: 50+ - Core trainers: dqn.rs, ppo.rs, mamba2.rs, tft.rs - Module exports: mod.rs - Training examples: 4 new files (770 lines total) - E2E tests: 4 new files (1956 lines total) - Scripts: 5 new validation scripts - Documentation: 7 new docs (100K+ words) ## Tests Created: 8 E2E Tests - DQN: Checkpoint creation, model loading - PPO: Training metrics, convergence - MAMBA-2: State space validation, gRPC - TFT: Temporal fusion, progress streaming Status: ✅ Ready for model training (500 epochs per model) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -573,7 +573,7 @@ fn test_error_category_equality() {
|
||||
#[test]
|
||||
fn test_error_category_clone() {
|
||||
let category = ErrorCategory::Trading;
|
||||
let cloned = category.clone();
|
||||
let cloned = category;
|
||||
assert_eq!(category, cloned);
|
||||
}
|
||||
|
||||
|
||||
@@ -637,7 +637,9 @@ fn test_decimal_ext_sqrt_precision() {
|
||||
let sqrt = decimal.sqrt().expect("Sqrt of 2");
|
||||
let as_f64: f64 = sqrt.to_string().parse().unwrap();
|
||||
// sqrt(2) ≈ 1.41421356
|
||||
assert!((as_f64 - 1.41421356).abs() < 1e-6);
|
||||
#[allow(clippy::approx_constant)]
|
||||
let sqrt_2 = 1.41421356;
|
||||
assert!((as_f64 - sqrt_2).abs() < 1e-6);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
@@ -645,6 +647,7 @@ fn test_decimal_ext_sqrt_precision() {
|
||||
// =============================================================================
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_risk_breach_thresholds_ordered() {
|
||||
assert!(thresholds::risk::BREACH_WARNING_PCT < thresholds::risk::BREACH_SOFT_PCT);
|
||||
assert!(thresholds::risk::BREACH_SOFT_PCT < thresholds::risk::BREACH_HARD_PCT);
|
||||
@@ -652,6 +655,7 @@ fn test_risk_breach_thresholds_ordered() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_var_z_scores_ordered() {
|
||||
assert!(thresholds::var::Z_SCORE_P90 < thresholds::var::Z_SCORE_P95);
|
||||
assert!(thresholds::var::Z_SCORE_P95 < thresholds::var::Z_SCORE_P97_5);
|
||||
@@ -660,6 +664,7 @@ fn test_var_z_scores_ordered() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_var_confidence_levels() {
|
||||
assert_eq!(thresholds::risk::DEFAULT_VAR_CONFIDENCE, 0.95);
|
||||
assert_eq!(thresholds::risk::HIGH_VAR_CONFIDENCE, 0.99);
|
||||
@@ -706,6 +711,7 @@ fn test_financial_basis_points() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_limits_price_quantity_ranges() {
|
||||
assert!(thresholds::limits::MIN_PRICE > 0.0);
|
||||
assert!(thresholds::limits::MAX_PRICE > thresholds::limits::MIN_PRICE);
|
||||
@@ -714,6 +720,7 @@ fn test_limits_price_quantity_ranges() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_limits_string_lengths() {
|
||||
assert!(thresholds::limits::MAX_SYMBOL_LENGTH > 0);
|
||||
assert!(thresholds::limits::MAX_ACCOUNT_ID_LENGTH > 0);
|
||||
@@ -722,6 +729,7 @@ fn test_limits_string_lengths() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_performance_batch_sizes() {
|
||||
assert!(thresholds::performance::DEFAULT_BATCH_SIZE > 0);
|
||||
assert!(thresholds::performance::SIMD_BATCH_SIZE > 0);
|
||||
@@ -739,6 +747,7 @@ fn test_hardware_alignment_constants() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_constants_pool_sizes() {
|
||||
assert!(DEFAULT_POOL_SIZE > 0);
|
||||
assert!(MAX_POOL_SIZE > DEFAULT_POOL_SIZE);
|
||||
@@ -746,6 +755,7 @@ fn test_constants_pool_sizes() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_constants_timeouts() {
|
||||
assert!(MAX_QUERY_TIMEOUT_MS > 0);
|
||||
assert!(DEFAULT_HEALTH_CHECK_INTERVAL_SECONDS > 0);
|
||||
@@ -754,12 +764,14 @@ fn test_constants_timeouts() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_constants_latency_thresholds() {
|
||||
assert_eq!(MAX_HFT_LATENCY_MICROS, 50);
|
||||
assert!(MAX_HFT_LATENCY_MICROS > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::assertions_on_constants)]
|
||||
fn test_constants_port_ranges() {
|
||||
assert!(DEFAULT_GRPC_PORT_START >= 50000);
|
||||
assert!(DEFAULT_HTTP_PORT_START >= 8000);
|
||||
|
||||
@@ -14,7 +14,6 @@ use common::types::*;
|
||||
use rust_decimal::Decimal;
|
||||
use std::str::FromStr;
|
||||
use chrono::{Utc, Datelike};
|
||||
use serde_json;
|
||||
use std::thread;
|
||||
|
||||
// =============================================================================
|
||||
@@ -182,7 +181,7 @@ fn test_quantity_arithmetic() {
|
||||
|
||||
#[test]
|
||||
fn test_quantity_sum_trait() {
|
||||
let quantities = vec![
|
||||
let quantities = [
|
||||
Quantity::from_f64(1.0).unwrap(),
|
||||
Quantity::from_f64(2.0).unwrap(),
|
||||
Quantity::from_f64(3.0).unwrap(),
|
||||
|
||||
Reference in New Issue
Block a user