From 200deaafe27ae6d4b20d550ebae3f811eb800257 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 00:03:12 +0100 Subject: [PATCH] fix(ml): delete mock data loader from train_tft binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace load_and_split_data() mock that generated 2000 synthetic samples with an error-returning stub directing users to ml_training_service. The binary retains its real infrastructure (TFTTrainer, CLI, checkpoint storage, progress callbacks) — only the fake data generation is removed. -120 lines of mock data, +16 lines error stub with documentation. Co-Authored-By: Claude Opus 4.6 --- ml/src/bin/train_tft.rs | 136 +++++----------------------------------- 1 file changed, 16 insertions(+), 120 deletions(-) diff --git a/ml/src/bin/train_tft.rs b/ml/src/bin/train_tft.rs index 053a43cca..5065c1871 100644 --- a/ml/src/bin/train_tft.rs +++ b/ml/src/bin/train_tft.rs @@ -34,7 +34,7 @@ use clap::Parser; use ndarray::{Array1, Array2}; use std::path::PathBuf; use std::sync::Arc; -use tracing::{error, info, warn}; +use tracing::{error, info}; use ml::checkpoint::FileSystemStorage; use ml::tft::training::TFTDataLoader; @@ -356,20 +356,22 @@ async fn main() -> Result<(), Box> { Ok(()) } -/// Load parquet data and split into train/validation sets +/// Load parquet data and split into train/validation sets. /// -/// This function: -/// 1. Loads raw market data from parquet files (using existing infrastructure) -/// 2. Engineers features (OHLCV + technical indicators) -/// 3. Creates rolling windows (lookback + forecast) -/// 4. Splits into train/validation sets +/// This function is intentionally unimplemented. The real data pipeline requires: /// -/// Returns: (`train_data`, `val_data`) +/// 1. `data::replay::ParquetDataLoader` to load OHLCV bars from parquet files +/// 2. `ml::features::extraction::extract_ml_features()` for feature engineering +/// 3. Rolling window creation with (static, historical, future, target) tuples +/// +/// Use `ml_training_service` for production TFT training. +/// See `ml/src/trainers/tft/trainer.rs` for the real trainer implementation. +/// See `ml/src/features/extraction.rs` for the 51-dim feature pipeline. async fn load_and_split_data( _files: &[PathBuf], - lookback: usize, - forecast: usize, - train_split: f64, + _lookback: usize, + _forecast: usize, + _train_split: f64, ) -> Result< ( Vec<(Array1, Array2, Array2, Array1)>, @@ -377,113 +379,7 @@ async fn load_and_split_data( ), Box, > { - warn!("\u{26a0}\u{fe0f} Using MOCK DATA for proof-of-concept"); - warn!(" Real parquet loading requires:"); - warn!(" 1. Integration with data::replay::ParquetDataLoader"); - warn!(" 2. Feature engineering pipeline (OHLCV \u{2192} TFT features)"); - warn!(" 3. Rolling window creation"); - warn!(""); - - // TODO: Real implementation - // - // use data::replay::ParquetDataLoader; - // use trading_engine::types::metrics::ParquetMarketDataEvent; - // - // let mut all_events = Vec::new(); - // for file in files { - // let loader = ParquetDataLoader::new(file); - // let events = loader.load_all().await?; - // all_events.extend(events); - // } - // - // // Engineer features - // let features = engineer_tft_features(&all_events)?; - // - // // Create rolling windows - // let samples = create_rolling_windows(features, lookback, forecast)?; - // - // // Split train/val - // split_by_ratio(samples, train_split) - - // Mock data generation (for now) - let num_samples = 2000; // Simulate 2000 timesteps - let mut samples = Vec::with_capacity(num_samples); - - info!(" Generating {} mock samples...", num_samples); - - for i in 0..num_samples { - // Static features (10 dimensions): asset metadata, regime indicators - let static_features = Array1::from_vec(vec![ - i as f64 / num_samples as f64, // Time progress (0-1) - 0.5, // Volatility regime - 0.3, // Trend strength - 1.0, // Market hours indicator - 0.0, // Weekend indicator - 0.5, // Liquidity score - 0.7, // Correlation to market - 0.2, // Sector indicator - 0.4, // Asset age - 0.6, // Trading volume indicator - ]); - - // Historical features (lookback × 64 dimensions): OHLCV + technical indicators - let mut hist_data = Vec::with_capacity(lookback * 64); - for t in 0..lookback { - // OHLCV (5) - let base_price = 50000.0 + (i + t) as f64 * 10.0; - hist_data.push(base_price); // Open - hist_data.push(base_price * 1.01); // High - hist_data.push(base_price * 0.99); // Low - hist_data.push(base_price * 1.005); // Close - hist_data.push(1000.0); // Volume - - // Technical indicators (59): SMA, EMA, RSI, MACD, etc. - for _ in 0..59 { - hist_data.push((i + t) as f64 * 0.1); - } - } - let historical_features = Array2::from_shape_vec((lookback, 64), hist_data)?; - - // Future features (forecast × 10 dimensions): known future info (time, calendar) - let mut fut_data = Vec::with_capacity(forecast * 10); - for t in 0..forecast { - // Hour of day - fut_data.push(((i + lookback + t) % 24) as f64 / 24.0); - // Day of week - fut_data.push(((i + lookback + t) % 7) as f64 / 7.0); - // Month indicator - fut_data.push(0.5); - // Holiday indicator - fut_data.push(0.0); - // Scheduled event indicator - fut_data.push(0.0); - // Market open/close indicator - fut_data.push(1.0); - // Padding (4) - for _ in 0..4 { - fut_data.push(0.0); - } - } - let future_features = Array2::from_shape_vec((forecast, 10), fut_data)?; - - // Targets (forecast dimensions): future prices to predict - let target_data: Vec = (0..forecast) - .map(|t| 50000.0 + (i + lookback + t) as f64 * 10.0) - .collect(); - let targets = Array1::from_vec(target_data); - - samples.push(( - static_features, - historical_features, - future_features, - targets, - )); - } - - // Split by ratio - let split_idx = (samples.len() as f64 * train_split) as usize; - let train_data = samples[..split_idx].to_vec(); - let val_data = samples[split_idx..].to_vec(); - - Ok((train_data, val_data)) + Err("Real data loading not implemented. Use ml_training_service for production TFT training. \ + Wire data::replay::ParquetDataLoader -> ml::features::extraction -> rolling windows." + .into()) }