gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ml): delete mock data loader from train_tft binary

Browse Source 
Replace load_and_split_data() mock that generated 2000 synthetic samples
with an error-returning stub directing users to ml_training_service.

The binary retains its real infrastructure (TFTTrainer, CLI, checkpoint
storage, progress callbacks) — only the fake data generation is removed.

-120 lines of mock data, +16 lines error stub with documentation.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-23 00:03:12 +01:00
parent edd5a503da
commit 200deaafe2
1 changed files with 16 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
ml/src/bin/train_tft.rs
View File

@@ -34,7 +34,7 @@ use clap::Parser;
use ndarray::{Array1, Array2};
use std::path::PathBuf;
use std::sync::Arc;
use tracing::{error, info, warn};
use tracing::{error, info};
use ml::checkpoint::FileSystemStorage;
use ml::tft::training::TFTDataLoader;
@@ -356,20 +356,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
/// Load parquet data and split into train/validation sets
/// Load parquet data and split into train/validation sets.
///
/// This function:
/// 1. Loads raw market data from parquet files (using existing infrastructure)
/// 2. Engineers features (OHLCV + technical indicators)
/// 3. Creates rolling windows (lookback + forecast)
/// 4. Splits into train/validation sets
/// This function is intentionally unimplemented. The real data pipeline requires:
///
/// Returns: (`train_data`, `val_data`)
/// 1. `data::replay::ParquetDataLoader` to load OHLCV bars from parquet files
/// 2. `ml::features::extraction::extract_ml_features()` for feature engineering
/// 3. Rolling window creation with (static, historical, future, target) tuples
///
/// Use `ml_training_service` for production TFT training.
/// See `ml/src/trainers/tft/trainer.rs` for the real trainer implementation.
/// See `ml/src/features/extraction.rs` for the 51-dim feature pipeline.
async fn load_and_split_data(
_files: &[PathBuf],
lookback: usize,
forecast: usize,
train_split: f64,
_lookback: usize,
_forecast: usize,
_train_split: f64,
) -> Result<
(
Vec<(Array1<f64>, Array2<f64>, Array2<f64>, Array1<f64>)>,
@@ -377,113 +379,7 @@ async fn load_and_split_data(
),
Box<dyn std::error::Error>,
> {
warn!("\u{26a0}\u{fe0f} Using MOCK DATA for proof-of-concept");
warn!(" Real parquet loading requires:");
warn!(" 1. Integration with data::replay::ParquetDataLoader");
warn!(" 2. Feature engineering pipeline (OHLCV \u{2192} TFT features)");
warn!(" 3. Rolling window creation");
warn!("");
// TODO: Real implementation
//
// use data::replay::ParquetDataLoader;
// use trading_engine::types::metrics::ParquetMarketDataEvent;
//
// let mut all_events = Vec::new();
// for file in files {
// let loader = ParquetDataLoader::new(file);
// let events = loader.load_all().await?;
// all_events.extend(events);
// }
//
// // Engineer features
// let features = engineer_tft_features(&all_events)?;
//
// // Create rolling windows
// let samples = create_rolling_windows(features, lookback, forecast)?;
//
// // Split train/val
// split_by_ratio(samples, train_split)
// Mock data generation (for now)
let num_samples = 2000; // Simulate 2000 timesteps
let mut samples = Vec::with_capacity(num_samples);
info!(" Generating {} mock samples...", num_samples);
for i in 0..num_samples {
// Static features (10 dimensions): asset metadata, regime indicators
let static_features = Array1::from_vec(vec![
i as f64 / num_samples as f64, // Time progress (0-1)
0.5, // Volatility regime
0.3, // Trend strength
1.0, // Market hours indicator
0.0, // Weekend indicator
0.5, // Liquidity score
0.7, // Correlation to market
0.2, // Sector indicator
0.4, // Asset age
0.6, // Trading volume indicator
]);
// Historical features (lookback × 64 dimensions): OHLCV + technical indicators
let mut hist_data = Vec::with_capacity(lookback * 64);
for t in 0..lookback {
// OHLCV (5)
let base_price = 50000.0 + (i + t) as f64 * 10.0;
hist_data.push(base_price); // Open
hist_data.push(base_price * 1.01); // High
hist_data.push(base_price * 0.99); // Low
hist_data.push(base_price * 1.005); // Close
hist_data.push(1000.0); // Volume
// Technical indicators (59): SMA, EMA, RSI, MACD, etc.
for _ in 0..59 {
hist_data.push((i + t) as f64 * 0.1);
}
}
let historical_features = Array2::from_shape_vec((lookback, 64), hist_data)?;
// Future features (forecast × 10 dimensions): known future info (time, calendar)
let mut fut_data = Vec::with_capacity(forecast * 10);
for t in 0..forecast {
// Hour of day
fut_data.push(((i + lookback + t) % 24) as f64 / 24.0);
// Day of week
fut_data.push(((i + lookback + t) % 7) as f64 / 7.0);
// Month indicator
fut_data.push(0.5);
// Holiday indicator
fut_data.push(0.0);
// Scheduled event indicator
fut_data.push(0.0);
// Market open/close indicator
fut_data.push(1.0);
// Padding (4)
for _ in 0..4 {
fut_data.push(0.0);
}
}
let future_features = Array2::from_shape_vec((forecast, 10), fut_data)?;
// Targets (forecast dimensions): future prices to predict
let target_data: Vec<f64> = (0..forecast)
.map(|t| 50000.0 + (i + lookback + t) as f64 * 10.0)
.collect();
let targets = Array1::from_vec(target_data);
samples.push((
static_features,
historical_features,
future_features,
targets,
));
}
// Split by ratio
let split_idx = (samples.len() as f64 * train_split) as usize;
let train_data = samples[..split_idx].to_vec();
let val_data = samples[split_idx..].to_vec();
Ok((train_data, val_data))
Err("Real data loading not implemented. Use ml_training_service for production TFT training. \
Wire data::replay::ParquetDataLoader -> ml::features::extraction -> rolling windows."
.into())
}