foxhunt/ml/examples/train_liquid_dbn.rs

// Pilot Training Example for Liquid Neural Network with DBN Real Market Data
//
// This example demonstrates training a Liquid Time-Constant Neural Network (LTC)
// on real market data from Databento (DBN format) for HFT price prediction.
//
// Architecture:
// - Input: 16 features (5 OHLCV + 10 technical indicators + 1 volume)
// - Hidden: 128 LTC neurons with adaptive time constants
// - Output: 3 classes (buy=0, hold=1, sell=2)
//
// Expected Performance:
// - Accuracy: 55-65% (better than random 33.3%)
// - Convergence: 20-30 epochs with early stopping
// - Training time: ~5 minutes (CPU) or ~30 seconds (GPU)
// - Inference latency: <100μs (fixed-point arithmetic)

use anyhow::Result;
use ml::data_loaders::dbn_sequence_loader::DbnSequenceLoader;
use ml::liquid::{
    ActivationType, FixedPoint, LayerConfig, LiquidNetwork, LiquidNetworkConfig,
    LiquidTrainer, LiquidTrainingConfig, OutputLayerConfig, SolverType,
    TrainingSample, TrainingUtils, PRECISION,
    LTCConfig, NetworkType,
};
use std::time::Instant;

#[tokio::main]
async fn main() -> Result<()> {
    println!("========================================");
    println!("Liquid Neural Network Pilot Training");
    println!("========================================");
    println!();
    println!("Architecture:");
    println!("  Input:  16 features (normalized OHLCV sequence)");
    println!("  Hidden: 128 LTC neurons (τ=0.01-1.0)");
    println!("  Output: 3 classes (buy/hold/sell)");
    println!("  Solver: RK4 (4th order accuracy)");
    println!();

    // Step 1: Load DBN market data
    println!("[1/6] Loading DBN market data (6E.FUT)...");

    // Create DbnSequenceLoader with sequence length 60 and feature dimension 16
    let mut loader = DbnSequenceLoader::new(60, 16).await?;

    // Load OHLCV sequences from the production data directory
    let data_dir = "test_data/real/databento/ml_training";
    let (train_sequences, _val_sequences) = loader.load_sequences(data_dir, 0.8).await?;

    println!("  ✓ Loaded {} training sequences", train_sequences.len());

    // Step 2: Convert sequences to training samples
    println!();
    println!("[2/6] Converting sequences to training samples...");

    let mut training_samples = Vec::new();

    for (input_tensor, _target_tensor) in train_sequences.iter() {
        // Extract the last timestep from the input sequence for feature extraction
        // Input shape: [seq_len, d_model] = [60, 16]
        // Target shape: [d_model] = [16] (next timestep prediction)
        let seq_data = input_tensor.to_vec2::<f64>()?;

        // Use the last timestep as features (16 features)
        if let Some(last_step) = seq_data.last() {
            // Convert input to FixedPoint
            let features: Vec<FixedPoint> = last_step
                .iter()
                .map(|&f| FixedPoint::from_f64(f))
                .collect();

            // For this pilot, we'll create synthetic labels based on the trend in the sequence
            // In production, you'd use actual price change labels from target_data
            let label = if seq_data.len() >= 2 {
                // Compare last few prices to determine trend
                let recent_prices: Vec<f64> = seq_data.iter().rev().take(5).map(|step| step[3]).collect(); // Close price at index 3
                let first = recent_prices.last().unwrap_or(&0.0);
                let last = recent_prices.first().unwrap_or(&0.0);
                let price_change = (last - first) / first.abs().max(1e-6);

                // Thresholds for buy/hold/sell (0.1% = 10 basis points)
                if price_change > 0.001 {
                    0  // Buy signal
                } else if price_change < -0.001 {
                    2  // Sell signal
                } else {
                    1  // Hold signal
                }
            } else {
                1  // Hold for insufficient data
            };

            // One-hot encode label [buy, hold, sell]
            let target = match label {
                0 => vec![FixedPoint::one(), FixedPoint::zero(), FixedPoint::zero()],  // Buy
                1 => vec![FixedPoint::zero(), FixedPoint::one(), FixedPoint::zero()],  // Hold
                2 => vec![FixedPoint::zero(), FixedPoint::zero(), FixedPoint::one()],  // Sell
                _ => vec![FixedPoint::zero(), FixedPoint::one(), FixedPoint::zero()],  // Default: Hold
            };

            training_samples.push(TrainingSample {
                input: features,
                target,
                timestamp: None,
                market_regime: None,
                volatility: None,
            });
        }
    }
    println!("  ✓ Created {} training samples from sequences", training_samples.len());

    // Step 3: Normalize features (Z-score normalization)
    println!();
    println!("[3/6] Normalizing features...");
    let (means, _stds) = TrainingUtils::normalize_features(&mut training_samples)?;
    println!("  ✓ Normalized {} features (mean=0, std=1)", means.len());

    // Step 4: Split into training and validation sets
    println!();
    println!("[4/6] Splitting data (80% train, 20% validation)...");
    let (train_samples, val_samples) = TrainingUtils::train_validation_split(
        training_samples,
        0.2,  // 20% validation
    );
    println!("  ✓ Training samples:   {}", train_samples.len());
    println!("  ✓ Validation samples: {}", val_samples.len());

    // Create batches
    let batch_size = 32;
    let train_batches = TrainingUtils::create_batches(train_samples, batch_size);
    let val_batches = TrainingUtils::create_batches(val_samples, batch_size);
    println!("  ✓ Training batches:   {}", train_batches.len());
    println!("  ✓ Validation batches: {}", val_batches.len());

    // Step 5: Create Liquid Neural Network
    println!();
    println!("[5/6] Creating Liquid Neural Network...");

    // Create LTC layer configuration
    let ltc_config = LTCConfig {
        input_size: 16,  // 5 OHLCV + 10 technical indicators + 1 volume
        hidden_size: 128,
        tau_min: FixedPoint(PRECISION / 100),  // 0.01
        tau_max: FixedPoint(PRECISION),         // 1.0
        use_bias: true,
        solver_type: SolverType::RK4,  // 4th order accuracy
        activation: ActivationType::Tanh,
    };

    let network_config = LiquidNetworkConfig {
        network_type: NetworkType::LTC,
        input_size: 16,  // 5 OHLCV + 10 technical indicators + 1 volume
        output_size: 3,  // buy/hold/sell
        layer_configs: vec![LayerConfig::LTC(ltc_config)],
        output_layer: OutputLayerConfig {
            use_linear_output: false,
            output_activation: Some(ActivationType::Sigmoid),
            dropout_rate: None,
        },
        default_dt: FixedPoint(PRECISION / 100),  // 0.01 time step
        market_regime_adaptation: true,
    };

    let mut network = LiquidNetwork::new(network_config)?;
    println!("  ✓ Network created with {} parameters", network.parameter_count());
    println!("  ✓ Memory footprint: ~{} KB", (network.parameter_count() * 8) / 1024);

    // Step 6: Train the network
    println!();
    println!("[6/6] Training Liquid Neural Network (50 epochs)...");
    println!();

    let training_config = LiquidTrainingConfig {
        learning_rate: FixedPoint(PRECISION / 1000),  // 0.001
        batch_size,
        max_epochs: 50,  // Pilot training
        early_stopping_patience: 10,
        gradient_clip_threshold: FixedPoint(PRECISION),  // 1.0
        l2_regularization: FixedPoint(PRECISION / 10000),  // 0.0001
        adaptive_learning_rate: true,
        market_regime_adaptation: false,  // No regime data in pilot
        validation_split: 0.2,
    };

    let mut trainer = LiquidTrainer::new(training_config);

    let training_start = Instant::now();
    trainer.train(&mut network, &train_batches, Some(&val_batches))?;
    let training_duration = training_start.elapsed();

    // Training results
    println!();
    println!("========================================");
    println!("Training Complete!");
    println!("========================================");
    println!();
    println!("Training Metrics:");
    println!("  Total time:     {:.2}s", training_duration.as_secs_f64());
    println!("  Epochs trained: {}", trainer.training_history.len());

    if let Some(final_metrics) = trainer.training_history.last() {
        println!("  Final loss:     {:.6}", final_metrics.training_loss);
        if let Some(val_loss) = final_metrics.validation_loss {
            println!("  Val loss:       {:.6}", val_loss);
        }
        println!("  Learning rate:  {:.6}", final_metrics.learning_rate);
        println!("  Gradient norm:  {:.4}", final_metrics.gradient_norm);
        println!("  Samples/sec:    {:.1}", final_metrics.samples_per_second);
    }

    // Test inference latency
    println!();
    println!("Inference Performance:");
    let test_input: Vec<FixedPoint> = (0..16)
        .map(|i| FixedPoint::from_f64((i as f64) / 16.0))
        .collect();

    let inference_start = Instant::now();
    for _ in 0..1000 {
        let _ = network.forward(&test_input)?;
    }
    let avg_inference_time = inference_start.elapsed().as_micros() / 1000;

    println!("  Average latency: {}μs (1000 runs)", avg_inference_time);
    println!("  Target latency:  <100μs");

    if avg_inference_time < 100 {
        println!("  ✓ Latency target MET");
    } else {
        println!("  ⚠ Latency target EXCEEDED (consider GPU optimization)");
    }

    // Save network state (optional - future work)
    println!();
    println!("Checkpoint Status:");
    println!("  ⚠ Checkpoint saving not implemented (future: MinIO/S3)");
    println!("  ✓ Network state can be serialized via serde");

    println!();
    println!("========================================");
    println!("Next Steps:");
    println!("========================================");
    println!("1. Run full training (100 epochs, 90 days data)");
    println!("2. Integrate with ML Training Service (gRPC)");
    println!("3. Add checkpoint saving (MinIO)");
    println!("4. GPU acceleration (if latency >100μs)");
    println!("5. Hyperparameter tuning (Optuna)");
    println!();

    Ok(())
}