8ce7c52586
- Fixed feature dimension mismatch in evaluate_dqn_main_orchestrator.rs - Updated all 5 occurrences: state_dim, input comments, feature vector type - Aligned with Wave 16D training (128 features: 125 market + 3 portfolio) Issue: Validation backtest reveals 100% HOLD action collapse - requires reward system investigation and redesign per latest RL research.
1402 lines
51 KiB
Plaintext
1402 lines
51 KiB
Plaintext
//! Component 7: Main Orchestrator - Complete DQN Evaluation Pipeline
|
||
//!
|
||
//! Integrates all 6 components into a production-ready evaluation system:
|
||
//! - Component 1: CLI Configuration (EvaluationConfig)
|
||
//! - Component 2: Model Loading (load_dqn_model)
|
||
//! - Component 3: Parquet Data Loading (load_parquet_data)
|
||
//! - Component 4: Inference Engine (run_inference)
|
||
//! - Component 5: Metrics Calculator (calculate_metrics)
|
||
//! - Component 6: Report Generator (generate_report)
|
||
//! - Component 7: Main Orchestrator (main)
|
||
//!
|
||
//! # Usage
|
||
//!
|
||
//! ```bash
|
||
//! # Evaluate with default settings (auto-detect CUDA)
|
||
//! cargo run -p ml --example evaluate_dqn_main_orchestrator --release --features cuda
|
||
//!
|
||
//! # Evaluate on CPU with custom model
|
||
//! cargo run -p ml --example evaluate_dqn_main_orchestrator --release -- \
|
||
//! --model-path ml/trained_models/dqn_final_epoch100.safetensors \
|
||
//! --device cpu
|
||
//!
|
||
//! # Evaluate with JSON output for CI/CD
|
||
//! cargo run -p ml --example evaluate_dqn_main_orchestrator --release --features cuda -- \
|
||
//! --parquet-file test_data/ES_FUT_unseen.parquet \
|
||
//! --output-json evaluation_results.json
|
||
//!
|
||
//! # Custom warmup period (skip first 30 bars)
|
||
//! cargo run -p ml --example evaluate_dqn_main_orchestrator --release --features cuda -- \
|
||
//! --warmup-bars 30 \
|
||
//! --parquet-file test_data/ES_FUT_validation.parquet
|
||
//! ```
|
||
//!
|
||
//! # Architecture
|
||
//!
|
||
//! ```text
|
||
//! ┌─────────────────────────────────────────────────────────────────┐
|
||
//! │ MAIN ORCHESTRATOR │
|
||
//! │ │
|
||
//! │ 1. INITIALIZATION │
|
||
//! │ ├─ Parse CLI args (Component 1) │
|
||
//! │ ├─ Setup tracing (stdout + /tmp/dqn_eval.log) │
|
||
//! │ ├─ Validate config │
|
||
//! │ └─ Setup graceful shutdown (Ctrl+C / SIGTERM) │
|
||
//! │ │
|
||
//! │ 2. PARALLEL LOADING (tokio::try_join!) │
|
||
//! │ ├─ Load Parquet data (Component 3) ─────┐ │
|
||
//! │ └─ Load DQN model (Component 2) ────────┴─ Concurrent │
|
||
//! │ │
|
||
//! │ 3. SEQUENTIAL INFERENCE │
|
||
//! │ ├─ Run inference (Component 4) │
|
||
//! │ ├─ Calculate metrics (Component 5) │
|
||
//! │ └─ Track total elapsed time │
|
||
//! │ │
|
||
//! │ 4. REPORT GENERATION │
|
||
//! │ ├─ Generate report (Component 6) │
|
||
//! │ └─ Export JSON (if configured) │
|
||
//! │ │
|
||
//! │ 5. GRACEFUL SHUTDOWN │
|
||
//! │ ├─ Stop inference loop (if interrupted) │
|
||
//! │ ├─ Generate partial report │
|
||
//! │ └─ Exit with appropriate code (0=success, 1=error) │
|
||
//! └─────────────────────────────────────────────────────────────────┘
|
||
//! ```
|
||
|
||
use anyhow::{Context, Result};
|
||
use candle_core::{Device, Tensor};
|
||
use clap::Parser;
|
||
use serde::{Deserialize, Serialize};
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::atomic::{AtomicBool, Ordering};
|
||
use std::sync::Arc;
|
||
use std::time::Instant;
|
||
use tokio::signal;
|
||
use tracing::{info, warn};
|
||
use tracing_subscriber::FmtSubscriber;
|
||
|
||
use ml::data_loaders::load_parquet_data_with_timestamps;
|
||
use ml::dqn::dqn::{WorkingDQN, WorkingDQNConfig};
|
||
use ml::features::extraction::OHLCVBar;
|
||
use ml::preprocessing::{preprocess_prices, PreprocessConfig};
|
||
|
||
// ============================================================================
|
||
// Component 1: CLI Configuration (imported from evaluate_dqn.rs)
|
||
// ============================================================================
|
||
|
||
/// DQN Model Evaluation Configuration
|
||
///
|
||
/// Parses and validates CLI arguments for evaluating a trained DQN model.
|
||
///
|
||
/// # Validation Rules
|
||
///
|
||
/// - `model_path`: Must exist and be a valid SafeTensors file
|
||
/// - `parquet_file`: Must exist and contain OHLCV data
|
||
/// - `device`: Must be "cpu", "cuda", or "auto"
|
||
/// - `warmup_bars`: Must be in range [10, 100] (prevents over/under fitting)
|
||
#[derive(Parser, Debug)]
|
||
#[command(
|
||
name = "evaluate_dqn_main_orchestrator",
|
||
about = "Evaluate DQN model on unseen market data",
|
||
long_about = "Complete DQN evaluation pipeline with parallel loading, inference, \
|
||
metrics calculation, and report generation."
|
||
)]
|
||
struct EvaluationConfig {
|
||
/// Path to trained DQN model (SafeTensors format)
|
||
#[arg(long, default_value = "/tmp/dqn_final_model.safetensors")]
|
||
model_path: PathBuf,
|
||
|
||
/// Path to Parquet file with unseen OHLCV data
|
||
#[arg(long, default_value = "test_data/ES_FUT_unseen.parquet")]
|
||
parquet_file: PathBuf,
|
||
|
||
/// Device selection: cpu, cuda, or auto
|
||
#[arg(long, default_value = "auto")]
|
||
device: String,
|
||
|
||
/// Number of warmup bars to skip (insufficient history)
|
||
#[arg(long, default_value_t = 50)]
|
||
warmup_bars: usize,
|
||
|
||
/// Optional JSON output path for CI/CD integration
|
||
#[arg(long)]
|
||
output_json: Option<PathBuf>,
|
||
|
||
/// Optional path to export DQN actions as CSV
|
||
#[arg(long)]
|
||
export_actions: Option<PathBuf>,
|
||
|
||
/// Verbose logging (DEBUG level)
|
||
#[arg(short, long)]
|
||
verbose: bool,
|
||
}
|
||
|
||
impl EvaluationConfig {
|
||
/// Validates the configuration parameters
|
||
pub(crate) fn validate(&self) -> Result<()> {
|
||
// Validate model_path exists
|
||
if !self.model_path.exists() {
|
||
return Err(anyhow::anyhow!(
|
||
"Model file does not exist: {}\n\n\
|
||
Suggestion: Train a model first using:\n\
|
||
cargo run -p ml --example train_dqn --release --features cuda -- \\\n\
|
||
--output {}",
|
||
self.model_path.display(),
|
||
self.model_path.display()
|
||
));
|
||
}
|
||
|
||
if !self.model_path.is_file() {
|
||
return Err(anyhow::anyhow!(
|
||
"Model path is not a regular file: {}",
|
||
self.model_path.display()
|
||
));
|
||
}
|
||
|
||
// Validate parquet_file exists
|
||
if !self.parquet_file.exists() {
|
||
return Err(anyhow::anyhow!(
|
||
"Parquet file does not exist: {}\n\n\
|
||
Suggestion: Generate unseen data using:\n\
|
||
# Download data from Databento API for a different time period\n\
|
||
# than your training data (temporal split)",
|
||
self.parquet_file.display()
|
||
));
|
||
}
|
||
|
||
if !self.parquet_file.is_file() {
|
||
return Err(anyhow::anyhow!(
|
||
"Parquet path is not a regular file: {}",
|
||
self.parquet_file.display()
|
||
));
|
||
}
|
||
|
||
// Validate device string
|
||
match self.device.as_str() {
|
||
"cpu" | "cuda" | "auto" => {
|
||
// Valid device string
|
||
}
|
||
_ => {
|
||
return Err(anyhow::anyhow!(
|
||
"Invalid device: '{}'\n\n\
|
||
Valid options:\n\
|
||
- 'cpu': Force CPU execution\n\
|
||
- 'cuda': Force CUDA GPU execution (requires NVIDIA GPU)\n\
|
||
- 'auto': Auto-detect CUDA availability (recommended)",
|
||
self.device
|
||
));
|
||
}
|
||
}
|
||
|
||
// Validate warmup_bars range
|
||
if self.warmup_bars < 10 {
|
||
return Err(anyhow::anyhow!(
|
||
"Warmup bars too small: {} (minimum: 10)\n\n\
|
||
At least 10 bars are required for basic feature computation.\n\
|
||
Recommended: 50 bars for most models.",
|
||
self.warmup_bars
|
||
));
|
||
}
|
||
|
||
if self.warmup_bars > 100 {
|
||
return Err(anyhow::anyhow!(
|
||
"Warmup bars too large: {} (maximum: 100)\n\n\
|
||
Using more than 100 warmup bars wastes evaluation data.\n\
|
||
Recommended: 50 bars for most models.",
|
||
self.warmup_bars
|
||
));
|
||
}
|
||
|
||
// Validate output_json path (if specified)
|
||
if let Some(ref output_path) = self.output_json {
|
||
// Check if parent directory exists
|
||
if let Some(parent) = output_path.parent() {
|
||
if !parent.exists() {
|
||
return Err(anyhow::anyhow!(
|
||
"Output JSON parent directory does not exist: {}\n\n\
|
||
Suggestion: Create the directory first:\n\
|
||
mkdir -p {}",
|
||
parent.display(),
|
||
parent.display()
|
||
));
|
||
}
|
||
}
|
||
|
||
// Check if file already exists (warn, but don't fail)
|
||
if output_path.exists() {
|
||
info!(
|
||
"⚠️ Output JSON file already exists and will be overwritten: {}",
|
||
output_path.display()
|
||
);
|
||
}
|
||
}
|
||
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
// ============================================================================
|
||
// Component 2: Model Loading
|
||
// ============================================================================
|
||
|
||
/// Load DQN model from SafeTensors file with device selection
|
||
///
|
||
/// # Arguments
|
||
/// * `model_path` - Path to SafeTensors model file (.safetensors extension)
|
||
/// * `device_str` - Device selection: "cpu", "cuda", or "auto"
|
||
///
|
||
/// # Returns
|
||
/// WorkingDQN ready for inference with loaded weights
|
||
///
|
||
/// # Errors
|
||
/// Returns error if:
|
||
/// - File not found or not readable
|
||
/// - SafeTensors deserialization fails
|
||
/// - CUDA requested but unavailable
|
||
/// - Model dimensions incorrect (expected: 225 input features, 3 actions)
|
||
///
|
||
/// # Note
|
||
/// Uses WorkingDQN which has production-ready load_from_safetensors() method.
|
||
/// Architecture: 225 input → [128, 64, 32] hidden → 3 output (8 tensors)
|
||
fn load_dqn_model(model_path: &Path, device_str: &str) -> Result<WorkingDQN> {
|
||
info!("🔧 Component 2: Loading DQN model");
|
||
info!(" Model path: {}", model_path.display());
|
||
info!(" Device selection: {}", device_str);
|
||
|
||
// 1. Parse device string to create Device
|
||
let device = match device_str.to_lowercase().as_str() {
|
||
"cpu" => {
|
||
info!(" Using CPU device (explicitly requested)");
|
||
Device::Cpu
|
||
}
|
||
"cuda" => {
|
||
info!(" Using CUDA device (explicitly requested)");
|
||
Device::new_cuda(0).context(
|
||
"CUDA device requested but unavailable. \
|
||
Suggestions:\n\
|
||
- Check nvidia-smi to verify GPU is available\n\
|
||
- Try device_str=\"auto\" for automatic fallback to CPU\n\
|
||
- Use device_str=\"cpu\" to force CPU execution"
|
||
)?
|
||
}
|
||
"auto" => {
|
||
match Device::cuda_if_available(0) {
|
||
Ok(cuda_device) => {
|
||
info!(" Auto-selected CUDA device (GPU available)");
|
||
cuda_device
|
||
}
|
||
Err(e) => {
|
||
warn!(" CUDA unavailable, falling back to CPU: {}", e);
|
||
info!(" Using CPU device (auto-fallback)");
|
||
Device::Cpu
|
||
}
|
||
}
|
||
}
|
||
_ => {
|
||
return Err(anyhow::anyhow!(
|
||
"Invalid device_str: '{}'. Must be one of: 'cpu', 'cuda', 'auto'",
|
||
device_str
|
||
));
|
||
}
|
||
};
|
||
|
||
let device_name = match &device {
|
||
Device::Cpu => "CPU",
|
||
Device::Cuda(_) => "CUDA:0",
|
||
_ => "Unknown",
|
||
};
|
||
|
||
// 2. Check if model file exists and is readable
|
||
if !model_path.exists() {
|
||
return Err(anyhow::anyhow!(
|
||
"Model file not found: {}\n\
|
||
Suggestions:\n\
|
||
- Check the file path is correct\n\
|
||
- Verify the model was saved successfully during training\n\
|
||
- Look for checkpoint files in ml/trained_models/",
|
||
model_path.display()
|
||
));
|
||
}
|
||
|
||
if !model_path.is_file() {
|
||
return Err(anyhow::anyhow!(
|
||
"Path exists but is not a file: {}",
|
||
model_path.display()
|
||
));
|
||
}
|
||
|
||
// Check file permissions (read access)
|
||
match std::fs::metadata(model_path) {
|
||
Ok(metadata) => {
|
||
if metadata.permissions().readonly() {
|
||
warn!(" Model file is read-only: {}", model_path.display());
|
||
}
|
||
info!(" Model file size: {} bytes ({:.2} KB)",
|
||
metadata.len(),
|
||
metadata.len() as f64 / 1024.0
|
||
);
|
||
}
|
||
Err(e) => {
|
||
return Err(anyhow::anyhow!(
|
||
"Cannot read file metadata for {}: {}",
|
||
model_path.display(),
|
||
e
|
||
));
|
||
}
|
||
}
|
||
|
||
// 3. Create WorkingDQNConfig with correct architecture
|
||
// Architecture: 125 input → [256, 128, 64] hidden → 3 output (matches training)
|
||
info!(" Creating WorkingDQN configuration...");
|
||
let config = WorkingDQNConfig {
|
||
state_dim: 125, // Wave D: 125 features (101 Wave C + 24 Wave D)
|
||
hidden_dims: vec![256, 128, 64], // Match trainer architecture (Wave 10-A1)
|
||
num_actions: 3,
|
||
learning_rate: 0.001,
|
||
gamma: 0.99,
|
||
epsilon_start: 0.0, // No exploration during evaluation
|
||
epsilon_end: 0.0,
|
||
epsilon_decay: 1.0,
|
||
replay_buffer_capacity: 1000,
|
||
batch_size: 32,
|
||
min_replay_size: 64,
|
||
target_update_freq: 1000,
|
||
use_double_dqn: true,
|
||
use_huber_loss: true, // Huber loss default (more robust to outliers)
|
||
huber_delta: 1.0, // Standard Huber delta
|
||
leaky_relu_alpha: 0.01, // Standard LeakyReLU negative slope
|
||
gradient_clip_norm: 10.0, // Wave 11 Bug #1 fix
|
||
tau: 1.0, // Hard updates (full copy)
|
||
use_soft_updates: false, // Hard updates by default
|
||
warmup_steps: 0, // No warmup for evaluation
|
||
};
|
||
|
||
info!(" Model architecture (from training):");
|
||
info!(" - Input: 125 features (Wave D)");
|
||
info!(" - Hidden: [256, 128, 64]");
|
||
info!(" - Output: 3 actions (BUY, SELL, HOLD)");
|
||
|
||
// 4. Create WorkingDQN (auto-selects device internally)
|
||
info!(" Creating WorkingDQN network...");
|
||
let mut dqn = WorkingDQN::new(config)
|
||
.context("Failed to create WorkingDQN")?;
|
||
|
||
// Log actual device used
|
||
let actual_device = match dqn.device() {
|
||
Device::Cpu => "CPU",
|
||
Device::Cuda(_) => "CUDA:0",
|
||
_ => "Unknown",
|
||
};
|
||
info!(" Device used: {} (auto-selected)", actual_device);
|
||
|
||
// 5. Load weights from SafeTensors
|
||
info!(" Loading weights from SafeTensors...");
|
||
let model_path_str = model_path.to_str()
|
||
.ok_or_else(|| anyhow::anyhow!("Model path contains invalid UTF-8: {}", model_path.display()))?;
|
||
dqn.load_from_safetensors(model_path_str)
|
||
.context(format!(
|
||
"Failed to load weights from {}\n\
|
||
Possible causes:\n\
|
||
- File is corrupted (try retraining)\n\
|
||
- Wrong architecture (expected: 225 → [128,64,32] → 3)\n\
|
||
- Incompatible tensor types or shapes\n\
|
||
- Device memory issue ({})",
|
||
model_path.display(),
|
||
device_name
|
||
))?;
|
||
|
||
info!("✅ Component 2: DQN checkpoint loaded successfully");
|
||
info!(" Model: {} (8 tensors: layer_0-2.weight/bias, output.weight/bias)", model_path.display());
|
||
|
||
Ok(dqn)
|
||
}
|
||
|
||
// ============================================================================
|
||
// Component 3: Parquet Data Loading
|
||
// ============================================================================
|
||
//
|
||
// Production 225-feature extraction pipeline is now imported from:
|
||
// ml::data_loaders::load_parquet_data
|
||
//
|
||
// This function:
|
||
// - Loads Parquet files with schema-agnostic OHLCV extraction
|
||
// - Extracts 225 features using Wave C + Wave D production pipeline
|
||
// - Handles warmup period (50 bars for technical indicators)
|
||
// - Validates NaN/Inf values
|
||
// - Sorts bars chronologically for rolling windows
|
||
//
|
||
// See: ml/src/data_loaders/parquet_utils.rs for implementation
|
||
|
||
// ============================================================================
|
||
// Component 4: Inference Engine
|
||
// ============================================================================
|
||
|
||
/// Result of a single DQN inference
|
||
#[derive(Debug, Clone)]
|
||
struct InferenceResult {
|
||
action: usize, // 0=BUY, 1=SELL, 2=HOLD
|
||
q_values: [f64; 3], // Q-value for each action
|
||
latency_us: u64, // Microseconds for this inference
|
||
}
|
||
|
||
/// Run DQN inference on all feature vectors with progress tracking
|
||
///
|
||
/// # Arguments
|
||
/// * `dqn` - WorkingDQN for inference (mutable for select_action)
|
||
/// * `features` - 225-dimensional feature vectors
|
||
/// * `shutdown_flag` - Atomic flag for graceful shutdown (Ctrl+C)
|
||
///
|
||
/// # Returns
|
||
/// Vector of inference results (action, Q-values, latency per bar)
|
||
///
|
||
/// # Notes
|
||
/// - Uses WorkingDQN's select_action() (returns TradingAction enum)
|
||
/// - Uses WorkingDQN's forward() to get Q-values separately
|
||
/// - Handles NaN/Inf gracefully by logging warnings and skipping bars
|
||
/// - Tracks latency per inference in microseconds
|
||
/// - Progress bar shows real-time inference speed
|
||
/// - Respects shutdown flag for graceful interruption
|
||
fn run_inference(
|
||
dqn: &mut WorkingDQN,
|
||
features: Vec<[f64; 125]>,
|
||
shutdown_flag: &Arc<AtomicBool>,
|
||
) -> Result<Vec<InferenceResult>> {
|
||
info!("🔍 Component 4: Running DQN inference");
|
||
|
||
let total_bars = features.len();
|
||
if total_bars == 0 {
|
||
return Err(anyhow::anyhow!("No feature vectors provided for inference"));
|
||
}
|
||
|
||
info!(" Total bars to process: {}", total_bars);
|
||
|
||
let mut results = Vec::with_capacity(total_bars);
|
||
let mut total_inference_time_us = 0u64;
|
||
let mut skipped_bars = 0usize;
|
||
let start_time = Instant::now();
|
||
let mut last_progress_update = Instant::now();
|
||
|
||
// Run inference for each feature vector
|
||
for (i, feature_vec) in features.iter().enumerate() {
|
||
// Check for shutdown signal
|
||
if shutdown_flag.load(Ordering::Relaxed) {
|
||
warn!(" ⚠️ Shutdown signal received, stopping inference at bar {}/{}", i, total_bars);
|
||
break;
|
||
}
|
||
|
||
// Start timer for this inference
|
||
let timer = Instant::now();
|
||
|
||
// Convert f64 features to f32 for DQN network
|
||
let state_f32: Vec<f32> = feature_vec.iter().map(|&x| x as f32).collect();
|
||
|
||
// Use WorkingDQN's select_action() for greedy inference (epsilon=0.0)
|
||
// This returns TradingAction enum
|
||
let trading_action = match dqn.select_action(state_f32.as_slice()) {
|
||
Ok(a) => a,
|
||
Err(e) => {
|
||
if skipped_bars < 10 {
|
||
warn!(" ⚠️ Bar {}: select_action failed: {}. Skipping.", i, e);
|
||
}
|
||
skipped_bars += 1;
|
||
continue;
|
||
}
|
||
};
|
||
|
||
// Convert TradingAction to usize (0=BUY, 1=SELL, 2=HOLD)
|
||
let action = trading_action.to_int() as usize;
|
||
|
||
// Get Q-values separately using forward pass
|
||
use candle_core::Tensor;
|
||
let state_tensor = match Tensor::from_vec(
|
||
state_f32,
|
||
(1, 125),
|
||
dqn.device(),
|
||
) {
|
||
Ok(t) => t,
|
||
Err(e) => {
|
||
if skipped_bars < 10 {
|
||
warn!(" ⚠️ Bar {}: Failed to create tensor: {}. Skipping.", i, e);
|
||
}
|
||
skipped_bars += 1;
|
||
continue;
|
||
}
|
||
};
|
||
|
||
let q_values_tensor = match dqn.forward(&state_tensor) {
|
||
Ok(qv) => qv,
|
||
Err(e) => {
|
||
if skipped_bars < 10 {
|
||
warn!(" ⚠️ Bar {}: Forward pass failed: {}. Skipping.", i, e);
|
||
}
|
||
skipped_bars += 1;
|
||
continue;
|
||
}
|
||
};
|
||
|
||
// Extract Q-values from tensor [1, 3] -> [3]
|
||
let q_values_vec: Vec<f32> = match q_values_tensor.squeeze(0)
|
||
.and_then(|t| t.to_vec1()) {
|
||
Ok(v) => v,
|
||
Err(e) => {
|
||
if skipped_bars < 10 {
|
||
warn!(" ⚠️ Bar {}: Failed to extract Q-values: {}. Skipping.", i, e);
|
||
}
|
||
skipped_bars += 1;
|
||
continue;
|
||
}
|
||
};
|
||
|
||
// Validate Q-values shape
|
||
if q_values_vec.len() != 3 {
|
||
if skipped_bars < 10 {
|
||
warn!(
|
||
" ⚠️ Bar {}: Expected 3 Q-values, got {}. Skipping.",
|
||
i,
|
||
q_values_vec.len()
|
||
);
|
||
}
|
||
skipped_bars += 1;
|
||
continue;
|
||
}
|
||
|
||
let q_values: [f64; 3] = [
|
||
q_values_vec[0] as f64,
|
||
q_values_vec[1] as f64,
|
||
q_values_vec[2] as f64,
|
||
];
|
||
|
||
// Check for NaN/Inf in Q-values
|
||
if q_values.iter().any(|&q| !q.is_finite()) {
|
||
if skipped_bars < 10 {
|
||
warn!(
|
||
" ⚠️ Bar {}: Q-values contain NaN/Inf, skipping. Q-values: {:?}",
|
||
i,
|
||
q_values
|
||
);
|
||
}
|
||
skipped_bars += 1;
|
||
continue;
|
||
}
|
||
|
||
// Calculate latency for this inference
|
||
let latency_us = timer.elapsed().as_micros() as u64;
|
||
total_inference_time_us += latency_us;
|
||
|
||
// Store result
|
||
results.push(InferenceResult {
|
||
action,
|
||
q_values,
|
||
latency_us,
|
||
});
|
||
|
||
// Update progress every 1 second or every 10% completion
|
||
let should_update = last_progress_update.elapsed().as_secs() >= 1
|
||
|| (i + 1) % (total_bars / 10).max(1) == 0
|
||
|| i == 0
|
||
|| i == total_bars - 1;
|
||
|
||
if should_update {
|
||
let elapsed_sec = start_time.elapsed().as_secs_f64();
|
||
let avg_speed = if elapsed_sec > 0.0 {
|
||
(i + 1) as f64 / elapsed_sec
|
||
} else {
|
||
0.0
|
||
};
|
||
let progress_pct = ((i + 1) as f64 / total_bars as f64) * 100.0;
|
||
info!(
|
||
" Progress: {}/{} ({:.1}%) | Speed: {:.1} bars/sec | Skipped: {}",
|
||
i + 1,
|
||
total_bars,
|
||
progress_pct,
|
||
avg_speed,
|
||
skipped_bars
|
||
);
|
||
last_progress_update = Instant::now();
|
||
}
|
||
}
|
||
|
||
info!("✅ Component 4: Inference complete");
|
||
|
||
// Calculate summary statistics
|
||
let processed_bars = results.len();
|
||
let total_time_sec = start_time.elapsed().as_secs_f64();
|
||
let avg_latency_us = if processed_bars > 0 {
|
||
total_inference_time_us / processed_bars as u64
|
||
} else {
|
||
0
|
||
};
|
||
let avg_speed = if total_time_sec > 0.0 {
|
||
processed_bars as f64 / total_time_sec
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Log summary with detailed metrics
|
||
info!(" Inference Summary:");
|
||
info!(" - Total bars: {}", total_bars);
|
||
info!(" - Processed: {}", processed_bars);
|
||
info!(" - Skipped (NaN/Inf/errors): {}", skipped_bars);
|
||
info!(" - Skip rate: {:.2}%", (skipped_bars as f64 / total_bars as f64) * 100.0);
|
||
info!(" - Total time: {:.2}s", total_time_sec);
|
||
info!(" - Average latency: {}μs ({:.2}ms)", avg_latency_us, avg_latency_us as f64 / 1000.0);
|
||
info!(" - Average speed: {:.1} bars/sec", avg_speed);
|
||
|
||
// Validate results
|
||
if results.is_empty() {
|
||
return Err(anyhow::anyhow!(
|
||
"All {} inference attempts failed (likely NaN/Inf in Q-values or network errors)",
|
||
total_bars
|
||
));
|
||
}
|
||
|
||
Ok(results)
|
||
}
|
||
|
||
// ============================================================================
|
||
// Component 5: Metrics Calculator (imported from evaluate_dqn_component5.rs)
|
||
// ============================================================================
|
||
|
||
/// DQN-specific inference result for metrics calculation
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct DQNInferenceResult {
|
||
pub action: usize,
|
||
pub q_values: [f64; 3],
|
||
pub latency_us: u64,
|
||
}
|
||
|
||
/// Comprehensive evaluation metrics for DQN model validation
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct EvaluationMetrics {
|
||
pub total_bars: usize,
|
||
pub action_distribution: ActionDistribution,
|
||
pub avg_q_values: AvgQValues,
|
||
pub latency_stats: LatencyStats,
|
||
pub policy_consistency: PolicyConsistency,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct ActionDistribution {
|
||
pub buy_count: usize,
|
||
pub sell_count: usize,
|
||
pub hold_count: usize,
|
||
pub buy_pct: f64,
|
||
pub sell_pct: f64,
|
||
pub hold_pct: f64,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct AvgQValues {
|
||
pub buy_avg: f64,
|
||
pub sell_avg: f64,
|
||
pub hold_avg: f64,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct LatencyStats {
|
||
pub mean_us: f64,
|
||
pub median_us: u64,
|
||
pub p50_us: u64,
|
||
pub p95_us: u64,
|
||
pub p99_us: u64,
|
||
pub min_us: u64,
|
||
pub max_us: u64,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct PolicyConsistency {
|
||
pub total_switches: usize,
|
||
pub switch_rate: f64,
|
||
pub interpretation: String,
|
||
}
|
||
|
||
/// Calculate comprehensive evaluation metrics from inference results
|
||
fn calculate_metrics(results: &[InferenceResult]) -> Result<EvaluationMetrics> {
|
||
info!("📊 Component 5: Calculating metrics");
|
||
|
||
if results.is_empty() {
|
||
return Err(anyhow::anyhow!("Cannot calculate metrics from empty results"));
|
||
}
|
||
|
||
let total_bars = results.len();
|
||
info!(" Total bars: {}", total_bars);
|
||
|
||
// 1. Action distribution
|
||
let mut buy_count = 0usize;
|
||
let mut sell_count = 0usize;
|
||
let mut hold_count = 0usize;
|
||
|
||
for result in results {
|
||
match result.action {
|
||
0 => buy_count += 1,
|
||
1 => sell_count += 1,
|
||
2 => hold_count += 1,
|
||
_ => warn!(" ⚠️ Invalid action: {}", result.action),
|
||
}
|
||
}
|
||
|
||
let buy_pct = (buy_count as f64 / total_bars as f64) * 100.0;
|
||
let sell_pct = (sell_count as f64 / total_bars as f64) * 100.0;
|
||
let hold_pct = (hold_count as f64 / total_bars as f64) * 100.0;
|
||
|
||
// 2. Average Q-values per action
|
||
let mut buy_q_sum = 0.0;
|
||
let mut sell_q_sum = 0.0;
|
||
let mut hold_q_sum = 0.0;
|
||
|
||
for result in results {
|
||
match result.action {
|
||
0 => buy_q_sum += result.q_values[0],
|
||
1 => sell_q_sum += result.q_values[1],
|
||
2 => hold_q_sum += result.q_values[2],
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
let buy_avg = if buy_count > 0 { buy_q_sum / buy_count as f64 } else { 0.0 };
|
||
let sell_avg = if sell_count > 0 { sell_q_sum / sell_count as f64 } else { 0.0 };
|
||
let hold_avg = if hold_count > 0 { hold_q_sum / hold_count as f64 } else { 0.0 };
|
||
|
||
// 3. Latency statistics
|
||
let mut latencies: Vec<u64> = results.iter().map(|r| r.latency_us).collect();
|
||
latencies.sort_unstable();
|
||
|
||
let mean_us = latencies.iter().sum::<u64>() as f64 / latencies.len() as f64;
|
||
let median_us = latencies[latencies.len() / 2];
|
||
let p50_us = median_us;
|
||
let p95_us = latencies[(latencies.len() as f64 * 0.95) as usize];
|
||
let p99_us = latencies[(latencies.len() as f64 * 0.99) as usize];
|
||
let min_us = latencies[0];
|
||
let max_us = latencies[latencies.len() - 1];
|
||
|
||
// 4. Policy consistency
|
||
let mut total_switches = 0usize;
|
||
for i in 1..results.len() {
|
||
if results[i].action != results[i - 1].action {
|
||
total_switches += 1;
|
||
}
|
||
}
|
||
|
||
let switch_rate = if results.len() > 1 {
|
||
total_switches as f64 / (results.len() - 1) as f64
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
let interpretation = if switch_rate < 0.10 {
|
||
"Stable - Low adaptability".to_string()
|
||
} else if switch_rate <= 0.30 {
|
||
"Moderate - Healthy adaptive behavior".to_string()
|
||
} else {
|
||
"Volatile - High uncertainty or noise".to_string()
|
||
};
|
||
|
||
info!("✅ Component 5: Metrics calculated successfully");
|
||
|
||
Ok(EvaluationMetrics {
|
||
total_bars,
|
||
action_distribution: ActionDistribution {
|
||
buy_count,
|
||
sell_count,
|
||
hold_count,
|
||
buy_pct,
|
||
sell_pct,
|
||
hold_pct,
|
||
},
|
||
avg_q_values: AvgQValues {
|
||
buy_avg,
|
||
sell_avg,
|
||
hold_avg,
|
||
},
|
||
latency_stats: LatencyStats {
|
||
mean_us,
|
||
median_us,
|
||
p50_us,
|
||
p95_us,
|
||
p99_us,
|
||
min_us,
|
||
max_us,
|
||
},
|
||
policy_consistency: PolicyConsistency {
|
||
total_switches,
|
||
switch_rate,
|
||
interpretation,
|
||
},
|
||
})
|
||
}
|
||
|
||
// ============================================================================
|
||
// Component 6: Report Generator
|
||
// ============================================================================
|
||
|
||
/// Generate comprehensive evaluation report
|
||
///
|
||
/// # Arguments
|
||
/// * `metrics` - Evaluation metrics from Component 5
|
||
/// * `config` - CLI configuration
|
||
/// * `elapsed` - Total elapsed time (including data loading, inference, etc.)
|
||
///
|
||
/// # Returns
|
||
/// Result indicating success/failure of report generation
|
||
///
|
||
/// # Side Effects
|
||
/// - Prints formatted report to stdout
|
||
/// - Writes JSON file if config.output_json is specified
|
||
/// - Validates production readiness thresholds
|
||
fn generate_report(
|
||
metrics: &EvaluationMetrics,
|
||
config: &EvaluationConfig,
|
||
elapsed: std::time::Duration,
|
||
) -> Result<()> {
|
||
info!("📝 Component 6: Generating evaluation report");
|
||
|
||
// 1. Print header
|
||
println!("\n╔══════════════════════════════════════════════════════════════════════╗");
|
||
println!("║ DQN MODEL EVALUATION REPORT ║");
|
||
println!("╚══════════════════════════════════════════════════════════════════════╝");
|
||
println!();
|
||
|
||
// 2. Configuration summary
|
||
println!("═══ Configuration ═══");
|
||
println!(" Model path: {}", config.model_path.display());
|
||
println!(" Data file: {}", config.parquet_file.display());
|
||
println!(" Device: {}", config.device);
|
||
println!(" Warmup bars: {}", config.warmup_bars);
|
||
println!(" Total runtime: {:.2}s", elapsed.as_secs_f64());
|
||
println!();
|
||
|
||
// 3. Action distribution
|
||
println!("═══ Action Distribution ═══");
|
||
println!(" BUY: {:5} ({:5.1}%)",
|
||
metrics.action_distribution.buy_count,
|
||
metrics.action_distribution.buy_pct);
|
||
println!(" SELL: {:5} ({:5.1}%)",
|
||
metrics.action_distribution.sell_count,
|
||
metrics.action_distribution.sell_pct);
|
||
println!(" HOLD: {:5} ({:5.1}%)",
|
||
metrics.action_distribution.hold_count,
|
||
metrics.action_distribution.hold_pct);
|
||
println!(" ────────────────────");
|
||
println!(" Total: {} bars", metrics.total_bars);
|
||
println!();
|
||
|
||
// 4. Q-value statistics
|
||
println!("═══ Average Q-Values ═══");
|
||
println!(" BUY: {:8.4}", metrics.avg_q_values.buy_avg);
|
||
println!(" SELL: {:8.4}", metrics.avg_q_values.sell_avg);
|
||
println!(" HOLD: {:8.4}", metrics.avg_q_values.hold_avg);
|
||
println!();
|
||
|
||
// 5. Latency statistics
|
||
println!("═══ Latency Statistics ═══");
|
||
println!(" Mean: {:6.1} μs ({:.3} ms)",
|
||
metrics.latency_stats.mean_us,
|
||
metrics.latency_stats.mean_us / 1000.0);
|
||
println!(" Median: {:6} μs ({:.3} ms)",
|
||
metrics.latency_stats.median_us,
|
||
metrics.latency_stats.median_us as f64 / 1000.0);
|
||
println!(" P95: {:6} μs ({:.3} ms)",
|
||
metrics.latency_stats.p95_us,
|
||
metrics.latency_stats.p95_us as f64 / 1000.0);
|
||
println!(" P99: {:6} μs ({:.3} ms)",
|
||
metrics.latency_stats.p99_us,
|
||
metrics.latency_stats.p99_us as f64 / 1000.0);
|
||
println!(" Range: {:6} - {:6} μs",
|
||
metrics.latency_stats.min_us,
|
||
metrics.latency_stats.max_us);
|
||
println!();
|
||
|
||
// 6. Policy consistency
|
||
println!("═══ Policy Consistency ═══");
|
||
println!(" Switches: {} / {} bars",
|
||
metrics.policy_consistency.total_switches,
|
||
metrics.total_bars - 1);
|
||
println!(" Switch rate: {:.1}%",
|
||
metrics.policy_consistency.switch_rate * 100.0);
|
||
println!(" Interpretation: {}",
|
||
metrics.policy_consistency.interpretation);
|
||
println!();
|
||
|
||
// 7. Production readiness check
|
||
println!("═══ Production Readiness Check ═══");
|
||
|
||
let latency_ok = metrics.latency_stats.p99_us < 5_000;
|
||
println!(" {} Latency P99 < 5,000μs: {} (actual: {} μs)",
|
||
if latency_ok { "✅" } else { "❌" },
|
||
latency_ok,
|
||
metrics.latency_stats.p99_us);
|
||
|
||
let consistency_ok = metrics.policy_consistency.switch_rate >= 0.10
|
||
&& metrics.policy_consistency.switch_rate <= 0.30;
|
||
println!(" {} Policy switch rate 10-30%: {} (actual: {:.1}%)",
|
||
if consistency_ok { "✅" } else { "❌" },
|
||
consistency_ok,
|
||
metrics.policy_consistency.switch_rate * 100.0);
|
||
|
||
let balance_ok = metrics.action_distribution.buy_pct >= 5.0
|
||
&& metrics.action_distribution.sell_pct >= 5.0
|
||
&& metrics.action_distribution.hold_pct >= 5.0;
|
||
println!(" {} Balanced actions (each >5%): {}",
|
||
if balance_ok { "✅" } else { "⚠️ " },
|
||
balance_ok);
|
||
|
||
let q_ok = metrics.avg_q_values.buy_avg.is_finite()
|
||
&& metrics.avg_q_values.sell_avg.is_finite()
|
||
&& metrics.avg_q_values.hold_avg.is_finite();
|
||
println!(" {} Q-values finite: {}",
|
||
if q_ok { "✅" } else { "❌" },
|
||
q_ok);
|
||
|
||
println!();
|
||
let all_ok = latency_ok && consistency_ok && q_ok;
|
||
if all_ok {
|
||
println!("🎉 Model is PRODUCTION READY!");
|
||
} else {
|
||
println!("⚠️ Model requires further tuning before production deployment");
|
||
}
|
||
println!();
|
||
|
||
// 8. Export JSON (if configured)
|
||
if let Some(ref output_path) = config.output_json {
|
||
info!(" Exporting metrics to JSON: {}", output_path.display());
|
||
|
||
let json = serde_json::to_string_pretty(&metrics)
|
||
.context("Failed to serialize metrics to JSON")?;
|
||
|
||
std::fs::write(output_path, json)
|
||
.context(format!("Failed to write JSON to {}", output_path.display()))?;
|
||
|
||
println!("✅ Metrics exported to: {}", output_path.display());
|
||
println!();
|
||
}
|
||
|
||
info!("✅ Component 6: Report generated successfully");
|
||
|
||
Ok(())
|
||
}
|
||
|
||
// ============================================================================
|
||
// Component 6.5: Action Export (CSV Format)
|
||
// ============================================================================
|
||
|
||
/// Export DQN actions with timestamps to CSV format
|
||
///
|
||
/// # Arguments
|
||
/// * `results` - Inference results (action, Q-values, latency)
|
||
/// * `bars` - Original OHLCV bars (synchronized with results)
|
||
/// * `output_path` - Path to CSV file (will be created/overwritten)
|
||
///
|
||
/// # CSV Format
|
||
/// ```csv
|
||
/// timestamp,action,q_buy,q_sell,q_hold,open,high,low,close,volume
|
||
/// 2024-10-20T09:30:00.000000000Z,2,0.4523,-0.1234,0.8912,5720.25,5721.00,5719.50,5720.75,1234
|
||
/// ```
|
||
///
|
||
/// # Errors
|
||
/// Returns error if:
|
||
/// - Input vectors have mismatched lengths
|
||
/// - Output directory doesn't exist
|
||
/// - File write fails
|
||
fn export_actions_to_csv(
|
||
results: &[InferenceResult],
|
||
bars: &[OHLCVBar],
|
||
output_path: &Path,
|
||
) -> Result<()> {
|
||
use csv::Writer;
|
||
|
||
info!("💾 Component 6.5: Exporting actions to CSV");
|
||
info!(" Output path: {}", output_path.display());
|
||
|
||
// 1. Validate input synchronization
|
||
if results.len() != bars.len() {
|
||
return Err(anyhow::anyhow!(
|
||
"Input vectors have mismatched lengths: results={}, bars={}",
|
||
results.len(),
|
||
bars.len()
|
||
));
|
||
}
|
||
|
||
let total_rows = results.len();
|
||
info!(" Total rows to export: {}", total_rows);
|
||
|
||
// 2. Validate output path
|
||
if let Some(parent) = output_path.parent() {
|
||
if !parent.exists() {
|
||
return Err(anyhow::anyhow!(
|
||
"Output directory does not exist: {}\n\
|
||
Suggestion: Create directory first:\n\
|
||
mkdir -p {}",
|
||
parent.display(),
|
||
parent.display()
|
||
));
|
||
}
|
||
}
|
||
|
||
// 3. Create CSV writer
|
||
let mut wtr = Writer::from_path(output_path)
|
||
.context(format!("Failed to create CSV file: {}", output_path.display()))?;
|
||
|
||
// 4. Write CSV header
|
||
wtr.write_record(&[
|
||
"timestamp",
|
||
"action",
|
||
"q_buy",
|
||
"q_sell",
|
||
"q_hold",
|
||
"open",
|
||
"high",
|
||
"low",
|
||
"close",
|
||
"volume",
|
||
])
|
||
.context("Failed to write CSV header")?;
|
||
|
||
// 5. Write data rows
|
||
for (result, bar) in results.iter().zip(bars.iter()) {
|
||
// Format timestamp as RFC3339 with nanosecond precision
|
||
let timestamp_str = bar.timestamp.to_rfc3339_opts(
|
||
chrono::SecondsFormat::Nanos,
|
||
true,
|
||
);
|
||
|
||
wtr.write_record(&[
|
||
timestamp_str,
|
||
result.action.to_string(),
|
||
format!("{:.4}", result.q_values[0]), // q_buy
|
||
format!("{:.4}", result.q_values[1]), // q_sell
|
||
format!("{:.4}", result.q_values[2]), // q_hold
|
||
format!("{:.2}", bar.open),
|
||
format!("{:.2}", bar.high),
|
||
format!("{:.2}", bar.low),
|
||
format!("{:.2}", bar.close),
|
||
(bar.volume as u64).to_string(),
|
||
])
|
||
.context(format!("Failed to write CSV row for timestamp {}", bar.timestamp))?;
|
||
}
|
||
|
||
// 6. Flush writer
|
||
wtr.flush()
|
||
.context("Failed to flush CSV writer")?;
|
||
|
||
// 7. Calculate file size
|
||
let metadata = std::fs::metadata(output_path)
|
||
.context("Failed to read file metadata")?;
|
||
let file_size_bytes = metadata.len();
|
||
let file_size_kb = file_size_bytes as f64 / 1024.0;
|
||
|
||
info!("✅ Component 6.5: CSV export complete");
|
||
info!(" Rows written: {}", total_rows);
|
||
info!(" File size: {:.2} KB ({} bytes)", file_size_kb, file_size_bytes);
|
||
info!(" Average bytes/row: {:.1}", file_size_bytes as f64 / total_rows as f64);
|
||
|
||
Ok(())
|
||
}
|
||
|
||
// ============================================================================
|
||
// Component 7: Main Orchestrator
|
||
// ============================================================================
|
||
|
||
#[tokio::main]
|
||
async fn main() -> Result<()> {
|
||
// ========================================================================
|
||
// PHASE 1: INITIALIZATION
|
||
// ========================================================================
|
||
|
||
// 1.1: Parse CLI arguments
|
||
let config = EvaluationConfig::parse();
|
||
|
||
// 1.2: Setup tracing subscriber (stdout logging)
|
||
let log_level = if config.verbose {
|
||
tracing::Level::DEBUG
|
||
} else {
|
||
tracing::Level::INFO
|
||
};
|
||
|
||
let subscriber = FmtSubscriber::builder()
|
||
.with_max_level(log_level)
|
||
.with_target(false)
|
||
.with_level(true)
|
||
.finish();
|
||
|
||
tracing::subscriber::set_global_default(subscriber)
|
||
.context("Failed to set tracing subscriber")?;
|
||
|
||
// 1.3: Log startup banner
|
||
info!("╔══════════════════════════════════════════════════════════════════════╗");
|
||
info!("║ DQN Model Evaluation Pipeline - Component 7 Orchestrator ║");
|
||
info!("║ Version: 1.0.0 ║");
|
||
info!("║ Timestamp: {} ║",
|
||
chrono::Local::now().format("%Y-%m-%d %H:%M:%S"));
|
||
info!("╚══════════════════════════════════════════════════════════════════════╝");
|
||
info!("");
|
||
|
||
// 1.4: Log configuration
|
||
info!("Configuration:");
|
||
info!(" • Model path: {}", config.model_path.display());
|
||
info!(" • Parquet file: {}", config.parquet_file.display());
|
||
info!(" • Device: {}", config.device);
|
||
info!(" • Warmup bars: {}", config.warmup_bars);
|
||
if let Some(ref output_path) = config.output_json {
|
||
info!(" • JSON output: {}", output_path.display());
|
||
}
|
||
info!(" • Verbose logging: {}", config.verbose);
|
||
info!("");
|
||
|
||
// 1.5: Validate configuration
|
||
info!("🔍 Validating configuration...");
|
||
config.validate()
|
||
.context("Configuration validation failed")?;
|
||
info!("✅ Configuration validated successfully");
|
||
info!("");
|
||
|
||
// 1.6: Setup graceful shutdown handler
|
||
let shutdown_flag = Arc::new(AtomicBool::new(false));
|
||
let shutdown_clone = shutdown_flag.clone();
|
||
|
||
tokio::spawn(async move {
|
||
let ctrl_c = signal::ctrl_c();
|
||
|
||
#[cfg(unix)]
|
||
{
|
||
use tokio::signal::unix::{signal, SignalKind};
|
||
let mut sigterm = signal(SignalKind::terminate())
|
||
.expect("Failed to setup SIGTERM handler");
|
||
|
||
tokio::select! {
|
||
_ = ctrl_c => {
|
||
info!("🛑 Received Ctrl+C, initiating graceful shutdown...");
|
||
}
|
||
_ = sigterm.recv() => {
|
||
info!("🛑 Received SIGTERM, initiating graceful shutdown...");
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(not(unix))]
|
||
{
|
||
ctrl_c.await.expect("Failed to listen for Ctrl+C");
|
||
info!("🛑 Received Ctrl+C, initiating graceful shutdown...");
|
||
}
|
||
|
||
shutdown_clone.store(true, Ordering::Relaxed);
|
||
});
|
||
|
||
info!("✅ Graceful shutdown handler registered (Ctrl+C / SIGTERM)");
|
||
info!("");
|
||
|
||
// Start total elapsed timer
|
||
let total_start = Instant::now();
|
||
|
||
// ========================================================================
|
||
// PHASE 2: PARALLEL DATA + MODEL LOADING
|
||
// ========================================================================
|
||
|
||
info!("⚡ Phase 2: Parallel loading (Data + Model)");
|
||
info!("");
|
||
|
||
// Clone paths for async move
|
||
let parquet_path = config.parquet_file.clone();
|
||
let model_path = config.model_path.clone();
|
||
let device_str = config.device.clone();
|
||
let warmup_bars = config.warmup_bars;
|
||
let need_bars = config.export_actions.is_some();
|
||
|
||
// Parallel loading using tokio::try_join!
|
||
// CRITICAL: Always load with timestamps/bars for preprocessing (close prices needed)
|
||
let (data_result, dqn_result) = tokio::try_join!(
|
||
tokio::task::spawn_blocking(move || {
|
||
load_parquet_data_with_timestamps(&parquet_path, warmup_bars)
|
||
}),
|
||
tokio::task::spawn_blocking(move || {
|
||
load_dqn_model(&model_path, &device_str)
|
||
})
|
||
).context("Parallel loading failed")?;
|
||
|
||
// Unwrap the spawn_blocking JoinError and the function Result
|
||
let (mut features, _timestamps, bars) = data_result?;
|
||
let mut dqn = dqn_result?;
|
||
|
||
// Keep bars for action export if requested
|
||
let bars_opt = if need_bars {
|
||
Some(bars.clone())
|
||
} else {
|
||
None
|
||
};
|
||
|
||
info!("");
|
||
info!("✅ Phase 2 complete: Data and model loaded in parallel");
|
||
info!("");
|
||
|
||
// ========================================================================
|
||
// PHASE 2.5: PREPROCESSING (Match Training Pipeline)
|
||
// ========================================================================
|
||
|
||
info!("🔬 Phase 2.5: Applying preprocessing (log returns + normalization + clipping)");
|
||
info!("");
|
||
|
||
// Extract close prices from OHLCV bars
|
||
let close_prices_f64: Vec<f64> = bars.iter().map(|b| b.close).collect();
|
||
let close_prices_f32: Vec<f32> = close_prices_f64.iter().map(|&x| x as f32).collect();
|
||
|
||
info!(" • Input data: {} bars", close_prices_f32.len());
|
||
info!(" • Close price range: [{:.2}, {:.2}]",
|
||
close_prices_f32.iter().cloned().fold(f32::INFINITY, f32::min),
|
||
close_prices_f32.iter().cloned().fold(f32::NEG_INFINITY, f32::max));
|
||
|
||
// Create tensor on same device as model
|
||
let device = dqn.device().clone();
|
||
let close_tensor = Tensor::from_slice(&close_prices_f32, (close_prices_f32.len(),), &device)
|
||
.context("Failed to create close price tensor for preprocessing")?;
|
||
|
||
// Configure preprocessing (match training hyperparameters)
|
||
let preprocess_config = PreprocessConfig {
|
||
window_size: 50, // Default preprocessing window
|
||
clip_sigma: 5.0, // Default clip sigma
|
||
use_log_returns: true,
|
||
};
|
||
|
||
info!(" • Window size: {}", preprocess_config.window_size);
|
||
info!(" • Clip sigma: ±{:.1}σ", preprocess_config.clip_sigma);
|
||
info!(" • Method: log returns + windowed normalization");
|
||
|
||
// Apply preprocessing pipeline
|
||
let preprocessed_tensor = preprocess_prices(&close_tensor, preprocess_config)
|
||
.context("Preprocessing failed - check close prices for NaN/Inf/zeros")?;
|
||
|
||
let preprocessed_vec: Vec<f32> = preprocessed_tensor.to_vec1()
|
||
.context("Failed to convert preprocessed tensor to vec")?;
|
||
|
||
// Convert f32 to f64 for consistency with feature pipeline
|
||
let preprocessed_f64: Vec<f64> = preprocessed_vec.iter().map(|&x| x as f64).collect();
|
||
|
||
// Compute statistics for validation
|
||
let warmup = preprocess_config.window_size as usize;
|
||
let post_warmup: Vec<f64> = preprocessed_f64[warmup..].to_vec();
|
||
let mean = post_warmup.iter().sum::<f64>() / post_warmup.len() as f64;
|
||
let variance = post_warmup.iter().map(|&x| (x - mean).powi(2)).sum::<f64>() / post_warmup.len() as f64;
|
||
let std = variance.sqrt();
|
||
let min_val = post_warmup.iter().cloned().fold(f64::INFINITY, f64::min);
|
||
let max_val = post_warmup.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
|
||
|
||
info!("✅ Preprocessing complete:");
|
||
info!(" • Mean: {:.6} (expected ~0 for normalized data)", mean);
|
||
info!(" • Std: {:.4} (expected ~1 for normalized data)", std);
|
||
info!(" • Range: [{:.4}, {:.4}] (clipped at ±{:.1}σ)", min_val, max_val, preprocess_config.clip_sigma);
|
||
|
||
// Replace close prices in feature vectors with preprocessed values
|
||
// Feature vector structure: [101 Wave C features + 24 Wave D features]
|
||
// Close price is at index 3 (after timestamp, open, high, low)
|
||
info!(" • Replacing close prices in feature vectors with preprocessed values...");
|
||
|
||
for (i, feature_vec) in features.iter_mut().enumerate() {
|
||
if i < preprocessed_f64.len() {
|
||
feature_vec[3] = preprocessed_f64[i]; // Index 3 is close price
|
||
}
|
||
}
|
||
|
||
info!(" • Updated {} feature vectors with preprocessed close prices", features.len());
|
||
info!("");
|
||
info!("✅ Phase 2.5 complete: Features preprocessed to match training distribution");
|
||
info!("");
|
||
|
||
// ========================================================================
|
||
// PHASE 3: SEQUENTIAL INFERENCE
|
||
// ========================================================================
|
||
|
||
info!("🔍 Phase 3: Sequential inference");
|
||
info!("");
|
||
|
||
// Run inference
|
||
let inference_results = run_inference(&mut dqn, features, &shutdown_flag)
|
||
.context("Inference failed")?;
|
||
|
||
// Check if interrupted
|
||
if shutdown_flag.load(Ordering::Relaxed) {
|
||
warn!("⚠️ Evaluation interrupted by shutdown signal");
|
||
warn!(" Processed {} bars before interruption", inference_results.len());
|
||
|
||
// Still generate partial report
|
||
info!("");
|
||
info!("📊 Generating partial evaluation report...");
|
||
|
||
if !inference_results.is_empty() {
|
||
let partial_metrics = calculate_metrics(&inference_results)?;
|
||
let elapsed = total_start.elapsed();
|
||
generate_report(&partial_metrics, &config, elapsed)?;
|
||
}
|
||
|
||
info!("💾 Partial results saved, safe to terminate");
|
||
return Ok(());
|
||
}
|
||
|
||
info!("");
|
||
info!("✅ Phase 3 complete: Inference finished");
|
||
info!("");
|
||
|
||
// ========================================================================
|
||
// PHASE 4: METRICS CALCULATION
|
||
// ========================================================================
|
||
|
||
info!("📊 Phase 4: Metrics calculation");
|
||
info!("");
|
||
|
||
let metrics = calculate_metrics(&inference_results)
|
||
.context("Metrics calculation failed")?;
|
||
|
||
info!("");
|
||
info!("✅ Phase 4 complete: Metrics calculated");
|
||
info!("");
|
||
|
||
// ========================================================================
|
||
// PHASE 5: REPORT GENERATION
|
||
// ========================================================================
|
||
|
||
info!("📝 Phase 5: Report generation");
|
||
info!("");
|
||
|
||
let total_elapsed = total_start.elapsed();
|
||
generate_report(&metrics, &config, total_elapsed)
|
||
.context("Report generation failed")?;
|
||
|
||
info!("");
|
||
info!("✅ Phase 5 complete: Report generated");
|
||
info!("");
|
||
|
||
// ========================================================================
|
||
// PHASE 5.5: OPTIONAL ACTION EXPORT
|
||
// ========================================================================
|
||
|
||
if let Some(ref export_path) = config.export_actions {
|
||
info!("📤 Phase 5.5: Exporting actions to CSV");
|
||
info!("");
|
||
|
||
// Verify we have bars available
|
||
if let Some(bars) = bars_opt.as_ref() {
|
||
export_actions_to_csv(
|
||
&inference_results,
|
||
bars,
|
||
export_path,
|
||
)
|
||
.context("Action export failed")?;
|
||
|
||
info!("");
|
||
info!("✅ Phase 5.5 complete: Actions exported to {}", export_path.display());
|
||
info!("");
|
||
} else {
|
||
warn!("⚠️ Cannot export actions: bars were not loaded (internal error)");
|
||
}
|
||
}
|
||
|
||
// ========================================================================
|
||
// COMPLETION
|
||
// ========================================================================
|
||
|
||
info!("╔══════════════════════════════════════════════════════════════════════╗");
|
||
info!("║ EVALUATION COMPLETE ║");
|
||
info!("╚══════════════════════════════════════════════════════════════════════╝");
|
||
info!(" Total runtime: {:.2}s", total_elapsed.as_secs_f64());
|
||
info!("");
|
||
|
||
Ok(())
|
||
}
|