#![allow( clippy::assertions_on_constants, clippy::assertions_on_result_states, clippy::clone_on_copy, clippy::decimal_literal_representation, clippy::doc_markdown, clippy::empty_line_after_doc_comments, clippy::field_reassign_with_default, clippy::get_unwrap, clippy::identity_op, clippy::inconsistent_digit_grouping, clippy::indexing_slicing, clippy::integer_division, clippy::len_zero, clippy::let_underscore_must_use, clippy::manual_div_ceil, clippy::manual_let_else, clippy::manual_range_contains, clippy::modulo_arithmetic, clippy::needless_range_loop, clippy::non_ascii_literal, clippy::redundant_clone, clippy::shadow_reuse, clippy::shadow_same, clippy::shadow_unrelated, clippy::single_match_else, clippy::str_to_string, clippy::string_slice, clippy::tests_outside_test_module, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::unseparated_literal_suffix, clippy::use_debug, clippy::useless_vec, clippy::wildcard_enum_match_arm, clippy::else_if_without_else, clippy::expect_used, clippy::missing_const_for_fn, clippy::similar_names, clippy::type_complexity, clippy::collapsible_else_if, clippy::doc_lazy_continuation, clippy::items_after_test_module, clippy::map_clone, clippy::multiple_unsafe_ops_per_block, clippy::unwrap_or_default, clippy::assign_op_pattern, clippy::needless_borrow, clippy::println_empty_string, clippy::unnecessary_cast, clippy::used_underscore_binding, clippy::create_dir, clippy::implicit_saturating_sub, clippy::exit, clippy::expect_fun_call, clippy::too_many_arguments, clippy::unnecessary_map_or, clippy::unwrap_used, dead_code, unused_imports, unused_variables, clippy::cloned_ref_to_slice_refs, clippy::neg_multiply, clippy::while_let_loop, clippy::bool_assert_comparison, clippy::excessive_precision, clippy::trivially_copy_pass_by_ref, clippy::op_ref, clippy::redundant_closure, clippy::unnecessary_lazy_evaluations, clippy::if_then_some_else_none, clippy::unnecessary_to_owned, clippy::single_component_path_imports, )] // WAVE 16Q: OHLCV Mutation Bug Regression Test // // **Bug Description**: DQN training corrupts OHLCV bar close prices from realistic values // ($4000-$5500 for ES futures) to preprocessed z-scores (0.001-2.0 range). This causes: // 1. max_position explosion to 100M (should be 10-50) // 2. P&L calculation completely broken // 3. Reward signals meaningless // // **Root Cause**: target[2] (raw_current) receives preprocessed z-scores instead of raw dollars, // suggesting either data aliasing, index mismatch, or hidden mutation in preprocessing/feature extraction. // // **Fix Strategy**: Extract raw close prices into separate vector before any preprocessing or // feature extraction to guarantee data preservation. use anyhow::Result; use ml::trainers::DQNHyperparameters; use ml::trainers::dqn::DQNTrainer; use tracing::{info, warn}; #[tokio::test] async fn test_ohlcv_raw_prices_preserved_after_preprocessing() -> Result<()> { // Initialize trainer with preprocessing enabled let mut hyperparams = DQNHyperparameters::conservative(); hyperparams.enable_preprocessing = true; hyperparams.preprocessing_window = 50; hyperparams.preprocessing_clip_sigma = 3.0; hyperparams.epochs = 1; // Single epoch test let mut trainer = DQNTrainer::new(hyperparams)?; // Use real DBN data (same as production) let data_dir = "test_data/real/databento/ml_training"; // Run 1 epoch training to trigger preprocessing and feature extraction // Use no-op checkpoint callback (just return empty string) let noop_callback = |_epoch: usize, _data: Vec, _is_best: bool| -> Result { Ok(String::new()) }; let result = trainer.train(data_dir, noop_callback).await; // Training may fail for other reasons, but we need to check OHLCV corruption // even if training fails (the bug manifests during data loading) if let Err(e) = result { warn!(error = %e, "Training failed (may be expected)"); } // Get validation data to inspect target values let val_data = trainer.get_val_data(); // Validation: target[2] should contain raw prices ($4000-$5500), NOT z-scores (0.001-2.0) let mut raw_prices_valid = 0; let mut preprocessed_zscores = 0; let mut min_raw = f64::MAX; let mut max_raw = f64::MIN; for (_features, target) in val_data.iter().take(100) { let raw_current = target[2]; min_raw = min_raw.min(raw_current); max_raw = max_raw.max(raw_current); // ES futures trade around $4000-$5500 // Z-scores are in range -3 to +3 (clipped), typically 0.001 to 2.0 if raw_current.abs() > 100.0 { // Realistic dollar price raw_prices_valid += 1; } else { // Suspiciously small (likely z-score) preprocessed_zscores += 1; } } info!( raw_prices_valid, preprocessed_zscores, min_raw = format_args!("{min_raw:.2}"), max_raw = format_args!("{max_raw:.2}"), "OHLCV validation results" ); // ASSERTION: At least 90% of samples should have realistic raw prices assert!( raw_prices_valid >= 90, "❌ BUG CONFIRMED: target[2] contains z-scores instead of raw prices! \ Valid: {}/100, Z-scores: {}/100, Range: [{:.2}, {:.2}]", raw_prices_valid, preprocessed_zscores, min_raw, max_raw ); // ASSERTION: Min/max should be in realistic ES futures range ($4000-$5500) assert!( min_raw > 3000.0 && max_raw < 6000.0, "❌ BUG CONFIRMED: Raw prices outside realistic range! \ Expected: $4000-$5500, Got: [{:.2}, {:.2}]", min_raw, max_raw ); info!("OHLCV raw prices preserved correctly"); Ok(()) } #[tokio::test] async fn test_target_2_is_raw_price_not_zscore() -> Result<()> { // Regression test: Verify target[2] contains raw dollars, not preprocessed z-scores let mut hyperparams = DQNHyperparameters::conservative(); hyperparams.enable_preprocessing = true; // Enable preprocessing to trigger bug hyperparams.epochs = 1; let mut trainer = DQNTrainer::new(hyperparams)?; let data_dir = "test_data/real/databento/ml_training"; let noop_callback = |_: usize, _: Vec, _: bool| -> Result { Ok(String::new()) }; let _ = trainer.train(data_dir, noop_callback).await; let val_data = trainer.get_val_data(); assert!(!val_data.is_empty(), "Validation data should not be empty"); // Check first 10 samples for (i, (_features, target)) in val_data.iter().take(10).enumerate() { let raw_current = target[2]; info!(sample = i, target_2 = format_args!("{raw_current:.2}"), "sample target[2]"); // Assertion: raw_current should be in realistic price range, NOT z-score range assert!( raw_current.abs() > 100.0, "Sample {}: target[2] = {:.6} looks like a z-score, not raw price", i, raw_current ); } info!("All samples have realistic raw prices in target[2]"); Ok(()) } #[tokio::test] async fn test_max_position_realistic_not_100m() -> Result<()> { // Regression test: max_position should be 10-50, NOT 100M (indicates price corruption) let mut hyperparams = DQNHyperparameters::conservative(); hyperparams.epochs = 1; let mut trainer = DQNTrainer::new(hyperparams)?; let data_dir = "test_data/real/databento/ml_training"; let noop_callback = |_: usize, _: Vec, _: bool| -> Result { Ok(String::new()) }; let _ = trainer.train(data_dir, noop_callback).await; let val_data = trainer.get_val_data(); // Simulate position calculation (simplified) let mut max_position = 0.0f64; for (_features, target) in val_data.iter() { let price_change = (target[1] - target[0]).abs(); // Typical position sizing based on price change let position = 10000.0 / price_change.max(0.01); // Avoid div by zero max_position = max_position.max(position); } info!(max_position = format_args!("{max_position:.2}"), "Max position calculated"); // ASSERTION: max_position should be reasonable (10-50), NOT astronomical (100M) assert!( max_position < 1000.0, "❌ BUG CONFIRMED: max_position = {:.2e} suggests price corruption (expected <1000)", max_position ); info!(max_position = format_args!("{max_position:.2}"), "max_position is realistic"); Ok(()) }