#![allow( clippy::assertions_on_constants, clippy::assertions_on_result_states, clippy::clone_on_copy, clippy::decimal_literal_representation, clippy::doc_markdown, clippy::empty_line_after_doc_comments, clippy::field_reassign_with_default, clippy::get_unwrap, clippy::identity_op, clippy::inconsistent_digit_grouping, clippy::indexing_slicing, clippy::integer_division, clippy::len_zero, clippy::let_underscore_must_use, clippy::manual_div_ceil, clippy::manual_let_else, clippy::manual_range_contains, clippy::modulo_arithmetic, clippy::needless_range_loop, clippy::non_ascii_literal, clippy::redundant_clone, clippy::shadow_reuse, clippy::shadow_same, clippy::shadow_unrelated, clippy::single_match_else, clippy::str_to_string, clippy::string_slice, clippy::tests_outside_test_module, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::unseparated_literal_suffix, clippy::use_debug, clippy::useless_vec, clippy::wildcard_enum_match_arm, clippy::else_if_without_else, clippy::expect_used, clippy::missing_const_for_fn, clippy::similar_names, clippy::type_complexity, clippy::collapsible_else_if, clippy::doc_lazy_continuation, clippy::items_after_test_module, clippy::map_clone, clippy::multiple_unsafe_ops_per_block, clippy::unwrap_or_default, clippy::assign_op_pattern, clippy::needless_borrow, clippy::println_empty_string, clippy::unnecessary_cast, clippy::used_underscore_binding, clippy::create_dir, clippy::implicit_saturating_sub, clippy::exit, clippy::expect_fun_call, clippy::too_many_arguments, clippy::unnecessary_map_or, clippy::unwrap_used, dead_code, unused_imports, unused_variables, clippy::cloned_ref_to_slice_refs, clippy::neg_multiply, clippy::while_let_loop, clippy::bool_assert_comparison, clippy::excessive_precision, clippy::trivially_copy_pass_by_ref, clippy::op_ref, clippy::redundant_closure, clippy::unnecessary_lazy_evaluations, clippy::if_then_some_else_none, clippy::unnecessary_to_owned, clippy::single_component_path_imports, )] //! Test suite to verify MAMBA-2 accuracy calculation fix //! //! This test suite validates the fix for the accuracy calculation bug where //! mean_all() was incorrectly used on incompatible tensor shapes, causing //! 99% error rates and 3-12% "accuracy" despite normal loss convergence. // candle eliminated — test uses native APIs use tracing::info; /// Test accuracy calculation with single-value target (basic case) #[test] fn test_accuracy_calculation_single_value() { // Simulate normalized predictions and targets (pure arithmetic — no GPU needed) let pred_val: f64 = 0.48; // Predict 0.48 let target_val: f64 = 0.50; // Target 0.50 // Expected MAPE: |0.48 - 0.50| / 0.50 = 0.04 = 4% error // Should be CORRECT with 30% threshold // NEW FIX (scalar extraction): let error = ((pred_val - target_val) / target_val).abs(); assert!( (error - 0.04).abs() < 1e-6, "Expected 4% error, got {}%", error * 100.0 ); // With 30% threshold, this should be marked "correct" assert!( error < 0.3, "4% error should be considered correct with 30% threshold" ); // Also verify it passes the stricter 10% threshold assert!( error < 0.1, "4% error should also pass 10% threshold (but 10% is too strict for production)" ); } /// Test accuracy calculation with multi-dimensional output (realistic MAMBA-2 case) #[test] fn test_accuracy_calculation_multi_dim_output() { // Simulate realistic MAMBA-2 output: [1, 1, 54] (pure arithmetic) let mut output_data = vec![0.0_f64; 54]; output_data[0] = 0.48; // First feature is regression target let target_val: f64 = 0.50; // OLD BUG (mean_all): Would give 99% error let old_pred_mean: f64 = output_data.iter().sum::() / output_data.len() as f64; // old_pred_mean ≈ 0.48/54 ≈ 0.0089 let old_target_mean = target_val; // 0.50 let old_error = ((old_pred_mean - old_target_mean) / old_target_mean).abs(); info!( old_pred_mean, old_target_mean, old_error_pct = old_error * 100.0, "OLD BUG: pred_mean, target_mean, error" ); assert!( old_error > 0.9, "OLD BUG: Should show ~99% error due to mean_all() on 54-dim output" ); // NEW FIX (scalar extraction from first feature): let new_pred_val = output_data[0]; // 0.48 let new_target_val = target_val; // 0.50 let new_error = ((new_pred_val - new_target_val) / new_target_val).abs(); info!( new_pred_val, new_target_val, new_error_pct = new_error * 100.0, "NEW FIX: pred_val, target_val, error" ); assert!( (new_error - 0.04).abs() < 1e-6, "NEW FIX: Should show 4% error, got {}%", new_error * 100.0 ); // NEW: 4% error -> "correct" with 30% threshold assert!( new_error < 0.3, "NEW FIX: 4% error should be considered correct" ); } /// Test edge case: target near zero (avoid division by zero) #[test] fn test_accuracy_calculation_near_zero_target() { let pred_val: f64 = 0.02; let target_val: f64 = 1e-9; // Very near zero (below 1e-8) // For targets near zero (< 1e-8), use absolute error instead of percentage let error = if target_val.abs() > 1e-8 { ((pred_val - target_val) / target_val).abs() } else { (pred_val - target_val).abs() }; info!( pred_val, target_val, error, using_absolute_error = target_val.abs() <= 1e-8, "Near-zero target" ); // Should use absolute error (0.02 - 1e-9 ≈ 0.02) assert!( (error - 0.02).abs() < 1e-6, "Expected absolute error ~0.02, got {}", error ); } /// Test threshold sensitivity: 10% vs 30% #[test] fn test_threshold_comparison() { // Test different error levels (pure arithmetic) let test_cases: Vec<(f64, f64, f64)> = vec![ (0.48, 0.50, 0.04), // 4% error - should pass both thresholds (0.42, 0.50, 0.16), // 16% error - should pass 30% but fail 10% (0.30, 0.50, 0.40), // 40% error - should fail both thresholds ]; for (pred_val, target_val, expected_error) in test_cases { let error = ((pred_val - target_val) / target_val).abs(); info!( pred_val, target_val, error_pct = error * 100.0, expected_error_pct = expected_error * 100.0, "Threshold comparison" ); assert!( (error - expected_error).abs() < 1e-6, "Expected {:.2}% error, got {:.2}%", expected_error * 100.0, error * 100.0 ); // Check threshold behavior let passes_10 = error < 0.1; let passes_30 = error < 0.3; match expected_error { e if e < 0.1 => { assert!( passes_10, "Error {:.2}% should pass 10% threshold", e * 100.0 ); assert!( passes_30, "Error {:.2}% should pass 30% threshold", e * 100.0 ); }, e if e < 0.3 => { assert!( !passes_10, "Error {:.2}% should fail 10% threshold", e * 100.0 ); assert!( passes_30, "Error {:.2}% should pass 30% threshold", e * 100.0 ); }, e => { assert!( !passes_10, "Error {:.2}% should fail 10% threshold", e * 100.0 ); assert!( !passes_30, "Error {:.2}% should fail 30% threshold", e * 100.0 ); }, } } } /// Test realistic ES futures price prediction scenario #[test] fn test_realistic_futures_prediction() { // ES futures: price range $5000-$5200 (normalized to 0.0-1.0) // Example: predict $5095, actual $5100 // Normalized: predict 0.475, actual 0.5 // Error: $5 out of $200 range = 2.5% in price space // MAPE: |0.475 - 0.5| / 0.5 = 5% in normalized space let pred_val: f64 = 0.475; let target_val: f64 = 0.50; let error_pct = ((pred_val - target_val) / target_val).abs(); info!( pred_val, target_val, error_pct = error_pct * 100.0, "Realistic ES prediction" ); // 5% error should be considered EXCELLENT for financial prediction assert!(error_pct < 0.1, "5% error should easily pass 10% threshold"); assert!(error_pct < 0.3, "5% error should easily pass 30% threshold"); // In price terms: $5 error on $5100 = 0.098% in absolute terms // This is EXCELLENT prediction accuracy for intraday futures } /// Test batch of predictions to estimate accuracy rate #[test] fn test_batch_accuracy_estimation() { // Simulate 100 predictions with varying errors (pure arithmetic) let mut errors = vec![]; // Generate predictions with normal distribution around target for i in 0..100 { let target_val: f64 = 0.5; // Add noise: +/-15% RMSE -> most predictions within +/-30% let noise = (i as f64 / 100.0 - 0.5) * 0.3; // -15% to +15% let pred_val = target_val + noise; let error = ((pred_val - target_val) / target_val).abs(); errors.push(error); } // Count how many predictions are "correct" with different thresholds let correct_10 = errors.iter().filter(|&&e| e < 0.1).count(); let correct_30 = errors.iter().filter(|&&e| e < 0.3).count(); let accuracy_10 = correct_10 as f64 / 100.0; let accuracy_30 = correct_30 as f64 / 100.0; info!( accuracy_10_pct = accuracy_10 * 100.0, accuracy_30_pct = accuracy_30 * 100.0, "Batch accuracy estimation (100 samples)" ); // With ±15% noise, expect: // - 10% threshold: ~33% accuracy (1/3 within ±10%) // - 30% threshold: ~100% accuracy (all within ±15%) assert!( accuracy_10 > 0.20 && accuracy_10 < 0.50, "Expected 20-50% accuracy with 10% threshold, got {:.1}%", accuracy_10 * 100.0 ); assert!( accuracy_30 > 0.90, "Expected >90% accuracy with 30% threshold, got {:.1}%", accuracy_30 * 100.0 ); } /// Integration test: verify fix aligns accuracy with loss #[test] fn test_accuracy_loss_alignment() { // Given: Training loss = 0.071 (MSE in normalized space) // RMSE = sqrt(0.071) = 0.266 = 26.6% error // // With 30% MAPE threshold: // - Predictions with <30% error marked "correct" // - RMSE 26.6% means ~68% of predictions within ±30% (assuming normal distribution) // - Expected accuracy: ~68-75% let expected_rmse = 0.266; let threshold = 0.3; // Approximate: for RMSE R and threshold T, accuracy ≈ erf(T/R*sqrt(2)) // For R=0.266, T=0.3: accuracy ≈ erf(1.13*sqrt(2)) ≈ erf(1.6) ≈ 0.976 // But this assumes normal distribution centered at target // // More conservative estimate: if RMSE=26.6%, about 68-75% within ±30% info!( expected_rmse_pct = expected_rmse * 100.0, threshold_pct = threshold * 100.0, "Loss-Accuracy Alignment" ); info!("Expected accuracy: 68-75% (most predictions within threshold)"); // Verify threshold is reasonable for this RMSE assert!( threshold > expected_rmse, "Threshold ({:.1}%) should be greater than RMSE ({:.1}%) for reasonable accuracy", threshold * 100.0, expected_rmse * 100.0 ); }