5d6e79263c
tft_real_dbn_data: StreamTensor::from_vec, quantile loss returns f32 ppo_recurrent_integration: PPO::new() API, get_policy_state &[f32] test_dbn_sequence_256: to_host + manual indexing instead of .i() ops ppo_checkpoint_roundtrip: save/load_checkpoint(&PathBuf) API mamba2_accuracy_fix: pure f64 arithmetic, no GPU tensors needed ppo_lstm_training_loop: PPO::new() API ppo_step_counter_fix: new checkpoint API ppo_recurrent_performance: forward_host, LSTM batch_size arg Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
377 lines
12 KiB
Rust
377 lines
12 KiB
Rust
#![allow(
|
|
clippy::assertions_on_constants,
|
|
clippy::assertions_on_result_states,
|
|
clippy::clone_on_copy,
|
|
clippy::decimal_literal_representation,
|
|
clippy::doc_markdown,
|
|
clippy::empty_line_after_doc_comments,
|
|
clippy::field_reassign_with_default,
|
|
clippy::get_unwrap,
|
|
clippy::identity_op,
|
|
clippy::inconsistent_digit_grouping,
|
|
clippy::indexing_slicing,
|
|
clippy::integer_division,
|
|
clippy::len_zero,
|
|
clippy::let_underscore_must_use,
|
|
clippy::manual_div_ceil,
|
|
clippy::manual_let_else,
|
|
clippy::manual_range_contains,
|
|
clippy::modulo_arithmetic,
|
|
clippy::needless_range_loop,
|
|
clippy::non_ascii_literal,
|
|
clippy::redundant_clone,
|
|
clippy::shadow_reuse,
|
|
clippy::shadow_same,
|
|
clippy::shadow_unrelated,
|
|
clippy::single_match_else,
|
|
clippy::str_to_string,
|
|
clippy::string_slice,
|
|
clippy::tests_outside_test_module,
|
|
clippy::too_many_lines,
|
|
clippy::unnecessary_wraps,
|
|
clippy::unseparated_literal_suffix,
|
|
clippy::use_debug,
|
|
clippy::useless_vec,
|
|
clippy::wildcard_enum_match_arm,
|
|
clippy::else_if_without_else,
|
|
clippy::expect_used,
|
|
clippy::missing_const_for_fn,
|
|
clippy::similar_names,
|
|
clippy::type_complexity,
|
|
clippy::collapsible_else_if,
|
|
clippy::doc_lazy_continuation,
|
|
clippy::items_after_test_module,
|
|
clippy::map_clone,
|
|
clippy::multiple_unsafe_ops_per_block,
|
|
clippy::unwrap_or_default,
|
|
clippy::assign_op_pattern,
|
|
clippy::needless_borrow,
|
|
clippy::println_empty_string,
|
|
clippy::unnecessary_cast,
|
|
clippy::used_underscore_binding,
|
|
clippy::create_dir,
|
|
clippy::implicit_saturating_sub,
|
|
clippy::exit,
|
|
clippy::expect_fun_call,
|
|
clippy::too_many_arguments,
|
|
clippy::unnecessary_map_or,
|
|
clippy::unwrap_used,
|
|
dead_code,
|
|
unused_imports,
|
|
unused_variables,
|
|
clippy::cloned_ref_to_slice_refs,
|
|
clippy::neg_multiply,
|
|
clippy::while_let_loop,
|
|
clippy::bool_assert_comparison,
|
|
clippy::excessive_precision,
|
|
clippy::trivially_copy_pass_by_ref,
|
|
clippy::op_ref,
|
|
clippy::redundant_closure,
|
|
clippy::unnecessary_lazy_evaluations,
|
|
clippy::if_then_some_else_none,
|
|
clippy::unnecessary_to_owned,
|
|
clippy::single_component_path_imports,
|
|
)]
|
|
//! Test suite to verify MAMBA-2 accuracy calculation fix
|
|
//!
|
|
//! This test suite validates the fix for the accuracy calculation bug where
|
|
//! mean_all() was incorrectly used on incompatible tensor shapes, causing
|
|
//! 99% error rates and 3-12% "accuracy" despite normal loss convergence.
|
|
|
|
// candle eliminated — test uses native APIs
|
|
use tracing::info;
|
|
|
|
/// Test accuracy calculation with single-value target (basic case)
|
|
#[test]
|
|
fn test_accuracy_calculation_single_value() {
|
|
// Simulate normalized predictions and targets (pure arithmetic — no GPU needed)
|
|
let pred_val: f64 = 0.48; // Predict 0.48
|
|
let target_val: f64 = 0.50; // Target 0.50
|
|
|
|
// Expected MAPE: |0.48 - 0.50| / 0.50 = 0.04 = 4% error
|
|
// Should be CORRECT with 30% threshold
|
|
|
|
// NEW FIX (scalar extraction):
|
|
let error = ((pred_val - target_val) / target_val).abs();
|
|
|
|
assert!(
|
|
(error - 0.04).abs() < 1e-6,
|
|
"Expected 4% error, got {}%",
|
|
error * 100.0
|
|
);
|
|
|
|
// With 30% threshold, this should be marked "correct"
|
|
assert!(
|
|
error < 0.3,
|
|
"4% error should be considered correct with 30% threshold"
|
|
);
|
|
|
|
// Also verify it passes the stricter 10% threshold
|
|
assert!(
|
|
error < 0.1,
|
|
"4% error should also pass 10% threshold (but 10% is too strict for production)"
|
|
);
|
|
}
|
|
|
|
/// Test accuracy calculation with multi-dimensional output (realistic MAMBA-2 case)
|
|
#[test]
|
|
fn test_accuracy_calculation_multi_dim_output() {
|
|
// Simulate realistic MAMBA-2 output: [1, 1, 54] (pure arithmetic)
|
|
let mut output_data = vec![0.0_f64; 54];
|
|
output_data[0] = 0.48; // First feature is regression target
|
|
|
|
let target_val: f64 = 0.50;
|
|
|
|
// OLD BUG (mean_all): Would give 99% error
|
|
let old_pred_mean: f64 = output_data.iter().sum::<f64>() / output_data.len() as f64;
|
|
// old_pred_mean ≈ 0.48/54 ≈ 0.0089
|
|
let old_target_mean = target_val; // 0.50
|
|
let old_error = ((old_pred_mean - old_target_mean) / old_target_mean).abs();
|
|
|
|
info!(
|
|
old_pred_mean,
|
|
old_target_mean,
|
|
old_error_pct = old_error * 100.0,
|
|
"OLD BUG: pred_mean, target_mean, error"
|
|
);
|
|
assert!(
|
|
old_error > 0.9,
|
|
"OLD BUG: Should show ~99% error due to mean_all() on 54-dim output"
|
|
);
|
|
|
|
// NEW FIX (scalar extraction from first feature):
|
|
let new_pred_val = output_data[0]; // 0.48
|
|
let new_target_val = target_val; // 0.50
|
|
let new_error = ((new_pred_val - new_target_val) / new_target_val).abs();
|
|
|
|
info!(
|
|
new_pred_val,
|
|
new_target_val,
|
|
new_error_pct = new_error * 100.0,
|
|
"NEW FIX: pred_val, target_val, error"
|
|
);
|
|
assert!(
|
|
(new_error - 0.04).abs() < 1e-6,
|
|
"NEW FIX: Should show 4% error, got {}%",
|
|
new_error * 100.0
|
|
);
|
|
|
|
// NEW: 4% error -> "correct" with 30% threshold
|
|
assert!(
|
|
new_error < 0.3,
|
|
"NEW FIX: 4% error should be considered correct"
|
|
);
|
|
}
|
|
|
|
/// Test edge case: target near zero (avoid division by zero)
|
|
#[test]
|
|
fn test_accuracy_calculation_near_zero_target() {
|
|
let pred_val: f64 = 0.02;
|
|
let target_val: f64 = 1e-9; // Very near zero (below 1e-8)
|
|
|
|
// For targets near zero (< 1e-8), use absolute error instead of percentage
|
|
let error = if target_val.abs() > 1e-8 {
|
|
((pred_val - target_val) / target_val).abs()
|
|
} else {
|
|
(pred_val - target_val).abs()
|
|
};
|
|
|
|
info!(
|
|
pred_val,
|
|
target_val,
|
|
error,
|
|
using_absolute_error = target_val.abs() <= 1e-8,
|
|
"Near-zero target"
|
|
);
|
|
|
|
// Should use absolute error (0.02 - 1e-9 ≈ 0.02)
|
|
assert!(
|
|
(error - 0.02).abs() < 1e-6,
|
|
"Expected absolute error ~0.02, got {}",
|
|
error
|
|
);
|
|
}
|
|
|
|
/// Test threshold sensitivity: 10% vs 30%
|
|
#[test]
|
|
fn test_threshold_comparison() {
|
|
// Test different error levels (pure arithmetic)
|
|
let test_cases: Vec<(f64, f64, f64)> = vec![
|
|
(0.48, 0.50, 0.04), // 4% error - should pass both thresholds
|
|
(0.42, 0.50, 0.16), // 16% error - should pass 30% but fail 10%
|
|
(0.30, 0.50, 0.40), // 40% error - should fail both thresholds
|
|
];
|
|
|
|
for (pred_val, target_val, expected_error) in test_cases {
|
|
let error = ((pred_val - target_val) / target_val).abs();
|
|
|
|
info!(
|
|
pred_val,
|
|
target_val,
|
|
error_pct = error * 100.0,
|
|
expected_error_pct = expected_error * 100.0,
|
|
"Threshold comparison"
|
|
);
|
|
|
|
assert!(
|
|
(error - expected_error).abs() < 1e-6,
|
|
"Expected {:.2}% error, got {:.2}%",
|
|
expected_error * 100.0,
|
|
error * 100.0
|
|
);
|
|
|
|
// Check threshold behavior
|
|
let passes_10 = error < 0.1;
|
|
let passes_30 = error < 0.3;
|
|
|
|
match expected_error {
|
|
e if e < 0.1 => {
|
|
assert!(
|
|
passes_10,
|
|
"Error {:.2}% should pass 10% threshold",
|
|
e * 100.0
|
|
);
|
|
assert!(
|
|
passes_30,
|
|
"Error {:.2}% should pass 30% threshold",
|
|
e * 100.0
|
|
);
|
|
},
|
|
e if e < 0.3 => {
|
|
assert!(
|
|
!passes_10,
|
|
"Error {:.2}% should fail 10% threshold",
|
|
e * 100.0
|
|
);
|
|
assert!(
|
|
passes_30,
|
|
"Error {:.2}% should pass 30% threshold",
|
|
e * 100.0
|
|
);
|
|
},
|
|
e => {
|
|
assert!(
|
|
!passes_10,
|
|
"Error {:.2}% should fail 10% threshold",
|
|
e * 100.0
|
|
);
|
|
assert!(
|
|
!passes_30,
|
|
"Error {:.2}% should fail 30% threshold",
|
|
e * 100.0
|
|
);
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Test realistic ES futures price prediction scenario
|
|
#[test]
|
|
fn test_realistic_futures_prediction() {
|
|
// ES futures: price range $5000-$5200 (normalized to 0.0-1.0)
|
|
// Example: predict $5095, actual $5100
|
|
// Normalized: predict 0.475, actual 0.5
|
|
// Error: $5 out of $200 range = 2.5% in price space
|
|
// MAPE: |0.475 - 0.5| / 0.5 = 5% in normalized space
|
|
|
|
let pred_val: f64 = 0.475;
|
|
let target_val: f64 = 0.50;
|
|
let error_pct = ((pred_val - target_val) / target_val).abs();
|
|
|
|
info!(
|
|
pred_val,
|
|
target_val,
|
|
error_pct = error_pct * 100.0,
|
|
"Realistic ES prediction"
|
|
);
|
|
|
|
// 5% error should be considered EXCELLENT for financial prediction
|
|
assert!(error_pct < 0.1, "5% error should easily pass 10% threshold");
|
|
assert!(error_pct < 0.3, "5% error should easily pass 30% threshold");
|
|
|
|
// In price terms: $5 error on $5100 = 0.098% in absolute terms
|
|
// This is EXCELLENT prediction accuracy for intraday futures
|
|
}
|
|
|
|
/// Test batch of predictions to estimate accuracy rate
|
|
#[test]
|
|
fn test_batch_accuracy_estimation() {
|
|
// Simulate 100 predictions with varying errors (pure arithmetic)
|
|
let mut errors = vec![];
|
|
|
|
// Generate predictions with normal distribution around target
|
|
for i in 0..100 {
|
|
let target_val: f64 = 0.5;
|
|
// Add noise: +/-15% RMSE -> most predictions within +/-30%
|
|
let noise = (i as f64 / 100.0 - 0.5) * 0.3; // -15% to +15%
|
|
let pred_val = target_val + noise;
|
|
|
|
let error = ((pred_val - target_val) / target_val).abs();
|
|
|
|
errors.push(error);
|
|
}
|
|
|
|
// Count how many predictions are "correct" with different thresholds
|
|
let correct_10 = errors.iter().filter(|&&e| e < 0.1).count();
|
|
let correct_30 = errors.iter().filter(|&&e| e < 0.3).count();
|
|
|
|
let accuracy_10 = correct_10 as f64 / 100.0;
|
|
let accuracy_30 = correct_30 as f64 / 100.0;
|
|
|
|
info!(
|
|
accuracy_10_pct = accuracy_10 * 100.0,
|
|
accuracy_30_pct = accuracy_30 * 100.0,
|
|
"Batch accuracy estimation (100 samples)"
|
|
);
|
|
|
|
// With ±15% noise, expect:
|
|
// - 10% threshold: ~33% accuracy (1/3 within ±10%)
|
|
// - 30% threshold: ~100% accuracy (all within ±15%)
|
|
assert!(
|
|
accuracy_10 > 0.20 && accuracy_10 < 0.50,
|
|
"Expected 20-50% accuracy with 10% threshold, got {:.1}%",
|
|
accuracy_10 * 100.0
|
|
);
|
|
assert!(
|
|
accuracy_30 > 0.90,
|
|
"Expected >90% accuracy with 30% threshold, got {:.1}%",
|
|
accuracy_30 * 100.0
|
|
);
|
|
}
|
|
|
|
/// Integration test: verify fix aligns accuracy with loss
|
|
#[test]
|
|
fn test_accuracy_loss_alignment() {
|
|
// Given: Training loss = 0.071 (MSE in normalized space)
|
|
// RMSE = sqrt(0.071) = 0.266 = 26.6% error
|
|
//
|
|
// With 30% MAPE threshold:
|
|
// - Predictions with <30% error marked "correct"
|
|
// - RMSE 26.6% means ~68% of predictions within ±30% (assuming normal distribution)
|
|
// - Expected accuracy: ~68-75%
|
|
|
|
let expected_rmse = 0.266;
|
|
let threshold = 0.3;
|
|
|
|
// Approximate: for RMSE R and threshold T, accuracy ≈ erf(T/R*sqrt(2))
|
|
// For R=0.266, T=0.3: accuracy ≈ erf(1.13*sqrt(2)) ≈ erf(1.6) ≈ 0.976
|
|
// But this assumes normal distribution centered at target
|
|
//
|
|
// More conservative estimate: if RMSE=26.6%, about 68-75% within ±30%
|
|
|
|
info!(
|
|
expected_rmse_pct = expected_rmse * 100.0,
|
|
threshold_pct = threshold * 100.0,
|
|
"Loss-Accuracy Alignment"
|
|
);
|
|
info!("Expected accuracy: 68-75% (most predictions within threshold)");
|
|
|
|
// Verify threshold is reasonable for this RMSE
|
|
assert!(
|
|
threshold > expected_rmse,
|
|
"Threshold ({:.1}%) should be greater than RMSE ({:.1}%) for reasonable accuracy",
|
|
threshold * 100.0,
|
|
expected_rmse * 100.0
|
|
);
|
|
}
|