ca4c38d921
- Reduce CI GPU test datasets 16x for walltime reduction - Reduce early-stop epochs 50→10, add --test-threads=1 - Serialize all GPU lib tests to prevent cuBLAS init race - Align state_dim to 16 for BF16 tensor core HMMA dispatch - BF16 precision tolerance in ml-dqn tests - Enable branching DQN + tracing subscriber in smoke tests - Prevent min_replay_size > buffer_size deadlock in early-stop tests - Prevent AutoReplaySizer from breaking gradient collapse warmup - Replace racy tokio::spawn checkpoint counter with AtomicUsize - Set warmup_steps=0 and max_training_steps_per_epoch=300 in early-stop tests - RealDataLoader respects TEST_DATA_DIR for CI PVC layout - Add collapse_warmup_capacity to gpu_smoketest DQNConfig - Drain CUDA context between test binaries - Detached HEAD checkout prevents local branch corruption - GPU pipeline tests: fix BF16 dtype and rank-1 squeeze assertions - OOD input handling tests use use_gpu: true Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
440 lines
17 KiB
Rust
440 lines
17 KiB
Rust
#![allow(
|
|
clippy::assertions_on_constants,
|
|
clippy::assertions_on_result_states,
|
|
clippy::clone_on_copy,
|
|
clippy::decimal_literal_representation,
|
|
clippy::doc_markdown,
|
|
clippy::empty_line_after_doc_comments,
|
|
clippy::field_reassign_with_default,
|
|
clippy::get_unwrap,
|
|
clippy::identity_op,
|
|
clippy::inconsistent_digit_grouping,
|
|
clippy::indexing_slicing,
|
|
clippy::integer_division,
|
|
clippy::len_zero,
|
|
clippy::let_underscore_must_use,
|
|
clippy::manual_div_ceil,
|
|
clippy::manual_let_else,
|
|
clippy::manual_range_contains,
|
|
clippy::modulo_arithmetic,
|
|
clippy::needless_range_loop,
|
|
clippy::non_ascii_literal,
|
|
clippy::redundant_clone,
|
|
clippy::shadow_reuse,
|
|
clippy::shadow_same,
|
|
clippy::shadow_unrelated,
|
|
clippy::single_match_else,
|
|
clippy::str_to_string,
|
|
clippy::string_slice,
|
|
clippy::tests_outside_test_module,
|
|
clippy::too_many_lines,
|
|
clippy::unnecessary_wraps,
|
|
clippy::unseparated_literal_suffix,
|
|
clippy::use_debug,
|
|
clippy::useless_vec,
|
|
clippy::wildcard_enum_match_arm,
|
|
clippy::else_if_without_else,
|
|
clippy::expect_used,
|
|
clippy::missing_const_for_fn,
|
|
clippy::similar_names,
|
|
clippy::type_complexity,
|
|
clippy::collapsible_else_if,
|
|
clippy::doc_lazy_continuation,
|
|
clippy::items_after_test_module,
|
|
clippy::map_clone,
|
|
clippy::multiple_unsafe_ops_per_block,
|
|
clippy::unwrap_or_default,
|
|
clippy::assign_op_pattern,
|
|
clippy::needless_borrow,
|
|
clippy::println_empty_string,
|
|
clippy::unnecessary_cast,
|
|
clippy::used_underscore_binding,
|
|
clippy::create_dir,
|
|
clippy::implicit_saturating_sub,
|
|
clippy::exit,
|
|
clippy::expect_fun_call,
|
|
clippy::too_many_arguments,
|
|
clippy::unnecessary_map_or,
|
|
clippy::unwrap_used,
|
|
dead_code,
|
|
unused_imports,
|
|
unused_variables,
|
|
clippy::cloned_ref_to_slice_refs,
|
|
clippy::neg_multiply,
|
|
clippy::while_let_loop,
|
|
clippy::bool_assert_comparison,
|
|
clippy::excessive_precision,
|
|
clippy::trivially_copy_pass_by_ref,
|
|
clippy::op_ref,
|
|
clippy::redundant_closure,
|
|
clippy::unnecessary_lazy_evaluations,
|
|
clippy::if_then_some_else_none,
|
|
clippy::unnecessary_to_owned,
|
|
clippy::single_component_path_imports,
|
|
)]
|
|
//! Integration tests for DQN hyperopt JSON export functionality
|
|
//!
|
|
//! Tests the automatic export of best trial hyperparameters to JSON
|
|
//! during hyperopt campaigns for production deployment.
|
|
//!
|
|
//! NOTE: These tests require GPU and training data, so they are marked
|
|
//! with #[ignore]. Run with: cargo test --test dqn_hyperopt_json_export_test -- --ignored
|
|
|
|
use anyhow::Result;
|
|
use serde_json;
|
|
use std::fs;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use ml::hyperopt::adapters::dqn::{BestTrialExport, DQNTrainer};
|
|
use ml::hyperopt::ArgminOptimizer;
|
|
use tracing::info;
|
|
|
|
#[cfg(test)]
|
|
mod hyperopt_json_export_tests {
|
|
use super::*;
|
|
|
|
/// Helper: Clean up any test JSON files
|
|
fn cleanup_test_jsons(pattern: &str) {
|
|
let results_dir = Path::new("ml/hyperopt_results");
|
|
if !results_dir.exists() {
|
|
return;
|
|
}
|
|
|
|
if let Ok(entries) = fs::read_dir(results_dir) {
|
|
for entry in entries.flatten() {
|
|
let path = entry.path();
|
|
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
|
|
if name.contains(pattern) && name.ends_with(".json") {
|
|
let _ = fs::remove_file(&path);
|
|
info!(path = ?path, "Cleaned up test file");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Helper: Find JSON file matching pattern in hyperopt_results/
|
|
fn find_json_file(pattern: &str) -> Option<PathBuf> {
|
|
let results_dir = Path::new("ml/hyperopt_results");
|
|
if !results_dir.exists() {
|
|
return None;
|
|
}
|
|
|
|
if let Ok(entries) = fs::read_dir(results_dir) {
|
|
for entry in entries.flatten() {
|
|
let path = entry.path();
|
|
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
|
|
if name.contains(pattern) && name.ends_with(".json") {
|
|
return Some(path);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Test 1: Mini hyperopt saves best trial JSON
|
|
///
|
|
/// Runs 3-trial hyperopt with 10 epochs each, verifies JSON export
|
|
#[test]
|
|
#[ignore] // Requires GPU and training data
|
|
fn test_hyperopt_saves_best_trial_json() -> Result<()> {
|
|
info!("TEST 1: Hyperopt saves best trial JSON");
|
|
|
|
// Cleanup any previous test files
|
|
cleanup_test_jsons("test_mini_");
|
|
|
|
// Create DQN trainer with minimal epochs for speed
|
|
let trainer = DQNTrainer::new(
|
|
"test_data/ES_FUT_180d.parquet",
|
|
10, // epochs per trial
|
|
)?;
|
|
|
|
// Run mini hyperopt (3 trials)
|
|
info!("Running 3-trial hyperopt campaign");
|
|
let optimizer = ArgminOptimizer::with_trials(3, 1);
|
|
let result = optimizer.optimize(trainer)?;
|
|
|
|
info!(best_objective = result.best_objective, "Hyperopt complete");
|
|
|
|
// Verify JSON file was created
|
|
let json_file = find_json_file("best_trial_sharpe_")
|
|
.expect("Best trial JSON should be created");
|
|
|
|
info!(path = ?json_file, "Found JSON file");
|
|
|
|
// Load and verify the JSON
|
|
let json_content = fs::read_to_string(&json_file)?;
|
|
let trial: BestTrialExport = serde_json::from_str(&json_content)?;
|
|
|
|
// Verify metadata exists
|
|
assert!(trial.trial_number > 0, "Trial number should be > 0");
|
|
assert!(trial.trial_number <= 3, "Trial number should be <= 3 (only 3 trials)");
|
|
assert!(!trial.timestamp.is_empty(), "Timestamp should not be empty");
|
|
assert!(trial.gradient_clip_norm > 0.0, "Gradient clip norm should be > 0");
|
|
|
|
// Verify hyperparameters are present (all 21 fields)
|
|
let params = trial.hyperparameters;
|
|
assert!(params.learning_rate > 0.0, "Learning rate should be > 0");
|
|
assert!(params.batch_size > 0, "Batch size should be > 0");
|
|
assert!(params.gamma > 0.0 && params.gamma < 1.0, "Gamma should be in (0, 1)");
|
|
assert!(params.buffer_size > 0, "Buffer size should be > 0");
|
|
|
|
info!(trial_number = trial.trial_number, sharpe = trial.sharpe, "JSON file created with best trial");
|
|
|
|
// Cleanup
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Test 2: Hyperopt updates JSON on new best trial
|
|
///
|
|
/// Verifies that when a better trial is found, the JSON is updated
|
|
#[test]
|
|
#[ignore] // Requires GPU and training data
|
|
fn test_hyperopt_updates_json_on_new_best() -> Result<()> {
|
|
info!("TEST 2: Hyperopt updates JSON on new best");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
let trainer = DQNTrainer::new(
|
|
"test_data/ES_FUT_180d.parquet",
|
|
10, // epochs per trial
|
|
)?;
|
|
|
|
// Run 5 trials to increase chance of finding better trial
|
|
info!("Running 5-trial hyperopt campaign");
|
|
let optimizer = ArgminOptimizer::with_trials(5, 1);
|
|
let _ = optimizer.optimize(trainer)?;
|
|
|
|
// Load the final JSON
|
|
let json_file = find_json_file("best_trial_sharpe_")
|
|
.expect("Best trial JSON should exist");
|
|
|
|
let json_content = fs::read_to_string(&json_file)?;
|
|
let final_trial: BestTrialExport = serde_json::from_str(&json_content)?;
|
|
|
|
// Verify the JSON contains the best trial from the campaign
|
|
// (Note: We can't predict which trial number will be best, but we can
|
|
// verify the data structure is valid and updated)
|
|
assert!(final_trial.trial_number > 0 && final_trial.trial_number <= 5,
|
|
"Best trial number should be in range [1, 5]");
|
|
|
|
info!(trial_number = final_trial.trial_number, sharpe = final_trial.sharpe, "Final best trial");
|
|
|
|
// Verify the file was written at least once
|
|
let metadata = fs::metadata(&json_file)?;
|
|
assert!(metadata.len() > 100, "JSON file should have substantial content");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Test 3: JSON roundtrip - hyperopt output can be reloaded
|
|
///
|
|
/// Saves a trial, loads it back, verifies all parameters match exactly
|
|
#[test]
|
|
#[ignore] // Requires GPU and training data
|
|
fn test_hyperopt_json_roundtrip() -> Result<()> {
|
|
info!("TEST 3: JSON roundtrip consistency");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
let trainer = DQNTrainer::new(
|
|
"test_data/ES_FUT_180d.parquet",
|
|
10,
|
|
)?;
|
|
|
|
info!("Running 2-trial hyperopt campaign");
|
|
let optimizer = ArgminOptimizer::with_trials(2, 1);
|
|
let _ = optimizer.optimize(trainer)?;
|
|
|
|
// Load the JSON that was saved
|
|
let json_file = find_json_file("best_trial_sharpe_")
|
|
.expect("Best trial JSON should exist");
|
|
|
|
let json_content = fs::read_to_string(&json_file)?;
|
|
let saved_trial: BestTrialExport = serde_json::from_str(&json_content)?;
|
|
|
|
// Serialize it again
|
|
let roundtrip_json = serde_json::to_string_pretty(&saved_trial)?;
|
|
|
|
// Deserialize the roundtrip version
|
|
let roundtrip_trial: BestTrialExport = serde_json::from_str(&roundtrip_json)?;
|
|
|
|
// Verify exact match on all fields
|
|
assert_eq!(saved_trial.trial_number, roundtrip_trial.trial_number,
|
|
"Trial number should match exactly");
|
|
assert_eq!(saved_trial.sharpe, roundtrip_trial.sharpe,
|
|
"Sharpe should match exactly");
|
|
assert_eq!(saved_trial.win_rate, roundtrip_trial.win_rate,
|
|
"Win rate should match exactly");
|
|
assert_eq!(saved_trial.max_drawdown, roundtrip_trial.max_drawdown,
|
|
"Max drawdown should match exactly");
|
|
assert_eq!(saved_trial.total_return, roundtrip_trial.total_return,
|
|
"Total return should match exactly");
|
|
assert_eq!(saved_trial.gradient_clip_norm, roundtrip_trial.gradient_clip_norm,
|
|
"Gradient clip norm should match exactly");
|
|
|
|
// Verify all 21 hyperparameters match
|
|
let saved_params = saved_trial.hyperparameters;
|
|
let roundtrip_params = roundtrip_trial.hyperparameters;
|
|
|
|
assert_eq!(saved_params.learning_rate, roundtrip_params.learning_rate);
|
|
assert_eq!(saved_params.batch_size, roundtrip_params.batch_size);
|
|
assert_eq!(saved_params.gamma, roundtrip_params.gamma);
|
|
assert_eq!(saved_params.buffer_size, roundtrip_params.buffer_size);
|
|
assert_eq!(saved_params.hold_penalty_weight, roundtrip_params.hold_penalty_weight);
|
|
assert_eq!(saved_params.max_position_absolute, roundtrip_params.max_position_absolute);
|
|
assert_eq!(saved_params.huber_delta, roundtrip_params.huber_delta);
|
|
assert_eq!(saved_params.entropy_coefficient, roundtrip_params.entropy_coefficient);
|
|
assert_eq!(saved_params.transaction_cost_multiplier, roundtrip_params.transaction_cost_multiplier);
|
|
assert_eq!(saved_params.use_per, roundtrip_params.use_per);
|
|
assert_eq!(saved_params.per_alpha, roundtrip_params.per_alpha);
|
|
assert_eq!(saved_params.per_beta_start, roundtrip_params.per_beta_start);
|
|
assert_eq!(saved_params.use_dueling, roundtrip_params.use_dueling);
|
|
assert_eq!(saved_params.dueling_hidden_dim, roundtrip_params.dueling_hidden_dim);
|
|
assert_eq!(saved_params.n_steps, roundtrip_params.n_steps);
|
|
assert_eq!(saved_params.tau, roundtrip_params.tau);
|
|
assert_eq!(saved_params.use_distributional, roundtrip_params.use_distributional);
|
|
assert_eq!(saved_params.num_atoms, roundtrip_params.num_atoms);
|
|
assert_eq!(saved_params.v_min, roundtrip_params.v_min);
|
|
assert_eq!(saved_params.v_max, roundtrip_params.v_max);
|
|
assert_eq!(saved_params.use_noisy_nets, roundtrip_params.use_noisy_nets);
|
|
assert_eq!(saved_params.noisy_sigma_init, roundtrip_params.noisy_sigma_init);
|
|
assert_eq!(saved_params.minimum_profit_factor, roundtrip_params.minimum_profit_factor);
|
|
|
|
info!("All 21 hyperparameters + metadata survive roundtrip with perfect fidelity");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Test 4: JSON contains all required metadata fields
|
|
///
|
|
/// Validates that exported JSON has comprehensive metadata for production use
|
|
#[test]
|
|
#[ignore] // Requires GPU and training data
|
|
fn test_hyperopt_json_contains_all_metadata() -> Result<()> {
|
|
info!("TEST 4: JSON metadata completeness");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
let trainer = DQNTrainer::new(
|
|
"test_data/ES_FUT_180d.parquet",
|
|
10,
|
|
)?;
|
|
|
|
info!("Running 2-trial hyperopt campaign");
|
|
let optimizer = ArgminOptimizer::with_trials(2, 1);
|
|
let _ = optimizer.optimize(trainer)?;
|
|
|
|
let json_file = find_json_file("best_trial_sharpe_")
|
|
.expect("Best trial JSON should exist");
|
|
|
|
// Load as raw JSON to verify field presence
|
|
let json_content = fs::read_to_string(&json_file)?;
|
|
let raw_json: serde_json::Value = serde_json::from_str(&json_content)?;
|
|
|
|
// Verify top-level fields
|
|
assert!(raw_json.get("trial_number").is_some(), "Should have trial_number");
|
|
assert!(raw_json.get("sharpe").is_some(), "Should have sharpe");
|
|
assert!(raw_json.get("win_rate").is_some(), "Should have win_rate");
|
|
assert!(raw_json.get("max_drawdown").is_some(), "Should have max_drawdown");
|
|
assert!(raw_json.get("total_return").is_some(), "Should have total_return");
|
|
assert!(raw_json.get("timestamp").is_some(), "Should have timestamp");
|
|
assert!(raw_json.get("gradient_clip_norm").is_some(), "Should have gradient_clip_norm");
|
|
assert!(raw_json.get("hyperparameters").is_some(), "Should have hyperparameters");
|
|
|
|
// Verify timestamp is ISO 8601 format
|
|
let timestamp = raw_json["timestamp"].as_str()
|
|
.expect("Timestamp should be a string");
|
|
assert!(timestamp.contains('T') && timestamp.contains('Z'),
|
|
"Timestamp should be ISO 8601 format: {}", timestamp);
|
|
|
|
// Verify hyperparameters object has all 21 fields
|
|
let hyperparams = raw_json["hyperparameters"].as_object()
|
|
.expect("Hyperparameters should be an object");
|
|
|
|
let required_fields = vec![
|
|
"learning_rate", "batch_size", "gamma", "buffer_size",
|
|
"hold_penalty_weight", "max_position_absolute", "huber_delta",
|
|
"entropy_coefficient", "transaction_cost_multiplier",
|
|
"use_per", "per_alpha", "per_beta_start",
|
|
"use_dueling", "dueling_hidden_dim", "n_steps", "tau",
|
|
"use_distributional", "num_atoms", "v_min", "v_max",
|
|
"use_noisy_nets", "noisy_sigma_init", "minimum_profit_factor",
|
|
];
|
|
|
|
for field in required_fields {
|
|
assert!(hyperparams.contains_key(field),
|
|
"Hyperparameters should contain field: {}", field);
|
|
}
|
|
|
|
info!("JSON contains all 8 metadata fields + 21 hyperparameter fields");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Test 5: Verify filename contains Sharpe ratio
|
|
#[test]
|
|
#[ignore] // Requires GPU and training data
|
|
fn test_json_filename_contains_sharpe() -> Result<()> {
|
|
info!("TEST 5: JSON filename includes Sharpe");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
let trainer = DQNTrainer::new(
|
|
"test_data/ES_FUT_180d.parquet",
|
|
10,
|
|
)?;
|
|
|
|
info!("Running 2-trial hyperopt campaign");
|
|
let optimizer = ArgminOptimizer::with_trials(2, 1);
|
|
let _ = optimizer.optimize(trainer)?;
|
|
|
|
let json_file = find_json_file("best_trial_sharpe_")
|
|
.expect("Best trial JSON should exist");
|
|
|
|
// Verify filename format
|
|
let filename = json_file.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.expect("Should have valid filename");
|
|
|
|
assert!(filename.starts_with("best_trial_sharpe_"),
|
|
"Filename should start with 'best_trial_sharpe_': {}", filename);
|
|
assert!(filename.ends_with(".json"),
|
|
"Filename should end with '.json': {}", filename);
|
|
|
|
// Load JSON to verify Sharpe in filename matches content
|
|
let json_content = fs::read_to_string(&json_file)?;
|
|
let trial: BestTrialExport = serde_json::from_str(&json_content)?;
|
|
|
|
// Extract Sharpe from filename (format: best_trial_sharpe_X.XXXX.json)
|
|
let sharpe_str = filename
|
|
.strip_prefix("best_trial_sharpe_")
|
|
.and_then(|s| s.strip_suffix(".json"))
|
|
.expect("Should be able to extract Sharpe from filename");
|
|
|
|
let filename_sharpe: f64 = sharpe_str.parse()
|
|
.expect(&format!("Should be able to parse Sharpe from filename: {}", sharpe_str));
|
|
|
|
// Verify filename Sharpe matches JSON content (within floating point tolerance)
|
|
let sharpe_diff = (filename_sharpe - trial.sharpe).abs();
|
|
assert!(sharpe_diff < 0.0001,
|
|
"Filename Sharpe ({}) should match JSON Sharpe ({}) within 0.0001",
|
|
filename_sharpe, trial.sharpe);
|
|
|
|
info!(filename_sharpe, trial_sharpe = trial.sharpe, "Filename Sharpe matches JSON Sharpe");
|
|
|
|
cleanup_test_jsons("best_trial_sharpe_");
|
|
|
|
Ok(())
|
|
}
|
|
}
|