- Reduce CI GPU test datasets 16x for walltime reduction - Reduce early-stop epochs 50→10, add --test-threads=1 - Serialize all GPU lib tests to prevent cuBLAS init race - Align state_dim to 16 for BF16 tensor core HMMA dispatch - BF16 precision tolerance in ml-dqn tests - Enable branching DQN + tracing subscriber in smoke tests - Prevent min_replay_size > buffer_size deadlock in early-stop tests - Prevent AutoReplaySizer from breaking gradient collapse warmup - Replace racy tokio::spawn checkpoint counter with AtomicUsize - Set warmup_steps=0 and max_training_steps_per_epoch=300 in early-stop tests - RealDataLoader respects TEST_DATA_DIR for CI PVC layout - Add collapse_warmup_capacity to gpu_smoketest DQNConfig - Drain CUDA context between test binaries - Detached HEAD checkout prevents local branch corruption - GPU pipeline tests: fix BF16 dtype and rank-1 squeeze assertions - OOD input handling tests use use_gpu: true Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
244 lines
8.2 KiB
Rust
244 lines
8.2 KiB
Rust
#![allow(
|
|
clippy::assertions_on_constants,
|
|
clippy::assertions_on_result_states,
|
|
clippy::clone_on_copy,
|
|
clippy::decimal_literal_representation,
|
|
clippy::doc_markdown,
|
|
clippy::empty_line_after_doc_comments,
|
|
clippy::field_reassign_with_default,
|
|
clippy::get_unwrap,
|
|
clippy::identity_op,
|
|
clippy::inconsistent_digit_grouping,
|
|
clippy::indexing_slicing,
|
|
clippy::integer_division,
|
|
clippy::len_zero,
|
|
clippy::let_underscore_must_use,
|
|
clippy::manual_div_ceil,
|
|
clippy::manual_let_else,
|
|
clippy::manual_range_contains,
|
|
clippy::modulo_arithmetic,
|
|
clippy::needless_range_loop,
|
|
clippy::non_ascii_literal,
|
|
clippy::redundant_clone,
|
|
clippy::shadow_reuse,
|
|
clippy::shadow_same,
|
|
clippy::shadow_unrelated,
|
|
clippy::single_match_else,
|
|
clippy::str_to_string,
|
|
clippy::string_slice,
|
|
clippy::tests_outside_test_module,
|
|
clippy::too_many_lines,
|
|
clippy::unnecessary_wraps,
|
|
clippy::unseparated_literal_suffix,
|
|
clippy::use_debug,
|
|
clippy::useless_vec,
|
|
clippy::wildcard_enum_match_arm,
|
|
clippy::else_if_without_else,
|
|
clippy::expect_used,
|
|
clippy::missing_const_for_fn,
|
|
clippy::similar_names,
|
|
clippy::type_complexity,
|
|
clippy::collapsible_else_if,
|
|
clippy::doc_lazy_continuation,
|
|
clippy::items_after_test_module,
|
|
clippy::map_clone,
|
|
clippy::multiple_unsafe_ops_per_block,
|
|
clippy::unwrap_or_default,
|
|
clippy::assign_op_pattern,
|
|
clippy::needless_borrow,
|
|
clippy::println_empty_string,
|
|
clippy::unnecessary_cast,
|
|
clippy::used_underscore_binding,
|
|
clippy::create_dir,
|
|
clippy::implicit_saturating_sub,
|
|
clippy::exit,
|
|
clippy::expect_fun_call,
|
|
clippy::too_many_arguments,
|
|
clippy::unnecessary_map_or,
|
|
clippy::unwrap_used,
|
|
dead_code,
|
|
unused_imports,
|
|
unused_variables,
|
|
clippy::cloned_ref_to_slice_refs,
|
|
clippy::neg_multiply,
|
|
clippy::while_let_loop,
|
|
clippy::bool_assert_comparison,
|
|
clippy::excessive_precision,
|
|
clippy::trivially_copy_pass_by_ref,
|
|
clippy::op_ref,
|
|
clippy::redundant_closure,
|
|
clippy::unnecessary_lazy_evaluations,
|
|
clippy::if_then_some_else_none,
|
|
clippy::unnecessary_to_owned,
|
|
clippy::single_component_path_imports,
|
|
)]
|
|
//! Test: DQN adapter uses configurable TrainingPaths (no hardcoded paths)
|
|
//!
|
|
//! This test verifies that:
|
|
//! 1. DQNTrainer accepts TrainingPaths configuration
|
|
//! 2. Training directories are created correctly
|
|
//! 3. No hardcoded checkpoint directories are used
|
|
//!
|
|
//! ## CRITICAL
|
|
//!
|
|
//! NO GPU training - only compilation and path verification
|
|
|
|
use ml::hyperopt::adapters::dqn::DQNTrainer;
|
|
use ml::hyperopt::paths::TrainingPaths;
|
|
use std::path::PathBuf;
|
|
use tempfile::TempDir;
|
|
use tracing::info;
|
|
|
|
#[test]
|
|
fn test_dqn_trainer_accepts_training_paths() {
|
|
// Create temporary directories
|
|
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
|
let base_dir = temp_dir.path().to_path_buf();
|
|
let dbn_data_dir = temp_dir.path().join("dbn_data");
|
|
std::fs::create_dir(&dbn_data_dir).expect("Failed to create DBN data dir");
|
|
|
|
// Create a dummy DBN file (empty is fine for this test)
|
|
std::fs::write(dbn_data_dir.join("dummy.dbn.zst"), b"dummy")
|
|
.expect("Failed to write dummy file");
|
|
|
|
// Create TrainingPaths
|
|
let training_paths = TrainingPaths::new(&base_dir, "dqn", "test_run_001");
|
|
|
|
// Create DQN trainer with training paths
|
|
let trainer = DQNTrainer::new(&dbn_data_dir, 1)
|
|
.expect("Failed to create DQN trainer")
|
|
.with_training_paths(training_paths.clone());
|
|
|
|
// Verify trainer was created (compilation test)
|
|
drop(trainer);
|
|
|
|
// Verify expected paths exist after TrainingPaths::create_all()
|
|
training_paths
|
|
.create_all()
|
|
.expect("Failed to create directories");
|
|
|
|
let expected_run_dir = base_dir.join("training_runs/dqn/run_test_run_001");
|
|
let expected_checkpoints = expected_run_dir.join("checkpoints");
|
|
let expected_logs = expected_run_dir.join("logs");
|
|
let expected_hyperopt = expected_run_dir.join("hyperopt");
|
|
|
|
assert!(expected_run_dir.exists(), "Run directory should exist");
|
|
assert!(
|
|
expected_checkpoints.exists(),
|
|
"Checkpoints directory should exist"
|
|
);
|
|
assert!(expected_logs.exists(), "Logs directory should exist");
|
|
assert!(
|
|
expected_hyperopt.exists(),
|
|
"Hyperopt directory should exist"
|
|
);
|
|
|
|
info!(run_dir = ?expected_run_dir, checkpoints = ?expected_checkpoints, "DQN adapter accepts TrainingPaths configuration");
|
|
}
|
|
|
|
#[test]
|
|
fn test_dqn_trainer_default_paths() {
|
|
// Create temporary directories
|
|
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
|
let dbn_data_dir = temp_dir.path().join("dbn_data");
|
|
std::fs::create_dir(&dbn_data_dir).expect("Failed to create DBN data dir");
|
|
|
|
// Create a dummy DBN file
|
|
std::fs::write(dbn_data_dir.join("dummy.dbn.zst"), b"dummy")
|
|
.expect("Failed to write dummy file");
|
|
|
|
// Create DQN trainer without setting training paths (uses /tmp/ml_training default)
|
|
let trainer = DQNTrainer::new(&dbn_data_dir, 1).expect("Failed to create DQN trainer");
|
|
|
|
// Verify trainer was created with default paths
|
|
drop(trainer);
|
|
|
|
info!("DQN adapter uses /tmp/ml_training as default (should be overridden in production)");
|
|
}
|
|
|
|
#[test]
|
|
fn test_dqn_training_paths_structure() {
|
|
let base_dir = PathBuf::from("/runpod-volume");
|
|
let training_paths = TrainingPaths::new(&base_dir, "dqn", "20251028_120000_hyperopt");
|
|
|
|
// Verify path structure
|
|
assert_eq!(
|
|
training_paths.run_dir(),
|
|
PathBuf::from("/runpod-volume/training_runs/dqn/run_20251028_120000_hyperopt")
|
|
);
|
|
assert_eq!(
|
|
training_paths.checkpoints_dir(),
|
|
PathBuf::from("/runpod-volume/training_runs/dqn/run_20251028_120000_hyperopt/checkpoints")
|
|
);
|
|
assert_eq!(
|
|
training_paths.logs_dir(),
|
|
PathBuf::from("/runpod-volume/training_runs/dqn/run_20251028_120000_hyperopt/logs")
|
|
);
|
|
assert_eq!(
|
|
training_paths.hyperopt_dir(),
|
|
PathBuf::from("/runpod-volume/training_runs/dqn/run_20251028_120000_hyperopt/hyperopt")
|
|
);
|
|
|
|
info!("DQN training paths structure matches expected layout");
|
|
}
|
|
|
|
#[test]
|
|
fn test_no_hardcoded_paths_in_dqn_adapter() {
|
|
// This is a compile-time check - if the code compiles, it means:
|
|
// 1. DQNTrainer has a training_paths field
|
|
// 2. with_training_paths() method exists
|
|
// 3. TrainingPaths is properly integrated
|
|
|
|
// Try to read the source file from multiple possible locations
|
|
let possible_paths = vec![
|
|
"ml/src/hyperopt/adapters/dqn.rs",
|
|
"src/hyperopt/adapters/dqn.rs",
|
|
"../src/hyperopt/adapters/dqn.rs",
|
|
];
|
|
|
|
let source_path = possible_paths
|
|
.iter()
|
|
.find(|p| std::path::Path::new(p).exists())
|
|
.expect("Could not find DQN adapter source file");
|
|
|
|
let source = std::fs::read_to_string(source_path).expect("Failed to read DQN adapter source");
|
|
|
|
// Check for forbidden patterns (except in comments/docs)
|
|
let forbidden_patterns = vec![
|
|
"/tmp/dqn", // Hardcoded temp paths
|
|
"/runpod-volume/dqn", // Hardcoded production paths
|
|
"checkpoint_dir:", // Old hardcoded field (should be training_paths now)
|
|
];
|
|
|
|
for pattern in forbidden_patterns {
|
|
// Count occurrences (allow in comments)
|
|
let count = source.matches(pattern).count();
|
|
if count > 0 {
|
|
// Check if it's only in comments
|
|
let lines_with_pattern: Vec<_> = source
|
|
.lines()
|
|
.filter(|line| line.contains(pattern))
|
|
.collect();
|
|
|
|
let real_occurrences: Vec<_> = lines_with_pattern
|
|
.iter()
|
|
.filter(|line| !line.trim_start().starts_with("//"))
|
|
.collect();
|
|
|
|
if !real_occurrences.is_empty() {
|
|
let lines_str = real_occurrences
|
|
.iter()
|
|
.map(|s| s.to_string())
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
panic!(
|
|
"Found hardcoded pattern '{}' in DQN adapter:\n{}",
|
|
pattern, lines_str
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
info!("No hardcoded paths found in DQN adapter");
|
|
}
|