Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
107 lines
3.5 KiB
Rust
107 lines
3.5 KiB
Rust
use ml::hyperopt::paths::{TrainingPaths, generate_run_id};
|
|
use tempfile::TempDir;
|
|
|
|
#[test]
|
|
fn test_paths_creation_and_cleanup() {
|
|
let temp_dir = TempDir::new().unwrap();
|
|
let paths = TrainingPaths::new(temp_dir.path(), "test_model", "test_run");
|
|
|
|
// Create directories
|
|
paths.create_all().unwrap();
|
|
|
|
// Verify all directories exist
|
|
assert!(paths.checkpoints_dir().exists());
|
|
assert!(paths.logs_dir().exists());
|
|
assert!(paths.hyperopt_dir().exists());
|
|
assert!(paths.metrics_dir().exists());
|
|
}
|
|
|
|
#[test]
|
|
fn test_run_id_uniqueness() {
|
|
let id1 = generate_run_id("test");
|
|
std::thread::sleep(std::time::Duration::from_secs(1));
|
|
let id2 = generate_run_id("test");
|
|
assert_ne!(id1, id2);
|
|
}
|
|
|
|
#[test]
|
|
fn test_path_structure() {
|
|
let temp_dir = TempDir::new().unwrap();
|
|
let paths = TrainingPaths::new(temp_dir.path(), "mamba2", "20251028_223000_hyperopt");
|
|
|
|
// Verify path structure
|
|
let expected_run_dir = temp_dir.path().join("training_runs/mamba2/run_20251028_223000_hyperopt");
|
|
assert_eq!(paths.run_dir(), expected_run_dir);
|
|
|
|
let expected_checkpoints = expected_run_dir.join("checkpoints");
|
|
assert_eq!(paths.checkpoints_dir(), expected_checkpoints);
|
|
|
|
let expected_logs = expected_run_dir.join("logs");
|
|
assert_eq!(paths.logs_dir(), expected_logs);
|
|
|
|
let expected_hyperopt = expected_run_dir.join("hyperopt");
|
|
assert_eq!(paths.hyperopt_dir(), expected_hyperopt);
|
|
|
|
let expected_metrics = expected_run_dir.join("metrics");
|
|
assert_eq!(paths.metrics_dir(), expected_metrics);
|
|
}
|
|
|
|
#[test]
|
|
fn test_multiple_models_isolation() {
|
|
let temp_dir = TempDir::new().unwrap();
|
|
|
|
let paths_mamba2 = TrainingPaths::new(temp_dir.path(), "mamba2", "run1");
|
|
let paths_tft = TrainingPaths::new(temp_dir.path(), "tft", "run1");
|
|
|
|
paths_mamba2.create_all().unwrap();
|
|
paths_tft.create_all().unwrap();
|
|
|
|
// Verify models are isolated
|
|
assert_ne!(paths_mamba2.run_dir(), paths_tft.run_dir());
|
|
assert!(paths_mamba2.run_dir().to_str().unwrap().contains("mamba2"));
|
|
assert!(paths_tft.run_dir().to_str().unwrap().contains("tft"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_run_id_format() {
|
|
let run_id = generate_run_id("hyperopt");
|
|
|
|
// Should have format: YYYYMMDD_HHMMSS_hyperopt
|
|
let parts: Vec<&str> = run_id.split('_').collect();
|
|
assert_eq!(parts.len(), 3);
|
|
|
|
// Date part should be 8 digits
|
|
assert_eq!(parts[0].len(), 8);
|
|
assert!(parts[0].chars().all(|c| c.is_ascii_digit()));
|
|
|
|
// Time part should be 6 digits
|
|
assert_eq!(parts[1].len(), 6);
|
|
assert!(parts[1].chars().all(|c| c.is_ascii_digit()));
|
|
|
|
// Type part should match input
|
|
assert_eq!(parts[2], "hyperopt");
|
|
}
|
|
|
|
#[test]
|
|
fn test_no_hardcoded_paths() {
|
|
// Test that paths are fully configurable
|
|
let custom_base = "/custom/path";
|
|
let custom_model = "custom_model";
|
|
let custom_run = "custom_run_id";
|
|
|
|
let paths = TrainingPaths::new(custom_base, custom_model, custom_run);
|
|
|
|
// All paths should start with custom base
|
|
assert!(paths.run_dir().starts_with(custom_base));
|
|
assert!(paths.checkpoints_dir().starts_with(custom_base));
|
|
assert!(paths.logs_dir().starts_with(custom_base));
|
|
assert!(paths.hyperopt_dir().starts_with(custom_base));
|
|
assert!(paths.metrics_dir().starts_with(custom_base));
|
|
|
|
// All paths should contain custom model name
|
|
assert!(paths.run_dir().to_str().unwrap().contains(custom_model));
|
|
|
|
// All paths should contain custom run id
|
|
assert!(paths.run_dir().to_str().unwrap().contains(custom_run));
|
|
}
|