Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
188 lines
5.7 KiB
Rust
188 lines
5.7 KiB
Rust
//! PPO Adapter Training Paths Integration Test
|
|
//!
|
|
//! Validates that PPO hyperopt adapter properly uses TrainingPaths
|
|
//! instead of hardcoded checkpoint directories.
|
|
//!
|
|
//! This test ensures:
|
|
//! - No hardcoded paths in PPOTrainer
|
|
//! - TrainingPaths can be configured via with_training_paths()
|
|
//! - Default paths use /tmp/ml_training as temporary default
|
|
//! - Paths are correctly propagated to training logic
|
|
|
|
use std::path::PathBuf;
|
|
use tempfile::TempDir;
|
|
|
|
use ml::hyperopt::adapters::ppo::PPOTrainer;
|
|
use ml::hyperopt::paths::{generate_run_id, TrainingPaths};
|
|
|
|
#[test]
|
|
fn test_ppo_trainer_default_paths() {
|
|
// Create trainer with default paths
|
|
let trainer = PPOTrainer::new(100).expect("Failed to create PPO trainer");
|
|
|
|
// Default paths should use /tmp/ml_training
|
|
// This is intentional - provides a safe temporary default
|
|
// Production code MUST call with_training_paths()
|
|
let expected_base = PathBuf::from("/tmp/ml_training");
|
|
|
|
// Access training_paths via debug formatting (struct is Debug)
|
|
let debug_str = format!("{:?}", trainer);
|
|
assert!(
|
|
debug_str.contains("/tmp/ml_training"),
|
|
"Default paths should use /tmp/ml_training, got: {}",
|
|
debug_str
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ppo_trainer_custom_paths() {
|
|
// Create temporary directory for test
|
|
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
|
let base_dir = temp_dir.path();
|
|
|
|
// Generate run ID
|
|
let run_id = generate_run_id("test");
|
|
|
|
// Create training paths
|
|
let training_paths = TrainingPaths::new(base_dir, "ppo", &run_id);
|
|
|
|
// Create directories
|
|
training_paths
|
|
.create_all()
|
|
.expect("Failed to create training directories");
|
|
|
|
// Create trainer with custom paths
|
|
let trainer = PPOTrainer::new(100)
|
|
.expect("Failed to create PPO trainer")
|
|
.with_training_paths(training_paths.clone());
|
|
|
|
// Validate paths are set correctly
|
|
let expected_run_dir = base_dir
|
|
.join("training_runs")
|
|
.join("ppo")
|
|
.join(format!("run_{}", run_id));
|
|
|
|
let expected_checkpoints = expected_run_dir.join("checkpoints");
|
|
|
|
// Verify directories exist
|
|
assert!(
|
|
expected_run_dir.exists(),
|
|
"Run directory should exist: {:?}",
|
|
expected_run_dir
|
|
);
|
|
assert!(
|
|
expected_checkpoints.exists(),
|
|
"Checkpoints directory should exist: {:?}",
|
|
expected_checkpoints
|
|
);
|
|
|
|
// Verify paths are used (via debug formatting)
|
|
let debug_str = format!("{:?}", trainer);
|
|
assert!(
|
|
debug_str.contains("model_name: \"ppo\""),
|
|
"Trainer should use model_name 'ppo', got: {}",
|
|
debug_str
|
|
);
|
|
assert!(
|
|
debug_str.contains(&run_id),
|
|
"Trainer should use provided run_id, got: {}",
|
|
debug_str
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ppo_trainer_path_structure() {
|
|
// Create temporary directory
|
|
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
|
let base_dir = temp_dir.path();
|
|
|
|
// Create training paths with known run_id
|
|
let run_id = "20251029_120000_test";
|
|
let training_paths = TrainingPaths::new(base_dir, "ppo", run_id);
|
|
|
|
// Create all directories
|
|
training_paths
|
|
.create_all()
|
|
.expect("Failed to create directories");
|
|
|
|
// Create trainer with paths
|
|
let _trainer = PPOTrainer::new(100)
|
|
.expect("Failed to create trainer")
|
|
.with_training_paths(training_paths.clone());
|
|
|
|
// Validate full directory structure
|
|
let run_dir = training_paths.run_dir();
|
|
let checkpoints = training_paths.checkpoints_dir();
|
|
let logs = training_paths.logs_dir();
|
|
let hyperopt = training_paths.hyperopt_dir();
|
|
let metrics = training_paths.metrics_dir();
|
|
|
|
assert!(run_dir.exists(), "Run directory should exist");
|
|
assert!(checkpoints.exists(), "Checkpoints directory should exist");
|
|
assert!(logs.exists(), "Logs directory should exist");
|
|
assert!(hyperopt.exists(), "Hyperopt directory should exist");
|
|
assert!(metrics.exists(), "Metrics directory should exist");
|
|
|
|
// Validate path relationships
|
|
assert_eq!(
|
|
checkpoints,
|
|
run_dir.join("checkpoints"),
|
|
"Checkpoints should be under run_dir"
|
|
);
|
|
assert_eq!(logs, run_dir.join("logs"), "Logs should be under run_dir");
|
|
assert_eq!(
|
|
hyperopt,
|
|
run_dir.join("hyperopt"),
|
|
"Hyperopt should be under run_dir"
|
|
);
|
|
assert_eq!(
|
|
metrics,
|
|
run_dir.join("metrics"),
|
|
"Metrics should be under run_dir"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_run_id_generation() {
|
|
// Generate run ID
|
|
let run_id = generate_run_id("hyperopt");
|
|
|
|
// Validate format
|
|
assert!(
|
|
run_id.contains("hyperopt"),
|
|
"Run ID should contain run type"
|
|
);
|
|
assert!(
|
|
run_id.len() > 15,
|
|
"Run ID should have timestamp (YYYYMMDD_HHMMSS)"
|
|
);
|
|
|
|
// Validate uniqueness (generate multiple)
|
|
let run_id_2 = generate_run_id("hyperopt");
|
|
// Should be the same or different depending on timing
|
|
// Just validate format consistency
|
|
assert!(run_id_2.contains("hyperopt"));
|
|
assert!(run_id_2.len() > 15);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ppo_trainer_builder_pattern() {
|
|
// Test builder pattern with training paths
|
|
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
|
let training_paths = TrainingPaths::new(temp_dir.path(), "ppo", "test_builder");
|
|
|
|
training_paths
|
|
.create_all()
|
|
.expect("Failed to create directories");
|
|
|
|
// Build trainer with method chaining
|
|
let trainer = PPOTrainer::new(100)
|
|
.expect("Failed to create trainer")
|
|
.with_training_paths(training_paths.clone());
|
|
|
|
// Verify trainer was created successfully
|
|
let debug_str = format!("{:?}", trainer);
|
|
assert!(debug_str.contains("test_builder"));
|
|
assert!(debug_str.contains("ppo"));
|
|
}
|