feat(hyperopt): Complete DQN hyperopt analysis and PSO optimizer fix

- Fixed PSO budget calculation bug in ml/src/hyperopt/optimizer.rs
  - Root cause: Division by n_particles in sequential execution
  - Now correctly calculates max_iters = remaining_trials (no division)
  - Result: 50 trials complete instead of 23 (100% vs 46%)

- Added comprehensive DQN hyperopt results analysis
  - 39/50 trials analyzed across 2 RunPod deployments
  - Best hyperparameters identified: LR 4.89e-5 (ultra-low)
  - Created DQN_HYPEROPT_RESULTS_SUMMARY.md with expert validation

- GitLab CI/CD pipeline operational (48 lines fixed)
  - Fixed YAML syntax errors (unquoted colons)
  - All 7 jobs validated and working

- Warning cleanup complete (136 → 0 warnings)
  - Removed 143 lines dead code
  - Fixed visibility, unused imports, Debug traits

- Archived Wave D reports to docs/archive/
  - 8 early stopping reports moved
  - Root directory cleaned up

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2025-11-02 21:49:07 +01:00
parent 2cf07a9086
commit 3853988af7
186 changed files with 42695 additions and 101 deletions

View File

@@ -0,0 +1,194 @@
// ml/src/backtesting/action_loader.rs
// DQN action loader for CSV-based backtesting
use chrono::{DateTime, Utc};
use serde::Deserialize;
use std::fs::File;
use std::io::BufReader;
use std::path::Path;
/// DQN action record from CSV (13,552 actions, 10 columns)
/// Format: timestamp,action,q_buy,q_sell,q_hold,open,high,low,close,volume
#[derive(Debug, Clone, Deserialize)]
pub struct DQNActionRecord {
/// ISO8601 timestamp (e.g., "2024-10-20T23:31:00.000000000Z")
pub timestamp: DateTime<Utc>,
/// Action: 0=Buy, 1=Sell, 2=Hold
pub action: u8,
/// Q-value for buy action
pub q_buy: f64,
/// Q-value for sell action
pub q_sell: f64,
/// Q-value for hold action
pub q_hold: f64,
/// OHLCV data (for validation/debugging)
pub open: f64,
pub high: f64,
pub low: f64,
pub close: f64,
pub volume: u64,
}
/// Load DQN actions from CSV file
///
/// # Arguments
/// * `csv_path` - Path to CSV file (e.g., "/tmp/dqn_actions_wave3.csv")
///
/// # Returns
/// * `Ok(Vec<DQNActionRecord>)` - Parsed and validated actions
/// * `Err(String)` - Validation errors (action bounds, Q-values, timestamps)
///
/// # Validations
/// 1. Action bounds: 0 <= action <= 2
/// 2. Finite Q-values: q_buy, q_sell, q_hold must be finite (no NaN/Inf)
/// 3. Timestamp ordering: timestamps must be monotonically increasing
///
/// # Example
/// ```ignore
/// let actions = load_actions_from_csv("/tmp/dqn_actions_wave3.csv")?;
/// assert_eq!(actions.len(), 13_552);
/// ```
pub fn load_actions_from_csv<P: AsRef<Path>>(csv_path: P) -> Result<Vec<DQNActionRecord>, String> {
let path = csv_path.as_ref();
// Open CSV file
let file = File::open(path)
.map_err(|e| format!("Failed to open CSV file '{}': {}", path.display(), e))?;
let reader = BufReader::new(file);
let mut csv_reader = csv::Reader::from_reader(reader);
let mut actions = Vec::new();
let mut prev_timestamp: Option<DateTime<Utc>> = None;
let mut row_num = 1; // Start at 1 (header is row 0)
for result in csv_reader.deserialize() {
row_num += 1;
let record: DQNActionRecord = result
.map_err(|e| format!("CSV parsing error at row {}: {}", row_num, e))?;
// Validation 1: Action bounds (0-2)
if record.action > 2 {
return Err(format!(
"Invalid action {} at row {} (timestamp {}): action must be 0 (Buy), 1 (Sell), or 2 (Hold)",
record.action,
row_num,
record.timestamp
));
}
// Validation 2: Finite Q-values
if !record.q_buy.is_finite() {
return Err(format!(
"Invalid q_buy {} at row {} (timestamp {}): Q-value must be finite",
record.q_buy,
row_num,
record.timestamp
));
}
if !record.q_sell.is_finite() {
return Err(format!(
"Invalid q_sell {} at row {} (timestamp {}): Q-value must be finite",
record.q_sell,
row_num,
record.timestamp
));
}
if !record.q_hold.is_finite() {
return Err(format!(
"Invalid q_hold {} at row {} (timestamp {}): Q-value must be finite",
record.q_hold,
row_num,
record.timestamp
));
}
// Validation 3: Timestamp ordering
if let Some(prev_ts) = prev_timestamp {
if record.timestamp < prev_ts {
return Err(format!(
"Timestamp ordering violation at row {}: {} is before previous timestamp {}",
row_num,
record.timestamp,
prev_ts
));
}
}
prev_timestamp = Some(record.timestamp);
actions.push(record);
}
if actions.is_empty() {
return Err(format!("CSV file '{}' contains no data rows", path.display()));
}
Ok(actions)
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::TimeZone;
#[test]
fn test_valid_action_record() {
// Test DQNActionRecord struct initialization
let record = DQNActionRecord {
timestamp: Utc.with_ymd_and_hms(2024, 10, 20, 23, 31, 0).unwrap(),
action: 2, // Hold
q_buy: -658.8440,
q_sell: 355.0268,
q_hold: 538.5875,
open: 5914.50,
high: 5914.75,
low: 5914.25,
close: 5914.25,
volume: 27,
};
assert_eq!(record.action, 2);
assert!(record.q_buy.is_finite());
assert!(record.q_sell.is_finite());
assert!(record.q_hold.is_finite());
}
#[test]
fn test_action_bounds_validation() {
// Test action bounds: 0 <= action <= 2
// Create a temporary CSV with invalid action
use std::io::Write;
let mut tmpfile = tempfile::NamedTempFile::new().unwrap();
writeln!(tmpfile, "timestamp,action,q_buy,q_sell,q_hold,open,high,low,close,volume").unwrap();
writeln!(tmpfile, "2024-10-20T23:31:00.000000000Z,3,-658.8440,355.0268,538.5875,5914.50,5914.75,5914.25,5914.25,27").unwrap();
tmpfile.flush().unwrap();
let result = load_actions_from_csv(tmpfile.path());
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.contains("Invalid action 3"));
assert!(err.contains("action must be 0 (Buy), 1 (Sell), or 2 (Hold)"));
}
#[test]
fn test_finite_qvalue_validation() {
// Test finite Q-value validation (NaN/Inf detection)
use std::io::Write;
let mut tmpfile = tempfile::NamedTempFile::new().unwrap();
writeln!(tmpfile, "timestamp,action,q_buy,q_sell,q_hold,open,high,low,close,volume").unwrap();
writeln!(tmpfile, "2024-10-20T23:31:00.000000000Z,2,NaN,355.0268,538.5875,5914.50,5914.75,5914.25,5914.25,27").unwrap();
tmpfile.flush().unwrap();
let result = load_actions_from_csv(tmpfile.path());
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.contains("Invalid q_buy"));
assert!(err.contains("Q-value must be finite"));
}
}