afd85b2f8f
- Delete 14 unused example files (-3,543 lines): config, adaptive-strategy, data, storage, trading_engine, api_gateway, backtesting, trading_service, chaos - Update ML training/eval binaries: improved CLI args, completion tracking, CUDA test cleanup, hyperopt enhancements - Fix KAN network and TFT module adjustments - Update risk test assertions for consistency - Fix backtesting repositories and promotion manager - Update .serena project config and Cargo dependencies Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
82 lines
2.9 KiB
Rust
82 lines
2.9 KiB
Rust
//! Training completion marker and metrics writer.
|
|
//!
|
|
//! After training finishes (successfully or not), the binary writes a marker
|
|
//! file (`DONE` or `FAILED`) plus an optional `metrics.json` into the output
|
|
//! directory. A K8s sidecar can watch for these files to detect completion
|
|
//! and trigger result uploads.
|
|
|
|
use std::path::Path;
|
|
|
|
use tracing::{error, info};
|
|
|
|
/// Summary metrics written alongside the `DONE` marker on success.
|
|
#[derive(serde::Serialize)]
|
|
#[allow(clippy::module_name_repetitions)]
|
|
pub struct CompletionMetrics {
|
|
/// Model name (e.g. "kan", "dqn", "ppo")
|
|
pub model: String,
|
|
/// Symbol trained on (e.g. "ES.FUT")
|
|
pub symbol: String,
|
|
/// Best validation loss across all folds (if available)
|
|
pub best_val_loss: Option<f64>,
|
|
/// Sharpe ratio from evaluation (if available)
|
|
pub sharpe_ratio: Option<f64>,
|
|
/// Total training epochs completed across all folds
|
|
pub epochs_completed: usize,
|
|
/// Number of walk-forward folds completed
|
|
pub folds_completed: usize,
|
|
}
|
|
|
|
/// Write a `DONE` marker and `metrics.json` into `output_dir`.
|
|
///
|
|
/// Errors are logged but never cause a panic -- the training result is
|
|
/// already persisted in checkpoints; the marker is best-effort.
|
|
#[allow(clippy::cognitive_complexity)]
|
|
pub fn write_success_marker(output_dir: &Path, metrics: &CompletionMetrics) {
|
|
// Write metrics.json
|
|
match serde_json::to_string_pretty(metrics) {
|
|
Ok(json) => {
|
|
let metrics_path = output_dir.join("metrics.json");
|
|
if let Err(e) = std::fs::write(&metrics_path, json) {
|
|
error!("Failed to write {}: {}", metrics_path.display(), e);
|
|
} else {
|
|
info!("Wrote {}", metrics_path.display());
|
|
}
|
|
}
|
|
Err(e) => {
|
|
error!("Failed to serialize CompletionMetrics: {}", e);
|
|
}
|
|
}
|
|
|
|
// Write empty DONE marker
|
|
let done_path = output_dir.join("DONE");
|
|
if let Err(e) = std::fs::write(&done_path, "") {
|
|
error!("Failed to write {}: {}", done_path.display(), e);
|
|
} else {
|
|
info!("Wrote {}", done_path.display());
|
|
}
|
|
}
|
|
|
|
/// Write a `FAILED` marker containing the error message.
|
|
///
|
|
/// Errors are logged but never cause a panic.
|
|
pub fn write_failure_marker(output_dir: &Path, error_msg: &str) {
|
|
// Ensure the output directory exists so the marker can be written even
|
|
// when training fails before creating any checkpoints.
|
|
if let Err(e) = std::fs::create_dir_all(output_dir) {
|
|
error!(
|
|
"Failed to create output dir {}: {} -- cannot write FAILED marker",
|
|
output_dir.display(),
|
|
e
|
|
);
|
|
return;
|
|
}
|
|
|
|
let failed_path = output_dir.join("FAILED");
|
|
if let Err(e) = std::fs::write(&failed_path, error_msg) {
|
|
error!("Failed to write {}: {}", failed_path.display(), e);
|
|
} else {
|
|
info!("Wrote {} ({})", failed_path.display(), error_msg);
|
|
}
|
|
}
|