//! Training completion marker and metrics writer. //! //! After training finishes (successfully or not), the binary writes a marker //! file (`DONE` or `FAILED`) plus an optional `metrics.json` into the output //! directory. A K8s sidecar can watch for these files to detect completion //! and trigger result uploads. use std::path::Path; use tracing::{error, info}; /// Summary metrics written alongside the `DONE` marker on success. #[derive(serde::Serialize)] #[allow(clippy::module_name_repetitions)] pub struct CompletionMetrics { /// Model name (e.g. "kan", "dqn", "ppo") pub model: String, /// Symbol trained on (e.g. "ES.FUT") pub symbol: String, /// Best validation loss across all folds (if available) pub best_val_loss: Option, /// Sharpe ratio from evaluation (if available) pub sharpe_ratio: Option, /// Total training epochs completed across all folds pub epochs_completed: usize, /// Number of walk-forward folds completed pub folds_completed: usize, } /// Write a `DONE` marker and `metrics.json` into `output_dir`. /// /// Errors are logged but never cause a panic -- the training result is /// already persisted in checkpoints; the marker is best-effort. #[allow(clippy::cognitive_complexity)] pub fn write_success_marker(output_dir: &Path, metrics: &CompletionMetrics) { // Write metrics.json match serde_json::to_string_pretty(metrics) { Ok(json) => { let metrics_path = output_dir.join("metrics.json"); if let Err(e) = std::fs::write(&metrics_path, json) { error!("Failed to write {}: {}", metrics_path.display(), e); } else { info!("Wrote {}", metrics_path.display()); } } Err(e) => { error!("Failed to serialize CompletionMetrics: {}", e); } } // Write empty DONE marker let done_path = output_dir.join("DONE"); if let Err(e) = std::fs::write(&done_path, "") { error!("Failed to write {}: {}", done_path.display(), e); } else { info!("Wrote {}", done_path.display()); } } /// Write a `FAILED` marker containing the error message. /// /// Errors are logged but never cause a panic. pub fn write_failure_marker(output_dir: &Path, error_msg: &str) { // Ensure the output directory exists so the marker can be written even // when training fails before creating any checkpoints. if let Err(e) = std::fs::create_dir_all(output_dir) { error!( "Failed to create output dir {}: {} -- cannot write FAILED marker", output_dir.display(), e ); return; } let failed_path = output_dir.join("FAILED"); if let Err(e) = std::fs::write(&failed_path, error_msg) { error!("Failed to write {}: {}", failed_path.display(), e); } else { info!("Wrote {} ({})", failed_path.display(), error_msg); } }