From afbc668dcdc3fd7285125b8b1e3d7d2c36e01aa7 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 13:38:58 +0100 Subject: [PATCH] feat(fxt): enhance watch TUI with full training metrics and gRPC helpers - Add connect_channel/connect_channel_lazy with TLS auto-detection - Expand TrainingSession to 35+ proto fields (RL diagnostics, hyperopt, eval metrics, health counters, checkpoints) - Add RL-specific rendering (entropy, action diversity, Q-values, rewards) - Add hyperopt progress panel (trial/best objective tracking) - Refactor all commands to use shared connect_channel helper - Default API gateway URL to https://api.fxhnt.ai Co-Authored-By: Claude Opus 4.6 --- bin/fxt/src/client/mod.rs | 32 ++ bin/fxt/src/commands/agent.rs | 8 +- bin/fxt/src/commands/auth.rs | 22 +- bin/fxt/src/commands/model/approve.rs | 8 +- bin/fxt/src/commands/model/list.rs | 8 +- bin/fxt/src/commands/model/reject.rs | 8 +- bin/fxt/src/commands/train/list.rs | 8 +- bin/fxt/src/commands/train/start.rs | 14 +- bin/fxt/src/commands/watch/event_loop.rs | 267 +++++++++++- bin/fxt/src/commands/watch/render.rs | 414 ++++++++++++++++++- bin/fxt/src/commands/watch/state.rs | 491 ++++++++++++++++++++++- bin/fxt/src/commands/watch/streams.rs | 52 ++- bin/fxt/src/main.rs | 40 +- 13 files changed, 1261 insertions(+), 111 deletions(-) diff --git a/bin/fxt/src/client/mod.rs b/bin/fxt/src/client/mod.rs index 2714efe00..d95553e1d 100644 --- a/bin/fxt/src/client/mod.rs +++ b/bin/fxt/src/client/mod.rs @@ -13,6 +13,38 @@ pub mod connection_manager; pub mod ml_training_client; pub mod trading_client; +use anyhow::{Context, Result}; +use tonic::transport::Channel; + +/// Create a gRPC channel that auto-configures TLS for `https://` URLs. +/// +/// Connects eagerly (awaits the TCP handshake). +pub async fn connect_channel(url: &str) -> Result { + let mut endpoint = Channel::from_shared(url.to_owned()).context("Invalid gRPC endpoint URL")?; + if url.starts_with("https://") { + endpoint = endpoint + .tls_config(tonic::transport::ClientTlsConfig::new().with_enabled_roots()) + .context("Failed to configure TLS")?; + } + endpoint + .connect() + .await + .context("Failed to connect to API Gateway") +} + +/// Create a lazily-connected gRPC channel with TLS auto-detection. +/// +/// Does not attempt the TCP connection until the first RPC call. +pub fn connect_channel_lazy(url: &str) -> Result { + let mut endpoint = Channel::from_shared(url.to_owned()).context("Invalid gRPC endpoint URL")?; + if url.starts_with("https://") { + endpoint = endpoint + .tls_config(tonic::transport::ClientTlsConfig::new().with_enabled_roots()) + .context("Failed to configure TLS")?; + } + Ok(endpoint.connect_lazy()) +} + /// Service endpoints configuration #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] pub struct ServiceEndpoints { diff --git a/bin/fxt/src/commands/agent.rs b/bin/fxt/src/commands/agent.rs index c5ee88afa..39f8b37a8 100644 --- a/bin/fxt/src/commands/agent.rs +++ b/bin/fxt/src/commands/agent.rs @@ -15,7 +15,7 @@ use anyhow::{Context, Result}; use clap::{Args, Subcommand}; use colored::Colorize; -use tonic::transport::Channel; +// Channel creation via crate::client::connect_channel use tonic::Request; // Import generated proto types @@ -181,11 +181,7 @@ pub async fn handle_allocate_portfolio( println!(); // Connect to API Gateway - let channel = Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; let mut client = TradingAgentServiceClient::new(channel); diff --git a/bin/fxt/src/commands/auth.rs b/bin/fxt/src/commands/auth.rs index c960555a9..e222d453d 100644 --- a/bin/fxt/src/commands/auth.rs +++ b/bin/fxt/src/commands/auth.rs @@ -34,7 +34,7 @@ pub enum AuthCommand { #[clap( long, env = "API_GATEWAY_URL", - default_value = "http://localhost:50051" + default_value = "https://api.fxhnt.ai" )] api_gateway_url: String, }, @@ -51,7 +51,7 @@ pub enum AuthCommand { #[clap( long, env = "API_GATEWAY_URL", - default_value = "http://localhost:50051" + default_value = "https://api.fxhnt.ai" )] api_gateway_url: String, }, @@ -98,11 +98,7 @@ async fn execute_login( println!("{}", "Connecting to API Gateway...".cyan()); // Connect to API Gateway - let channel = tonic::transport::Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; // Create auth components let storage = FileTokenStorage::new().context("Failed to initialize token storage")?; @@ -155,11 +151,7 @@ async fn execute_interactive_login(api_gateway_url: &str) -> Result<()> { println!(); println!("{}", "Connecting to API Gateway...".cyan()); - let channel = tonic::transport::Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; // Create auth components let storage = FileTokenStorage::new().context("Failed to initialize token storage")?; @@ -289,11 +281,7 @@ async fn execute_refresh(api_gateway_url: &str) -> Result<()> { println!("{}", "Refreshing tokens...".cyan()); // Connect to API Gateway - let channel = tonic::transport::Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; // Create auth components let storage = FileTokenStorage::new().context("Failed to initialize token storage")?; diff --git a/bin/fxt/src/commands/model/approve.rs b/bin/fxt/src/commands/model/approve.rs index 8a8e13e18..bc4b461bd 100644 --- a/bin/fxt/src/commands/model/approve.rs +++ b/bin/fxt/src/commands/model/approve.rs @@ -12,18 +12,14 @@ use crate::proto::ml_training::{ ml_training_service_client::MlTrainingServiceClient, ApproveModelRequest, }; use anyhow::{Context, Result}; -use tonic::{metadata::MetadataValue, transport::Channel, Request}; +use tonic::{metadata::MetadataValue, Request}; /// Approve a pending model promotion. /// /// Calls `ApproveModel` on the ML training service via the API gateway /// to promote the model to production. pub async fn run(api_gateway_url: &str, jwt_token: &str, model_id: &str) -> Result<()> { - let channel = Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; let mut client = MlTrainingServiceClient::new(channel); diff --git a/bin/fxt/src/commands/model/list.rs b/bin/fxt/src/commands/model/list.rs index cee33a6e7..15ca9518c 100644 --- a/bin/fxt/src/commands/model/list.rs +++ b/bin/fxt/src/commands/model/list.rs @@ -12,18 +12,14 @@ use crate::proto::ml_training::{ ml_training_service_client::MlTrainingServiceClient, ListAvailableModelsRequest, }; use anyhow::{Context, Result}; -use tonic::{metadata::MetadataValue, transport::Channel, Request}; +use tonic::{metadata::MetadataValue, Request}; /// List available models for training. /// /// Calls `ListAvailableModels` on the ML training service via the API gateway /// and displays a formatted table of available model types. pub async fn run(api_gateway_url: &str, jwt_token: &str) -> Result<()> { - let channel = Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; let mut client = MlTrainingServiceClient::new(channel); diff --git a/bin/fxt/src/commands/model/reject.rs b/bin/fxt/src/commands/model/reject.rs index 3370eecb7..d1a6167e5 100644 --- a/bin/fxt/src/commands/model/reject.rs +++ b/bin/fxt/src/commands/model/reject.rs @@ -13,7 +13,7 @@ use crate::proto::ml_training::{ ml_training_service_client::MlTrainingServiceClient, RejectModelRequest, }; use anyhow::{Context, Result}; -use tonic::{metadata::MetadataValue, transport::Channel, Request}; +use tonic::{metadata::MetadataValue, Request}; /// Reject a pending model promotion. /// @@ -25,11 +25,7 @@ pub async fn run( model_id: &str, reason: &str, ) -> Result<()> { - let channel = Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; let mut client = MlTrainingServiceClient::new(channel); diff --git a/bin/fxt/src/commands/train/list.rs b/bin/fxt/src/commands/train/list.rs index b82d83774..e7a30f229 100644 --- a/bin/fxt/src/commands/train/list.rs +++ b/bin/fxt/src/commands/train/list.rs @@ -42,7 +42,7 @@ use crate::proto::ml_training::{ ml_training_service_client::MlTrainingServiceClient, ListTrainingJobsRequest, TrainingJobSummary, TrainingStatus, }; -use tonic::{metadata::MetadataValue, transport::Channel, Request}; +use tonic::{metadata::MetadataValue, Request}; /// List training jobs with filtering and sorting #[derive(Parser, Debug)] @@ -84,11 +84,7 @@ impl ListCommand { /// Execute the list command pub async fn run(&self, api_gateway_url: &str, jwt_token: &str) -> Result<()> { // Connect to API Gateway - let channel = Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; let mut client = MlTrainingServiceClient::new(channel); diff --git a/bin/fxt/src/commands/train/start.rs b/bin/fxt/src/commands/train/start.rs index b229a7447..e2f0ec0ff 100644 --- a/bin/fxt/src/commands/train/start.rs +++ b/bin/fxt/src/commands/train/start.rs @@ -12,7 +12,7 @@ use crate::proto::ml_training::{ data_source::Source, ml_training_service_client::MlTrainingServiceClient, DataSource, StartTrainingRequest, TrainingStatus, }; -use tonic::{metadata::MetadataValue, transport::Channel, Request}; +use tonic::{metadata::MetadataValue, Request}; /// Start an on-demand training job #[derive(Args, Debug)] @@ -67,17 +67,7 @@ impl StartCommand { ); // Connect to API Gateway - let mut endpoint = Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")?; - if api_gateway_url.starts_with("https://") { - endpoint = endpoint - .tls_config(tonic::transport::ClientTlsConfig::new().with_enabled_roots()) - .context("Failed to configure TLS")?; - } - let channel = endpoint - .connect() - .await - .context("Failed to connect to API Gateway")?; + let channel = crate::client::connect_channel(api_gateway_url).await?; let mut client = MlTrainingServiceClient::new(channel); diff --git a/bin/fxt/src/commands/watch/event_loop.rs b/bin/fxt/src/commands/watch/event_loop.rs index c94df8518..bd87e7717 100644 --- a/bin/fxt/src/commands/watch/event_loop.rs +++ b/bin/fxt/src/commands/watch/event_loop.rs @@ -16,7 +16,7 @@ use tokio_stream::StreamExt; use super::render; use super::state::{ - CircuitBreakerRow, DashboardState, FillRow, ServiceRow, + CircuitBreakerRow, DashboardState, FillRow, ServiceRow, Tab, TrainingViewMode, }; use super::streams::{self, StreamEvent}; @@ -101,11 +101,29 @@ async fn run_event_loop( fn apply_stream_event(state: &mut DashboardState, event: StreamEvent) { match event { StreamEvent::TrainingUpdate { sessions, gpu } => { - // Push first session's epoch_loss to loss sparkline. - if let Some(first) = sessions.first() { + // Filter out uninitialised or aggregate rows: + // - empty model name (default proto) + // - model present but empty fold with zero epoch (summary row) + let active_sessions: Vec<_> = sessions + .into_iter() + .filter(|s| !(s.model.is_empty() || s.fold.is_empty() && s.epoch == 0.0)) + .collect(); + // Push first session's epoch_loss to global loss sparkline. + if let Some(first) = active_sessions.first() { state.training.push_loss(f64::from(first.epoch_loss)); } - state.training.sessions = sessions; + // Accumulate per-session metric history for detail sparklines. + for session in &active_sessions { + let key = format!("{}:{}", session.model, session.fold); + state + .training + .session_histories + .entry(key) + .or_default() + .push(session); + } + state.training.sessions = active_sessions; + state.training.clamp_selection(); state.training.gpu = gpu.clone(); state.system.gpu = gpu; state.training.connected = true; @@ -236,9 +254,78 @@ fn apply_stream_event(state: &mut DashboardState, event: StreamEvent) { /// Returns `true` if the user wants to quit. #[allow(clippy::wildcard_enum_match_arm)] // KeyCode has 25+ variants; exhaustive match is noise fn handle_key(state: &mut DashboardState, key: KeyEvent) -> bool { + // Global quit match key.code { KeyCode::Char('q') => return true, KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => return true, + _ => {} + } + + // Training tab has special modal key handling. + if state.current_tab() == Tab::Training { + match state.training.view_mode { + TrainingViewMode::List => match key.code { + KeyCode::Char('j') | KeyCode::Down => { + state.training.select_down(); + state.dirty = true; + } + KeyCode::Char('k') | KeyCode::Up => { + state.training.select_up(); + state.dirty = true; + } + KeyCode::Enter => { + state.training.enter_detail(); + state.dirty = true; + } + KeyCode::Char('1') => state.select_tab(0), + KeyCode::Char('2') => state.select_tab(1), + KeyCode::Char('3') => state.select_tab(2), + KeyCode::Char('4') => state.select_tab(3), + _ => {} + }, + TrainingViewMode::Detail => match key.code { + KeyCode::Esc => { + state.training.exit_detail(); + state.dirty = true; + } + KeyCode::Tab => { + state.training.next_sub_tab(); + state.dirty = true; + } + KeyCode::BackTab => { + state.training.prev_sub_tab(); + state.dirty = true; + } + KeyCode::Char('l') | KeyCode::Right => { + // Next session + state.training.select_down(); + state.dirty = true; + } + KeyCode::Char('h') | KeyCode::Left => { + // Previous session + state.training.select_up(); + state.dirty = true; + } + KeyCode::Char('j') | KeyCode::Down => { + state.training.scroll_down(); + state.dirty = true; + } + KeyCode::Char('k') | KeyCode::Up => { + state.training.scroll_up(); + state.dirty = true; + } + KeyCode::Char(c @ '1'..='4') => { + state.training.exit_detail(); + state.select_tab((c as u8 - b'1') as usize); + } + _ => {} + }, + } + return false; + } + + // Non-Training tabs: standard key handling. + match key.code { KeyCode::Char('1') => state.select_tab(0), KeyCode::Char('2') => state.select_tab(1), KeyCode::Char('3') => state.select_tab(2), @@ -302,29 +389,30 @@ mod tests { } #[test] - fn test_handle_key_scroll() { + fn test_handle_key_scroll_non_training_tab() { + // Non-training tabs still use scroll on j/k. let mut state = DashboardState::default(); - // Scroll down with 'j'. + state.select_tab(1); // Trading tab let j = KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE); handle_key(&mut state, j); - assert_eq!(state.training.scroll_offset, 1); + assert_eq!(state.trading.scroll_offset, 1); - // Scroll up with 'k'. let k = KeyEvent::new(KeyCode::Char('k'), KeyModifiers::NONE); handle_key(&mut state, k); - assert_eq!(state.training.scroll_offset, 0); + assert_eq!(state.trading.scroll_offset, 0); } #[test] - fn test_handle_key_arrow_scroll() { + fn test_handle_key_arrow_scroll_non_training_tab() { let mut state = DashboardState::default(); + state.select_tab(1); // Trading tab let down = KeyEvent::new(KeyCode::Down, KeyModifiers::NONE); handle_key(&mut state, down); - assert_eq!(state.training.scroll_offset, 1); + assert_eq!(state.trading.scroll_offset, 1); let up = KeyEvent::new(KeyCode::Up, KeyModifiers::NONE); handle_key(&mut state, up); - assert_eq!(state.training.scroll_offset, 0); + assert_eq!(state.trading.scroll_offset, 0); } #[test] @@ -334,6 +422,120 @@ mod tests { assert!(!handle_key(&mut state, key)); } + // -- Training list mode keys ------------------------------------------- + + fn make_state_with_sessions(n: usize) -> DashboardState { + let mut state = DashboardState::default(); + for i in 0..n { + state.training.sessions.push(TrainingSession { + model: format!("DQN_{i}"), + fold: format!("fold_{i}"), + ..Default::default() + }); + } + state + } + + #[test] + fn test_training_list_jk_selects() { + let mut state = make_state_with_sessions(3); + // j selects first row + let j = KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE); + handle_key(&mut state, j); + assert_eq!(state.training.selected_index, Some(0)); + // j again moves to second + handle_key(&mut state, j); + assert_eq!(state.training.selected_index, Some(1)); + // k moves back + let k = KeyEvent::new(KeyCode::Char('k'), KeyModifiers::NONE); + handle_key(&mut state, k); + assert_eq!(state.training.selected_index, Some(0)); + } + + #[test] + fn test_training_list_enter_opens_detail() { + let mut state = make_state_with_sessions(3); + state.training.selected_index = Some(1); + let enter = KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE); + handle_key(&mut state, enter); + assert_eq!(state.training.view_mode, TrainingViewMode::Detail); + } + + // -- Training detail mode keys ----------------------------------------- + + #[test] + fn test_training_detail_esc_exits() { + let mut state = make_state_with_sessions(3); + state.training.selected_index = Some(0); + state.training.enter_detail(); + let esc = KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE); + handle_key(&mut state, esc); + assert_eq!(state.training.view_mode, TrainingViewMode::List); + } + + #[test] + fn test_training_detail_tab_cycles_sub_tabs() { + let mut state = make_state_with_sessions(1); + state.training.selected_index = Some(0); + state.training.enter_detail(); + // DQN_0 -> Overview, Loss, RL, Health + let tab = KeyEvent::new(KeyCode::Tab, KeyModifiers::NONE); + handle_key(&mut state, tab); + assert_eq!( + state.training.detail_sub_tab, + super::super::state::DetailSubTab::Loss + ); + } + + #[test] + fn test_training_detail_backtab_cycles_back() { + let mut state = make_state_with_sessions(1); + state.training.selected_index = Some(0); + state.training.enter_detail(); + state.training.detail_sub_tab = super::super::state::DetailSubTab::Loss; + let backtab = KeyEvent::new(KeyCode::BackTab, KeyModifiers::SHIFT); + handle_key(&mut state, backtab); + assert_eq!( + state.training.detail_sub_tab, + super::super::state::DetailSubTab::Overview + ); + } + + #[test] + fn test_training_detail_lr_navigates_sessions() { + let mut state = make_state_with_sessions(3); + state.training.selected_index = Some(0); + state.training.enter_detail(); + // Right moves to next session + let right = KeyEvent::new(KeyCode::Right, KeyModifiers::NONE); + handle_key(&mut state, right); + assert_eq!(state.training.selected_index, Some(1)); + // Left moves back + let left = KeyEvent::new(KeyCode::Left, KeyModifiers::NONE); + handle_key(&mut state, left); + assert_eq!(state.training.selected_index, Some(0)); + } + + #[test] + fn test_training_detail_number_exits_and_switches_tab() { + let mut state = make_state_with_sessions(3); + state.training.selected_index = Some(0); + state.training.enter_detail(); + let key2 = KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE); + handle_key(&mut state, key2); + assert_eq!(state.training.view_mode, TrainingViewMode::List); + assert_eq!(state.active_tab, 1); // Trading tab + } + + #[test] + fn test_training_detail_q_quits() { + let mut state = make_state_with_sessions(1); + state.training.selected_index = Some(0); + state.training.enter_detail(); + let q = KeyEvent::new(KeyCode::Char('q'), KeyModifiers::NONE); + assert!(handle_key(&mut state, q)); + } + // -- format_nanos -------------------------------------------------------- #[test] @@ -370,6 +572,7 @@ mod tests { batches_per_second: 42.0, gradient_norm: 1.5, epoch_duration_seconds: 12.3, + ..Default::default() }; let gpu = GpuInfo { utilization_percent: 85.0, @@ -393,6 +596,11 @@ mod tests { assert!((state.training.loss_history.back().unwrap() - 0.123_f64).abs() < 1e-3); // GPU should be cloned to both training and system. assert!((state.system.gpu.utilization_percent - 85.0).abs() < f32::EPSILON); + // Per-session history should be accumulated. + assert_eq!(state.training.session_histories.len(), 1); + let hist = state.training.session_histories.get("DQN:fold0").unwrap(); + assert_eq!(hist.loss.len(), 1); + assert!((hist.loss[0] - 0.123_f64).abs() < 1e-3); assert!(state.dirty); } @@ -606,4 +814,39 @@ mod tests { // Dirty flag should be set after each event. assert!(state.dirty); } + + #[test] + fn test_training_update_filters_empty_model() { + let mut state = DashboardState::default(); + let valid = TrainingSession { + model: "DQN".into(), + fold: "fold0".into(), + epoch_loss: 0.5, + ..Default::default() + }; + let empty = TrainingSession { + model: String::new(), + fold: String::new(), + ..Default::default() + }; + // Summary row: model present but no fold and epoch=0 (aggregate row from proto) + let summary = TrainingSession { + model: "DQN".into(), + fold: String::new(), + epoch: 0.0, + ..Default::default() + }; + apply_stream_event( + &mut state, + StreamEvent::TrainingUpdate { + sessions: vec![empty, summary, valid], + gpu: GpuInfo::default(), + }, + ); + // Only the session with a non-empty model AND a fold (or non-zero epoch) should remain. + assert_eq!(state.training.sessions.len(), 1); + assert_eq!(state.training.sessions[0].model, "DQN"); + assert_eq!(state.training.session_histories.len(), 1); + assert!(state.training.session_histories.contains_key("DQN:fold0")); + } } diff --git a/bin/fxt/src/commands/watch/render.rs b/bin/fxt/src/commands/watch/render.rs index 7b1056058..a96b7347c 100644 --- a/bin/fxt/src/commands/watch/render.rs +++ b/bin/fxt/src/commands/watch/render.rs @@ -7,12 +7,13 @@ use ratatui::{ layout::{Constraint, Direction, Layout, Rect}, style::{Color, Modifier, Style}, text::{Line, Span}, - widgets::{Block, Borders, Cell, Paragraph, Row, Sparkline, Table, Tabs}, + widgets::{Block, Borders, Cell, Gauge, Paragraph, Row, Sparkline, Table, TableState, Tabs}, Frame, }; use super::state::{ - DashboardState, RiskTabState, SystemTabState, Tab, TradingTabState, TrainingTabState, + DashboardState, DetailSubTab, RiskTabState, SessionHistory, SystemTabState, Tab, + TradingTabState, TrainingTabState, TrainingViewMode, }; // --------------------------------------------------------------------------- @@ -33,13 +34,18 @@ pub(super) fn render(frame: &mut Frame, state: &DashboardState) { render_tab_bar(frame, chunks[0], state); match state.current_tab() { - Tab::Training => render_training_tab(frame, chunks[1], &state.training), + Tab::Training => match state.training.view_mode { + TrainingViewMode::List => render_training_list(frame, chunks[1], &state.training), + TrainingViewMode::Detail => { + render_training_detail(frame, chunks[1], &state.training); + } + }, Tab::Trading => render_trading_tab(frame, chunks[1], &state.trading), Tab::Risk => render_risk_tab(frame, chunks[1], &state.risk), Tab::System => render_system_tab(frame, chunks[1], &state.system), } - render_status_bar(frame, chunks[2]); + render_status_bar(frame, chunks[2], state); } // --------------------------------------------------------------------------- @@ -72,15 +78,43 @@ fn render_tab_bar(frame: &mut Frame, area: Rect, state: &DashboardState) { // Status bar // --------------------------------------------------------------------------- -fn render_status_bar(frame: &mut Frame, area: Rect) { - let bar = Paragraph::new(Line::from(vec![ - Span::styled("1-4", Style::default().fg(Color::Cyan)), - Span::raw(" tabs "), - Span::styled("j/k", Style::default().fg(Color::Cyan)), - Span::raw(" scroll "), - Span::styled("q", Style::default().fg(Color::Cyan)), - Span::raw(" quit"), - ])); +fn render_status_bar(frame: &mut Frame, area: Rect, state: &DashboardState) { + let hints: Vec> = + if state.current_tab() == Tab::Training && state.training.view_mode == TrainingViewMode::Detail { + vec![ + Span::styled("Esc", Style::default().fg(Color::Cyan)), + Span::raw(" back "), + Span::styled("Tab/S-Tab", Style::default().fg(Color::Cyan)), + Span::raw(" sub-tab "), + Span::styled("h/l", Style::default().fg(Color::Cyan)), + Span::raw(" prev/next "), + Span::styled("j/k", Style::default().fg(Color::Cyan)), + Span::raw(" scroll "), + Span::styled("q", Style::default().fg(Color::Cyan)), + Span::raw(" quit"), + ] + } else if state.current_tab() == Tab::Training { + vec![ + Span::styled("1-4", Style::default().fg(Color::Cyan)), + Span::raw(" tabs "), + Span::styled("j/k", Style::default().fg(Color::Cyan)), + Span::raw(" select "), + Span::styled("Enter", Style::default().fg(Color::Cyan)), + Span::raw(" detail "), + Span::styled("q", Style::default().fg(Color::Cyan)), + Span::raw(" quit"), + ] + } else { + vec![ + Span::styled("1-4", Style::default().fg(Color::Cyan)), + Span::raw(" tabs "), + Span::styled("j/k", Style::default().fg(Color::Cyan)), + Span::raw(" scroll "), + Span::styled("q", Style::default().fg(Color::Cyan)), + Span::raw(" quit"), + ] + }; + let bar = Paragraph::new(Line::from(hints)); frame.render_widget(bar, area); } @@ -88,7 +122,7 @@ fn render_status_bar(frame: &mut Frame, area: Rect) { // Training tab // --------------------------------------------------------------------------- -fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { +fn render_training_list(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { let conn_label = if tab.connected { "connected" } else { @@ -103,13 +137,15 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { ]) .split(area); - // -- Sessions table --------------------------------------------------- + // -- Sessions table with selection ------------------------------------ let header = Row::new(vec![ Cell::from("Model"), Cell::from("Fold"), Cell::from("Epoch"), Cell::from("Loss"), Cell::from("Val Loss"), + Cell::from("Acc"), + Cell::from("F1"), Cell::from("LR"), Cell::from("Batch/s"), Cell::from("Grad"), @@ -123,7 +159,6 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { let rows: Vec> = tab .sessions .iter() - .skip(tab.scroll_offset) .map(|s| { Row::new(vec![ Cell::from(s.model.as_str()), @@ -131,6 +166,8 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { Cell::from(format!("{:.1}", s.epoch)), Cell::from(format!("{:.4}", s.epoch_loss)), Cell::from(format!("{:.4}", s.validation_loss)), + Cell::from(format!("{:.4}", s.eval_accuracy)), + Cell::from(format!("{:.4}", s.eval_f1)), Cell::from(format!("{:.2e}", s.learning_rate)), Cell::from(format!("{:.1}", s.batches_per_second)), Cell::from(format!("{:.3}", s.gradient_norm)), @@ -146,19 +183,27 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { Constraint::Length(8), Constraint::Length(10), Constraint::Length(10), + Constraint::Length(8), + Constraint::Length(8), Constraint::Length(10), Constraint::Length(8), Constraint::Length(8), ], ) .header(header) + .row_highlight_style( + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ) .block( Block::default() .borders(Borders::ALL) .title(format!(" Training [{conn_label}] ")), ); - frame.render_widget(table, chunks[0]); + let mut table_state = TableState::default().with_selected(tab.selected_index); + frame.render_stateful_widget(table, chunks[0], &mut table_state); // -- Bottom row: GPU info (left) + loss sparkline (right) ------------- let bottom = Layout::default() @@ -200,6 +245,341 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { frame.render_widget(sparkline, bottom[1]); } +// --------------------------------------------------------------------------- +// Training detail view +// --------------------------------------------------------------------------- + +/// Convert a `&[f64]` metric history to Sparkline-compatible `Vec`. +fn spark_u64(data: &[f64], max_clamp: f64) -> Vec { + data.iter() + .map(|&v| { + let clamped = v.clamp(0.0, max_clamp); + (clamped * 100.0) as u64 + }) + .collect() +} + +fn render_sparkline_row(frame: &mut Frame, area: Rect, title: &str, data: &[f64], max: f64, color: Color) { + let vals = spark_u64(data, max); + let sparkline = Sparkline::default() + .block(Block::default().borders(Borders::ALL).title(format!(" {title} "))) + .data(&vals) + .style(Style::default().fg(color)); + frame.render_widget(sparkline, area); +} + +fn render_training_detail(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { + let Some(session) = tab.selected_session() else { + let msg = Paragraph::new("No session selected") + .block(Block::default().borders(Borders::ALL).title(" Detail ")); + frame.render_widget(msg, area); + return; + }; + + let vis_tabs = tab.visible_tabs(); + let history_key = format!("{}:{}", session.model, session.fold); + let empty_hist = SessionHistory::default(); + let history = tab.session_histories.get(&history_key).unwrap_or(&empty_hist); + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(3), // title bar + Constraint::Length(3), // sub-tab bar + Constraint::Min(0), // content + ]) + .split(area); + + // -- Title bar -------------------------------------------------------- + let status_label = if session.is_hyperopt { + format!( + "Hyperopt trial {}/{}", + session.hyperopt_trial_current, session.hyperopt_trial_total, + ) + } else { + format!("Epoch {:.1}", session.epoch) + }; + let title_text = format!( + " {} | {} | {} | GPU {:.0}%", + session.model, session.fold, status_label, tab.gpu.utilization_percent, + ); + let title_bar = Paragraph::new(title_text).block( + Block::default() + .borders(Borders::ALL) + .title(" Session Detail "), + ); + frame.render_widget(title_bar, chunks[0]); + + // -- Sub-tab bar ------------------------------------------------------ + let tab_titles: Vec> = vis_tabs + .iter() + .map(|t| Line::from(Span::raw(t.title()))) + .collect(); + let active_idx = vis_tabs + .iter() + .position(|t| *t == tab.detail_sub_tab) + .unwrap_or(0); + let sub_tabs = Tabs::new(tab_titles) + .select(active_idx) + .highlight_style( + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ) + .block(Block::default().borders(Borders::ALL)); + frame.render_widget(sub_tabs, chunks[1]); + + // -- Content area: dispatch to sub-tab renderer ----------------------- + match tab.detail_sub_tab { + DetailSubTab::Overview => render_detail_overview(frame, chunks[2], session, history), + DetailSubTab::Loss => render_detail_loss(frame, chunks[2], history), + DetailSubTab::RlDiagnostics => render_detail_rl(frame, chunks[2], session, history), + DetailSubTab::Metrics => render_detail_metrics(frame, chunks[2], session, history), + DetailSubTab::Hyperopt => render_detail_hyperopt(frame, chunks[2], session, history), + DetailSubTab::Health => render_detail_health(frame, chunks[2], session), + } +} + +// -- Sub-tab renderers ---------------------------------------------------- + +fn render_detail_overview( + frame: &mut Frame, + area: Rect, + session: &super::state::TrainingSession, + history: &SessionHistory, +) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(8), // key stats + Constraint::Min(0), // loss sparkline + ]) + .split(area); + + // Key stats grid + let stats = Paragraph::new(vec![ + Line::from(format!( + " Loss: {:.4} Val Loss: {:.4} LR: {:.2e}", + session.epoch_loss, session.validation_loss, session.learning_rate, + )), + Line::from(format!( + " Batch/s: {:.1} Grad Norm: {:.3} Epoch Time: {:.1}s", + session.batches_per_second, session.gradient_norm, session.epoch_duration_seconds, + )), + Line::from(format!( + " Batches: {:.0} Iter: {:.3}s Checkpoints: {} ({} failed)", + session.batches_processed, + session.iteration_seconds, + session.checkpoint_saves, + session.checkpoint_failures, + )), + Line::from(format!( + " NaN: {} Grad Explosions: {} Feature Errors: {}", + session.nan_detected, session.gradient_explosions, session.feature_errors, + )), + ]) + .block( + Block::default() + .borders(Borders::ALL) + .title(" Overview "), + ); + frame.render_widget(stats, chunks[0]); + + render_sparkline_row(frame, chunks[1], "Loss", &history.loss, 10.0, Color::Cyan); +} + +fn render_detail_loss(frame: &mut Frame, area: Rect, history: &SessionHistory) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Percentage(25), + Constraint::Percentage(25), + Constraint::Percentage(25), + Constraint::Percentage(25), + ]) + .split(area); + + render_sparkline_row(frame, chunks[0], "Loss", &history.loss, 10.0, Color::Cyan); + render_sparkline_row(frame, chunks[1], "Val Loss", &history.val_loss, 10.0, Color::Yellow); + render_sparkline_row(frame, chunks[2], "Learning Rate", &history.learning_rate, 1.0, Color::Green); + render_sparkline_row(frame, chunks[3], "Grad Norm", &history.gradient_norm, 100.0, Color::Red); +} + +fn render_detail_rl( + frame: &mut Frame, + area: Rect, + session: &super::state::TrainingSession, + history: &SessionHistory, +) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Percentage(20), + Constraint::Percentage(20), + Constraint::Percentage(20), + Constraint::Percentage(20), + Constraint::Percentage(20), + ]) + .split(area); + + render_sparkline_row(frame, chunks[0], "Q-Value Mean", &history.q_value_mean, 100.0, Color::Cyan); + render_sparkline_row(frame, chunks[1], "Policy Entropy", &history.policy_entropy, 10.0, Color::Yellow); + render_sparkline_row(frame, chunks[2], "KL Divergence", &history.kl_divergence, 1.0, Color::Red); + render_sparkline_row(frame, chunks[3], "Advantage Mean", &history.advantage_mean, 10.0, Color::Green); + + // Replay buffer gauge + let buf_size = session.replay_buffer_size; + let ratio = if buf_size > 0 { + (buf_size as f64 / 100_000.0).min(1.0) + } else { + 0.0 + }; + let gauge = Gauge::default() + .block( + Block::default() + .borders(Borders::ALL) + .title(format!(" Replay Buffer: {buf_size} ")), + ) + .gauge_style(Style::default().fg(Color::Cyan)) + .ratio(ratio); + frame.render_widget(gauge, chunks[4]); +} + +fn render_detail_metrics( + frame: &mut Frame, + area: Rect, + session: &super::state::TrainingSession, + history: &SessionHistory, +) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(5), // current values + Constraint::Min(0), // sparklines + ]) + .split(area); + + let current = Paragraph::new(vec![ + Line::from(format!( + " Accuracy: {:.4} F1: {:.4}", + session.eval_accuracy, session.eval_f1, + )), + Line::from(format!( + " Precision: {:.4} Recall: {:.4}", + session.eval_precision, session.eval_recall, + )), + ]) + .block( + Block::default() + .borders(Borders::ALL) + .title(" Current Metrics "), + ); + frame.render_widget(current, chunks[0]); + + let spark_area = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Percentage(25), + Constraint::Percentage(25), + Constraint::Percentage(25), + Constraint::Percentage(25), + ]) + .split(chunks[1]); + + render_sparkline_row(frame, spark_area[0], "Accuracy", &history.accuracy, 1.0, Color::Green); + render_sparkline_row(frame, spark_area[1], "Precision", &history.precision, 1.0, Color::Cyan); + render_sparkline_row(frame, spark_area[2], "Recall", &history.recall, 1.0, Color::Magenta); + render_sparkline_row(frame, spark_area[3], "F1", &history.f1, 1.0, Color::Yellow); +} + +fn render_detail_hyperopt( + frame: &mut Frame, + area: Rect, + session: &super::state::TrainingSession, + history: &SessionHistory, +) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(7), // stats + Constraint::Min(0), // best objective sparkline + ]) + .split(area); + + let pct = if session.hyperopt_trial_total > 0 { + (session.hyperopt_trial_current as f32 / session.hyperopt_trial_total as f32) * 100.0 + } else { + 0.0 + }; + let stats = Paragraph::new(vec![ + Line::from(format!( + " Trial: {}/{} ({:.0}%)", + session.hyperopt_trial_current, session.hyperopt_trial_total, pct, + )), + Line::from(format!( + " Best Objective: {:.6} Trial Best Loss: {:.6}", + session.hyperopt_best_objective, session.hyperopt_trial_best_loss, + )), + Line::from(format!( + " Trial Epoch: {} Failed: {} Elapsed: {:.0}s", + session.hyperopt_trial_epoch, + session.hyperopt_trials_failed, + session.hyperopt_elapsed_seconds, + )), + ]) + .block( + Block::default() + .borders(Borders::ALL) + .title(" Hyperopt Progress "), + ); + frame.render_widget(stats, chunks[0]); + + render_sparkline_row( + frame, + chunks[1], + "Best Objective", + &history.hyperopt_best_objective, + 10.0, + Color::Magenta, + ); +} + +fn render_detail_health( + frame: &mut Frame, + area: Rect, + session: &super::state::TrainingSession, +) { + let health = Paragraph::new(vec![ + Line::from(format!( + " NaN Detected: {}", + session.nan_detected, + )), + Line::from(format!( + " Gradient Explosions: {}", + session.gradient_explosions, + )), + Line::from(format!( + " Feature Errors: {}", + session.feature_errors, + )), + Line::from(""), + Line::from(format!( + " Checkpoint Saves: {} Failures: {}", + session.checkpoint_saves, session.checkpoint_failures, + )), + Line::from(format!( + " Checkpoint Size: {:.1} MB", + session.checkpoint_size_bytes / (1024.0 * 1024.0), + )), + ]) + .block( + Block::default() + .borders(Borders::ALL) + .title(" Health "), + ); + frame.render_widget(health, area); +} + // --------------------------------------------------------------------------- // Trading tab // --------------------------------------------------------------------------- diff --git a/bin/fxt/src/commands/watch/state.rs b/bin/fxt/src/commands/watch/state.rs index cefbfa637..60ffc534e 100644 --- a/bin/fxt/src/commands/watch/state.rs +++ b/bin/fxt/src/commands/watch/state.rs @@ -3,7 +3,7 @@ //! All mutable state lives here so that the render and event-loop modules //! can borrow it without circular dependencies. -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; // --------------------------------------------------------------------------- // Tab enum @@ -47,20 +47,149 @@ impl Tab { // Training types // --------------------------------------------------------------------------- -/// A single in-progress training session. -#[derive(Debug, Clone)] +/// Returns `true` if the model name indicates an RL model (DQN or PPO). +pub fn is_rl_model(model: &str) -> bool { + let lower = model.to_ascii_lowercase(); + lower.contains("dqn") || lower.contains("ppo") +} + +/// A single in-progress training session (all 35 proto fields + local gpu_percent). +#[derive(Debug, Clone, Default)] pub struct TrainingSession { pub model: String, pub fold: String, pub is_hyperopt: bool, + // Epoch / progress pub epoch: f32, pub epoch_loss: f32, pub validation_loss: f32, - pub learning_rate: f32, - pub gpu_percent: f32, + // Throughput pub batches_per_second: f32, + pub batches_processed: f32, + pub iteration_seconds: f32, + // Eval metrics + pub eval_accuracy: f32, + pub eval_precision: f32, + pub eval_recall: f32, + pub eval_f1: f32, + // Checkpoint + pub checkpoint_size_bytes: f32, + pub checkpoint_saves: u32, + pub checkpoint_failures: u32, + // Health counters + pub nan_detected: u32, + pub gradient_explosions: u32, + pub feature_errors: u32, + // Hyperopt + pub hyperopt_trial_current: u32, + pub hyperopt_trial_total: u32, + pub hyperopt_best_objective: f32, + pub hyperopt_trials_failed: u32, + // RL diagnostics + pub q_value_mean: f32, + pub q_value_max: f32, + pub policy_entropy: f32, + pub kl_divergence: f32, + pub advantage_mean: f32, + pub replay_buffer_size: u32, + // Gradient & training health pub gradient_norm: f32, + pub learning_rate: f32, pub epoch_duration_seconds: f32, + // Hyperopt intra-trial + pub hyperopt_trial_epoch: u32, + pub hyperopt_trial_best_loss: f32, + pub hyperopt_elapsed_seconds: f32, + // Local (not from proto) + pub gpu_percent: f32, +} + +/// Whether the Training tab shows the session list or a single-session detail. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TrainingViewMode { + #[default] + List, + Detail, +} + +/// Sub-tabs within the training detail view. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DetailSubTab { + #[default] + Overview, + Loss, + RlDiagnostics, + Metrics, + Hyperopt, + Health, +} + +impl DetailSubTab { + /// Human-readable label for the sub-tab bar. + pub fn title(self) -> &'static str { + match self { + Self::Overview => "Overview", + Self::Loss => "Loss", + Self::RlDiagnostics => "RL", + Self::Metrics => "Metrics", + Self::Hyperopt => "Hyperopt", + Self::Health => "Health", + } + } +} + +/// Returns the sub-tabs visible for a given model/session configuration. +pub fn visible_sub_tabs(model: &str, is_hyperopt: bool) -> Vec { + let mut tabs = vec![DetailSubTab::Overview, DetailSubTab::Loss]; + if is_rl_model(model) { + tabs.push(DetailSubTab::RlDiagnostics); + } else { + tabs.push(DetailSubTab::Metrics); + } + if is_hyperopt { + tabs.push(DetailSubTab::Hyperopt); + } + tabs.push(DetailSubTab::Health); + tabs +} + +/// Per-session unbounded metric history for sparklines. +#[derive(Debug, Clone, Default)] +pub struct SessionHistory { + pub loss: Vec, + pub val_loss: Vec, + pub learning_rate: Vec, + pub gradient_norm: Vec, + pub batches_per_second: Vec, + pub accuracy: Vec, + pub precision: Vec, + pub recall: Vec, + pub f1: Vec, + pub q_value_mean: Vec, + pub policy_entropy: Vec, + pub kl_divergence: Vec, + pub advantage_mean: Vec, + pub hyperopt_best_objective: Vec, +} + +impl SessionHistory { + /// Append one sample from a training session snapshot. + pub fn push(&mut self, s: &TrainingSession) { + self.loss.push(f64::from(s.epoch_loss)); + self.val_loss.push(f64::from(s.validation_loss)); + self.learning_rate.push(f64::from(s.learning_rate)); + self.gradient_norm.push(f64::from(s.gradient_norm)); + self.batches_per_second.push(f64::from(s.batches_per_second)); + self.accuracy.push(f64::from(s.eval_accuracy)); + self.precision.push(f64::from(s.eval_precision)); + self.recall.push(f64::from(s.eval_recall)); + self.f1.push(f64::from(s.eval_f1)); + self.q_value_mean.push(f64::from(s.q_value_mean)); + self.policy_entropy.push(f64::from(s.policy_entropy)); + self.kl_divergence.push(f64::from(s.kl_divergence)); + self.advantage_mean.push(f64::from(s.advantage_mean)); + self.hyperopt_best_objective.push(f64::from(s.hyperopt_best_objective)); + } } /// GPU telemetry snapshot. @@ -81,6 +210,11 @@ pub struct TrainingTabState { pub loss_history: VecDeque, pub scroll_offset: usize, pub connected: bool, + // -- Detail view state -- + pub selected_index: Option, + pub view_mode: TrainingViewMode, + pub detail_sub_tab: DetailSubTab, + pub session_histories: HashMap, } const MAX_LOSS_HISTORY: usize = 200; @@ -101,6 +235,95 @@ impl TrainingTabState { pub fn scroll_down(&mut self) { self.scroll_offset = self.scroll_offset.saturating_add(1); } + + /// Move selection cursor up (list mode). + pub fn select_up(&mut self) { + if let Some(idx) = self.selected_index { + self.selected_index = Some(idx.saturating_sub(1)); + } + } + + /// Move selection cursor down (list mode). + pub fn select_down(&mut self) { + match self.selected_index { + Some(idx) => { + let max = self.sessions.len().saturating_sub(1); + if idx < max { + self.selected_index = Some(idx.saturating_add(1)); + } + } + None => { + if !self.sessions.is_empty() { + self.selected_index = Some(0); + } + } + } + } + + /// Enter detail view for the currently selected session. + pub fn enter_detail(&mut self) { + if self.selected_index.is_some() && !self.sessions.is_empty() { + self.view_mode = TrainingViewMode::Detail; + self.detail_sub_tab = DetailSubTab::Overview; + self.scroll_offset = 0; + } + } + + /// Return to list view. + pub fn exit_detail(&mut self) { + self.view_mode = TrainingViewMode::List; + self.scroll_offset = 0; + } + + /// Advance to the next visible sub-tab. + pub fn next_sub_tab(&mut self) { + let tabs = match self.selected_index.and_then(|i| self.sessions.get(i)) { + Some(s) => visible_sub_tabs(&s.model, s.is_hyperopt), + None => return, + }; + if let Some(pos) = tabs.iter().position(|t| *t == self.detail_sub_tab) { + if let Some(&next) = tabs.get(pos + 1) { + self.detail_sub_tab = next; + } + } + } + + /// Go back to the previous visible sub-tab. + pub fn prev_sub_tab(&mut self) { + let tabs = match self.selected_index.and_then(|i| self.sessions.get(i)) { + Some(s) => visible_sub_tabs(&s.model, s.is_hyperopt), + None => return, + }; + if let Some(pos) = tabs.iter().position(|t| *t == self.detail_sub_tab) { + if let Some(&prev) = pos.checked_sub(1).and_then(|p| tabs.get(p)) { + self.detail_sub_tab = prev; + } + } + } + + /// The currently selected session, if any. + pub fn selected_session(&self) -> Option<&TrainingSession> { + self.selected_index.and_then(|i| self.sessions.get(i)) + } + + /// Sub-tabs visible for the currently selected session. + pub fn visible_tabs(&self) -> Vec { + match self.selected_session() { + Some(s) => visible_sub_tabs(&s.model, s.is_hyperopt), + None => vec![DetailSubTab::Overview], + } + } + + /// Clamp selected_index after sessions list changes. + pub fn clamp_selection(&mut self) { + match (self.sessions.is_empty(), self.selected_index) { + (true, _) => self.selected_index = None, + (false, Some(idx)) if idx >= self.sessions.len() => { + self.selected_index = Some(self.sessions.len().saturating_sub(1)); + } + _ => {} + } + } } // --------------------------------------------------------------------------- @@ -362,4 +585,262 @@ mod tests { state.scroll_up(); assert_eq!(state.training.scroll_offset, 0); } + + // -- is_rl_model -------------------------------------------------------- + + #[test] + fn test_is_rl_model() { + assert!(is_rl_model("DQN")); + assert!(is_rl_model("dqn")); + assert!(is_rl_model("PPO")); + assert!(is_rl_model("ContinuousPPO")); + assert!(!is_rl_model("TFT")); + assert!(!is_rl_model("Mamba2")); + assert!(!is_rl_model("XLSTM")); + } + + // -- visible_sub_tabs --------------------------------------------------- + + #[test] + fn test_visible_sub_tabs_rl() { + let tabs = visible_sub_tabs("DQN", false); + assert_eq!( + tabs, + vec![ + DetailSubTab::Overview, + DetailSubTab::Loss, + DetailSubTab::RlDiagnostics, + DetailSubTab::Health, + ] + ); + } + + #[test] + fn test_visible_sub_tabs_supervised() { + let tabs = visible_sub_tabs("TFT", false); + assert_eq!( + tabs, + vec![ + DetailSubTab::Overview, + DetailSubTab::Loss, + DetailSubTab::Metrics, + DetailSubTab::Health, + ] + ); + } + + #[test] + fn test_visible_sub_tabs_hyperopt_rl() { + let tabs = visible_sub_tabs("PPO", true); + assert_eq!( + tabs, + vec![ + DetailSubTab::Overview, + DetailSubTab::Loss, + DetailSubTab::RlDiagnostics, + DetailSubTab::Hyperopt, + DetailSubTab::Health, + ] + ); + } + + #[test] + fn test_visible_sub_tabs_hyperopt_supervised() { + let tabs = visible_sub_tabs("TFT", true); + assert_eq!( + tabs, + vec![ + DetailSubTab::Overview, + DetailSubTab::Loss, + DetailSubTab::Metrics, + DetailSubTab::Hyperopt, + DetailSubTab::Health, + ] + ); + } + + // -- SessionHistory::push ----------------------------------------------- + + #[test] + fn test_session_history_push() { + let mut hist = SessionHistory::default(); + let session = TrainingSession { + epoch_loss: 0.5, + validation_loss: 0.6, + learning_rate: 1e-3, + gradient_norm: 1.2, + batches_per_second: 42.0, + eval_accuracy: 0.85, + eval_f1: 0.82, + q_value_mean: 3.5, + policy_entropy: 1.1, + kl_divergence: 0.02, + advantage_mean: 0.1, + hyperopt_best_objective: 0.45, + ..Default::default() + }; + + hist.push(&session); + hist.push(&session); + + assert_eq!(hist.loss.len(), 2); + assert!((hist.loss[0] - 0.5).abs() < f64::EPSILON); + assert!((hist.accuracy[0] - 0.85).abs() < 1e-6); + assert!((hist.q_value_mean[0] - 3.5).abs() < 1e-6); + } + + // -- select_up / select_down -------------------------------------------- + + fn make_tab_with_sessions(n: usize) -> TrainingTabState { + let mut tab = TrainingTabState::default(); + for i in 0..n { + tab.sessions.push(TrainingSession { + model: format!("DQN_{i}"), + fold: format!("fold_{i}"), + ..Default::default() + }); + } + tab + } + + #[test] + fn test_select_down_from_none() { + let mut tab = make_tab_with_sessions(3); + assert_eq!(tab.selected_index, None); + tab.select_down(); + assert_eq!(tab.selected_index, Some(0)); + } + + #[test] + fn test_select_down_clamps_at_end() { + let mut tab = make_tab_with_sessions(3); + tab.selected_index = Some(2); + tab.select_down(); + assert_eq!(tab.selected_index, Some(2)); + } + + #[test] + fn test_select_up_clamps_at_zero() { + let mut tab = make_tab_with_sessions(3); + tab.selected_index = Some(0); + tab.select_up(); + assert_eq!(tab.selected_index, Some(0)); + } + + #[test] + fn test_select_down_empty_sessions() { + let mut tab = TrainingTabState::default(); + tab.select_down(); + assert_eq!(tab.selected_index, None); + } + + #[test] + fn test_select_up_down_navigation() { + let mut tab = make_tab_with_sessions(5); + tab.selected_index = Some(0); + tab.select_down(); + tab.select_down(); + assert_eq!(tab.selected_index, Some(2)); + tab.select_up(); + assert_eq!(tab.selected_index, Some(1)); + } + + // -- mode transitions --------------------------------------------------- + + #[test] + fn test_enter_detail_with_selection() { + let mut tab = make_tab_with_sessions(3); + tab.selected_index = Some(1); + tab.enter_detail(); + assert_eq!(tab.view_mode, TrainingViewMode::Detail); + assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview); + } + + #[test] + fn test_enter_detail_without_selection_noop() { + let mut tab = make_tab_with_sessions(3); + tab.enter_detail(); + assert_eq!(tab.view_mode, TrainingViewMode::List); + } + + #[test] + fn test_exit_detail() { + let mut tab = make_tab_with_sessions(3); + tab.selected_index = Some(0); + tab.enter_detail(); + tab.scroll_offset = 5; + tab.exit_detail(); + assert_eq!(tab.view_mode, TrainingViewMode::List); + assert_eq!(tab.scroll_offset, 0); + } + + // -- sub-tab navigation ------------------------------------------------- + + #[test] + fn test_next_prev_sub_tab() { + let mut tab = make_tab_with_sessions(1); + tab.selected_index = Some(0); + tab.enter_detail(); + + // DQN -> Overview, Loss, RL, Health + assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview); + tab.next_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::Loss); + tab.next_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::RlDiagnostics); + tab.next_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::Health); + // At the end — should stay + tab.next_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::Health); + + // Go back + tab.prev_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::RlDiagnostics); + tab.prev_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::Loss); + tab.prev_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview); + // At start — should stay + tab.prev_sub_tab(); + assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview); + } + + // -- clamp_selection ---------------------------------------------------- + + #[test] + fn test_clamp_selection_empty() { + let mut tab = TrainingTabState::default(); + tab.selected_index = Some(5); + tab.clamp_selection(); + assert_eq!(tab.selected_index, None); + } + + #[test] + fn test_clamp_selection_out_of_bounds() { + let mut tab = make_tab_with_sessions(3); + tab.selected_index = Some(10); + tab.clamp_selection(); + assert_eq!(tab.selected_index, Some(2)); + } + + #[test] + fn test_clamp_selection_valid() { + let mut tab = make_tab_with_sessions(3); + tab.selected_index = Some(1); + tab.clamp_selection(); + assert_eq!(tab.selected_index, Some(1)); + } + + // -- DetailSubTab::title ------------------------------------------------ + + #[test] + fn test_detail_sub_tab_titles() { + assert_eq!(DetailSubTab::Overview.title(), "Overview"); + assert_eq!(DetailSubTab::Loss.title(), "Loss"); + assert_eq!(DetailSubTab::RlDiagnostics.title(), "RL"); + assert_eq!(DetailSubTab::Metrics.title(), "Metrics"); + assert_eq!(DetailSubTab::Hyperopt.title(), "Hyperopt"); + assert_eq!(DetailSubTab::Health.title(), "Health"); + } } diff --git a/bin/fxt/src/commands/watch/streams.rs b/bin/fxt/src/commands/watch/streams.rs index 409627cf2..f9c674bb8 100644 --- a/bin/fxt/src/commands/watch/streams.rs +++ b/bin/fxt/src/commands/watch/streams.rs @@ -83,9 +83,7 @@ pub(super) enum StreamEvent { /// Create a lazily-connected gRPC channel to the given URL. fn connect_channel(url: &str) -> Result { - Ok(Channel::from_shared(url.to_owned()) - .context("invalid gRPC endpoint URL")? - .connect_lazy()) + crate::client::connect_channel_lazy(url) } /// Parse a JWT string into a tonic `MetadataValue` suitable for the @@ -123,14 +121,49 @@ fn convert_training_session(s: &mon::TrainingSession) -> TrainingSession { model: s.model.clone(), fold: s.fold.clone(), is_hyperopt: s.is_hyperopt, + // Epoch / progress epoch: s.current_epoch, epoch_loss: s.epoch_loss, validation_loss: s.validation_loss, - learning_rate: s.learning_rate, - gpu_percent: 0.0, // per-session GPU not in proto; filled from GpuSnapshot + // Throughput batches_per_second: s.batches_per_second, + batches_processed: s.batches_processed, + iteration_seconds: s.iteration_seconds, + // Eval metrics + eval_accuracy: s.eval_accuracy, + eval_precision: s.eval_precision, + eval_recall: s.eval_recall, + eval_f1: s.eval_f1, + // Checkpoint + checkpoint_size_bytes: s.checkpoint_size_bytes, + checkpoint_saves: s.checkpoint_saves, + checkpoint_failures: s.checkpoint_failures, + // Health counters + nan_detected: s.nan_detected, + gradient_explosions: s.gradient_explosions, + feature_errors: s.feature_errors, + // Hyperopt + hyperopt_trial_current: s.hyperopt_trial_current, + hyperopt_trial_total: s.hyperopt_trial_total, + hyperopt_best_objective: s.hyperopt_best_objective, + hyperopt_trials_failed: s.hyperopt_trials_failed, + // RL diagnostics + q_value_mean: s.q_value_mean, + q_value_max: s.q_value_max, + policy_entropy: s.policy_entropy, + kl_divergence: s.kl_divergence, + advantage_mean: s.advantage_mean, + replay_buffer_size: s.replay_buffer_size, + // Gradient & training health gradient_norm: s.gradient_norm, + learning_rate: s.learning_rate, epoch_duration_seconds: s.epoch_duration_seconds, + // Hyperopt intra-trial + hyperopt_trial_epoch: s.hyperopt_trial_epoch, + hyperopt_trial_best_loss: s.hyperopt_trial_best_loss, + hyperopt_elapsed_seconds: s.hyperopt_elapsed_seconds, + // Local + gpu_percent: 0.0, // per-session GPU not in proto; filled from GpuSnapshot } } @@ -503,9 +536,16 @@ async fn try_stream_system_status( while let Some(frame) = stream.next().await { let heartbeat = frame.context("system status stream message error")?; + let status_label = match heartbeat.status { + 1 => "HEALTHY", + 2 => "DEGRADED", + 3 => "UNHEALTHY", + 4 => "CRITICAL", + _ => "UNKNOWN", + }; tx.send(StreamEvent::SystemStatus { service: heartbeat.service_name, - status: format!("{}", heartbeat.status), + status: status_label.to_owned(), message: heartbeat.message, }) .await diff --git a/bin/fxt/src/main.rs b/bin/fxt/src/main.rs index 5b863f852..1c2f756b0 100644 --- a/bin/fxt/src/main.rs +++ b/bin/fxt/src/main.rs @@ -81,7 +81,7 @@ use zeroize as _; about = "Foxhunt Trading System Terminal Interface", long_about = "Foxhunt Trading System Terminal Interface\n\n\ Environment Variables:\n\ - API_GATEWAY_URL API Gateway URL (default: http://localhost:50051)\n\ + API_GATEWAY_URL API Gateway URL (default: https://api.fxhnt.ai)\n\ TLI_LOG_LEVEL Log level (default: info)\n\ TLI_TOKEN_STORAGE Token storage backend (default: keyring)\n\n\ Precedence: CLI args > Environment variables > Config file > Defaults" @@ -95,7 +95,7 @@ struct Cli { #[clap( long, env = "API_GATEWAY_URL", - default_value = "http://localhost:50051", + default_value = "https://api.fxhnt.ai", help = "API Gateway URL (env: API_GATEWAY_URL)" )] api_gateway_url: String, @@ -266,7 +266,6 @@ struct Claims { async fn load_jwt_token(api_gateway_url: &str) -> Result { use fxt::auth::login::LoginClient; use fxt::auth::token_manager::{AuthTokenManager, TokenStorage}; - use tonic::transport::Channel; let storage = FileTokenStorage::new().context("Failed to initialize file token storage")?; @@ -292,9 +291,8 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result { // Refresh tokens let auth_manager = AuthTokenManager::new(storage.clone()); - let channel = Channel::from_shared(api_gateway_url.to_owned()) - .context("Invalid API Gateway URL")? - .connect_lazy(); + let channel = fxt::client::connect_channel_lazy(api_gateway_url) + .context("Invalid API Gateway URL")?; let login_client = LoginClient::new(channel); @@ -400,7 +398,7 @@ async fn main() -> Result<()> { let mut cli = Cli::parse(); // Merge: CLI args override config file (precedence: CLI > Config > Default) - if cli.api_gateway_url == "http://localhost:50051" { + if cli.api_gateway_url == "https://api.fxhnt.ai" { // Using default, check if config has override cli.api_gateway_url = config.api_gateway_url.clone(); } @@ -420,11 +418,29 @@ async fn main() -> Result<()> { _ => Level::INFO, }; - // Initialize tracing with configured log level - let subscriber = FmtSubscriber::builder().with_max_level(log_level).finish(); + // Initialize tracing with configured log level. + // For the Watch (TUI) command, redirect logs to a file so they don't + // corrupt the ratatui alternate-screen display. + let is_tui = matches!(cli.command, Commands::Watch); - if let Err(e) = tracing::subscriber::set_global_default(subscriber) { - eprintln!("Warning: Failed to set global tracing subscriber: {}", e); + if is_tui { + let log_dir = dirs::home_dir() + .unwrap_or_default() + .join(".foxhunt"); + std::fs::create_dir_all(&log_dir).ok(); + if let Ok(file) = std::fs::File::create(log_dir.join("watch.log")) { + let subscriber = FmtSubscriber::builder() + .with_max_level(log_level) + .with_writer(std::sync::Mutex::new(file)) + .with_ansi(false) + .finish(); + tracing::subscriber::set_global_default(subscriber).ok(); + } + } else { + let subscriber = FmtSubscriber::builder().with_max_level(log_level).finish(); + if let Err(e) = tracing::subscriber::set_global_default(subscriber) { + eprintln!("Warning: Failed to set global tracing subscriber: {}", e); + } } // Route to subcommands @@ -514,7 +530,7 @@ mod tests { // Test default values when no flags provided let cli = Cli::parse_from(&["fxt", "auth", "status"]); - assert_eq!(cli.api_gateway_url, "http://localhost:50051"); + assert_eq!(cli.api_gateway_url, "https://api.fxhnt.ai"); assert_eq!(cli.log_level, "info"); assert_eq!(cli.token_storage, "keyring"); }