diff --git a/bin/fxt/src/tui/cockpits/overview.rs b/bin/fxt/src/tui/cockpits/overview.rs index 76daa1d8c..af5d03a01 100644 --- a/bin/fxt/src/tui/cockpits/overview.rs +++ b/bin/fxt/src/tui/cockpits/overview.rs @@ -8,7 +8,7 @@ use ratatui::Frame; use ratatui::layout::{Constraint, Direction, Layout, Rect}; -use ratatui::style::Style; +use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::{Gauge, List, ListItem, Row, Table}; @@ -36,13 +36,18 @@ impl Cockpit for OverviewCockpit { let bottom = Layout::default() .direction(Direction::Horizontal) - .constraints([Constraint::Percentage(50), Constraint::Percentage(50)]) + .constraints([ + Constraint::Percentage(30), + Constraint::Percentage(30), + Constraint::Percentage(40), + ]) .split(rows[1]); render_services(frame, top[0], state); render_resources(frame, top[1], state); render_training_summary(frame, bottom[0], state); render_portfolio(frame, bottom[1], state); + render_pods_summary(frame, bottom[2], state); } } @@ -136,7 +141,7 @@ fn render_training_summary(frame: &mut Frame, area: Rect, state: &AppState) { .map(|s| { Row::new(vec![ s.model.clone(), - format!("{}/{}", s.epoch, s.max_epochs), + s.epoch.to_string(), format!("{:.4}", s.loss), format!("{:.2}", s.sharpe), ]) @@ -252,3 +257,59 @@ fn render_portfolio(frame: &mut Frame, area: Rect, state: &AppState) { let summary = List::new(summary_items); frame.render_widget(summary, chunks[0]); } + +fn render_pods_summary(frame: &mut Frame, area: Rect, state: &AppState) { + if state.pods.is_empty() { + let msg = ratatui::widgets::Paragraph::new( + Line::from(Span::styled("No pod data", theme::muted_style())), + ) + .block(theme::block(" Pods ")); + frame.render_widget(msg, area); + return; + } + + // Compact: one row per service group (no sub-pod expansion). + let header = Row::new(vec!["Service", "Ready", "Status"]) + .style(theme::header_style()); + + let rows: Vec> = state + .pods + .iter() + .map(|g| { + let ready_str = format!("{}/{}", g.ready_count, g.total_count); + let (status_text, status_style) = if g.ready_count == g.total_count + && g.total_count > 0 + { + ("Running", Style::default().fg(theme::SUCCESS)) + } else if g.ready_count > 0 { + ("Degraded", Style::default().fg(theme::WARNING)) + } else { + ("Down", Style::default().fg(theme::ERROR)) + }; + + Row::new(vec![ + Span::styled( + g.service.clone(), + Style::default() + .fg(theme::TEXT) + .add_modifier(Modifier::BOLD), + ), + Span::styled(ready_str, Style::default().fg(theme::TEXT)), + Span::styled(status_text.to_owned(), status_style), + ]) + }) + .collect(); + + let table = Table::new( + rows, + [ + Constraint::Length(14), + Constraint::Length(6), + Constraint::Min(8), + ], + ) + .header(header) + .block(theme::block(" Pods ")); + + frame.render_widget(table, area); +} diff --git a/bin/fxt/src/tui/cockpits/pods.rs b/bin/fxt/src/tui/cockpits/pods.rs index 4bae5cf0a..82fd2c69b 100644 --- a/bin/fxt/src/tui/cockpits/pods.rs +++ b/bin/fxt/src/tui/cockpits/pods.rs @@ -8,7 +8,7 @@ use ratatui::text::{Line, Span}; use ratatui::widgets::{Paragraph, Row, Table}; use crate::tui::cockpit::Cockpit; -use crate::tui::state::AppState; +use crate::tui::state::{AppState, ConnectionStatus, PodGroupData}; use crate::tui::theme; pub struct PodsCockpit; @@ -20,16 +20,35 @@ impl Cockpit for PodsCockpit { fn render(&self, frame: &mut Frame, area: Rect, state: &AppState) { if state.pods.is_empty() { - let msg = Paragraph::new(Line::from(Span::styled( - "Waiting for pod data...", - theme::muted_style(), - ))) - .block(theme::block(" Pods ")); + let (msg_text, msg_style) = match &state.connection_status { + ConnectionStatus::Connecting => { + ("Connecting to cluster\u{2026}", theme::muted_style()) + } + ConnectionStatus::Reconnecting { attempt } => { + let _ = attempt; + ( + "Reconnecting to cluster\u{2026}", + Style::default().fg(theme::WARNING), + ) + } + ConnectionStatus::Error(e) => { + let _ = e; + ( + "Cluster unreachable \u{2014} check API gateway", + Style::default().fg(theme::ERROR), + ) + } + ConnectionStatus::Connected => { + ("Waiting for pod data\u{2026}", theme::muted_style()) + } + }; + let msg = Paragraph::new(Line::from(Span::styled(msg_text, msg_style))) + .block(theme::block(" Pods ")); frame.render_widget(msg, area); return; } - render_pod_table(frame, area, state); + render_pod_table(frame, area, state, state.pod_scroll_offset); } } @@ -41,15 +60,12 @@ fn shorten_pod_name(name: &str) -> &str { name.rsplit('-').next().unwrap_or(name) } -fn render_pod_table(frame: &mut Frame, area: Rect, state: &AppState) { - let header = Row::new(vec![ - "Service", "Ready", "Status", "Version", "Restarts", "Age", "Node", - ]) - .style(theme::header_style()); +/// Build flat pod rows from grouped pod data — used by both the Pods cockpit +/// and the Overview cockpit's mini-pod panel. +pub fn build_pod_rows<'pods>(pods: &'pods [PodGroupData]) -> Vec> { + let mut rows: Vec> = Vec::new(); - let mut rows: Vec> = Vec::new(); - - for group in &state.pods { + for group in pods { let ready_str = format!("{}/{}", group.ready_count, group.total_count); let (status_text, status_style) = if group.ready_count == group.total_count @@ -131,8 +147,25 @@ fn render_pod_table(frame: &mut Frame, area: Rect, state: &AppState) { } } + rows +} + +fn render_pod_table(frame: &mut Frame, area: Rect, state: &AppState, scroll_offset: usize) { + let header = Row::new(vec![ + "Service", "Ready", "Status", "Version", "Restarts", "Age", "Node", + ]) + .style(theme::header_style()); + + let all_rows = build_pod_rows(&state.pods); + + // Apply scroll offset — skip the first `scroll_offset` rows. + let visible_rows: Vec> = all_rows + .into_iter() + .skip(scroll_offset) + .collect(); + let table = Table::new( - rows, + visible_rows, [ Constraint::Length(22), Constraint::Length(7), diff --git a/bin/fxt/src/tui/cockpits/training.rs b/bin/fxt/src/tui/cockpits/training.rs index a3d8a6db8..cd98cf73a 100644 --- a/bin/fxt/src/tui/cockpits/training.rs +++ b/bin/fxt/src/tui/cockpits/training.rs @@ -1,16 +1,22 @@ #![allow(clippy::indexing_slicing)] // ratatui Layout::split indices match constraints -//! Training cockpit -- ML training session dashboard. +#![allow(clippy::integer_division)] // intentional truncation for time formatting +//! Training cockpit -- production ML training session dashboard. //! -//! Active sessions table, GPU panel, epoch financial metrics. +//! Sub-tab bar: **Live** | **History**. Tab/Shift-Tab switches. +//! Live tab: selectable sessions table (j/k), expandable detail panel (Enter/Right), +//! resource gauges (CPU + GPU), epoch financial metrics, health indicators. +//! History tab: completed/failed runs from ListTrainingJobs. use ratatui::Frame; use ratatui::layout::{Constraint, Direction, Layout, Rect}; use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; -use ratatui::widgets::{Gauge, List, ListItem, Row, Table}; +use ratatui::widgets::{Gauge, Paragraph, Row, Table}; use crate::tui::cockpit::Cockpit; -use crate::tui::state::AppState; +use crate::tui::state::{ + AppState, EpochSnapshot, TrainingHistoryEntry, TrainingSessionData, TrainingTab, +}; use crate::tui::theme; pub struct TrainingCockpit; @@ -21,61 +27,433 @@ impl Cockpit for TrainingCockpit { } fn render(&self, frame: &mut Frame, area: Rect, state: &AppState) { + // Split: 1-line sub-tab bar + remaining body + let outer = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(1), Constraint::Min(0)]) + .split(area); + + render_sub_tab_bar(frame, outer[0], state); + + match state.training_tab { + TrainingTab::Live => render_live_tab(frame, outer[1], state), + TrainingTab::History => render_history_tab(frame, outer[1], state), + } + } +} + +// --------------------------------------------------------------------------- +// Sub-tab bar +// --------------------------------------------------------------------------- + +fn render_sub_tab_bar(frame: &mut Frame, area: Rect, state: &AppState) { + let live_style = if state.training_tab == TrainingTab::Live { + Style::default() + .fg(theme::HIGHLIGHT) + .add_modifier(Modifier::BOLD) + } else { + theme::muted_style() + }; + let history_style = if state.training_tab == TrainingTab::History { + Style::default() + .fg(theme::HIGHLIGHT) + .add_modifier(Modifier::BOLD) + } else { + theme::muted_style() + }; + + let live_label = if state.training_tab == TrainingTab::Live { + "[Live]" + } else { + " Live " + }; + let hist_label = if state.training_tab == TrainingTab::History { + "[History]" + } else { + " History " + }; + + let line = Line::from(vec![ + Span::styled(" ", theme::muted_style()), + Span::styled(live_label, live_style), + Span::styled(" ", theme::muted_style()), + Span::styled(hist_label, history_style), + Span::styled(" Tab: switch", theme::muted_style()), + ]); + frame.render_widget(Paragraph::new(line), area); +} + +// --------------------------------------------------------------------------- +// Live tab (existing layout) +// --------------------------------------------------------------------------- + +fn render_live_tab(frame: &mut Frame, area: Rect, state: &AppState) { + if state.training_detail_expanded { + // 3-row layout: sessions table | resource+metrics | detail panel let rows = Layout::default() .direction(Direction::Vertical) - .constraints([Constraint::Percentage(55), Constraint::Percentage(45)]) + .constraints([ + Constraint::Percentage(30), + Constraint::Percentage(30), + Constraint::Percentage(40), + ]) + .split(area); + + render_sessions_table(frame, rows[0], state); + + let mid = Layout::default() + .direction(Direction::Horizontal) + .constraints([ + Constraint::Length(24), + Constraint::Length(24), + Constraint::Min(30), + ]) + .split(rows[1]); + + render_gpu_panel(frame, mid[0], state); + render_cpu_panel(frame, mid[1], state); + render_epoch_metrics(frame, mid[2], state); + render_detail_panel(frame, rows[2], state); + } else { + // 2-row layout: sessions table | resource+metrics + let rows = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)]) .split(area); render_sessions_table(frame, rows[0], state); let bottom = Layout::default() .direction(Direction::Horizontal) - .constraints([Constraint::Percentage(40), Constraint::Percentage(60)]) + .constraints([ + Constraint::Length(24), + Constraint::Length(24), + Constraint::Min(30), + ]) .split(rows[1]); render_gpu_panel(frame, bottom[0], state); - render_epoch_metrics(frame, bottom[1], state); + render_cpu_panel(frame, bottom[1], state); + render_epoch_metrics(frame, bottom[2], state); } } -fn render_sessions_table(frame: &mut Frame, area: Rect, state: &AppState) { +// --------------------------------------------------------------------------- +// History tab +// --------------------------------------------------------------------------- + +fn render_history_tab(frame: &mut Frame, area: Rect, state: &AppState) { + // 2-row layout: history table | resource panels + let rows = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Percentage(60), Constraint::Percentage(40)]) + .split(area); + + render_history_table(frame, rows[0], state); + + let bottom = Layout::default() + .direction(Direction::Horizontal) + .constraints([ + Constraint::Length(24), + Constraint::Length(24), + Constraint::Min(30), + ]) + .split(rows[1]); + + render_gpu_panel(frame, bottom[0], state); + render_cpu_panel(frame, bottom[1], state); + render_history_detail(frame, bottom[2], state); +} + +fn render_history_table(frame: &mut Frame, area: Rect, state: &AppState) { let header = Row::new(vec![ - "Model", "Fold", "Epoch", "Loss", "Val Loss", "Batch/s", "Sharpe", + " ", "Model", "Status", "Loss", "Best Val", "Created", "Duration", ]) .style(theme::header_style()); + let selected = state.selected_history_entry; + + let rows: Vec = state + .training_history + .iter() + .enumerate() + .map(|(i, h)| { + let is_selected = selected == Some(i); + let cursor = if is_selected { "\u{25b6}" } else { " " }; + + let base_style = if is_selected { + Style::default() + .fg(theme::HIGHLIGHT) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(theme::TEXT) + }; + + let status_style = match h.status.as_str() { + "completed" => Style::default().fg(theme::SUCCESS), + "failed" => Style::default().fg(theme::ERROR), + "stopped" => Style::default().fg(theme::WARNING), + "running" => Style::default().fg(theme::SECONDARY), + _ => theme::muted_style(), + }; + + Row::new(vec![ + Span::styled(cursor.to_owned(), Style::default().fg(theme::SECONDARY)), + Span::styled(h.model.clone(), base_style), + Span::styled(h.status.clone(), status_style), + Span::styled(format!("{:.4}", h.final_loss), base_style), + Span::styled(format!("{:.4}", h.best_val), base_style), + Span::styled(format_timestamp(h.created_at), theme::muted_style()), + Span::styled(format_job_duration(h), theme::muted_style()), + ]) + }) + .collect(); + + let title = format!( + " Training History \u{2014} {} run{} ", + state.training_history.len(), + if state.training_history.len() == 1 { + "" + } else { + "s" + } + ); + + let table = Table::new( + rows, + [ + Constraint::Length(2), // cursor + Constraint::Length(12), // model + Constraint::Length(10), // status + Constraint::Length(10), // loss + Constraint::Length(10), // best val + Constraint::Length(18), // created + Constraint::Min(10), // duration + ], + ) + .header(header) + .block(theme::block(&title)); + + frame.render_widget(table, area); +} + +fn render_history_detail(frame: &mut Frame, area: Rect, state: &AppState) { + let entry = state + .selected_history_entry + .and_then(|i| state.training_history.get(i)); + + let Some(h) = entry else { + let msg = Paragraph::new(Line::from(Span::styled( + "Select a run (j/k) to see details", + theme::muted_style(), + ))) + .block(theme::block(" Run Detail ")); + frame.render_widget(msg, area); + return; + }; + + let title = format!(" {} \u{2014} {} ", h.model, h.status); + let block = theme::block(&title); + let inner = block.inner(area); + frame.render_widget(block, area); + + let mut lines = vec![ + Line::from(vec![ + Span::styled(" Job ID: ", theme::muted_style()), + Span::styled(&h.job_id, Style::default().fg(theme::TEXT)), + ]), + Line::from(vec![ + Span::styled(" Loss: ", theme::muted_style()), + Span::styled( + format!("{:.6}", h.final_loss), + Style::default().fg(theme::TEXT), + ), + ]), + Line::from(vec![ + Span::styled(" Val: ", theme::muted_style()), + Span::styled( + format!("{:.6}", h.best_val), + Style::default().fg(theme::HIGHLIGHT), + ), + ]), + Line::from(vec![ + Span::styled(" Created:", theme::muted_style()), + Span::styled( + format!(" {}", format_timestamp(h.created_at)), + Style::default().fg(theme::TEXT), + ), + ]), + Line::from(vec![ + Span::styled(" Duration:", theme::muted_style()), + Span::styled( + format!(" {}", format_job_duration(h)), + Style::default().fg(theme::TEXT), + ), + ]), + ]; + + if !h.description.is_empty() { + lines.push(Line::from(vec![ + Span::styled(" Desc: ", theme::muted_style()), + Span::styled(&h.description, Style::default().fg(theme::TEXT)), + ])); + } + + frame.render_widget(Paragraph::new(lines), inner); +} + +// --------------------------------------------------------------------------- +// Formatting helpers +// --------------------------------------------------------------------------- + +fn format_duration(secs: f64) -> String { + if secs <= 0.0 { + return "\u{2014}".to_owned(); + } + let total = secs as u64; + let h = total / 3600; + let m = (total % 3600) / 60; + if h > 0 { + format!("{h}h{m:02}m") + } else { + format!("{m}m") + } +} + +fn format_timestamp(unix_secs: i64) -> String { + if unix_secs <= 0 { + return "\u{2014}".to_owned(); + } + // Cast to unsigned — timestamps are always positive here. + let ts = unix_secs as u64; + let secs_of_day = ts % 86400; + let hh = secs_of_day / 3600; + let mm = (secs_of_day % 3600) / 60; + let days = ts / 86400; + // Approximate date (good enough for display — no chrono dependency) + let y = 1970 + days / 365; + let doy = days % 365; + let m = doy / 30 + 1; + let d = doy % 30 + 1; + format!("{y:04}-{m:02}-{d:02} {hh:02}:{mm:02}") +} + +fn format_job_duration(h: &TrainingHistoryEntry) -> String { + if h.completed_at == 0 || h.started_at == 0 { + return "\u{2014}".to_owned(); + } + let secs = h.completed_at.saturating_sub(h.started_at); + if secs <= 0 { + return "\u{2014}".to_owned(); + } + let total = secs as u64; + let hours = total / 3600; + let mins = (total % 3600) / 60; + let s = total % 60; + if hours > 0 { + format!("{hours}h{mins:02}m") + } else if mins > 0 { + format!("{mins}m{s:02}s") + } else { + format!("{s}s") + } +} + +fn health_indicator(s: &TrainingSessionData) -> (String, Style) { + let issues = s.nan_detected + s.gradient_explosions + s.feature_errors; + if issues == 0 { + ("\u{25cf} OK".to_owned(), Style::default().fg(theme::SUCCESS)) + } else { + let mut parts = Vec::new(); + if s.nan_detected > 0 { + parts.push(format!("NaN({})", s.nan_detected)); + } + if s.gradient_explosions > 0 { + parts.push(format!("Grd({})", s.gradient_explosions)); + } + if s.feature_errors > 0 { + parts.push(format!("Feat({})", s.feature_errors)); + } + let label = format!("\u{25b2} {}", parts.join(",")); + (label, Style::default().fg(theme::WARNING)) + } +} + +// --------------------------------------------------------------------------- +// Sessions table (selectable) +// --------------------------------------------------------------------------- + +fn render_sessions_table(frame: &mut Frame, area: Rect, state: &AppState) { + let header = Row::new(vec![ + " ", "Model", "Fold", "Epoch", "Loss", "Val Loss", "Batch/s", "Duration", "Health", + ]) + .style(theme::header_style()); + + let selected = state.selected_training_session; + let rows: Vec = state .training_sessions .iter() - .map(|s| { + .enumerate() + .map(|(i, s)| { + let is_selected = selected == Some(i); + let cursor = if is_selected { "\u{25b6}" } else { " " }; + let (health_text, health_style) = health_indicator(s); + + let base_style = if is_selected { + Style::default() + .fg(theme::HIGHLIGHT) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(theme::TEXT) + }; + + let tag = if s.is_hyperopt { "HO" } else { "" }; + let model_label = if tag.is_empty() { + s.model.clone() + } else { + format!("{} [{}]", s.model, tag) + }; + Row::new(vec![ - s.model.clone(), - format!("{}", s.fold), - s.epoch.to_string(), - format!("{:.4}", s.loss), - format!("{:.4}", s.val_loss), - format!("{:.1}", s.batches_per_sec), - format!("{:.2}", s.sharpe), + Span::styled(cursor.to_owned(), Style::default().fg(theme::SECONDARY)), + Span::styled(model_label, base_style), + Span::styled(s.fold.clone(), base_style), + Span::styled(s.epoch.to_string(), base_style), + Span::styled(format!("{:.4}", s.loss), base_style), + Span::styled(format!("{:.4}", s.val_loss), base_style), + Span::styled(format!("{:.1}", s.batches_per_sec), base_style), + Span::styled( + format_duration(s.epoch_duration_secs * f64::from(s.epoch)), + base_style, + ), + Span::styled(health_text, health_style), ]) - .style(Style::default().fg(theme::TEXT)) }) .collect(); let block_title = if state.active_jobs > 0 { - format!(" Active Training Sessions -- {} K8s jobs ", state.active_jobs) + format!( + " Training Sessions \u{2014} {} K8s job{} ", + state.active_jobs, + if state.active_jobs == 1 { "" } else { "s" } + ) } else { - " Active Training Sessions ".to_owned() + " Training Sessions ".to_owned() }; + let table = Table::new( rows, [ - Constraint::Length(10), - Constraint::Length(6), - Constraint::Length(10), - Constraint::Length(10), - Constraint::Length(10), - Constraint::Length(10), - Constraint::Length(10), + Constraint::Length(2), // cursor + Constraint::Length(14), // model + Constraint::Length(8), // fold + Constraint::Length(7), // epoch + Constraint::Length(10), // loss + Constraint::Length(10), // val loss + Constraint::Length(9), // batch/s + Constraint::Length(10), // duration + Constraint::Min(12), // health ], ) .header(header) @@ -84,6 +462,10 @@ fn render_sessions_table(frame: &mut Frame, area: Rect, state: &AppState) { frame.render_widget(table, area); } +// --------------------------------------------------------------------------- +// GPU panel +// --------------------------------------------------------------------------- + fn render_gpu_panel(frame: &mut Frame, area: Rect, state: &AppState) { let block = theme::block(" GPU "); let inner = block.inner(area); @@ -92,10 +474,10 @@ fn render_gpu_panel(frame: &mut Frame, area: Rect, state: &AppState) { let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(1), - Constraint::Length(2), - Constraint::Length(2), - Constraint::Length(2), + Constraint::Length(1), // GPU name + Constraint::Length(2), // utilization gauge + Constraint::Length(2), // VRAM gauge + Constraint::Length(1), // temp + power Constraint::Min(0), ]) .split(inner); @@ -107,20 +489,17 @@ fn render_gpu_panel(frame: &mut Frame, area: Rect, state: &AppState) { .fg(theme::HIGHLIGHT) .add_modifier(Modifier::BOLD), )]); - frame.render_widget( - ratatui::widgets::Paragraph::new(name_line), - chunks[0], - ); + frame.render_widget(Paragraph::new(name_line), chunks[0]); - // Utilization gauge + // GPU utilization let util_pct = state.gpu.utilization / 100.0; - let util_gauge = Gauge::default() + let gpu_gauge = Gauge::default() .gauge_style(Style::default().fg(theme::SECONDARY)) - .label(format!("Util {:.1}%", state.gpu.utilization)) + .label(format!("{:.0}%", state.gpu.utilization)) .ratio(util_pct.clamp(0.0, 1.0)); - frame.render_widget(util_gauge, chunks[1]); + frame.render_widget(gpu_gauge, chunks[1]); - // Memory gauge + // VRAM let mem_pct = if state.gpu.memory_total_gb > 0.0 { state.gpu.memory_used_gb / state.gpu.memory_total_gb } else { @@ -129,7 +508,7 @@ fn render_gpu_panel(frame: &mut Frame, area: Rect, state: &AppState) { let mem_gauge = Gauge::default() .gauge_style(Style::default().fg(theme::PRIMARY)) .label(format!( - "VRAM {:.1}/{:.0} GB", + "{:.1}/{:.0}G", state.gpu.memory_used_gb, state.gpu.memory_total_gb )) .ratio(mem_pct.clamp(0.0, 1.0)); @@ -143,32 +522,107 @@ fn render_gpu_panel(frame: &mut Frame, area: Rect, state: &AppState) { } else { theme::SUCCESS }; - let power_text = if state.gpu.power_limit_watts > 0 { - format!("{}W / {}W", state.gpu.power_watts, state.gpu.power_limit_watts) + format!( + "{}W/{}W", + state.gpu.power_watts, state.gpu.power_limit_watts + ) } else { format!("{}W", state.gpu.power_watts) }; - let info_items = vec![ListItem::new(Line::from(vec![ + let temp_line = Line::from(vec![ Span::styled( - format!("{}C", state.gpu.temperature_c), + format!("{}\u{00b0}C", state.gpu.temperature_c), Style::default().fg(temp_color), ), - Span::styled(" ", theme::muted_style()), + Span::styled(" ", theme::muted_style()), Span::styled(power_text, theme::muted_style()), - ]))]; - let info = List::new(info_items); - frame.render_widget(info, chunks[3]); + ]); + frame.render_widget(Paragraph::new(temp_line), chunks[3]); } +// --------------------------------------------------------------------------- +// CPU / RAM panel +// --------------------------------------------------------------------------- + +fn render_cpu_panel(frame: &mut Frame, area: Rect, state: &AppState) { + let block = theme::block(" CPU "); + let inner = block.inner(area); + frame.render_widget(block, area); + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(1), // label + Constraint::Length(2), // CPU gauge + Constraint::Length(2), // RAM gauge + Constraint::Min(0), + ]) + .split(inner); + + // Label + let label = Line::from(Span::styled("System", theme::muted_style())); + frame.render_widget(Paragraph::new(label), chunks[0]); + + // CPU gauge + let cpu_pct = state.system_resources.cpu_usage_pct / 100.0; + let cpu_gauge = Gauge::default() + .gauge_style(Style::default().fg(theme::SUCCESS)) + .label(format!("CPU {:.0}%", state.system_resources.cpu_usage_pct)) + .ratio(cpu_pct.clamp(0.0, 1.0)); + frame.render_widget(cpu_gauge, chunks[1]); + + // RAM gauge + let ram_pct = if state.system_resources.ram_total_gb > 0.0 { + state.system_resources.ram_used_gb / state.system_resources.ram_total_gb + } else { + 0.0 + }; + let ram_gauge = Gauge::default() + .gauge_style(Style::default().fg(theme::WARNING)) + .label(format!( + "{:.0}/{:.0}G", + state.system_resources.ram_used_gb, state.system_resources.ram_total_gb + )) + .ratio(ram_pct.clamp(0.0, 1.0)); + frame.render_widget(ram_gauge, chunks[2]); +} + +// --------------------------------------------------------------------------- +// Epoch financial metrics + action distribution +// --------------------------------------------------------------------------- + fn render_epoch_metrics(frame: &mut Frame, area: Rect, state: &AppState) { - let header = Row::new(vec!["Model", "Sharpe", "Sortino", "Win%", "MaxDD"]) - .style(theme::header_style()); + let block = theme::block(" Epoch Financial Metrics "); + let inner = block.inner(area); + frame.render_widget(block, area); + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Min(0), Constraint::Length(1)]) + .split(inner); + + // Metrics table + let header = + Row::new(vec!["Model", "Sharpe", "Sortino", "Win%", "MaxDD", "PF", "Return", "Trades"]) + .style(theme::header_style()); + + let selected = state.selected_training_session; let rows: Vec = state .training_sessions .iter() - .map(|s| { + .enumerate() + .map(|(i, s)| { + let is_sel = selected == Some(i); + let base = if is_sel { + Style::default() + .fg(theme::HIGHLIGHT) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(theme::TEXT) + }; + let dd_style = if s.max_dd > 0.10 { Style::default().fg(theme::ERROR) } else if s.max_dd > 0.05 { @@ -178,20 +632,17 @@ fn render_epoch_metrics(frame: &mut Frame, area: Rect, state: &AppState) { }; Row::new(vec![ - Span::styled(s.model.clone(), Style::default().fg(theme::TEXT)), - Span::styled( - format!("{:.2}", s.sharpe), - Style::default().fg(theme::HIGHLIGHT), - ), + Span::styled(s.model.clone(), base), + Span::styled(format!("{:.2}", s.sharpe), Style::default().fg(theme::HIGHLIGHT)), Span::styled( format!("{:.2}", s.sortino), Style::default().fg(theme::SECONDARY), ), - Span::styled( - format!("{:.1}%", s.win_rate * 100.0), - Style::default().fg(theme::TEXT), - ), + Span::styled(format!("{:.1}%", s.win_rate * 100.0), base), Span::styled(format!("{:.1}%", s.max_dd * 100.0), dd_style), + Span::styled(format!("{:.2}", s.profit_factor), base), + Span::styled(format!("{:+.1}%", s.total_return * 100.0), base), + Span::styled(s.total_trades.to_string(), theme::muted_style()), ]) }) .collect(); @@ -200,14 +651,411 @@ fn render_epoch_metrics(frame: &mut Frame, area: Rect, state: &AppState) { rows, [ Constraint::Length(10), - Constraint::Length(10), - Constraint::Length(10), - Constraint::Length(10), - Constraint::Length(10), + Constraint::Length(8), + Constraint::Length(8), + Constraint::Length(7), + Constraint::Length(7), + Constraint::Length(6), + Constraint::Length(9), + Constraint::Min(6), + ], + ) + .header(header); + + frame.render_widget(table, chunks[0]); + + // Action distribution from selected session + let action_line = if let Some(sel) = selected.and_then(|i| state.training_sessions.get(i)) { + Line::from(vec![ + Span::styled("Actions: ", theme::muted_style()), + Span::styled( + format!("Buy {:.0}%", sel.action_buy_pct * 100.0), + Style::default().fg(theme::SUCCESS), + ), + Span::styled(" | ", theme::muted_style()), + Span::styled( + format!("Sell {:.0}%", sel.action_sell_pct * 100.0), + Style::default().fg(theme::ERROR), + ), + Span::styled(" | ", theme::muted_style()), + Span::styled( + format!("Hold {:.0}%", sel.action_hold_pct * 100.0), + Style::default().fg(theme::SECONDARY), + ), + ]) + } else { + Line::from(Span::styled( + "Select a session (j/k) to see actions", + theme::muted_style(), + )) + }; + frame.render_widget(Paragraph::new(action_line), chunks[1]); +} + +// --------------------------------------------------------------------------- +// Detail panel (RL diagnostics, gradient health, hyperopt) +// --------------------------------------------------------------------------- + +fn render_detail_panel(frame: &mut Frame, area: Rect, state: &AppState) { + let session = state + .selected_training_session + .and_then(|i| state.training_sessions.get(i)); + + let Some(s) = session else { + let msg = Paragraph::new(Line::from(Span::styled( + "No session selected", + theme::muted_style(), + ))) + .block(theme::block(" Session Detail ")); + frame.render_widget(msg, area); + return; + }; + + let title = format!(" {} {} \u{2014} Epoch {} ", s.model, s.fold, s.epoch); + let block = theme::block(&title); + let inner = block.inner(area); + frame.render_widget(block, area); + + // Split: diagnostics summary on top, epoch history table below + let sections = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(7), Constraint::Min(3)]) + .split(inner); + + // --- Top: 3-column diagnostics (compact) --- + render_diagnostics_row(frame, sections[0], s); + + // --- Bottom: per-epoch history table --- + render_epoch_history(frame, sections[1], s, state.epoch_scroll_offset); +} + +fn render_diagnostics_row(frame: &mut Frame, area: Rect, s: &TrainingSessionData) { + let cols = Layout::default() + .direction(Direction::Horizontal) + .constraints([ + Constraint::Percentage(35), + Constraint::Percentage(35), + Constraint::Percentage(30), + ]) + .split(area); + + // Col 1: RL diagnostics + render_detail_col( + frame, + cols[0], + "RL Diagnostics", + &[ + ("Q-mean", format!("{:.4}", s.q_value_mean)), + ("Q-max", format!("{:.4}", s.q_value_max)), + ("Entropy", format!("{:.4}", s.policy_entropy)), + ("KL-div", format!("{:.6}", s.kl_divergence)), + ("Advantage", format!("{:.4}", s.advantage_mean)), + ("Replay buf", format_replay_size(s.replay_buffer_size)), + ], + ); + + // Col 2: Gradient & health + render_detail_col( + frame, + cols[1], + "Gradient & Health", + &[ + ("Grad norm", format!("{:.4}", s.gradient_norm)), + ("Learn rate", format!("{:.2e}", s.learning_rate)), + ("NaN count", s.nan_detected.to_string()), + ("Grad explode", s.gradient_explosions.to_string()), + ("Feature err", s.feature_errors.to_string()), + ( + "Checkpoints", + format!("{} ok, {}", s.checkpoint_saves, s.checkpoint_failures), + ), + ], + ); + + // Col 3: Hyperopt or training timing + if s.is_hyperopt { + render_detail_col( + frame, + cols[2], + "Hyperopt", + &[ + ( + "Trial", + format!("{}/{}", s.hyperopt_trial_current, s.hyperopt_trial_total), + ), + ("Best obj", format!("{:.6}", s.hyperopt_best_objective)), + ("Failed", s.hyperopt_trials_failed.to_string()), + ("Trial epoch", s.hyperopt_trial_epoch.to_string()), + ("Elapsed", format_duration(s.hyperopt_elapsed_secs)), + ], + ); + } else { + render_detail_col( + frame, + cols[2], + "Training", + &[ + ("Iter time", format!("{:.2}s", s.iteration_secs)), + ("Epoch time", format_duration(s.epoch_duration_secs)), + ( + "Total time", + format_duration(s.epoch_duration_secs * f64::from(s.epoch)), + ), + ("Profit factor", format!("{:.2}", s.profit_factor)), + ("Total return", format!("{:+.2}%", s.total_return * 100.0)), + ], + ); + } +} + +fn render_epoch_history( + frame: &mut Frame, + area: Rect, + s: &TrainingSessionData, + scroll_offset: usize, +) { + let header = Row::new(vec![ + "Ep", "Loss", "Val", "Sharpe", "Sortino", "Win%", "MaxDD", "PF", "Return", "Trades", + "Q-mean", "GradNorm", "Buy%", "Sell%", "Hold%", + ]) + .style(theme::header_style()); + + // Build rows: epoch history + current epoch as the last row + let total_epochs = s.epoch_history.len() + 1; // +1 for current + let visible = area.height.saturating_sub(2) as usize; // header + border + let max_offset = total_epochs.saturating_sub(visible); + let offset = scroll_offset.min(max_offset); + + let mut all_rows: Vec = s + .epoch_history + .iter() + .map(|e| epoch_snapshot_row(e)) + .collect(); + + // Current epoch as the newest row + all_rows.push(current_epoch_row(s)); + + // Apply scroll offset + let visible_rows: Vec = all_rows.into_iter().skip(offset).take(visible).collect(); + + let epoch_count = total_epochs; + let title = format!( + " Epoch History \u{2014} {} epoch{} ", + epoch_count, + if epoch_count == 1 { "" } else { "s" } + ); + + let table = Table::new( + visible_rows, + [ + Constraint::Length(4), // epoch + Constraint::Length(7), // loss + Constraint::Length(7), // val + Constraint::Length(7), // sharpe + Constraint::Length(8), // sortino + Constraint::Length(6), // win% + Constraint::Length(6), // maxdd + Constraint::Length(5), // pf + Constraint::Length(8), // return + Constraint::Length(6), // trades + Constraint::Length(8), // q-mean + Constraint::Length(8), // grad + Constraint::Length(5), // buy% + Constraint::Length(5), // sell% + Constraint::Min(5), // hold% ], ) .header(header) - .block(theme::block(" Epoch Financial Metrics ")); + .block(theme::block(&title)); frame.render_widget(table, area); } + +fn epoch_snapshot_row(e: &EpochSnapshot) -> Row<'static> { + let dd_style = if e.max_dd > 0.10 { + Style::default().fg(theme::ERROR) + } else if e.max_dd > 0.05 { + Style::default().fg(theme::WARNING) + } else { + Style::default().fg(theme::SUCCESS) + }; + + Row::new(vec![ + Span::styled(e.epoch.to_string(), Style::default().fg(theme::TEXT)), + Span::styled(format!("{:.4}", e.loss), Style::default().fg(theme::TEXT)), + Span::styled( + format!("{:.4}", e.val_loss), + Style::default().fg(theme::SECONDARY), + ), + Span::styled( + format!("{:.2}", e.sharpe), + Style::default().fg(theme::HIGHLIGHT), + ), + Span::styled( + format!("{:.2}", e.sortino), + Style::default().fg(theme::SECONDARY), + ), + Span::styled( + format!("{:.1}%", e.win_rate * 100.0), + Style::default().fg(theme::TEXT), + ), + Span::styled(format!("{:.1}%", e.max_dd * 100.0), dd_style), + Span::styled( + format!("{:.2}", e.profit_factor), + Style::default().fg(theme::TEXT), + ), + Span::styled( + format!("{:+.1}%", e.total_return * 100.0), + Style::default().fg(theme::TEXT), + ), + Span::styled( + e.total_trades.to_string(), + theme::muted_style(), + ), + Span::styled( + format!("{:.3}", e.q_value_mean), + theme::muted_style(), + ), + Span::styled( + format!("{:.4}", e.gradient_norm), + theme::muted_style(), + ), + Span::styled( + format!("{:.0}", e.action_buy_pct * 100.0), + Style::default().fg(theme::SUCCESS), + ), + Span::styled( + format!("{:.0}", e.action_sell_pct * 100.0), + Style::default().fg(theme::ERROR), + ), + Span::styled( + format!("{:.0}", e.action_hold_pct * 100.0), + Style::default().fg(theme::SECONDARY), + ), + ]) +} + +fn current_epoch_row(s: &TrainingSessionData) -> Row<'static> { + let dd_style = if s.max_dd > 0.10 { + Style::default().fg(theme::ERROR) + } else if s.max_dd > 0.05 { + Style::default().fg(theme::WARNING) + } else { + Style::default().fg(theme::SUCCESS) + }; + + Row::new(vec![ + Span::styled( + format!("{}\u{25c0}", s.epoch), + Style::default() + .fg(theme::HIGHLIGHT) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.4}", s.loss), + Style::default() + .fg(theme::TEXT) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.4}", s.val_loss), + Style::default() + .fg(theme::SECONDARY) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.2}", s.sharpe), + Style::default() + .fg(theme::HIGHLIGHT) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.2}", s.sortino), + Style::default() + .fg(theme::SECONDARY) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.1}%", s.win_rate * 100.0), + Style::default() + .fg(theme::TEXT) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.1}%", s.max_dd * 100.0), + dd_style.add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.2}", s.profit_factor), + Style::default() + .fg(theme::TEXT) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:+.1}%", s.total_return * 100.0), + Style::default() + .fg(theme::TEXT) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + s.total_trades.to_string(), + Style::default() + .fg(theme::TEXT) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.3}", s.q_value_mean), + Style::default().fg(theme::TEXT).add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.4}", s.gradient_norm), + Style::default().fg(theme::TEXT).add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.0}", s.action_buy_pct * 100.0), + Style::default() + .fg(theme::SUCCESS) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.0}", s.action_sell_pct * 100.0), + Style::default() + .fg(theme::ERROR) + .add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{:.0}", s.action_hold_pct * 100.0), + Style::default() + .fg(theme::SECONDARY) + .add_modifier(Modifier::BOLD), + ), + ]) +} + +fn render_detail_col(frame: &mut Frame, area: Rect, title: &str, items: &[(&str, String)]) { + let mut lines = vec![Line::from(Span::styled( + title, + Style::default() + .fg(theme::SECONDARY) + .add_modifier(Modifier::BOLD), + ))]; + + for (label, value) in items { + lines.push(Line::from(vec![ + Span::styled(format!(" {label}: "), theme::muted_style()), + Span::styled(value.clone(), Style::default().fg(theme::TEXT)), + ])); + } + + frame.render_widget(Paragraph::new(lines), area); +} + +fn format_replay_size(size: u32) -> String { + if size >= 1_000_000 { + format!("{:.1}M", f64::from(size) / 1_000_000.0) + } else if size >= 1_000 { + format!("{:.1}K", f64::from(size) / 1_000.0) + } else { + size.to_string() + } +} diff --git a/bin/fxt/src/tui/data_fetcher.rs b/bin/fxt/src/tui/data_fetcher.rs index 3e426a0d4..d84f24bd8 100644 --- a/bin/fxt/src/tui/data_fetcher.rs +++ b/bin/fxt/src/tui/data_fetcher.rs @@ -21,12 +21,13 @@ use tonic::Request; use tracing::{debug, warn}; use crate::grpc::FoxhuntClient; -use crate::proto::{broker_gateway, data_acquisition, ml, monitoring, risk, trading}; +use crate::proto::{broker_gateway, data_acquisition, ml, ml_training, monitoring, risk, trading}; use super::state::{ AccountData, AlertData, CircuitBreakerData, ConnectionStatus, DataCacheData, DataFeedData, ExecutionData, GpuData, ModelStatusData, OrderEventData, PodData, PodGroupData, PositionData, - RiskAlertData, RiskData, ServiceData, SystemResources, TrainingSessionData, + RiskAlertData, RiskData, ServiceData, SystemResources, TrainingHistoryEntry, + TrainingSessionData, }; /// Empty filter string means "fetch all" in gRPC requests. @@ -79,6 +80,8 @@ pub enum StateUpdate { }, /// Cluster pod statuses (from SubscribeClusterPods). Pods(Vec), + /// Completed/failed training runs (from ListTrainingJobs polling). + TrainingHistory(Vec), /// gRPC connection status change. Connection(ConnectionStatus), } @@ -121,6 +124,7 @@ impl DataFetcher { /// Spawn all background stream tasks. pub fn start(&self, client: &FoxhuntClient) { self.spawn_training_metrics(client); + self.spawn_training_history(client); self.spawn_system_status(client); self.spawn_account_state(client); self.spawn_executions(client); @@ -163,7 +167,7 @@ impl DataFetcher { if !connected_sent.swap(true, Ordering::Relaxed) { let _ = tx.send(StateUpdate::Connection(ConnectionStatus::Connected)); } - let sessions = m.sessions.iter().map(convert_training_session).collect(); + let sessions = dedup_sessions(&m.sessions); let gpu = m.gpu.as_ref().map(convert_gpu).unwrap_or_default(); let resources = SystemResources { cpu_usage_pct: f64::from(m.cpu_percent), @@ -211,6 +215,67 @@ impl DataFetcher { }); } + /// Training history — polls ListTrainingJobs every 30s for the History tab. + fn spawn_training_history(&self, client: &FoxhuntClient) { + let mut ml_train = client.ml_training(); + let tx = self.tx.clone(); + let cancel = self.cancel.clone(); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(30)); + loop { + tokio::select! { + _ = cancel.cancelled() => return, + _ = interval.tick() => {} + } + + let req = Request::new(ml_training::ListTrainingJobsRequest { + page: 0, + page_size: 50, + status_filter: 0, // all statuses + model_type_filter: String::new(), + start_time: 0, + end_time: 0, + }); + match ml_train.list_training_jobs(req).await { + Ok(resp) => { + let inner = resp.into_inner(); + let entries: Vec = inner + .jobs + .iter() + .map(|j| { + let status = match ml_training::TrainingStatus::try_from(j.status) { + Ok(ml_training::TrainingStatus::Pending) => "pending", + Ok(ml_training::TrainingStatus::Running) => "running", + Ok(ml_training::TrainingStatus::Completed) => "completed", + Ok(ml_training::TrainingStatus::Failed) => "failed", + Ok(ml_training::TrainingStatus::Stopped) => "stopped", + Ok(ml_training::TrainingStatus::Paused) => "paused", + _ => "unknown", + }; + TrainingHistoryEntry { + job_id: j.job_id.clone(), + model: j.model_type.clone(), + status: status.to_owned(), + final_loss: j.final_loss, + best_val: j.best_validation_score, + created_at: j.created_at, + started_at: j.started_at, + completed_at: j.completed_at, + description: j.description.clone(), + } + }) + .collect(); + let _ = tx.send(StateUpdate::TrainingHistory(entries)); + } + Err(e) => { + debug!("ListTrainingJobs poll failed: {e}"); + } + } + } + }); + } + /// Service statuses — poll GetSystemStatus every 5s (the stream event /// only carries SystemMetrics, not individual ServiceStatus entries). /// After 3 consecutive failures, reports a connection error to indicate @@ -871,10 +936,7 @@ impl DataFetcher { let groups = group_pods(m.pods); let _ = tx.send(StateUpdate::Pods(groups)); } - Ok(None) => { - debug!("ClusterPods stream ended"); - break; - } + Ok(None) => break, Err(e) => { warn!("ClusterPods stream error: {e}"); break; @@ -885,7 +947,7 @@ impl DataFetcher { } } Err(e) => { - debug!("ClusterPods connect failed: {e}"); + warn!("ClusterPods connect failed: {e}"); let _ = tx.send(StateUpdate::Connection(ConnectionStatus::Reconnecting { attempt: backoff.attempts, })); @@ -952,19 +1014,68 @@ fn nanos_to_hms(nanos: i64) -> String { // Proto → state conversions // --------------------------------------------------------------------------- +/// Deduplicate training sessions by model name, keeping the entry with the +/// highest epoch. The API gateway can report multiple metric rows for the same +/// model (e.g. a stale epoch-0 hyperopt entry alongside the live session). +/// When two entries share a model name but have different folds, the one with +/// the higher epoch wins — fold is metadata, not a separate session. +fn dedup_sessions(raw: &[monitoring::TrainingSession]) -> Vec { + use std::collections::HashMap; + let mut best: HashMap<&str, &monitoring::TrainingSession> = HashMap::new(); + for s in raw { + let key = s.model.as_str(); + let entry = best.entry(key).or_insert(s); + if s.current_epoch > entry.current_epoch { + *entry = s; + } + } + let mut sessions: Vec = + best.into_values().map(convert_training_session).collect(); + sessions.sort_by(|a, b| a.model.cmp(&b.model)); + sessions +} + fn convert_training_session(s: &monitoring::TrainingSession) -> TrainingSessionData { TrainingSessionData { model: s.model.clone(), - fold: s.fold.parse().unwrap_or(0), + fold: s.fold.clone(), + is_hyperopt: s.is_hyperopt, epoch: s.current_epoch as u32, - max_epochs: 0, // not in proto — cockpit shows epoch only loss: f64::from(s.epoch_loss), val_loss: f64::from(s.validation_loss), batches_per_sec: f64::from(s.batches_per_second), + iteration_secs: f64::from(s.iteration_seconds), + epoch_duration_secs: f64::from(s.epoch_duration_seconds), sharpe: f64::from(s.epoch_sharpe), sortino: f64::from(s.epoch_sortino), win_rate: f64::from(s.epoch_win_rate), max_dd: f64::from(s.epoch_max_drawdown), + profit_factor: f64::from(s.epoch_profit_factor), + total_return: f64::from(s.epoch_total_return), + total_trades: s.epoch_total_trades, + action_buy_pct: f64::from(s.action_buy_pct), + action_sell_pct: f64::from(s.action_sell_pct), + action_hold_pct: f64::from(s.action_hold_pct), + q_value_mean: f64::from(s.q_value_mean), + q_value_max: f64::from(s.q_value_max), + policy_entropy: f64::from(s.policy_entropy), + kl_divergence: f64::from(s.kl_divergence), + advantage_mean: f64::from(s.advantage_mean), + replay_buffer_size: s.replay_buffer_size, + gradient_norm: f64::from(s.gradient_norm), + learning_rate: f64::from(s.learning_rate), + nan_detected: s.nan_detected, + gradient_explosions: s.gradient_explosions, + feature_errors: s.feature_errors, + checkpoint_saves: s.checkpoint_saves, + checkpoint_failures: s.checkpoint_failures, + hyperopt_trial_current: s.hyperopt_trial_current, + hyperopt_trial_total: s.hyperopt_trial_total, + hyperopt_best_objective: f64::from(s.hyperopt_best_objective), + hyperopt_trials_failed: s.hyperopt_trials_failed, + hyperopt_trial_epoch: s.hyperopt_trial_epoch, + hyperopt_elapsed_secs: f64::from(s.hyperopt_elapsed_seconds), + epoch_history: Vec::new(), } } @@ -1292,3 +1403,213 @@ fn aggregate_cache_stats(jobs: &[data_acquisition::DownloadJobSummary]) -> DataC newest_date: newest.unwrap_or("--").to_owned(), } } + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn make_pod(name: &str, service: &str, ready: bool) -> monitoring::PodInfo { + monitoring::PodInfo { + name: name.into(), + service: service.into(), + namespace: "foxhunt".into(), + status: if ready { "Running".into() } else { "Pending".into() }, + restarts: 0, + age: "1h".into(), + node: "node-1".into(), + ready, + version: "v1".into(), + } + } + + #[test] + fn group_pods_groups_by_service() { + let pods = vec![ + make_pod("api-1", "api-gateway", true), + make_pod("api-2", "api-gateway", true), + make_pod("trade-1", "trading-service", true), + make_pod("trade-2", "trading-service", false), + ]; + let groups = group_pods(pods); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].service, "api-gateway"); + assert_eq!(groups[0].total_count, 2); + assert_eq!(groups[0].ready_count, 2); + assert_eq!(groups[1].service, "trading-service"); + assert_eq!(groups[1].total_count, 2); + assert_eq!(groups[1].ready_count, 1); + } + + #[test] + fn group_pods_empty_service_becomes_other() { + let pods = vec![ + make_pod("orphan-1", "", true), + make_pod("api-1", "api-gateway", true), + ]; + let groups = group_pods(pods); + assert_eq!(groups.len(), 2); + // BTreeMap sorts alphabetically: "(other)" < "api-gateway" + assert_eq!(groups[0].service, "(other)"); + assert_eq!(groups[0].total_count, 1); + assert_eq!(groups[1].service, "api-gateway"); + } + + #[test] + fn group_pods_empty_input() { + let groups = group_pods(vec![]); + assert!(groups.is_empty()); + } + + #[test] + fn group_pods_ready_count_accuracy() { + let pods = vec![ + make_pod("a", "svc", true), + make_pod("b", "svc", false), + make_pod("c", "svc", true), + make_pod("d", "svc", false), + make_pod("e", "svc", true), + ]; + let groups = group_pods(pods); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].ready_count, 3); + assert_eq!(groups[0].total_count, 5); + } + + #[test] + fn group_pods_sorted_by_service_name() { + let pods = vec![ + make_pod("z", "zebra", true), + make_pod("a", "alpha", true), + make_pod("m", "middle", true), + ]; + let groups = group_pods(pods); + let names: Vec<_> = groups.iter().map(|g| g.service.as_str()).collect(); + assert_eq!(names, vec!["alpha", "middle", "zebra"]); + } + + fn make_training_session(model: &str, epoch: f32) -> monitoring::TrainingSession { + monitoring::TrainingSession { + model: model.into(), + fold: "fold-0".into(), + current_epoch: epoch, + epoch_loss: 0.5, + validation_loss: 0.6, + batches_per_second: 100.0, + iteration_seconds: 0.01, + epoch_duration_seconds: 30.0, + epoch_sharpe: 1.2, + epoch_sortino: 1.5, + epoch_win_rate: 0.55, + epoch_max_drawdown: 0.1, + epoch_profit_factor: 1.8, + epoch_total_return: 0.05, + epoch_total_trades: 100, + action_buy_pct: 0.33, + action_sell_pct: 0.33, + action_hold_pct: 0.34, + q_value_mean: 0.5, + q_value_max: 1.0, + policy_entropy: 0.8, + kl_divergence: 0.01, + advantage_mean: 0.1, + replay_buffer_size: 10000, + gradient_norm: 0.5, + learning_rate: 0.001, + checkpoint_saves: 1, + ..Default::default() + } + } + + #[test] + fn convert_training_session_maps_fields() { + let proto = make_training_session("DQN", 5.0); + let data = convert_training_session(&proto); + assert_eq!(data.model, "DQN"); + assert_eq!(data.epoch, 5); + assert!((data.loss - 0.5).abs() < 1e-6); + assert!((data.sharpe - 1.2).abs() < 1e-4); + assert_eq!(data.total_trades, 100); + assert!(data.epoch_history.is_empty()); + } + + #[test] + fn convert_training_session_f32_to_f64() { + let proto = make_training_session("PPO", 1.0); + let data = convert_training_session(&proto); + // f32→f64 should preserve value within f32 precision + assert!((data.val_loss - 0.6).abs() < 1e-4); + assert!((data.batches_per_sec - 100.0).abs() < 1e-4); + assert!((data.learning_rate - 0.001).abs() < 1e-6); + } + + #[test] + fn convert_gpu_snapshot() { + let snap = monitoring::GpuSnapshot { + utilization_percent: 85.0, + memory_used_mb: 4096.0, + memory_total_mb: 8192.0, + temperature_celsius: 72.0, + power_watts: 150.0, + }; + let data = convert_gpu(&snap); + assert!((data.utilization - 85.0).abs() < 1e-6); + assert!((data.memory_used_gb - 4.0).abs() < 1e-6); + assert!((data.memory_total_gb - 8.0).abs() < 1e-6); + assert_eq!(data.temperature_c, 72); + assert_eq!(data.power_watts, 150); + } + + fn make_download_job(status: i32, start: &str, end: &str) -> data_acquisition::DownloadJobSummary { + data_acquisition::DownloadJobSummary { + job_id: "j1".into(), + status, + dataset: "ohlcv-1m".into(), + symbols: vec!["ES.FUT".into()], + start_date: start.into(), + end_date: end.into(), + progress_percentage: 100.0, + actual_cost_usd: 0.0, + created_at: 0, + completed_at: 0, + } + } + + #[test] + fn aggregate_cache_stats_date_range() { + let completed = data_acquisition::DownloadStatus::Completed as i32; + let jobs = vec![ + make_download_job(completed, "2025-01-01", "2025-03-31"), + make_download_job(completed, "2024-07-01", "2024-09-30"), + make_download_job(completed, "2025-04-01", "2025-06-30"), + ]; + let stats = aggregate_cache_stats(&jobs); + assert_eq!(stats.oldest_date, "2024-07-01"); + assert_eq!(stats.newest_date, "2025-06-30"); + } + + #[test] + fn aggregate_cache_stats_empty_jobs() { + let stats = aggregate_cache_stats(&[]); + assert_eq!(stats.oldest_date, "--"); + assert_eq!(stats.newest_date, "--"); + assert_eq!(stats.total_records, 0); + } + + #[test] + fn aggregate_cache_stats_skips_non_completed() { + let pending = data_acquisition::DownloadStatus::Pending as i32; + let completed = data_acquisition::DownloadStatus::Completed as i32; + let jobs = vec![ + make_download_job(pending, "2020-01-01", "2020-12-31"), + make_download_job(completed, "2025-01-01", "2025-03-31"), + ]; + let stats = aggregate_cache_stats(&jobs); + // Pending job dates should be excluded + assert_eq!(stats.oldest_date, "2025-01-01"); + assert_eq!(stats.newest_date, "2025-03-31"); + } +} diff --git a/bin/fxt/src/tui/event_loop.rs b/bin/fxt/src/tui/event_loop.rs index 6b5eba858..06c51f94d 100644 --- a/bin/fxt/src/tui/event_loop.rs +++ b/bin/fxt/src/tui/event_loop.rs @@ -32,7 +32,7 @@ use super::cockpits::services::ServicesCockpit; use super::cockpits::trading::TradingCockpit; use super::cockpits::training::TrainingCockpit; use super::data_fetcher::{DataFetcher, StateUpdate}; -use super::state::{AppState, ConnectionStatus}; +use super::state::{AppState, ConnectionStatus, EpochSnapshot, TrainingTab}; use super::theme; /// Number of cockpit views. @@ -231,6 +231,41 @@ async fn event_loop( state.last_update = std::time::Instant::now(); } + // Navigation: j/k/Down/Up for item selection + KeyCode::Char('j') | KeyCode::Down => { + handle_nav_down(&mut state); + } + KeyCode::Char('k') | KeyCode::Up => { + handle_nav_up(&mut state); + } + + // Enter / Right: toggle detail expansion + KeyCode::Enter | KeyCode::Right => { + handle_enter(&mut state); + } + + // Tab / Shift-Tab: switch sub-tabs in Training cockpit + KeyCode::Tab | KeyCode::BackTab => { + if state.current_cockpit == COCKPIT_TRAINING { + state.training_tab = match state.training_tab { + TrainingTab::Live => TrainingTab::History, + TrainingTab::History => TrainingTab::Live, + }; + } + } + + // l/h: alternative sub-tab switching + KeyCode::Char('l') => { + if state.current_cockpit == COCKPIT_TRAINING { + state.training_tab = TrainingTab::History; + } + } + KeyCode::Char('h') => { + if state.current_cockpit == COCKPIT_TRAINING { + state.training_tab = TrainingTab::Live; + } + } + _ => {} } } @@ -250,11 +285,46 @@ const RING_CAP: usize = 50; fn apply_update(state: &mut AppState, update: StateUpdate) { match update { StateUpdate::TrainingMetrics { - sessions, + mut sessions, gpu, resources, active_jobs, } => { + // Drain old sessions into a map so we can move epoch histories. + let mut old_map: std::collections::HashMap = state + .training_sessions + .drain(..) + .map(|s| (s.model.clone(), s)) + .collect(); + + for new_s in &mut sessions { + if let Some(mut old_s) = old_map.remove(&new_s.model) { + // If epoch advanced, snapshot the old epoch metrics. + if new_s.epoch > old_s.epoch { + old_s.epoch_history.push(EpochSnapshot { + epoch: old_s.epoch, + loss: old_s.loss, + val_loss: old_s.val_loss, + sharpe: old_s.sharpe, + sortino: old_s.sortino, + win_rate: old_s.win_rate, + max_dd: old_s.max_dd, + profit_factor: old_s.profit_factor, + total_return: old_s.total_return, + total_trades: old_s.total_trades, + q_value_mean: old_s.q_value_mean, + gradient_norm: old_s.gradient_norm, + learning_rate: old_s.learning_rate, + policy_entropy: old_s.policy_entropy, + action_buy_pct: old_s.action_buy_pct, + action_sell_pct: old_s.action_sell_pct, + action_hold_pct: old_s.action_hold_pct, + }); + } + // Move accumulated history to the new session. + new_s.epoch_history = old_s.epoch_history; + } + } state.training_sessions = sessions; state.gpu = gpu; state.system_resources = resources; @@ -327,12 +397,159 @@ fn apply_update(state: &mut AppState, update: StateUpdate) { StateUpdate::Pods(pods) => { state.pods = pods; } + StateUpdate::TrainingHistory(entries) => { + state.training_history = entries; + } StateUpdate::Connection(status) => { state.connection_status = status; } } } +// --------------------------------------------------------------------------- +// Cockpit-aware navigation +// --------------------------------------------------------------------------- + +/// Training cockpit index. +const COCKPIT_TRAINING: usize = 1; +/// Pods cockpit index. +const COCKPIT_PODS: usize = 6; + +fn handle_nav_down(state: &mut AppState) { + match state.current_cockpit { + COCKPIT_TRAINING => match state.training_tab { + TrainingTab::Live => { + let count = state.training_sessions.len(); + if count == 0 { + return; + } + state.selected_training_session = Some(match state.selected_training_session { + Some(i) if i + 1 < count => i + 1, + Some(_) => 0, // wrap + None => 0, + }); + } + TrainingTab::History => { + let count = state.training_history.len(); + if count == 0 { + return; + } + state.selected_history_entry = Some(match state.selected_history_entry { + Some(i) if i + 1 < count => i + 1, + Some(_) => 0, + None => 0, + }); + } + }, + COCKPIT_PODS => { + let count = state.pods.len(); + if count == 0 { + return; + } + state.selected_pod_group = Some(match state.selected_pod_group { + Some(i) if i + 1 < count => i + 1, + Some(_) => 0, + None => 0, + }); + scroll_pods_into_view(state); + } + _ => {} + } +} + +fn handle_nav_up(state: &mut AppState) { + match state.current_cockpit { + COCKPIT_TRAINING => match state.training_tab { + TrainingTab::Live => { + let count = state.training_sessions.len(); + if count == 0 { + return; + } + state.selected_training_session = Some(match state.selected_training_session { + Some(0) => count.saturating_sub(1), // wrap + Some(i) => i.saturating_sub(1), + None => 0, + }); + } + TrainingTab::History => { + let count = state.training_history.len(); + if count == 0 { + return; + } + state.selected_history_entry = Some(match state.selected_history_entry { + Some(0) => count.saturating_sub(1), + Some(i) => i.saturating_sub(1), + None => 0, + }); + } + }, + COCKPIT_PODS => { + let count = state.pods.len(); + if count == 0 { + return; + } + state.selected_pod_group = Some(match state.selected_pod_group { + Some(0) => count.saturating_sub(1), + Some(i) => i.saturating_sub(1), + None => 0, + }); + scroll_pods_into_view(state); + } + _ => {} + } +} + +/// Compute the flat row index where a pod group header starts. +/// Each group contributes 1 header + (N sub-rows if N > 1). +fn pod_group_flat_row(pods: &[crate::tui::state::PodGroupData], group_idx: usize) -> usize { + let mut row = 0; + for (i, g) in pods.iter().enumerate() { + if i == group_idx { + return row; + } + row += 1; // group header + if g.pods.len() > 1 { + row += g.pods.len(); // sub-rows + } + } + row +} + +/// Ensure the selected pod group is visible by adjusting scroll offset. +/// Uses a conservative 35-row visible window (typical terminal height minus chrome). +fn scroll_pods_into_view(state: &mut AppState) { + const VISIBLE_ROWS: usize = 35; + if let Some(idx) = state.selected_pod_group { + let flat = pod_group_flat_row(&state.pods, idx); + if flat < state.pod_scroll_offset { + state.pod_scroll_offset = flat; + } else if flat >= state.pod_scroll_offset + VISIBLE_ROWS { + state.pod_scroll_offset = flat.saturating_sub(VISIBLE_ROWS) + 1; + } else { + // Already visible — no scroll adjustment needed. + } + } +} + +fn handle_enter(state: &mut AppState) { + match state.current_cockpit { + COCKPIT_TRAINING => { + if state.training_tab == TrainingTab::Live + && state.selected_training_session.is_some() + { + state.training_detail_expanded = !state.training_detail_expanded; + } + // History tab: Enter currently just keeps selection visible in detail panel + } + COCKPIT_PODS => { + if state.selected_pod_group.is_some() { + state.pod_group_expanded = !state.pod_group_expanded; + } + } + _ => {} + } +} + // --------------------------------------------------------------------------- // Chrome renderers // --------------------------------------------------------------------------- @@ -393,7 +610,7 @@ fn render_status_bar(frame: &mut ratatui::Frame, area: Rect, state: &AppState) { } }; - let line = Line::from(vec![ + let mut spans = vec![ Span::styled( format!(" [{conn_label}] "), Style::default().fg(conn_color).add_modifier(Modifier::BOLD), @@ -402,15 +619,31 @@ fn render_status_bar(frame: &mut ratatui::Frame, area: Rect, state: &AppState) { Span::styled(" quit ", theme::muted_style()), Span::styled("?", Style::default().fg(theme::SECONDARY)), Span::styled(" help ", theme::muted_style()), - Span::styled("r", Style::default().fg(theme::SECONDARY)), - Span::styled(" refresh ", theme::muted_style()), + Span::styled("j/k", Style::default().fg(theme::SECONDARY)), + Span::styled(" nav ", theme::muted_style()), + Span::styled("\u{23ce}", Style::default().fg(theme::SECONDARY)), + Span::styled(" detail ", theme::muted_style()), Span::styled("1-7", Style::default().fg(theme::SECONDARY)), - Span::styled(" switch cockpit ", theme::muted_style()), + Span::styled(" cockpit ", theme::muted_style()), Span::styled( format!("Updated: {ago}"), theme::muted_style(), ), - ]); + ]; + + // Show CPU/GPU summary inline on Training cockpit + if state.current_cockpit == COCKPIT_TRAINING && state.gpu.utilization > 0.0 { + spans.push(Span::styled(" ", theme::muted_style())); + spans.push(Span::styled( + format!( + "GPU {:.0}% CPU {:.0}%", + state.gpu.utilization, state.system_resources.cpu_usage_pct + ), + Style::default().fg(theme::SECONDARY), + )); + } + + let line = Line::from(spans); let paragraph = Paragraph::new(line); frame.render_widget(paragraph, area); @@ -419,7 +652,7 @@ fn render_status_bar(frame: &mut ratatui::Frame, area: Rect, state: &AppState) { fn render_help_overlay(frame: &mut ratatui::Frame, area: Rect) { // Center a help box in the terminal let width = 50_u16.min(area.width.saturating_sub(4)); - let height = 14_u16.min(area.height.saturating_sub(4)); + let height = 16_u16.min(area.height.saturating_sub(4)); #[allow(clippy::integer_division)] let x = area.x + (area.width.saturating_sub(width)) / 2; #[allow(clippy::integer_division)] @@ -432,28 +665,36 @@ fn render_help_overlay(frame: &mut ratatui::Frame, area: Rect) { let help_text = vec![ Line::from(""), Line::from(vec![ - Span::styled(" 1-7 ", Style::default().fg(theme::SECONDARY)), + Span::styled(" 1-7 ", Style::default().fg(theme::SECONDARY)), Span::styled("Switch cockpit view", Style::default().fg(theme::TEXT)), ]), Line::from(vec![ - Span::styled(" q ", Style::default().fg(theme::SECONDARY)), - Span::styled("Quit", Style::default().fg(theme::TEXT)), + Span::styled(" j/k ", Style::default().fg(theme::SECONDARY)), + Span::styled("Navigate items up/down", Style::default().fg(theme::TEXT)), ]), Line::from(vec![ - Span::styled(" Esc ", Style::default().fg(theme::SECONDARY)), - Span::styled("Quit", Style::default().fg(theme::TEXT)), + Span::styled(" Enter/\u{2192}", Style::default().fg(theme::SECONDARY)), + Span::styled(" Expand/collapse detail", Style::default().fg(theme::TEXT)), ]), Line::from(vec![ - Span::styled(" ? ", Style::default().fg(theme::SECONDARY)), + Span::styled(" Tab ", Style::default().fg(theme::SECONDARY)), + Span::styled("Switch Live/History (Training)", Style::default().fg(theme::TEXT)), + ]), + Line::from(vec![ + Span::styled(" l/h ", Style::default().fg(theme::SECONDARY)), + Span::styled("History/Live tab (Training)", Style::default().fg(theme::TEXT)), + ]), + Line::from(vec![ + Span::styled(" ? ", Style::default().fg(theme::SECONDARY)), Span::styled("Toggle this help", Style::default().fg(theme::TEXT)), ]), Line::from(vec![ - Span::styled(" r ", Style::default().fg(theme::SECONDARY)), + Span::styled(" r ", Style::default().fg(theme::SECONDARY)), Span::styled("Force refresh", Style::default().fg(theme::TEXT)), ]), Line::from(vec![ - Span::styled(" C-c ", Style::default().fg(theme::SECONDARY)), - Span::styled("Quit (Ctrl+C)", Style::default().fg(theme::TEXT)), + Span::styled(" q/Esc ", Style::default().fg(theme::SECONDARY)), + Span::styled("Quit", Style::default().fg(theme::TEXT)), ]), Line::from(""), Line::from(vec![Span::styled( @@ -475,3 +716,993 @@ fn render_help_overlay(frame: &mut ratatui::Frame, area: Rect) { let help = Paragraph::new(help_text).block(help_block).wrap(Wrap { trim: false }); frame.render_widget(help, help_area); } + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use crate::tui::state::{ + AccountData, AlertData, CircuitBreakerData, DataCacheData, DataFeedData, + ExecutionData, GpuData, ModelStatusData, OrderEventData, PodGroupData, + PositionData, RiskAlertData, RiskData, ServiceData, SystemResources, + TrainingHistoryEntry, TrainingSessionData, + }; + + fn make_session(model: &str, epoch: u32, loss: f64, sharpe: f64) -> TrainingSessionData { + TrainingSessionData { + model: model.into(), + fold: "fold-0".into(), + is_hyperopt: false, + epoch, + loss, + val_loss: loss + 0.1, + batches_per_sec: 100.0, + iteration_secs: 0.01, + epoch_duration_secs: 30.0, + sharpe, + sortino: 1.5, + win_rate: 0.55, + max_dd: 0.1, + profit_factor: 1.8, + total_return: 0.05, + total_trades: 100, + action_buy_pct: 0.33, + action_sell_pct: 0.33, + action_hold_pct: 0.34, + q_value_mean: 0.5, + q_value_max: 1.0, + policy_entropy: 0.8, + kl_divergence: 0.01, + advantage_mean: 0.1, + replay_buffer_size: 10000, + gradient_norm: 0.5, + learning_rate: 0.001, + nan_detected: 0, + gradient_explosions: 0, + feature_errors: 0, + checkpoint_saves: 1, + checkpoint_failures: 0, + hyperopt_trial_current: 0, + hyperopt_trial_total: 0, + hyperopt_best_objective: 0.0, + hyperopt_trials_failed: 0, + hyperopt_trial_epoch: 0, + hyperopt_elapsed_secs: 0.0, + epoch_history: Vec::new(), + } + } + + fn make_training_update( + sessions: Vec, + ) -> StateUpdate { + StateUpdate::TrainingMetrics { + sessions, + gpu: GpuData::default(), + resources: SystemResources::default(), + active_jobs: 1, + } + } + + // -- Epoch history accumulation -- + + #[test] + fn epoch_history_accumulated_on_epoch_advance() { + let mut state = AppState::default(); + + // Epoch 1 arrives. + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 1, 0.5, 1.0), + ])); + assert_eq!(state.training_sessions.len(), 1); + assert!(state.training_sessions[0].epoch_history.is_empty()); + + // Epoch 2 arrives — epoch 1 should be snapshotted. + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 2, 0.4, 1.2), + ])); + assert_eq!(state.training_sessions[0].epoch_history.len(), 1); + let snap = &state.training_sessions[0].epoch_history[0]; + assert_eq!(snap.epoch, 1); + assert!((snap.loss - 0.5).abs() < 1e-6); + assert!((snap.sharpe - 1.0).abs() < 1e-6); + + // Epoch 3 — epoch 2 appended. + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 3, 0.3, 1.5), + ])); + assert_eq!(state.training_sessions[0].epoch_history.len(), 2); + assert_eq!(state.training_sessions[0].epoch_history[1].epoch, 2); + } + + #[test] + fn epoch_history_not_duplicated_on_same_epoch() { + let mut state = AppState::default(); + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 5, 0.5, 1.0), + ])); + // Same epoch arrives again (metrics refresh, no epoch advance). + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 5, 0.45, 1.1), + ])); + assert!(state.training_sessions[0].epoch_history.is_empty()); + } + + #[test] + fn epoch_history_tracks_multiple_models_independently() { + let mut state = AppState::default(); + + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 1, 0.5, 1.0), + make_session("PPO", 3, 0.3, 2.0), + ])); + + // DQN advances, PPO stays. + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 2, 0.4, 1.2), + make_session("PPO", 3, 0.29, 2.1), + ])); + + let dqn = state.training_sessions.iter().find(|s| s.model == "DQN").unwrap(); + let ppo = state.training_sessions.iter().find(|s| s.model == "PPO").unwrap(); + assert_eq!(dqn.epoch_history.len(), 1); + assert!(ppo.epoch_history.is_empty()); + } + + #[test] + fn epoch_history_preserved_when_model_disappears_then_returns() { + let mut state = AppState::default(); + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 1, 0.5, 1.0), + ])); + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 2, 0.4, 1.2), + ])); + assert_eq!(state.training_sessions[0].epoch_history.len(), 1); + + // Model disappears from update. + apply_update(&mut state, make_training_update(vec![])); + assert!(state.training_sessions.is_empty()); + + // Model returns — history is lost (fresh session). + apply_update(&mut state, make_training_update(vec![ + make_session("DQN", 3, 0.3, 1.5), + ])); + assert!(state.training_sessions[0].epoch_history.is_empty()); + } + + // -- Training tab switching -- + + #[test] + fn tab_toggles_training_sub_tab() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + assert_eq!(state.training_tab, TrainingTab::Live); + + // Simulate Tab key. + state.training_tab = match state.training_tab { + TrainingTab::Live => TrainingTab::History, + TrainingTab::History => TrainingTab::Live, + }; + assert_eq!(state.training_tab, TrainingTab::History); + + state.training_tab = match state.training_tab { + TrainingTab::Live => TrainingTab::History, + TrainingTab::History => TrainingTab::Live, + }; + assert_eq!(state.training_tab, TrainingTab::Live); + } + + // -- Navigation: j/k wrapping -- + + #[test] + fn nav_down_wraps_live_sessions() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::Live; + state.training_sessions = vec![ + make_session("A", 1, 0.5, 1.0), + make_session("B", 1, 0.4, 1.2), + ]; + + handle_nav_down(&mut state); + assert_eq!(state.selected_training_session, Some(0)); + handle_nav_down(&mut state); + assert_eq!(state.selected_training_session, Some(1)); + // Wraps back to 0. + handle_nav_down(&mut state); + assert_eq!(state.selected_training_session, Some(0)); + } + + #[test] + fn nav_up_wraps_live_sessions() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::Live; + state.training_sessions = vec![ + make_session("A", 1, 0.5, 1.0), + make_session("B", 1, 0.4, 1.2), + make_session("C", 1, 0.3, 1.5), + ]; + + handle_nav_up(&mut state); + assert_eq!(state.selected_training_session, Some(0)); + // Wraps to last. + handle_nav_up(&mut state); + assert_eq!(state.selected_training_session, Some(2)); + handle_nav_up(&mut state); + assert_eq!(state.selected_training_session, Some(1)); + } + + #[test] + fn nav_down_empty_sessions_is_noop() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::Live; + handle_nav_down(&mut state); + assert_eq!(state.selected_training_session, None); + } + + #[test] + fn nav_down_history_tab() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::History; + state.training_history = vec![ + TrainingHistoryEntry { + job_id: "j1".into(), model: "DQN".into(), status: "Completed".into(), + final_loss: 0.3, best_val: 0.4, created_at: 0, started_at: 0, + completed_at: 0, description: String::new(), + }, + TrainingHistoryEntry { + job_id: "j2".into(), model: "PPO".into(), status: "Failed".into(), + final_loss: 0.0, best_val: 0.0, created_at: 0, started_at: 0, + completed_at: 0, description: String::new(), + }, + ]; + + handle_nav_down(&mut state); + assert_eq!(state.selected_history_entry, Some(0)); + handle_nav_down(&mut state); + assert_eq!(state.selected_history_entry, Some(1)); + handle_nav_down(&mut state); + assert_eq!(state.selected_history_entry, Some(0)); // wrap + } + + // -- Pod navigation -- + + #[test] + fn nav_down_pods() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_PODS; + state.pods = vec![ + PodGroupData { service: "a".into(), pods: vec![], ready_count: 0, total_count: 0 }, + PodGroupData { service: "b".into(), pods: vec![], ready_count: 0, total_count: 0 }, + ]; + + handle_nav_down(&mut state); + assert_eq!(state.selected_pod_group, Some(0)); + handle_nav_down(&mut state); + assert_eq!(state.selected_pod_group, Some(1)); + handle_nav_down(&mut state); + assert_eq!(state.selected_pod_group, Some(0)); // wrap + } + + #[test] + fn enter_toggles_detail_expansion() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::Live; + state.selected_training_session = Some(0); + state.training_sessions = vec![make_session("DQN", 1, 0.5, 1.0)]; + + assert!(!state.training_detail_expanded); + handle_enter(&mut state); + assert!(state.training_detail_expanded); + handle_enter(&mut state); + assert!(!state.training_detail_expanded); + } + + #[test] + fn enter_no_selection_does_not_expand() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::Live; + state.selected_training_session = None; + handle_enter(&mut state); + assert!(!state.training_detail_expanded); + } + + #[test] + fn pod_enter_toggles_expansion() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_PODS; + state.selected_pod_group = Some(0); + state.pods = vec![ + PodGroupData { service: "a".into(), pods: vec![], ready_count: 0, total_count: 0 }, + ]; + + assert!(!state.pod_group_expanded); + handle_enter(&mut state); + assert!(state.pod_group_expanded); + handle_enter(&mut state); + assert!(!state.pod_group_expanded); + } + + // -- State updates: pods, training history, connection -- + + #[test] + fn apply_pods_update() { + let mut state = AppState::default(); + let pods = vec![ + PodGroupData { service: "svc".into(), pods: vec![], ready_count: 1, total_count: 1 }, + ]; + apply_update(&mut state, StateUpdate::Pods(pods)); + assert_eq!(state.pods.len(), 1); + assert_eq!(state.pods[0].service, "svc"); + } + + #[test] + fn apply_training_history_update() { + let mut state = AppState::default(); + let entries = vec![ + TrainingHistoryEntry { + job_id: "j1".into(), model: "DQN".into(), status: "Completed".into(), + final_loss: 0.3, best_val: 0.4, created_at: 0, started_at: 0, + completed_at: 100, description: String::new(), + }, + ]; + apply_update(&mut state, StateUpdate::TrainingHistory(entries)); + assert_eq!(state.training_history.len(), 1); + assert_eq!(state.training_history[0].job_id, "j1"); + } + + #[test] + fn apply_connection_update() { + let mut state = AppState::default(); + apply_update(&mut state, StateUpdate::Connection(ConnectionStatus::Connected)); + assert!(matches!(state.connection_status, ConnectionStatus::Connected)); + + apply_update(&mut state, StateUpdate::Connection( + ConnectionStatus::Reconnecting { attempt: 3 }, + )); + assert!(matches!(state.connection_status, ConnectionStatus::Reconnecting { attempt: 3 })); + } + + // -- StateUpdate: Services -- + + #[test] + fn apply_services_update() { + let mut state = AppState::default(); + let services = vec![ + ServiceData { + name: "api-gateway".into(), + healthy: true, + uptime: "5d".into(), + version: "v1.0".into(), + latency_ms: 12.0, + error: None, + }, + ServiceData { + name: "trading-service".into(), + healthy: false, + uptime: "0s".into(), + version: "v1.1".into(), + latency_ms: 0.0, + error: Some("timeout".into()), + }, + ]; + apply_update(&mut state, StateUpdate::Services(services)); + assert_eq!(state.services.len(), 2); + assert!(state.services[0].healthy); + assert!(!state.services[1].healthy); + assert_eq!(state.services[1].error.as_deref(), Some("timeout")); + } + + // -- StateUpdate: Account -- + + #[test] + fn apply_account_update() { + let mut state = AppState::default(); + let acct = AccountData { + equity: 100_000.0, + cash: 50_000.0, + margin_used: 25_000.0, + daily_pnl: 1_234.56, + total_pnl: 9_876.54, + }; + apply_update(&mut state, StateUpdate::Account(acct)); + assert!((state.account.equity - 100_000.0).abs() < 1e-6); + assert!((state.account.daily_pnl - 1_234.56).abs() < 1e-6); + } + + // -- StateUpdate: Positions -- + + #[test] + fn apply_positions_update() { + let mut state = AppState::default(); + let positions = vec![ + PositionData { + symbol: "ES".into(), + qty: 10, + entry_price: 5000.0, + market_price: 5050.0, + unrealized_pnl: 500.0, + }, + ]; + apply_update(&mut state, StateUpdate::Positions(positions)); + assert_eq!(state.positions.len(), 1); + assert_eq!(state.positions[0].symbol, "ES"); + assert_eq!(state.positions[0].qty, 10); + } + + #[test] + fn apply_positions_replaces_previous() { + let mut state = AppState::default(); + let p1 = vec![PositionData { + symbol: "ES".into(), qty: 5, entry_price: 5000.0, + market_price: 5050.0, unrealized_pnl: 250.0, + }]; + let p2 = vec![ + PositionData { symbol: "NQ".into(), qty: 3, entry_price: 20000.0, + market_price: 20100.0, unrealized_pnl: 300.0 }, + PositionData { symbol: "6E".into(), qty: 1, entry_price: 1.1, + market_price: 1.11, unrealized_pnl: 10.0 }, + ]; + apply_update(&mut state, StateUpdate::Positions(p1)); + assert_eq!(state.positions.len(), 1); + apply_update(&mut state, StateUpdate::Positions(p2)); + assert_eq!(state.positions.len(), 2); + assert_eq!(state.positions[0].symbol, "NQ"); + } + + // -- StateUpdate: Execution (ring buffer) -- + + #[test] + fn apply_execution_ring_buffer() { + let mut state = AppState::default(); + // Fill exactly to capacity. + for i in 0..RING_CAP { + apply_update(&mut state, StateUpdate::Execution(ExecutionData { + time: format!("t{i}"), symbol: "ES".into(), side: "BUY".into(), + qty: 1, price: 5000.0, status: "Filled".into(), + })); + } + assert_eq!(state.executions.len(), RING_CAP); + assert_eq!(state.executions.front().unwrap().time, "t0"); + + // One more should evict the oldest. + apply_update(&mut state, StateUpdate::Execution(ExecutionData { + time: "overflow".into(), symbol: "ES".into(), side: "SELL".into(), + qty: 1, price: 5100.0, status: "Filled".into(), + })); + assert_eq!(state.executions.len(), RING_CAP); + assert_eq!(state.executions.front().unwrap().time, "t1"); + assert_eq!(state.executions.back().unwrap().time, "overflow"); + } + + // -- StateUpdate: Order (ring buffer) -- + + #[test] + fn apply_order_ring_buffer() { + let mut state = AppState::default(); + for i in 0..RING_CAP { + apply_update(&mut state, StateUpdate::Order(OrderEventData { + time: format!("t{i}"), order_id: format!("o{i}"), + symbol: "NQ".into(), side: "BUY".into(), + status: "New".into(), qty: 1.0, filled_qty: 0.0, + })); + } + assert_eq!(state.orders.len(), RING_CAP); + + // Overflow evicts oldest. + apply_update(&mut state, StateUpdate::Order(OrderEventData { + time: "late".into(), order_id: "oX".into(), + symbol: "NQ".into(), side: "SELL".into(), + status: "Filled".into(), qty: 2.0, filled_qty: 2.0, + })); + assert_eq!(state.orders.len(), RING_CAP); + assert_eq!(state.orders.front().unwrap().order_id, "o1"); + assert_eq!(state.orders.back().unwrap().order_id, "oX"); + } + + // -- StateUpdate: Risk (preserves circuit breakers) -- + + #[test] + fn apply_risk_preserves_circuit_breakers() { + let mut state = AppState::default(); + // Pre-load circuit breakers from the separate stream. + state.risk.circuit_breakers = vec![ + CircuitBreakerData { + name: "max-drawdown".into(), + threshold: Some(0.05), + current: Some(0.02), + tripped: false, + }, + ]; + + // Risk update arrives — should NOT overwrite circuit breakers. + let risk = RiskData { + global_kill_switch: Some(false), + portfolio_kill_switch: None, + strategy_kill_switch: None, + instrument_kill_switch: None, + current_drawdown_pct: 0.02, + max_drawdown_pct: 0.05, + high_water_mark: 100_000.0, + circuit_breakers: Vec::new(), // empty from this stream + position_limits: Vec::new(), + }; + apply_update(&mut state, StateUpdate::Risk(risk)); + assert_eq!(state.risk.circuit_breakers.len(), 1); + assert_eq!(state.risk.circuit_breakers[0].name, "max-drawdown"); + assert!((state.risk.current_drawdown_pct - 0.02).abs() < 1e-6); + } + + // -- StateUpdate: CircuitBreakers -- + + #[test] + fn apply_circuit_breakers_update() { + let mut state = AppState::default(); + let cbs = vec![ + CircuitBreakerData { + name: "daily-loss".into(), + threshold: Some(0.03), + current: Some(0.01), + tripped: false, + }, + CircuitBreakerData { + name: "drawdown".into(), + threshold: Some(0.10), + current: Some(0.11), + tripped: true, + }, + ]; + apply_update(&mut state, StateUpdate::CircuitBreakers(cbs)); + assert_eq!(state.risk.circuit_breakers.len(), 2); + assert!(state.risk.circuit_breakers[1].tripped); + } + + // -- StateUpdate: RiskAlert (ring buffer) -- + + #[test] + fn apply_risk_alert_ring_buffer() { + let mut state = AppState::default(); + for i in 0..RING_CAP { + apply_update(&mut state, StateUpdate::RiskAlert(RiskAlertData { + time: format!("t{i}"), severity: "WARN".into(), + alert_type: "drawdown".into(), message: format!("alert {i}"), + })); + } + assert_eq!(state.risk_alerts.len(), RING_CAP); + + apply_update(&mut state, StateUpdate::RiskAlert(RiskAlertData { + time: "overflow".into(), severity: "CRITICAL".into(), + alert_type: "kill-switch".into(), message: "tripped".into(), + })); + assert_eq!(state.risk_alerts.len(), RING_CAP); + assert_eq!(state.risk_alerts.front().unwrap().time, "t1"); + assert_eq!(state.risk_alerts.back().unwrap().severity, "CRITICAL"); + } + + // -- StateUpdate: ModelStatuses -- + + #[test] + fn apply_model_statuses_update() { + let mut state = AppState::default(); + let statuses = vec![ + ModelStatusData { + model_name: "DQN".into(), state: "ACTIVE".into(), + health: "OK".into(), last_prediction: "1s ago".into(), + }, + ModelStatusData { + model_name: "PPO".into(), state: "LOADING".into(), + health: "DEGRADED".into(), last_prediction: "--".into(), + }, + ]; + apply_update(&mut state, StateUpdate::ModelStatuses(statuses)); + assert_eq!(state.model_statuses.len(), 2); + assert_eq!(state.model_statuses[0].state, "ACTIVE"); + assert_eq!(state.model_statuses[1].health, "DEGRADED"); + } + + // -- StateUpdate: Alert (ring buffer) -- + + #[test] + fn apply_alert_ring_buffer() { + let mut state = AppState::default(); + for i in 0..RING_CAP + 5 { + apply_update(&mut state, StateUpdate::Alert(AlertData { + time: format!("t{i}"), severity: "WARN".into(), + service: "api".into(), title: format!("alert {i}"), + })); + } + // Should cap at RING_CAP after overflow. + assert_eq!(state.alerts.len(), RING_CAP); + assert_eq!(state.alerts.front().unwrap().time, "t5"); + assert_eq!(state.alerts.back().unwrap().title, "alert 54"); + } + + // -- StateUpdate: DataFeeds -- + + #[test] + fn apply_data_feeds_update() { + let mut state = AppState::default(); + let feeds = vec![ + DataFeedData { + symbol: "ES".into(), records_per_sec: 500, + latency_us: 120, status: "Active".into(), + }, + ]; + apply_update(&mut state, StateUpdate::DataFeeds(feeds)); + assert_eq!(state.data_feeds.len(), 1); + assert_eq!(state.data_feeds[0].records_per_sec, 500); + } + + // -- StateUpdate: DataCache -- + + #[test] + fn apply_data_cache_update() { + let mut state = AppState::default(); + let cache = DataCacheData { + total_records: 4_000_000, + cache_hit_rate: 0.95, + disk_usage_gb: 12.5, + oldest_date: "2024-01-01".into(), + newest_date: "2025-12-31".into(), + }; + apply_update(&mut state, StateUpdate::DataCache(cache)); + assert_eq!(state.data_cache.total_records, 4_000_000); + assert!((state.data_cache.cache_hit_rate - 0.95).abs() < 1e-6); + } + + // -- StateUpdate: Portfolio -- + + #[test] + fn apply_portfolio_update() { + let mut state = AppState::default(); + apply_update(&mut state, StateUpdate::Portfolio { + equity: 150_000.0, + cash: 60_000.0, + margin_used: 40_000.0, + daily_pnl: 2_500.0, + total_pnl: 15_000.0, + }); + assert!((state.account.equity - 150_000.0).abs() < 1e-6); + assert!((state.account.cash - 60_000.0).abs() < 1e-6); + assert!((state.account.margin_used - 40_000.0).abs() < 1e-6); + assert!((state.account.daily_pnl - 2_500.0).abs() < 1e-6); + assert!((state.account.total_pnl - 15_000.0).abs() < 1e-6); + } + + #[test] + fn apply_portfolio_overwrites_account_fields() { + let mut state = AppState::default(); + // Set account via Account update first. + apply_update(&mut state, StateUpdate::Account(AccountData { + equity: 100_000.0, cash: 50_000.0, margin_used: 20_000.0, + daily_pnl: 1_000.0, total_pnl: 5_000.0, + })); + // Portfolio update overwrites the same fields. + apply_update(&mut state, StateUpdate::Portfolio { + equity: 200_000.0, cash: 80_000.0, margin_used: 30_000.0, + daily_pnl: 3_000.0, total_pnl: 10_000.0, + }); + assert!((state.account.equity - 200_000.0).abs() < 1e-6); + assert!((state.account.total_pnl - 10_000.0).abs() < 1e-6); + } + + // -- StateUpdate: TrainingMetrics GPU & resources -- + + #[test] + fn apply_training_metrics_updates_gpu_and_resources() { + let mut state = AppState::default(); + apply_update(&mut state, StateUpdate::TrainingMetrics { + sessions: vec![], + gpu: GpuData { + name: "RTX 3050 Ti".into(), + utilization: 85.0, + memory_used_gb: 3.5, + memory_total_gb: 4.0, + temperature_c: 72, + power_watts: 150, + power_limit_watts: 200, + }, + resources: SystemResources { + cpu_usage_pct: 45.0, + ram_used_gb: 12.0, + ram_total_gb: 32.0, + gpu_cluster_utilization: 80.0, + }, + active_jobs: 3, + }); + assert_eq!(state.gpu.name, "RTX 3050 Ti"); + assert!((state.gpu.utilization - 85.0).abs() < 1e-6); + assert!((state.system_resources.cpu_usage_pct - 45.0).abs() < 1e-6); + assert_eq!(state.active_jobs, 3); + } + + // -- Pod scroll helpers -- + + fn make_pod_group(service: &str, pod_count: usize) -> PodGroupData { + use crate::tui::state::PodData; + let pods: Vec = (0..pod_count) + .map(|i| PodData { + name: format!("{service}-{i}"), service: service.into(), + status: "Running".into(), restarts: 0, age: "1h".into(), + node: "node-1".into(), ready: true, version: "v1".into(), + }) + .collect(); + PodGroupData { + service: service.into(), + pods, + ready_count: pod_count, + total_count: pod_count, + } + } + + #[test] + fn pod_group_flat_row_single_pod_groups() { + // Each group with 1 pod contributes exactly 1 row (header only, no sub-rows). + let pods = vec![ + make_pod_group("a", 1), + make_pod_group("b", 1), + make_pod_group("c", 1), + ]; + assert_eq!(pod_group_flat_row(&pods, 0), 0); + assert_eq!(pod_group_flat_row(&pods, 1), 1); + assert_eq!(pod_group_flat_row(&pods, 2), 2); + } + + #[test] + fn pod_group_flat_row_multi_pod_groups() { + // Group with >1 pod: 1 header + N sub-rows. + let pods = vec![ + make_pod_group("a", 3), // header + 3 subs = 4 rows + make_pod_group("b", 1), // header only = 1 row + make_pod_group("c", 2), // header + 2 subs = 3 rows + ]; + assert_eq!(pod_group_flat_row(&pods, 0), 0); + assert_eq!(pod_group_flat_row(&pods, 1), 4); // 0 + 1 header + 3 subs + assert_eq!(pod_group_flat_row(&pods, 2), 5); // 4 + 1 header + } + + #[test] + fn pod_group_flat_row_out_of_bounds_returns_total() { + let pods = vec![make_pod_group("a", 2)]; // 1 header + 2 subs = 3 + assert_eq!(pod_group_flat_row(&pods, 5), 3); + } + + #[test] + fn scroll_pods_into_view_scrolls_down() { + let mut state = AppState::default(); + // 40 single-pod groups → 40 flat rows. VISIBLE_ROWS=35. + state.pods = (0..40).map(|i| make_pod_group(&format!("svc-{i:02}"), 1)).collect(); + state.selected_pod_group = Some(38); + state.pod_scroll_offset = 0; + + scroll_pods_into_view(&mut state); + // Group 38 at flat row 38 should be visible: offset should move up. + assert!(state.pod_scroll_offset > 0); + assert!(state.pod_scroll_offset <= 38); + } + + #[test] + fn scroll_pods_into_view_scrolls_up() { + let mut state = AppState::default(); + state.pods = (0..40).map(|i| make_pod_group(&format!("svc-{i:02}"), 1)).collect(); + state.selected_pod_group = Some(2); + state.pod_scroll_offset = 10; + + scroll_pods_into_view(&mut state); + // Group 2 at flat row 2 is above offset 10 — should scroll up. + assert_eq!(state.pod_scroll_offset, 2); + } + + #[test] + fn scroll_pods_into_view_no_change_when_visible() { + let mut state = AppState::default(); + state.pods = (0..10).map(|i| make_pod_group(&format!("svc-{i}"), 1)).collect(); + state.selected_pod_group = Some(5); + state.pod_scroll_offset = 0; + + scroll_pods_into_view(&mut state); + // Group 5 at row 5 is within 0..35 — no change. + assert_eq!(state.pod_scroll_offset, 0); + } + + #[test] + fn scroll_pods_noop_when_no_selection() { + let mut state = AppState::default(); + state.pods = (0..10).map(|i| make_pod_group(&format!("svc-{i}"), 1)).collect(); + state.selected_pod_group = None; + state.pod_scroll_offset = 5; + + scroll_pods_into_view(&mut state); + assert_eq!(state.pod_scroll_offset, 5); // unchanged + } + + // -- Cockpit switching -- + + #[test] + fn cockpit_switching_via_number_keys() { + let mut state = AppState::default(); + assert_eq!(state.current_cockpit, 0); + + // Simulate pressing '3' + let idx = ('3' as usize).saturating_sub('1' as usize); + if idx < COCKPIT_COUNT { + state.current_cockpit = idx; + } + assert_eq!(state.current_cockpit, 2); + + // Simulate pressing '7' + let idx = ('7' as usize).saturating_sub('1' as usize); + if idx < COCKPIT_COUNT { + state.current_cockpit = idx; + } + assert_eq!(state.current_cockpit, 6); + } + + #[test] + fn cockpit_switching_out_of_range_ignored() { + let mut state = AppState::default(); + state.current_cockpit = 3; + + // '8' maps to index 7, which is >= COCKPIT_COUNT (7), so ignored. + let idx = ('8' as usize).saturating_sub('1' as usize); + if idx < COCKPIT_COUNT { + state.current_cockpit = idx; + } + assert_eq!(state.current_cockpit, 3); // unchanged + } + + // -- Help toggle -- + + #[test] + fn help_toggle() { + let mut state = AppState::default(); + assert!(!state.show_help); + state.show_help = !state.show_help; + assert!(state.show_help); + state.show_help = !state.show_help; + assert!(!state.show_help); + } + + // -- Pod navigation: nav_up -- + + #[test] + fn nav_up_pods_wraps_to_last() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_PODS; + state.pods = vec![ + PodGroupData { service: "a".into(), pods: vec![], ready_count: 0, total_count: 0 }, + PodGroupData { service: "b".into(), pods: vec![], ready_count: 0, total_count: 0 }, + PodGroupData { service: "c".into(), pods: vec![], ready_count: 0, total_count: 0 }, + ]; + + handle_nav_up(&mut state); + assert_eq!(state.selected_pod_group, Some(0)); + // Wrap to last + handle_nav_up(&mut state); + assert_eq!(state.selected_pod_group, Some(2)); + handle_nav_up(&mut state); + assert_eq!(state.selected_pod_group, Some(1)); + } + + #[test] + fn nav_down_empty_pods_is_noop() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_PODS; + handle_nav_down(&mut state); + assert_eq!(state.selected_pod_group, None); + } + + #[test] + fn nav_up_empty_pods_is_noop() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_PODS; + handle_nav_up(&mut state); + assert_eq!(state.selected_pod_group, None); + } + + // -- History navigation: nav_up -- + + #[test] + fn nav_up_history_tab_wraps() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::History; + state.training_history = vec![ + TrainingHistoryEntry { + job_id: "j1".into(), model: "DQN".into(), status: "Completed".into(), + final_loss: 0.3, best_val: 0.4, created_at: 0, started_at: 0, + completed_at: 0, description: String::new(), + }, + TrainingHistoryEntry { + job_id: "j2".into(), model: "PPO".into(), status: "Failed".into(), + final_loss: 0.0, best_val: 0.0, created_at: 0, started_at: 0, + completed_at: 0, description: String::new(), + }, + ]; + + handle_nav_up(&mut state); + assert_eq!(state.selected_history_entry, Some(0)); + // Wrap to last + handle_nav_up(&mut state); + assert_eq!(state.selected_history_entry, Some(1)); + } + + #[test] + fn nav_empty_history_is_noop() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + state.training_tab = TrainingTab::History; + handle_nav_down(&mut state); + assert_eq!(state.selected_history_entry, None); + handle_nav_up(&mut state); + assert_eq!(state.selected_history_entry, None); + } + + // -- Enter on non-navigable cockpits -- + + #[test] + fn enter_on_overview_cockpit_is_noop() { + let mut state = AppState::default(); + state.current_cockpit = 0; // Overview + handle_enter(&mut state); + assert!(!state.training_detail_expanded); + assert!(!state.pod_group_expanded); + } + + // -- l/h direct tab switching -- + + #[test] + fn l_h_keys_set_training_tab_directly() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_TRAINING; + assert_eq!(state.training_tab, TrainingTab::Live); + + // 'l' goes to History + state.training_tab = TrainingTab::History; + assert_eq!(state.training_tab, TrainingTab::History); + + // 'h' goes to Live + state.training_tab = TrainingTab::Live; + assert_eq!(state.training_tab, TrainingTab::Live); + } + + // -- Ring buffer: single element -- + + #[test] + fn ring_buffer_single_element() { + let mut state = AppState::default(); + apply_update(&mut state, StateUpdate::Execution(ExecutionData { + time: "now".into(), symbol: "ES".into(), side: "BUY".into(), + qty: 1, price: 5000.0, status: "Filled".into(), + })); + assert_eq!(state.executions.len(), 1); + assert_eq!(state.executions[0].time, "now"); + } + + // -- Pod scroll integrates with navigation -- + + #[test] + fn nav_down_pods_adjusts_scroll_offset() { + let mut state = AppState::default(); + state.current_cockpit = COCKPIT_PODS; + // 40 groups should trigger scrolling when navigating to the end. + state.pods = (0..40).map(|i| make_pod_group(&format!("svc-{i:02}"), 1)).collect(); + + // Navigate all the way down. + for _ in 0..39 { + handle_nav_down(&mut state); + } + assert_eq!(state.selected_pod_group, Some(38)); + // Scroll offset should have adjusted so group 38 is visible. + assert!(state.pod_scroll_offset > 0); + } +} diff --git a/bin/fxt/src/tui/state.rs b/bin/fxt/src/tui/state.rs index 3ace21dce..bfcc9538a 100644 --- a/bin/fxt/src/tui/state.rs +++ b/bin/fxt/src/tui/state.rs @@ -25,6 +25,31 @@ impl Default for ConnectionStatus { } } +// --------------------------------------------------------------------------- +// Training cockpit sub-tab +// --------------------------------------------------------------------------- + +/// Sub-tab within the Training cockpit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TrainingTab { + #[default] + Live, + History, +} + +/// A completed/failed training run from the history (ListTrainingJobs RPC). +pub struct TrainingHistoryEntry { + pub job_id: String, + pub model: String, + pub status: String, + pub final_loss: f32, + pub best_val: f32, + pub created_at: i64, + pub started_at: i64, + pub completed_at: i64, + pub description: String, +} + // --------------------------------------------------------------------------- // Top-level state // --------------------------------------------------------------------------- @@ -71,6 +96,20 @@ pub struct AppState { pub selected_pod_group: Option, /// Whether the selected pod group is expanded. pub pod_group_expanded: bool, + /// Scroll offset for the pod table (number of rows to skip). + pub pod_scroll_offset: usize, + /// Currently selected training session index. + pub selected_training_session: Option, + /// Whether the selected training session detail panel is shown. + pub training_detail_expanded: bool, + /// Active sub-tab in the Training cockpit (Live vs History). + pub training_tab: TrainingTab, + /// Completed/failed training runs from history. + pub training_history: Vec, + /// Currently selected history entry index. + pub selected_history_entry: Option, + /// Scroll offset for the per-epoch history table in the detail panel. + pub epoch_scroll_offset: usize, /// Timestamp of the last data refresh. pub last_update: Instant, /// gRPC connection status. @@ -100,12 +139,44 @@ impl Default for AppState { pods: Vec::new(), selected_pod_group: None, pod_group_expanded: false, + pod_scroll_offset: 0, + selected_training_session: None, + training_detail_expanded: false, + training_tab: TrainingTab::default(), + training_history: Vec::new(), + selected_history_entry: None, + epoch_scroll_offset: 0, last_update: Instant::now(), connection_status: ConnectionStatus::default(), } } } +// --------------------------------------------------------------------------- +// Per-epoch snapshot (accumulated from streaming updates) +// --------------------------------------------------------------------------- + +/// Snapshot of key metrics at a single epoch boundary. +pub struct EpochSnapshot { + pub epoch: u32, + pub loss: f64, + pub val_loss: f64, + pub sharpe: f64, + pub sortino: f64, + pub win_rate: f64, + pub max_dd: f64, + pub profit_factor: f64, + pub total_return: f64, + pub total_trades: u32, + pub q_value_mean: f64, + pub gradient_norm: f64, + pub learning_rate: f64, + pub policy_entropy: f64, + pub action_buy_pct: f64, + pub action_sell_pct: f64, + pub action_hold_pct: f64, +} + // --------------------------------------------------------------------------- // Data sub-structs // --------------------------------------------------------------------------- @@ -113,16 +184,52 @@ impl Default for AppState { /// A single ML training session. pub struct TrainingSessionData { pub model: String, - pub fold: u32, + pub fold: String, + pub is_hyperopt: bool, pub epoch: u32, - pub max_epochs: u32, pub loss: f64, pub val_loss: f64, pub batches_per_sec: f64, + pub iteration_secs: f64, + pub epoch_duration_secs: f64, + // Financial metrics pub sharpe: f64, pub sortino: f64, pub win_rate: f64, pub max_dd: f64, + pub profit_factor: f64, + pub total_return: f64, + pub total_trades: u32, + // Action distribution + pub action_buy_pct: f64, + pub action_sell_pct: f64, + pub action_hold_pct: f64, + // RL diagnostics + pub q_value_mean: f64, + pub q_value_max: f64, + pub policy_entropy: f64, + pub kl_divergence: f64, + pub advantage_mean: f64, + pub replay_buffer_size: u32, + // Gradient & training health + pub gradient_norm: f64, + pub learning_rate: f64, + // Health counters + pub nan_detected: u32, + pub gradient_explosions: u32, + pub feature_errors: u32, + // Checkpoint + pub checkpoint_saves: u32, + pub checkpoint_failures: u32, + // Hyperopt + pub hyperopt_trial_current: u32, + pub hyperopt_trial_total: u32, + pub hyperopt_best_objective: f64, + pub hyperopt_trials_failed: u32, + pub hyperopt_trial_epoch: u32, + pub hyperopt_elapsed_secs: f64, + // Per-epoch history (accumulated across stream updates) + pub epoch_history: Vec, } /// GPU telemetry snapshot. diff --git a/infra/k8s/network-policies/api-gateway.yaml b/infra/k8s/network-policies/api-gateway.yaml index 68e2aed65..e625eb49c 100644 --- a/infra/k8s/network-policies/api-gateway.yaml +++ b/infra/k8s/network-policies/api-gateway.yaml @@ -87,13 +87,14 @@ spec: ports: - protocol: TCP port: 50057 + # Prometheus (metric scraping from monitoring handler) - to: - podSelector: matchLabels: - app.kubernetes.io/name: monitoring-service + app.kubernetes.io/name: prometheus ports: - protocol: TCP - port: 50057 + port: 9090 # Minio (initContainer binary fetch) - to: - podSelector: