gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(fxt): enhance watch TUI with full training metrics and gRPC helpers

Browse Source 
- Add connect_channel/connect_channel_lazy with TLS auto-detection
- Expand TrainingSession to 35+ proto fields (RL diagnostics, hyperopt,
  eval metrics, health counters, checkpoints)
- Add RL-specific rendering (entropy, action diversity, Q-values, rewards)
- Add hyperopt progress panel (trial/best objective tracking)
- Refactor all commands to use shared connect_channel helper
- Default API gateway URL to https://api.fxhnt.ai

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-03 13:38:58 +01:00
parent 25f8d514b1
commit afbc668dcd
13 changed files with 1261 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
bin/fxt/src/client/mod.rs
View File

@@ -13,6 +13,38 @@ pub mod connection_manager;
pub mod ml_training_client;
pub mod trading_client;
use anyhow::{Context, Result};
use tonic::transport::Channel;
/// Create a gRPC channel that auto-configures TLS for `https://` URLs.
///
/// Connects eagerly (awaits the TCP handshake).
pub async fn connect_channel(url: &str) -> Result<Channel> {
let mut endpoint = Channel::from_shared(url.to_owned()).context("Invalid gRPC endpoint URL")?;
if url.starts_with("https://") {
endpoint = endpoint
.tls_config(tonic::transport::ClientTlsConfig::new().with_enabled_roots())
.context("Failed to configure TLS")?;
}
endpoint
.connect()
.await
.context("Failed to connect to API Gateway")
}
/// Create a lazily-connected gRPC channel with TLS auto-detection.
///
/// Does not attempt the TCP connection until the first RPC call.
pub fn connect_channel_lazy(url: &str) -> Result<Channel> {
let mut endpoint = Channel::from_shared(url.to_owned()).context("Invalid gRPC endpoint URL")?;
if url.starts_with("https://") {
endpoint = endpoint
.tls_config(tonic::transport::ClientTlsConfig::new().with_enabled_roots())
.context("Failed to configure TLS")?;
}
Ok(endpoint.connect_lazy())
}
/// Service endpoints configuration
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
pub struct ServiceEndpoints {

8
bin/fxt/src/commands/agent.rs
View File

@@ -15,7 +15,7 @@
use anyhow::{Context, Result};
use clap::{Args, Subcommand};
use colored::Colorize;
use tonic::transport::Channel;
// Channel creation via crate::client::connect_channel
use tonic::Request;
// Import generated proto types
@@ -181,11 +181,7 @@ pub async fn handle_allocate_portfolio(
println!();
// Connect to API Gateway
let channel = Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
let mut client = TradingAgentServiceClient::new(channel);

22
bin/fxt/src/commands/auth.rs
View File

@@ -34,7 +34,7 @@ pub enum AuthCommand {
#[clap(
long,
env = "API_GATEWAY_URL",
default_value = "http://localhost:50051"
default_value = "https://api.fxhnt.ai"
)]
api_gateway_url: String,
},
@@ -51,7 +51,7 @@ pub enum AuthCommand {
#[clap(
long,
env = "API_GATEWAY_URL",
default_value = "http://localhost:50051"
default_value = "https://api.fxhnt.ai"
)]
api_gateway_url: String,
},
@@ -98,11 +98,7 @@ async fn execute_login(
println!("{}", "Connecting to API Gateway...".cyan());
// Connect to API Gateway
let channel = tonic::transport::Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
// Create auth components
let storage = FileTokenStorage::new().context("Failed to initialize token storage")?;
@@ -155,11 +151,7 @@ async fn execute_interactive_login(api_gateway_url: &str) -> Result<()> {
println!();
println!("{}", "Connecting to API Gateway...".cyan());
let channel = tonic::transport::Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
// Create auth components
let storage = FileTokenStorage::new().context("Failed to initialize token storage")?;
@@ -289,11 +281,7 @@ async fn execute_refresh(api_gateway_url: &str) -> Result<()> {
println!("{}", "Refreshing tokens...".cyan());
// Connect to API Gateway
let channel = tonic::transport::Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
// Create auth components
let storage = FileTokenStorage::new().context("Failed to initialize token storage")?;

8
bin/fxt/src/commands/model/approve.rs
View File

@@ -12,18 +12,14 @@ use crate::proto::ml_training::{
ml_training_service_client::MlTrainingServiceClient, ApproveModelRequest,
};
use anyhow::{Context, Result};
use tonic::{metadata::MetadataValue, transport::Channel, Request};
use tonic::{metadata::MetadataValue, Request};
/// Approve a pending model promotion.
///
/// Calls `ApproveModel` on the ML training service via the API gateway
/// to promote the model to production.
pub async fn run(api_gateway_url: &str, jwt_token: &str, model_id: &str) -> Result<()> {
let channel = Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
let mut client = MlTrainingServiceClient::new(channel);

8
bin/fxt/src/commands/model/list.rs
View File

@@ -12,18 +12,14 @@ use crate::proto::ml_training::{
ml_training_service_client::MlTrainingServiceClient, ListAvailableModelsRequest,
};
use anyhow::{Context, Result};
use tonic::{metadata::MetadataValue, transport::Channel, Request};
use tonic::{metadata::MetadataValue, Request};
/// List available models for training.
///
/// Calls `ListAvailableModels` on the ML training service via the API gateway
/// and displays a formatted table of available model types.
pub async fn run(api_gateway_url: &str, jwt_token: &str) -> Result<()> {
let channel = Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
let mut client = MlTrainingServiceClient::new(channel);

8
bin/fxt/src/commands/model/reject.rs
View File

@@ -13,7 +13,7 @@ use crate::proto::ml_training::{
ml_training_service_client::MlTrainingServiceClient, RejectModelRequest,
};
use anyhow::{Context, Result};
use tonic::{metadata::MetadataValue, transport::Channel, Request};
use tonic::{metadata::MetadataValue, Request};
/// Reject a pending model promotion.
///
@@ -25,11 +25,7 @@ pub async fn run(
model_id: &str,
reason: &str,
) -> Result<()> {
let channel = Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
let mut client = MlTrainingServiceClient::new(channel);

8
bin/fxt/src/commands/train/list.rs
View File

@@ -42,7 +42,7 @@ use crate::proto::ml_training::{
ml_training_service_client::MlTrainingServiceClient, ListTrainingJobsRequest,
TrainingJobSummary, TrainingStatus,
};
use tonic::{metadata::MetadataValue, transport::Channel, Request};
use tonic::{metadata::MetadataValue, Request};
/// List training jobs with filtering and sorting
#[derive(Parser, Debug)]
@@ -84,11 +84,7 @@ impl ListCommand {
/// Execute the list command
pub async fn run(&self, api_gateway_url: &str, jwt_token: &str) -> Result<()> {
// Connect to API Gateway
let channel = Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
let mut client = MlTrainingServiceClient::new(channel);

14
bin/fxt/src/commands/train/start.rs
View File

@@ -12,7 +12,7 @@ use crate::proto::ml_training::{
data_source::Source, ml_training_service_client::MlTrainingServiceClient, DataSource,
StartTrainingRequest, TrainingStatus,
};
use tonic::{metadata::MetadataValue, transport::Channel, Request};
use tonic::{metadata::MetadataValue, Request};
/// Start an on-demand training job
#[derive(Args, Debug)]
@@ -67,17 +67,7 @@ impl StartCommand {
);
// Connect to API Gateway
let mut endpoint = Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?;
if api_gateway_url.starts_with("https://") {
endpoint = endpoint
.tls_config(tonic::transport::ClientTlsConfig::new().with_enabled_roots())
.context("Failed to configure TLS")?;
}
let channel = endpoint
.connect()
.await
.context("Failed to connect to API Gateway")?;
let channel = crate::client::connect_channel(api_gateway_url).await?;
let mut client = MlTrainingServiceClient::new(channel);

267
bin/fxt/src/commands/watch/event_loop.rs
View File

@@ -16,7 +16,7 @@ use tokio_stream::StreamExt;
use super::render;
use super::state::{
CircuitBreakerRow, DashboardState, FillRow, ServiceRow,
CircuitBreakerRow, DashboardState, FillRow, ServiceRow, Tab, TrainingViewMode,
};
use super::streams::{self, StreamEvent};
@@ -101,11 +101,29 @@ async fn run_event_loop(
fn apply_stream_event(state: &mut DashboardState, event: StreamEvent) {
match event {
StreamEvent::TrainingUpdate { sessions, gpu } => {
// Push first session's epoch_loss to loss sparkline.
if let Some(first) = sessions.first() {
// Filter out uninitialised or aggregate rows:
// - empty model name (default proto)
// - model present but empty fold with zero epoch (summary row)
let active_sessions: Vec<_> = sessions
.into_iter()
.filter(|s| !(s.model.is_empty() || s.fold.is_empty() && s.epoch == 0.0))
.collect();
// Push first session's epoch_loss to global loss sparkline.
if let Some(first) = active_sessions.first() {
state.training.push_loss(f64::from(first.epoch_loss));
}
state.training.sessions = sessions;
// Accumulate per-session metric history for detail sparklines.
for session in &active_sessions {
let key = format!("{}:{}", session.model, session.fold);
state
.training
.session_histories
.entry(key)
.or_default()
.push(session);
}
state.training.sessions = active_sessions;
state.training.clamp_selection();
state.training.gpu = gpu.clone();
state.system.gpu = gpu;
state.training.connected = true;
@@ -236,9 +254,78 @@ fn apply_stream_event(state: &mut DashboardState, event: StreamEvent) {
/// Returns `true` if the user wants to quit.
#[allow(clippy::wildcard_enum_match_arm)] // KeyCode has 25+ variants; exhaustive match is noise
fn handle_key(state: &mut DashboardState, key: KeyEvent) -> bool {
// Global quit
match key.code {
KeyCode::Char('q') => return true,
KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => return true,
_ => {}
}
// Training tab has special modal key handling.
if state.current_tab() == Tab::Training {
match state.training.view_mode {
TrainingViewMode::List => match key.code {
KeyCode::Char('j') | KeyCode::Down => {
state.training.select_down();
state.dirty = true;
}
KeyCode::Char('k') | KeyCode::Up => {
state.training.select_up();
state.dirty = true;
}
KeyCode::Enter => {
state.training.enter_detail();
state.dirty = true;
}
KeyCode::Char('1') => state.select_tab(0),
KeyCode::Char('2') => state.select_tab(1),
KeyCode::Char('3') => state.select_tab(2),
KeyCode::Char('4') => state.select_tab(3),
_ => {}
},
TrainingViewMode::Detail => match key.code {
KeyCode::Esc => {
state.training.exit_detail();
state.dirty = true;
}
KeyCode::Tab => {
state.training.next_sub_tab();
state.dirty = true;
}
KeyCode::BackTab => {
state.training.prev_sub_tab();
state.dirty = true;
}
KeyCode::Char('l') | KeyCode::Right => {
// Next session
state.training.select_down();
state.dirty = true;
}
KeyCode::Char('h') | KeyCode::Left => {
// Previous session
state.training.select_up();
state.dirty = true;
}
KeyCode::Char('j') | KeyCode::Down => {
state.training.scroll_down();
state.dirty = true;
}
KeyCode::Char('k') | KeyCode::Up => {
state.training.scroll_up();
state.dirty = true;
}
KeyCode::Char(c @ '1'..='4') => {
state.training.exit_detail();
state.select_tab((c as u8 - b'1') as usize);
}
_ => {}
},
}
return false;
}
// Non-Training tabs: standard key handling.
match key.code {
KeyCode::Char('1') => state.select_tab(0),
KeyCode::Char('2') => state.select_tab(1),
KeyCode::Char('3') => state.select_tab(2),
@@ -302,29 +389,30 @@ mod tests {
}
#[test]
fn test_handle_key_scroll() {
fn test_handle_key_scroll_non_training_tab() {
// Non-training tabs still use scroll on j/k.
let mut state = DashboardState::default();
// Scroll down with 'j'.
state.select_tab(1); // Trading tab
let j = KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE);
handle_key(&mut state, j);
assert_eq!(state.training.scroll_offset, 1);
assert_eq!(state.trading.scroll_offset, 1);
// Scroll up with 'k'.
let k = KeyEvent::new(KeyCode::Char('k'), KeyModifiers::NONE);
handle_key(&mut state, k);
assert_eq!(state.training.scroll_offset, 0);
assert_eq!(state.trading.scroll_offset, 0);
}
#[test]
fn test_handle_key_arrow_scroll() {
fn test_handle_key_arrow_scroll_non_training_tab() {
let mut state = DashboardState::default();
state.select_tab(1); // Trading tab
let down = KeyEvent::new(KeyCode::Down, KeyModifiers::NONE);
handle_key(&mut state, down);
assert_eq!(state.training.scroll_offset, 1);
assert_eq!(state.trading.scroll_offset, 1);
let up = KeyEvent::new(KeyCode::Up, KeyModifiers::NONE);
handle_key(&mut state, up);
assert_eq!(state.training.scroll_offset, 0);
assert_eq!(state.trading.scroll_offset, 0);
}
#[test]
@@ -334,6 +422,120 @@ mod tests {
assert!(!handle_key(&mut state, key));
}
// -- Training list mode keys -------------------------------------------
fn make_state_with_sessions(n: usize) -> DashboardState {
let mut state = DashboardState::default();
for i in 0..n {
state.training.sessions.push(TrainingSession {
model: format!("DQN_{i}"),
fold: format!("fold_{i}"),
..Default::default()
});
}
state
}
#[test]
fn test_training_list_jk_selects() {
let mut state = make_state_with_sessions(3);
// j selects first row
let j = KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE);
handle_key(&mut state, j);
assert_eq!(state.training.selected_index, Some(0));
// j again moves to second
handle_key(&mut state, j);
assert_eq!(state.training.selected_index, Some(1));
// k moves back
let k = KeyEvent::new(KeyCode::Char('k'), KeyModifiers::NONE);
handle_key(&mut state, k);
assert_eq!(state.training.selected_index, Some(0));
}
#[test]
fn test_training_list_enter_opens_detail() {
let mut state = make_state_with_sessions(3);
state.training.selected_index = Some(1);
let enter = KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE);
handle_key(&mut state, enter);
assert_eq!(state.training.view_mode, TrainingViewMode::Detail);
}
// -- Training detail mode keys -----------------------------------------
#[test]
fn test_training_detail_esc_exits() {
let mut state = make_state_with_sessions(3);
state.training.selected_index = Some(0);
state.training.enter_detail();
let esc = KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE);
handle_key(&mut state, esc);
assert_eq!(state.training.view_mode, TrainingViewMode::List);
}
#[test]
fn test_training_detail_tab_cycles_sub_tabs() {
let mut state = make_state_with_sessions(1);
state.training.selected_index = Some(0);
state.training.enter_detail();
// DQN_0 -> Overview, Loss, RL, Health
let tab = KeyEvent::new(KeyCode::Tab, KeyModifiers::NONE);
handle_key(&mut state, tab);
assert_eq!(
state.training.detail_sub_tab,
super::super::state::DetailSubTab::Loss
);
}
#[test]
fn test_training_detail_backtab_cycles_back() {
let mut state = make_state_with_sessions(1);
state.training.selected_index = Some(0);
state.training.enter_detail();
state.training.detail_sub_tab = super::super::state::DetailSubTab::Loss;
let backtab = KeyEvent::new(KeyCode::BackTab, KeyModifiers::SHIFT);
handle_key(&mut state, backtab);
assert_eq!(
state.training.detail_sub_tab,
super::super::state::DetailSubTab::Overview
);
}
#[test]
fn test_training_detail_lr_navigates_sessions() {
let mut state = make_state_with_sessions(3);
state.training.selected_index = Some(0);
state.training.enter_detail();
// Right moves to next session
let right = KeyEvent::new(KeyCode::Right, KeyModifiers::NONE);
handle_key(&mut state, right);
assert_eq!(state.training.selected_index, Some(1));
// Left moves back
let left = KeyEvent::new(KeyCode::Left, KeyModifiers::NONE);
handle_key(&mut state, left);
assert_eq!(state.training.selected_index, Some(0));
}
#[test]
fn test_training_detail_number_exits_and_switches_tab() {
let mut state = make_state_with_sessions(3);
state.training.selected_index = Some(0);
state.training.enter_detail();
let key2 = KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE);
handle_key(&mut state, key2);
assert_eq!(state.training.view_mode, TrainingViewMode::List);
assert_eq!(state.active_tab, 1); // Trading tab
}
#[test]
fn test_training_detail_q_quits() {
let mut state = make_state_with_sessions(1);
state.training.selected_index = Some(0);
state.training.enter_detail();
let q = KeyEvent::new(KeyCode::Char('q'), KeyModifiers::NONE);
assert!(handle_key(&mut state, q));
}
// -- format_nanos --------------------------------------------------------
#[test]
@@ -370,6 +572,7 @@ mod tests {
batches_per_second: 42.0,
gradient_norm: 1.5,
epoch_duration_seconds: 12.3,
..Default::default()
};
let gpu = GpuInfo {
utilization_percent: 85.0,
@@ -393,6 +596,11 @@ mod tests {
assert!((state.training.loss_history.back().unwrap() - 0.123_f64).abs() < 1e-3);
// GPU should be cloned to both training and system.
assert!((state.system.gpu.utilization_percent - 85.0).abs() < f32::EPSILON);
// Per-session history should be accumulated.
assert_eq!(state.training.session_histories.len(), 1);
let hist = state.training.session_histories.get("DQN:fold0").unwrap();
assert_eq!(hist.loss.len(), 1);
assert!((hist.loss[0] - 0.123_f64).abs() < 1e-3);
assert!(state.dirty);
}
@@ -606,4 +814,39 @@ mod tests {
// Dirty flag should be set after each event.
assert!(state.dirty);
}
#[test]
fn test_training_update_filters_empty_model() {
let mut state = DashboardState::default();
let valid = TrainingSession {
model: "DQN".into(),
fold: "fold0".into(),
epoch_loss: 0.5,
..Default::default()
};
let empty = TrainingSession {
model: String::new(),
fold: String::new(),
..Default::default()
};
// Summary row: model present but no fold and epoch=0 (aggregate row from proto)
let summary = TrainingSession {
model: "DQN".into(),
fold: String::new(),
epoch: 0.0,
..Default::default()
};
apply_stream_event(
&mut state,
StreamEvent::TrainingUpdate {
sessions: vec![empty, summary, valid],
gpu: GpuInfo::default(),
},
);
// Only the session with a non-empty model AND a fold (or non-zero epoch) should remain.
assert_eq!(state.training.sessions.len(), 1);
assert_eq!(state.training.sessions[0].model, "DQN");
assert_eq!(state.training.session_histories.len(), 1);
assert!(state.training.session_histories.contains_key("DQN:fold0"));
}
}

414
bin/fxt/src/commands/watch/render.rs
View File

@@ -7,12 +7,13 @@ use ratatui::{
layout::{Constraint, Direction, Layout, Rect},
style::{Color, Modifier, Style},
text::{Line, Span},
widgets::{Block, Borders, Cell, Paragraph, Row, Sparkline, Table, Tabs},
widgets::{Block, Borders, Cell, Gauge, Paragraph, Row, Sparkline, Table, TableState, Tabs},
Frame,
};
use super::state::{
DashboardState, RiskTabState, SystemTabState, Tab, TradingTabState, TrainingTabState,
DashboardState, DetailSubTab, RiskTabState, SessionHistory, SystemTabState, Tab,
TradingTabState, TrainingTabState, TrainingViewMode,
};
// ---------------------------------------------------------------------------
@@ -33,13 +34,18 @@ pub(super) fn render(frame: &mut Frame, state: &DashboardState) {
render_tab_bar(frame, chunks[0], state);
match state.current_tab() {
Tab::Training => render_training_tab(frame, chunks[1], &state.training),
Tab::Training => match state.training.view_mode {
TrainingViewMode::List => render_training_list(frame, chunks[1], &state.training),
TrainingViewMode::Detail => {
render_training_detail(frame, chunks[1], &state.training);
}
},
Tab::Trading => render_trading_tab(frame, chunks[1], &state.trading),
Tab::Risk => render_risk_tab(frame, chunks[1], &state.risk),
Tab::System => render_system_tab(frame, chunks[1], &state.system),
}
render_status_bar(frame, chunks[2]);
render_status_bar(frame, chunks[2], state);
}
// ---------------------------------------------------------------------------
@@ -72,15 +78,43 @@ fn render_tab_bar(frame: &mut Frame, area: Rect, state: &DashboardState) {
// Status bar
// ---------------------------------------------------------------------------
fn render_status_bar(frame: &mut Frame, area: Rect) {
let bar = Paragraph::new(Line::from(vec![
Span::styled("1-4", Style::default().fg(Color::Cyan)),
Span::raw(" tabs "),
Span::styled("j/k", Style::default().fg(Color::Cyan)),
Span::raw(" scroll "),
Span::styled("q", Style::default().fg(Color::Cyan)),
Span::raw(" quit"),
]));
fn render_status_bar(frame: &mut Frame, area: Rect, state: &DashboardState) {
let hints: Vec<Span<'_>> =
if state.current_tab() == Tab::Training && state.training.view_mode == TrainingViewMode::Detail {
vec![
Span::styled("Esc", Style::default().fg(Color::Cyan)),
Span::raw(" back "),
Span::styled("Tab/S-Tab", Style::default().fg(Color::Cyan)),
Span::raw(" sub-tab "),
Span::styled("h/l", Style::default().fg(Color::Cyan)),
Span::raw(" prev/next "),
Span::styled("j/k", Style::default().fg(Color::Cyan)),
Span::raw(" scroll "),
Span::styled("q", Style::default().fg(Color::Cyan)),
Span::raw(" quit"),
]
} else if state.current_tab() == Tab::Training {
vec![
Span::styled("1-4", Style::default().fg(Color::Cyan)),
Span::raw(" tabs "),
Span::styled("j/k", Style::default().fg(Color::Cyan)),
Span::raw(" select "),
Span::styled("Enter", Style::default().fg(Color::Cyan)),
Span::raw(" detail "),
Span::styled("q", Style::default().fg(Color::Cyan)),
Span::raw(" quit"),
]
} else {
vec![
Span::styled("1-4", Style::default().fg(Color::Cyan)),
Span::raw(" tabs "),
Span::styled("j/k", Style::default().fg(Color::Cyan)),
Span::raw(" scroll "),
Span::styled("q", Style::default().fg(Color::Cyan)),
Span::raw(" quit"),
]
};
let bar = Paragraph::new(Line::from(hints));
frame.render_widget(bar, area);
}
@@ -88,7 +122,7 @@ fn render_status_bar(frame: &mut Frame, area: Rect) {
// Training tab
// ---------------------------------------------------------------------------
fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
fn render_training_list(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
let conn_label = if tab.connected {
"connected"
} else {
@@ -103,13 +137,15 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
])
.split(area);
// -- Sessions table ---------------------------------------------------
// -- Sessions table with selection ------------------------------------
let header = Row::new(vec![
Cell::from("Model"),
Cell::from("Fold"),
Cell::from("Epoch"),
Cell::from("Loss"),
Cell::from("Val Loss"),
Cell::from("Acc"),
Cell::from("F1"),
Cell::from("LR"),
Cell::from("Batch/s"),
Cell::from("Grad"),
@@ -123,7 +159,6 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
let rows: Vec<Row<'_>> = tab
.sessions
.iter()
.skip(tab.scroll_offset)
.map(|s| {
Row::new(vec![
Cell::from(s.model.as_str()),
@@ -131,6 +166,8 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
Cell::from(format!("{:.1}", s.epoch)),
Cell::from(format!("{:.4}", s.epoch_loss)),
Cell::from(format!("{:.4}", s.validation_loss)),
Cell::from(format!("{:.4}", s.eval_accuracy)),
Cell::from(format!("{:.4}", s.eval_f1)),
Cell::from(format!("{:.2e}", s.learning_rate)),
Cell::from(format!("{:.1}", s.batches_per_second)),
Cell::from(format!("{:.3}", s.gradient_norm)),
@@ -146,19 +183,27 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
Constraint::Length(8),
Constraint::Length(10),
Constraint::Length(10),
Constraint::Length(8),
Constraint::Length(8),
Constraint::Length(10),
Constraint::Length(8),
Constraint::Length(8),
],
)
.header(header)
.row_highlight_style(
Style::default()
.fg(Color::Cyan)
.add_modifier(Modifier::BOLD),
)
.block(
Block::default()
.borders(Borders::ALL)
.title(format!(" Training [{conn_label}] ")),
);
frame.render_widget(table, chunks[0]);
let mut table_state = TableState::default().with_selected(tab.selected_index);
frame.render_stateful_widget(table, chunks[0], &mut table_state);
// -- Bottom row: GPU info (left) + loss sparkline (right) -------------
let bottom = Layout::default()
@@ -200,6 +245,341 @@ fn render_training_tab(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
frame.render_widget(sparkline, bottom[1]);
}
// ---------------------------------------------------------------------------
// Training detail view
// ---------------------------------------------------------------------------
/// Convert a `&[f64]` metric history to Sparkline-compatible `Vec<u64>`.
fn spark_u64(data: &[f64], max_clamp: f64) -> Vec<u64> {
data.iter()
.map(|&v| {
let clamped = v.clamp(0.0, max_clamp);
(clamped * 100.0) as u64
})
.collect()
}
fn render_sparkline_row(frame: &mut Frame, area: Rect, title: &str, data: &[f64], max: f64, color: Color) {
let vals = spark_u64(data, max);
let sparkline = Sparkline::default()
.block(Block::default().borders(Borders::ALL).title(format!(" {title} ")))
.data(&vals)
.style(Style::default().fg(color));
frame.render_widget(sparkline, area);
}
fn render_training_detail(frame: &mut Frame, area: Rect, tab: &TrainingTabState) {
let Some(session) = tab.selected_session() else {
let msg = Paragraph::new("No session selected")
.block(Block::default().borders(Borders::ALL).title(" Detail "));
frame.render_widget(msg, area);
return;
};
let vis_tabs = tab.visible_tabs();
let history_key = format!("{}:{}", session.model, session.fold);
let empty_hist = SessionHistory::default();
let history = tab.session_histories.get(&history_key).unwrap_or(&empty_hist);
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(3), // title bar
Constraint::Length(3), // sub-tab bar
Constraint::Min(0), // content
])
.split(area);
// -- Title bar --------------------------------------------------------
let status_label = if session.is_hyperopt {
format!(
"Hyperopt trial {}/{}",
session.hyperopt_trial_current, session.hyperopt_trial_total,
)
} else {
format!("Epoch {:.1}", session.epoch)
};
let title_text = format!(
" {} | {} | {} | GPU {:.0}%",
session.model, session.fold, status_label, tab.gpu.utilization_percent,
);
let title_bar = Paragraph::new(title_text).block(
Block::default()
.borders(Borders::ALL)
.title(" Session Detail "),
);
frame.render_widget(title_bar, chunks[0]);
// -- Sub-tab bar ------------------------------------------------------
let tab_titles: Vec<Line<'_>> = vis_tabs
.iter()
.map(|t| Line::from(Span::raw(t.title())))
.collect();
let active_idx = vis_tabs
.iter()
.position(|t| *t == tab.detail_sub_tab)
.unwrap_or(0);
let sub_tabs = Tabs::new(tab_titles)
.select(active_idx)
.highlight_style(
Style::default()
.fg(Color::Cyan)
.add_modifier(Modifier::BOLD),
)
.block(Block::default().borders(Borders::ALL));
frame.render_widget(sub_tabs, chunks[1]);
// -- Content area: dispatch to sub-tab renderer -----------------------
match tab.detail_sub_tab {
DetailSubTab::Overview => render_detail_overview(frame, chunks[2], session, history),
DetailSubTab::Loss => render_detail_loss(frame, chunks[2], history),
DetailSubTab::RlDiagnostics => render_detail_rl(frame, chunks[2], session, history),
DetailSubTab::Metrics => render_detail_metrics(frame, chunks[2], session, history),
DetailSubTab::Hyperopt => render_detail_hyperopt(frame, chunks[2], session, history),
DetailSubTab::Health => render_detail_health(frame, chunks[2], session),
}
}
// -- Sub-tab renderers ----------------------------------------------------
fn render_detail_overview(
frame: &mut Frame,
area: Rect,
session: &super::state::TrainingSession,
history: &SessionHistory,
) {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(8), // key stats
Constraint::Min(0), // loss sparkline
])
.split(area);
// Key stats grid
let stats = Paragraph::new(vec![
Line::from(format!(
" Loss: {:.4} Val Loss: {:.4} LR: {:.2e}",
session.epoch_loss, session.validation_loss, session.learning_rate,
)),
Line::from(format!(
" Batch/s: {:.1} Grad Norm: {:.3} Epoch Time: {:.1}s",
session.batches_per_second, session.gradient_norm, session.epoch_duration_seconds,
)),
Line::from(format!(
" Batches: {:.0} Iter: {:.3}s Checkpoints: {} ({} failed)",
session.batches_processed,
session.iteration_seconds,
session.checkpoint_saves,
session.checkpoint_failures,
)),
Line::from(format!(
" NaN: {} Grad Explosions: {} Feature Errors: {}",
session.nan_detected, session.gradient_explosions, session.feature_errors,
)),
])
.block(
Block::default()
.borders(Borders::ALL)
.title(" Overview "),
);
frame.render_widget(stats, chunks[0]);
render_sparkline_row(frame, chunks[1], "Loss", &history.loss, 10.0, Color::Cyan);
}
fn render_detail_loss(frame: &mut Frame, area: Rect, history: &SessionHistory) {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Percentage(25),
Constraint::Percentage(25),
Constraint::Percentage(25),
Constraint::Percentage(25),
])
.split(area);
render_sparkline_row(frame, chunks[0], "Loss", &history.loss, 10.0, Color::Cyan);
render_sparkline_row(frame, chunks[1], "Val Loss", &history.val_loss, 10.0, Color::Yellow);
render_sparkline_row(frame, chunks[2], "Learning Rate", &history.learning_rate, 1.0, Color::Green);
render_sparkline_row(frame, chunks[3], "Grad Norm", &history.gradient_norm, 100.0, Color::Red);
}
fn render_detail_rl(
frame: &mut Frame,
area: Rect,
session: &super::state::TrainingSession,
history: &SessionHistory,
) {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Percentage(20),
Constraint::Percentage(20),
Constraint::Percentage(20),
Constraint::Percentage(20),
Constraint::Percentage(20),
])
.split(area);
render_sparkline_row(frame, chunks[0], "Q-Value Mean", &history.q_value_mean, 100.0, Color::Cyan);
render_sparkline_row(frame, chunks[1], "Policy Entropy", &history.policy_entropy, 10.0, Color::Yellow);
render_sparkline_row(frame, chunks[2], "KL Divergence", &history.kl_divergence, 1.0, Color::Red);
render_sparkline_row(frame, chunks[3], "Advantage Mean", &history.advantage_mean, 10.0, Color::Green);
// Replay buffer gauge
let buf_size = session.replay_buffer_size;
let ratio = if buf_size > 0 {
(buf_size as f64 / 100_000.0).min(1.0)
} else {
0.0
};
let gauge = Gauge::default()
.block(
Block::default()
.borders(Borders::ALL)
.title(format!(" Replay Buffer: {buf_size} ")),
)
.gauge_style(Style::default().fg(Color::Cyan))
.ratio(ratio);
frame.render_widget(gauge, chunks[4]);
}
fn render_detail_metrics(
frame: &mut Frame,
area: Rect,
session: &super::state::TrainingSession,
history: &SessionHistory,
) {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(5), // current values
Constraint::Min(0), // sparklines
])
.split(area);
let current = Paragraph::new(vec![
Line::from(format!(
" Accuracy: {:.4} F1: {:.4}",
session.eval_accuracy, session.eval_f1,
)),
Line::from(format!(
" Precision: {:.4} Recall: {:.4}",
session.eval_precision, session.eval_recall,
)),
])
.block(
Block::default()
.borders(Borders::ALL)
.title(" Current Metrics "),
);
frame.render_widget(current, chunks[0]);
let spark_area = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Percentage(25),
Constraint::Percentage(25),
Constraint::Percentage(25),
Constraint::Percentage(25),
])
.split(chunks[1]);
render_sparkline_row(frame, spark_area[0], "Accuracy", &history.accuracy, 1.0, Color::Green);
render_sparkline_row(frame, spark_area[1], "Precision", &history.precision, 1.0, Color::Cyan);
render_sparkline_row(frame, spark_area[2], "Recall", &history.recall, 1.0, Color::Magenta);
render_sparkline_row(frame, spark_area[3], "F1", &history.f1, 1.0, Color::Yellow);
}
fn render_detail_hyperopt(
frame: &mut Frame,
area: Rect,
session: &super::state::TrainingSession,
history: &SessionHistory,
) {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(7), // stats
Constraint::Min(0), // best objective sparkline
])
.split(area);
let pct = if session.hyperopt_trial_total > 0 {
(session.hyperopt_trial_current as f32 / session.hyperopt_trial_total as f32) * 100.0
} else {
0.0
};
let stats = Paragraph::new(vec![
Line::from(format!(
" Trial: {}/{} ({:.0}%)",
session.hyperopt_trial_current, session.hyperopt_trial_total, pct,
)),
Line::from(format!(
" Best Objective: {:.6} Trial Best Loss: {:.6}",
session.hyperopt_best_objective, session.hyperopt_trial_best_loss,
)),
Line::from(format!(
" Trial Epoch: {} Failed: {} Elapsed: {:.0}s",
session.hyperopt_trial_epoch,
session.hyperopt_trials_failed,
session.hyperopt_elapsed_seconds,
)),
])
.block(
Block::default()
.borders(Borders::ALL)
.title(" Hyperopt Progress "),
);
frame.render_widget(stats, chunks[0]);
render_sparkline_row(
frame,
chunks[1],
"Best Objective",
&history.hyperopt_best_objective,
10.0,
Color::Magenta,
);
}
fn render_detail_health(
frame: &mut Frame,
area: Rect,
session: &super::state::TrainingSession,
) {
let health = Paragraph::new(vec![
Line::from(format!(
" NaN Detected: {}",
session.nan_detected,
)),
Line::from(format!(
" Gradient Explosions: {}",
session.gradient_explosions,
)),
Line::from(format!(
" Feature Errors: {}",
session.feature_errors,
)),
Line::from(""),
Line::from(format!(
" Checkpoint Saves: {} Failures: {}",
session.checkpoint_saves, session.checkpoint_failures,
)),
Line::from(format!(
" Checkpoint Size: {:.1} MB",
session.checkpoint_size_bytes / (1024.0 * 1024.0),
)),
])
.block(
Block::default()
.borders(Borders::ALL)
.title(" Health "),
);
frame.render_widget(health, area);
}
// ---------------------------------------------------------------------------
// Trading tab
// ---------------------------------------------------------------------------

491
bin/fxt/src/commands/watch/state.rs
View File

@@ -3,7 +3,7 @@
//! All mutable state lives here so that the render and event-loop modules
//! can borrow it without circular dependencies.
use std::collections::VecDeque;
use std::collections::{HashMap, VecDeque};
// ---------------------------------------------------------------------------
// Tab enum
@@ -47,20 +47,149 @@ impl Tab {
// Training types
// ---------------------------------------------------------------------------
/// A single in-progress training session.
#[derive(Debug, Clone)]
/// Returns `true` if the model name indicates an RL model (DQN or PPO).
pub fn is_rl_model(model: &str) -> bool {
let lower = model.to_ascii_lowercase();
lower.contains("dqn") || lower.contains("ppo")
}
/// A single in-progress training session (all 35 proto fields + local gpu_percent).
#[derive(Debug, Clone, Default)]
pub struct TrainingSession {
pub model: String,
pub fold: String,
pub is_hyperopt: bool,
// Epoch / progress
pub epoch: f32,
pub epoch_loss: f32,
pub validation_loss: f32,
pub learning_rate: f32,
pub gpu_percent: f32,
// Throughput
pub batches_per_second: f32,
pub batches_processed: f32,
pub iteration_seconds: f32,
// Eval metrics
pub eval_accuracy: f32,
pub eval_precision: f32,
pub eval_recall: f32,
pub eval_f1: f32,
// Checkpoint
pub checkpoint_size_bytes: f32,
pub checkpoint_saves: u32,
pub checkpoint_failures: u32,
// Health counters
pub nan_detected: u32,
pub gradient_explosions: u32,
pub feature_errors: u32,
// Hyperopt
pub hyperopt_trial_current: u32,
pub hyperopt_trial_total: u32,
pub hyperopt_best_objective: f32,
pub hyperopt_trials_failed: u32,
// RL diagnostics
pub q_value_mean: f32,
pub q_value_max: f32,
pub policy_entropy: f32,
pub kl_divergence: f32,
pub advantage_mean: f32,
pub replay_buffer_size: u32,
// Gradient & training health
pub gradient_norm: f32,
pub learning_rate: f32,
pub epoch_duration_seconds: f32,
// Hyperopt intra-trial
pub hyperopt_trial_epoch: u32,
pub hyperopt_trial_best_loss: f32,
pub hyperopt_elapsed_seconds: f32,
// Local (not from proto)
pub gpu_percent: f32,
}
/// Whether the Training tab shows the session list or a single-session detail.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum TrainingViewMode {
#[default]
List,
Detail,
}
/// Sub-tabs within the training detail view.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum DetailSubTab {
#[default]
Overview,
Loss,
RlDiagnostics,
Metrics,
Hyperopt,
Health,
}
impl DetailSubTab {
/// Human-readable label for the sub-tab bar.
pub fn title(self) -> &'static str {
match self {
Self::Overview => "Overview",
Self::Loss => "Loss",
Self::RlDiagnostics => "RL",
Self::Metrics => "Metrics",
Self::Hyperopt => "Hyperopt",
Self::Health => "Health",
}
}
}
/// Returns the sub-tabs visible for a given model/session configuration.
pub fn visible_sub_tabs(model: &str, is_hyperopt: bool) -> Vec<DetailSubTab> {
let mut tabs = vec![DetailSubTab::Overview, DetailSubTab::Loss];
if is_rl_model(model) {
tabs.push(DetailSubTab::RlDiagnostics);
} else {
tabs.push(DetailSubTab::Metrics);
}
if is_hyperopt {
tabs.push(DetailSubTab::Hyperopt);
}
tabs.push(DetailSubTab::Health);
tabs
}
/// Per-session unbounded metric history for sparklines.
#[derive(Debug, Clone, Default)]
pub struct SessionHistory {
pub loss: Vec<f64>,
pub val_loss: Vec<f64>,
pub learning_rate: Vec<f64>,
pub gradient_norm: Vec<f64>,
pub batches_per_second: Vec<f64>,
pub accuracy: Vec<f64>,
pub precision: Vec<f64>,
pub recall: Vec<f64>,
pub f1: Vec<f64>,
pub q_value_mean: Vec<f64>,
pub policy_entropy: Vec<f64>,
pub kl_divergence: Vec<f64>,
pub advantage_mean: Vec<f64>,
pub hyperopt_best_objective: Vec<f64>,
}
impl SessionHistory {
/// Append one sample from a training session snapshot.
pub fn push(&mut self, s: &TrainingSession) {
self.loss.push(f64::from(s.epoch_loss));
self.val_loss.push(f64::from(s.validation_loss));
self.learning_rate.push(f64::from(s.learning_rate));
self.gradient_norm.push(f64::from(s.gradient_norm));
self.batches_per_second.push(f64::from(s.batches_per_second));
self.accuracy.push(f64::from(s.eval_accuracy));
self.precision.push(f64::from(s.eval_precision));
self.recall.push(f64::from(s.eval_recall));
self.f1.push(f64::from(s.eval_f1));
self.q_value_mean.push(f64::from(s.q_value_mean));
self.policy_entropy.push(f64::from(s.policy_entropy));
self.kl_divergence.push(f64::from(s.kl_divergence));
self.advantage_mean.push(f64::from(s.advantage_mean));
self.hyperopt_best_objective.push(f64::from(s.hyperopt_best_objective));
}
}
/// GPU telemetry snapshot.
@@ -81,6 +210,11 @@ pub struct TrainingTabState {
pub loss_history: VecDeque<f64>,
pub scroll_offset: usize,
pub connected: bool,
// -- Detail view state --
pub selected_index: Option<usize>,
pub view_mode: TrainingViewMode,
pub detail_sub_tab: DetailSubTab,
pub session_histories: HashMap<String, SessionHistory>,
}
const MAX_LOSS_HISTORY: usize = 200;
@@ -101,6 +235,95 @@ impl TrainingTabState {
pub fn scroll_down(&mut self) {
self.scroll_offset = self.scroll_offset.saturating_add(1);
}
/// Move selection cursor up (list mode).
pub fn select_up(&mut self) {
if let Some(idx) = self.selected_index {
self.selected_index = Some(idx.saturating_sub(1));
}
}
/// Move selection cursor down (list mode).
pub fn select_down(&mut self) {
match self.selected_index {
Some(idx) => {
let max = self.sessions.len().saturating_sub(1);
if idx < max {
self.selected_index = Some(idx.saturating_add(1));
}
}
None => {
if !self.sessions.is_empty() {
self.selected_index = Some(0);
}
}
}
}
/// Enter detail view for the currently selected session.
pub fn enter_detail(&mut self) {
if self.selected_index.is_some() && !self.sessions.is_empty() {
self.view_mode = TrainingViewMode::Detail;
self.detail_sub_tab = DetailSubTab::Overview;
self.scroll_offset = 0;
}
}
/// Return to list view.
pub fn exit_detail(&mut self) {
self.view_mode = TrainingViewMode::List;
self.scroll_offset = 0;
}
/// Advance to the next visible sub-tab.
pub fn next_sub_tab(&mut self) {
let tabs = match self.selected_index.and_then(|i| self.sessions.get(i)) {
Some(s) => visible_sub_tabs(&s.model, s.is_hyperopt),
None => return,
};
if let Some(pos) = tabs.iter().position(|t| *t == self.detail_sub_tab) {
if let Some(&next) = tabs.get(pos + 1) {
self.detail_sub_tab = next;
}
}
}
/// Go back to the previous visible sub-tab.
pub fn prev_sub_tab(&mut self) {
let tabs = match self.selected_index.and_then(|i| self.sessions.get(i)) {
Some(s) => visible_sub_tabs(&s.model, s.is_hyperopt),
None => return,
};
if let Some(pos) = tabs.iter().position(|t| *t == self.detail_sub_tab) {
if let Some(&prev) = pos.checked_sub(1).and_then(|p| tabs.get(p)) {
self.detail_sub_tab = prev;
}
}
}
/// The currently selected session, if any.
pub fn selected_session(&self) -> Option<&TrainingSession> {
self.selected_index.and_then(|i| self.sessions.get(i))
}
/// Sub-tabs visible for the currently selected session.
pub fn visible_tabs(&self) -> Vec<DetailSubTab> {
match self.selected_session() {
Some(s) => visible_sub_tabs(&s.model, s.is_hyperopt),
None => vec![DetailSubTab::Overview],
}
}
/// Clamp selected_index after sessions list changes.
pub fn clamp_selection(&mut self) {
match (self.sessions.is_empty(), self.selected_index) {
(true, _) => self.selected_index = None,
(false, Some(idx)) if idx >= self.sessions.len() => {
self.selected_index = Some(self.sessions.len().saturating_sub(1));
}
_ => {}
}
}
}
// ---------------------------------------------------------------------------
@@ -362,4 +585,262 @@ mod tests {
state.scroll_up();
assert_eq!(state.training.scroll_offset, 0);
}
// -- is_rl_model --------------------------------------------------------
#[test]
fn test_is_rl_model() {
assert!(is_rl_model("DQN"));
assert!(is_rl_model("dqn"));
assert!(is_rl_model("PPO"));
assert!(is_rl_model("ContinuousPPO"));
assert!(!is_rl_model("TFT"));
assert!(!is_rl_model("Mamba2"));
assert!(!is_rl_model("XLSTM"));
}
// -- visible_sub_tabs ---------------------------------------------------
#[test]
fn test_visible_sub_tabs_rl() {
let tabs = visible_sub_tabs("DQN", false);
assert_eq!(
tabs,
vec![
DetailSubTab::Overview,
DetailSubTab::Loss,
DetailSubTab::RlDiagnostics,
DetailSubTab::Health,
]
);
}
#[test]
fn test_visible_sub_tabs_supervised() {
let tabs = visible_sub_tabs("TFT", false);
assert_eq!(
tabs,
vec![
DetailSubTab::Overview,
DetailSubTab::Loss,
DetailSubTab::Metrics,
DetailSubTab::Health,
]
);
}
#[test]
fn test_visible_sub_tabs_hyperopt_rl() {
let tabs = visible_sub_tabs("PPO", true);
assert_eq!(
tabs,
vec![
DetailSubTab::Overview,
DetailSubTab::Loss,
DetailSubTab::RlDiagnostics,
DetailSubTab::Hyperopt,
DetailSubTab::Health,
]
);
}
#[test]
fn test_visible_sub_tabs_hyperopt_supervised() {
let tabs = visible_sub_tabs("TFT", true);
assert_eq!(
tabs,
vec![
DetailSubTab::Overview,
DetailSubTab::Loss,
DetailSubTab::Metrics,
DetailSubTab::Hyperopt,
DetailSubTab::Health,
]
);
}
// -- SessionHistory::push -----------------------------------------------
#[test]
fn test_session_history_push() {
let mut hist = SessionHistory::default();
let session = TrainingSession {
epoch_loss: 0.5,
validation_loss: 0.6,
learning_rate: 1e-3,
gradient_norm: 1.2,
batches_per_second: 42.0,
eval_accuracy: 0.85,
eval_f1: 0.82,
q_value_mean: 3.5,
policy_entropy: 1.1,
kl_divergence: 0.02,
advantage_mean: 0.1,
hyperopt_best_objective: 0.45,
..Default::default()
};
hist.push(&session);
hist.push(&session);
assert_eq!(hist.loss.len(), 2);
assert!((hist.loss[0] - 0.5).abs() < f64::EPSILON);
assert!((hist.accuracy[0] - 0.85).abs() < 1e-6);
assert!((hist.q_value_mean[0] - 3.5).abs() < 1e-6);
}
// -- select_up / select_down --------------------------------------------
fn make_tab_with_sessions(n: usize) -> TrainingTabState {
let mut tab = TrainingTabState::default();
for i in 0..n {
tab.sessions.push(TrainingSession {
model: format!("DQN_{i}"),
fold: format!("fold_{i}"),
..Default::default()
});
}
tab
}
#[test]
fn test_select_down_from_none() {
let mut tab = make_tab_with_sessions(3);
assert_eq!(tab.selected_index, None);
tab.select_down();
assert_eq!(tab.selected_index, Some(0));
}
#[test]
fn test_select_down_clamps_at_end() {
let mut tab = make_tab_with_sessions(3);
tab.selected_index = Some(2);
tab.select_down();
assert_eq!(tab.selected_index, Some(2));
}
#[test]
fn test_select_up_clamps_at_zero() {
let mut tab = make_tab_with_sessions(3);
tab.selected_index = Some(0);
tab.select_up();
assert_eq!(tab.selected_index, Some(0));
}
#[test]
fn test_select_down_empty_sessions() {
let mut tab = TrainingTabState::default();
tab.select_down();
assert_eq!(tab.selected_index, None);
}
#[test]
fn test_select_up_down_navigation() {
let mut tab = make_tab_with_sessions(5);
tab.selected_index = Some(0);
tab.select_down();
tab.select_down();
assert_eq!(tab.selected_index, Some(2));
tab.select_up();
assert_eq!(tab.selected_index, Some(1));
}
// -- mode transitions ---------------------------------------------------
#[test]
fn test_enter_detail_with_selection() {
let mut tab = make_tab_with_sessions(3);
tab.selected_index = Some(1);
tab.enter_detail();
assert_eq!(tab.view_mode, TrainingViewMode::Detail);
assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview);
}
#[test]
fn test_enter_detail_without_selection_noop() {
let mut tab = make_tab_with_sessions(3);
tab.enter_detail();
assert_eq!(tab.view_mode, TrainingViewMode::List);
}
#[test]
fn test_exit_detail() {
let mut tab = make_tab_with_sessions(3);
tab.selected_index = Some(0);
tab.enter_detail();
tab.scroll_offset = 5;
tab.exit_detail();
assert_eq!(tab.view_mode, TrainingViewMode::List);
assert_eq!(tab.scroll_offset, 0);
}
// -- sub-tab navigation -------------------------------------------------
#[test]
fn test_next_prev_sub_tab() {
let mut tab = make_tab_with_sessions(1);
tab.selected_index = Some(0);
tab.enter_detail();
// DQN -> Overview, Loss, RL, Health
assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview);
tab.next_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::Loss);
tab.next_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::RlDiagnostics);
tab.next_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::Health);
// At the end — should stay
tab.next_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::Health);
// Go back
tab.prev_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::RlDiagnostics);
tab.prev_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::Loss);
tab.prev_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview);
// At start — should stay
tab.prev_sub_tab();
assert_eq!(tab.detail_sub_tab, DetailSubTab::Overview);
}
// -- clamp_selection ----------------------------------------------------
#[test]
fn test_clamp_selection_empty() {
let mut tab = TrainingTabState::default();
tab.selected_index = Some(5);
tab.clamp_selection();
assert_eq!(tab.selected_index, None);
}
#[test]
fn test_clamp_selection_out_of_bounds() {
let mut tab = make_tab_with_sessions(3);
tab.selected_index = Some(10);
tab.clamp_selection();
assert_eq!(tab.selected_index, Some(2));
}
#[test]
fn test_clamp_selection_valid() {
let mut tab = make_tab_with_sessions(3);
tab.selected_index = Some(1);
tab.clamp_selection();
assert_eq!(tab.selected_index, Some(1));
}
// -- DetailSubTab::title ------------------------------------------------
#[test]
fn test_detail_sub_tab_titles() {
assert_eq!(DetailSubTab::Overview.title(), "Overview");
assert_eq!(DetailSubTab::Loss.title(), "Loss");
assert_eq!(DetailSubTab::RlDiagnostics.title(), "RL");
assert_eq!(DetailSubTab::Metrics.title(), "Metrics");
assert_eq!(DetailSubTab::Hyperopt.title(), "Hyperopt");
assert_eq!(DetailSubTab::Health.title(), "Health");
}
}

52
bin/fxt/src/commands/watch/streams.rs
View File

@@ -83,9 +83,7 @@ pub(super) enum StreamEvent {
/// Create a lazily-connected gRPC channel to the given URL.
fn connect_channel(url: &str) -> Result<Channel> {
Ok(Channel::from_shared(url.to_owned())
.context("invalid gRPC endpoint URL")?
.connect_lazy())
crate::client::connect_channel_lazy(url)
}
/// Parse a JWT string into a tonic `MetadataValue` suitable for the
@@ -123,14 +121,49 @@ fn convert_training_session(s: &mon::TrainingSession) -> TrainingSession {
model: s.model.clone(),
fold: s.fold.clone(),
is_hyperopt: s.is_hyperopt,
// Epoch / progress
epoch: s.current_epoch,
epoch_loss: s.epoch_loss,
validation_loss: s.validation_loss,
learning_rate: s.learning_rate,
gpu_percent: 0.0, // per-session GPU not in proto; filled from GpuSnapshot
// Throughput
batches_per_second: s.batches_per_second,
batches_processed: s.batches_processed,
iteration_seconds: s.iteration_seconds,
// Eval metrics
eval_accuracy: s.eval_accuracy,
eval_precision: s.eval_precision,
eval_recall: s.eval_recall,
eval_f1: s.eval_f1,
// Checkpoint
checkpoint_size_bytes: s.checkpoint_size_bytes,
checkpoint_saves: s.checkpoint_saves,
checkpoint_failures: s.checkpoint_failures,
// Health counters
nan_detected: s.nan_detected,
gradient_explosions: s.gradient_explosions,
feature_errors: s.feature_errors,
// Hyperopt
hyperopt_trial_current: s.hyperopt_trial_current,
hyperopt_trial_total: s.hyperopt_trial_total,
hyperopt_best_objective: s.hyperopt_best_objective,
hyperopt_trials_failed: s.hyperopt_trials_failed,
// RL diagnostics
q_value_mean: s.q_value_mean,
q_value_max: s.q_value_max,
policy_entropy: s.policy_entropy,
kl_divergence: s.kl_divergence,
advantage_mean: s.advantage_mean,
replay_buffer_size: s.replay_buffer_size,
// Gradient & training health
gradient_norm: s.gradient_norm,
learning_rate: s.learning_rate,
epoch_duration_seconds: s.epoch_duration_seconds,
// Hyperopt intra-trial
hyperopt_trial_epoch: s.hyperopt_trial_epoch,
hyperopt_trial_best_loss: s.hyperopt_trial_best_loss,
hyperopt_elapsed_seconds: s.hyperopt_elapsed_seconds,
// Local
gpu_percent: 0.0, // per-session GPU not in proto; filled from GpuSnapshot
}
}
@@ -503,9 +536,16 @@ async fn try_stream_system_status(
while let Some(frame) = stream.next().await {
let heartbeat = frame.context("system status stream message error")?;
let status_label = match heartbeat.status {
1 => "HEALTHY",
2 => "DEGRADED",
3 => "UNHEALTHY",
4 => "CRITICAL",
_ => "UNKNOWN",
};
tx.send(StreamEvent::SystemStatus {
service: heartbeat.service_name,
status: format!("{}", heartbeat.status),
status: status_label.to_owned(),
message: heartbeat.message,
})
.await

40
bin/fxt/src/main.rs
View File

@@ -81,7 +81,7 @@ use zeroize as _;
about = "Foxhunt Trading System Terminal Interface",
long_about = "Foxhunt Trading System Terminal Interface\n\n\
Environment Variables:\n\
API_GATEWAY_URL API Gateway URL (default: http://localhost:50051)\n\
API_GATEWAY_URL API Gateway URL (default: https://api.fxhnt.ai)\n\
TLI_LOG_LEVEL Log level (default: info)\n\
TLI_TOKEN_STORAGE Token storage backend (default: keyring)\n\n\
Precedence: CLI args > Environment variables > Config file > Defaults"
@@ -95,7 +95,7 @@ struct Cli {
#[clap(
long,
env = "API_GATEWAY_URL",
default_value = "http://localhost:50051",
default_value = "https://api.fxhnt.ai",
help = "API Gateway URL (env: API_GATEWAY_URL)"
)]
api_gateway_url: String,
@@ -266,7 +266,6 @@ struct Claims {
async fn load_jwt_token(api_gateway_url: &str) -> Result<String> {
use fxt::auth::login::LoginClient;
use fxt::auth::token_manager::{AuthTokenManager, TokenStorage};
use tonic::transport::Channel;
let storage = FileTokenStorage::new().context("Failed to initialize file token storage")?;
@@ -292,9 +291,8 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result<String> {
// Refresh tokens
let auth_manager = AuthTokenManager::new(storage.clone());
let channel = Channel::from_shared(api_gateway_url.to_owned())
.context("Invalid API Gateway URL")?
.connect_lazy();
let channel = fxt::client::connect_channel_lazy(api_gateway_url)
.context("Invalid API Gateway URL")?;
let login_client = LoginClient::new(channel);
@@ -400,7 +398,7 @@ async fn main() -> Result<()> {
let mut cli = Cli::parse();
// Merge: CLI args override config file (precedence: CLI > Config > Default)
if cli.api_gateway_url == "http://localhost:50051" {
if cli.api_gateway_url == "https://api.fxhnt.ai" {
// Using default, check if config has override
cli.api_gateway_url = config.api_gateway_url.clone();
}
@@ -420,11 +418,29 @@ async fn main() -> Result<()> {
_ => Level::INFO,
};
// Initialize tracing with configured log level
let subscriber = FmtSubscriber::builder().with_max_level(log_level).finish();
// Initialize tracing with configured log level.
// For the Watch (TUI) command, redirect logs to a file so they don't
// corrupt the ratatui alternate-screen display.
let is_tui = matches!(cli.command, Commands::Watch);
if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
eprintln!("Warning: Failed to set global tracing subscriber: {}", e);
if is_tui {
let log_dir = dirs::home_dir()
.unwrap_or_default()
.join(".foxhunt");
std::fs::create_dir_all(&log_dir).ok();
if let Ok(file) = std::fs::File::create(log_dir.join("watch.log")) {
let subscriber = FmtSubscriber::builder()
.with_max_level(log_level)
.with_writer(std::sync::Mutex::new(file))
.with_ansi(false)
.finish();
tracing::subscriber::set_global_default(subscriber).ok();
}
} else {
let subscriber = FmtSubscriber::builder().with_max_level(log_level).finish();
if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
eprintln!("Warning: Failed to set global tracing subscriber: {}", e);
}
}
// Route to subcommands
@@ -514,7 +530,7 @@ mod tests {
// Test default values when no flags provided
let cli = Cli::parse_from(&["fxt", "auth", "status"]);
assert_eq!(cli.api_gateway_url, "http://localhost:50051");
assert_eq!(cli.api_gateway_url, "https://api.fxhnt.ai");
assert_eq!(cli.log_level, "info");
assert_eq!(cli.token_storage, "keyring");
}