48ca3db92f
Add Training cockpit sub-tabs (Live/History) with expandable per-epoch detail panel showing financial metrics, RL diagnostics, and gradient health. History tab fetches completed/failed runs from ListTrainingJobs RPC. Add scrollable pod table to Pods cockpit and compact pods summary to Overview bottom row. Fix K8s API egress in api-gateway NetworkPolicy (monitoring-service→prometheus, apply existing K8s API CIDR rules). Add 62 new unit tests across event_loop (51) and data_fetcher (11) covering all StateUpdate variants, ring buffer eviction, epoch history accumulation, pod scroll helpers, and navigation edge cases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1709 lines
60 KiB
Rust
1709 lines
60 KiB
Rust
#![allow(clippy::indexing_slicing)] // ratatui Layout::split indices match constraints
|
|
//! TUI event loop -- terminal setup, input handling, rendering.
|
|
|
|
use std::io::{self, Stdout};
|
|
use std::time::Duration;
|
|
|
|
use crossterm::event::{self, Event, KeyCode, KeyModifiers};
|
|
use crossterm::terminal::{
|
|
EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode,
|
|
};
|
|
use crossterm::ExecutableCommand;
|
|
use ratatui::Terminal;
|
|
use ratatui::backend::CrosstermBackend;
|
|
use ratatui::layout::{Constraint, Direction, Layout, Rect};
|
|
use ratatui::style::{Modifier, Style};
|
|
use ratatui::text::{Line, Span};
|
|
use ratatui::widgets::{Block, Borders, Clear, Paragraph, Wrap};
|
|
use tokio::sync::mpsc;
|
|
use tokio_util::sync::CancellationToken;
|
|
use tonic::transport::Channel;
|
|
|
|
use crate::auth::login::LoginClient;
|
|
use crate::auth::token_manager::{AuthTokenManager, FileTokenStorage};
|
|
use crate::grpc::FoxhuntClient;
|
|
|
|
use super::cockpit::Cockpit;
|
|
use super::cockpits::data::DataCockpit;
|
|
use super::cockpits::overview::OverviewCockpit;
|
|
use super::cockpits::pods::PodsCockpit;
|
|
use super::cockpits::risk::RiskCockpit;
|
|
use super::cockpits::services::ServicesCockpit;
|
|
use super::cockpits::trading::TradingCockpit;
|
|
use super::cockpits::training::TrainingCockpit;
|
|
use super::data_fetcher::{DataFetcher, StateUpdate};
|
|
use super::state::{AppState, ConnectionStatus, EpochSnapshot, TrainingTab};
|
|
use super::theme;
|
|
|
|
/// Number of cockpit views.
|
|
const COCKPIT_COUNT: usize = 7;
|
|
|
|
/// Tick interval for the event loop (1 second).
|
|
const TICK_RATE: Duration = Duration::from_secs(1);
|
|
|
|
/// Run the TUI event loop with live gRPC data.
|
|
pub async fn run(api_url: &str) -> anyhow::Result<()> {
|
|
// Connect to the API Gateway (lazy — no TCP handshake until first RPC).
|
|
let client = FoxhuntClient::connect(api_url).await?;
|
|
|
|
// Ensure we have a valid JWT before starting streams.
|
|
ensure_auth(&client).await;
|
|
|
|
// Create data fetcher — spawns background stream tasks.
|
|
let (fetcher, rx) = DataFetcher::new();
|
|
let cancel = fetcher.cancel_token();
|
|
fetcher.start(&client);
|
|
|
|
// Spawn periodic token refresh (JWT expires after 1h, refresh at 45m).
|
|
spawn_token_refresher(client.channel(), cancel.clone());
|
|
|
|
// Setup terminal
|
|
let mut terminal = setup_terminal()?;
|
|
let result = event_loop(&mut terminal, rx).await;
|
|
|
|
// Signal all stream tasks to stop.
|
|
cancel.cancel();
|
|
|
|
// Always restore terminal, even on error
|
|
if let Err(e) = teardown_terminal(&mut terminal) {
|
|
eprintln!("terminal teardown failed: {e}");
|
|
}
|
|
result
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Token refresh
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Ensure we have a non-expired access token before opening streams.
|
|
///
|
|
/// If the stored token is expired, performs a silent refresh using the locally
|
|
/// configured JWT secret (from `~/.foxhunt/config.toml` or `JWT_SECRET` env).
|
|
async fn ensure_auth(client: &FoxhuntClient) {
|
|
let Ok(storage) = FileTokenStorage::new() else {
|
|
return;
|
|
};
|
|
let manager = AuthTokenManager::new(storage);
|
|
|
|
// Already have a valid token — nothing to do.
|
|
if manager.has_valid_token().await {
|
|
return;
|
|
}
|
|
|
|
// Attempt silent refresh (generates a fresh JWT locally).
|
|
let login = LoginClient::new(client.channel());
|
|
match login.silent_login(&manager).await {
|
|
Ok(true) => tracing::info!("Token refreshed for TUI streams"),
|
|
Ok(false) => tracing::warn!("No refresh token - streams will fail auth"),
|
|
Err(e) => tracing::warn!("Token refresh failed: {e}"),
|
|
}
|
|
}
|
|
|
|
/// Periodic token refresh — runs every 45 min in the background.
|
|
///
|
|
/// If the stored token is within 60 s of expiry (or already expired),
|
|
/// a silent refresh regenerates it so reconnecting streams pick it up.
|
|
fn spawn_token_refresher(channel: Channel, cancel: CancellationToken) {
|
|
tokio::spawn(async move {
|
|
// 45 minutes — well within the 1 h gateway token lifetime.
|
|
let mut interval = tokio::time::interval(Duration::from_secs(45 * 60));
|
|
interval.tick().await; // skip the immediate first tick
|
|
loop {
|
|
tokio::select! {
|
|
() = cancel.cancelled() => return,
|
|
_ = interval.tick() => {
|
|
let Ok(storage) = FileTokenStorage::new() else { continue };
|
|
let manager = AuthTokenManager::new(storage);
|
|
if manager.has_valid_token().await {
|
|
continue;
|
|
}
|
|
let login = LoginClient::new(channel.clone());
|
|
match login.silent_login(&manager).await {
|
|
Ok(true) => tracing::info!("Token auto-refreshed"),
|
|
Ok(false) => tracing::warn!("Token refresh: no refresh token"),
|
|
Err(e) => tracing::warn!("Token auto-refresh failed: {e}"),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Terminal lifecycle
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn setup_terminal() -> anyhow::Result<Terminal<CrosstermBackend<Stdout>>> {
|
|
enable_raw_mode()?;
|
|
io::stdout().execute(EnterAlternateScreen)?;
|
|
let backend = CrosstermBackend::new(io::stdout());
|
|
let terminal = Terminal::new(backend)?;
|
|
Ok(terminal)
|
|
}
|
|
|
|
fn teardown_terminal(
|
|
terminal: &mut Terminal<CrosstermBackend<Stdout>>,
|
|
) -> anyhow::Result<()> {
|
|
disable_raw_mode()?;
|
|
terminal.backend_mut().execute(LeaveAlternateScreen)?;
|
|
terminal.show_cursor()?;
|
|
Ok(())
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Event loop
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async fn event_loop(
|
|
terminal: &mut Terminal<CrosstermBackend<Stdout>>,
|
|
mut rx: mpsc::UnboundedReceiver<StateUpdate>,
|
|
) -> anyhow::Result<()> {
|
|
let mut state = AppState::default();
|
|
|
|
let cockpits: Vec<Box<dyn Cockpit>> = vec![
|
|
Box::new(OverviewCockpit),
|
|
Box::new(TrainingCockpit),
|
|
Box::new(TradingCockpit),
|
|
Box::new(ServicesCockpit),
|
|
Box::new(RiskCockpit),
|
|
Box::new(DataCockpit),
|
|
Box::new(PodsCockpit),
|
|
];
|
|
|
|
loop {
|
|
// Drain all pending updates from the data fetcher.
|
|
while let Ok(update) = rx.try_recv() {
|
|
apply_update(&mut state, update);
|
|
}
|
|
|
|
// Render
|
|
terminal.draw(|frame| {
|
|
let size = frame.area();
|
|
|
|
let layout = Layout::default()
|
|
.direction(Direction::Vertical)
|
|
.constraints([
|
|
Constraint::Length(1), // tab bar
|
|
Constraint::Min(0), // cockpit body
|
|
Constraint::Length(1), // status bar
|
|
])
|
|
.split(size);
|
|
|
|
render_tab_bar(frame, layout[0], &cockpits, state.current_cockpit);
|
|
|
|
if let Some(cockpit) = cockpits.get(state.current_cockpit) {
|
|
cockpit.render(frame, layout[1], &state);
|
|
}
|
|
|
|
render_status_bar(frame, layout[2], &state);
|
|
|
|
if state.show_help {
|
|
render_help_overlay(frame, size);
|
|
}
|
|
})?;
|
|
|
|
// Poll for events with timeout
|
|
if crossterm::event::poll(TICK_RATE)? {
|
|
if let Event::Key(key) = event::read()? {
|
|
#[allow(clippy::wildcard_enum_match_arm)]
|
|
match key.code {
|
|
// Quit
|
|
KeyCode::Char('q') | KeyCode::Esc => break,
|
|
KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => {
|
|
break;
|
|
}
|
|
|
|
// Cockpit switching (1-6)
|
|
KeyCode::Char(c @ '1'..='7') => {
|
|
let idx = (c as usize).saturating_sub('1' as usize);
|
|
if idx < COCKPIT_COUNT {
|
|
state.current_cockpit = idx;
|
|
}
|
|
}
|
|
|
|
// Help toggle
|
|
KeyCode::Char('?') => {
|
|
state.show_help = !state.show_help;
|
|
}
|
|
|
|
// Refresh
|
|
KeyCode::Char('r') => {
|
|
state.last_update = std::time::Instant::now();
|
|
}
|
|
|
|
// Navigation: j/k/Down/Up for item selection
|
|
KeyCode::Char('j') | KeyCode::Down => {
|
|
handle_nav_down(&mut state);
|
|
}
|
|
KeyCode::Char('k') | KeyCode::Up => {
|
|
handle_nav_up(&mut state);
|
|
}
|
|
|
|
// Enter / Right: toggle detail expansion
|
|
KeyCode::Enter | KeyCode::Right => {
|
|
handle_enter(&mut state);
|
|
}
|
|
|
|
// Tab / Shift-Tab: switch sub-tabs in Training cockpit
|
|
KeyCode::Tab | KeyCode::BackTab => {
|
|
if state.current_cockpit == COCKPIT_TRAINING {
|
|
state.training_tab = match state.training_tab {
|
|
TrainingTab::Live => TrainingTab::History,
|
|
TrainingTab::History => TrainingTab::Live,
|
|
};
|
|
}
|
|
}
|
|
|
|
// l/h: alternative sub-tab switching
|
|
KeyCode::Char('l') => {
|
|
if state.current_cockpit == COCKPIT_TRAINING {
|
|
state.training_tab = TrainingTab::History;
|
|
}
|
|
}
|
|
KeyCode::Char('h') => {
|
|
if state.current_cockpit == COCKPIT_TRAINING {
|
|
state.training_tab = TrainingTab::Live;
|
|
}
|
|
}
|
|
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tick timestamp
|
|
state.last_update = std::time::Instant::now();
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Ring-buffer capacity for event lists.
|
|
const RING_CAP: usize = 50;
|
|
|
|
/// Apply a single [`StateUpdate`] to the shared [`AppState`].
|
|
fn apply_update(state: &mut AppState, update: StateUpdate) {
|
|
match update {
|
|
StateUpdate::TrainingMetrics {
|
|
mut sessions,
|
|
gpu,
|
|
resources,
|
|
active_jobs,
|
|
} => {
|
|
// Drain old sessions into a map so we can move epoch histories.
|
|
let mut old_map: std::collections::HashMap<String, _> = state
|
|
.training_sessions
|
|
.drain(..)
|
|
.map(|s| (s.model.clone(), s))
|
|
.collect();
|
|
|
|
for new_s in &mut sessions {
|
|
if let Some(mut old_s) = old_map.remove(&new_s.model) {
|
|
// If epoch advanced, snapshot the old epoch metrics.
|
|
if new_s.epoch > old_s.epoch {
|
|
old_s.epoch_history.push(EpochSnapshot {
|
|
epoch: old_s.epoch,
|
|
loss: old_s.loss,
|
|
val_loss: old_s.val_loss,
|
|
sharpe: old_s.sharpe,
|
|
sortino: old_s.sortino,
|
|
win_rate: old_s.win_rate,
|
|
max_dd: old_s.max_dd,
|
|
profit_factor: old_s.profit_factor,
|
|
total_return: old_s.total_return,
|
|
total_trades: old_s.total_trades,
|
|
q_value_mean: old_s.q_value_mean,
|
|
gradient_norm: old_s.gradient_norm,
|
|
learning_rate: old_s.learning_rate,
|
|
policy_entropy: old_s.policy_entropy,
|
|
action_buy_pct: old_s.action_buy_pct,
|
|
action_sell_pct: old_s.action_sell_pct,
|
|
action_hold_pct: old_s.action_hold_pct,
|
|
});
|
|
}
|
|
// Move accumulated history to the new session.
|
|
new_s.epoch_history = old_s.epoch_history;
|
|
}
|
|
}
|
|
state.training_sessions = sessions;
|
|
state.gpu = gpu;
|
|
state.system_resources = resources;
|
|
state.active_jobs = active_jobs;
|
|
}
|
|
StateUpdate::Services(services) => {
|
|
state.services = services;
|
|
}
|
|
StateUpdate::Account(acct) => {
|
|
state.account = acct;
|
|
}
|
|
StateUpdate::Positions(positions) => {
|
|
state.positions = positions;
|
|
}
|
|
StateUpdate::Execution(exec) => {
|
|
if state.executions.len() >= RING_CAP {
|
|
state.executions.pop_front();
|
|
}
|
|
state.executions.push_back(exec);
|
|
}
|
|
StateUpdate::Order(order) => {
|
|
if state.orders.len() >= RING_CAP {
|
|
state.orders.pop_front();
|
|
}
|
|
state.orders.push_back(order);
|
|
}
|
|
StateUpdate::Risk(risk) => {
|
|
// Preserve circuit breakers from separate stream.
|
|
let cbs = std::mem::take(&mut state.risk.circuit_breakers);
|
|
state.risk = risk;
|
|
state.risk.circuit_breakers = cbs;
|
|
}
|
|
StateUpdate::CircuitBreakers(cbs) => {
|
|
state.risk.circuit_breakers = cbs;
|
|
}
|
|
StateUpdate::RiskAlert(alert) => {
|
|
if state.risk_alerts.len() >= RING_CAP {
|
|
state.risk_alerts.pop_front();
|
|
}
|
|
state.risk_alerts.push_back(alert);
|
|
}
|
|
StateUpdate::ModelStatuses(statuses) => {
|
|
state.model_statuses = statuses;
|
|
}
|
|
StateUpdate::Alert(alert) => {
|
|
if state.alerts.len() >= RING_CAP {
|
|
state.alerts.pop_front();
|
|
}
|
|
state.alerts.push_back(alert);
|
|
}
|
|
StateUpdate::DataFeeds(feeds) => {
|
|
state.data_feeds = feeds;
|
|
}
|
|
StateUpdate::DataCache(cache) => {
|
|
state.data_cache = cache;
|
|
}
|
|
StateUpdate::Portfolio {
|
|
equity,
|
|
cash,
|
|
margin_used,
|
|
daily_pnl,
|
|
total_pnl,
|
|
} => {
|
|
state.account.equity = equity;
|
|
state.account.cash = cash;
|
|
state.account.margin_used = margin_used;
|
|
state.account.daily_pnl = daily_pnl;
|
|
state.account.total_pnl = total_pnl;
|
|
}
|
|
StateUpdate::Pods(pods) => {
|
|
state.pods = pods;
|
|
}
|
|
StateUpdate::TrainingHistory(entries) => {
|
|
state.training_history = entries;
|
|
}
|
|
StateUpdate::Connection(status) => {
|
|
state.connection_status = status;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Cockpit-aware navigation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Training cockpit index.
|
|
const COCKPIT_TRAINING: usize = 1;
|
|
/// Pods cockpit index.
|
|
const COCKPIT_PODS: usize = 6;
|
|
|
|
fn handle_nav_down(state: &mut AppState) {
|
|
match state.current_cockpit {
|
|
COCKPIT_TRAINING => match state.training_tab {
|
|
TrainingTab::Live => {
|
|
let count = state.training_sessions.len();
|
|
if count == 0 {
|
|
return;
|
|
}
|
|
state.selected_training_session = Some(match state.selected_training_session {
|
|
Some(i) if i + 1 < count => i + 1,
|
|
Some(_) => 0, // wrap
|
|
None => 0,
|
|
});
|
|
}
|
|
TrainingTab::History => {
|
|
let count = state.training_history.len();
|
|
if count == 0 {
|
|
return;
|
|
}
|
|
state.selected_history_entry = Some(match state.selected_history_entry {
|
|
Some(i) if i + 1 < count => i + 1,
|
|
Some(_) => 0,
|
|
None => 0,
|
|
});
|
|
}
|
|
},
|
|
COCKPIT_PODS => {
|
|
let count = state.pods.len();
|
|
if count == 0 {
|
|
return;
|
|
}
|
|
state.selected_pod_group = Some(match state.selected_pod_group {
|
|
Some(i) if i + 1 < count => i + 1,
|
|
Some(_) => 0,
|
|
None => 0,
|
|
});
|
|
scroll_pods_into_view(state);
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
fn handle_nav_up(state: &mut AppState) {
|
|
match state.current_cockpit {
|
|
COCKPIT_TRAINING => match state.training_tab {
|
|
TrainingTab::Live => {
|
|
let count = state.training_sessions.len();
|
|
if count == 0 {
|
|
return;
|
|
}
|
|
state.selected_training_session = Some(match state.selected_training_session {
|
|
Some(0) => count.saturating_sub(1), // wrap
|
|
Some(i) => i.saturating_sub(1),
|
|
None => 0,
|
|
});
|
|
}
|
|
TrainingTab::History => {
|
|
let count = state.training_history.len();
|
|
if count == 0 {
|
|
return;
|
|
}
|
|
state.selected_history_entry = Some(match state.selected_history_entry {
|
|
Some(0) => count.saturating_sub(1),
|
|
Some(i) => i.saturating_sub(1),
|
|
None => 0,
|
|
});
|
|
}
|
|
},
|
|
COCKPIT_PODS => {
|
|
let count = state.pods.len();
|
|
if count == 0 {
|
|
return;
|
|
}
|
|
state.selected_pod_group = Some(match state.selected_pod_group {
|
|
Some(0) => count.saturating_sub(1),
|
|
Some(i) => i.saturating_sub(1),
|
|
None => 0,
|
|
});
|
|
scroll_pods_into_view(state);
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
/// Compute the flat row index where a pod group header starts.
|
|
/// Each group contributes 1 header + (N sub-rows if N > 1).
|
|
fn pod_group_flat_row(pods: &[crate::tui::state::PodGroupData], group_idx: usize) -> usize {
|
|
let mut row = 0;
|
|
for (i, g) in pods.iter().enumerate() {
|
|
if i == group_idx {
|
|
return row;
|
|
}
|
|
row += 1; // group header
|
|
if g.pods.len() > 1 {
|
|
row += g.pods.len(); // sub-rows
|
|
}
|
|
}
|
|
row
|
|
}
|
|
|
|
/// Ensure the selected pod group is visible by adjusting scroll offset.
|
|
/// Uses a conservative 35-row visible window (typical terminal height minus chrome).
|
|
fn scroll_pods_into_view(state: &mut AppState) {
|
|
const VISIBLE_ROWS: usize = 35;
|
|
if let Some(idx) = state.selected_pod_group {
|
|
let flat = pod_group_flat_row(&state.pods, idx);
|
|
if flat < state.pod_scroll_offset {
|
|
state.pod_scroll_offset = flat;
|
|
} else if flat >= state.pod_scroll_offset + VISIBLE_ROWS {
|
|
state.pod_scroll_offset = flat.saturating_sub(VISIBLE_ROWS) + 1;
|
|
} else {
|
|
// Already visible — no scroll adjustment needed.
|
|
}
|
|
}
|
|
}
|
|
|
|
fn handle_enter(state: &mut AppState) {
|
|
match state.current_cockpit {
|
|
COCKPIT_TRAINING => {
|
|
if state.training_tab == TrainingTab::Live
|
|
&& state.selected_training_session.is_some()
|
|
{
|
|
state.training_detail_expanded = !state.training_detail_expanded;
|
|
}
|
|
// History tab: Enter currently just keeps selection visible in detail panel
|
|
}
|
|
COCKPIT_PODS => {
|
|
if state.selected_pod_group.is_some() {
|
|
state.pod_group_expanded = !state.pod_group_expanded;
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Chrome renderers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn render_tab_bar(
|
|
frame: &mut ratatui::Frame,
|
|
area: Rect,
|
|
cockpits: &[Box<dyn Cockpit>],
|
|
active: usize,
|
|
) {
|
|
let mut spans = Vec::new();
|
|
spans.push(Span::styled(" FXT ", Style::default().fg(theme::PRIMARY).add_modifier(Modifier::BOLD)));
|
|
spans.push(Span::styled(" | ", theme::muted_style()));
|
|
|
|
for (i, cockpit) in cockpits.iter().enumerate() {
|
|
let num = format!("{}", i + 1);
|
|
if i == active {
|
|
spans.push(Span::styled(
|
|
format!("[{num}] {}", cockpit.name()),
|
|
Style::default()
|
|
.fg(theme::HIGHLIGHT)
|
|
.add_modifier(Modifier::BOLD),
|
|
));
|
|
} else {
|
|
spans.push(Span::styled(
|
|
format!(" {num} {}", cockpit.name()),
|
|
theme::muted_style(),
|
|
));
|
|
}
|
|
if i + 1 < cockpits.len() {
|
|
spans.push(Span::styled(" ", theme::muted_style()));
|
|
}
|
|
}
|
|
|
|
let tab_line = Line::from(spans);
|
|
let paragraph = Paragraph::new(tab_line);
|
|
frame.render_widget(paragraph, area);
|
|
}
|
|
|
|
fn render_status_bar(frame: &mut ratatui::Frame, area: Rect, state: &AppState) {
|
|
let elapsed = state.last_update.elapsed();
|
|
let ago = if elapsed.as_secs() < 2 {
|
|
"just now".to_owned()
|
|
} else {
|
|
format!("{}s ago", elapsed.as_secs())
|
|
};
|
|
|
|
let (conn_label, conn_color) = match &state.connection_status {
|
|
ConnectionStatus::Connecting => ("CONNECTING", theme::WARNING),
|
|
ConnectionStatus::Connected => ("LIVE", theme::SUCCESS),
|
|
ConnectionStatus::Reconnecting { attempt } => {
|
|
let _ = attempt; // shown implicitly via color
|
|
("RECONNECTING", theme::WARNING)
|
|
}
|
|
ConnectionStatus::Error(msg) => {
|
|
let _ = msg;
|
|
("OFFLINE", theme::ERROR)
|
|
}
|
|
};
|
|
|
|
let mut spans = vec![
|
|
Span::styled(
|
|
format!(" [{conn_label}] "),
|
|
Style::default().fg(conn_color).add_modifier(Modifier::BOLD),
|
|
),
|
|
Span::styled(" q", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled(" quit ", theme::muted_style()),
|
|
Span::styled("?", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled(" help ", theme::muted_style()),
|
|
Span::styled("j/k", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled(" nav ", theme::muted_style()),
|
|
Span::styled("\u{23ce}", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled(" detail ", theme::muted_style()),
|
|
Span::styled("1-7", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled(" cockpit ", theme::muted_style()),
|
|
Span::styled(
|
|
format!("Updated: {ago}"),
|
|
theme::muted_style(),
|
|
),
|
|
];
|
|
|
|
// Show CPU/GPU summary inline on Training cockpit
|
|
if state.current_cockpit == COCKPIT_TRAINING && state.gpu.utilization > 0.0 {
|
|
spans.push(Span::styled(" ", theme::muted_style()));
|
|
spans.push(Span::styled(
|
|
format!(
|
|
"GPU {:.0}% CPU {:.0}%",
|
|
state.gpu.utilization, state.system_resources.cpu_usage_pct
|
|
),
|
|
Style::default().fg(theme::SECONDARY),
|
|
));
|
|
}
|
|
|
|
let line = Line::from(spans);
|
|
|
|
let paragraph = Paragraph::new(line);
|
|
frame.render_widget(paragraph, area);
|
|
}
|
|
|
|
fn render_help_overlay(frame: &mut ratatui::Frame, area: Rect) {
|
|
// Center a help box in the terminal
|
|
let width = 50_u16.min(area.width.saturating_sub(4));
|
|
let height = 16_u16.min(area.height.saturating_sub(4));
|
|
#[allow(clippy::integer_division)]
|
|
let x = area.x + (area.width.saturating_sub(width)) / 2;
|
|
#[allow(clippy::integer_division)]
|
|
let y = area.y + (area.height.saturating_sub(height)) / 2;
|
|
let help_area = Rect::new(x, y, width, height);
|
|
|
|
// Clear behind the overlay
|
|
frame.render_widget(Clear, help_area);
|
|
|
|
let help_text = vec![
|
|
Line::from(""),
|
|
Line::from(vec![
|
|
Span::styled(" 1-7 ", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled("Switch cockpit view", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(vec![
|
|
Span::styled(" j/k ", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled("Navigate items up/down", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(vec![
|
|
Span::styled(" Enter/\u{2192}", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled(" Expand/collapse detail", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(vec![
|
|
Span::styled(" Tab ", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled("Switch Live/History (Training)", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(vec![
|
|
Span::styled(" l/h ", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled("History/Live tab (Training)", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(vec![
|
|
Span::styled(" ? ", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled("Toggle this help", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(vec![
|
|
Span::styled(" r ", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled("Force refresh", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(vec![
|
|
Span::styled(" q/Esc ", Style::default().fg(theme::SECONDARY)),
|
|
Span::styled("Quit", Style::default().fg(theme::TEXT)),
|
|
]),
|
|
Line::from(""),
|
|
Line::from(vec![Span::styled(
|
|
" Cockpits: Overview | Training | Trading",
|
|
theme::muted_style(),
|
|
)]),
|
|
Line::from(vec![Span::styled(
|
|
" Services | Risk | Data | Pods",
|
|
theme::muted_style(),
|
|
)]),
|
|
];
|
|
|
|
let help_block = Block::default()
|
|
.borders(Borders::ALL)
|
|
.border_style(Style::default().fg(theme::PRIMARY))
|
|
.title(" Keyboard Shortcuts ")
|
|
.title_style(theme::title_style());
|
|
|
|
let help = Paragraph::new(help_text).block(help_block).wrap(Wrap { trim: false });
|
|
frame.render_widget(help, help_area);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[cfg(test)]
|
|
#[allow(clippy::unwrap_used)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::tui::state::{
|
|
AccountData, AlertData, CircuitBreakerData, DataCacheData, DataFeedData,
|
|
ExecutionData, GpuData, ModelStatusData, OrderEventData, PodGroupData,
|
|
PositionData, RiskAlertData, RiskData, ServiceData, SystemResources,
|
|
TrainingHistoryEntry, TrainingSessionData,
|
|
};
|
|
|
|
fn make_session(model: &str, epoch: u32, loss: f64, sharpe: f64) -> TrainingSessionData {
|
|
TrainingSessionData {
|
|
model: model.into(),
|
|
fold: "fold-0".into(),
|
|
is_hyperopt: false,
|
|
epoch,
|
|
loss,
|
|
val_loss: loss + 0.1,
|
|
batches_per_sec: 100.0,
|
|
iteration_secs: 0.01,
|
|
epoch_duration_secs: 30.0,
|
|
sharpe,
|
|
sortino: 1.5,
|
|
win_rate: 0.55,
|
|
max_dd: 0.1,
|
|
profit_factor: 1.8,
|
|
total_return: 0.05,
|
|
total_trades: 100,
|
|
action_buy_pct: 0.33,
|
|
action_sell_pct: 0.33,
|
|
action_hold_pct: 0.34,
|
|
q_value_mean: 0.5,
|
|
q_value_max: 1.0,
|
|
policy_entropy: 0.8,
|
|
kl_divergence: 0.01,
|
|
advantage_mean: 0.1,
|
|
replay_buffer_size: 10000,
|
|
gradient_norm: 0.5,
|
|
learning_rate: 0.001,
|
|
nan_detected: 0,
|
|
gradient_explosions: 0,
|
|
feature_errors: 0,
|
|
checkpoint_saves: 1,
|
|
checkpoint_failures: 0,
|
|
hyperopt_trial_current: 0,
|
|
hyperopt_trial_total: 0,
|
|
hyperopt_best_objective: 0.0,
|
|
hyperopt_trials_failed: 0,
|
|
hyperopt_trial_epoch: 0,
|
|
hyperopt_elapsed_secs: 0.0,
|
|
epoch_history: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn make_training_update(
|
|
sessions: Vec<TrainingSessionData>,
|
|
) -> StateUpdate {
|
|
StateUpdate::TrainingMetrics {
|
|
sessions,
|
|
gpu: GpuData::default(),
|
|
resources: SystemResources::default(),
|
|
active_jobs: 1,
|
|
}
|
|
}
|
|
|
|
// -- Epoch history accumulation --
|
|
|
|
#[test]
|
|
fn epoch_history_accumulated_on_epoch_advance() {
|
|
let mut state = AppState::default();
|
|
|
|
// Epoch 1 arrives.
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 1, 0.5, 1.0),
|
|
]));
|
|
assert_eq!(state.training_sessions.len(), 1);
|
|
assert!(state.training_sessions[0].epoch_history.is_empty());
|
|
|
|
// Epoch 2 arrives — epoch 1 should be snapshotted.
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 2, 0.4, 1.2),
|
|
]));
|
|
assert_eq!(state.training_sessions[0].epoch_history.len(), 1);
|
|
let snap = &state.training_sessions[0].epoch_history[0];
|
|
assert_eq!(snap.epoch, 1);
|
|
assert!((snap.loss - 0.5).abs() < 1e-6);
|
|
assert!((snap.sharpe - 1.0).abs() < 1e-6);
|
|
|
|
// Epoch 3 — epoch 2 appended.
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 3, 0.3, 1.5),
|
|
]));
|
|
assert_eq!(state.training_sessions[0].epoch_history.len(), 2);
|
|
assert_eq!(state.training_sessions[0].epoch_history[1].epoch, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn epoch_history_not_duplicated_on_same_epoch() {
|
|
let mut state = AppState::default();
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 5, 0.5, 1.0),
|
|
]));
|
|
// Same epoch arrives again (metrics refresh, no epoch advance).
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 5, 0.45, 1.1),
|
|
]));
|
|
assert!(state.training_sessions[0].epoch_history.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn epoch_history_tracks_multiple_models_independently() {
|
|
let mut state = AppState::default();
|
|
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 1, 0.5, 1.0),
|
|
make_session("PPO", 3, 0.3, 2.0),
|
|
]));
|
|
|
|
// DQN advances, PPO stays.
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 2, 0.4, 1.2),
|
|
make_session("PPO", 3, 0.29, 2.1),
|
|
]));
|
|
|
|
let dqn = state.training_sessions.iter().find(|s| s.model == "DQN").unwrap();
|
|
let ppo = state.training_sessions.iter().find(|s| s.model == "PPO").unwrap();
|
|
assert_eq!(dqn.epoch_history.len(), 1);
|
|
assert!(ppo.epoch_history.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn epoch_history_preserved_when_model_disappears_then_returns() {
|
|
let mut state = AppState::default();
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 1, 0.5, 1.0),
|
|
]));
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 2, 0.4, 1.2),
|
|
]));
|
|
assert_eq!(state.training_sessions[0].epoch_history.len(), 1);
|
|
|
|
// Model disappears from update.
|
|
apply_update(&mut state, make_training_update(vec![]));
|
|
assert!(state.training_sessions.is_empty());
|
|
|
|
// Model returns — history is lost (fresh session).
|
|
apply_update(&mut state, make_training_update(vec![
|
|
make_session("DQN", 3, 0.3, 1.5),
|
|
]));
|
|
assert!(state.training_sessions[0].epoch_history.is_empty());
|
|
}
|
|
|
|
// -- Training tab switching --
|
|
|
|
#[test]
|
|
fn tab_toggles_training_sub_tab() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
assert_eq!(state.training_tab, TrainingTab::Live);
|
|
|
|
// Simulate Tab key.
|
|
state.training_tab = match state.training_tab {
|
|
TrainingTab::Live => TrainingTab::History,
|
|
TrainingTab::History => TrainingTab::Live,
|
|
};
|
|
assert_eq!(state.training_tab, TrainingTab::History);
|
|
|
|
state.training_tab = match state.training_tab {
|
|
TrainingTab::Live => TrainingTab::History,
|
|
TrainingTab::History => TrainingTab::Live,
|
|
};
|
|
assert_eq!(state.training_tab, TrainingTab::Live);
|
|
}
|
|
|
|
// -- Navigation: j/k wrapping --
|
|
|
|
#[test]
|
|
fn nav_down_wraps_live_sessions() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::Live;
|
|
state.training_sessions = vec![
|
|
make_session("A", 1, 0.5, 1.0),
|
|
make_session("B", 1, 0.4, 1.2),
|
|
];
|
|
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_training_session, Some(0));
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_training_session, Some(1));
|
|
// Wraps back to 0.
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_training_session, Some(0));
|
|
}
|
|
|
|
#[test]
|
|
fn nav_up_wraps_live_sessions() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::Live;
|
|
state.training_sessions = vec![
|
|
make_session("A", 1, 0.5, 1.0),
|
|
make_session("B", 1, 0.4, 1.2),
|
|
make_session("C", 1, 0.3, 1.5),
|
|
];
|
|
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_training_session, Some(0));
|
|
// Wraps to last.
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_training_session, Some(2));
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_training_session, Some(1));
|
|
}
|
|
|
|
#[test]
|
|
fn nav_down_empty_sessions_is_noop() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::Live;
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_training_session, None);
|
|
}
|
|
|
|
#[test]
|
|
fn nav_down_history_tab() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::History;
|
|
state.training_history = vec![
|
|
TrainingHistoryEntry {
|
|
job_id: "j1".into(), model: "DQN".into(), status: "Completed".into(),
|
|
final_loss: 0.3, best_val: 0.4, created_at: 0, started_at: 0,
|
|
completed_at: 0, description: String::new(),
|
|
},
|
|
TrainingHistoryEntry {
|
|
job_id: "j2".into(), model: "PPO".into(), status: "Failed".into(),
|
|
final_loss: 0.0, best_val: 0.0, created_at: 0, started_at: 0,
|
|
completed_at: 0, description: String::new(),
|
|
},
|
|
];
|
|
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_history_entry, Some(0));
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_history_entry, Some(1));
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_history_entry, Some(0)); // wrap
|
|
}
|
|
|
|
// -- Pod navigation --
|
|
|
|
#[test]
|
|
fn nav_down_pods() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_PODS;
|
|
state.pods = vec![
|
|
PodGroupData { service: "a".into(), pods: vec![], ready_count: 0, total_count: 0 },
|
|
PodGroupData { service: "b".into(), pods: vec![], ready_count: 0, total_count: 0 },
|
|
];
|
|
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_pod_group, Some(0));
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_pod_group, Some(1));
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_pod_group, Some(0)); // wrap
|
|
}
|
|
|
|
#[test]
|
|
fn enter_toggles_detail_expansion() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::Live;
|
|
state.selected_training_session = Some(0);
|
|
state.training_sessions = vec![make_session("DQN", 1, 0.5, 1.0)];
|
|
|
|
assert!(!state.training_detail_expanded);
|
|
handle_enter(&mut state);
|
|
assert!(state.training_detail_expanded);
|
|
handle_enter(&mut state);
|
|
assert!(!state.training_detail_expanded);
|
|
}
|
|
|
|
#[test]
|
|
fn enter_no_selection_does_not_expand() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::Live;
|
|
state.selected_training_session = None;
|
|
handle_enter(&mut state);
|
|
assert!(!state.training_detail_expanded);
|
|
}
|
|
|
|
#[test]
|
|
fn pod_enter_toggles_expansion() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_PODS;
|
|
state.selected_pod_group = Some(0);
|
|
state.pods = vec![
|
|
PodGroupData { service: "a".into(), pods: vec![], ready_count: 0, total_count: 0 },
|
|
];
|
|
|
|
assert!(!state.pod_group_expanded);
|
|
handle_enter(&mut state);
|
|
assert!(state.pod_group_expanded);
|
|
handle_enter(&mut state);
|
|
assert!(!state.pod_group_expanded);
|
|
}
|
|
|
|
// -- State updates: pods, training history, connection --
|
|
|
|
#[test]
|
|
fn apply_pods_update() {
|
|
let mut state = AppState::default();
|
|
let pods = vec![
|
|
PodGroupData { service: "svc".into(), pods: vec![], ready_count: 1, total_count: 1 },
|
|
];
|
|
apply_update(&mut state, StateUpdate::Pods(pods));
|
|
assert_eq!(state.pods.len(), 1);
|
|
assert_eq!(state.pods[0].service, "svc");
|
|
}
|
|
|
|
#[test]
|
|
fn apply_training_history_update() {
|
|
let mut state = AppState::default();
|
|
let entries = vec![
|
|
TrainingHistoryEntry {
|
|
job_id: "j1".into(), model: "DQN".into(), status: "Completed".into(),
|
|
final_loss: 0.3, best_val: 0.4, created_at: 0, started_at: 0,
|
|
completed_at: 100, description: String::new(),
|
|
},
|
|
];
|
|
apply_update(&mut state, StateUpdate::TrainingHistory(entries));
|
|
assert_eq!(state.training_history.len(), 1);
|
|
assert_eq!(state.training_history[0].job_id, "j1");
|
|
}
|
|
|
|
#[test]
|
|
fn apply_connection_update() {
|
|
let mut state = AppState::default();
|
|
apply_update(&mut state, StateUpdate::Connection(ConnectionStatus::Connected));
|
|
assert!(matches!(state.connection_status, ConnectionStatus::Connected));
|
|
|
|
apply_update(&mut state, StateUpdate::Connection(
|
|
ConnectionStatus::Reconnecting { attempt: 3 },
|
|
));
|
|
assert!(matches!(state.connection_status, ConnectionStatus::Reconnecting { attempt: 3 }));
|
|
}
|
|
|
|
// -- StateUpdate: Services --
|
|
|
|
#[test]
|
|
fn apply_services_update() {
|
|
let mut state = AppState::default();
|
|
let services = vec![
|
|
ServiceData {
|
|
name: "api-gateway".into(),
|
|
healthy: true,
|
|
uptime: "5d".into(),
|
|
version: "v1.0".into(),
|
|
latency_ms: 12.0,
|
|
error: None,
|
|
},
|
|
ServiceData {
|
|
name: "trading-service".into(),
|
|
healthy: false,
|
|
uptime: "0s".into(),
|
|
version: "v1.1".into(),
|
|
latency_ms: 0.0,
|
|
error: Some("timeout".into()),
|
|
},
|
|
];
|
|
apply_update(&mut state, StateUpdate::Services(services));
|
|
assert_eq!(state.services.len(), 2);
|
|
assert!(state.services[0].healthy);
|
|
assert!(!state.services[1].healthy);
|
|
assert_eq!(state.services[1].error.as_deref(), Some("timeout"));
|
|
}
|
|
|
|
// -- StateUpdate: Account --
|
|
|
|
#[test]
|
|
fn apply_account_update() {
|
|
let mut state = AppState::default();
|
|
let acct = AccountData {
|
|
equity: 100_000.0,
|
|
cash: 50_000.0,
|
|
margin_used: 25_000.0,
|
|
daily_pnl: 1_234.56,
|
|
total_pnl: 9_876.54,
|
|
};
|
|
apply_update(&mut state, StateUpdate::Account(acct));
|
|
assert!((state.account.equity - 100_000.0).abs() < 1e-6);
|
|
assert!((state.account.daily_pnl - 1_234.56).abs() < 1e-6);
|
|
}
|
|
|
|
// -- StateUpdate: Positions --
|
|
|
|
#[test]
|
|
fn apply_positions_update() {
|
|
let mut state = AppState::default();
|
|
let positions = vec![
|
|
PositionData {
|
|
symbol: "ES".into(),
|
|
qty: 10,
|
|
entry_price: 5000.0,
|
|
market_price: 5050.0,
|
|
unrealized_pnl: 500.0,
|
|
},
|
|
];
|
|
apply_update(&mut state, StateUpdate::Positions(positions));
|
|
assert_eq!(state.positions.len(), 1);
|
|
assert_eq!(state.positions[0].symbol, "ES");
|
|
assert_eq!(state.positions[0].qty, 10);
|
|
}
|
|
|
|
#[test]
|
|
fn apply_positions_replaces_previous() {
|
|
let mut state = AppState::default();
|
|
let p1 = vec![PositionData {
|
|
symbol: "ES".into(), qty: 5, entry_price: 5000.0,
|
|
market_price: 5050.0, unrealized_pnl: 250.0,
|
|
}];
|
|
let p2 = vec![
|
|
PositionData { symbol: "NQ".into(), qty: 3, entry_price: 20000.0,
|
|
market_price: 20100.0, unrealized_pnl: 300.0 },
|
|
PositionData { symbol: "6E".into(), qty: 1, entry_price: 1.1,
|
|
market_price: 1.11, unrealized_pnl: 10.0 },
|
|
];
|
|
apply_update(&mut state, StateUpdate::Positions(p1));
|
|
assert_eq!(state.positions.len(), 1);
|
|
apply_update(&mut state, StateUpdate::Positions(p2));
|
|
assert_eq!(state.positions.len(), 2);
|
|
assert_eq!(state.positions[0].symbol, "NQ");
|
|
}
|
|
|
|
// -- StateUpdate: Execution (ring buffer) --
|
|
|
|
#[test]
|
|
fn apply_execution_ring_buffer() {
|
|
let mut state = AppState::default();
|
|
// Fill exactly to capacity.
|
|
for i in 0..RING_CAP {
|
|
apply_update(&mut state, StateUpdate::Execution(ExecutionData {
|
|
time: format!("t{i}"), symbol: "ES".into(), side: "BUY".into(),
|
|
qty: 1, price: 5000.0, status: "Filled".into(),
|
|
}));
|
|
}
|
|
assert_eq!(state.executions.len(), RING_CAP);
|
|
assert_eq!(state.executions.front().unwrap().time, "t0");
|
|
|
|
// One more should evict the oldest.
|
|
apply_update(&mut state, StateUpdate::Execution(ExecutionData {
|
|
time: "overflow".into(), symbol: "ES".into(), side: "SELL".into(),
|
|
qty: 1, price: 5100.0, status: "Filled".into(),
|
|
}));
|
|
assert_eq!(state.executions.len(), RING_CAP);
|
|
assert_eq!(state.executions.front().unwrap().time, "t1");
|
|
assert_eq!(state.executions.back().unwrap().time, "overflow");
|
|
}
|
|
|
|
// -- StateUpdate: Order (ring buffer) --
|
|
|
|
#[test]
|
|
fn apply_order_ring_buffer() {
|
|
let mut state = AppState::default();
|
|
for i in 0..RING_CAP {
|
|
apply_update(&mut state, StateUpdate::Order(OrderEventData {
|
|
time: format!("t{i}"), order_id: format!("o{i}"),
|
|
symbol: "NQ".into(), side: "BUY".into(),
|
|
status: "New".into(), qty: 1.0, filled_qty: 0.0,
|
|
}));
|
|
}
|
|
assert_eq!(state.orders.len(), RING_CAP);
|
|
|
|
// Overflow evicts oldest.
|
|
apply_update(&mut state, StateUpdate::Order(OrderEventData {
|
|
time: "late".into(), order_id: "oX".into(),
|
|
symbol: "NQ".into(), side: "SELL".into(),
|
|
status: "Filled".into(), qty: 2.0, filled_qty: 2.0,
|
|
}));
|
|
assert_eq!(state.orders.len(), RING_CAP);
|
|
assert_eq!(state.orders.front().unwrap().order_id, "o1");
|
|
assert_eq!(state.orders.back().unwrap().order_id, "oX");
|
|
}
|
|
|
|
// -- StateUpdate: Risk (preserves circuit breakers) --
|
|
|
|
#[test]
|
|
fn apply_risk_preserves_circuit_breakers() {
|
|
let mut state = AppState::default();
|
|
// Pre-load circuit breakers from the separate stream.
|
|
state.risk.circuit_breakers = vec![
|
|
CircuitBreakerData {
|
|
name: "max-drawdown".into(),
|
|
threshold: Some(0.05),
|
|
current: Some(0.02),
|
|
tripped: false,
|
|
},
|
|
];
|
|
|
|
// Risk update arrives — should NOT overwrite circuit breakers.
|
|
let risk = RiskData {
|
|
global_kill_switch: Some(false),
|
|
portfolio_kill_switch: None,
|
|
strategy_kill_switch: None,
|
|
instrument_kill_switch: None,
|
|
current_drawdown_pct: 0.02,
|
|
max_drawdown_pct: 0.05,
|
|
high_water_mark: 100_000.0,
|
|
circuit_breakers: Vec::new(), // empty from this stream
|
|
position_limits: Vec::new(),
|
|
};
|
|
apply_update(&mut state, StateUpdate::Risk(risk));
|
|
assert_eq!(state.risk.circuit_breakers.len(), 1);
|
|
assert_eq!(state.risk.circuit_breakers[0].name, "max-drawdown");
|
|
assert!((state.risk.current_drawdown_pct - 0.02).abs() < 1e-6);
|
|
}
|
|
|
|
// -- StateUpdate: CircuitBreakers --
|
|
|
|
#[test]
|
|
fn apply_circuit_breakers_update() {
|
|
let mut state = AppState::default();
|
|
let cbs = vec![
|
|
CircuitBreakerData {
|
|
name: "daily-loss".into(),
|
|
threshold: Some(0.03),
|
|
current: Some(0.01),
|
|
tripped: false,
|
|
},
|
|
CircuitBreakerData {
|
|
name: "drawdown".into(),
|
|
threshold: Some(0.10),
|
|
current: Some(0.11),
|
|
tripped: true,
|
|
},
|
|
];
|
|
apply_update(&mut state, StateUpdate::CircuitBreakers(cbs));
|
|
assert_eq!(state.risk.circuit_breakers.len(), 2);
|
|
assert!(state.risk.circuit_breakers[1].tripped);
|
|
}
|
|
|
|
// -- StateUpdate: RiskAlert (ring buffer) --
|
|
|
|
#[test]
|
|
fn apply_risk_alert_ring_buffer() {
|
|
let mut state = AppState::default();
|
|
for i in 0..RING_CAP {
|
|
apply_update(&mut state, StateUpdate::RiskAlert(RiskAlertData {
|
|
time: format!("t{i}"), severity: "WARN".into(),
|
|
alert_type: "drawdown".into(), message: format!("alert {i}"),
|
|
}));
|
|
}
|
|
assert_eq!(state.risk_alerts.len(), RING_CAP);
|
|
|
|
apply_update(&mut state, StateUpdate::RiskAlert(RiskAlertData {
|
|
time: "overflow".into(), severity: "CRITICAL".into(),
|
|
alert_type: "kill-switch".into(), message: "tripped".into(),
|
|
}));
|
|
assert_eq!(state.risk_alerts.len(), RING_CAP);
|
|
assert_eq!(state.risk_alerts.front().unwrap().time, "t1");
|
|
assert_eq!(state.risk_alerts.back().unwrap().severity, "CRITICAL");
|
|
}
|
|
|
|
// -- StateUpdate: ModelStatuses --
|
|
|
|
#[test]
|
|
fn apply_model_statuses_update() {
|
|
let mut state = AppState::default();
|
|
let statuses = vec![
|
|
ModelStatusData {
|
|
model_name: "DQN".into(), state: "ACTIVE".into(),
|
|
health: "OK".into(), last_prediction: "1s ago".into(),
|
|
},
|
|
ModelStatusData {
|
|
model_name: "PPO".into(), state: "LOADING".into(),
|
|
health: "DEGRADED".into(), last_prediction: "--".into(),
|
|
},
|
|
];
|
|
apply_update(&mut state, StateUpdate::ModelStatuses(statuses));
|
|
assert_eq!(state.model_statuses.len(), 2);
|
|
assert_eq!(state.model_statuses[0].state, "ACTIVE");
|
|
assert_eq!(state.model_statuses[1].health, "DEGRADED");
|
|
}
|
|
|
|
// -- StateUpdate: Alert (ring buffer) --
|
|
|
|
#[test]
|
|
fn apply_alert_ring_buffer() {
|
|
let mut state = AppState::default();
|
|
for i in 0..RING_CAP + 5 {
|
|
apply_update(&mut state, StateUpdate::Alert(AlertData {
|
|
time: format!("t{i}"), severity: "WARN".into(),
|
|
service: "api".into(), title: format!("alert {i}"),
|
|
}));
|
|
}
|
|
// Should cap at RING_CAP after overflow.
|
|
assert_eq!(state.alerts.len(), RING_CAP);
|
|
assert_eq!(state.alerts.front().unwrap().time, "t5");
|
|
assert_eq!(state.alerts.back().unwrap().title, "alert 54");
|
|
}
|
|
|
|
// -- StateUpdate: DataFeeds --
|
|
|
|
#[test]
|
|
fn apply_data_feeds_update() {
|
|
let mut state = AppState::default();
|
|
let feeds = vec![
|
|
DataFeedData {
|
|
symbol: "ES".into(), records_per_sec: 500,
|
|
latency_us: 120, status: "Active".into(),
|
|
},
|
|
];
|
|
apply_update(&mut state, StateUpdate::DataFeeds(feeds));
|
|
assert_eq!(state.data_feeds.len(), 1);
|
|
assert_eq!(state.data_feeds[0].records_per_sec, 500);
|
|
}
|
|
|
|
// -- StateUpdate: DataCache --
|
|
|
|
#[test]
|
|
fn apply_data_cache_update() {
|
|
let mut state = AppState::default();
|
|
let cache = DataCacheData {
|
|
total_records: 4_000_000,
|
|
cache_hit_rate: 0.95,
|
|
disk_usage_gb: 12.5,
|
|
oldest_date: "2024-01-01".into(),
|
|
newest_date: "2025-12-31".into(),
|
|
};
|
|
apply_update(&mut state, StateUpdate::DataCache(cache));
|
|
assert_eq!(state.data_cache.total_records, 4_000_000);
|
|
assert!((state.data_cache.cache_hit_rate - 0.95).abs() < 1e-6);
|
|
}
|
|
|
|
// -- StateUpdate: Portfolio --
|
|
|
|
#[test]
|
|
fn apply_portfolio_update() {
|
|
let mut state = AppState::default();
|
|
apply_update(&mut state, StateUpdate::Portfolio {
|
|
equity: 150_000.0,
|
|
cash: 60_000.0,
|
|
margin_used: 40_000.0,
|
|
daily_pnl: 2_500.0,
|
|
total_pnl: 15_000.0,
|
|
});
|
|
assert!((state.account.equity - 150_000.0).abs() < 1e-6);
|
|
assert!((state.account.cash - 60_000.0).abs() < 1e-6);
|
|
assert!((state.account.margin_used - 40_000.0).abs() < 1e-6);
|
|
assert!((state.account.daily_pnl - 2_500.0).abs() < 1e-6);
|
|
assert!((state.account.total_pnl - 15_000.0).abs() < 1e-6);
|
|
}
|
|
|
|
#[test]
|
|
fn apply_portfolio_overwrites_account_fields() {
|
|
let mut state = AppState::default();
|
|
// Set account via Account update first.
|
|
apply_update(&mut state, StateUpdate::Account(AccountData {
|
|
equity: 100_000.0, cash: 50_000.0, margin_used: 20_000.0,
|
|
daily_pnl: 1_000.0, total_pnl: 5_000.0,
|
|
}));
|
|
// Portfolio update overwrites the same fields.
|
|
apply_update(&mut state, StateUpdate::Portfolio {
|
|
equity: 200_000.0, cash: 80_000.0, margin_used: 30_000.0,
|
|
daily_pnl: 3_000.0, total_pnl: 10_000.0,
|
|
});
|
|
assert!((state.account.equity - 200_000.0).abs() < 1e-6);
|
|
assert!((state.account.total_pnl - 10_000.0).abs() < 1e-6);
|
|
}
|
|
|
|
// -- StateUpdate: TrainingMetrics GPU & resources --
|
|
|
|
#[test]
|
|
fn apply_training_metrics_updates_gpu_and_resources() {
|
|
let mut state = AppState::default();
|
|
apply_update(&mut state, StateUpdate::TrainingMetrics {
|
|
sessions: vec![],
|
|
gpu: GpuData {
|
|
name: "RTX 3050 Ti".into(),
|
|
utilization: 85.0,
|
|
memory_used_gb: 3.5,
|
|
memory_total_gb: 4.0,
|
|
temperature_c: 72,
|
|
power_watts: 150,
|
|
power_limit_watts: 200,
|
|
},
|
|
resources: SystemResources {
|
|
cpu_usage_pct: 45.0,
|
|
ram_used_gb: 12.0,
|
|
ram_total_gb: 32.0,
|
|
gpu_cluster_utilization: 80.0,
|
|
},
|
|
active_jobs: 3,
|
|
});
|
|
assert_eq!(state.gpu.name, "RTX 3050 Ti");
|
|
assert!((state.gpu.utilization - 85.0).abs() < 1e-6);
|
|
assert!((state.system_resources.cpu_usage_pct - 45.0).abs() < 1e-6);
|
|
assert_eq!(state.active_jobs, 3);
|
|
}
|
|
|
|
// -- Pod scroll helpers --
|
|
|
|
fn make_pod_group(service: &str, pod_count: usize) -> PodGroupData {
|
|
use crate::tui::state::PodData;
|
|
let pods: Vec<PodData> = (0..pod_count)
|
|
.map(|i| PodData {
|
|
name: format!("{service}-{i}"), service: service.into(),
|
|
status: "Running".into(), restarts: 0, age: "1h".into(),
|
|
node: "node-1".into(), ready: true, version: "v1".into(),
|
|
})
|
|
.collect();
|
|
PodGroupData {
|
|
service: service.into(),
|
|
pods,
|
|
ready_count: pod_count,
|
|
total_count: pod_count,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn pod_group_flat_row_single_pod_groups() {
|
|
// Each group with 1 pod contributes exactly 1 row (header only, no sub-rows).
|
|
let pods = vec![
|
|
make_pod_group("a", 1),
|
|
make_pod_group("b", 1),
|
|
make_pod_group("c", 1),
|
|
];
|
|
assert_eq!(pod_group_flat_row(&pods, 0), 0);
|
|
assert_eq!(pod_group_flat_row(&pods, 1), 1);
|
|
assert_eq!(pod_group_flat_row(&pods, 2), 2);
|
|
}
|
|
|
|
#[test]
|
|
fn pod_group_flat_row_multi_pod_groups() {
|
|
// Group with >1 pod: 1 header + N sub-rows.
|
|
let pods = vec![
|
|
make_pod_group("a", 3), // header + 3 subs = 4 rows
|
|
make_pod_group("b", 1), // header only = 1 row
|
|
make_pod_group("c", 2), // header + 2 subs = 3 rows
|
|
];
|
|
assert_eq!(pod_group_flat_row(&pods, 0), 0);
|
|
assert_eq!(pod_group_flat_row(&pods, 1), 4); // 0 + 1 header + 3 subs
|
|
assert_eq!(pod_group_flat_row(&pods, 2), 5); // 4 + 1 header
|
|
}
|
|
|
|
#[test]
|
|
fn pod_group_flat_row_out_of_bounds_returns_total() {
|
|
let pods = vec![make_pod_group("a", 2)]; // 1 header + 2 subs = 3
|
|
assert_eq!(pod_group_flat_row(&pods, 5), 3);
|
|
}
|
|
|
|
#[test]
|
|
fn scroll_pods_into_view_scrolls_down() {
|
|
let mut state = AppState::default();
|
|
// 40 single-pod groups → 40 flat rows. VISIBLE_ROWS=35.
|
|
state.pods = (0..40).map(|i| make_pod_group(&format!("svc-{i:02}"), 1)).collect();
|
|
state.selected_pod_group = Some(38);
|
|
state.pod_scroll_offset = 0;
|
|
|
|
scroll_pods_into_view(&mut state);
|
|
// Group 38 at flat row 38 should be visible: offset should move up.
|
|
assert!(state.pod_scroll_offset > 0);
|
|
assert!(state.pod_scroll_offset <= 38);
|
|
}
|
|
|
|
#[test]
|
|
fn scroll_pods_into_view_scrolls_up() {
|
|
let mut state = AppState::default();
|
|
state.pods = (0..40).map(|i| make_pod_group(&format!("svc-{i:02}"), 1)).collect();
|
|
state.selected_pod_group = Some(2);
|
|
state.pod_scroll_offset = 10;
|
|
|
|
scroll_pods_into_view(&mut state);
|
|
// Group 2 at flat row 2 is above offset 10 — should scroll up.
|
|
assert_eq!(state.pod_scroll_offset, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn scroll_pods_into_view_no_change_when_visible() {
|
|
let mut state = AppState::default();
|
|
state.pods = (0..10).map(|i| make_pod_group(&format!("svc-{i}"), 1)).collect();
|
|
state.selected_pod_group = Some(5);
|
|
state.pod_scroll_offset = 0;
|
|
|
|
scroll_pods_into_view(&mut state);
|
|
// Group 5 at row 5 is within 0..35 — no change.
|
|
assert_eq!(state.pod_scroll_offset, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn scroll_pods_noop_when_no_selection() {
|
|
let mut state = AppState::default();
|
|
state.pods = (0..10).map(|i| make_pod_group(&format!("svc-{i}"), 1)).collect();
|
|
state.selected_pod_group = None;
|
|
state.pod_scroll_offset = 5;
|
|
|
|
scroll_pods_into_view(&mut state);
|
|
assert_eq!(state.pod_scroll_offset, 5); // unchanged
|
|
}
|
|
|
|
// -- Cockpit switching --
|
|
|
|
#[test]
|
|
fn cockpit_switching_via_number_keys() {
|
|
let mut state = AppState::default();
|
|
assert_eq!(state.current_cockpit, 0);
|
|
|
|
// Simulate pressing '3'
|
|
let idx = ('3' as usize).saturating_sub('1' as usize);
|
|
if idx < COCKPIT_COUNT {
|
|
state.current_cockpit = idx;
|
|
}
|
|
assert_eq!(state.current_cockpit, 2);
|
|
|
|
// Simulate pressing '7'
|
|
let idx = ('7' as usize).saturating_sub('1' as usize);
|
|
if idx < COCKPIT_COUNT {
|
|
state.current_cockpit = idx;
|
|
}
|
|
assert_eq!(state.current_cockpit, 6);
|
|
}
|
|
|
|
#[test]
|
|
fn cockpit_switching_out_of_range_ignored() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = 3;
|
|
|
|
// '8' maps to index 7, which is >= COCKPIT_COUNT (7), so ignored.
|
|
let idx = ('8' as usize).saturating_sub('1' as usize);
|
|
if idx < COCKPIT_COUNT {
|
|
state.current_cockpit = idx;
|
|
}
|
|
assert_eq!(state.current_cockpit, 3); // unchanged
|
|
}
|
|
|
|
// -- Help toggle --
|
|
|
|
#[test]
|
|
fn help_toggle() {
|
|
let mut state = AppState::default();
|
|
assert!(!state.show_help);
|
|
state.show_help = !state.show_help;
|
|
assert!(state.show_help);
|
|
state.show_help = !state.show_help;
|
|
assert!(!state.show_help);
|
|
}
|
|
|
|
// -- Pod navigation: nav_up --
|
|
|
|
#[test]
|
|
fn nav_up_pods_wraps_to_last() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_PODS;
|
|
state.pods = vec![
|
|
PodGroupData { service: "a".into(), pods: vec![], ready_count: 0, total_count: 0 },
|
|
PodGroupData { service: "b".into(), pods: vec![], ready_count: 0, total_count: 0 },
|
|
PodGroupData { service: "c".into(), pods: vec![], ready_count: 0, total_count: 0 },
|
|
];
|
|
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_pod_group, Some(0));
|
|
// Wrap to last
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_pod_group, Some(2));
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_pod_group, Some(1));
|
|
}
|
|
|
|
#[test]
|
|
fn nav_down_empty_pods_is_noop() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_PODS;
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_pod_group, None);
|
|
}
|
|
|
|
#[test]
|
|
fn nav_up_empty_pods_is_noop() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_PODS;
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_pod_group, None);
|
|
}
|
|
|
|
// -- History navigation: nav_up --
|
|
|
|
#[test]
|
|
fn nav_up_history_tab_wraps() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::History;
|
|
state.training_history = vec![
|
|
TrainingHistoryEntry {
|
|
job_id: "j1".into(), model: "DQN".into(), status: "Completed".into(),
|
|
final_loss: 0.3, best_val: 0.4, created_at: 0, started_at: 0,
|
|
completed_at: 0, description: String::new(),
|
|
},
|
|
TrainingHistoryEntry {
|
|
job_id: "j2".into(), model: "PPO".into(), status: "Failed".into(),
|
|
final_loss: 0.0, best_val: 0.0, created_at: 0, started_at: 0,
|
|
completed_at: 0, description: String::new(),
|
|
},
|
|
];
|
|
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_history_entry, Some(0));
|
|
// Wrap to last
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_history_entry, Some(1));
|
|
}
|
|
|
|
#[test]
|
|
fn nav_empty_history_is_noop() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
state.training_tab = TrainingTab::History;
|
|
handle_nav_down(&mut state);
|
|
assert_eq!(state.selected_history_entry, None);
|
|
handle_nav_up(&mut state);
|
|
assert_eq!(state.selected_history_entry, None);
|
|
}
|
|
|
|
// -- Enter on non-navigable cockpits --
|
|
|
|
#[test]
|
|
fn enter_on_overview_cockpit_is_noop() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = 0; // Overview
|
|
handle_enter(&mut state);
|
|
assert!(!state.training_detail_expanded);
|
|
assert!(!state.pod_group_expanded);
|
|
}
|
|
|
|
// -- l/h direct tab switching --
|
|
|
|
#[test]
|
|
fn l_h_keys_set_training_tab_directly() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_TRAINING;
|
|
assert_eq!(state.training_tab, TrainingTab::Live);
|
|
|
|
// 'l' goes to History
|
|
state.training_tab = TrainingTab::History;
|
|
assert_eq!(state.training_tab, TrainingTab::History);
|
|
|
|
// 'h' goes to Live
|
|
state.training_tab = TrainingTab::Live;
|
|
assert_eq!(state.training_tab, TrainingTab::Live);
|
|
}
|
|
|
|
// -- Ring buffer: single element --
|
|
|
|
#[test]
|
|
fn ring_buffer_single_element() {
|
|
let mut state = AppState::default();
|
|
apply_update(&mut state, StateUpdate::Execution(ExecutionData {
|
|
time: "now".into(), symbol: "ES".into(), side: "BUY".into(),
|
|
qty: 1, price: 5000.0, status: "Filled".into(),
|
|
}));
|
|
assert_eq!(state.executions.len(), 1);
|
|
assert_eq!(state.executions[0].time, "now");
|
|
}
|
|
|
|
// -- Pod scroll integrates with navigation --
|
|
|
|
#[test]
|
|
fn nav_down_pods_adjusts_scroll_offset() {
|
|
let mut state = AppState::default();
|
|
state.current_cockpit = COCKPIT_PODS;
|
|
// 40 groups should trigger scrolling when navigating to the end.
|
|
state.pods = (0..40).map(|i| make_pod_group(&format!("svc-{i:02}"), 1)).collect();
|
|
|
|
// Navigate all the way down.
|
|
for _ in 0..39 {
|
|
handle_nav_down(&mut state);
|
|
}
|
|
assert_eq!(state.selected_pod_group, Some(38));
|
|
// Scroll offset should have adjusted so group 38 is visible.
|
|
assert!(state.pod_scroll_offset > 0);
|
|
}
|
|
}
|