gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tui): training history tab, pods scrolling, overview pods panel, 136 tests

Browse Source 
Add Training cockpit sub-tabs (Live/History) with expandable per-epoch
detail panel showing financial metrics, RL diagnostics, and gradient health.
History tab fetches completed/failed runs from ListTrainingJobs RPC.

Add scrollable pod table to Pods cockpit and compact pods summary to
Overview bottom row. Fix K8s API egress in api-gateway NetworkPolicy
(monitoring-service→prometheus, apply existing K8s API CIDR rules).

Add 62 new unit tests across event_loop (51) and data_fetcher (11)
covering all StateUpdate variants, ring buffer eviction, epoch history
accumulation, pod scroll helpers, and navigation edge cases.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-04 20:48:15 +01:00
parent 70f5f4ff52
commit 48ca3db92f
7 changed files with 2718 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
bin/fxt/src/tui/cockpits/overview.rs
View File

@@ -8,7 +8,7 @@
use ratatui::Frame;
use ratatui::layout::{Constraint, Direction, Layout, Rect};
use ratatui::style::Style;
use ratatui::style::{Modifier, Style};
use ratatui::text::{Line, Span};
use ratatui::widgets::{Gauge, List, ListItem, Row, Table};
@@ -36,13 +36,18 @@ impl Cockpit for OverviewCockpit {
let bottom = Layout::default()
.direction(Direction::Horizontal)
.constraints([Constraint::Percentage(50), Constraint::Percentage(50)])
.constraints([
Constraint::Percentage(30),
Constraint::Percentage(30),
Constraint::Percentage(40),
])
.split(rows[1]);
render_services(frame, top[0], state);
render_resources(frame, top[1], state);
render_training_summary(frame, bottom[0], state);
render_portfolio(frame, bottom[1], state);
render_pods_summary(frame, bottom[2], state);
}
}
@@ -136,7 +141,7 @@ fn render_training_summary(frame: &mut Frame, area: Rect, state: &AppState) {
.map(|s| {
Row::new(vec![
s.model.clone(),
format!("{}/{}", s.epoch, s.max_epochs),
s.epoch.to_string(),
format!("{:.4}", s.loss),
format!("{:.2}", s.sharpe),
])
@@ -252,3 +257,59 @@ fn render_portfolio(frame: &mut Frame, area: Rect, state: &AppState) {
let summary = List::new(summary_items);
frame.render_widget(summary, chunks[0]);
}
fn render_pods_summary(frame: &mut Frame, area: Rect, state: &AppState) {
if state.pods.is_empty() {
let msg = ratatui::widgets::Paragraph::new(
Line::from(Span::styled("No pod data", theme::muted_style())),
)
.block(theme::block(" Pods "));
frame.render_widget(msg, area);
return;
}
// Compact: one row per service group (no sub-pod expansion).
let header = Row::new(vec!["Service", "Ready", "Status"])
.style(theme::header_style());
let rows: Vec<Row<'_>> = state
.pods
.iter()
.map(|g| {
let ready_str = format!("{}/{}", g.ready_count, g.total_count);
let (status_text, status_style) = if g.ready_count == g.total_count
&& g.total_count > 0
{
("Running", Style::default().fg(theme::SUCCESS))
} else if g.ready_count > 0 {
("Degraded", Style::default().fg(theme::WARNING))
} else {
("Down", Style::default().fg(theme::ERROR))
};
Row::new(vec![
Span::styled(
g.service.clone(),
Style::default()
.fg(theme::TEXT)
.add_modifier(Modifier::BOLD),
),
Span::styled(ready_str, Style::default().fg(theme::TEXT)),
Span::styled(status_text.to_owned(), status_style),
])
})
.collect();
let table = Table::new(
rows,
[
Constraint::Length(14),
Constraint::Length(6),
Constraint::Min(8),
],
)
.header(header)
.block(theme::block(" Pods "));
frame.render_widget(table, area);
}

65
bin/fxt/src/tui/cockpits/pods.rs
View File

@@ -8,7 +8,7 @@ use ratatui::text::{Line, Span};
use ratatui::widgets::{Paragraph, Row, Table};
use crate::tui::cockpit::Cockpit;
use crate::tui::state::AppState;
use crate::tui::state::{AppState, ConnectionStatus, PodGroupData};
use crate::tui::theme;
pub struct PodsCockpit;
@@ -20,16 +20,35 @@ impl Cockpit for PodsCockpit {
fn render(&self, frame: &mut Frame, area: Rect, state: &AppState) {
if state.pods.is_empty() {
let msg = Paragraph::new(Line::from(Span::styled(
"Waiting for pod data...",
theme::muted_style(),
)))
.block(theme::block(" Pods "));
let (msg_text, msg_style) = match &state.connection_status {
ConnectionStatus::Connecting => {
("Connecting to cluster\u{2026}", theme::muted_style())
}
ConnectionStatus::Reconnecting { attempt } => {
let _ = attempt;
(
"Reconnecting to cluster\u{2026}",
Style::default().fg(theme::WARNING),
)
}
ConnectionStatus::Error(e) => {
let _ = e;
(
"Cluster unreachable \u{2014} check API gateway",
Style::default().fg(theme::ERROR),
)
}
ConnectionStatus::Connected => {
("Waiting for pod data\u{2026}", theme::muted_style())
}
};
let msg = Paragraph::new(Line::from(Span::styled(msg_text, msg_style)))
.block(theme::block(" Pods "));
frame.render_widget(msg, area);
return;
}
render_pod_table(frame, area, state);
render_pod_table(frame, area, state, state.pod_scroll_offset);
}
}
@@ -41,15 +60,12 @@ fn shorten_pod_name(name: &str) -> &str {
name.rsplit('-').next().unwrap_or(name)
}
fn render_pod_table(frame: &mut Frame, area: Rect, state: &AppState) {
let header = Row::new(vec![
"Service", "Ready", "Status", "Version", "Restarts", "Age", "Node",
])
.style(theme::header_style());
/// Build flat pod rows from grouped pod data — used by both the Pods cockpit
/// and the Overview cockpit's mini-pod panel.
pub fn build_pod_rows<'pods>(pods: &'pods [PodGroupData]) -> Vec<Row<'pods>> {
let mut rows: Vec<Row<'pods>> = Vec::new();
let mut rows: Vec<Row<'_>> = Vec::new();
for group in &state.pods {
for group in pods {
let ready_str = format!("{}/{}", group.ready_count, group.total_count);
let (status_text, status_style) = if group.ready_count == group.total_count
@@ -131,8 +147,25 @@ fn render_pod_table(frame: &mut Frame, area: Rect, state: &AppState) {
}
}
rows
}
fn render_pod_table(frame: &mut Frame, area: Rect, state: &AppState, scroll_offset: usize) {
let header = Row::new(vec![
"Service", "Ready", "Status", "Version", "Restarts", "Age", "Node",
])
.style(theme::header_style());
let all_rows = build_pod_rows(&state.pods);
// Apply scroll offset — skip the first `scroll_offset` rows.
let visible_rows: Vec<Row<'_>> = all_rows
.into_iter()
.skip(scroll_offset)
.collect();
let table = Table::new(
rows,
visible_rows,
[
Constraint::Length(22),
Constraint::Length(7),

980
bin/fxt/src/tui/cockpits/training.rs
View File

File diff suppressed because it is too large Load Diff

341
bin/fxt/src/tui/data_fetcher.rs
View File

@@ -21,12 +21,13 @@ use tonic::Request;
use tracing::{debug, warn};
use crate::grpc::FoxhuntClient;
use crate::proto::{broker_gateway, data_acquisition, ml, monitoring, risk, trading};
use crate::proto::{broker_gateway, data_acquisition, ml, ml_training, monitoring, risk, trading};
use super::state::{
AccountData, AlertData, CircuitBreakerData, ConnectionStatus, DataCacheData, DataFeedData,
ExecutionData, GpuData, ModelStatusData, OrderEventData, PodData, PodGroupData, PositionData,
RiskAlertData, RiskData, ServiceData, SystemResources, TrainingSessionData,
RiskAlertData, RiskData, ServiceData, SystemResources, TrainingHistoryEntry,
TrainingSessionData,
};
/// Empty filter string means "fetch all" in gRPC requests.
@@ -79,6 +80,8 @@ pub enum StateUpdate {
},
/// Cluster pod statuses (from SubscribeClusterPods).
Pods(Vec<PodGroupData>),
/// Completed/failed training runs (from ListTrainingJobs polling).
TrainingHistory(Vec<TrainingHistoryEntry>),
/// gRPC connection status change.
Connection(ConnectionStatus),
}
@@ -121,6 +124,7 @@ impl DataFetcher {
/// Spawn all background stream tasks.
pub fn start(&self, client: &FoxhuntClient) {
self.spawn_training_metrics(client);
self.spawn_training_history(client);
self.spawn_system_status(client);
self.spawn_account_state(client);
self.spawn_executions(client);
@@ -163,7 +167,7 @@ impl DataFetcher {
if !connected_sent.swap(true, Ordering::Relaxed) {
let _ = tx.send(StateUpdate::Connection(ConnectionStatus::Connected));
}
let sessions = m.sessions.iter().map(convert_training_session).collect();
let sessions = dedup_sessions(&m.sessions);
let gpu = m.gpu.as_ref().map(convert_gpu).unwrap_or_default();
let resources = SystemResources {
cpu_usage_pct: f64::from(m.cpu_percent),
@@ -211,6 +215,67 @@ impl DataFetcher {
});
}
/// Training history — polls ListTrainingJobs every 30s for the History tab.
fn spawn_training_history(&self, client: &FoxhuntClient) {
let mut ml_train = client.ml_training();
let tx = self.tx.clone();
let cancel = self.cancel.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(30));
loop {
tokio::select! {
_ = cancel.cancelled() => return,
_ = interval.tick() => {}
}
let req = Request::new(ml_training::ListTrainingJobsRequest {
page: 0,
page_size: 50,
status_filter: 0, // all statuses
model_type_filter: String::new(),
start_time: 0,
end_time: 0,
});
match ml_train.list_training_jobs(req).await {
Ok(resp) => {
let inner = resp.into_inner();
let entries: Vec<TrainingHistoryEntry> = inner
.jobs
.iter()
.map(|j| {
let status = match ml_training::TrainingStatus::try_from(j.status) {
Ok(ml_training::TrainingStatus::Pending) => "pending",
Ok(ml_training::TrainingStatus::Running) => "running",
Ok(ml_training::TrainingStatus::Completed) => "completed",
Ok(ml_training::TrainingStatus::Failed) => "failed",
Ok(ml_training::TrainingStatus::Stopped) => "stopped",
Ok(ml_training::TrainingStatus::Paused) => "paused",
_ => "unknown",
};
TrainingHistoryEntry {
job_id: j.job_id.clone(),
model: j.model_type.clone(),
status: status.to_owned(),
final_loss: j.final_loss,
best_val: j.best_validation_score,
created_at: j.created_at,
started_at: j.started_at,
completed_at: j.completed_at,
description: j.description.clone(),
}
})
.collect();
let _ = tx.send(StateUpdate::TrainingHistory(entries));
}
Err(e) => {
debug!("ListTrainingJobs poll failed: {e}");
}
}
}
});
}
/// Service statuses — poll GetSystemStatus every 5s (the stream event
/// only carries SystemMetrics, not individual ServiceStatus entries).
/// After 3 consecutive failures, reports a connection error to indicate
@@ -871,10 +936,7 @@ impl DataFetcher {
let groups = group_pods(m.pods);
let _ = tx.send(StateUpdate::Pods(groups));
}
Ok(None) => {
debug!("ClusterPods stream ended");
break;
}
Ok(None) => break,
Err(e) => {
warn!("ClusterPods stream error: {e}");
break;
@@ -885,7 +947,7 @@ impl DataFetcher {
}
}
Err(e) => {
debug!("ClusterPods connect failed: {e}");
warn!("ClusterPods connect failed: {e}");
let _ = tx.send(StateUpdate::Connection(ConnectionStatus::Reconnecting {
attempt: backoff.attempts,
}));
@@ -952,19 +1014,68 @@ fn nanos_to_hms(nanos: i64) -> String {
// Proto → state conversions
// ---------------------------------------------------------------------------
/// Deduplicate training sessions by model name, keeping the entry with the
/// highest epoch. The API gateway can report multiple metric rows for the same
/// model (e.g. a stale epoch-0 hyperopt entry alongside the live session).
/// When two entries share a model name but have different folds, the one with
/// the higher epoch wins — fold is metadata, not a separate session.
fn dedup_sessions(raw: &[monitoring::TrainingSession]) -> Vec<TrainingSessionData> {
use std::collections::HashMap;
let mut best: HashMap<&str, &monitoring::TrainingSession> = HashMap::new();
for s in raw {
let key = s.model.as_str();
let entry = best.entry(key).or_insert(s);
if s.current_epoch > entry.current_epoch {
*entry = s;
}
}
let mut sessions: Vec<TrainingSessionData> =
best.into_values().map(convert_training_session).collect();
sessions.sort_by(|a, b| a.model.cmp(&b.model));
sessions
}
fn convert_training_session(s: &monitoring::TrainingSession) -> TrainingSessionData {
TrainingSessionData {
model: s.model.clone(),
fold: s.fold.parse().unwrap_or(0),
fold: s.fold.clone(),
is_hyperopt: s.is_hyperopt,
epoch: s.current_epoch as u32,
max_epochs: 0, // not in proto — cockpit shows epoch only
loss: f64::from(s.epoch_loss),
val_loss: f64::from(s.validation_loss),
batches_per_sec: f64::from(s.batches_per_second),
iteration_secs: f64::from(s.iteration_seconds),
epoch_duration_secs: f64::from(s.epoch_duration_seconds),
sharpe: f64::from(s.epoch_sharpe),
sortino: f64::from(s.epoch_sortino),
win_rate: f64::from(s.epoch_win_rate),
max_dd: f64::from(s.epoch_max_drawdown),
profit_factor: f64::from(s.epoch_profit_factor),
total_return: f64::from(s.epoch_total_return),
total_trades: s.epoch_total_trades,
action_buy_pct: f64::from(s.action_buy_pct),
action_sell_pct: f64::from(s.action_sell_pct),
action_hold_pct: f64::from(s.action_hold_pct),
q_value_mean: f64::from(s.q_value_mean),
q_value_max: f64::from(s.q_value_max),
policy_entropy: f64::from(s.policy_entropy),
kl_divergence: f64::from(s.kl_divergence),
advantage_mean: f64::from(s.advantage_mean),
replay_buffer_size: s.replay_buffer_size,
gradient_norm: f64::from(s.gradient_norm),
learning_rate: f64::from(s.learning_rate),
nan_detected: s.nan_detected,
gradient_explosions: s.gradient_explosions,
feature_errors: s.feature_errors,
checkpoint_saves: s.checkpoint_saves,
checkpoint_failures: s.checkpoint_failures,
hyperopt_trial_current: s.hyperopt_trial_current,
hyperopt_trial_total: s.hyperopt_trial_total,
hyperopt_best_objective: f64::from(s.hyperopt_best_objective),
hyperopt_trials_failed: s.hyperopt_trials_failed,
hyperopt_trial_epoch: s.hyperopt_trial_epoch,
hyperopt_elapsed_secs: f64::from(s.hyperopt_elapsed_seconds),
epoch_history: Vec::new(),
}
}
@@ -1292,3 +1403,213 @@ fn aggregate_cache_stats(jobs: &[data_acquisition::DownloadJobSummary]) -> DataC
newest_date: newest.unwrap_or("--").to_owned(),
}
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
fn make_pod(name: &str, service: &str, ready: bool) -> monitoring::PodInfo {
monitoring::PodInfo {
name: name.into(),
service: service.into(),
namespace: "foxhunt".into(),
status: if ready { "Running".into() } else { "Pending".into() },
restarts: 0,
age: "1h".into(),
node: "node-1".into(),
ready,
version: "v1".into(),
}
}
#[test]
fn group_pods_groups_by_service() {
let pods = vec![
make_pod("api-1", "api-gateway", true),
make_pod("api-2", "api-gateway", true),
make_pod("trade-1", "trading-service", true),
make_pod("trade-2", "trading-service", false),
];
let groups = group_pods(pods);
assert_eq!(groups.len(), 2);
assert_eq!(groups[0].service, "api-gateway");
assert_eq!(groups[0].total_count, 2);
assert_eq!(groups[0].ready_count, 2);
assert_eq!(groups[1].service, "trading-service");
assert_eq!(groups[1].total_count, 2);
assert_eq!(groups[1].ready_count, 1);
}
#[test]
fn group_pods_empty_service_becomes_other() {
let pods = vec![
make_pod("orphan-1", "", true),
make_pod("api-1", "api-gateway", true),
];
let groups = group_pods(pods);
assert_eq!(groups.len(), 2);
// BTreeMap sorts alphabetically: "(other)" < "api-gateway"
assert_eq!(groups[0].service, "(other)");
assert_eq!(groups[0].total_count, 1);
assert_eq!(groups[1].service, "api-gateway");
}
#[test]
fn group_pods_empty_input() {
let groups = group_pods(vec![]);
assert!(groups.is_empty());
}
#[test]
fn group_pods_ready_count_accuracy() {
let pods = vec![
make_pod("a", "svc", true),
make_pod("b", "svc", false),
make_pod("c", "svc", true),
make_pod("d", "svc", false),
make_pod("e", "svc", true),
];
let groups = group_pods(pods);
assert_eq!(groups.len(), 1);
assert_eq!(groups[0].ready_count, 3);
assert_eq!(groups[0].total_count, 5);
}
#[test]
fn group_pods_sorted_by_service_name() {
let pods = vec![
make_pod("z", "zebra", true),
make_pod("a", "alpha", true),
make_pod("m", "middle", true),
];
let groups = group_pods(pods);
let names: Vec<_> = groups.iter().map(|g| g.service.as_str()).collect();
assert_eq!(names, vec!["alpha", "middle", "zebra"]);
}
fn make_training_session(model: &str, epoch: f32) -> monitoring::TrainingSession {
monitoring::TrainingSession {
model: model.into(),
fold: "fold-0".into(),
current_epoch: epoch,
epoch_loss: 0.5,
validation_loss: 0.6,
batches_per_second: 100.0,
iteration_seconds: 0.01,
epoch_duration_seconds: 30.0,
epoch_sharpe: 1.2,
epoch_sortino: 1.5,
epoch_win_rate: 0.55,
epoch_max_drawdown: 0.1,
epoch_profit_factor: 1.8,
epoch_total_return: 0.05,
epoch_total_trades: 100,
action_buy_pct: 0.33,
action_sell_pct: 0.33,
action_hold_pct: 0.34,
q_value_mean: 0.5,
q_value_max: 1.0,
policy_entropy: 0.8,
kl_divergence: 0.01,
advantage_mean: 0.1,
replay_buffer_size: 10000,
gradient_norm: 0.5,
learning_rate: 0.001,
checkpoint_saves: 1,
..Default::default()
}
}
#[test]
fn convert_training_session_maps_fields() {
let proto = make_training_session("DQN", 5.0);
let data = convert_training_session(&proto);
assert_eq!(data.model, "DQN");
assert_eq!(data.epoch, 5);
assert!((data.loss - 0.5).abs() < 1e-6);
assert!((data.sharpe - 1.2).abs() < 1e-4);
assert_eq!(data.total_trades, 100);
assert!(data.epoch_history.is_empty());
}
#[test]
fn convert_training_session_f32_to_f64() {
let proto = make_training_session("PPO", 1.0);
let data = convert_training_session(&proto);
// f32→f64 should preserve value within f32 precision
assert!((data.val_loss - 0.6).abs() < 1e-4);
assert!((data.batches_per_sec - 100.0).abs() < 1e-4);
assert!((data.learning_rate - 0.001).abs() < 1e-6);
}
#[test]
fn convert_gpu_snapshot() {
let snap = monitoring::GpuSnapshot {
utilization_percent: 85.0,
memory_used_mb: 4096.0,
memory_total_mb: 8192.0,
temperature_celsius: 72.0,
power_watts: 150.0,
};
let data = convert_gpu(&snap);
assert!((data.utilization - 85.0).abs() < 1e-6);
assert!((data.memory_used_gb - 4.0).abs() < 1e-6);
assert!((data.memory_total_gb - 8.0).abs() < 1e-6);
assert_eq!(data.temperature_c, 72);
assert_eq!(data.power_watts, 150);
}
fn make_download_job(status: i32, start: &str, end: &str) -> data_acquisition::DownloadJobSummary {
data_acquisition::DownloadJobSummary {
job_id: "j1".into(),
status,
dataset: "ohlcv-1m".into(),
symbols: vec!["ES.FUT".into()],
start_date: start.into(),
end_date: end.into(),
progress_percentage: 100.0,
actual_cost_usd: 0.0,
created_at: 0,
completed_at: 0,
}
}
#[test]
fn aggregate_cache_stats_date_range() {
let completed = data_acquisition::DownloadStatus::Completed as i32;
let jobs = vec![
make_download_job(completed, "2025-01-01", "2025-03-31"),
make_download_job(completed, "2024-07-01", "2024-09-30"),
make_download_job(completed, "2025-04-01", "2025-06-30"),
];
let stats = aggregate_cache_stats(&jobs);
assert_eq!(stats.oldest_date, "2024-07-01");
assert_eq!(stats.newest_date, "2025-06-30");
}
#[test]
fn aggregate_cache_stats_empty_jobs() {
let stats = aggregate_cache_stats(&[]);
assert_eq!(stats.oldest_date, "--");
assert_eq!(stats.newest_date, "--");
assert_eq!(stats.total_records, 0);
}
#[test]
fn aggregate_cache_stats_skips_non_completed() {
let pending = data_acquisition::DownloadStatus::Pending as i32;
let completed = data_acquisition::DownloadStatus::Completed as i32;
let jobs = vec![
make_download_job(pending, "2020-01-01", "2020-12-31"),
make_download_job(completed, "2025-01-01", "2025-03-31"),
];
let stats = aggregate_cache_stats(&jobs);
// Pending job dates should be excluded
assert_eq!(stats.oldest_date, "2025-01-01");
assert_eq!(stats.newest_date, "2025-03-31");
}
}

1265
bin/fxt/src/tui/event_loop.rs
View File

File diff suppressed because it is too large Load Diff

111
bin/fxt/src/tui/state.rs
View File

@@ -25,6 +25,31 @@ impl Default for ConnectionStatus {
}
}
// ---------------------------------------------------------------------------
// Training cockpit sub-tab
// ---------------------------------------------------------------------------
/// Sub-tab within the Training cockpit.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum TrainingTab {
#[default]
Live,
History,
}
/// A completed/failed training run from the history (ListTrainingJobs RPC).
pub struct TrainingHistoryEntry {
pub job_id: String,
pub model: String,
pub status: String,
pub final_loss: f32,
pub best_val: f32,
pub created_at: i64,
pub started_at: i64,
pub completed_at: i64,
pub description: String,
}
// ---------------------------------------------------------------------------
// Top-level state
// ---------------------------------------------------------------------------
@@ -71,6 +96,20 @@ pub struct AppState {
pub selected_pod_group: Option<usize>,
/// Whether the selected pod group is expanded.
pub pod_group_expanded: bool,
/// Scroll offset for the pod table (number of rows to skip).
pub pod_scroll_offset: usize,
/// Currently selected training session index.
pub selected_training_session: Option<usize>,
/// Whether the selected training session detail panel is shown.
pub training_detail_expanded: bool,
/// Active sub-tab in the Training cockpit (Live vs History).
pub training_tab: TrainingTab,
/// Completed/failed training runs from history.
pub training_history: Vec<TrainingHistoryEntry>,
/// Currently selected history entry index.
pub selected_history_entry: Option<usize>,
/// Scroll offset for the per-epoch history table in the detail panel.
pub epoch_scroll_offset: usize,
/// Timestamp of the last data refresh.
pub last_update: Instant,
/// gRPC connection status.
@@ -100,12 +139,44 @@ impl Default for AppState {
pods: Vec::new(),
selected_pod_group: None,
pod_group_expanded: false,
pod_scroll_offset: 0,
selected_training_session: None,
training_detail_expanded: false,
training_tab: TrainingTab::default(),
training_history: Vec::new(),
selected_history_entry: None,
epoch_scroll_offset: 0,
last_update: Instant::now(),
connection_status: ConnectionStatus::default(),
}
}
}
// ---------------------------------------------------------------------------
// Per-epoch snapshot (accumulated from streaming updates)
// ---------------------------------------------------------------------------
/// Snapshot of key metrics at a single epoch boundary.
pub struct EpochSnapshot {
pub epoch: u32,
pub loss: f64,
pub val_loss: f64,
pub sharpe: f64,
pub sortino: f64,
pub win_rate: f64,
pub max_dd: f64,
pub profit_factor: f64,
pub total_return: f64,
pub total_trades: u32,
pub q_value_mean: f64,
pub gradient_norm: f64,
pub learning_rate: f64,
pub policy_entropy: f64,
pub action_buy_pct: f64,
pub action_sell_pct: f64,
pub action_hold_pct: f64,
}
// ---------------------------------------------------------------------------
// Data sub-structs
// ---------------------------------------------------------------------------
@@ -113,16 +184,52 @@ impl Default for AppState {
/// A single ML training session.
pub struct TrainingSessionData {
pub model: String,
pub fold: u32,
pub fold: String,
pub is_hyperopt: bool,
pub epoch: u32,
pub max_epochs: u32,
pub loss: f64,
pub val_loss: f64,
pub batches_per_sec: f64,
pub iteration_secs: f64,
pub epoch_duration_secs: f64,
// Financial metrics
pub sharpe: f64,
pub sortino: f64,
pub win_rate: f64,
pub max_dd: f64,
pub profit_factor: f64,
pub total_return: f64,
pub total_trades: u32,
// Action distribution
pub action_buy_pct: f64,
pub action_sell_pct: f64,
pub action_hold_pct: f64,
// RL diagnostics
pub q_value_mean: f64,
pub q_value_max: f64,
pub policy_entropy: f64,
pub kl_divergence: f64,
pub advantage_mean: f64,
pub replay_buffer_size: u32,
// Gradient & training health
pub gradient_norm: f64,
pub learning_rate: f64,
// Health counters
pub nan_detected: u32,
pub gradient_explosions: u32,
pub feature_errors: u32,
// Checkpoint
pub checkpoint_saves: u32,
pub checkpoint_failures: u32,
// Hyperopt
pub hyperopt_trial_current: u32,
pub hyperopt_trial_total: u32,
pub hyperopt_best_objective: f64,
pub hyperopt_trials_failed: u32,
pub hyperopt_trial_epoch: u32,
pub hyperopt_elapsed_secs: f64,
// Per-epoch history (accumulated across stream updates)
pub epoch_history: Vec<EpochSnapshot>,
}
/// GPU telemetry snapshot.

5
infra/k8s/network-policies/api-gateway.yaml
View File

@@ -87,13 +87,14 @@ spec:
ports:
- protocol: TCP
port: 50057
# Prometheus (metric scraping from monitoring handler)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: monitoring-service
app.kubernetes.io/name: prometheus
ports:
- protocol: TCP
port: 50057
port: 9090
# Minio (initContainer binary fetch)
- to:
- podSelector: