diff --git a/bin/fxt/src/commands/train/monitor.rs b/bin/fxt/src/commands/train/monitor.rs index 94496fbac..410fa4784 100644 --- a/bin/fxt/src/commands/train/monitor.rs +++ b/bin/fxt/src/commands/train/monitor.rs @@ -33,9 +33,16 @@ pub struct MonitorCommand { impl MonitorCommand { /// Execute the monitor command -- connect to monitoring service and display metrics pub async fn run(&self, api_gateway_url: &str, _jwt_token: &str) -> Result<()> { - // monitoring_service runs on a different port -- derive URL from env or default - let monitoring_url = std::env::var("MONITORING_SERVICE_URL") - .unwrap_or_else(|_| api_gateway_url.replace(":50050", ":50057")); + // monitoring_service runs on a different port -- derive URL from env or default. + // When connecting via Tailscale (*.fxhnt.ai), use monitor.fxhnt.ai:443 (nginx proxy). + // Otherwise, replace the api-gateway port with the monitoring port. + let monitoring_url = std::env::var("MONITORING_SERVICE_URL").unwrap_or_else(|_| { + if api_gateway_url.contains("fxhnt.ai") { + "https://monitor.fxhnt.ai".to_owned() + } else { + api_gateway_url.replace(":50050", ":50057") + } + }); let mut client = MonitoringServiceClient::connect(monitoring_url) .await @@ -72,11 +79,6 @@ impl MonitorCommand { } fn render_snapshot(resp: &GetLiveTrainingMetricsResponse) { - if resp.sessions.is_empty() { - println!("No active training sessions found."); - return; - } - if let Some(gpu) = &resp.gpu { println!( "GPU: {:.0}% util | {:.1}/{:.1} GB VRAM | {:.0}C | {:.0}W", @@ -90,6 +92,11 @@ fn render_snapshot(resp: &GetLiveTrainingMetricsResponse) { println!("Active K8s training jobs: {}", resp.active_k8s_jobs); println!(); + if resp.sessions.is_empty() { + println!("No active training sessions with metrics."); + return; + } + print_session_table(&resp.sessions); print_hyperopt_summary(&resp.sessions); print_health_summary(&resp.sessions); diff --git a/infra/k8s/gitlab/tailscale-proxy.yaml b/infra/k8s/gitlab/tailscale-proxy.yaml index 9b0d837f6..716d87699 100644 --- a/infra/k8s/gitlab/tailscale-proxy.yaml +++ b/infra/k8s/gitlab/tailscale-proxy.yaml @@ -253,3 +253,21 @@ data: grpc_connect_timeout 30s; } } + + # Monitoring Service (gRPC) — monitor.fxhnt.ai + server { + listen 443 ssl; + http2 on; + server_name monitor.fxhnt.ai; + + ssl_certificate /etc/nginx/certs/tls.crt; + ssl_certificate_key /etc/nginx/certs/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + + location / { + grpc_pass grpc://monitoring-service.foxhunt.svc.cluster.local:50057; + grpc_read_timeout 3600s; + grpc_send_timeout 3600s; + grpc_connect_timeout 30s; + } + } diff --git a/infra/k8s/network-policies/monitoring-service.yaml b/infra/k8s/network-policies/monitoring-service.yaml index f5ae526bb..9b82d3caa 100644 --- a/infra/k8s/network-policies/monitoring-service.yaml +++ b/infra/k8s/network-policies/monitoring-service.yaml @@ -11,16 +11,19 @@ spec: - Ingress - Egress ingress: - # gRPC from web-gateway + # gRPC from web-gateway and Tailscale proxy (monitor.fxhnt.ai) - from: - podSelector: matchLabels: app.kubernetes.io/name: web-gateway + - podSelector: + matchLabels: + app.kubernetes.io/name: tailscale-gitlab-proxy ports: - protocol: TCP port: 50057 egress: - # Prometheus HTTP API (port 80) + # Prometheus HTTP API - to: - podSelector: matchLabels: @@ -28,10 +31,5 @@ spec: ports: - protocol: TCP port: 80 - - to: - - podSelector: - matchLabels: - app.kubernetes.io/name: prometheus - ports: - protocol: TCP port: 9090 diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf index a4be0aa0b..94ca1f3e7 100644 --- a/infra/modules/dns/main.tf +++ b/infra/modules/dns/main.tf @@ -39,3 +39,11 @@ resource "scaleway_domain_record" "api" { data = var.git_ip ttl = 300 } + +resource "scaleway_domain_record" "monitor" { + dns_zone = var.dns_zone + name = "monitor" + type = "A" + data = var.git_ip + ttl = 300 +} diff --git a/infra/modules/dns/outputs.tf b/infra/modules/dns/outputs.tf index 71a2e6145..2d2daea89 100644 --- a/infra/modules/dns/outputs.tf +++ b/infra/modules/dns/outputs.tf @@ -17,3 +17,8 @@ output "api_fqdn" { description = "FQDN for the API Gateway (gRPC)" value = "${scaleway_domain_record.api.name}.${scaleway_domain_record.api.dns_zone}" } + +output "monitor_fqdn" { + description = "FQDN for the Monitoring Service (gRPC)" + value = "${scaleway_domain_record.monitor.name}.${scaleway_domain_record.monitor.dns_zone}" +}