Files
foxhunt/testing/integration/standalone/watch_tuning_progress_updated.rs
jgrusewski 68b6aa8313 feat(infra): migrate container registry from SCW to internal GitLab
Full migration off Scaleway Container Registry to internal GitLab
registry backed by MinIO S3. All 4 images (ci-builder, ci-builder-cpu,
foxhunt-runtime, foxhunt-training-runtime) rebuilt in internal registry.

Registry & images:
- All image refs → gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/
- imagePullSecrets: scw-registry → gitlab-registry
- Kaniko build template: two-step DAG (git-clone → kaniko-build) with shared PVC
- Kaniko layer cache enabled at root/foxhunt/cache
- AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY → $MINIO_ACCESS_KEY in .gitlab-ci.yml

Network policies:
- ci-pipeline: add HTTP/80, registry/5000, webservice/8181 egress rules

DNS & Tailscale proxy cleanup:
- Remove ci, prometheus, monitor DNS records (no longer exposed)
- Rename s3 → minio DNS record
- Remove Argo UI, Prometheus, monitor nginx server blocks
- Remove argo-htpasswd volume mount
- Tailscale proxy nodeSelector: infra → platform

Terraform cleanup:
- Delete infra/modules/registry/ (SCW CR namespace)
- Delete infra/modules/object-storage/ (SCW S3 buckets)
- Delete infra/modules/secrets/ (SCW secrets)
- Delete corresponding live configs
- TF state backend: S3 → GitLab HTTP

Argo workflows:
- Add events/ (GitLab push eventsource + ci-pipeline sensor)
- ci-pipeline + training templates: SCW → internal registry
- Delete obsolete compile-training-template.yaml

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 13:52:24 +01:00

159 lines
6.0 KiB
Rust

/// Watch tuning progress with live updates (poll every 5 seconds)
async fn watch_tuning_progress(
api_url: &str,
jwt_token: &str,
job_id: &str,
) -> AnyhowResult<()> {
use tokio::time::{sleep, Duration};
let mut last_trial: u32 = 0;
let mut iteration: u32 = 0;
// Validate job ID format once at the start
let job_id_uuid = Uuid::parse_str(job_id)
.context("❌ Invalid job ID format (expected UUID)")?;
// Create gRPC client once (reuse connection)
let mut client = MlTrainingServiceClient::connect(api_url.to_string())
.await
.context("Failed to connect to API")?;
loop {
iteration += 1;
// Create gRPC request with JWT metadata
let mut request = tonic::Request::new(GetTuningJobStatusRequest {
job_id: job_id_uuid.to_string(),
});
request.metadata_mut().insert(
"authorization",
format!("Bearer {}", jwt_token)
.parse()
.context("Failed to parse JWT token")?
);
// Execute gRPC call to get live status
let response = client
.get_tuning_job_status(request)
.await
.context("Failed to get tuning job status")?;
let status_response = response.into_inner();
// Calculate progress percentage
let progress_percent = if status_response.total_trials > 0 {
(status_response.current_trial as f32 / status_response.total_trials as f32) * 100.0
} else {
0.0
};
// Calculate elapsed time
let elapsed_seconds = if status_response.started_at > 0 {
chrono::Utc::now().timestamp() - status_response.started_at
} else {
0
};
// Extract best Sharpe ratio from metrics
let best_sharpe_ratio = status_response
.best_metrics
.get("sharpe_ratio")
.copied()
.unwrap_or(0.0);
// Format status string
let status_str = format_tuning_status(status_response.status());
// Clear previous output (move cursor up and clear lines)
if iteration > 1 {
// Clear the previous display (9 lines)
print!("\x1B[9A\x1B[J");
}
// Display rich progress UI
println!("┌─────────────────────────────────────────────────────────┐");
println!("{} Tuning Job: {}", "🎯".bright_cyan(), job_id.chars().take(8).collect::<String>());
println!("├─────────────────────────────────────────────────────────┤");
println!("│ Trials: {}/{} ({:.1}%) │",
status_response.current_trial,
status_response.total_trials,
progress_percent
);
println!("{} Best Sharpe Ratio: {}",
"🏆".bright_yellow(),
format!("{:.4}", best_sharpe_ratio).bright_green()
);
// Show trial progress indicator if trial changed
if status_response.current_trial > last_trial {
println!("{} Current Trial #{}: Running... │",
"🔄".bright_blue(),
status_response.current_trial
);
last_trial = status_response.current_trial;
} else {
println!("{} Status: {}",
"📊".bright_white(),
format_status_colored(&status_str)
);
}
// Progress bar
let progress_bar = create_progress_bar(progress_percent);
println!("{}", progress_bar);
// Elapsed time
let elapsed_minutes = elapsed_seconds / 60;
let elapsed_seconds_remainder = elapsed_seconds % 60;
println!("│ ⏱️ Elapsed: {}m {}s │",
elapsed_minutes, elapsed_seconds_remainder
);
println!("└─────────────────────────────────────────────────────────┘");
// Check if job is complete
match status_response.status() {
TuningJobStatus::TuningCompleted => {
println!("\n✅ Tuning job completed successfully!");
println!(" Best Sharpe Ratio: {}", format!("{:.4}", best_sharpe_ratio).bright_green());
println!("\n💡 Get best parameters with:");
println!(" tli tune best --job-id {}", job_id);
break;
}
TuningJobStatus::TuningFailed => {
println!("\n❌ Tuning job failed!");
if !status_response.message.is_empty() {
println!(" Error: {}", status_response.message);
}
break;
}
TuningJobStatus::TuningStopped => {
println!("\n🛑 Tuning job stopped by user");
println!("\n💡 Get partial results with:");
println!(" tli tune best --job-id {}", job_id);
break;
}
_ => {
// Still running, continue polling
}
}
// Wait 5 seconds before next poll
sleep(Duration::from_secs(5)).await;
}
Ok(())
}
/// Format tuning job status as string
fn format_tuning_status(status: TuningJobStatus) -> String {
match status {
TuningJobStatus::TuningUnknown => "UNKNOWN",
TuningJobStatus::TuningPending => "PENDING",
TuningJobStatus::TuningRunning => "RUNNING",
TuningJobStatus::TuningCompleted => "COMPLETED",
TuningJobStatus::TuningFailed => "FAILED",
TuningJobStatus::TuningStopped => "STOPPED",
}.to_string()
}