Full migration off Scaleway Container Registry to internal GitLab registry backed by MinIO S3. All 4 images (ci-builder, ci-builder-cpu, foxhunt-runtime, foxhunt-training-runtime) rebuilt in internal registry. Registry & images: - All image refs → gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ - imagePullSecrets: scw-registry → gitlab-registry - Kaniko build template: two-step DAG (git-clone → kaniko-build) with shared PVC - Kaniko layer cache enabled at root/foxhunt/cache - AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY → $MINIO_ACCESS_KEY in .gitlab-ci.yml Network policies: - ci-pipeline: add HTTP/80, registry/5000, webservice/8181 egress rules DNS & Tailscale proxy cleanup: - Remove ci, prometheus, monitor DNS records (no longer exposed) - Rename s3 → minio DNS record - Remove Argo UI, Prometheus, monitor nginx server blocks - Remove argo-htpasswd volume mount - Tailscale proxy nodeSelector: infra → platform Terraform cleanup: - Delete infra/modules/registry/ (SCW CR namespace) - Delete infra/modules/object-storage/ (SCW S3 buckets) - Delete infra/modules/secrets/ (SCW secrets) - Delete corresponding live configs - TF state backend: S3 → GitLab HTTP Argo workflows: - Add events/ (GitLab push eventsource + ci-pipeline sensor) - ci-pipeline + training templates: SCW → internal registry - Delete obsolete compile-training-template.yaml Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
159 lines
6.0 KiB
Rust
159 lines
6.0 KiB
Rust
/// Watch tuning progress with live updates (poll every 5 seconds)
|
|
async fn watch_tuning_progress(
|
|
api_url: &str,
|
|
jwt_token: &str,
|
|
job_id: &str,
|
|
) -> AnyhowResult<()> {
|
|
use tokio::time::{sleep, Duration};
|
|
|
|
let mut last_trial: u32 = 0;
|
|
let mut iteration: u32 = 0;
|
|
|
|
// Validate job ID format once at the start
|
|
let job_id_uuid = Uuid::parse_str(job_id)
|
|
.context("❌ Invalid job ID format (expected UUID)")?;
|
|
|
|
// Create gRPC client once (reuse connection)
|
|
let mut client = MlTrainingServiceClient::connect(api_url.to_string())
|
|
.await
|
|
.context("Failed to connect to API")?;
|
|
|
|
loop {
|
|
iteration += 1;
|
|
|
|
// Create gRPC request with JWT metadata
|
|
let mut request = tonic::Request::new(GetTuningJobStatusRequest {
|
|
job_id: job_id_uuid.to_string(),
|
|
});
|
|
|
|
request.metadata_mut().insert(
|
|
"authorization",
|
|
format!("Bearer {}", jwt_token)
|
|
.parse()
|
|
.context("Failed to parse JWT token")?
|
|
);
|
|
|
|
// Execute gRPC call to get live status
|
|
let response = client
|
|
.get_tuning_job_status(request)
|
|
.await
|
|
.context("Failed to get tuning job status")?;
|
|
|
|
let status_response = response.into_inner();
|
|
|
|
// Calculate progress percentage
|
|
let progress_percent = if status_response.total_trials > 0 {
|
|
(status_response.current_trial as f32 / status_response.total_trials as f32) * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
// Calculate elapsed time
|
|
let elapsed_seconds = if status_response.started_at > 0 {
|
|
chrono::Utc::now().timestamp() - status_response.started_at
|
|
} else {
|
|
0
|
|
};
|
|
|
|
// Extract best Sharpe ratio from metrics
|
|
let best_sharpe_ratio = status_response
|
|
.best_metrics
|
|
.get("sharpe_ratio")
|
|
.copied()
|
|
.unwrap_or(0.0);
|
|
|
|
// Format status string
|
|
let status_str = format_tuning_status(status_response.status());
|
|
|
|
// Clear previous output (move cursor up and clear lines)
|
|
if iteration > 1 {
|
|
// Clear the previous display (9 lines)
|
|
print!("\x1B[9A\x1B[J");
|
|
}
|
|
|
|
// Display rich progress UI
|
|
println!("┌─────────────────────────────────────────────────────────┐");
|
|
println!("│ {} Tuning Job: {} │", "🎯".bright_cyan(), job_id.chars().take(8).collect::<String>());
|
|
println!("├─────────────────────────────────────────────────────────┤");
|
|
println!("│ Trials: {}/{} ({:.1}%) │",
|
|
status_response.current_trial,
|
|
status_response.total_trials,
|
|
progress_percent
|
|
);
|
|
println!("│ {} Best Sharpe Ratio: {} │",
|
|
"🏆".bright_yellow(),
|
|
format!("{:.4}", best_sharpe_ratio).bright_green()
|
|
);
|
|
|
|
// Show trial progress indicator if trial changed
|
|
if status_response.current_trial > last_trial {
|
|
println!("│ {} Current Trial #{}: Running... │",
|
|
"🔄".bright_blue(),
|
|
status_response.current_trial
|
|
);
|
|
last_trial = status_response.current_trial;
|
|
} else {
|
|
println!("│ {} Status: {} │",
|
|
"📊".bright_white(),
|
|
format_status_colored(&status_str)
|
|
);
|
|
}
|
|
|
|
// Progress bar
|
|
let progress_bar = create_progress_bar(progress_percent);
|
|
println!("│ {} │", progress_bar);
|
|
|
|
// Elapsed time
|
|
let elapsed_minutes = elapsed_seconds / 60;
|
|
let elapsed_seconds_remainder = elapsed_seconds % 60;
|
|
println!("│ ⏱️ Elapsed: {}m {}s │",
|
|
elapsed_minutes, elapsed_seconds_remainder
|
|
);
|
|
println!("└─────────────────────────────────────────────────────────┘");
|
|
|
|
// Check if job is complete
|
|
match status_response.status() {
|
|
TuningJobStatus::TuningCompleted => {
|
|
println!("\n✅ Tuning job completed successfully!");
|
|
println!(" Best Sharpe Ratio: {}", format!("{:.4}", best_sharpe_ratio).bright_green());
|
|
println!("\n💡 Get best parameters with:");
|
|
println!(" tli tune best --job-id {}", job_id);
|
|
break;
|
|
}
|
|
TuningJobStatus::TuningFailed => {
|
|
println!("\n❌ Tuning job failed!");
|
|
if !status_response.message.is_empty() {
|
|
println!(" Error: {}", status_response.message);
|
|
}
|
|
break;
|
|
}
|
|
TuningJobStatus::TuningStopped => {
|
|
println!("\n🛑 Tuning job stopped by user");
|
|
println!("\n💡 Get partial results with:");
|
|
println!(" tli tune best --job-id {}", job_id);
|
|
break;
|
|
}
|
|
_ => {
|
|
// Still running, continue polling
|
|
}
|
|
}
|
|
|
|
// Wait 5 seconds before next poll
|
|
sleep(Duration::from_secs(5)).await;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Format tuning job status as string
|
|
fn format_tuning_status(status: TuningJobStatus) -> String {
|
|
match status {
|
|
TuningJobStatus::TuningUnknown => "UNKNOWN",
|
|
TuningJobStatus::TuningPending => "PENDING",
|
|
TuningJobStatus::TuningRunning => "RUNNING",
|
|
TuningJobStatus::TuningCompleted => "COMPLETED",
|
|
TuningJobStatus::TuningFailed => "FAILED",
|
|
TuningJobStatus::TuningStopped => "STOPPED",
|
|
}.to_string()
|
|
}
|