Files
foxhunt/testing/e2e/src/bin/service_orchestrator.rs
jgrusewski 08d070bb95 refactor: rename JWT issuer foxhunt-api-gateway → foxhunt-api, drop compat aliases
- JWT issuer now foxhunt-api across all 16 files (services, tests, config, docker-compose)
- Remove serde alias api_gateway_url from FxtConfig (no backwards compat)
- Remove api_gateway CLI alias from e2e orchestrator
- All services must deploy simultaneously for JWT validation to match

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 00:22:04 +01:00

848 lines
30 KiB
Rust

//! Service Orchestrator for E2E Testing
//!
//! **CRITICAL ARCHITECTURAL ISSUE**:
//! This orchestrator currently starts services on individual ports (50051, 50052, 50053, etc.)
//! WITHOUT starting the API Gateway. This violates the Foxhunt architecture where ALL client
//! connections must go through API Gateway (port 50051) with JWT authentication.
//!
//! **Current Behavior** (INCORRECT):
//! - Trading Service: port 50051 (directly exposed)
//! - Backtesting Service: port 50052 (directly exposed)
//! - ML Training Service: port 50053 (directly exposed)
//! - NO API Gateway running
//!
//! **Correct Architecture** (per CLAUDE.md):
//! - API Gateway: port 50051 (single entry point with JWT auth)
//! - Trading Service: port 50052 (behind gateway)
//! - Backtesting Service: port 50053 (behind gateway)
//! - ML Training Service: port 50054 (behind gateway)
//!
//! **Impact**:
//! E2ETestFramework correctly tries to connect to API Gateway (50051) but finds Trading Service
//! instead, causing authentication failures and incorrect routing.
//!
//! **Fix Required**:
//! 1. Add API Gateway startup logic on port 50051
//! 2. Adjust backend service ports to 50052+ (not 50051+)
//! 3. Configure API Gateway to route to backend services
//! 4. Ensure JWT_SECRET environment variable is set for authentication
//!
//! See: Wave 2 Agent 19 - E2E Fix (WAVE_2_AGENT_19_E2E_FIX.md)
use anyhow::Result;
use clap::{Arg, ArgMatches, Command};
use foxhunt_e2e::{
database::TestDatabase,
services::{ServiceConfig, ServiceManager, ServiceType},
utils::{PerformanceProfiler, TestUtils},
};
use std::collections::HashMap;
use std::time::Duration;
use tokio::signal;
use tracing::{debug, info, warn};
#[tokio::main]
async fn main() -> Result<()> {
TestUtils::setup_test_logging();
let matches = build_cli().get_matches();
match matches.subcommand() {
Some(("start", sub_matches)) => start_services(sub_matches).await,
Some(("stop", sub_matches)) => stop_services(sub_matches).await,
Some(("restart", sub_matches)) => restart_services(sub_matches).await,
Some(("status", _)) => check_status().await,
Some(("logs", sub_matches)) => show_logs(sub_matches).await,
Some(("benchmark", sub_matches)) => run_benchmark(sub_matches).await,
_ => {
eprintln!("Use --help for available commands");
Ok(())
},
}
}
fn build_cli() -> Command {
Command::new("foxhunt-service-orchestrator")
.version("1.0.0")
.about("Foxhunt Service Orchestrator for E2E Testing")
.subcommand(
Command::new("start")
.about("Start services for E2E testing")
.arg(
Arg::new("services")
.long("services")
.short('s')
.value_name("SERVICE_LIST")
.help("Comma-separated list of services to start (trading,backtesting,ml_training,database,all)")
.default_value("all")
)
.arg(
Arg::new("wait")
.long("wait")
.short('w')
.action(clap::ArgAction::SetTrue)
.help("Wait for all services to be ready before returning")
)
.arg(
Arg::new("timeout")
.long("timeout")
.value_name("SECONDS")
.help("Startup timeout in seconds")
.default_value("120")
)
.arg(
Arg::new("port-base")
.long("port-base")
.value_name("PORT")
.help("Base port for services (trading=base, backtesting=base+1, ml=base+2)")
.default_value("50051")
)
.arg(
Arg::new("background")
.long("background")
.short('d')
.action(clap::ArgAction::SetTrue)
.help("Run services in background (daemon mode)")
)
)
.subcommand(
Command::new("stop")
.about("Stop running services")
.arg(
Arg::new("services")
.long("services")
.short('s')
.value_name("SERVICE_LIST")
.help("Comma-separated list of services to stop (trading,backtesting,ml_training,database,all)")
.default_value("all")
)
.arg(
Arg::new("force")
.long("force")
.short('f')
.action(clap::ArgAction::SetTrue)
.help("Force kill services if graceful shutdown fails")
)
)
.subcommand(
Command::new("restart")
.about("Restart services")
.arg(
Arg::new("services")
.long("services")
.short('s')
.value_name("SERVICE_LIST")
.help("Comma-separated list of services to restart")
.default_value("all")
)
)
.subcommand(
Command::new("status")
.about("Check status of all services")
)
.subcommand(
Command::new("logs")
.about("Show service logs")
.arg(
Arg::new("service")
.value_name("SERVICE")
.help("Service name to show logs for")
.required(true)
)
.arg(
Arg::new("follow")
.long("follow")
.short('f')
.action(clap::ArgAction::SetTrue)
.help("Follow log output")
)
.arg(
Arg::new("lines")
.long("lines")
.short('n')
.value_name("COUNT")
.help("Number of lines to show")
.default_value("100")
)
)
.subcommand(
Command::new("benchmark")
.about("Run service performance benchmarks")
.arg(
Arg::new("duration")
.long("duration")
.short('d')
.value_name("SECONDS")
.help("Benchmark duration in seconds")
.default_value("60")
)
.arg(
Arg::new("connections")
.long("connections")
.short('c')
.value_name("COUNT")
.help("Number of concurrent connections")
.default_value("10")
)
)
}
async fn start_services(matches: &ArgMatches) -> Result<()> {
let services_arg = matches.get_one::<String>("services").unwrap();
let wait_ready = matches.get_flag("wait");
let timeout: u64 = matches.get_one::<String>("timeout").unwrap().parse()?;
let background = matches.get_flag("background");
// API Gateway gets port 50051, backend services start at 50052
let api_port: u16 = 50051;
let backend_base_port: u16 = 50052;
// Get JWT_SECRET from environment (required for API Gateway)
let jwt_secret = std::env::var("JWT_SECRET").unwrap_or_else(|_| {
warn!("JWT_SECRET not set, using default development secret");
"dev_secret_key_change_in_production".to_string()
});
// Backend service URLs for API Gateway configuration
let trading_service_url = format!("http://localhost:{}", backend_base_port);
let backtesting_service_url = format!("http://localhost:{}", backend_base_port + 1);
let ml_training_service_url = format!("http://localhost:{}", backend_base_port + 2);
info!("Starting services: {}", services_arg);
info!("API service port: {}", api_port);
info!("Backend services starting at port: {}", backend_base_port);
info!("Background mode: {}", background);
let services_to_start = parse_service_list(services_arg)?;
let mut profiler = PerformanceProfiler::new();
// Initialize service manager
let mut service_manager = ServiceManager::new();
profiler.checkpoint("service_manager_init");
// Start database first if requested
if services_to_start.contains(&ServiceType::Database) {
info!("Starting database service...");
let db_harness = TestDatabase::new("postgresql://localhost/foxhunt_test".to_string());
db_harness.setup().await?;
profiler.checkpoint("database_started");
// Wait for database to be ready
TestUtils::wait_for_condition(
|| async {
TestUtils::check_service_health("http://localhost:5432")
.await
.unwrap_or(false)
},
30,
1000,
)
.await?;
info!("✅ Database service is ready");
}
// Start API Gateway FIRST if requested
if services_to_start.contains(&ServiceType::ApiGateway) || services_to_start.len() > 1 {
info!("Starting API service on port {}...", api_port);
let mut api_env = HashMap::new();
api_env.insert("JWT_SECRET".to_string(), jwt_secret.clone());
api_env.insert(
"TRADING_SERVICE_URL".to_string(),
trading_service_url.clone(),
);
api_env.insert(
"BACKTESTING_SERVICE_URL".to_string(),
backtesting_service_url.clone(),
);
api_env.insert(
"ML_TRAINING_SERVICE_URL".to_string(),
ml_training_service_url.clone(),
);
api_env.insert("GRPC_PORT".to_string(), api_port.to_string());
api_env.insert("HTTP_PORT".to_string(), "8080".to_string());
api_env.insert("METRICS_PORT".to_string(), "9091".to_string());
api_env.insert("RUST_LOG".to_string(), "info".to_string());
api_env.insert("FOXHUNT_TEST_MODE".to_string(), "true".to_string());
let api_config = ServiceConfig {
service_type: ServiceType::ApiGateway,
executable_path: "target/debug/api".to_string(),
port: api_port,
health_endpoint: "http://localhost:8080/health".to_string(),
startup_timeout: Duration::from_secs(30),
environment: api_env,
working_directory: std::env::current_dir()?,
log_file: Some("/tmp/foxhunt_api_service.log".to_string()),
};
service_manager.start_service(api_config).await?;
profiler.checkpoint("api_started");
// Wait for API Gateway to be ready
if wait_ready {
TestUtils::wait_for_condition(
|| async {
TestUtils::check_service_health("http://localhost:8080/health")
.await
.unwrap_or(false)
},
timeout,
2000,
)
.await?;
info!("✅ API Gateway service is ready");
}
}
// Start backend services on ports 50052+
for service_type in services_to_start.iter() {
if matches!(
service_type,
ServiceType::Database | ServiceType::ApiGateway
) {
continue; // Already started
}
let port = match service_type {
ServiceType::TradingService => backend_base_port,
ServiceType::BacktestingService => backend_base_port + 1,
ServiceType::MLTrainingService => backend_base_port + 2,
_ => continue,
};
let config = create_service_config(service_type, port)?;
info!(
"Starting {} service on port {}...",
service_type.as_str(),
port
);
service_manager.start_service(config).await?;
profiler.checkpoint(&format!("{}_started", service_type.as_str()));
}
if wait_ready {
info!("Waiting for all services to be ready...");
// Wait for services to be healthy
for service_type in &services_to_start {
if matches!(service_type, ServiceType::Database) {
continue;
}
let endpoint = match service_type {
ServiceType::ApiGateway => "http://localhost:8080/health".to_string(),
ServiceType::TradingService => format!("http://localhost:{}", backend_base_port),
ServiceType::BacktestingService => {
format!("http://localhost:{}", backend_base_port + 1)
},
ServiceType::MLTrainingService => {
format!("http://localhost:{}", backend_base_port + 2)
},
ServiceType::Database => continue, // Already handled above
};
TestUtils::wait_for_condition(
|| async {
TestUtils::check_service_health(&endpoint)
.await
.unwrap_or(false)
},
timeout,
2000,
)
.await
.map_err(|_| {
anyhow::anyhow!(
"Service {} failed to become ready within {}s",
service_type.as_str(),
timeout
)
})?;
info!("✅ {} service is ready", service_type.as_str());
}
profiler.checkpoint("all_services_ready");
profiler.print_summary();
}
if background {
info!("Services started in background mode");
info!("Use 'service_orchestrator status' to check service status");
info!("Use 'service_orchestrator stop' to stop services");
// Keep running until interrupted
signal::ctrl_c().await?;
info!("Received interrupt signal, shutting down services...");
service_manager.stop_all_services().await?;
info!("All services stopped");
} else {
info!("Services started in foreground mode");
info!("Press Ctrl+C to stop all services");
// Wait for interrupt signal
signal::ctrl_c().await?;
info!("Received interrupt signal, shutting down services...");
service_manager.stop_all_services().await?;
info!("All services stopped");
}
Ok(())
}
async fn stop_services(matches: &ArgMatches) -> Result<()> {
let services_arg = matches.get_one::<String>("services").unwrap();
let force = matches.get_flag("force");
info!("Stopping services: {}", services_arg);
let services_to_stop = parse_service_list(services_arg)?;
let _service_manager = ServiceManager::new();
for service_type in &services_to_stop {
info!("Stopping {} service...", service_type.as_str());
if force {
// Force kill the service
match tokio::process::Command::new("pkill")
.args(["-f", &format!("{}_service", service_type.as_str())])
.output()
.await
{
Ok(_) => info!("Force killed {} service", service_type.as_str()),
Err(e) => warn!(
"Failed to force kill {} service: {}",
service_type.as_str(),
e
),
}
} else {
// Graceful shutdown
// Note: This would typically send SIGTERM to the service
info!("Gracefully stopping {} service", service_type.as_str());
}
}
// Stop database last
if services_to_stop.contains(&ServiceType::Database) {
info!("Stopping database service...");
// Database cleanup would be handled by DatabaseTestHarness
}
info!("All requested services stopped");
Ok(())
}
async fn restart_services(matches: &ArgMatches) -> Result<()> {
let services_arg = matches.get_one::<String>("services").unwrap();
info!("Restarting services: {}", services_arg);
// Stop services first
let stop_matches = Command::new("stop")
.arg(Arg::new("services").short('s').long("services"))
.arg(Arg::new("force").action(clap::ArgAction::SetTrue))
.get_matches_from(vec!["stop", "--services", services_arg.as_str()]);
stop_services(&stop_matches).await?;
// Wait a moment for cleanup
tokio::time::sleep(Duration::from_secs(2)).await;
// Start services
let start_matches = Command::new("start")
.arg(Arg::new("services").short('s').long("services"))
.arg(Arg::new("wait").action(clap::ArgAction::SetTrue))
.get_matches_from(vec!["start", "--services", services_arg.as_str(), "--wait"]);
start_services(&start_matches).await?;
Ok(())
}
async fn check_status() -> Result<()> {
println!("🔍 Checking Foxhunt Service Status\n");
let services = [
("API Gateway", "http://localhost:8080/health"),
("Trading Service", "http://localhost:8081/health"),
("Backtesting Service", "http://localhost:8082/health"),
("ML Training Service", "http://localhost:8095/health"),
(
"PostgreSQL Database",
"postgresql://localhost:5432/foxhunt_test",
),
];
let mut all_healthy = true;
for (name, endpoint) in services {
print!("Checking {}... ", name);
let healthy = if endpoint.starts_with("http") {
TestUtils::check_service_health(endpoint)
.await
.unwrap_or(false)
} else {
// Database connection check
check_database_connection().await.unwrap_or(false)
};
if healthy {
println!("✅ Healthy");
} else {
println!("❌ Unhealthy");
all_healthy = false;
}
}
println!();
if all_healthy {
println!("🎉 All services are healthy and ready for E2E testing!");
} else {
println!("⚠️ Some services are not healthy. Check logs for details.");
println!(" Run 'service_orchestrator start --wait' to start missing services.");
}
// Check for running test processes
match tokio::process::Command::new("pgrep")
.args(["-f", "foxhunt"])
.output()
.await
{
Ok(output) => {
if !output.stdout.is_empty() {
let pids = String::from_utf8_lossy(&output.stdout);
println!("🔄 Running Foxhunt processes:");
for pid in pids.lines() {
if !pid.trim().is_empty() {
println!(" PID: {}", pid.trim());
}
}
}
},
Err(_) => debug!("Could not check for running processes"),
}
Ok(())
}
async fn show_logs(matches: &ArgMatches) -> Result<()> {
let service = matches.get_one::<String>("service").unwrap();
let follow = matches.get_flag("follow");
let lines: usize = matches.get_one::<String>("lines").unwrap().parse()?;
info!(
"Showing logs for {} service (last {} lines)",
service, lines
);
let log_file = format!("/tmp/foxhunt_{}_service.log", service);
if !tokio::fs::try_exists(&log_file).await.unwrap_or(false) {
println!("❌ Log file not found: {}", log_file);
println!(" Services may not be running or logging to a different location.");
return Ok(());
}
if follow {
// Follow log file
let mut command = tokio::process::Command::new("tail");
command.args(["-f", "-n", &lines.to_string(), &log_file]);
let mut child = command.spawn()?;
// Handle Ctrl+C to stop following
tokio::select! {
_ = child.wait() => {},
_ = signal::ctrl_c() => {
child.kill().await?;
}
}
} else {
// Show last N lines
let output = tokio::process::Command::new("tail")
.args(["-n", &lines.to_string(), &log_file])
.output()
.await?;
println!("{}", String::from_utf8_lossy(&output.stdout));
}
Ok(())
}
async fn run_benchmark(matches: &ArgMatches) -> Result<()> {
let duration: u64 = matches.get_one::<String>("duration").unwrap().parse()?;
let connections: u32 = matches.get_one::<String>("connections").unwrap().parse()?;
info!("Running service performance benchmark");
info!("Duration: {}s, Connections: {}", duration, connections);
let mut profiler = PerformanceProfiler::new();
// Check if services are running
let services = [
("Trading", "http://localhost:50051/health"),
("Backtesting", "http://localhost:50052/health"),
("ML Training", "http://localhost:50053/health"),
];
println!("🚀 Starting benchmark...\n");
for (name, endpoint) in services {
if !TestUtils::check_service_health(endpoint)
.await
.unwrap_or(false)
{
warn!("Service {} is not running, skipping benchmark", name);
continue;
}
println!("📊 Benchmarking {} Service", name);
// Simple load test - make concurrent requests
let start = tokio::time::Instant::now();
let mut tasks = Vec::new();
for i in 0..connections {
let endpoint = endpoint.to_string();
let task = tokio::spawn(async move {
let client = reqwest::Client::new();
let mut request_count = 0;
let mut error_count = 0;
let test_duration = tokio::time::Duration::from_secs(duration);
let start_time = tokio::time::Instant::now();
while start_time.elapsed() < test_duration {
match client.get(&endpoint).send().await {
Ok(response) => {
request_count += 1;
if !response.status().is_success() {
error_count += 1;
}
},
Err(_) => {
error_count += 1;
},
}
// Small delay between requests
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
(i, request_count, error_count)
});
tasks.push(task);
}
// Wait for all tasks to complete
let mut total_requests = 0;
let mut total_errors = 0;
for task in tasks {
let (_, requests, errors) = task.await?;
total_requests += requests;
total_errors += errors;
}
let elapsed = start.elapsed();
let requests_per_second = total_requests as f64 / elapsed.as_secs_f64();
let error_rate = if total_requests > 0 {
(total_errors as f64 / total_requests as f64) * 100.0
} else {
0.0
};
println!(" Total Requests: {}", total_requests);
println!(" Total Errors: {}", total_errors);
println!(" Requests/sec: {:.2}", requests_per_second);
println!(" Error Rate: {:.2}%", error_rate);
println!(" Duration: {:?}", elapsed);
println!();
profiler.checkpoint(&format!("benchmark_{}", name.to_lowercase()));
}
profiler.print_summary();
println!("🎯 Benchmark completed!");
Ok(())
}
// Helper functions
fn parse_service_list(services_str: &str) -> Result<Vec<ServiceType>> {
let mut services = Vec::new();
for service in services_str.split(',') {
let service = service.trim().to_lowercase();
match service.as_str() {
"all" => {
services = vec![
ServiceType::ApiGateway,
ServiceType::Database,
ServiceType::TradingService,
ServiceType::BacktestingService,
ServiceType::MLTrainingService,
];
break;
},
"api" | "gateway" => services.push(ServiceType::ApiGateway),
"trading" => services.push(ServiceType::TradingService),
"backtesting" => services.push(ServiceType::BacktestingService),
"ml_training" | "ml" => services.push(ServiceType::MLTrainingService),
"database" | "db" => services.push(ServiceType::Database),
_ => return Err(anyhow::anyhow!("Unknown service: {}", service)),
}
}
Ok(services)
}
fn create_service_config(service_type: &ServiceType, port: u16) -> Result<ServiceConfig> {
let (health_endpoint, executable_path) = match service_type {
ServiceType::ApiGateway => (
"http://localhost:8080/health".to_string(),
"api".to_string(),
),
ServiceType::TradingService => (
"http://localhost:8081/health".to_string(),
"trading_service".to_string(),
),
ServiceType::BacktestingService => (
"http://localhost:8082/health".to_string(),
"backtesting_service".to_string(),
),
ServiceType::MLTrainingService => (
"http://localhost:8095/health".to_string(),
"ml_training_service".to_string(),
),
ServiceType::Database => {
return Err(anyhow::anyhow!("Database service config not supported"))
},
};
let config = ServiceConfig {
service_type: service_type.clone(),
executable_path: format!("cargo run --bin {}", executable_path),
port,
health_endpoint,
startup_timeout: Duration::from_secs(30),
environment: create_service_environment(service_type, port)?,
working_directory: std::env::current_dir()?,
log_file: Some(format!(
"/tmp/foxhunt_{}_service.log",
service_type.as_str()
)),
};
Ok(config)
}
fn create_service_environment(
service_type: &ServiceType,
port: u16,
) -> Result<HashMap<String, String>> {
let mut env = HashMap::new();
// Common environment
env.insert("RUST_LOG".to_string(), "info".to_string());
env.insert("FOXHUNT_TEST_MODE".to_string(), "true".to_string());
env.insert(
"DATABASE_URL".to_string(),
"postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt".to_string(),
);
env.insert(
"REDIS_URL".to_string(),
"redis://localhost:6379".to_string(),
);
// Service-specific environment
match service_type {
ServiceType::ApiGateway => {
env.insert("API_GATEWAY_PORT".to_string(), port.to_string());
env.insert("GRPC_PORT".to_string(), port.to_string());
env.insert("HTTP_PORT".to_string(), "8080".to_string());
env.insert("METRICS_PORT".to_string(), "9091".to_string());
env.insert(
"JWT_SECRET".to_string(),
std::env::var("JWT_SECRET")
.unwrap_or_else(|_| "dev_secret_key_change_in_production".to_string()),
);
// Backend service URLs
env.insert(
"TRADING_SERVICE_URL".to_string(),
"http://localhost:50052".to_string(),
);
env.insert(
"BACKTESTING_SERVICE_URL".to_string(),
"http://localhost:50053".to_string(),
);
env.insert(
"ML_TRAINING_SERVICE_URL".to_string(),
"http://localhost:50054".to_string(),
);
},
ServiceType::TradingService => {
env.insert("TRADING_SERVICE_PORT".to_string(), port.to_string());
env.insert("GRPC_PORT".to_string(), port.to_string());
env.insert("HTTP_PORT".to_string(), "8081".to_string());
env.insert("METRICS_PORT".to_string(), "9092".to_string());
},
ServiceType::BacktestingService => {
env.insert("BACKTESTING_SERVICE_PORT".to_string(), port.to_string());
env.insert("GRPC_PORT".to_string(), port.to_string());
env.insert("HTTP_PORT".to_string(), "8082".to_string());
env.insert("METRICS_PORT".to_string(), "9093".to_string());
},
ServiceType::MLTrainingService => {
env.insert("ML_TRAINING_SERVICE_PORT".to_string(), port.to_string());
env.insert("GRPC_PORT".to_string(), port.to_string());
env.insert("HTTP_PORT".to_string(), "8095".to_string());
env.insert("METRICS_PORT".to_string(), "9094".to_string());
env.insert("TORCH_DEVICE".to_string(), "cpu".to_string());
},
ServiceType::Database => {
env.insert("PGPORT".to_string(), "5432".to_string());
env.insert("PGDATABASE".to_string(), "foxhunt_test".to_string());
},
}
Ok(env)
}
async fn check_database_connection() -> Result<bool> {
use sqlx::postgres::PgPoolOptions;
let database_url = std::env::var("DATABASE_URL")
.unwrap_or_else(|_| "postgresql://localhost/foxhunt_test".to_string());
match PgPoolOptions::new()
.max_connections(1)
.acquire_timeout(Duration::from_secs(5))
.connect(&database_url)
.await
{
Ok(pool) => {
// Try a simple query
match sqlx::query("SELECT 1").fetch_one(&pool).await {
Ok(_) => Ok(true),
Err(_) => Ok(false),
}
},
Err(_) => Ok(false),
}
}