6093eac7bf
Wave 64-65 cleanup: Proto regeneration and build system updates from Tonic 0.12→0.14 upgrade Files updated: - Cargo.lock: Dependency resolution for Tonic 0.14.2 - All build.rs: Updated for tonic-prost-build - Proto files: Regenerated with tonic-prost 0.14 - Examples/tests: Updated for new gRPC API 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
945 lines
35 KiB
Rust
945 lines
35 KiB
Rust
//! Circuit Breaker for Risk Management Service
|
|
//! Circuit Breaker Module
|
|
//!
|
|
//! Implements dynamic portfolio-based circuit breakers with distributed Redis coordination.
|
|
//! Eliminates fixed $1M daily loss limits in favor of dynamic 2% portfolio-based limits.
|
|
|
|
// #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic)] // COMMENTED: Crate-level allows applied
|
|
#![warn(clippy::indexing_slicing)]
|
|
|
|
use std::collections::HashMap;
|
|
use std::marker::Send;
|
|
use std::sync::{
|
|
atomic::{AtomicU32, Ordering},
|
|
Arc,
|
|
};
|
|
// Removed foxhunt_infrastructure - not available in this simplified risk crate
|
|
|
|
use async_trait::async_trait;
|
|
use chrono::{DateTime, Utc};
|
|
use common::{Position, Price, Quantity, Symbol};
|
|
use redis::{AsyncCommands, RedisResult};
|
|
use rust_decimal::Decimal;
|
|
// REMOVED: Direct Decimal usage - use canonical types
|
|
use serde::{Deserialize, Serialize};
|
|
use tokio::sync::RwLock;
|
|
use tracing::{debug, error, info, warn};
|
|
|
|
// Import types using established patterns
|
|
use crate::error::{
|
|
decimal_to_f64_safe, f64_to_decimal_safe, f64_to_price_safe, RiskError, RiskResult,
|
|
};
|
|
|
|
/// Circuit breaker state with Redis coordination
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CircuitBreakerState {
|
|
/// Whether circuit breaker is currently active
|
|
pub is_active: bool,
|
|
/// Current portfolio value
|
|
pub portfolio_value: Price,
|
|
/// Dynamic daily loss limit (percentage of portfolio)
|
|
pub daily_loss_limit: Price,
|
|
/// Current realized daily loss
|
|
pub current_daily_loss: Price,
|
|
/// Reason for activation
|
|
pub activation_reason: Option<String>,
|
|
/// When circuit breaker was activated
|
|
pub activated_at: Option<DateTime<Utc>>,
|
|
/// Last state update timestamp
|
|
pub last_updated: DateTime<Utc>,
|
|
/// Associated account ID
|
|
pub account_id: String,
|
|
/// Number of consecutive violations
|
|
pub consecutive_violations: u32,
|
|
}
|
|
|
|
impl Default for CircuitBreakerState {
|
|
fn default() -> Self {
|
|
Self {
|
|
is_active: false,
|
|
portfolio_value: Price::ZERO,
|
|
daily_loss_limit: Price::ZERO,
|
|
current_daily_loss: Price::ZERO,
|
|
activation_reason: None,
|
|
activated_at: None,
|
|
last_updated: Utc::now(),
|
|
account_id: "default".to_owned(),
|
|
consecutive_violations: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Circuit breaker configuration with dynamic limits
|
|
#[derive(Debug, Clone)]
|
|
pub struct CircuitBreakerConfig {
|
|
/// Enable circuit breaker functionality
|
|
pub enabled: bool,
|
|
/// Daily loss limit as percentage of portfolio (e.g., 2.0 = 2%)
|
|
pub daily_loss_percentage: Price,
|
|
/// Position size limit as percentage of portfolio (e.g., 5.0 = 5%)
|
|
pub position_limit_percentage: Price,
|
|
/// Maximum consecutive violations before emergency stop
|
|
pub max_consecutive_violations: u32,
|
|
/// Redis connection URL for distributed coordination
|
|
pub redis_url: String,
|
|
/// Redis key prefix for namespacing
|
|
pub redis_key_prefix: String,
|
|
/// Enable automatic recovery from circuit breaker state
|
|
pub auto_recovery_enabled: bool,
|
|
/// Interval for refreshing portfolio values (seconds)
|
|
pub portfolio_refresh_interval_secs: u64,
|
|
/// Cooldown period before allowing new positions after breach (seconds)
|
|
pub cooldown_period_secs: u64,
|
|
}
|
|
|
|
impl Default for CircuitBreakerConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enabled: true,
|
|
daily_loss_percentage: f64_to_price_safe(2.0, "default daily loss percentage")
|
|
.unwrap_or_else(|_| {
|
|
warn!("Failed to create default daily loss percentage, using ZERO");
|
|
Price::ZERO
|
|
}), // 2.00%
|
|
position_limit_percentage: f64_to_price_safe(5.0, "default position limit percentage")
|
|
.unwrap_or_else(|_| {
|
|
warn!("Failed to create default position limit percentage, using ZERO");
|
|
Price::ZERO
|
|
}), // 5.00%
|
|
max_consecutive_violations: 5,
|
|
redis_url: std::env::var("REDIS_URL").unwrap_or_else(|_| {
|
|
std::env::var("FOXHUNT_REDIS_URL").unwrap_or_else(|_| {
|
|
let redis_host =
|
|
std::env::var("REDIS_HOST").unwrap_or_else(|_| "localhost".to_owned());
|
|
let redis_port =
|
|
std::env::var("REDIS_PORT").unwrap_or_else(|_| "6379".to_owned());
|
|
format!("redis://{redis_host}:{redis_port}")
|
|
})
|
|
}),
|
|
redis_key_prefix: "foxhunt:circuit_breaker".to_owned(),
|
|
auto_recovery_enabled: false, // Manual recovery for safety
|
|
portfolio_refresh_interval_secs: 60, // 1 minute
|
|
cooldown_period_secs: 300, // 5 minutes
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Trait for broker account services
|
|
#[async_trait]
|
|
pub trait BrokerAccountService: Send + Sync {
|
|
/// Get current portfolio value
|
|
async fn get_portfolio_value(&self, account_id: &str) -> RiskResult<Decimal>;
|
|
|
|
/// Get daily `PnL` for account
|
|
async fn get_daily_pnl(&self, account_id: &str) -> RiskResult<Decimal>;
|
|
|
|
/// Get current positions for account
|
|
async fn get_positions(&self, account_id: &str) -> RiskResult<Vec<Position>>;
|
|
}
|
|
|
|
/// `PnL` metrics for risk calculations
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct PnLMetrics {
|
|
/// Unrealized profit/loss from open positions
|
|
pub unrealized_pnl: Decimal,
|
|
/// Realized profit/loss from closed positions
|
|
pub realized_pnl: Decimal,
|
|
/// Total profit/loss (realized + unrealized)
|
|
pub total_pnl: Decimal,
|
|
/// Daily profit/loss for the current trading day
|
|
pub daily_pnl: Decimal,
|
|
}
|
|
|
|
/// Real circuit breaker with dynamic portfolio-based limits
|
|
pub struct RealCircuitBreaker {
|
|
config: CircuitBreakerConfig,
|
|
broker_service: Arc<dyn BrokerAccountService>,
|
|
state: Arc<RwLock<HashMap<String, CircuitBreakerState>>>,
|
|
redis_client: Option<redis::Client>,
|
|
consecutive_violations: AtomicU32,
|
|
last_portfolio_refresh: Arc<RwLock<HashMap<String, DateTime<Utc>>>>,
|
|
}
|
|
|
|
impl RealCircuitBreaker {
|
|
/// Create new circuit breaker with real broker integration
|
|
pub async fn new(
|
|
config: CircuitBreakerConfig,
|
|
broker_service: Arc<dyn BrokerAccountService>,
|
|
) -> RiskResult<Self> {
|
|
info!("\u{1f512} Initializing REAL Circuit Breaker");
|
|
info!(
|
|
" Daily Loss Limit: {}% of portfolio value",
|
|
config.daily_loss_percentage
|
|
);
|
|
info!(" Redis Coordination: {}", config.redis_url);
|
|
|
|
// Initialize Redis connection
|
|
let redis_client = if config.enabled {
|
|
match redis::Client::open(config.redis_url.as_str()) {
|
|
Ok(client) => {
|
|
// Test Redis connection
|
|
match client.get_multiplexed_async_connection().await {
|
|
Ok(mut conn) => {
|
|
match redis::cmd("PING").query_async::<String>(&mut conn).await {
|
|
Ok(_) => {
|
|
info!("\u{2705} Redis connection established for circuit breaker coordination");
|
|
Some(client)
|
|
},
|
|
Err(e) => {
|
|
warn!("\u{26a0}\u{fe0f} Redis connection test failed: {}", e);
|
|
Some(client) // Still store client for retry attempts
|
|
},
|
|
}
|
|
},
|
|
Err(e) => {
|
|
warn!(
|
|
"\u{26a0}\u{fe0f} Could not establish initial Redis connection: {}",
|
|
e
|
|
);
|
|
Some(client) // Still store client for retry attempts
|
|
},
|
|
}
|
|
},
|
|
Err(e) => {
|
|
error!("\u{274c} Failed to create Redis client: {}", e);
|
|
return Err(RiskError::Network(format!(
|
|
"Failed to create Redis client: {e}"
|
|
)));
|
|
},
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
Ok(Self {
|
|
config,
|
|
broker_service,
|
|
state: Arc::new(RwLock::new(HashMap::new())),
|
|
redis_client,
|
|
consecutive_violations: AtomicU32::new(0),
|
|
last_portfolio_refresh: Arc::new(RwLock::new(HashMap::new())),
|
|
})
|
|
}
|
|
|
|
/// Check if circuit breaker should be triggered
|
|
pub async fn check_circuit_breaker(&self, account_id: &str) -> RiskResult<bool> {
|
|
if !self.config.enabled {
|
|
return Ok(false);
|
|
}
|
|
|
|
debug!("Checking circuit breaker for account: {}", account_id);
|
|
|
|
// Get or create state for account
|
|
let mut state = self.get_or_create_state(account_id).await?;
|
|
|
|
// Refresh portfolio value if needed
|
|
if self.should_refresh_portfolio(&state).await? {
|
|
self.refresh_portfolio_value(&mut state).await?;
|
|
}
|
|
|
|
// Check daily loss against dynamic limit - use safe conversions
|
|
let loss_percentage =
|
|
if state.portfolio_value > Price::ZERO {
|
|
let current_loss_decimal = state.current_daily_loss.to_decimal().map_err(|_| {
|
|
RiskError::TypeConversion {
|
|
from_type: "Price".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: "current daily loss conversion failed".to_owned(),
|
|
}
|
|
})?;
|
|
let portfolio_decimal =
|
|
state
|
|
.portfolio_value
|
|
.to_decimal()
|
|
.map_err(|_| RiskError::TypeConversion {
|
|
from_type: "Price".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: "portfolio value conversion failed".to_owned(),
|
|
})?;
|
|
let ratio = current_loss_decimal / portfolio_decimal;
|
|
let ratio_f64 = decimal_to_f64_safe(ratio, "loss ratio calculation")?;
|
|
f64_to_decimal_safe(ratio_f64 * 100.0, "loss percentage calculation")?
|
|
} else {
|
|
Decimal::ZERO
|
|
};
|
|
|
|
let daily_loss_limit_decimal =
|
|
self.config
|
|
.daily_loss_percentage
|
|
.to_decimal()
|
|
.map_err(|_| RiskError::TypeConversion {
|
|
from_type: "Price".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: "daily loss limit conversion failed".to_owned(),
|
|
})?;
|
|
let should_activate = loss_percentage >= daily_loss_limit_decimal;
|
|
|
|
if should_activate && !state.is_active {
|
|
self.activate_circuit_breaker(
|
|
&mut state,
|
|
format!(
|
|
"Daily loss {}% exceeds limit {}%",
|
|
loss_percentage, self.config.daily_loss_percentage
|
|
),
|
|
)
|
|
.await?;
|
|
}
|
|
|
|
Ok(state.is_active)
|
|
}
|
|
|
|
/// Get current circuit breaker state
|
|
pub async fn get_state(&self, account_id: &str) -> RiskResult<CircuitBreakerState> {
|
|
let state_map = self.state.read().await;
|
|
Ok(state_map.get(account_id).cloned().unwrap_or_else(|| {
|
|
warn!(
|
|
"No circuit breaker state found for account {}, using default",
|
|
account_id
|
|
);
|
|
CircuitBreakerState::default()
|
|
}))
|
|
}
|
|
|
|
/// Check if circuit breaker is active for an account
|
|
pub async fn is_active(&self, account_id: &str) -> bool {
|
|
let state_map = self.state.read().await;
|
|
state_map
|
|
.get(account_id)
|
|
.is_some_and(|state| state.is_active)
|
|
}
|
|
|
|
/// Record a violation and potentially activate circuit breaker
|
|
pub async fn record_violation(&self, violation_type: &str) {
|
|
warn!(
|
|
"\u{1f6a8} Circuit breaker violation recorded: {}",
|
|
violation_type
|
|
);
|
|
self.consecutive_violations.fetch_add(1, Ordering::SeqCst);
|
|
}
|
|
|
|
/// Manually reset circuit breaker
|
|
pub async fn reset_circuit_breaker(&self, account_id: &str, reason: String) -> RiskResult<()> {
|
|
info!(
|
|
"\u{1f513} Manually resetting circuit breaker for account {}: {}",
|
|
account_id, reason
|
|
);
|
|
|
|
let mut state_map = self.state.write().await;
|
|
if let Some(state) = state_map.get_mut(account_id) {
|
|
state.is_active = false;
|
|
state.activation_reason = None;
|
|
state.activated_at = None;
|
|
state.consecutive_violations = 0;
|
|
state.last_updated = Utc::now();
|
|
|
|
// Persist to Redis
|
|
if let Err(e) = self.persist_state_to_redis(state).await {
|
|
warn!("Failed to persist reset state to Redis: {}", e);
|
|
}
|
|
}
|
|
|
|
self.consecutive_violations.store(0, Ordering::SeqCst);
|
|
info!(
|
|
"\u{2705} Circuit breaker reset completed for account {}",
|
|
account_id
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// Check position size limits
|
|
pub async fn check_position_limit(
|
|
&self,
|
|
account_id: &str,
|
|
symbol: &Symbol,
|
|
quantity: Quantity,
|
|
) -> RiskResult<bool> {
|
|
if !self.config.enabled {
|
|
return Ok(true); // Allow all positions if circuit breaker disabled
|
|
}
|
|
|
|
let state = self.get_or_create_state(account_id).await?;
|
|
|
|
if state.portfolio_value <= Price::ZERO {
|
|
return Ok(false); // Block if no portfolio value
|
|
}
|
|
|
|
// Calculate position value (simplified - would need current price in real implementation)
|
|
let estimated_position_value = quantity.to_f64(); // Convert to f64 for calculation
|
|
let estimated_decimal =
|
|
f64_to_decimal_safe(estimated_position_value, "position value calculation")
|
|
.unwrap_or_else(|_| {
|
|
warn!("Failed to convert position value to decimal, using ZERO");
|
|
Decimal::ZERO
|
|
});
|
|
let portfolio_decimal = state
|
|
.portfolio_value
|
|
.to_decimal()
|
|
.map_err(|_| RiskError::TypeConversion {
|
|
from_type: "Price".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: "portfolio value conversion for position limit failed".to_owned(),
|
|
})
|
|
.unwrap_or_else(|e| {
|
|
warn!("Portfolio value conversion failed: {}, using default", e);
|
|
Decimal::from(1) // Use 1 to avoid division by zero
|
|
});
|
|
let position_percentage = estimated_decimal / portfolio_decimal * Decimal::from(100);
|
|
|
|
let position_limit_decimal = self
|
|
.config
|
|
.position_limit_percentage
|
|
.to_decimal()
|
|
.map_err(|_| RiskError::TypeConversion {
|
|
from_type: "Price".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: "position limit percentage conversion failed".to_owned(),
|
|
})
|
|
.unwrap_or_else(|e| {
|
|
warn!("Position limit conversion failed: {}, using default 5%", e);
|
|
Decimal::from(5) // 5% default
|
|
});
|
|
let within_limit = position_percentage <= position_limit_decimal;
|
|
|
|
if !within_limit {
|
|
warn!(
|
|
"Position size limit exceeded: {}% > {}% for {} in account {}",
|
|
position_percentage, self.config.position_limit_percentage, symbol, account_id
|
|
);
|
|
}
|
|
|
|
Ok(within_limit)
|
|
}
|
|
|
|
/// Get or create state for account
|
|
async fn get_or_create_state(&self, account_id: &str) -> RiskResult<CircuitBreakerState> {
|
|
let mut state_map = self.state.write().await;
|
|
|
|
if let Some(existing_state) = state_map.get(account_id) {
|
|
return Ok(existing_state.clone());
|
|
}
|
|
|
|
// Try to load from Redis first
|
|
if let Some(redis_state) = self.load_state_from_redis(account_id).await? {
|
|
state_map.insert(account_id.to_owned(), redis_state.clone());
|
|
return Ok(redis_state);
|
|
}
|
|
|
|
// Create new state
|
|
let mut new_state = CircuitBreakerState::default();
|
|
new_state.account_id = account_id.to_owned();
|
|
|
|
// Initialize portfolio value
|
|
self.refresh_portfolio_value(&mut new_state).await?;
|
|
|
|
state_map.insert(account_id.to_owned(), new_state.clone());
|
|
Ok(new_state)
|
|
}
|
|
|
|
/// Check if portfolio should be refreshed
|
|
async fn should_refresh_portfolio(&self, state: &CircuitBreakerState) -> RiskResult<bool> {
|
|
let refresh_map = self.last_portfolio_refresh.read().await;
|
|
|
|
if let Some(last_refresh) = refresh_map.get(&state.account_id) {
|
|
let elapsed = Utc::now().signed_duration_since(*last_refresh);
|
|
Ok(elapsed.num_seconds() >= self.config.portfolio_refresh_interval_secs as i64)
|
|
} else {
|
|
Ok(true) // Refresh if never refreshed
|
|
}
|
|
}
|
|
|
|
/// Refresh portfolio value from broker
|
|
async fn refresh_portfolio_value(&self, state: &mut CircuitBreakerState) -> RiskResult<()> {
|
|
debug!(
|
|
"Refreshing portfolio value for account: {}",
|
|
state.account_id
|
|
);
|
|
|
|
// Get portfolio value from broker
|
|
let portfolio_value = self
|
|
.broker_service
|
|
.get_portfolio_value(&state.account_id)
|
|
.await?;
|
|
let daily_pnl = self.broker_service.get_daily_pnl(&state.account_id).await?;
|
|
|
|
state.portfolio_value = portfolio_value.into();
|
|
state.current_daily_loss = if daily_pnl < Decimal::ZERO {
|
|
daily_pnl.abs().into()
|
|
} else {
|
|
Price::ZERO
|
|
};
|
|
|
|
let daily_loss_percentage_decimal = self
|
|
.config
|
|
.daily_loss_percentage
|
|
.to_decimal()
|
|
.map_err(|_| RiskError::TypeConversion {
|
|
from_type: "Price".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: "daily loss percentage conversion failed".to_owned(),
|
|
})
|
|
.unwrap_or_else(|e| {
|
|
warn!(
|
|
"Daily loss percentage conversion failed: {}, using default 2%",
|
|
e
|
|
);
|
|
Decimal::from(2) // 2% default
|
|
});
|
|
|
|
state.daily_loss_limit =
|
|
((portfolio_value * daily_loss_percentage_decimal) / Decimal::from(100)).into();
|
|
state.last_updated = Utc::now();
|
|
|
|
// Update refresh timestamp
|
|
{
|
|
let mut refresh_map = self.last_portfolio_refresh.write().await;
|
|
refresh_map.insert(state.account_id.clone(), Utc::now());
|
|
}
|
|
|
|
debug!(
|
|
"Portfolio refreshed - Value: {}, Daily Loss: {}, Limit: {}",
|
|
state.portfolio_value, state.current_daily_loss, state.daily_loss_limit
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Activate circuit breaker
|
|
async fn activate_circuit_breaker(
|
|
&self,
|
|
state: &mut CircuitBreakerState,
|
|
reason: String,
|
|
) -> RiskResult<()> {
|
|
warn!(
|
|
"\u{1f6a8} ACTIVATING CIRCUIT BREAKER for account {}: {}",
|
|
state.account_id, reason
|
|
);
|
|
|
|
state.is_active = true;
|
|
state.activation_reason = Some(reason.clone());
|
|
state.activated_at = Some(Utc::now());
|
|
state.consecutive_violations += 1;
|
|
state.last_updated = Utc::now();
|
|
|
|
// Update global violation counter
|
|
self.consecutive_violations.fetch_add(1, Ordering::SeqCst);
|
|
|
|
// Persist to Redis
|
|
if let Err(e) = self.persist_state_to_redis(state).await {
|
|
error!("Failed to persist circuit breaker state to Redis: {}", e);
|
|
}
|
|
|
|
// Update in-memory state
|
|
{
|
|
let mut state_map = self.state.write().await;
|
|
state_map.insert(state.account_id.clone(), state.clone());
|
|
}
|
|
|
|
warn!(
|
|
"\u{26d4} Circuit breaker ACTIVE - Trading halted for account {}",
|
|
state.account_id
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// Load state from Redis
|
|
async fn load_state_from_redis(
|
|
&self,
|
|
account_id: &str,
|
|
) -> RiskResult<Option<CircuitBreakerState>> {
|
|
let Some(ref client) = self.redis_client else {
|
|
return Ok(None);
|
|
};
|
|
|
|
match client.get_multiplexed_async_connection().await {
|
|
Ok(mut conn) => {
|
|
let key = format!("{}:{}", self.config.redis_key_prefix, account_id);
|
|
match conn.get::<_, Option<String>>(&key).await {
|
|
Ok(Some(json_data)) => {
|
|
match serde_json::from_str::<CircuitBreakerState>(&json_data) {
|
|
Ok(state) => Ok(Some(state)),
|
|
Err(e) => {
|
|
warn!(
|
|
"Failed to deserialize circuit breaker state from Redis: {}",
|
|
e
|
|
);
|
|
Ok(None)
|
|
},
|
|
}
|
|
},
|
|
Ok(None) => Ok(None),
|
|
Err(e) => {
|
|
warn!("Failed to load circuit breaker state from Redis: {}", e);
|
|
Ok(None)
|
|
},
|
|
}
|
|
},
|
|
Err(e) => {
|
|
warn!("Failed to connect to Redis for state loading: {}", e);
|
|
Ok(None)
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Persist state to Redis
|
|
async fn persist_state_to_redis(&self, state: &CircuitBreakerState) -> RiskResult<()> {
|
|
let Some(ref client) = self.redis_client else {
|
|
return Ok(()); // No Redis client, skip persistence
|
|
};
|
|
|
|
match client.get_multiplexed_async_connection().await {
|
|
Ok(mut conn) => {
|
|
let key = format!("{}:{}", self.config.redis_key_prefix, state.account_id);
|
|
let json_data = serde_json::to_string(state)?;
|
|
|
|
// Set with expiration (24 hours)
|
|
let _: RedisResult<()> = conn.set_ex(&key, json_data, 86400).await;
|
|
|
|
debug!(
|
|
"Persisted circuit breaker state to Redis for account {}",
|
|
state.account_id
|
|
);
|
|
Ok(())
|
|
},
|
|
Err(e) => {
|
|
warn!("Failed to connect to Redis for state persistence: {}", e);
|
|
Ok(()) // Don't fail the operation if Redis is unavailable
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Get circuit breaker metrics
|
|
pub async fn get_metrics(&self) -> HashMap<String, f64> {
|
|
let mut metrics = HashMap::new();
|
|
|
|
let state_map = self.state.read().await;
|
|
let active_count = state_map.values().filter(|s| s.is_active).count();
|
|
let total_violations = self.consecutive_violations.load(Ordering::SeqCst);
|
|
|
|
metrics.insert("active_circuit_breakers".to_owned(), active_count as f64);
|
|
metrics.insert("total_violations".to_owned(), f64::from(total_violations));
|
|
metrics.insert("accounts_monitored".to_owned(), state_map.len() as f64);
|
|
|
|
metrics
|
|
}
|
|
|
|
/// Health check for circuit breaker
|
|
pub async fn health_check(&self) -> bool {
|
|
// Check Redis connectivity if enabled
|
|
if let Some(ref client) = self.redis_client {
|
|
match client.get_multiplexed_async_connection().await {
|
|
Ok(mut conn) => (redis::cmd("PING").query_async::<String>(&mut conn).await).is_ok(),
|
|
Err(_) => false,
|
|
}
|
|
} else {
|
|
true // Always healthy if Redis not configured
|
|
}
|
|
}
|
|
}
|
|
|
|
// REAL BROKER CLIENT - NO MOCKS IN PRODUCTION CODE
|
|
/// Real broker client implementation for production use
|
|
pub struct RealBrokerClient {
|
|
/// HTTP endpoint URL for the broker service
|
|
endpoint: String,
|
|
}
|
|
|
|
impl RealBrokerClient {
|
|
#[must_use]
|
|
pub const fn new(endpoint: String) -> Self {
|
|
Self { endpoint }
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl BrokerAccountService for RealBrokerClient {
|
|
async fn get_portfolio_value(&self, account_id: &str) -> RiskResult<Decimal> {
|
|
// Real HTTP call to broker service
|
|
let client = reqwest::Client::new();
|
|
let response = client
|
|
.get(format!(
|
|
"{}/accounts/{}/portfolio/value",
|
|
self.endpoint, account_id
|
|
))
|
|
.send()
|
|
.await
|
|
.map_err(|e| RiskError::BrokerError(format!("Portfolio value request failed: {e}")))?;
|
|
|
|
if !response.status().is_success() {
|
|
return Err(RiskError::BrokerError(format!(
|
|
"Broker returned error: {}",
|
|
response.status()
|
|
)));
|
|
}
|
|
|
|
let data: serde_json::Value = response
|
|
.json()
|
|
.await
|
|
.map_err(|e| RiskError::BrokerError(format!("Invalid response format: {e}")))?;
|
|
|
|
let portfolio_value = data["portfolio_value"].as_f64().ok_or_else(|| {
|
|
RiskError::BrokerError("Missing portfolio_value in response".to_owned())
|
|
})?;
|
|
|
|
Decimal::try_from(portfolio_value).map_err(|e| RiskError::TypeConversion {
|
|
from_type: "f64".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: format!("portfolio value conversion failed: {e}"),
|
|
})
|
|
}
|
|
|
|
async fn get_daily_pnl(&self, account_id: &str) -> RiskResult<Decimal> {
|
|
// Real HTTP call to broker service
|
|
let client = reqwest::Client::new();
|
|
let response = client
|
|
.get(format!(
|
|
"{}/accounts/{}/pnl/daily",
|
|
self.endpoint, account_id
|
|
))
|
|
.send()
|
|
.await
|
|
.map_err(|e| RiskError::BrokerError(format!("Daily PnL request failed: {e}")))?;
|
|
|
|
if !response.status().is_success() {
|
|
return Err(RiskError::BrokerError(format!(
|
|
"Broker returned error: {}",
|
|
response.status()
|
|
)));
|
|
}
|
|
|
|
let data: serde_json::Value = response
|
|
.json()
|
|
.await
|
|
.map_err(|e| RiskError::BrokerError(format!("Invalid response format: {e}")))?;
|
|
|
|
let daily_pnl = data["daily_pnl"]
|
|
.as_f64()
|
|
.ok_or_else(|| RiskError::BrokerError("Missing daily_pnl in response".to_owned()))?;
|
|
|
|
Decimal::try_from(daily_pnl).map_err(|e| RiskError::TypeConversion {
|
|
from_type: "f64".to_owned(),
|
|
to_type: "Decimal".to_owned(),
|
|
reason: format!("daily PnL conversion failed: {e}"),
|
|
})
|
|
}
|
|
|
|
async fn get_positions(&self, account_id: &str) -> RiskResult<Vec<Position>> {
|
|
// Real HTTP call to broker service
|
|
let client = reqwest::Client::new();
|
|
let response = client
|
|
.get(format!(
|
|
"{}/accounts/{}/positions",
|
|
self.endpoint, account_id
|
|
))
|
|
.send()
|
|
.await
|
|
.map_err(|e| RiskError::BrokerError(format!("Positions request failed: {e}")))?;
|
|
|
|
if !response.status().is_success() {
|
|
return Err(RiskError::BrokerError(format!(
|
|
"Broker returned error: {}",
|
|
response.status()
|
|
)));
|
|
}
|
|
|
|
let data: serde_json::Value = response
|
|
.json()
|
|
.await
|
|
.map_err(|e| RiskError::BrokerError(format!("Invalid response format: {e}")))?;
|
|
|
|
// Parse positions array from real broker response
|
|
let positions_array = data["positions"].as_array().ok_or_else(|| {
|
|
RiskError::BrokerError("Missing positions array in response".to_owned())
|
|
})?;
|
|
|
|
let mut positions = Vec::new();
|
|
for pos_data in positions_array {
|
|
let symbol = pos_data["symbol"]
|
|
.as_str()
|
|
.ok_or_else(|| RiskError::BrokerError("Missing symbol in position".to_owned()))?;
|
|
|
|
let quantity_raw = pos_data["quantity"]
|
|
.as_f64()
|
|
.ok_or_else(|| RiskError::BrokerError("Missing quantity in position".to_owned()))?;
|
|
|
|
let market_value_raw = pos_data["market_value"].as_f64().ok_or_else(|| {
|
|
RiskError::BrokerError("Missing market_value in position".to_owned())
|
|
})?;
|
|
|
|
let quantity = Decimal::try_from(quantity_raw).map_err(|_| {
|
|
RiskError::CalculationError("Failed to convert quantity_raw to decimal".to_owned())
|
|
})?;
|
|
let market_value = Decimal::try_from(market_value_raw).map_err(|_| {
|
|
RiskError::CalculationError(
|
|
"Failed to convert market_value_raw to decimal".to_owned(),
|
|
)
|
|
})?;
|
|
|
|
// Use Position::new constructor for consistency
|
|
let mut position = Position::new(
|
|
symbol.to_owned(),
|
|
quantity,
|
|
market_value / quantity.abs().max(Decimal::ONE), // Derive avg_price from market_value
|
|
);
|
|
|
|
// Update market_value to match the actual market value from broker
|
|
position.market_value = market_value;
|
|
position.last_updated = Utc::now();
|
|
positions.push(position);
|
|
}
|
|
|
|
Ok(positions)
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::sync::Arc;
|
|
use tokio;
|
|
// CANONICAL TYPE IMPORTS - Use types::prelude for dec! macro
|
|
|
|
fn create_test_config() -> Result<CircuitBreakerConfig, Box<dyn std::error::Error>> {
|
|
Ok(CircuitBreakerConfig {
|
|
enabled: true,
|
|
daily_loss_percentage: Price::from_f64(2.00)?, // 2%
|
|
position_limit_percentage: Price::from_f64(5.00)?, // 5%
|
|
redis_url: "redis://${REDIS_HOST:-localhost}:6379".to_string(), // Different port for tests
|
|
..Default::default()
|
|
})
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_creation() -> Result<(), Box<dyn std::error::Error>> {
|
|
let config = create_test_config()?;
|
|
let broker_service = Arc::new(RealBrokerClient::new(
|
|
std::env::var("FOXHUNT_BROKER_SERVICE_ENDPOINT").unwrap_or_else(|_| {
|
|
let service_host =
|
|
std::env::var("SERVICE_HOST").unwrap_or_else(|_| "localhost".to_string());
|
|
format!("http://{}:50054", service_host)
|
|
}), // Real broker service endpoint
|
|
));
|
|
|
|
// Circuit breaker creation might fail if Redis is not available, which is fine for tests
|
|
let _result = RealCircuitBreaker::new(config, broker_service).await;
|
|
// Don't assert success since Redis might not be available in test environment
|
|
// Debug output removed for production
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_daily_loss_check() -> Result<(), Box<dyn std::error::Error>> {
|
|
let config = create_test_config()?;
|
|
let broker_service = Arc::new(RealBrokerClient::new(
|
|
"http://${SERVICE_HOST:-localhost}:50054".to_string(), // Real broker service endpoint
|
|
));
|
|
|
|
// Skip test if broker service is not available
|
|
if let Ok(circuit_breaker) = RealCircuitBreaker::new(config, broker_service).await {
|
|
let account_id = "TEST_ACCOUNT";
|
|
let result = circuit_breaker.check_circuit_breaker(account_id).await;
|
|
|
|
match result {
|
|
Ok(_should_trigger) => {
|
|
// Debug output removed for production
|
|
},
|
|
Err(_e) => {
|
|
// Debug output removed for production
|
|
},
|
|
}
|
|
} else {
|
|
// Debug output removed for production
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_disabled() -> Result<(), Box<dyn std::error::Error>> {
|
|
let mut config = create_test_config()?;
|
|
config.enabled = false;
|
|
let broker_service = Arc::new(RealBrokerClient::new("http://localhost:50054".to_string()));
|
|
|
|
if let Ok(circuit_breaker) = RealCircuitBreaker::new(config, broker_service).await {
|
|
let account_id = "TEST_ACCOUNT";
|
|
let result = circuit_breaker.check_circuit_breaker(account_id).await?;
|
|
assert!(!result, "Circuit breaker should not trigger when disabled");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_position_limit_zero_portfolio(
|
|
) -> Result<(), Box<dyn std::error::Error>> {
|
|
let config = create_test_config()?;
|
|
let broker_service = Arc::new(RealBrokerClient::new("http://localhost:50054".to_string()));
|
|
|
|
if let Ok(circuit_breaker) = RealCircuitBreaker::new(config, broker_service).await {
|
|
let account_id = "TEST_ACCOUNT";
|
|
let symbol = Symbol::from("AAPL");
|
|
let quantity = Quantity::from_f64(100.0)?;
|
|
|
|
let result = circuit_breaker
|
|
.check_position_limit(account_id, &symbol, quantity)
|
|
.await?;
|
|
assert!(
|
|
!result,
|
|
"Should block positions when portfolio value is zero"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_consecutive_violations() -> Result<(), Box<dyn std::error::Error>>
|
|
{
|
|
let config = create_test_config()?;
|
|
let broker_service = Arc::new(RealBrokerClient::new("http://localhost:50054".to_string()));
|
|
|
|
if let Ok(circuit_breaker) = RealCircuitBreaker::new(config, broker_service).await {
|
|
circuit_breaker.record_violation("Test violation 1").await;
|
|
circuit_breaker.record_violation("Test violation 2").await;
|
|
circuit_breaker.record_violation("Test violation 3").await;
|
|
|
|
let metrics = circuit_breaker.get_metrics().await;
|
|
assert_eq!(metrics.get("total_violations").copied(), Some(3.0));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_reset() -> Result<(), Box<dyn std::error::Error>> {
|
|
let config = create_test_config()?;
|
|
let broker_service = Arc::new(RealBrokerClient::new("http://localhost:50054".to_string()));
|
|
|
|
if let Ok(circuit_breaker) = RealCircuitBreaker::new(config, broker_service).await {
|
|
let account_id = "TEST_ACCOUNT";
|
|
|
|
// Manually activate by setting state (would normally be done through check_circuit_breaker)
|
|
let mut state = CircuitBreakerState::default();
|
|
state.account_id = account_id.to_string();
|
|
state.is_active = true;
|
|
state.consecutive_violations = 3;
|
|
|
|
// Reset the circuit breaker
|
|
circuit_breaker
|
|
.reset_circuit_breaker(account_id, "Manual reset for testing".to_string())
|
|
.await?;
|
|
|
|
// Verify it was reset
|
|
assert!(!circuit_breaker.is_active(account_id).await);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_circuit_breaker_health_check() -> Result<(), Box<dyn std::error::Error>> {
|
|
let config = create_test_config()?;
|
|
let broker_service = Arc::new(RealBrokerClient::new("http://localhost:50054".to_string()));
|
|
|
|
if let Ok(circuit_breaker) = RealCircuitBreaker::new(config, broker_service).await {
|
|
// Health check might pass or fail depending on Redis availability
|
|
let _ = circuit_breaker.health_check().await;
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|