foxhunt/docs/archive/ml_models/ENSEMBLE_IMPLEMENTATION_GUIDE.md

Ensemble Implementation Quick Reference

Target Audience: Engineers implementing the ensemble deployment strategy Last Updated: 2025-10-14

---

File Structure

foxhunt/
├── services/trading_service/src/
│   ├── ensemble_coordinator.rs          # NEW: Main ensemble orchestration
│   ├── ensemble_metrics.rs              # NEW: Prometheus metrics
│   ├── ab_test_router.rs                # NEW: A/B testing logic
│   ├── circuit_breaker.rs               # NEW: Risk mitigation
│   ├── state.rs                         # MODIFIED: Add ensemble integration
│   └── main.rs                          # MODIFIED: Initialize ensemble
├── ml/src/ensemble/
│   ├── aggregator.rs                    # EXISTS: Signal aggregation
│   ├── confidence.rs                    # EXISTS: Confidence calculation
│   ├── model.rs                         # EXISTS: Model trait
│   └── voting.rs                        # EXISTS: Voting mechanisms
└── tli/src/commands/
    ├── ab_test.rs                       # NEW: A/B test CLI commands
    └── rollout.rs                       # NEW: Rollout CLI commands

---

1. Ensemble Coordinator Implementation

File: /services/trading_service/src/ensemble_coordinator.rs

use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use parking_lot::RwLock;
use candle_core::Tensor;
use anyhow::Result;

use ml::ensemble::aggregator::{SignalAggregator, AggregatorConfig, ModelSignal};
use ml::ensemble::confidence::ConfidenceCalculator;
use ml::ensemble::weights::ModelWeights;

/// Ensemble coordinator for multi-model prediction
pub struct EnsembleCoordinator {
    /// Active model registry (serving production traffic)
    active_models: Arc<RwLock<ModelRegistry>>,

    /// Shadow model registry (staging new checkpoints)
    shadow_models: Arc<RwLock<ModelRegistry>>,

    /// Signal aggregator for weighted voting
    aggregator: Arc<SignalAggregator>,

    /// A/B test router
    ab_router: Option<Arc<ABTestRouter>>,

    /// Metrics collector
    metrics: Arc<EnsembleMetricsCollector>,

    /// Circuit breaker
    circuit_breaker: Arc<EnsembleCircuitBreaker>,
}

impl EnsembleCoordinator {
    /// Create new ensemble coordinator
    pub fn new(config: EnsembleConfig) -> Result<Self> {
        let aggregator_config = AggregatorConfig {
            max_signals: 100,
            confidence_threshold: config.min_confidence,
            enable_simd: true,
        };

        let weights = ModelWeights::new(config.initial_weights.clone());
        let confidence_calc = ConfidenceCalculator::new();
        let aggregator = Arc::new(SignalAggregator::new(
            aggregator_config,
            weights,
            confidence_calc,
        ));

        Ok(Self {
            active_models: Arc::new(RwLock::new(ModelRegistry::new())),
            shadow_models: Arc::new(RwLock::new(ModelRegistry::new())),
            aggregator,
            ab_router: None,
            metrics: Arc::new(EnsembleMetricsCollector::new()),
            circuit_breaker: Arc::new(EnsembleCircuitBreaker::new(config.circuit_breaker_config)),
        })
    }

    /// Get ensemble prediction
    pub async fn predict(&self, features: &Tensor) -> Result<EnsembleDecision> {
        // Check circuit breaker
        let halt_decision = self.circuit_breaker.should_halt().await?;
        if matches!(halt_decision, HaltDecision::Halt(_)) {
            return Err(anyhow::anyhow!("Circuit breaker open: {:?}", halt_decision));
        }

        let start_time = std::time::Instant::now();

        // Get predictions from all active models
        let models = self.active_models.read();
        let mut signals = Vec::new();

        for (model_id, model_pair) in models.models.iter() {
            let active_model = model_pair.active.read();

            match active_model.predict(features).await {
                Ok(prediction) => {
                    signals.push(ModelSignal {
                        model_id: model_id.clone(),
                        signal: prediction.signal,
                        confidence: prediction.confidence,
                        timestamp: chrono::Utc::now().timestamp_nanos() as u64,
                        metadata: prediction.metadata,
                    });

                    // Record per-model metrics
                    crate::ml_metrics::ML_INFERENCE_LATENCY_US
                        .with_label_values(&[model_id])
                        .observe(prediction.latency_us as f64);
                }
                Err(e) => {
                    tracing::error!("Model {} prediction failed: {}", model_id, e);
                    crate::ml_metrics::ML_PREDICTION_ERRORS_TOTAL
                        .with_label_values(&[model_id, "inference_error"])
                        .inc();
                }
            }
        }

        // Aggregate signals
        let aggregated_signal = self.aggregator.aggregate_signals(signals.clone())?;

        // Calculate disagreement rate
        let disagreement_rate = self.calculate_disagreement(&signals, aggregated_signal);

        // Calculate ensemble confidence
        let confidence = self.calculate_ensemble_confidence(&signals);

        let latency_us = start_time.elapsed().as_micros() as u64;

        // Record ensemble metrics
        crate::ensemble_metrics::ENSEMBLE_AGGREGATION_LATENCY_US
            .with_label_values(&["weighted_average"])
            .observe(latency_us as f64);

        crate::ensemble_metrics::ENSEMBLE_CONFIDENCE
            .with_label_values(&["ES.FUT"]) // TODO: Extract from features
            .set(confidence);

        crate::ensemble_metrics::ENSEMBLE_DISAGREEMENT_RATE
            .with_label_values(&["ES.FUT"])
            .set(disagreement_rate);

        // High disagreement alert
        if disagreement_rate > 0.5 {
            crate::ensemble_metrics::ENSEMBLE_HIGH_DISAGREEMENT_TOTAL
                .with_label_values(&["ES.FUT", "0.5"])
                .inc();
        }

        Ok(EnsembleDecision {
            signal: aggregated_signal,
            confidence,
            disagreement_rate,
            model_votes: signals,
            latency_us,
            timestamp: chrono::Utc::now(),
        })
    }

    /// Hot-swap checkpoint (zero downtime)
    pub async fn swap_checkpoint(&self, model_id: &str, checkpoint_path: &Path) -> Result<()> {
        tracing::info!("Starting checkpoint swap for model {} from {:?}", model_id, checkpoint_path);

        // Step 1: Stage checkpoint in shadow buffer
        {
            let mut shadow = self.shadow_models.write();
            shadow.stage_checkpoint(model_id, checkpoint_path).await?;
        }

        // Step 2: Validate shadow checkpoint
        let validation_result = self.validate_shadow_checkpoint(model_id).await?;
        if !validation_result.is_valid {
            tracing::error!("Shadow checkpoint validation failed: {}", validation_result.reason);
            crate::ensemble_metrics::CHECKPOINT_SWAPS_TOTAL
                .with_label_values(&[model_id, "failed"])
                .inc();
            return Err(anyhow::anyhow!("Checkpoint validation failed: {}", validation_result.reason));
        }

        // Step 3: Atomic swap (< 1μs)
        {
            let mut active = self.active_models.write();
            let mut shadow = self.shadow_models.write();

            // Swap active and shadow pointers
            active.commit_swap(model_id).await?;
            shadow.commit_swap(model_id).await?;
        }

        tracing::info!("Checkpoint swap completed for model {}", model_id);
        crate::ensemble_metrics::CHECKPOINT_SWAPS_TOTAL
            .with_label_values(&[model_id, "success"])
            .inc();

        // Step 4: Canary monitoring (5 minutes)
        tokio::spawn(async move {
            let canary_result = self.circuit_breaker.monitor_canary(model_id).await;
            match canary_result {
                Ok(CanaryResult::Success) => {
                    tracing::info!("Canary validation passed for {}", model_id);
                }
                Ok(CanaryResult::Failed(reason)) => {
                    tracing::error!("Canary validation failed for {}: {}", model_id, reason);
                    // Automatic rollback
                    if let Err(e) = self.rollback(model_id).await {
                        tracing::error!("Rollback failed for {}: {}", model_id, e);
                    }
                }
                Err(e) => {
                    tracing::error!("Canary monitoring error for {}: {}", model_id, e);
                }
            }
        });

        Ok(())
    }

    /// Validate shadow checkpoint before swapping
    async fn validate_shadow_checkpoint(&self, model_id: &str) -> Result<ValidationResult> {
        let shadow = self.shadow_models.read();
        let model_pair = shadow.models.get(model_id)
            .ok_or_else(|| anyhow::anyhow!("Model {} not found in shadow registry", model_id))?;

        let shadow_model = model_pair.shadow.read();
        let shadow_model = shadow_model.as_ref()
            .ok_or_else(|| anyhow::anyhow!("No shadow model staged for {}", model_id))?;

        // Run 1,000 inference calls with historical data
        const VALIDATION_SAMPLES: usize = 1000;
        let historical_features = self.load_historical_features(VALIDATION_SAMPLES).await?;

        let mut latencies = Vec::new();
        let mut predictions = Vec::new();

        for features in historical_features.iter() {
            let start = std::time::Instant::now();
            let prediction = shadow_model.predict(features).await?;
            let latency_us = start.elapsed().as_micros() as u64;

            latencies.push(latency_us);
            predictions.push(prediction.signal);
        }

        // Check P99 latency
        latencies.sort_unstable();
        let p99_latency = latencies[(latencies.len() * 99) / 100];
        if p99_latency > 50 {
            return Ok(ValidationResult {
                is_valid: false,
                reason: format!("P99 latency {}μs exceeds 50μs threshold", p99_latency),
            });
        }

        // Check predictions are within expected ranges (-1.0 to 1.0)
        for pred in predictions.iter() {
            if *pred < -1.0 || *pred > 1.0 {
                return Ok(ValidationResult {
                    is_valid: false,
                    reason: format!("Prediction {} out of range [-1.0, 1.0]", pred),
                });
            }
        }

        Ok(ValidationResult {
            is_valid: true,
            reason: "All validation checks passed".to_string(),
        })
    }

    /// Calculate disagreement rate among models
    fn calculate_disagreement(&self, signals: &[ModelSignal], aggregated: f32) -> f64 {
        if signals.is_empty() {
            return 0.0;
        }

        let disagreements = signals.iter()
            .filter(|s| (s.signal - aggregated).abs() > 0.3) // Threshold: 30% difference
            .count();

        disagreements as f64 / signals.len() as f64
    }

    /// Calculate ensemble confidence (weighted average of model confidences)
    fn calculate_ensemble_confidence(&self, signals: &[ModelSignal]) -> f64 {
        if signals.is_empty() {
            return 0.0;
        }

        let total_confidence: f32 = signals.iter().map(|s| s.confidence).sum();
        (total_confidence / signals.len() as f32) as f64
    }
}

/// Ensemble decision structure
#[derive(Debug, Clone)]
pub struct EnsembleDecision {
    pub signal: f32,              // -1.0 to 1.0 (bearish to bullish)
    pub confidence: f64,          // 0.0 to 1.0
    pub disagreement_rate: f64,   // 0.0 to 1.0
    pub model_votes: Vec<ModelSignal>,
    pub latency_us: u64,
    pub timestamp: chrono::DateTime<chrono::Utc>,
}

/// Model registry with dual-buffer support
pub struct ModelRegistry {
    pub models: HashMap<String, ModelBufferPair>,
}

impl ModelRegistry {
    pub fn new() -> Self {
        Self {
            models: HashMap::new(),
        }
    }

    /// Stage checkpoint in shadow buffer
    pub async fn stage_checkpoint(&mut self, model_id: &str, checkpoint_path: &Path) -> Result<()> {
        let pair = self.models.get_mut(model_id)
            .ok_or_else(|| anyhow::anyhow!("Model {} not found", model_id))?;

        // Load checkpoint using safetensors
        let new_model = self.load_checkpoint(checkpoint_path).await?;
        *pair.shadow.write() = Some(new_model);

        tracing::info!("Staged checkpoint for model {} in shadow buffer", model_id);
        Ok(())
    }

    /// Commit swap (atomic operation)
    pub async fn commit_swap(&mut self, model_id: &str) -> Result<()> {
        let pair = self.models.get_mut(model_id)
            .ok_or_else(|| anyhow::anyhow!("Model {} not found", model_id))?;

        // Acquire swap lock
        let _guard = pair.swap_lock.lock().await;

        // Swap pointers
        let mut active = pair.active.write();
        let mut shadow = pair.shadow.write();

        if let Some(new_model) = shadow.take() {
            let old_model = std::mem::replace(&mut *active, new_model);
            *shadow = Some(old_model);

            tracing::info!("Committed checkpoint swap for model {}", model_id);
        }

        Ok(())
    }

    async fn load_checkpoint(&self, checkpoint_path: &Path) -> Result<Box<dyn MLModel>> {
        // TODO: Implement safetensors loading
        // Use candle_core::safetensors::load
        unimplemented!("Checkpoint loading not implemented")
    }
}

/// Model buffer pair (active + shadow)
pub struct ModelBufferPair {
    pub active: Arc<parking_lot::RwLock<Box<dyn MLModel>>>,
    pub shadow: Arc<parking_lot::RwLock<Option<Box<dyn MLModel>>>>,
    pub swap_lock: Arc<tokio::sync::Mutex<()>>,
}

---

2. A/B Testing Router Implementation

File: /services/trading_service/src/ab_test_router.rs

use std::collections::HashMap;
use std::sync::Arc;
use parking_lot::RwLock;
use anyhow::Result;

/// A/B test group assignment
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ABGroup {
    Control,   // Single model (e.g., DQN only)
    Treatment, // Ensemble
}

/// A/B test router
pub struct ABTestRouter {
    config: ABTestConfig,
    group_assignments: Arc<RwLock<HashMap<String, ABGroup>>>,
    metrics_tracker: Arc<ABMetricsTracker>,
}

/// A/B test configuration
#[derive(Debug, Clone)]
pub struct ABTestConfig {
    pub test_id: String,
    pub control_model: String,      // e.g., "DQN"
    pub treatment_model: String,    // e.g., "Ensemble"
    pub traffic_split: f64,         // 0.5 = 50/50
    pub min_sample_size: usize,     // 1,000 predictions
    pub significance_level: f64,    // 0.05 (95% confidence)
    pub max_duration_hours: u64,    // 168 hours (1 week)
    pub start_time: chrono::DateTime<chrono::Utc>,
}

impl ABTestRouter {
    pub fn new(config: ABTestConfig) -> Self {
        Self {
            config,
            group_assignments: Arc::new(RwLock::new(HashMap::new())),
            metrics_tracker: Arc::new(ABMetricsTracker::new(config.test_id.clone())),
        }
    }

    /// Assign user to control or treatment group (deterministic)
    pub fn assign_group(&self, user_id: &str) -> ABGroup {
        // Check existing assignment
        {
            let assignments = self.group_assignments.read();
            if let Some(&group) = assignments.get(user_id) {
                return group;
            }
        }

        // Deterministic hash-based assignment
        let hash = self.hash_user_id(user_id);
        let assignment = if (hash % 100) < (self.config.traffic_split * 100.0) as u64 {
            ABGroup::Treatment
        } else {
            ABGroup::Control
        };

        // Cache assignment
        self.group_assignments.write().insert(user_id.to_string(), assignment);

        // Record metric
        crate::ensemble_metrics::AB_TEST_ASSIGNMENTS_TOTAL
            .with_label_values(&[&self.config.test_id, self.group_str(assignment)])
            .inc();

        assignment
    }

    fn hash_user_id(&self, user_id: &str) -> u64 {
        use std::collections::hash_map::DefaultHasher;
        use std::hash::{Hash, Hasher};

        let mut hasher = DefaultHasher::new();
        user_id.hash(&mut hasher);
        hasher.finish()
    }

    fn group_str(&self, group: ABGroup) -> &'static str {
        match group {
            ABGroup::Control => "control",
            ABGroup::Treatment => "treatment",
        }
    }
}

/// A/B metrics tracker
pub struct ABMetricsTracker {
    test_id: String,
    control_metrics: Arc<RwLock<GroupMetrics>>,
    treatment_metrics: Arc<RwLock<GroupMetrics>>,
}

#[derive(Debug, Clone, Default)]
pub struct GroupMetrics {
    pub predictions: u64,
    pub correct_predictions: u64,
    pub total_pnl: f64,
    pub sharpe_ratio: f64,
    pub win_rate: f64,
}

impl ABMetricsTracker {
    pub fn new(test_id: String) -> Self {
        Self {
            test_id,
            control_metrics: Arc::new(RwLock::new(GroupMetrics::default())),
            treatment_metrics: Arc::new(RwLock::new(GroupMetrics::default())),
        }
    }

    /// Record prediction result
    pub fn record_prediction(&self, group: ABGroup, result: &PredictionResult) {
        let metrics = match group {
            ABGroup::Control => &self.control_metrics,
            ABGroup::Treatment => &self.treatment_metrics,
        };

        let mut m = metrics.write();
        m.predictions += 1;
        if result.correct {
            m.correct_predictions += 1;
        }
        m.total_pnl += result.pnl;
        m.win_rate = m.correct_predictions as f64 / m.predictions as f64;
        // TODO: Calculate Sharpe ratio from returns
    }

    /// Compute statistical significance
    pub fn compute_significance(&self) -> ABTestResults {
        let control = self.control_metrics.read().clone();
        let treatment = self.treatment_metrics.read().clone();

        // Sharpe ratio difference
        let sharpe_diff = treatment.sharpe_ratio - control.sharpe_ratio;

        // Win rate difference
        let win_rate_diff = treatment.win_rate - control.win_rate;

        // P&L difference
        let pnl_diff = treatment.total_pnl - control.total_pnl;

        // TODO: Implement Welch's t-test for Sharpe ratio
        let sharpe_pvalue = 0.05; // Placeholder

        // Update Prometheus metric
        crate::ensemble_metrics::AB_TEST_METRIC_DIFF
            .with_label_values(&[&self.test_id, "sharpe_ratio"])
            .set(sharpe_diff);

        ABTestResults {
            test_id: self.test_id.clone(),
            control_group: control,
            treatment_group: treatment,
            sharpe_diff,
            win_rate_diff,
            pnl_diff,
            is_significant: sharpe_pvalue < 0.05,
        }
    }
}

---

3. Circuit Breaker Implementation

File: /services/trading_service/src/circuit_breaker.rs

use std::sync::Arc;
use parking_lot::RwLock;
use std::time::{Duration, Instant};
use anyhow::Result;

/// Ensemble circuit breaker
pub struct EnsembleCircuitBreaker {
    config: CircuitBreakerConfig,
    state: Arc<RwLock<CircuitBreakerState>>,
}

/// Circuit breaker configuration
#[derive(Debug, Clone)]
pub struct CircuitBreakerConfig {
    pub max_consecutive_losses: u32,       // 3
    pub max_daily_drawdown: f64,           // 0.05 (5%)
    pub max_disagreement_rate: f64,        // 0.70 (70%)
    pub latency_threshold_us: u64,         // 100μs P99
    pub min_confidence_threshold: f64,     // 0.60
    pub canary_duration_secs: u64,         // 300 seconds (5 minutes)
}

#[derive(Debug, Clone)]
struct CircuitBreakerState {
    consecutive_losses: u32,
    daily_pnl: f64,
    starting_capital: f64,
    last_reset: Instant,
}

impl EnsembleCircuitBreaker {
    pub fn new(config: CircuitBreakerConfig) -> Self {
        Self {
            config,
            state: Arc::new(RwLock::new(CircuitBreakerState {
                consecutive_losses: 0,
                daily_pnl: 0.0,
                starting_capital: 5_000_000.0, // $5M
                last_reset: Instant::now(),
            })),
        }
    }

    /// Check if trading should halt
    pub async fn should_halt(&self) -> Result<HaltDecision> {
        let state = self.state.read();

        // Check consecutive losses
        if state.consecutive_losses >= self.config.max_consecutive_losses {
            return Ok(HaltDecision::Halt(
                format!("{} consecutive losses exceeded threshold", state.consecutive_losses)
            ));
        }

        // Check daily drawdown
        let drawdown = -state.daily_pnl / state.starting_capital;
        if drawdown > self.config.max_daily_drawdown {
            return Ok(HaltDecision::Halt(
                format!("Daily drawdown {:.2}% exceeds {:.2}%",
                        drawdown * 100.0, self.config.max_daily_drawdown * 100.0)
            ));
        }

        Ok(HaltDecision::Continue)
    }

    /// Monitor canary deployment
    pub async fn monitor_canary(&self, model_id: &str) -> Result<CanaryResult> {
        let start_time = Instant::now();

        while start_time.elapsed() < Duration::from_secs(self.config.canary_duration_secs) {
            // TODO: Get real-time metrics from EnsembleMetricsCollector

            tokio::time::sleep(Duration::from_secs(10)).await;
        }

        Ok(CanaryResult::Success)
    }
}

/// Halt decision
#[derive(Debug)]
pub enum HaltDecision {
    Continue,
    ReduceExposure(String),
    Halt(String),
}

/// Canary validation result
#[derive(Debug)]
pub enum CanaryResult {
    Success,
    Failed(String),
}

---

4. Trading Service Integration

File: /services/trading_service/src/state.rs (modifications)

// Add to TradingServiceState struct:
pub struct TradingServiceState {
    // ... existing fields ...

    /// NEW: Ensemble coordinator for ML predictions
    pub ensemble_coordinator: Option<Arc<EnsembleCoordinator>>,
}

// Add to initialization:
impl TradingServiceState {
    pub async fn new_with_repositories(
        // ... existing params ...
    ) -> TradingServiceResult<Self> {
        // ... existing initialization ...

        // Initialize ensemble coordinator if enabled
        let ensemble_coordinator = if std::env::var("ENABLE_ENSEMBLE").unwrap_or_default() == "true" {
            let config = EnsembleConfig::from_env();
            Some(Arc::new(EnsembleCoordinator::new(config)?))
        } else {
            None
        };

        Ok(Self {
            // ... existing fields ...
            ensemble_coordinator,
        })
    }

    /// Get trading signal using ensemble
    pub async fn get_trading_signal(&self, symbol: &str) -> Result<TradingSignal> {
        // Extract features
        let features = self.extract_features(symbol).await?;

        // Get ensemble prediction
        let ensemble_decision = self.ensemble_coordinator
            .as_ref()
            .ok_or(Error::EnsembleNotInitialized)?
            .predict(&features)
            .await?;

        // Convert to trading signal
        let action = if ensemble_decision.signal > 0.3 {
            TradingAction::Buy
        } else if ensemble_decision.signal < -0.3 {
            TradingAction::Sell
        } else {
            TradingAction::Hold
        };

        // Calculate position size (confidence-weighted)
        let position_size = self.calculate_position_size(ensemble_decision.confidence);

        Ok(TradingSignal {
            action,
            confidence: ensemble_decision.confidence,
            position_size,
            metadata: serde_json::to_value(ensemble_decision)?,
        })
    }

    fn calculate_position_size(&self, confidence: f64) -> f64 {
        let base_position = 100_000.0; // $100K
        let max_position = 1_000_000.0; // $1M

        // Confidence-weighted sizing (Kelly Criterion approximation)
        let confidence_multiplier = (confidence - 0.5) * 2.0; // 0.5 -> 0.0, 1.0 -> 1.0
        let kelly_fraction = confidence_multiplier * 0.25; // Max 25% capital

        let position_size = base_position * kelly_fraction;
        position_size.clamp(0.0, max_position)
    }
}

---

5. TLI Commands

File: /tli/src/commands/ab_test.rs

use clap::{Args, Subcommand};

#[derive(Debug, Args)]
pub struct ABTestCommand {
    #[command(subcommand)]
    pub action: ABTestAction,
}

#[derive(Debug, Subcommand)]
pub enum ABTestAction {
    /// Start new A/B test
    Start {
        #[arg(long)]
        control: String,

        #[arg(long)]
        treatment: String,

        #[arg(long, default_value = "0.5")]
        split: f64,

        #[arg(long, default_value = "7d")]
        duration: String,
    },

    /// Check A/B test status
    Status {
        #[arg(long)]
        test_id: String,
    },

    /// Stop A/B test
    Stop {
        #[arg(long)]
        test_id: String,
    },

    /// Get A/B test results
    Results {
        #[arg(long)]
        test_id: String,

        #[arg(long, default_value = "text")]
        format: String, // text, json
    },
}

pub async fn handle_ab_test(cmd: ABTestCommand) -> Result<()> {
    match cmd.action {
        ABTestAction::Start { control, treatment, split, duration } => {
            // TODO: Call API Gateway StartABTest RPC
            println!("Starting A/B test: {} vs {}", control, treatment);
        }
        ABTestAction::Status { test_id } => {
            // TODO: Call API Gateway GetABTestStatus RPC
            println!("A/B Test Status: {}", test_id);
        }
        // ... other actions ...
    }
    Ok(())
}

---

6. Environment Variables

# Enable ensemble mode
export ENABLE_ENSEMBLE=true

# Ensemble configuration
export ENSEMBLE_MIN_CONFIDENCE=0.6
export ENSEMBLE_AGGREGATION_METHOD=weighted_average # or majority_vote

# Circuit breaker
export CIRCUIT_BREAKER_MAX_LOSSES=3
export CIRCUIT_BREAKER_MAX_DRAWDOWN=0.05
export CIRCUIT_BREAKER_MAX_DISAGREEMENT=0.70

# A/B testing
export ENABLE_AB_TESTING=true
export AB_TEST_TRAFFIC_SPLIT=0.5

# Model checkpoints
export MODEL_CHECKPOINT_DIR=/home/jgrusewski/Work/foxhunt/ml/trained_models/production
export DQN_CHECKPOINT=dqn_final_epoch1.safetensors
export PPO_CHECKPOINT=ppo_final_epoch1.safetensors
export MAMBA2_CHECKPOINT=mamba2_final_epoch1.safetensors
export TFT_CHECKPOINT=tft_final_epoch1.safetensors

---

7. Testing Commands

# Build all services
cargo build --workspace --release

# Run unit tests
cargo test -p trading_service -- ensemble_coordinator
cargo test -p trading_service -- ab_test_router
cargo test -p trading_service -- circuit_breaker

# Run integration tests
cargo test -p trading_service --test ensemble_integration_test

# Start Trading Service with ensemble enabled
ENABLE_ENSEMBLE=true cargo run -p trading_service --release

# Test hot-swap (manual)
tli checkpoint swap --model DQN --path /path/to/new_checkpoint.safetensors

# Test A/B framework
tli ab start --control DQN --treatment Ensemble --split 50/50 --duration 1h
tli ab status --test-id <uuid>

# Test circuit breaker (inject failures)
# TODO: Implement failure injection endpoint

---

Next Steps

1. Week 1: Implement EnsembleCoordinator and ModelRegistry
2. Week 2: Implement ABTestRouter and statistical tests
3. Week 3: Implement CircuitBreaker and rollout automation
4. Week 4: Integration testing and validation
5. Week 5: Pre-production deployment

---

Contact: ML Engineering Team (ml-team@foxhunt.trading)