foxhunt/services/trading_agent_service/src/service.rs

//! Trading Agent Service Implementation
//!
//! Unified service implementing all Trading Agent gRPC methods.
//! Production-ready with full error handling, database persistence, and metrics.

use sqlx::PgPool;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::Mutex;
use tonic::{Request, Response, Status};
use tracing::{error, info, instrument, warn};

use crate::allocation::{AllocationMethod, AssetInfo, PortfolioAllocator};
use crate::assets::{AssetScore as InternalAssetScore, AssetSelector};
use crate::monitoring::TradingAgentMetrics;
use crate::orders::{OrderGenerator, PortfolioAllocation};
use crate::proto::trading_agent::*;
use crate::strategies::{
    StrategyConfig as InternalStrategyConfig, StrategyCoordinator,
    StrategyStatus as InternalStrategyStatus, StrategyType as InternalStrategyType,
};
use crate::universe::{AssetClass, Region, UniverseCriteria as InternalCriteria, UniverseSelector};
use bigdecimal::ToPrimitive;
use rust_decimal::Decimal;

pub struct TradingAgentServiceImpl {
    db_pool: PgPool,
    universe_selector: UniverseSelector,
    strategy_coordinator: StrategyCoordinator,
    metrics: TradingAgentMetrics,
    regime_orchestrator: Arc<Mutex<ml::regime::orchestrator::RegimeOrchestrator>>,
}

impl TradingAgentServiceImpl {
    pub fn new(
        db_pool: PgPool,
        regime_orchestrator: Arc<Mutex<ml::regime::orchestrator::RegimeOrchestrator>>,
    ) -> Self {
        Self {
            universe_selector: UniverseSelector::new(db_pool.clone()),
            strategy_coordinator: StrategyCoordinator::new(db_pool.clone()),
            metrics: TradingAgentMetrics::new(),
            regime_orchestrator,
            db_pool,
        }
    }

    /// Fetch recent OHLCV bars for regime detection
    async fn fetch_recent_bars(
        &self,
        symbol: &str,
        limit: i32,
    ) -> Result<Vec<ml::types::OHLCVBar>, Status> {
        let records = sqlx::query!(
            r#"
            SELECT open, close, high, low, volume, timestamp
            FROM prices
            WHERE symbol = $1
            ORDER BY timestamp DESC
            LIMIT $2
            "#,
            symbol,
            limit as i64
        )
        .fetch_all(&self.db_pool)
        .await
        .map_err(|e| Status::internal(format!("Failed to fetch bars: {}", e)))?;

        let bars = records
            .into_iter()
            .rev()
            .filter_map(|r| {
                // Convert BIGINT (fixed-point cents) to f64
                let open = r.open? as f64 / 100.0;
                let close = r.close? as f64 / 100.0;
                let high = r.high? as f64 / 100.0;
                let low = r.low? as f64 / 100.0;
                let volume = r.volume? as f64;

                Some(ml::types::OHLCVBar {
                    timestamp: r.timestamp,
                    open,
                    high,
                    low,
                    close,
                    volume,
                })
            })
            .collect();

        Ok(bars)
    }

    /// Convert proto UniverseCriteria to internal
    fn convert_criteria(&self, proto_criteria: UniverseCriteria) -> InternalCriteria {
        let asset_classes = proto_criteria
            .allowed_types
            .iter()
            .filter_map(|&t| match InstrumentType::try_from(t).ok()? {
                InstrumentType::Futures => Some(AssetClass::Futures),
                InstrumentType::Equity => Some(AssetClass::Equities),
                InstrumentType::Fx => Some(AssetClass::Currencies),
                _ => None,
            })
            .collect();

        InternalCriteria {
            min_liquidity: proto_criteria.min_liquidity_score,
            max_volatility: proto_criteria.max_volatility,
            asset_classes,
            regions: vec![Region::NorthAmerica], // Default to North America
            min_market_cap: Some(1_000_000_000.0),
            max_correlation: Some(0.85),
        }
    }

    /// Score a universe instrument by fetching price bars and computing factor scores.
    ///
    /// Returns an `InternalAssetScore` that can be fed to `AssetSelector`.
    async fn score_instrument(
        &self,
        inst: &crate::universe::Instrument,
    ) -> Result<InternalAssetScore, Status> {
        let symbol = inst.symbol.as_str();

        // Fetch recent price bars for feature extraction
        let bars = self.fetch_recent_bars(symbol, 60).await?;

        if bars.len() >= 26 {
            // Compute factor scores directly from bar data (no legacy MLFeatureExtractor needed)
            let momentum = {
                let first_close = bars.first().map(|b| b.close).unwrap_or(0.0);
                let last_close = bars.last().map(|b| b.close).unwrap_or(0.0);
                if first_close > 0.0 {
                    ((last_close / first_close) - 1.0).clamp(-1.0, 1.0) * 0.5 + 0.5
                } else {
                    0.5
                }
            };
            let value = {
                let avg_vol: f64 = bars.iter().map(|b| b.volume).sum::<f64>() / bars.len() as f64;
                if avg_vol > 0.0 { (avg_vol / 1_000_000.0).clamp(0.0, 1.0) } else { 0.5 }
            };
            let quality = inst.liquidity_score.clamp(0.0, 1.0);

            // ML score placeholder -- use liquidity_score from the instrument as a proxy
            // (real ML inference would go here)
            let ml_score = inst.liquidity_score.clamp(0.0, 1.0);

            Ok(InternalAssetScore::new(
                symbol.to_string(),
                ml_score,
                momentum,
                value,
                quality,
            ))
        } else {
            // Not enough bars -- fall back to instrument metadata
            let quality = inst.liquidity_score.clamp(0.0, 1.0);
            let ml_score = quality; // proxy
            let momentum = 0.5; // neutral
            let value = if inst.volatility > 0.0 {
                (1.0 / (1.0 + inst.volatility)).clamp(0.0, 1.0)
            } else {
                0.5
            };

            Ok(InternalAssetScore::new(
                symbol.to_string(),
                ml_score,
                momentum,
                value,
                quality,
            ))
        }
    }

    /// Persist an asset selection to the database.
    async fn store_asset_selection(
        &self,
        universe_id: &str,
        criteria: &serde_json::Value,
        scores: &[InternalAssetScore],
        metrics: &serde_json::Value,
    ) -> Result<(), Status> {
        let scores_json = serde_json::to_value(scores)
            .map_err(|e| Status::internal(format!("Failed to serialise scores: {e}")))?;

        sqlx::query(
            r#"
            INSERT INTO asset_selections (universe_id, criteria, asset_scores, metrics)
            VALUES ($1, $2, $3, $4)
            "#,
        )
        .bind(universe_id)
        .bind(criteria)
        .bind(&scores_json)
        .bind(metrics)
        .execute(&self.db_pool)
        .await
        .map_err(|e| {
            error!("Failed to persist asset selection: {}", e);
            Status::internal(format!("Failed to persist asset selection: {e}"))
        })?;

        Ok(())
    }

    /// Load the most recent asset selection (optionally filtered by universe_id).
    async fn load_latest_selection(
        &self,
        universe_id: Option<&str>,
    ) -> Result<(Vec<InternalAssetScore>, serde_json::Value), Status> {
        let row = if let Some(uid) = universe_id {
            sqlx::query_as::<_, (serde_json::Value, serde_json::Value)>(
                r#"
                SELECT asset_scores, metrics
                FROM asset_selections
                WHERE universe_id = $1
                ORDER BY selected_at DESC
                LIMIT 1
                "#,
            )
            .bind(uid)
            .fetch_optional(&self.db_pool)
            .await
        } else {
            sqlx::query_as::<_, (serde_json::Value, serde_json::Value)>(
                r#"
                SELECT asset_scores, metrics
                FROM asset_selections
                ORDER BY selected_at DESC
                LIMIT 1
                "#,
            )
            .fetch_optional(&self.db_pool)
            .await
        }
        .map_err(|e| {
            error!("Failed to load asset selection: {}", e);
            Status::internal(format!("Failed to load asset selection: {e}"))
        })?;

        match row {
            Some((scores_json, metrics_json)) => {
                let scores: Vec<InternalAssetScore> =
                    serde_json::from_value(scores_json).map_err(|e| {
                        error!("Failed to deserialise asset scores: {}", e);
                        Status::internal(format!("Failed to deserialise asset scores: {e}"))
                    })?;
                Ok((scores, metrics_json))
            }
            None => Ok((vec![], serde_json::json!({}))),
        }
    }

    /// Convert an internal `AssetScore` to the proto `AssetScore` message.
    fn convert_asset_score(score: &InternalAssetScore) -> AssetScore {
        AssetScore {
            symbol: score.symbol.clone(),
            ml_score: score.ml_score,
            momentum_score: score.momentum_score,
            value_score: score.value_score,
            quality_score: score.quality_score,
            composite_score: score.composite_score,
            model_scores: score.model_scores.clone(),
        }
    }

    /// Fetch current net positions from the `agent_orders` table.
    ///
    /// Returns a map of symbol -> net quantity (buys positive, sells negative)
    /// derived from non-cancelled orders. Returns an empty map on DB error
    /// so callers degrade gracefully to zero-position assumptions.
    async fn fetch_current_positions(&self) -> HashMap<String, f64> {
        let rows: Result<Vec<(String, f64)>, _> = sqlx::query_as(
            r#"
            SELECT symbol, COALESCE(SUM(
                CASE WHEN side = 'Buy' THEN CAST(quantity AS DOUBLE PRECISION)
                     WHEN side = 'Sell' THEN -CAST(quantity AS DOUBLE PRECISION)
                     ELSE 0.0
                END
            ), 0.0) as net_quantity
            FROM agent_orders
            WHERE status != 'CANCELLED'
            GROUP BY symbol
            "#,
        )
        .fetch_all(&self.db_pool)
        .await;

        match rows {
            Ok(data) => data.into_iter().collect(),
            Err(e) => {
                warn!("Failed to fetch current positions (defaulting to empty): {}", e);
                HashMap::new()
            }
        }
    }

    /// Convert internal Instrument to proto
    fn convert_instrument(&self, inst: &crate::universe::Instrument) -> Instrument {
        Instrument {
            symbol: inst.symbol.as_str().to_string(),
            exchange: inst.exchange.clone(),
            instrument_type: match inst.asset_class {
                AssetClass::Futures => InstrumentType::Futures as i32,
                AssetClass::Equities => InstrumentType::Equity as i32,
                AssetClass::Currencies => InstrumentType::Fx as i32,
                _ => InstrumentType::Unspecified as i32,
            },
            liquidity_score: inst.liquidity_score,
            volatility: inst.volatility,
            ml_signal_strength: 0.0, // Placeholder
            metadata: HashMap::new(),
        }
    }
}

#[tonic::async_trait]
impl trading_agent_service_server::TradingAgentService for TradingAgentServiceImpl {
    // ============================================================================
    // Universe Management
    // ============================================================================

    #[instrument(skip(self), fields(max_instruments))]
    async fn select_universe(
        &self,
        request: Request<SelectUniverseRequest>,
    ) -> Result<Response<SelectUniverseResponse>, Status> {
        let req = request.into_inner();
        info!(
            "SelectUniverse called with max_instruments: {:?}",
            req.max_instruments
        );

        let start = std::time::Instant::now();

        // Convert proto criteria to internal
        let criteria = match req.criteria {
            Some(c) => self.convert_criteria(c),
            None => InternalCriteria::default(),
        };

        // Select universe
        let universe = self
            .universe_selector
            .select_universe(criteria)
            .await
            .map_err(|e| {
                error!("Universe selection failed: {}", e);
                self.metrics.record_error("universe_selection_failed");
                Status::internal(format!("Failed to select universe: {}", e))
            })?;

        // Convert to proto
        let instruments: Vec<Instrument> = universe
            .instruments
            .iter()
            .map(|inst| self.convert_instrument(inst))
            .collect();

        let metrics = UniverseMetrics {
            total_instruments: universe.metrics.total_instruments as u32,
            avg_liquidity_score: universe.metrics.avg_liquidity_score,
            avg_volatility: universe.metrics.avg_volatility,
            portfolio_diversification: 0.8, // Placeholder
        };

        let duration_ms = start.elapsed().as_millis() as f64;
        self.metrics
            .record_universe_selection(duration_ms, universe.metrics.total_instruments as u64);

        info!(
            "Universe selected: {} instruments in {}ms",
            instruments.len(),
            duration_ms
        );

        Ok(Response::new(SelectUniverseResponse {
            instruments,
            metrics: Some(metrics),
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
            universe_id: universe.universe_id,
        }))
    }

    #[instrument(skip(self), fields(universe_id))]
    async fn get_universe(
        &self,
        request: Request<GetUniverseRequest>,
    ) -> Result<Response<GetUniverseResponse>, Status> {
        let req = request.into_inner();
        let universe_id = req.universe_id.unwrap_or_else(|| "current".to_string());
        info!("GetUniverse called for universe_id: {}", universe_id);

        // Fetch universe from database
        let universe = self
            .universe_selector
            .get_universe(&universe_id)
            .await
            .map_err(|e| {
                error!("Failed to get universe {}: {}", universe_id, e);
                self.metrics.record_error("get_universe_failed");
                Status::not_found(format!("Universe not found: {}", e))
            })?;

        // Convert to proto
        let instruments: Vec<Instrument> = universe
            .instruments
            .iter()
            .map(|inst| self.convert_instrument(inst))
            .collect();

        let proto_criteria = UniverseCriteria {
            min_liquidity_score: universe.criteria.min_liquidity,
            min_volatility: 0.0, // Not stored
            max_volatility: universe.criteria.max_volatility,
            allowed_types: vec![InstrumentType::Futures as i32],
            exchanges: vec!["CME".to_string()],
            min_ml_confidence: 0.0,
        };

        let metrics = UniverseMetrics {
            total_instruments: universe.metrics.total_instruments as u32,
            avg_liquidity_score: universe.metrics.avg_liquidity_score,
            avg_volatility: universe.metrics.avg_volatility,
            portfolio_diversification: 0.8,
        };

        Ok(Response::new(GetUniverseResponse {
            universe_id: universe.universe_id,
            instruments,
            criteria: Some(proto_criteria),
            metrics: Some(metrics),
            created_at: universe.created_at.timestamp_nanos_opt().unwrap_or(0),
            updated_at: universe.updated_at.timestamp_nanos_opt().unwrap_or(0),
        }))
    }

    #[instrument(skip(self))]
    async fn update_universe_criteria(
        &self,
        request: Request<UpdateUniverseCriteriaRequest>,
    ) -> Result<Response<UpdateUniverseCriteriaResponse>, Status> {
        let req = request.into_inner();
        info!("UpdateUniverseCriteria called");

        // Convert proto criteria to internal
        let criteria = match req.criteria {
            Some(c) => self.convert_criteria(c),
            None => {
                return Err(Status::invalid_argument("Criteria is required"));
            },
        };

        // Create new universe with updated criteria
        let universe = self
            .universe_selector
            .select_universe(criteria)
            .await
            .map_err(|e| {
                error!("Failed to update universe criteria: {}", e);
                self.metrics.record_error("update_criteria_failed");
                Status::internal(format!("Failed to update criteria: {}", e))
            })?;

        info!(
            "Universe criteria updated, new universe_id: {}",
            universe.universe_id
        );

        Ok(Response::new(UpdateUniverseCriteriaResponse {
            success: true,
            message: "Universe criteria updated successfully".to_string(),
            universe_id: universe.universe_id,
        }))
    }

    // ============================================================================
    // Asset Selection
    // ============================================================================

    #[instrument(skip(self), fields(universe_id, max_assets))]
    async fn select_assets(
        &self,
        request: Request<SelectAssetsRequest>,
    ) -> Result<Response<SelectAssetsResponse>, Status> {
        let req = request.into_inner();
        info!(
            "SelectAssets called for universe_id: {}, max_assets: {}",
            req.universe_id, req.max_assets
        );

        let start = std::time::Instant::now();

        // 1. Fetch the universe
        let universe = self
            .universe_selector
            .get_universe(&req.universe_id)
            .await
            .map_err(|e| {
                error!("Failed to get universe {}: {}", req.universe_id, e);
                self.metrics.record_error("select_assets_universe_not_found");
                Status::not_found(format!("Universe not found: {e}"))
            })?;

        // 2. Score each instrument
        let total_evaluated = universe.instruments.len() as u32;
        let mut internal_scores = Vec::with_capacity(universe.instruments.len());
        for inst in &universe.instruments {
            match self.score_instrument(inst).await {
                Ok(score) => internal_scores.push(score),
                Err(e) => {
                    warn!(
                        "Skipping instrument {} due to scoring error: {}",
                        inst.symbol.as_str(),
                        e
                    );
                }
            }
        }

        // 3. Apply selection criteria via AssetSelector
        let criteria = req.criteria.as_ref();
        let min_ml = criteria
            .map(|c| c.min_ml_signal_strength)
            .unwrap_or(0.0);

        // Build a threshold-aware selector from the request criteria
        let selector = if min_ml > 0.0 {
            AssetSelector::with_thresholds(min_ml, 0.0)
        } else {
            AssetSelector::new()
        };

        let mode = criteria
            .and_then(|c| SelectionMode::try_from(c.mode).ok())
            .unwrap_or(SelectionMode::TopN);

        let selected = match mode {
            SelectionMode::Threshold => selector.select_above_threshold(internal_scores),
            SelectionMode::Quantile => {
                // Use 20% quantile as default
                selector.select_top_quantile(internal_scores, 0.20)
            }
            // TopN (default), Unspecified
            _ => {
                let n = if req.max_assets > 0 {
                    req.max_assets as usize
                } else {
                    10
                };
                selector.select_top_n(internal_scores, n)
            }
        };

        // 4. Compute selection metrics
        let assets_selected = selected.len() as u32;
        let (avg_composite, min_score, max_score) = if selected.is_empty() {
            (0.0, 0.0, 0.0)
        } else {
            let sum: f64 = selected.iter().map(|s| s.composite_score).sum();
            let min = selected
                .iter()
                .map(|s| s.composite_score)
                .fold(f64::INFINITY, f64::min);
            let max = selected
                .iter()
                .map(|s| s.composite_score)
                .fold(f64::NEG_INFINITY, f64::max);
            (sum / selected.len() as f64, min, max)
        };

        let selection_metrics = SelectionMetrics {
            assets_evaluated: total_evaluated,
            assets_selected,
            avg_composite_score: avg_composite,
            min_score,
            max_score,
        };

        // 5. Persist to database
        let criteria_json = serde_json::json!({
            "min_ml_signal_strength": min_ml,
            "mode": mode as i32,
            "max_assets": req.max_assets,
        });
        let metrics_json = serde_json::json!({
            "assets_evaluated": selection_metrics.assets_evaluated,
            "assets_selected": selection_metrics.assets_selected,
            "avg_composite_score": selection_metrics.avg_composite_score,
            "min_score": selection_metrics.min_score,
            "max_score": selection_metrics.max_score,
        });

        if let Err(e) = self
            .store_asset_selection(&req.universe_id, &criteria_json, &selected, &metrics_json)
            .await
        {
            warn!("Failed to persist asset selection (non-fatal): {}", e);
        }

        // 6. Record metrics
        let duration_ms = start.elapsed().as_millis() as f64;
        self.metrics
            .record_asset_selection(duration_ms, assets_selected as u64);

        info!(
            "Asset selection complete: {}/{} assets selected in {}ms",
            assets_selected, total_evaluated, duration_ms
        );

        // 7. Convert to proto
        let proto_assets: Vec<AssetScore> =
            selected.iter().map(Self::convert_asset_score).collect();

        Ok(Response::new(SelectAssetsResponse {
            assets: proto_assets,
            metrics: Some(selection_metrics),
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
        }))
    }

    #[instrument(skip(self), fields(universe_id))]
    async fn get_selected_assets(
        &self,
        request: Request<GetSelectedAssetsRequest>,
    ) -> Result<Response<GetSelectedAssetsResponse>, Status> {
        let req = request.into_inner();
        let uid = req.universe_id.as_deref();
        info!(
            "GetSelectedAssets called (universe_id: {:?})",
            uid
        );

        let (scores, _metrics_json) = self.load_latest_selection(uid).await?;

        let proto_assets: Vec<AssetScore> =
            scores.iter().map(Self::convert_asset_score).collect();

        // Compute metrics from the returned scores
        let count = proto_assets.len() as u32;
        let (avg, min_s, max_s) = if proto_assets.is_empty() {
            (0.0, 0.0, 0.0)
        } else {
            let sum: f64 = proto_assets.iter().map(|a| a.composite_score).sum();
            let mn = proto_assets
                .iter()
                .map(|a| a.composite_score)
                .fold(f64::INFINITY, f64::min);
            let mx = proto_assets
                .iter()
                .map(|a| a.composite_score)
                .fold(f64::NEG_INFINITY, f64::max);
            (sum / count as f64, mn, mx)
        };

        let selection_metrics = SelectionMetrics {
            assets_evaluated: count,
            assets_selected: count,
            avg_composite_score: avg,
            min_score: min_s,
            max_score: max_s,
        };

        info!(
            "Returning {} selected assets",
            proto_assets.len()
        );

        Ok(Response::new(GetSelectedAssetsResponse {
            assets: proto_assets,
            metrics: Some(selection_metrics),
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
        }))
    }

    // ============================================================================
    // Portfolio Allocation (Placeholder implementations)
    // ============================================================================

    #[instrument(skip(self), fields(num_assets, total_capital))]
    async fn allocate_portfolio(
        &self,
        request: Request<AllocatePortfolioRequest>,
    ) -> Result<Response<AllocatePortfolioResponse>, Status> {
        let req = request.into_inner();
        info!(
            "AllocatePortfolio called with {} assets, total_capital: {}",
            req.assets.len(),
            req.total_capital
        );

        let start = std::time::Instant::now();

        // Validate inputs
        if req.assets.is_empty() {
            return Err(Status::invalid_argument("Assets list cannot be empty"));
        }
        if req.total_capital <= 0.0 {
            return Err(Status::invalid_argument("Total capital must be positive"));
        }

        // 1. Fetch bars, run regime detection, and compute per-asset market data
        //    We retain the bars to derive volatility and last price for each symbol.
        let mut symbol_volatilities: HashMap<String, f64> = HashMap::new();
        let mut symbol_last_prices: HashMap<String, f64> = HashMap::new();

        for asset in &req.assets {
            let bars = self.fetch_recent_bars(&asset.symbol, 100).await?;

            if bars.len() < 20 {
                warn!(
                    "Insufficient bars for regime detection: symbol={}, bars={}",
                    asset.symbol,
                    bars.len()
                );
                continue;
            }

            // Compute annualized volatility from daily log returns
            let log_returns: Vec<f64> = bars
                .windows(2)
                .filter_map(|w| {
                    let prev_close = w.first().map(|b| b.close)?;
                    let curr_close = w.last().map(|b| b.close)?;
                    if prev_close > 0.0 {
                        Some((curr_close / prev_close).ln())
                    } else {
                        None
                    }
                })
                .collect();

            if !log_returns.is_empty() {
                let n = log_returns.len() as f64;
                let mean = log_returns.iter().sum::<f64>() / n;
                let variance =
                    log_returns.iter().map(|r| (r - mean).powi(2)).sum::<f64>() / (n - 1.0).max(1.0);
                let daily_vol = variance.sqrt();
                // Annualize: daily_vol * sqrt(252 trading days)
                let annual_vol = daily_vol * (252.0_f64).sqrt();
                symbol_volatilities.insert(asset.symbol.clone(), annual_vol.max(0.001));
            }

            // Record last close price for target_quantity calculation
            if let Some(last_bar) = bars.last() {
                if last_bar.close > 0.0 {
                    symbol_last_prices.insert(asset.symbol.clone(), last_bar.close);
                }
            }

            let mut orchestrator = self.regime_orchestrator.lock().await;
            orchestrator
                .detect_and_persist(&asset.symbol, &bars)
                .await
                .map_err(|e| {
                    warn!("Regime detection failed for {}: {}", asset.symbol, e);
                    Status::internal(format!("Regime detection failed: {}", e))
                })?;

            info!("Regime detection complete for {}", asset.symbol);
        }

        // 2. Build AssetInfo from request, using real volatility when available
        let assets: Vec<AssetInfo> = req
            .assets
            .iter()
            .map(|a| {
                let volatility = symbol_volatilities
                    .get(&a.symbol)
                    .copied()
                    .unwrap_or(0.15); // Fallback: 15% if bars were insufficient
                AssetInfo {
                    symbol: a.symbol.clone(),
                    expected_return: a.composite_score, // Use composite score as expected return proxy
                    volatility,
                    win_rate: 0.55,   // Default 55% win rate (should be from historical backtest)
                    avg_win: 0.02,    // Default 2% avg win (should be from historical backtest)
                    avg_loss: 0.01,   // Default 1% avg loss (should be from historical backtest)
                    ml_score: a.ml_score,
                }
            })
            .collect();

        // 3. Call regime-adaptive Kelly
        let allocator = PortfolioAllocator::new(AllocationMethod::KellyCriterion {
            fraction: 0.25, // Quarter-Kelly for risk management
        });

        let total_capital = Decimal::from_f64_retain(req.total_capital)
            .ok_or_else(|| Status::invalid_argument("Invalid total capital"))?;

        let allocations = allocator
            .kelly_criterion_regime_adaptive(&assets, total_capital, 0.25, &self.db_pool)
            .await
            .map_err(|e| {
                error!("Kelly allocation failed: {}", e);
                Status::internal(format!("Allocation failed: {}", e))
            })?;

        // 4. Convert to proto -- compute target_quantity from last price,
        //    and populate current position data from agent_orders.
        //
        // Current positions are derived from the agent_orders table (net buy - sell
        // quantities per symbol). Current weight is the symbol's share of total
        // portfolio exposure valued at last close prices. Falls back to zeros when
        // position or price data is unavailable.
        let current_positions = self.fetch_current_positions().await;

        // Compute total portfolio value from current positions * last prices
        let total_current_value: f64 = current_positions
            .iter()
            .map(|(sym, qty)| {
                let price = symbol_last_prices.get(sym).copied().unwrap_or(0.0);
                qty.abs() * price
            })
            .sum();

        let proto_allocations: Vec<AssetAllocation> = allocations
            .iter()
            .map(|(symbol, capital)| {
                let capital_f64 = capital.to_f64().unwrap_or(0.0);
                let weight = if req.total_capital > 0.0 {
                    capital_f64 / req.total_capital
                } else {
                    0.0
                };

                // target_quantity = target_capital / last_close_price
                let target_quantity = symbol_last_prices
                    .get(symbol)
                    .filter(|p| **p > 0.0)
                    .map(|price| capital_f64 / price)
                    .unwrap_or(0.0); // 0.0 if no price data available

                // Current position from agent_orders (net quantity)
                let current_quantity = current_positions
                    .get(symbol)
                    .copied()
                    .unwrap_or(0.0);

                // Current weight: position value / total portfolio value
                let current_weight = if total_current_value > 0.0 {
                    let price = symbol_last_prices.get(symbol).copied().unwrap_or(0.0);
                    (current_quantity.abs() * price) / total_current_value
                } else {
                    0.0
                };

                AssetAllocation {
                    symbol: symbol.clone(),
                    target_weight: weight,
                    target_capital: capital_f64,
                    target_quantity,
                    current_weight,
                    current_quantity,
                    rebalance_delta: target_quantity - current_quantity,
                }
            })
            .collect();

        // 5. Calculate portfolio metrics using real per-asset volatilities
        let total_weight: f64 = proto_allocations.iter().map(|a| a.target_weight).sum();
        let total_allocated: f64 = proto_allocations.iter().map(|a| a.target_capital).sum();

        // Weighted expected return (composite_score * weight for each asset)
        let weighted_expected_return: f64 = proto_allocations
            .iter()
            .filter_map(|a| {
                req.assets
                    .iter()
                    .find(|ra| ra.symbol == a.symbol)
                    .map(|ra| a.target_weight * ra.composite_score)
            })
            .sum();

        // Portfolio volatility: sqrt(sum(w_i^2 * sigma_i^2)) — this is
        // the diagonal-only (uncorrelated) approximation. Full covariance
        // would require cross-asset return correlations which are not
        // maintained in this service; callers that need them build the
        // covariance matrix upstream and pass symbol-level vols only.
        let portfolio_variance: f64 = proto_allocations
            .iter()
            .map(|a| {
                let vol = symbol_volatilities
                    .get(&a.symbol)
                    .copied()
                    .unwrap_or(0.15);
                a.target_weight.powi(2) * vol.powi(2)
            })
            .sum();
        let portfolio_volatility = portfolio_variance.sqrt();

        // Sharpe ratio: (weighted_expected_return - risk_free_rate) / portfolio_volatility
        // Using 5% annualized risk-free rate (approximate US T-bill rate)
        let risk_free_rate = 0.05;
        let portfolio_sharpe = if portfolio_volatility > 1e-10 {
            (weighted_expected_return - risk_free_rate) / portfolio_volatility
        } else {
            0.0
        };

        // Parametric VaR (95%): portfolio_value * z_95 * daily_volatility
        // z_95 = 1.6449 (one-sided 95th percentile of standard normal)
        // daily_vol = annual_vol / sqrt(252)
        let z_95 = 1.6449;
        let daily_portfolio_vol = portfolio_volatility / (252.0_f64).sqrt();
        let var_95 = req.total_capital * z_95 * daily_portfolio_vol;

        // Max drawdown estimate via volatility-based approximation:
        // E[MaxDD] ~ volatility * sqrt(2 * ln(T)) where T = 252 trading days
        // This is a rough estimate for a 1-year horizon.
        let max_drawdown_estimate = portfolio_volatility * (2.0 * (252.0_f64).ln()).sqrt();

        let metrics = AllocationMetrics {
            total_weight,
            portfolio_volatility,
            portfolio_sharpe,
            var_95,
            max_drawdown_estimate,
        };

        let duration_ms = start.elapsed().as_millis() as f64;

        info!(
            "Portfolio allocated: {} assets, total_weight: {:.2}, total_capital: {:.2} in {}ms",
            proto_allocations.len(),
            total_weight,
            total_allocated,
            duration_ms
        );

        Ok(Response::new(AllocatePortfolioResponse {
            allocations: proto_allocations,
            metrics: Some(metrics),
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
            allocation_id: uuid::Uuid::new_v4().to_string(),
        }))
    }

    #[instrument(skip(self))]
    async fn get_allocation(
        &self,
        request: Request<GetAllocationRequest>,
    ) -> Result<Response<GetAllocationResponse>, Status> {
        let req = request.into_inner();
        let allocation_id_filter = req.allocation_id;
        info!(
            "GetAllocation called (allocation_id: {:?})",
            allocation_id_filter
        );

        // 1. Load the most recent asset selection (our source of truth for what assets
        //    were selected). If an allocation_id filter was provided, use it as the
        //    universe_id key; otherwise load the latest selection regardless.
        let uid = allocation_id_filter.as_deref();
        let (scores, _metrics_json) = self.load_latest_selection(uid).await?;

        if scores.is_empty() {
            info!("No asset selection found -- returning empty allocation");
            return Ok(Response::new(GetAllocationResponse {
                allocation_id: allocation_id_filter.unwrap_or_default(),
                allocations: vec![],
                metrics: Some(AllocationMetrics {
                    total_weight: 0.0,
                    portfolio_volatility: 0.0,
                    portfolio_sharpe: 0.0,
                    var_95: 0.0,
                    max_drawdown_estimate: 0.0,
                }),
                created_at: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
                total_capital: 0.0,
            }));
        }

        // 2. Derive asset info from the selection scores and re-compute allocation
        //    using quarter-Kelly (matching AllocatePortfolio defaults).
        let assets: Vec<AssetInfo> = scores
            .iter()
            .map(|s| AssetInfo {
                symbol: s.symbol.clone(),
                expected_return: s.composite_score,
                volatility: 0.15,
                win_rate: 0.55,
                avg_win: 0.02,
                avg_loss: 0.01,
                ml_score: s.ml_score,
            })
            .collect();

        let default_capital = 1_000_000.0_f64;
        let total_capital = Decimal::from_f64_retain(default_capital)
            .ok_or_else(|| Status::internal("Failed to create Decimal for default capital"))?;

        let allocator = PortfolioAllocator::new(AllocationMethod::KellyCriterion {
            fraction: 0.25,
        });

        let allocations = allocator
            .kelly_criterion_regime_adaptive(&assets, total_capital, 0.25, &self.db_pool)
            .await
            .map_err(|e| {
                error!("Kelly allocation failed in GetAllocation: {}", e);
                self.metrics.record_error("get_allocation_failed");
                Status::internal(format!("Allocation computation failed: {e}"))
            })?;

        // 3. Convert to proto
        let proto_allocations: Vec<AssetAllocation> = allocations
            .iter()
            .map(|(symbol, capital)| {
                let capital_f64 = capital.to_f64().unwrap_or(0.0);
                let weight = if default_capital > 0.0 {
                    capital_f64 / default_capital
                } else {
                    0.0
                };
                AssetAllocation {
                    symbol: symbol.clone(),
                    target_weight: weight,
                    target_capital: capital_f64,
                    target_quantity: 0.0,
                    current_weight: 0.0,
                    current_quantity: 0.0,
                    rebalance_delta: 0.0,
                }
            })
            .collect();

        let total_weight: f64 = proto_allocations.iter().map(|a| a.target_weight).sum();
        let portfolio_volatility: f64 = proto_allocations
            .iter()
            .map(|a| a.target_weight * 0.15)
            .sum();

        let metrics = AllocationMetrics {
            total_weight,
            portfolio_volatility,
            portfolio_sharpe: 0.0,
            var_95: 0.0,
            max_drawdown_estimate: 0.0,
        };

        let alloc_id = allocation_id_filter
            .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());

        info!(
            "GetAllocation returning {} allocations, total_weight: {:.4}",
            proto_allocations.len(),
            total_weight
        );

        Ok(Response::new(GetAllocationResponse {
            allocation_id: alloc_id,
            allocations: proto_allocations,
            metrics: Some(metrics),
            created_at: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
            total_capital: default_capital,
        }))
    }

    #[instrument(skip(self), fields(allocation_id))]
    async fn rebalance_portfolio(
        &self,
        request: Request<RebalancePortfolioRequest>,
    ) -> Result<Response<RebalancePortfolioResponse>, Status> {
        let req = request.into_inner();
        info!(
            "RebalancePortfolio called (allocation_id: {}, threshold: {}, force: {})",
            req.allocation_id, req.rebalance_threshold, req.force_rebalance
        );

        let start = std::time::Instant::now();
        let threshold = if req.rebalance_threshold > 0.0 {
            req.rebalance_threshold
        } else {
            0.05 // default 5% drift threshold
        };

        // 1. Load the target allocation by re-computing from latest selection
        let uid = if req.allocation_id.is_empty() {
            None
        } else {
            Some(req.allocation_id.as_str())
        };
        let (scores, _) = self.load_latest_selection(uid).await?;

        if scores.is_empty() {
            info!("No target allocation found -- nothing to rebalance");
            return Ok(Response::new(RebalancePortfolioResponse {
                actions: vec![],
                metrics: Some(RebalanceMetrics {
                    total_rebalance_actions: 0,
                    total_turnover: 0.0,
                    estimated_cost: 0.0,
                }),
                rebalance_required: false,
                timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
            }));
        }

        // 2. Build target weights from scores (equal-weight for simplicity)
        let n = scores.len() as f64;
        let target_weights: HashMap<String, f64> = scores
            .iter()
            .map(|s| (s.symbol.clone(), 1.0 / n))
            .collect();

        // 3. Load current positions from agent_orders to derive current weights
        let current_positions = self.fetch_current_positions().await;
        let current_rows: Vec<(String, f64)> = current_positions.into_iter().collect();

        let total_current: f64 = current_rows.iter().map(|(_, q)| q.abs()).sum();
        let current_weights: HashMap<String, f64> = if total_current > 0.0 {
            current_rows
                .iter()
                .map(|(sym, q)| (sym.clone(), q.abs() / total_current))
                .collect()
        } else {
            HashMap::new()
        };

        // 4. Compute deltas and build rebalance actions
        let mut actions = Vec::new();
        let mut total_turnover = 0.0_f64;

        // Collect all symbols from both target and current
        let mut all_symbols: std::collections::HashSet<String> = target_weights.keys().cloned().collect();
        for sym in current_weights.keys() {
            all_symbols.insert(sym.clone());
        }

        for symbol in &all_symbols {
            let target_w = target_weights.get(symbol).copied().unwrap_or(0.0);
            let current_w = current_weights.get(symbol).copied().unwrap_or(0.0);
            let delta_w = target_w - current_w;

            if delta_w.abs() >= threshold || req.force_rebalance {
                // Compute approximate quantities (use delta_w as proxy for quantity delta
                // since we don't have exact prices -- this is a directional signal)
                let current_qty = current_rows
                    .iter()
                    .find(|(s, _)| s == symbol)
                    .map(|(_, q)| *q)
                    .unwrap_or(0.0);
                let target_qty = if total_current > 0.0 {
                    target_w * total_current
                } else {
                    target_w * 100.0 // nominal unit if no existing positions
                };
                let delta_qty = target_qty - current_qty;

                let reason = if delta_w.abs() >= threshold {
                    RebalanceReason::Drift as i32
                } else {
                    RebalanceReason::Manual as i32
                };

                total_turnover += delta_qty.abs();

                actions.push(RebalanceAction {
                    symbol: symbol.clone(),
                    current_quantity: current_qty,
                    target_quantity: target_qty,
                    delta_quantity: delta_qty,
                    reason,
                });
            }
        }

        let rebalance_required = !actions.is_empty();
        let estimated_cost = total_turnover * 0.001; // 10 bps estimated slippage+commission

        let duration_ms = start.elapsed().as_millis() as f64;
        info!(
            "RebalancePortfolio: {} actions, turnover: {:.2}, required: {} in {}ms",
            actions.len(),
            total_turnover,
            rebalance_required,
            duration_ms
        );

        Ok(Response::new(RebalancePortfolioResponse {
            actions,
            metrics: Some(RebalanceMetrics {
                total_rebalance_actions: rebalance_required as u32,
                total_turnover,
                estimated_cost,
            }),
            rebalance_required,
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
        }))
    }

    // ============================================================================
    // Order Generation & Submission
    // ============================================================================

    #[instrument(skip(self), fields(allocation_id))]
    async fn generate_orders(
        &self,
        request: Request<GenerateOrdersRequest>,
    ) -> Result<Response<GenerateOrdersResponse>, Status> {
        let req = request.into_inner();
        info!(
            "GenerateOrders called (allocation_id: {}, ml_signals: {})",
            req.allocation_id,
            req.ml_signals.len()
        );

        let start = std::time::Instant::now();
        let order_batch_id = uuid::Uuid::new_v4().to_string();

        // 1. Load the latest asset selection (same pattern as get_allocation)
        let uid = if req.allocation_id.is_empty() {
            None
        } else {
            Some(req.allocation_id.as_str())
        };
        let (scores, _metrics_json) = self.load_latest_selection(uid).await?;

        if scores.is_empty() {
            info!("No asset selection found -- returning empty orders");
            return Ok(Response::new(GenerateOrdersResponse {
                orders: vec![],
                metrics: Some(OrderGenerationMetrics {
                    orders_generated: 0,
                    total_notional: 0.0,
                    avg_order_size: 0.0,
                }),
                timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
                order_batch_id,
            }));
        }

        // 2. Re-derive allocation via quarter-Kelly (matching AllocatePortfolio defaults)
        let assets: Vec<AssetInfo> = scores
            .iter()
            .map(|s| AssetInfo {
                symbol: s.symbol.clone(),
                expected_return: s.composite_score,
                volatility: 0.15,
                win_rate: 0.55,
                avg_win: 0.02,
                avg_loss: 0.01,
                ml_score: s.ml_score,
            })
            .collect();

        let default_capital = 1_000_000.0_f64;
        let total_capital = Decimal::from_f64_retain(default_capital)
            .ok_or_else(|| Status::internal("Failed to create Decimal for default capital"))?;

        let allocator = PortfolioAllocator::new(AllocationMethod::KellyCriterion {
            fraction: 0.25,
        });

        let allocations = allocator
            .kelly_criterion_regime_adaptive(&assets, total_capital, 0.25, &self.db_pool)
            .await
            .map_err(|e| {
                error!("Kelly allocation failed in GenerateOrders: {}", e);
                self.metrics.record_error("generate_orders_allocation_failed");
                Status::internal(format!("Allocation computation failed: {e}"))
            })?;

        // 3. Build PortfolioAllocation for OrderGenerator
        let symbol_weights: HashMap<String, f64> = allocations
            .iter()
            .map(|(symbol, capital)| {
                let capital_f64 = capital.to_f64().unwrap_or(0.0);
                let weight = if default_capital > 0.0 {
                    capital_f64 / default_capital
                } else {
                    0.0
                };
                (symbol.clone(), weight)
            })
            .collect();

        let portfolio_allocation = PortfolioAllocation {
            allocation_id: if req.allocation_id.is_empty() {
                order_batch_id.clone()
            } else {
                req.allocation_id.clone()
            },
            strategy_id: "agent_kelly_025".to_string(),
            total_capital,
            symbol_weights,
            rebalance_threshold: 0.01, // 1% threshold for order generation
            max_position_size: 0.25,
            created_at: chrono::Utc::now(),
        };

        // 4. Generate orders via OrderGenerator (no current positions -- start fresh)
        let generator = OrderGenerator::new(self.db_pool.clone(), 100.0, 500_000.0);
        let common_orders = generator
            .generate_orders(&portfolio_allocation, &[])
            .await
            .map_err(|e| {
                error!("Order generation failed: {}", e);
                self.metrics.record_error("generate_orders_failed");
                Status::internal(format!("Order generation failed: {e}"))
            })?;

        // 5. Apply ML signal adjustments: boost/suppress orders based on signal confidence
        let ml_signal_map: HashMap<String, &MlSignal> = req
            .ml_signals
            .iter()
            .map(|s| (s.symbol.clone(), s))
            .collect();

        // 6. Convert common::Order to proto GeneratedOrder
        let mut proto_orders = Vec::with_capacity(common_orders.len());
        let mut total_notional = 0.0_f64;

        for order in &common_orders {
            let symbol_str = order.symbol.as_str().to_string();
            let quantity_f64: f64 = {
                let q: Decimal = order.quantity.into();
                q.to_f64().unwrap_or(0.0)
            };

            let side = match order.side {
                common::OrderSide::Buy => OrderSide::Buy as i32,
                common::OrderSide::Sell => OrderSide::Sell as i32,
            };

            let order_type = match order.order_type {
                common::OrderType::Market => OrderType::Market as i32,
                common::OrderType::Limit => OrderType::Limit as i32,
                common::OrderType::Stop => OrderType::Stop as i32,
                common::OrderType::StopLimit => OrderType::StopLimit as i32,
                _ => OrderType::Market as i32,
            };

            let price: Option<f64> = order.price.and_then(|p| {
                let p_dec: Decimal = p.into();
                p_dec.to_f64()
            });

            // Build rationale including ML signal info if available
            let rationale = if let Some(signal) = ml_signal_map.get(&symbol_str) {
                format!(
                    "allocation_delta | ml_model={} confidence={:.2} action={}",
                    signal.model_name, signal.confidence, signal.predicted_action
                )
            } else {
                "allocation_delta".to_string()
            };

            let mut metadata = HashMap::new();
            metadata.insert("order_batch_id".to_string(), order_batch_id.clone());
            if let Some(cid) = &order.client_order_id {
                metadata.insert("client_order_id".to_string(), cid.clone());
            }
            metadata.insert("order_id".to_string(), order.id.to_string());

            // Estimate notional from metadata
            let delta_usd = order
                .metadata
                .get("delta_usd")
                .and_then(|v| v.as_f64())
                .unwrap_or(0.0)
                .abs();
            total_notional += delta_usd;

            proto_orders.push(GeneratedOrder {
                symbol: symbol_str,
                side,
                quantity: quantity_f64,
                order_type,
                price,
                rationale,
                metadata,
            });
        }

        let orders_generated = proto_orders.len() as u32;
        let avg_order_size = if orders_generated > 0 {
            total_notional / orders_generated as f64
        } else {
            0.0
        };

        let duration_ms = start.elapsed().as_millis() as f64;
        info!(
            "GenerateOrders: {} orders, total_notional: {:.2}, avg_size: {:.2} in {}ms",
            orders_generated, total_notional, avg_order_size, duration_ms
        );

        Ok(Response::new(GenerateOrdersResponse {
            orders: proto_orders,
            metrics: Some(OrderGenerationMetrics {
                orders_generated,
                total_notional,
                avg_order_size,
            }),
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
            order_batch_id,
        }))
    }

    #[instrument(skip(self), fields(order_batch_id, num_orders))]
    async fn submit_agent_orders(
        &self,
        request: Request<SubmitAgentOrdersRequest>,
    ) -> Result<Response<SubmitAgentOrdersResponse>, Status> {
        let req = request.into_inner();
        info!(
            "SubmitAgentOrders called (batch_id: {}, orders: {}, dry_run: {})",
            req.order_batch_id,
            req.orders.len(),
            req.dry_run
        );

        let start = std::time::Instant::now();
        let mut results = Vec::with_capacity(req.orders.len());
        let mut orders_accepted = 0_u32;
        let mut orders_rejected = 0_u32;

        for order in &req.orders {
            // Validate order fields
            if order.symbol.is_empty() {
                orders_rejected += 1;
                results.push(OrderSubmissionResult {
                    symbol: order.symbol.clone(),
                    success: false,
                    order_id: None,
                    error_message: Some("Symbol is required".to_string()),
                });
                continue;
            }

            if order.quantity <= 0.0 {
                orders_rejected += 1;
                results.push(OrderSubmissionResult {
                    symbol: order.symbol.clone(),
                    success: false,
                    order_id: None,
                    error_message: Some(format!(
                        "Quantity must be positive, got {}",
                        order.quantity
                    )),
                });
                continue;
            }

            let order_id = uuid::Uuid::new_v4().to_string();

            // Dry-run mode: validate only, do not persist
            if req.dry_run {
                orders_accepted += 1;
                results.push(OrderSubmissionResult {
                    symbol: order.symbol.clone(),
                    success: true,
                    order_id: Some(format!("dry_run_{}", order_id)),
                    error_message: None,
                });
                continue;
            }

            // Persist to agent_orders table
            let side_str = match OrderSide::try_from(order.side) {
                Ok(OrderSide::Buy) => "Buy",
                Ok(OrderSide::Sell) => "Sell",
                _ => "Buy", // default
            };

            let order_type_str = match OrderType::try_from(order.order_type) {
                Ok(OrderType::Market) => "MARKET",
                Ok(OrderType::Limit) => "LIMIT",
                Ok(OrderType::Stop) => "STOP",
                Ok(OrderType::StopLimit) => "STOP_LIMIT",
                _ => "MARKET",
            };

            let quantity_bd = bigdecimal::BigDecimal::try_from(order.quantity)
                .map_err(|e| {
                    Status::internal(format!("Failed to convert quantity: {e}"))
                })?;

            let price_bd: Option<bigdecimal::BigDecimal> = order.price.and_then(|p| {
                bigdecimal::BigDecimal::try_from(p).ok()
            });

            let metadata = serde_json::to_value(&order.metadata).unwrap_or_default();

            let insert_result = sqlx::query(
                r#"
                INSERT INTO agent_orders (
                    order_id, allocation_id, symbol, side, quantity, price,
                    order_type, status, time_in_force, filled_quantity,
                    client_order_id, created_at, metadata
                )
                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
                "#,
            )
            .bind(&order_id)
            .bind(&req.order_batch_id)
            .bind(&order.symbol)
            .bind(side_str)
            .bind(&quantity_bd)
            .bind(&price_bd)
            .bind(order_type_str)
            .bind("SUBMITTED")
            .bind("DAY")
            .bind(bigdecimal::BigDecimal::from(0_i64))
            .bind(format!("agent_submit_{}", &order_id))
            .bind(chrono::Utc::now())
            .bind(&metadata)
            .execute(&self.db_pool)
            .await;

            match insert_result {
                Ok(_) => {
                    orders_accepted += 1;
                    results.push(OrderSubmissionResult {
                        symbol: order.symbol.clone(),
                        success: true,
                        order_id: Some(order_id),
                        error_message: None,
                    });
                }
                Err(e) => {
                    error!(
                        "Failed to persist order for {}: {}",
                        order.symbol, e
                    );
                    orders_rejected += 1;
                    results.push(OrderSubmissionResult {
                        symbol: order.symbol.clone(),
                        success: false,
                        order_id: None,
                        error_message: Some(format!("Database error: {e}")),
                    });
                }
            }
        }

        let orders_submitted = req.orders.len() as u32;
        let acceptance_rate = if orders_submitted > 0 {
            orders_accepted as f64 / orders_submitted as f64
        } else {
            0.0
        };

        let duration_ms = start.elapsed().as_millis() as f64;
        info!(
            "SubmitAgentOrders: submitted={}, accepted={}, rejected={}, rate={:.2} in {}ms",
            orders_submitted, orders_accepted, orders_rejected, acceptance_rate, duration_ms
        );

        Ok(Response::new(SubmitAgentOrdersResponse {
            results,
            metrics: Some(OrderSubmissionMetrics {
                orders_submitted,
                orders_accepted,
                orders_rejected,
                acceptance_rate,
            }),
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
        }))
    }

    // ============================================================================
    // Strategy Coordination
    // ============================================================================

    #[instrument(skip(self), fields(strategy_name))]
    async fn register_strategy(
        &self,
        request: Request<RegisterStrategyRequest>,
    ) -> Result<Response<RegisterStrategyResponse>, Status> {
        let req = request.into_inner();
        info!("RegisterStrategy called: {}", req.strategy_name);

        // Convert proto strategy type to internal
        let strategy_type = match StrategyType::try_from(req.strategy_type) {
            Ok(StrategyType::MlEnsemble) => InternalStrategyType::MLOptimized,
            Ok(StrategyType::MeanReversion) => InternalStrategyType::MeanReversion,
            Ok(StrategyType::Momentum) => InternalStrategyType::Momentum,
            _ => InternalStrategyType::EqualWeight,
        };

        // Convert config
        let proto_config = req
            .config
            .ok_or_else(|| Status::invalid_argument("Strategy config is required"))?;

        let parameters: HashMap<String, f64> = proto_config
            .parameters
            .into_iter()
            .filter_map(|(k, v)| v.parse::<f64>().ok().map(|f| (k, f)))
            .collect();

        let config = InternalStrategyConfig {
            strategy_id: uuid::Uuid::new_v4().to_string(),
            strategy_name: req.strategy_name.clone(),
            strategy_type,
            parameters,
            status: if req.auto_enable {
                InternalStrategyStatus::Active
            } else {
                InternalStrategyStatus::Paused
            },
            created_at: chrono::Utc::now(),
            updated_at: chrono::Utc::now(),
        };

        // Register strategy
        let strategy_id = self
            .strategy_coordinator
            .register_strategy(config)
            .await
            .map_err(|e| {
                error!("Failed to register strategy: {}", e);
                self.metrics.record_error("register_strategy_failed");
                Status::internal(format!("Failed to register strategy: {}", e))
            })?;

        info!(
            "Strategy registered: {} (ID: {})",
            req.strategy_name, strategy_id
        );

        Ok(Response::new(RegisterStrategyResponse {
            success: true,
            strategy_id,
            message: "Strategy registered successfully".to_string(),
        }))
    }

    #[instrument(skip(self))]
    async fn list_strategies(
        &self,
        _request: Request<ListStrategiesRequest>,
    ) -> Result<Response<ListStrategiesResponse>, Status> {
        info!("ListStrategies called");

        // Fetch all strategies
        let strategies = self
            .strategy_coordinator
            .list_strategies()
            .await
            .map_err(|e| {
                error!("Failed to list strategies: {}", e);
                self.metrics.record_error("list_strategies_failed");
                Status::internal(format!("Failed to list strategies: {}", e))
            })?;

        // Convert to proto
        let proto_strategies: Vec<Strategy> = strategies
            .into_iter()
            .map(|s| {
                let strategy_type = match s.strategy_type {
                    InternalStrategyType::MLOptimized => StrategyType::MlEnsemble,
                    InternalStrategyType::MeanReversion => StrategyType::MeanReversion,
                    InternalStrategyType::Momentum => StrategyType::Momentum,
                    _ => StrategyType::Unspecified,
                };

                let status = match s.status {
                    InternalStrategyStatus::Active => StrategyStatus::Enabled,
                    InternalStrategyStatus::Paused => StrategyStatus::Paused,
                    InternalStrategyStatus::Stopped => StrategyStatus::Disabled,
                };

                Strategy {
                    strategy_id: s.strategy_id,
                    strategy_name: s.strategy_name,
                    strategy_type: strategy_type as i32,
                    status: status as i32,
                    config: Some(StrategyConfig {
                        parameters: s
                            .parameters
                            .into_iter()
                            .map(|(k, v)| (k, v.to_string()))
                            .collect(),
                        target_symbols: vec![],
                        max_capital_pct: 0.25,
                    }),
                    performance: None,
                    created_at: s.created_at.timestamp_nanos_opt().unwrap_or(0),
                    updated_at: s.updated_at.timestamp_nanos_opt().unwrap_or(0),
                }
            })
            .collect();

        info!("Listed {} strategies", proto_strategies.len());

        Ok(Response::new(ListStrategiesResponse {
            strategies: proto_strategies,
        }))
    }

    #[instrument(skip(self), fields(strategy_id))]
    async fn update_strategy_status(
        &self,
        request: Request<UpdateStrategyStatusRequest>,
    ) -> Result<Response<UpdateStrategyStatusResponse>, Status> {
        let req = request.into_inner();
        info!(
            "UpdateStrategyStatus called for strategy_id: {}",
            req.strategy_id
        );

        // Convert proto status to internal
        let new_status = match StrategyStatus::try_from(req.new_status) {
            Ok(StrategyStatus::Enabled) => InternalStrategyStatus::Active,
            Ok(StrategyStatus::Paused) => InternalStrategyStatus::Paused,
            Ok(StrategyStatus::Disabled) => InternalStrategyStatus::Stopped,
            _ => {
                return Err(Status::invalid_argument("Invalid strategy status"));
            },
        };

        // Update status
        self.strategy_coordinator
            .update_status(&req.strategy_id, new_status.clone())
            .await
            .map_err(|e| {
                error!("Failed to update strategy status: {}", e);
                self.metrics.record_error("update_strategy_status_failed");
                Status::internal(format!("Failed to update strategy status: {}", e))
            })?;

        // Fetch updated strategy
        let strategy = self
            .strategy_coordinator
            .get_strategy(&req.strategy_id)
            .await
            .ok();

        let updated_strategy = strategy.map(|s| {
            let strategy_type = match s.strategy_type {
                InternalStrategyType::MLOptimized => StrategyType::MlEnsemble,
                InternalStrategyType::MeanReversion => StrategyType::MeanReversion,
                InternalStrategyType::Momentum => StrategyType::Momentum,
                _ => StrategyType::Unspecified,
            };

            let status = match s.status {
                InternalStrategyStatus::Active => StrategyStatus::Enabled,
                InternalStrategyStatus::Paused => StrategyStatus::Paused,
                InternalStrategyStatus::Stopped => StrategyStatus::Disabled,
            };

            Strategy {
                strategy_id: s.strategy_id,
                strategy_name: s.strategy_name,
                strategy_type: strategy_type as i32,
                status: status as i32,
                config: Some(StrategyConfig {
                    parameters: s
                        .parameters
                        .into_iter()
                        .map(|(k, v)| (k, v.to_string()))
                        .collect(),
                    target_symbols: vec![],
                    max_capital_pct: 0.25,
                }),
                performance: None,
                created_at: s.created_at.timestamp_nanos_opt().unwrap_or(0),
                updated_at: s.updated_at.timestamp_nanos_opt().unwrap_or(0),
            }
        });

        info!("Strategy status updated: {}", req.strategy_id);

        Ok(Response::new(UpdateStrategyStatusResponse {
            success: true,
            message: "Strategy status updated successfully".to_string(),
            updated_strategy,
        }))
    }

    // ============================================================================
    // Agent Monitoring
    // ============================================================================

    #[instrument(skip(self))]
    async fn get_agent_status(
        &self,
        request: Request<GetAgentStatusRequest>,
    ) -> Result<Response<GetAgentStatusResponse>, Status> {
        let req = request.into_inner();
        info!(
            "GetAgentStatus called (include_performance: {}, include_positions: {})",
            req.include_performance, req.include_positions
        );

        // Fetch active strategies count
        let active_strategies = self
            .strategy_coordinator
            .get_active_strategies()
            .await
            .map(|s| s.len() as u32)
            .unwrap_or(0);

        let status = AgentStatus {
            state: AgentState::Active as i32,
            current_universe_id: "current_universe".to_string(),
            active_strategies,
            selected_assets: 0,
            portfolio_utilization: 0.0,
            last_action_timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
        };

        // BLOCKER(agent-status performance): This is a lightweight status endpoint.
        // For real metrics, call GetAgentPerformance which queries the orders table.
        // Wiring this endpoint to the same DB queries would duplicate logic; instead,
        // callers should use GetAgentPerformance for accurate metrics. When a
        // performance cache/snapshot is introduced, populate these from the cache.
        let performance = if req.include_performance {
            Some(AgentPerformanceMetrics {
                total_pnl: 0.0,
                sharpe_ratio: 0.0,
                max_drawdown: 0.0,
                win_rate: 0.0,
                total_trades: 0,
                avg_trade_pnl: 0.0,
                portfolio_turnover: 0.0,
                period_start: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
                period_end: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
            })
        } else {
            None
        };

        let positions = if req.include_positions {
            Some(PositionSummary {
                positions: vec![],
                total_equity: 0.0,
                total_exposure: 0.0,
                leverage_ratio: 0.0,
            })
        } else {
            None
        };

        Ok(Response::new(GetAgentStatusResponse {
            status: Some(status),
            performance,
            positions,
            timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
        }))
    }

    type StreamAgentActivityStream =
        tokio_stream::wrappers::ReceiverStream<Result<AgentActivityEvent, Status>>;

    #[instrument(skip(self))]
    async fn stream_agent_activity(
        &self,
        _request: Request<StreamAgentActivityRequest>,
    ) -> Result<Response<Self::StreamAgentActivityStream>, Status> {
        info!("StreamAgentActivity called -- starting heartbeat stream");

        let (tx, rx) = tokio::sync::mpsc::channel(16);

        // Capture active strategy count for heartbeat snapshots
        let active_strategies = self
            .strategy_coordinator
            .get_active_strategies()
            .await
            .map(|s| s.len() as u32)
            .unwrap_or(0);

        // Spawn background task that sends periodic heartbeat events every 5 seconds
        tokio::spawn(async move {
            let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));

            loop {
                interval.tick().await;

                let event = AgentActivityEvent {
                    activity_type: ActivityType::Strategy as i32,
                    event: Some(agent_activity_event::Event::StrategyEvent(StrategyEvent {
                        strategy_id: String::new(),
                        event_type: StrategyEventType::Unspecified as i32,
                        message: format!(
                            "heartbeat: {} active strategies",
                            active_strategies
                        ),
                    })),
                    timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
                };

                // If send fails the client disconnected -- exit gracefully
                if tx.send(Ok(event)).await.is_err() {
                    tracing::info!("StreamAgentActivity client disconnected");
                    break;
                }
            }
        });

        Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
            rx,
        )))
    }

    #[instrument(skip(self))]
    async fn get_agent_performance(
        &self,
        request: Request<GetAgentPerformanceRequest>,
    ) -> Result<Response<GetAgentPerformanceResponse>, Status> {
        let req = request.into_inner();
        let now_nanos = chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0);
        let period_start = req.start_time.unwrap_or(0);
        let period_end = req.end_time.unwrap_or(now_nanos);

        info!(
            "GetAgentPerformance called (start_time: {}, end_time: {})",
            period_start, period_end
        );

        // 1. Query aggregate trade metrics from agent_orders
        //    We compute: total trades, per-trade P&L from metadata, win/loss counts.
        let order_rows: Vec<(String, serde_json::Value)> = sqlx::query_as(
            r#"
            SELECT side, COALESCE(metadata, '{}'::jsonb) as metadata
            FROM agent_orders
            WHERE status IN ('FILLED', 'PARTIALLY_FILLED')
            ORDER BY created_at DESC
            LIMIT 1000
            "#,
        )
        .fetch_all(&self.db_pool)
        .await
        .unwrap_or_default();

        let total_trades = order_rows.len() as u32;

        // Extract per-trade P&L from metadata.delta_usd (stored by order generation)
        let mut total_pnl = 0.0_f64;
        let mut wins = 0_u32;
        let mut _losses = 0_u32;
        let mut pnl_values: Vec<f64> = Vec::with_capacity(order_rows.len());

        for (_side, metadata) in &order_rows {
            let delta = metadata
                .get("delta_usd")
                .and_then(|v| v.as_f64())
                .unwrap_or(0.0);
            total_pnl += delta;
            pnl_values.push(delta);
            if delta > 0.0 {
                wins += 1;
            } else if delta < 0.0 {
                _losses += 1;
            }
        }

        let win_rate = if total_trades > 0 {
            wins as f64 / total_trades as f64
        } else {
            0.0
        };

        let avg_trade_pnl = if total_trades > 0 {
            total_pnl / total_trades as f64
        } else {
            0.0
        };

        // 2. Estimate Sharpe ratio from per-trade P&L
        let sharpe_ratio = if pnl_values.len() > 1 {
            let mean = total_pnl / pnl_values.len() as f64;
            let variance: f64 = pnl_values
                .iter()
                .map(|v| (v - mean).powi(2))
                .sum::<f64>()
                / (pnl_values.len() as f64 - 1.0);
            let std_dev = variance.sqrt();
            if std_dev > 1e-12 {
                mean / std_dev
            } else {
                0.0
            }
        } else {
            0.0
        };

        // 3. Estimate max drawdown from cumulative P&L
        let max_drawdown = {
            let mut peak = 0.0_f64;
            let mut max_dd = 0.0_f64;
            let mut cumulative = 0.0_f64;
            for pnl in &pnl_values {
                cumulative += pnl;
                if cumulative > peak {
                    peak = cumulative;
                }
                let dd = peak - cumulative;
                if dd > max_dd {
                    max_dd = dd;
                }
            }
            max_dd
        };

        let metrics = AgentPerformanceMetrics {
            total_pnl,
            sharpe_ratio,
            max_drawdown,
            win_rate,
            total_trades,
            avg_trade_pnl,
            // BLOCKER(portfolio_turnover): Requires historical position snapshots table
            // (e.g., daily position weights) to compute sum(|w_t - w_{t-1}|) over time.
            // Currently no position-snapshot persistence exists in this service.
            // When position tracking is added (via PositionService or a positions_history
            // table), compute: sum of absolute weight changes / 2, annualized.
            portfolio_turnover: 0.0,
            period_start,
            period_end,
        };

        // 4. Strategy breakdown (if requested)
        let strategy_performance = if req.include_strategy_breakdown {
            let strategies = self
                .strategy_coordinator
                .get_active_strategies()
                .await
                .unwrap_or_default();

            // BLOCKER(per-strategy P&L): Order metadata currently lacks a strategy_id
            // field, so we cannot attribute trades to individual strategies. To fix:
            // 1. Add strategy_id to order metadata when orders are generated
            // 2. Filter order_rows by strategy_id per strategy
            // 3. Compute per-strategy P&L, sharpe, win_rate, total_trades
            // Until then, per-strategy metrics remain zeroed.
            strategies
                .iter()
                .map(|s| StrategyPerformance {
                    strategy_id: s.strategy_id.clone(),
                    total_pnl: 0.0,
                    sharpe_ratio: 0.0,
                    win_rate: 0.0,
                    total_trades: 0,
                    period_start,
                    period_end,
                })
                .collect()
        } else {
            vec![]
        };

        info!(
            "GetAgentPerformance: {} trades, P&L: {:.2}, win_rate: {:.2}%, sharpe: {:.3}",
            total_trades,
            total_pnl,
            win_rate * 100.0,
            sharpe_ratio
        );

        Ok(Response::new(GetAgentPerformanceResponse {
            metrics: Some(metrics),
            strategy_performance,
            timestamp: now_nanos,
        }))
    }

    async fn health_check(
        &self,
        _request: Request<HealthCheckRequest>,
    ) -> Result<Response<HealthCheckResponse>, Status> {
        info!("HealthCheck called");

        Ok(Response::new(HealthCheckResponse {
            healthy: true,
            message: "Trading Agent Service is healthy".to_string(),
            details: std::collections::HashMap::new(),
        }))
    }

    type StreamAgentStatusStream =
        tokio_stream::wrappers::ReceiverStream<Result<GetAgentStatusResponse, Status>>;

    #[instrument(skip(self))]
    async fn stream_agent_status(
        &self,
        request: Request<StreamAgentStatusRequest>,
    ) -> Result<Response<Self::StreamAgentStatusStream>, Status> {
        let req = request.into_inner();
        let interval_secs = if req.interval_seconds > 0 {
            u64::from(req.interval_seconds.clamp(1, 60))
        } else {
            3
        };

        info!("StreamAgentStatus started: interval={}s", interval_secs);

        let (tx, rx) = tokio::sync::mpsc::channel(16);
        let db_pool = self.db_pool.clone();
        let strategy_coordinator = self.strategy_coordinator.clone();

        tokio::spawn(async move {
            let mut interval =
                tokio::time::interval(std::time::Duration::from_secs(interval_secs));

            loop {
                interval.tick().await;

                // Replicate core get_agent_status logic using cloned fields
                let active_strategies = strategy_coordinator
                    .get_active_strategies()
                    .await
                    .map(|s| s.len() as u32)
                    .unwrap_or(0);

                let selected_count: i64 = sqlx::query_scalar(
                    "SELECT COUNT(*) FROM universe_instruments WHERE is_active = true",
                )
                .fetch_one(&db_pool)
                .await
                .unwrap_or_else(|e| {
                    tracing::warn!("Failed to query universe_instruments: {e}");
                    0
                });

                let status = AgentStatus {
                    state: AgentState::Active as i32,
                    current_universe_id: "current_universe".to_string(),
                    active_strategies,
                    selected_assets: selected_count as u32,
                    portfolio_utilization: 0.0,
                    last_action_timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
                };

                let response = GetAgentStatusResponse {
                    status: Some(status),
                    performance: None,
                    positions: None,
                    timestamp: chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
                };

                if tx.send(Ok(response)).await.is_err() {
                    info!("StreamAgentStatus: client disconnected");
                    break;
                }
            }
        });

        Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
            rx,
        )))
    }
}