foxhunt/services/trading_service/src/questdb_metrics.rs

//! QuestDB-backed MetricsProvider for the feedback loop
//!
//! Queries QuestDB via PostgreSQL wire protocol (port 8812) to provide
//! rolling model metrics, confidence buckets, and ensemble Sharpe for
//! the autonomous weight/gate optimizers.

use crate::feedback_loop::MetricsProvider;
use ml::ensemble::conviction_gates::ConvictionGateConfig;
use ml::ensemble::gate_optimizer::GateBucketMetrics;
use ml::ensemble::weight_optimizer::ModelRollingMetrics;
use sqlx::postgres::PgPool;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::RwLock;
use tracing::{debug, info};

/// QuestDB metrics provider that queries real time-series data.
///
/// Tables expected in QuestDB (created on first write via ILP):
/// - `model_predictions`: model_id, signal, confidence, timestamp
/// - `trade_outcomes`: model_id, realized_pnl, signal_alignment, timestamp
/// - `ensemble_metrics`: sharpe_7d, sharpe_30d, timestamp
pub struct QuestDBMetricsProvider {
    pool: PgPool,
    /// Cached gate config (updated from external source)
    gate_config: Arc<RwLock<ConvictionGateConfig>>,
    /// Cached weights (updated by the feedback loop itself)
    current_weights: Arc<RwLock<HashMap<String, f64>>>,
}

impl QuestDBMetricsProvider {
    pub async fn new(questdb_pg_url: &str) -> Result<Self, sqlx::Error> {
        let pool = PgPool::connect(questdb_pg_url).await?;

        Ok(Self {
            pool,
            gate_config: Arc::new(RwLock::new(ConvictionGateConfig::default())),
            current_weights: Arc::new(RwLock::new(HashMap::new())),
        })
    }

    /// Try to connect, returning None if QuestDB is unavailable
    pub async fn try_new(questdb_pg_url: &str) -> Option<Self> {
        match Self::new(questdb_pg_url).await {
            Ok(provider) => Some(provider),
            Err(e) => {
                info!("QuestDB unavailable at {}: {} — feedback loop will use defaults", questdb_pg_url, e);
                None
            }
        }
    }

    /// Update the cached gate config (called when config changes)
    pub async fn set_gate_config(&self, config: ConvictionGateConfig) {
        *self.gate_config.write().await = config;
    }

    /// Update the cached weights (called after weight adjustments)
    pub async fn set_weights(&self, weights: HashMap<String, f64>) {
        *self.current_weights.write().await = weights;
    }

    /// Create the required tables if they don't exist.
    /// QuestDB auto-creates tables on ILP write, but we create them
    /// explicitly for querying so tests can verify structure.
    pub async fn ensure_tables(&self) -> Result<(), sqlx::Error> {
        // QuestDB uses CREATE TABLE IF NOT EXISTS with designated timestamp
        sqlx::query(
            "CREATE TABLE IF NOT EXISTS model_predictions (
                model_id SYMBOL,
                signal DOUBLE,
                confidence DOUBLE,
                prediction_accuracy DOUBLE,
                win_rate DOUBLE,
                timestamp TIMESTAMP
            ) TIMESTAMP(timestamp) PARTITION BY DAY;"
        )
        .execute(&self.pool)
        .await?;

        sqlx::query(
            "CREATE TABLE IF NOT EXISTS trade_outcomes (
                model_id SYMBOL,
                realized_pnl DOUBLE,
                signal_alignment DOUBLE,
                confidence_bucket_lower DOUBLE,
                confidence_bucket_upper DOUBLE,
                timestamp TIMESTAMP
            ) TIMESTAMP(timestamp) PARTITION BY DAY;"
        )
        .execute(&self.pool)
        .await?;

        sqlx::query(
            "CREATE TABLE IF NOT EXISTS ensemble_metrics (
                sharpe_7d DOUBLE,
                sharpe_30d DOUBLE,
                total_pnl DOUBLE,
                timestamp TIMESTAMP
            ) TIMESTAMP(timestamp) PARTITION BY DAY;"
        )
        .execute(&self.pool)
        .await?;

        Ok(())
    }

    /// Query 30-day rolling Sharpe per model
    async fn query_model_sharpe(&self) -> HashMap<String, f64> {
        let result: Result<Vec<(String, f64)>, _> = sqlx::query_as(
            "SELECT model_id, coalesce(avg(signal) / (stddev(signal) + 1e-10), 0.0) as sharpe
             FROM model_predictions
             WHERE timestamp > dateadd('d', -30, now())
             GROUP BY model_id"
        )
        .fetch_all(&self.pool)
        .await;

        match result {
            Ok(rows) => rows.into_iter().collect(),
            Err(e) => {
                debug!("QuestDB model Sharpe query failed: {}", e);
                HashMap::new()
            }
        }
    }

    /// Query 30-day win rate per model
    async fn query_model_win_rates(&self) -> HashMap<String, (f64, u64)> {
        let result: Result<Vec<(String, f64, i64)>, _> = sqlx::query_as(
            "SELECT model_id,
                    sum(CASE WHEN signal_alignment > 0 THEN 1.0 ELSE 0.0 END) / count(1) as win_rate,
                    count(*) as trade_count
             FROM trade_outcomes
             WHERE timestamp > dateadd('d', -30, now())
             GROUP BY model_id"
        )
        .fetch_all(&self.pool)
        .await;

        match result {
            Ok(rows) => rows
                .into_iter()
                .map(|(id, wr, cnt)| (id, (wr, cnt as u64)))
                .collect(),
            Err(e) => {
                debug!("QuestDB win rate query failed: {}", e);
                HashMap::new()
            }
        }
    }

    /// Query confidence-bucketed win rates for gate optimization
    async fn query_confidence_buckets(&self) -> Vec<GateBucketMetrics> {
        // 5 buckets: [0.5-0.6), [0.6-0.7), [0.7-0.8), [0.8-0.9), [0.9-1.0]
        let result: Result<Vec<(f64, f64, f64, i64, f64)>, _> = sqlx::query_as(
            "SELECT confidence_bucket_lower,
                    confidence_bucket_upper,
                    sum(CASE WHEN signal_alignment > 0 THEN 1.0 ELSE 0.0 END) / count(1) as win_rate,
                    count(*) as trade_count,
                    avg(realized_pnl) as avg_pnl
             FROM trade_outcomes
             WHERE timestamp > dateadd('d', -30, now())
               AND confidence_bucket_lower IS NOT NULL
             GROUP BY confidence_bucket_lower, confidence_bucket_upper
             ORDER BY confidence_bucket_lower"
        )
        .fetch_all(&self.pool)
        .await;

        match result {
            Ok(rows) => rows
                .into_iter()
                .map(|(lower, upper, win_rate, count, avg_pnl)| GateBucketMetrics {
                    confidence_lower: lower,
                    confidence_upper: upper,
                    win_rate,
                    trade_count: count as u64,
                    avg_pnl,
                })
                .collect(),
            Err(e) => {
                debug!("QuestDB confidence bucket query failed: {}", e);
                Vec::new()
            }
        }
    }

    /// Query latest ensemble Sharpe (7-day)
    async fn query_ensemble_sharpe(&self) -> f64 {
        let result: Result<Option<(f64,)>, _> = sqlx::query_as(
            "SELECT sharpe_7d FROM ensemble_metrics ORDER BY timestamp DESC LIMIT 1"
        )
        .fetch_optional(&self.pool)
        .await;

        match result {
            Ok(Some((sharpe,))) => sharpe,
            Ok(None) => 0.0,
            Err(e) => {
                debug!("QuestDB ensemble Sharpe query failed: {}", e);
                0.0
            }
        }
    }
}

impl MetricsProvider for QuestDBMetricsProvider {
    fn get_model_metrics(&self) -> Vec<ModelRollingMetrics> {
        // block_in_place allows blocking inside a multi-threaded tokio runtime
        // by moving other tasks off this thread. Safe because the feedback loop
        // runs periodically (24h cycle) — not on the hot path.
        let handle = tokio::runtime::Handle::current();
        tokio::task::block_in_place(|| {
            let sharpes = handle.block_on(self.query_model_sharpe());
            let win_rates = handle.block_on(self.query_model_win_rates());

            let mut metrics = Vec::new();
            for (model_id, sharpe) in &sharpes {
                let (win_rate, trade_count) = win_rates
                    .get(model_id)
                    .copied()
                    .unwrap_or((0.5, 0));

                metrics.push(ModelRollingMetrics {
                    model_id: model_id.clone(),
                    sharpe_30d: *sharpe,
                    win_rate_30d: win_rate,
                    prediction_accuracy: win_rate, // Using win_rate as proxy
                    trade_count,
                    deployed_at: Instant::now() - std::time::Duration::from_secs(30 * 24 * 3600),
                });
            }
            metrics
        })
    }

    fn get_gate_buckets(&self) -> Vec<GateBucketMetrics> {
        let handle = tokio::runtime::Handle::current();
        tokio::task::block_in_place(|| handle.block_on(self.query_confidence_buckets()))
    }

    fn get_current_weights(&self) -> HashMap<String, f64> {
        let handle = tokio::runtime::Handle::current();
        tokio::task::block_in_place(|| {
            handle.block_on(async { self.current_weights.read().await.clone() })
        })
    }

    fn get_gate_config(&self) -> ConvictionGateConfig {
        let handle = tokio::runtime::Handle::current();
        tokio::task::block_in_place(|| {
            handle.block_on(async { self.gate_config.read().await.clone() })
        })
    }

    fn get_ensemble_sharpe_7d(&self) -> f64 {
        let handle = tokio::runtime::Handle::current();
        tokio::task::block_in_place(|| handle.block_on(self.query_ensemble_sharpe()))
    }

    fn get_model_correlations(&self) -> Vec<((String, String), f64)> {
        // Correlation requires cross-model signal comparison
        // For now, query is deferred until we have enough data
        Vec::new()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Drop and recreate all test tables for clean isolation between runs.
    /// QuestDB TRUNCATE via PG wire can be flaky; DROP + CREATE is reliable.
    async fn reset_tables(pool: &PgPool) {
        for table in &["model_predictions", "trade_outcomes", "ensemble_metrics"] {
            let drop_q = format!("DROP TABLE IF EXISTS {table};");
            let _ = sqlx::query(&drop_q).execute(pool).await;
        }
    }

    /// Poll QuestDB until a table has at least `min_rows` rows (up to 5s).
    async fn wait_for_rows(pool: &PgPool, table: &str, min_rows: i64) {
        for _ in 0..50 {
            let query = format!("SELECT count(*) FROM {table}");
            let row: Option<(i64,)> = sqlx::query_as(&query)
                .fetch_optional(pool)
                .await
                .ok()
                .flatten();
            if row.map(|(c,)| c).unwrap_or(0) >= min_rows {
                return;
            }
            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
        }
    }

    /// Comprehensive integration test: connect, write, read, verify metrics.
    /// Single test avoids parallel interference on shared QuestDB tables.
    ///
    /// Run with: docker-compose up -d questdb
    /// Then: SQLX_OFFLINE=true cargo test -p trading_service --lib -- questdb_metrics --include-ignored
    #[tokio::test(flavor = "multi_thread")]
    #[ignore = "Requires QuestDB on localhost:8812"]
    async fn test_questdb_provider_full_lifecycle() {
        let provider = QuestDBMetricsProvider::new("postgresql://admin:quest@localhost:8812/qdb")
            .await
            .expect("QuestDB should be running");

        // Drop + recreate for clean slate (prior test runs may have left data)
        reset_tables(&provider.pool).await;
        provider.ensure_tables().await.expect("Tables should be created");

        // Phase 1: Empty tables — metrics should return defaults
        let metrics = provider.get_model_metrics();
        assert!(metrics.is_empty(), "Empty tables should yield no metrics");

        let sharpe = provider.get_ensemble_sharpe_7d();
        assert!((sharpe - 0.0).abs() < 1e-10, "Empty ensemble Sharpe should be 0.0");

        // Phase 2: Write test data (multiple rows per model for valid stddev)
        sqlx::query(
            "INSERT INTO model_predictions(model_id, signal, confidence, prediction_accuracy, win_rate, timestamp)
             VALUES ('dqn', 0.8, 0.7, 0.62, 0.58, systimestamp()),
                    ('dqn', 0.6, 0.8, 0.65, 0.60, systimestamp()),
                    ('ppo', 0.3, 0.6, 0.55, 0.52, systimestamp()),
                    ('ppo', 0.5, 0.7, 0.58, 0.54, systimestamp())"
        )
        .execute(&provider.pool)
        .await
        .expect("Insert predictions should succeed");

        sqlx::query(
            "INSERT INTO trade_outcomes(model_id, realized_pnl, signal_alignment, confidence_bucket_lower, confidence_bucket_upper, timestamp)
             VALUES ('dqn', 100.0, 1.0, 0.60, 0.70, systimestamp()),
                    ('dqn', -50.0, -1.0, 0.60, 0.70, systimestamp()),
                    ('ppo', 75.0, 1.0, 0.50, 0.60, systimestamp())"
        )
        .execute(&provider.pool)
        .await
        .expect("Insert outcomes should succeed");

        sqlx::query(
            "INSERT INTO ensemble_metrics(sharpe_7d, sharpe_30d, total_pnl, timestamp)
             VALUES (1.5, 1.2, 5000.0, systimestamp())"
        )
        .execute(&provider.pool)
        .await
        .expect("Insert ensemble metrics should succeed");

        // Wait for QuestDB WAL to commit all tables
        wait_for_rows(&provider.pool, "model_predictions", 4).await;
        wait_for_rows(&provider.pool, "trade_outcomes", 3).await;
        wait_for_rows(&provider.pool, "ensemble_metrics", 1).await;

        // Phase 3: Read back and verify
        let metrics = provider.get_model_metrics();
        assert!(!metrics.is_empty(), "Should have model metrics after insert");
        assert!(metrics.len() >= 2, "Should have at least DQN and PPO");

        let sharpe = provider.get_ensemble_sharpe_7d();
        assert!((sharpe - 1.5).abs() < 0.1, "Sharpe should be ~1.5, got {sharpe}");

        let buckets = provider.get_gate_buckets();
        assert!(!buckets.is_empty(), "Should have confidence buckets");
    }

    #[tokio::test(flavor = "multi_thread")]
    #[ignore = "Requires QuestDB on localhost:8812"]
    async fn test_questdb_provider_try_new_succeeds() {
        let provider = QuestDBMetricsProvider::try_new("postgresql://admin:quest@localhost:8812/qdb").await;
        assert!(provider.is_some(), "Should connect to QuestDB");
    }

    #[tokio::test(flavor = "multi_thread")]
    async fn test_questdb_provider_try_new_fails_gracefully() {
        // Connect to a non-existent QuestDB — should return None, not panic
        let provider = QuestDBMetricsProvider::try_new("postgresql://admin:quest@localhost:19999/qdb").await;
        assert!(provider.is_none(), "Should fail gracefully");
    }

    #[tokio::test(flavor = "multi_thread")]
    #[ignore = "Requires QuestDB on localhost:8812"]
    async fn test_feedback_loop_with_questdb() {
        use crate::feedback_loop::{FeedbackLoop, FeedbackLoopConfig};
        use ml::ensemble::gate_optimizer::GateOptimizerConfig;
        use ml::ensemble::weight_optimizer::WeightOptimizerConfig;
        use std::time::Duration;

        let provider = QuestDBMetricsProvider::new("postgresql://admin:quest@localhost:8812/qdb")
            .await
            .expect("QuestDB should be running");

        provider.ensure_tables().await.expect("Tables should be created");

        // Set up some weights
        let mut weights = HashMap::new();
        weights.insert("dqn".to_string(), 0.5);
        weights.insert("ppo".to_string(), 0.5);
        provider.set_weights(weights).await;

        let feedback = FeedbackLoop::with_optimizers(
            FeedbackLoopConfig {
                cycle_interval: Duration::from_millis(100),
                ..Default::default()
            },
            WeightOptimizerConfig {
                cooldown: Duration::ZERO,
                ..Default::default()
            },
            GateOptimizerConfig {
                cooldown: Duration::ZERO,
                ..Default::default()
            },
        );

        // Run a cycle — with empty QuestDB should report InsufficientData
        let result = feedback.run_cycle(&provider).await;
        assert!(!result.kill_switch_activated);
    }
}