foxhunt/migrations/025_query_optimization.sql

-- ================================================================================================
-- Migration 025: Query Performance Optimization
-- Additional optimizations for paper trading validation queries
-- ================================================================================================
-- Target: <5ms P99 query latency for aggregation queries, optimize TimescaleDB chunk exclusion
-- ================================================================================================

-- ================================================================================================
-- PART 1: ENHANCED COMPOSITE INDEXES FOR COMMON QUERY PATTERNS
-- ================================================================================================

-- Note: TimescaleDB hypertables do not support CONCURRENTLY, using regular CREATE INDEX
-- Optimize symbol-filtered aggregation queries (from paper trading validation)
-- Pattern: WHERE symbol = ? AND timestamp > ?
CREATE INDEX IF NOT EXISTS idx_ensemble_predictions_symbol_time
ON ensemble_predictions (symbol, timestamp DESC)
WHERE timestamp > NOW() - INTERVAL '30 days';

-- Optimize real-time dashboard queries (last 24 hours)
-- Pattern: WHERE timestamp > NOW() - INTERVAL '1 day'
-- Note: This is a partial index covering only recent data for faster scans
CREATE INDEX IF NOT EXISTS idx_ensemble_predictions_recent_24h
ON ensemble_predictions (timestamp DESC)
INCLUDE (ensemble_confidence, disagreement_rate, ensemble_action, symbol)
WHERE timestamp > NOW() - INTERVAL '24 hours';

-- Optimize model-specific queries (individual model performance)
-- Pattern: WHERE dqn_signal IS NOT NULL
CREATE INDEX IF NOT EXISTS idx_ensemble_predictions_dqn_active
ON ensemble_predictions (timestamp DESC)
WHERE dqn_signal IS NOT NULL;

CREATE INDEX IF NOT EXISTS idx_ensemble_predictions_ppo_active
ON ensemble_predictions (timestamp DESC)
WHERE ppo_signal IS NOT NULL;

CREATE INDEX IF NOT EXISTS idx_ensemble_predictions_mamba2_active
ON ensemble_predictions (timestamp DESC)
WHERE mamba2_signal IS NOT NULL;

CREATE INDEX IF NOT EXISTS idx_ensemble_predictions_tft_active
ON ensemble_predictions (timestamp DESC)
WHERE tft_signal IS NOT NULL;

-- Optimize order execution tracking (predictions that converted to orders)
CREATE INDEX IF NOT EXISTS idx_ensemble_predictions_executed
ON ensemble_predictions (timestamp DESC)
INCLUDE (order_id, executed_price, position_size, pnl)
WHERE order_id IS NOT NULL;

-- ================================================================================================
-- PART 2: MATERIALIZED VIEWS FOR SLOW AGGREGATION QUERIES
-- ================================================================================================

-- Real-time model activity summary (for debugging NULL model votes)
-- Refreshes every minute to catch inactive models quickly
DROP MATERIALIZED VIEW IF EXISTS model_activity_realtime CASCADE;

CREATE MATERIALIZED VIEW model_activity_realtime AS
SELECT
    time_bucket('1 minute', timestamp) AS minute,
    symbol,
    COUNT(*) AS total_predictions,
    COUNT(dqn_signal) AS dqn_active_count,
    COUNT(ppo_signal) AS ppo_active_count,
    COUNT(mamba2_signal) AS mamba2_active_count,
    COUNT(tft_signal) AS tft_active_count,
    ROUND(100.0 * COUNT(dqn_signal) / NULLIF(COUNT(*), 0), 2) AS dqn_active_pct,
    ROUND(100.0 * COUNT(ppo_signal) / NULLIF(COUNT(*), 0), 2) AS ppo_active_pct,
    ROUND(100.0 * COUNT(mamba2_signal) / NULLIF(COUNT(*), 0), 2) AS mamba2_active_pct,
    ROUND(100.0 * COUNT(tft_signal) / NULLIF(COUNT(*), 0), 2) AS tft_active_pct,
    AVG(ensemble_confidence) AS avg_confidence,
    AVG(disagreement_rate) AS avg_disagreement
FROM ensemble_predictions
WHERE timestamp > NOW() - INTERVAL '1 hour'
GROUP BY minute, symbol
ORDER BY minute DESC;

CREATE INDEX ON model_activity_realtime (minute DESC);
CREATE INDEX ON model_activity_realtime (symbol);

COMMENT ON MATERIALIZED VIEW model_activity_realtime IS 'Real-time model activity tracking (last 1 hour, 1-minute buckets)';

-- Paper trading execution summary (for monitoring order conversion rate)
DROP MATERIALIZED VIEW IF EXISTS paper_trading_execution_summary CASCADE;

CREATE MATERIALIZED VIEW paper_trading_execution_summary AS
SELECT
    time_bucket('5 minutes', timestamp) AS bucket,
    symbol,
    COUNT(*) AS total_predictions,
    COUNT(order_id) AS executed_orders,
    ROUND(100.0 * COUNT(order_id) / NULLIF(COUNT(*), 0), 2) AS execution_rate_pct,
    COUNT(CASE WHEN pnl > 0 THEN 1 END) AS winning_trades,
    COUNT(CASE WHEN pnl < 0 THEN 1 END) AS losing_trades,
    COUNT(CASE WHEN pnl IS NOT NULL THEN 1 END) AS total_trades,
    ROUND(100.0 * COUNT(CASE WHEN pnl > 0 THEN 1 END) / NULLIF(COUNT(CASE WHEN pnl IS NOT NULL THEN 1 END), 0), 2) AS win_rate_pct,
    SUM(pnl) AS total_pnl,
    AVG(pnl) AS avg_pnl,
    STDDEV(pnl) AS stddev_pnl,
    MIN(pnl) AS worst_trade,
    MAX(pnl) AS best_trade
FROM ensemble_predictions
WHERE timestamp > NOW() - INTERVAL '24 hours'
GROUP BY bucket, symbol
ORDER BY bucket DESC;

CREATE INDEX ON paper_trading_execution_summary (bucket DESC);
CREATE INDEX ON paper_trading_execution_summary (symbol);

COMMENT ON MATERIALIZED VIEW paper_trading_execution_summary IS 'Paper trading execution rate and P&L summary (last 24 hours)';

-- ================================================================================================
-- PART 3: OPTIMIZED QUERY FUNCTIONS (PRE-COMPUTED AGGREGATIONS)
-- ================================================================================================

-- Fast aggregation function for real-time dashboard queries
-- Uses continuous aggregates instead of scanning raw table
CREATE OR REPLACE FUNCTION get_ensemble_performance_summary(
    p_interval INTERVAL DEFAULT INTERVAL '1 day',
    p_symbol VARCHAR(20) DEFAULT NULL
)
RETURNS TABLE (
    avg_confidence DOUBLE PRECISION,
    avg_disagreement DOUBLE PRECISION,
    total_predictions BIGINT,
    total_trades BIGINT,
    win_rate DOUBLE PRECISION,
    total_pnl NUMERIC,
    avg_latency_us NUMERIC,
    p99_latency_us DOUBLE PRECISION
) AS $$
BEGIN
    RETURN QUERY
    SELECT
        AVG(ep5m.avg_confidence)::DOUBLE PRECISION,
        AVG(ep5m.avg_disagreement)::DOUBLE PRECISION,
        SUM(ep5m.prediction_count)::BIGINT,
        SUM(ep5m.total_trades)::BIGINT,
        (100.0 * SUM(ep5m.winning_trades) / NULLIF(SUM(ep5m.total_trades), 0))::DOUBLE PRECISION,
        SUM(ep5m.total_pnl),
        AVG(ep5m.avg_latency_us),
        MAX(ep5m.p99_latency_us)::DOUBLE PRECISION
    FROM ensemble_performance_5min ep5m
    WHERE ep5m.bucket > NOW() - p_interval
        AND (p_symbol IS NULL OR ep5m.symbol = p_symbol);
END;
$$ LANGUAGE plpgsql STABLE;

COMMENT ON FUNCTION get_ensemble_performance_summary IS 'Fast aggregation using continuous aggregates (avoids raw table scan)';

-- Model activity health check function
-- Quickly identifies inactive models
CREATE OR REPLACE FUNCTION check_model_activity_health(
    p_lookback_minutes INTEGER DEFAULT 60
)
RETURNS TABLE (
    model_name VARCHAR(20),
    is_active BOOLEAN,
    last_prediction_time TIMESTAMPTZ,
    minutes_since_last_prediction INTEGER,
    predictions_in_window BIGINT,
    activity_rate_pct DOUBLE PRECISION
) AS $$
BEGIN
    RETURN QUERY
    WITH recent_predictions AS (
        SELECT
            timestamp,
            dqn_signal IS NOT NULL AS dqn_active,
            ppo_signal IS NOT NULL AS ppo_active,
            mamba2_signal IS NOT NULL AS mamba2_active,
            tft_signal IS NOT NULL AS tft_active
        FROM ensemble_predictions
        WHERE timestamp > NOW() - INTERVAL '1 minute' * p_lookback_minutes
    ),
    model_stats AS (
        SELECT
            'DQN' AS model,
            MAX(CASE WHEN dqn_active THEN timestamp END) AS last_pred,
            COUNT(CASE WHEN dqn_active THEN 1 END) AS pred_count,
            COUNT(*) AS total_count
        FROM recent_predictions
        UNION ALL
        SELECT
            'PPO',
            MAX(CASE WHEN ppo_active THEN timestamp END),
            COUNT(CASE WHEN ppo_active THEN 1 END),
            COUNT(*)
        FROM recent_predictions
        UNION ALL
        SELECT
            'MAMBA-2',
            MAX(CASE WHEN mamba2_active THEN timestamp END),
            COUNT(CASE WHEN mamba2_active THEN 1 END),
            COUNT(*)
        FROM recent_predictions
        UNION ALL
        SELECT
            'TFT',
            MAX(CASE WHEN tft_active THEN timestamp END),
            COUNT(CASE WHEN tft_active THEN 1 END),
            COUNT(*)
        FROM recent_predictions
    )
    SELECT
        ms.model::VARCHAR(20),
        (ms.pred_count > 0)::BOOLEAN,
        ms.last_pred,
        EXTRACT(EPOCH FROM (NOW() - COALESCE(ms.last_pred, NOW() - INTERVAL '1 year')))::INTEGER / 60,
        ms.pred_count::BIGINT,
        (100.0 * ms.pred_count / NULLIF(ms.total_count, 0))::DOUBLE PRECISION
    FROM model_stats ms;
END;
$$ LANGUAGE plpgsql STABLE;

COMMENT ON FUNCTION check_model_activity_health IS 'Quickly identifies inactive models (NULL signal issue)';

-- ================================================================================================
-- PART 4: QUERY PERFORMANCE MONITORING
-- ================================================================================================

-- Create extension for query statistics if not exists
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;

-- View for monitoring slow queries (updated from migration 023)
CREATE OR REPLACE VIEW ensemble_slow_queries AS
SELECT
    LEFT(query, 150) AS query_preview,
    calls,
    ROUND(total_exec_time::NUMERIC / 1000.0, 2) AS total_time_sec,
    ROUND(mean_exec_time::NUMERIC, 2) AS avg_time_ms,
    ROUND(max_exec_time::NUMERIC, 2) AS max_time_ms,
    ROUND(stddev_exec_time::NUMERIC, 2) AS stddev_time_ms,
    rows / NULLIF(calls, 0) AS avg_rows_per_call,
    ROUND(100.0 * shared_blks_hit / NULLIF(shared_blks_hit + shared_blks_read, 0), 2) AS cache_hit_ratio
FROM pg_stat_statements
WHERE query LIKE '%ensemble_predictions%'
   OR query LIKE '%model_performance_attribution%'
   OR query LIKE '%paper_trading_predictions%'
ORDER BY mean_exec_time DESC
LIMIT 30;

COMMENT ON VIEW ensemble_slow_queries IS 'Top 30 slowest ensemble/paper trading queries with cache hit ratio';

-- ================================================================================================
-- PART 5: VACUUM AND ANALYZE OPTIMIZATION
-- ================================================================================================

-- Optimize autovacuum settings for high-write tables
ALTER TABLE ensemble_predictions SET (
    autovacuum_vacuum_scale_factor = 0.05,  -- Vacuum when 5% of rows change (default 20%)
    autovacuum_analyze_scale_factor = 0.025, -- Analyze when 2.5% change (default 10%)
    autovacuum_vacuum_cost_delay = 10        -- Speed up vacuum (default 20ms)
);

ALTER TABLE model_performance_attribution SET (
    autovacuum_vacuum_scale_factor = 0.05,
    autovacuum_analyze_scale_factor = 0.025,
    autovacuum_vacuum_cost_delay = 10
);

ALTER TABLE paper_trading_predictions SET (
    autovacuum_vacuum_scale_factor = 0.05,
    autovacuum_analyze_scale_factor = 0.025,
    autovacuum_vacuum_cost_delay = 10
);

-- Force immediate vacuum and analyze
VACUUM ANALYZE ensemble_predictions;
VACUUM ANALYZE model_performance_attribution;
VACUUM ANALYZE paper_trading_predictions;

-- ================================================================================================
-- PART 6: GRANT PERMISSIONS
-- ================================================================================================

GRANT SELECT ON model_activity_realtime TO foxhunt;
GRANT SELECT ON paper_trading_execution_summary TO foxhunt;
GRANT SELECT ON ensemble_slow_queries TO foxhunt;
GRANT EXECUTE ON FUNCTION get_ensemble_performance_summary TO foxhunt;
GRANT EXECUTE ON FUNCTION check_model_activity_health TO foxhunt;

-- ================================================================================================
-- PART 7: REFRESH MATERIALIZED VIEWS
-- ================================================================================================

REFRESH MATERIALIZED VIEW model_activity_realtime;
REFRESH MATERIALIZED VIEW paper_trading_execution_summary;

-- ================================================================================================
-- END MIGRATION 025
-- ================================================================================================