foxhunt/migrations/023_ensemble_performance_tuning.sql

-- ================================================================================================
-- Migration 023: Ensemble ML Performance Tuning
-- High-frequency write optimization for 1000+ predictions/sec
-- ================================================================================================
-- Target: >1000 inserts/sec, <100ms P99 query latency, >5x compression ratio
-- ================================================================================================

-- ================================================================================================
-- PART 1: ENHANCED INDEXING STRATEGY
-- ================================================================================================

-- Drop redundant indexes from migration 022 that are covered by hypertable time-space indexes
DROP INDEX IF EXISTS idx_ensemble_predictions_symbol_timestamp;
DROP INDEX IF EXISTS idx_model_performance_symbol_timestamp;

-- Composite index for high-frequency writes (model_id + timestamp for fast lookups)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_model_performance_composite
ON model_performance_attribution (model_id, symbol, window_hours, timestamp DESC);

-- Index for ensemble prediction lookups by symbol (most common query pattern)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ensemble_predictions_symbol_action_timestamp
ON ensemble_predictions (symbol, ensemble_action, timestamp DESC)
WHERE timestamp > NOW() - INTERVAL '30 days'; -- Partial index for recent data only

-- Index for model checkpoint tracking (frequent lookup by checkpoint_id)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ensemble_predictions_checkpoints
ON ensemble_predictions (dqn_checkpoint_id, ppo_checkpoint_id, mamba2_checkpoint_id, tft_checkpoint_id)
WHERE timestamp > NOW() - INTERVAL '7 days';

-- Index for real-time performance monitoring (last 24 hours only)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_model_performance_realtime
ON model_performance_attribution (model_id, timestamp DESC)
WHERE timestamp > NOW() - INTERVAL '24 hours';

-- Covering index for P&L attribution queries (includes all needed columns)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ensemble_predictions_pnl_covering
ON ensemble_predictions (symbol, timestamp DESC)
INCLUDE (ensemble_action, ensemble_signal, pnl, order_id)
WHERE pnl IS NOT NULL AND timestamp > NOW() - INTERVAL '90 days';

-- Index for inference latency monitoring (P99 latency tracking)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ensemble_predictions_latency
ON ensemble_predictions (inference_latency_us DESC)
WHERE inference_latency_us IS NOT NULL AND timestamp > NOW() - INTERVAL '7 days';

-- ================================================================================================
-- PART 2: TIMESCALEDB COMPRESSION OPTIMIZATION
-- ================================================================================================

-- Drop old compression policies
SELECT remove_compression_policy('ensemble_predictions', if_exists => true);
SELECT remove_compression_policy('model_performance_attribution', if_exists => true);

-- Enhanced compression for ensemble_predictions (compress after 7 days, target >5x ratio)
ALTER TABLE ensemble_predictions SET (
    timescaledb.compress = true,
    timescaledb.compress_segmentby = 'symbol, ensemble_action',
    timescaledb.compress_orderby = 'timestamp DESC, id',
    timescaledb.compress_chunk_time_interval = '1 day'
);

-- Aggressive compression policy (7-day retention for hot data)
SELECT add_compression_policy('ensemble_predictions',
    compress_after => INTERVAL '7 days',
    if_not_exists => true
);

-- Enhanced compression for model_performance_attribution (compress after 14 days)
ALTER TABLE model_performance_attribution SET (
    timescaledb.compress = true,
    timescaledb.compress_segmentby = 'model_id, symbol, window_hours',
    timescaledb.compress_orderby = 'timestamp DESC, id',
    timescaledb.compress_chunk_time_interval = '1 day'
);

SELECT add_compression_policy('model_performance_attribution',
    compress_after => INTERVAL '14 days',
    if_not_exists => true
);

-- ================================================================================================
-- PART 3: CONTINUOUS AGGREGATES FOR REAL-TIME DASHBOARDS
-- ================================================================================================

-- Drop old continuous aggregates if they exist (to recreate with better config)
DROP MATERIALIZED VIEW IF EXISTS ensemble_performance_5min CASCADE;
DROP MATERIALIZED VIEW IF EXISTS model_performance_hourly CASCADE;

-- 5-minute ensemble performance (for real-time dashboards)
CREATE MATERIALIZED VIEW ensemble_performance_5min
WITH (timescaledb.continuous) AS
SELECT
    time_bucket('5 minutes', timestamp) AS bucket,
    symbol,
    ensemble_action,
    COUNT(*) AS prediction_count,
    AVG(ensemble_confidence) AS avg_confidence,
    STDDEV(ensemble_confidence) AS stddev_confidence,
    AVG(disagreement_rate) AS avg_disagreement,
    MAX(disagreement_rate) AS max_disagreement,
    AVG(inference_latency_us) AS avg_latency_us,
    PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us) AS p99_latency_us,
    SUM(CASE WHEN pnl IS NOT NULL THEN pnl ELSE 0 END) AS total_pnl,
    COUNT(CASE WHEN pnl > 0 THEN 1 END) AS winning_trades,
    COUNT(CASE WHEN pnl < 0 THEN 1 END) AS losing_trades,
    COUNT(CASE WHEN pnl IS NOT NULL THEN 1 END) AS total_trades,
    AVG(CASE WHEN pnl IS NOT NULL THEN pnl END) AS avg_trade_pnl
FROM ensemble_predictions
GROUP BY bucket, symbol, ensemble_action;

-- Refresh every 5 minutes (near real-time)
SELECT add_continuous_aggregate_policy('ensemble_performance_5min',
    start_offset => INTERVAL '30 minutes',
    end_offset => INTERVAL '5 minutes',
    schedule_interval => INTERVAL '5 minutes',
    if_not_exists => true
);

-- Hourly model performance comparison (detailed attribution)
CREATE MATERIALIZED VIEW model_performance_hourly
WITH (timescaledb.continuous) AS
SELECT
    time_bucket('1 hour', timestamp) AS bucket,
    model_id,
    symbol,
    window_hours,
    SUM(total_predictions) AS total_predictions,
    SUM(correct_predictions) AS correct_predictions,
    AVG(accuracy) AS avg_accuracy,
    STDDEV(accuracy) AS stddev_accuracy,
    SUM(total_pnl) AS total_pnl,
    AVG(sharpe_ratio) AS avg_sharpe_ratio,
    MAX(sharpe_ratio) AS max_sharpe_ratio,
    AVG(sortino_ratio) AS avg_sortino_ratio,
    AVG(max_drawdown) AS avg_max_drawdown,
    AVG(win_rate) AS avg_win_rate,
    AVG(avg_weight) AS avg_weight,
    AVG(avg_confidence) AS avg_confidence,
    AVG(disagreement_rate) AS avg_disagreement_rate,
    SUM(disagreement_count) AS total_disagreements
FROM model_performance_attribution
GROUP BY bucket, model_id, symbol, window_hours;

-- Refresh every hour
SELECT add_continuous_aggregate_policy('model_performance_hourly',
    start_offset => INTERVAL '6 hours',
    end_offset => INTERVAL '1 hour',
    schedule_interval => INTERVAL '1 hour',
    if_not_exists => true
);

-- Weekly ensemble summary (for long-term trend analysis)
CREATE MATERIALIZED VIEW ensemble_performance_weekly
WITH (timescaledb.continuous) AS
SELECT
    time_bucket('1 week', timestamp) AS bucket,
    symbol,
    COUNT(*) AS total_predictions,
    AVG(ensemble_confidence) AS avg_confidence,
    AVG(disagreement_rate) AS avg_disagreement,
    SUM(CASE WHEN pnl IS NOT NULL THEN pnl ELSE 0 END) AS total_pnl,
    COUNT(CASE WHEN pnl > 0 THEN 1 END) AS winning_trades,
    COUNT(CASE WHEN pnl IS NOT NULL THEN 1 END) AS total_trades,
    -- Sharpe ratio approximation
    AVG(CASE WHEN pnl IS NOT NULL THEN pnl END) /
        NULLIF(STDDEV(CASE WHEN pnl IS NOT NULL THEN pnl END), 0) AS sharpe_approx,
    -- Max drawdown approximation
    MIN(pnl) AS worst_trade,
    MAX(pnl) AS best_trade
FROM ensemble_predictions
GROUP BY bucket, symbol;

-- Refresh daily
SELECT add_continuous_aggregate_policy('ensemble_performance_weekly',
    start_offset => INTERVAL '1 month',
    end_offset => INTERVAL '1 day',
    schedule_interval => INTERVAL '1 day',
    if_not_exists => true
);

-- ================================================================================================
-- PART 4: STATISTICS COLLECTION TUNING
-- ================================================================================================

-- Increase statistics target for critical columns (better query planning)
ALTER TABLE ensemble_predictions ALTER COLUMN timestamp SET STATISTICS 1000;
ALTER TABLE ensemble_predictions ALTER COLUMN symbol SET STATISTICS 500;
ALTER TABLE ensemble_predictions ALTER COLUMN ensemble_action SET STATISTICS 200;
ALTER TABLE ensemble_predictions ALTER COLUMN disagreement_rate SET STATISTICS 200;

ALTER TABLE model_performance_attribution ALTER COLUMN model_id SET STATISTICS 500;
ALTER TABLE model_performance_attribution ALTER COLUMN timestamp SET STATISTICS 1000;
ALTER TABLE model_performance_attribution ALTER COLUMN symbol SET STATISTICS 500;
ALTER TABLE model_performance_attribution ALTER COLUMN sharpe_ratio SET STATISTICS 500;

-- Force statistics update
ANALYZE ensemble_predictions;
ANALYZE model_performance_attribution;

-- ================================================================================================
-- PART 5: RETENTION POLICIES (DATA LIFECYCLE MANAGEMENT)
-- ================================================================================================

-- Drop old retention policies if they exist
SELECT remove_retention_policy('ensemble_predictions', if_exists => true);
SELECT remove_retention_policy('model_performance_attribution', if_exists => true);

-- Retain ensemble predictions for 90 days (compressed after 7 days, deleted after 90)
SELECT add_retention_policy('ensemble_predictions',
    drop_after => INTERVAL '90 days',
    if_not_exists => true
);

-- Retain model performance for 180 days (6 months for long-term analysis)
SELECT add_retention_policy('model_performance_attribution',
    drop_after => INTERVAL '180 days',
    if_not_exists => true
);

-- ================================================================================================
-- PART 6: WRITE OPTIMIZATION FUNCTIONS
-- ================================================================================================

-- Bulk insert function for high-frequency predictions (batched writes)
CREATE OR REPLACE FUNCTION insert_ensemble_predictions_bulk(
    p_predictions JSONB -- Array of prediction objects
)
RETURNS INTEGER AS $$
DECLARE
    v_count INTEGER;
BEGIN
    -- Insert all predictions from JSONB array
    INSERT INTO ensemble_predictions (
        timestamp, symbol, account_id, strategy_id,
        ensemble_action, ensemble_signal, ensemble_confidence, disagreement_rate,
        dqn_signal, dqn_confidence, dqn_weight, dqn_vote,
        ppo_signal, ppo_confidence, ppo_weight, ppo_vote,
        mamba2_signal, mamba2_confidence, mamba2_weight, mamba2_vote,
        tft_signal, tft_confidence, tft_weight, tft_vote,
        inference_latency_us, aggregation_latency_us,
        feature_snapshot, metadata
    )
    SELECT
        (pred->>'timestamp')::TIMESTAMPTZ,
        pred->>'symbol',
        pred->>'account_id',
        pred->>'strategy_id',
        pred->>'ensemble_action',
        (pred->>'ensemble_signal')::DOUBLE PRECISION,
        (pred->>'ensemble_confidence')::DOUBLE PRECISION,
        (pred->>'disagreement_rate')::DOUBLE PRECISION,
        (pred->>'dqn_signal')::DOUBLE PRECISION,
        (pred->>'dqn_confidence')::DOUBLE PRECISION,
        (pred->>'dqn_weight')::DOUBLE PRECISION,
        pred->>'dqn_vote',
        (pred->>'ppo_signal')::DOUBLE PRECISION,
        (pred->>'ppo_confidence')::DOUBLE PRECISION,
        (pred->>'ppo_weight')::DOUBLE PRECISION,
        pred->>'ppo_vote',
        (pred->>'mamba2_signal')::DOUBLE PRECISION,
        (pred->>'mamba2_confidence')::DOUBLE PRECISION,
        (pred->>'mamba2_weight')::DOUBLE PRECISION,
        pred->>'mamba2_vote',
        (pred->>'tft_signal')::DOUBLE PRECISION,
        (pred->>'tft_confidence')::DOUBLE PRECISION,
        (pred->>'tft_weight')::DOUBLE PRECISION,
        pred->>'tft_vote',
        (pred->>'inference_latency_us')::INTEGER,
        (pred->>'aggregation_latency_us')::INTEGER,
        (pred->'feature_snapshot')::JSONB,
        (pred->'metadata')::JSONB
    FROM jsonb_array_elements(p_predictions) AS pred;

    GET DIAGNOSTICS v_count = ROW_COUNT;
    RETURN v_count;
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION insert_ensemble_predictions_bulk IS 'Bulk insert function for high-frequency predictions (batched writes, >1000/sec)';

-- Bulk update function for P&L attribution (post-trade execution)
CREATE OR REPLACE FUNCTION update_ensemble_pnl_bulk(
    p_updates JSONB -- Array of {id, order_id, executed_price, position_size, pnl, commission, slippage_bps}
)
RETURNS INTEGER AS $$
DECLARE
    v_count INTEGER := 0;
    v_update JSONB;
BEGIN
    -- Update each prediction with execution data
    FOR v_update IN SELECT jsonb_array_elements(p_updates)
    LOOP
        UPDATE ensemble_predictions
        SET
            order_id = (v_update->>'order_id')::UUID,
            executed_price = (v_update->>'executed_price')::BIGINT,
            position_size = (v_update->>'position_size')::BIGINT,
            pnl = (v_update->>'pnl')::BIGINT,
            commission = COALESCE((v_update->>'commission')::BIGINT, 0),
            slippage_bps = (v_update->>'slippage_bps')::INTEGER
        WHERE id = (v_update->>'id')::UUID;

        v_count := v_count + 1;
    END LOOP;

    RETURN v_count;
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION update_ensemble_pnl_bulk IS 'Bulk update P&L for executed predictions (post-trade attribution)';

-- ================================================================================================
-- PART 7: MONITORING VIEWS
-- ================================================================================================

-- Real-time write throughput view (last 5 minutes)
CREATE OR REPLACE VIEW ensemble_write_throughput_5min AS
SELECT
    time_bucket('1 minute', timestamp) AS minute,
    COUNT(*) AS inserts_per_minute,
    COUNT(*) / 60.0 AS inserts_per_second,
    AVG(inference_latency_us) AS avg_inference_us,
    PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us) AS p99_inference_us
FROM ensemble_predictions
WHERE timestamp >= NOW() - INTERVAL '5 minutes'
GROUP BY minute
ORDER BY minute DESC;

COMMENT ON VIEW ensemble_write_throughput_5min IS 'Real-time write throughput monitoring (last 5 minutes)';

-- Compression efficiency view
CREATE OR REPLACE VIEW ensemble_compression_stats AS
SELECT
    hypertable_name,
    chunk_name,
    before_compression_total_bytes / 1024.0 / 1024.0 AS uncompressed_mb,
    after_compression_total_bytes / 1024.0 / 1024.0 AS compressed_mb,
    before_compression_total_bytes::FLOAT / NULLIF(after_compression_total_bytes, 0) AS compression_ratio,
    number_compressed_rows,
    pg_size_pretty(before_compression_total_bytes) AS uncompressed_size,
    pg_size_pretty(after_compression_total_bytes) AS compressed_size
FROM timescaledb_information.compressed_chunk_stats
WHERE hypertable_name IN ('ensemble_predictions', 'model_performance_attribution')
ORDER BY before_compression_total_bytes DESC;

COMMENT ON VIEW ensemble_compression_stats IS 'TimescaleDB compression efficiency metrics';

-- Query performance view (pg_stat_statements required)
CREATE OR REPLACE VIEW ensemble_query_performance AS
SELECT
    LEFT(query, 100) AS query_preview,
    calls,
    total_exec_time / 1000.0 AS total_time_sec,
    mean_exec_time AS avg_time_ms,
    max_exec_time AS max_time_ms,
    stddev_exec_time AS stddev_time_ms,
    rows / NULLIF(calls, 0) AS avg_rows_per_call
FROM pg_stat_statements
WHERE query LIKE '%ensemble_predictions%' OR query LIKE '%model_performance_attribution%'
ORDER BY mean_exec_time DESC
LIMIT 20;

COMMENT ON VIEW ensemble_query_performance IS 'Top 20 slowest ensemble queries (requires pg_stat_statements)';

-- ================================================================================================
-- PART 8: GRANT PERMISSIONS
-- ================================================================================================

GRANT SELECT ON ensemble_performance_5min TO foxhunt;
GRANT SELECT ON model_performance_hourly TO foxhunt;
GRANT SELECT ON ensemble_performance_weekly TO foxhunt;
GRANT SELECT ON ensemble_write_throughput_5min TO foxhunt;
GRANT SELECT ON ensemble_compression_stats TO foxhunt;
GRANT SELECT ON ensemble_query_performance TO foxhunt;

GRANT EXECUTE ON FUNCTION insert_ensemble_predictions_bulk TO foxhunt;
GRANT EXECUTE ON FUNCTION update_ensemble_pnl_bulk TO foxhunt;

-- ================================================================================================
-- END MIGRATION 023
-- ================================================================================================