foxhunt/sql/paper_trading_schema.sql

-- Paper Trading Database Schema
-- Created: 2025-10-14
-- Purpose: Track ensemble predictions and simulated trades during Phase 1 paper trading

-- ============================================================================
-- Table 1: paper_trading_predictions
-- ============================================================================
-- Stores every ensemble prediction with per-model votes and simulated execution

CREATE TABLE IF NOT EXISTS paper_trading_predictions (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    symbol VARCHAR(20) NOT NULL,

    -- Ensemble decision
    ensemble_action VARCHAR(10) NOT NULL,  -- BUY, SELL, HOLD
    ensemble_signal DOUBLE PRECISION NOT NULL,  -- -1.0 to 1.0 (bearish to bullish)
    ensemble_confidence DOUBLE PRECISION NOT NULL,  -- 0.0 to 1.0
    disagreement_rate DOUBLE PRECISION NOT NULL,  -- 0.0 to 1.0 (% models disagree)

    -- Per-model votes (DQN)
    dqn_signal DOUBLE PRECISION,
    dqn_confidence DOUBLE PRECISION,
    dqn_weight DOUBLE PRECISION,

    -- Per-model votes (PPO)
    ppo_signal DOUBLE PRECISION,
    ppo_confidence DOUBLE PRECISION,
    ppo_weight DOUBLE PRECISION,

    -- Per-model votes (TFT) - Reserved for Phase 2
    tft_signal DOUBLE PRECISION,
    tft_confidence DOUBLE PRECISION,
    tft_weight DOUBLE PRECISION,

    -- Per-model votes (MAMBA-2) - Reserved for Phase 2
    mamba2_signal DOUBLE PRECISION,
    mamba2_confidence DOUBLE PRECISION,
    mamba2_weight DOUBLE PRECISION,

    -- Simulated execution (paper trading)
    executed BOOLEAN DEFAULT FALSE,
    execution_price DOUBLE PRECISION,  -- Price at which trade was executed
    position_size DOUBLE PRECISION,  -- Number of contracts/shares
    position_value DOUBLE PRECISION,  -- USD value of position

    -- Position tracking
    entry_price DOUBLE PRECISION,  -- Entry price for open positions
    exit_price DOUBLE PRECISION,  -- Exit price when position closed
    position_duration_seconds INTEGER,  -- How long position was held

    -- P&L tracking
    pnl DOUBLE PRECISION,  -- Realized P&L (USD)
    pnl_percentage DOUBLE PRECISION,  -- Realized P&L (%)
    commission_fees DOUBLE PRECISION DEFAULT 0.0,  -- Simulated commission
    slippage_cost DOUBLE PRECISION DEFAULT 0.0,  -- Simulated slippage

    -- Baseline comparison (current production strategy)
    baseline_action VARCHAR(10),  -- What current production would have done
    baseline_pnl DOUBLE PRECISION,  -- What current production would have made

    -- Metadata
    prediction_latency_us BIGINT,  -- Time to generate prediction (microseconds)
    aggregation_method VARCHAR(50) DEFAULT 'weighted_average',
    trading_mode VARCHAR(20) DEFAULT 'paper'  -- paper, live
);

COMMENT ON TABLE paper_trading_predictions IS 'Tracks all ensemble predictions and simulated trades during paper trading validation';
COMMENT ON COLUMN paper_trading_predictions.disagreement_rate IS 'Percentage of models disagreeing with ensemble decision (0.0-1.0)';
COMMENT ON COLUMN paper_trading_predictions.executed IS 'Whether this prediction resulted in a simulated trade';

-- Create indexes separately
CREATE INDEX IF NOT EXISTS idx_paper_trading_timestamp ON paper_trading_predictions (timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_paper_trading_symbol_timestamp ON paper_trading_predictions (symbol, timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_paper_trading_ensemble_action ON paper_trading_predictions (ensemble_action);
CREATE INDEX IF NOT EXISTS idx_paper_trading_executed ON paper_trading_predictions (executed);
CREATE INDEX IF NOT EXISTS idx_paper_trading_pnl ON paper_trading_predictions (pnl DESC);

-- TimescaleDB hypertable for time-series optimization (if TimescaleDB extension available)
DO $$
BEGIN
    IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN
        PERFORM create_hypertable('paper_trading_predictions', 'timestamp', if_not_exists => TRUE);
    END IF;
END $$;

-- ============================================================================
-- Table 2: model_performance_attribution
-- ============================================================================
-- Rolling window performance metrics per model and symbol

CREATE TABLE IF NOT EXISTS model_performance_attribution (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    model_id VARCHAR(50) NOT NULL,  -- DQN, PPO, TFT, MAMBA2
    symbol VARCHAR(20) NOT NULL,

    -- Performance metrics
    total_predictions INTEGER NOT NULL DEFAULT 0,
    correct_predictions INTEGER NOT NULL DEFAULT 0,
    accuracy DOUBLE PRECISION NOT NULL DEFAULT 0.0,  -- correct / total
    total_pnl DOUBLE PRECISION NOT NULL DEFAULT 0.0,
    sharpe_ratio DOUBLE PRECISION,
    max_drawdown DOUBLE PRECISION,

    -- Contribution to ensemble
    avg_weight DOUBLE PRECISION NOT NULL,
    avg_confidence DOUBLE PRECISION NOT NULL,
    avg_signal DOUBLE PRECISION,

    -- Rolling window
    window_hours INTEGER NOT NULL DEFAULT 24,  -- 1, 24, 168 (1h, 1d, 1w)

    -- Constraints
    CONSTRAINT valid_model_id CHECK (model_id IN ('DQN', 'PPO', 'TFT', 'MAMBA2', 'TLOB', 'Liquid')),
    CONSTRAINT valid_accuracy CHECK (accuracy >= 0.0 AND accuracy <= 1.0),
    CONSTRAINT valid_window CHECK (window_hours IN (1, 24, 168))
);

COMMENT ON TABLE model_performance_attribution IS 'Rolling window performance metrics for each model in the ensemble';

-- Create indexes separately
CREATE INDEX IF NOT EXISTS idx_model_perf_model_timestamp ON model_performance_attribution (model_id, timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_model_perf_symbol_timestamp ON model_performance_attribution (symbol, timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_model_perf_window ON model_performance_attribution (window_hours);

-- TimescaleDB hypertable
DO $$
BEGIN
    IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN
        PERFORM create_hypertable('model_performance_attribution', 'timestamp', if_not_exists => TRUE);
    END IF;
END $$;

-- ============================================================================
-- Materialized View 1: paper_trading_daily_performance
-- ============================================================================
-- Daily aggregated performance summary (refreshed nightly)

CREATE MATERIALIZED VIEW IF NOT EXISTS paper_trading_daily_performance AS
SELECT
    DATE(timestamp) AS date,
    symbol,
    COUNT(*) AS total_trades,
    SUM(CASE WHEN pnl > 0 THEN 1 ELSE 0 END) AS winning_trades,
    SUM(CASE WHEN pnl < 0 THEN 1 ELSE 0 END) AS losing_trades,
    SUM(CASE WHEN pnl = 0 THEN 1 ELSE 0 END) AS breakeven_trades,

    -- P&L metrics
    SUM(pnl) AS total_pnl,
    AVG(pnl) AS avg_pnl,
    STDDEV(pnl) AS pnl_stddev,
    MAX(pnl) AS max_win,
    MIN(pnl) AS max_loss,

    -- Ensemble metrics
    AVG(ensemble_confidence) AS avg_confidence,
    AVG(disagreement_rate) AS avg_disagreement,
    MAX(disagreement_rate) AS max_disagreement,

    -- Model weights
    AVG(dqn_weight) AS avg_dqn_weight,
    AVG(ppo_weight) AS avg_ppo_weight,
    AVG(tft_weight) AS avg_tft_weight,
    AVG(mamba2_weight) AS avg_mamba2_weight,

    -- Performance latency
    AVG(prediction_latency_us) AS avg_prediction_latency_us,
    PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY prediction_latency_us) AS p99_prediction_latency_us

FROM paper_trading_predictions
WHERE executed = TRUE
GROUP BY DATE(timestamp), symbol
ORDER BY date DESC, total_pnl DESC;

CREATE INDEX IF NOT EXISTS idx_daily_perf_date_symbol ON paper_trading_daily_performance (date DESC, symbol);

COMMENT ON MATERIALIZED VIEW paper_trading_daily_performance IS 'Daily aggregated performance metrics for paper trading validation';

-- ============================================================================
-- Materialized View 2: paper_trading_weekly_summary
-- ============================================================================
-- Weekly performance summary for Phase 1 completion report

CREATE MATERIALIZED VIEW IF NOT EXISTS paper_trading_weekly_summary AS
SELECT
    DATE_TRUNC('week', timestamp) AS week_start,
    symbol,
    COUNT(*) AS total_trades,
    SUM(CASE WHEN pnl > 0 THEN 1 ELSE 0 END) AS winning_trades,
    (SUM(CASE WHEN pnl > 0 THEN 1 ELSE 0 END)::FLOAT / NULLIF(COUNT(*), 0) * 100)::NUMERIC(5,2) AS win_rate_pct,

    -- P&L
    SUM(pnl) AS total_pnl,
    AVG(pnl) AS avg_pnl_per_trade,

    -- Risk metrics
    MAX(pnl) - MIN(pnl) AS pnl_range,
    STDDEV(pnl) / NULLIF(AVG(pnl), 0) AS coefficient_of_variation,

    -- Sharpe ratio (annualized)
    CASE
        WHEN STDDEV(pnl) > 0 THEN
            (AVG(pnl) / STDDEV(pnl)) * SQRT(252)  -- 252 trading days
        ELSE NULL
    END AS sharpe_ratio,

    -- Ensemble health
    AVG(ensemble_confidence) AS avg_confidence,
    AVG(disagreement_rate) AS avg_disagreement,

    -- Model contribution
    AVG(dqn_weight) AS avg_dqn_weight,
    AVG(ppo_weight) AS avg_ppo_weight,

    -- Baseline comparison
    SUM(baseline_pnl) AS baseline_total_pnl,
    (SUM(pnl) - SUM(baseline_pnl))::NUMERIC(12,2) AS pnl_vs_baseline,
    ((SUM(pnl) - SUM(baseline_pnl)) / NULLIF(ABS(SUM(baseline_pnl)), 0) * 100)::NUMERIC(5,2) AS pnl_vs_baseline_pct

FROM paper_trading_predictions
WHERE executed = TRUE
GROUP BY DATE_TRUNC('week', timestamp), symbol
ORDER BY week_start DESC, total_pnl DESC;

CREATE INDEX IF NOT EXISTS idx_weekly_summary_week_symbol ON paper_trading_weekly_summary (week_start DESC, symbol);

COMMENT ON MATERIALIZED VIEW paper_trading_weekly_summary IS '7-day rolling summary for Phase 1 completion report';

-- ============================================================================
-- View 1: high_disagreement_events
-- ============================================================================
-- Real-time view of high disagreement predictions (>50%)

CREATE OR REPLACE VIEW high_disagreement_events AS
SELECT
    timestamp,
    symbol,
    ensemble_action,
    ensemble_signal,
    ensemble_confidence,
    disagreement_rate,
    dqn_signal,
    ppo_signal,
    tft_signal,
    mamba2_signal,
    CASE
        WHEN disagreement_rate > 0.7 THEN 'CRITICAL'
        WHEN disagreement_rate > 0.5 THEN 'HIGH'
        ELSE 'NORMAL'
    END AS disagreement_severity
FROM paper_trading_predictions
WHERE disagreement_rate > 0.5
ORDER BY timestamp DESC;

COMMENT ON VIEW high_disagreement_events IS 'Real-time view of predictions with high model disagreement (>50%)';

-- ============================================================================
-- View 2: model_performance_comparison
-- ============================================================================
-- Compare per-model performance for attribution analysis

CREATE OR REPLACE VIEW model_performance_comparison AS
SELECT
    symbol,

    -- DQN metrics
    AVG(dqn_weight) AS dqn_avg_weight,
    SUM(CASE WHEN dqn_signal * pnl > 0 THEN pnl ELSE 0 END) AS dqn_pnl_contribution,

    -- PPO metrics
    AVG(ppo_weight) AS ppo_avg_weight,
    SUM(CASE WHEN ppo_signal * pnl > 0 THEN pnl ELSE 0 END) AS ppo_pnl_contribution,

    -- TFT metrics (Phase 2)
    AVG(tft_weight) AS tft_avg_weight,
    SUM(CASE WHEN tft_signal * pnl > 0 THEN pnl ELSE 0 END) AS tft_pnl_contribution,

    -- MAMBA-2 metrics (Phase 2)
    AVG(mamba2_weight) AS mamba2_avg_weight,
    SUM(CASE WHEN mamba2_signal * pnl > 0 THEN pnl ELSE 0 END) AS mamba2_pnl_contribution,

    -- Totals
    COUNT(*) AS total_predictions,
    SUM(pnl) AS total_ensemble_pnl

FROM paper_trading_predictions
WHERE executed = TRUE
  AND timestamp >= NOW() - INTERVAL '7 days'
GROUP BY symbol
ORDER BY total_ensemble_pnl DESC;

COMMENT ON VIEW model_performance_comparison IS 'Per-model P&L contribution analysis for 7-day rolling window';

-- ============================================================================
-- Function 1: refresh_paper_trading_views
-- ============================================================================
-- Refresh materialized views (call nightly via cron)

CREATE OR REPLACE FUNCTION refresh_paper_trading_views()
RETURNS void AS $$
BEGIN
    REFRESH MATERIALIZED VIEW CONCURRENTLY paper_trading_daily_performance;
    REFRESH MATERIALIZED VIEW CONCURRENTLY paper_trading_weekly_summary;
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION refresh_paper_trading_views() IS 'Refresh materialized views for paper trading metrics (run nightly)';

-- ============================================================================
-- Function 2: calculate_sharpe_ratio
-- ============================================================================
-- Calculate Sharpe ratio for a given symbol and time window

CREATE OR REPLACE FUNCTION calculate_sharpe_ratio(
    p_symbol VARCHAR(20),
    p_days INTEGER DEFAULT 7
)
RETURNS NUMERIC AS $$
DECLARE
    v_avg_return DOUBLE PRECISION;
    v_stddev DOUBLE PRECISION;
    v_sharpe NUMERIC;
BEGIN
    SELECT
        AVG(pnl),
        STDDEV(pnl)
    INTO v_avg_return, v_stddev
    FROM paper_trading_predictions
    WHERE symbol = p_symbol
      AND executed = TRUE
      AND timestamp >= NOW() - (p_days || ' days')::INTERVAL;

    IF v_stddev IS NULL OR v_stddev = 0 THEN
        RETURN NULL;
    END IF;

    -- Annualized Sharpe ratio (252 trading days)
    v_sharpe := (v_avg_return / v_stddev) * SQRT(252);

    RETURN v_sharpe::NUMERIC(10,4);
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION calculate_sharpe_ratio(VARCHAR, INTEGER) IS 'Calculate annualized Sharpe ratio for a symbol over N days';

-- ============================================================================
-- Function 3: calculate_max_drawdown
-- ============================================================================
-- Calculate maximum drawdown for a given symbol

CREATE OR REPLACE FUNCTION calculate_max_drawdown(
    p_symbol VARCHAR(20),
    p_days INTEGER DEFAULT 7
)
RETURNS NUMERIC AS $$
DECLARE
    v_max_drawdown NUMERIC;
BEGIN
    WITH cumulative_pnl AS (
        SELECT
            timestamp,
            SUM(pnl) OVER (ORDER BY timestamp) AS cum_pnl
        FROM paper_trading_predictions
        WHERE symbol = p_symbol
          AND executed = TRUE
          AND timestamp >= NOW() - (p_days || ' days')::INTERVAL
    ),
    running_max AS (
        SELECT
            timestamp,
            cum_pnl,
            MAX(cum_pnl) OVER (ORDER BY timestamp) AS peak
        FROM cumulative_pnl
    )
    SELECT
        MIN((cum_pnl - peak) / NULLIF(peak, 0) * 100) AS max_drawdown_pct
    INTO v_max_drawdown
    FROM running_max
    WHERE peak > 0;

    RETURN COALESCE(v_max_drawdown, 0)::NUMERIC(10,4);
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION calculate_max_drawdown(VARCHAR, INTEGER) IS 'Calculate maximum drawdown percentage for a symbol over N days';

-- ============================================================================
-- Table 3: paper_trading_circuit_breaker_log
-- ============================================================================
-- Log of circuit breaker activations

CREATE TABLE IF NOT EXISTS paper_trading_circuit_breaker_log (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    trigger_type VARCHAR(50) NOT NULL,  -- max_daily_loss, consecutive_losses, high_disagreement
    trigger_value DOUBLE PRECISION NOT NULL,
    threshold_value DOUBLE PRECISION NOT NULL,
    symbol VARCHAR(20),
    action_taken VARCHAR(100) NOT NULL,  -- halt_trading, reduce_position_size, alert_only
    resolved_at TIMESTAMPTZ,
    resolution_notes TEXT
);

COMMENT ON TABLE paper_trading_circuit_breaker_log IS 'Log of circuit breaker activations and resolutions';

-- Create indexes separately
CREATE INDEX IF NOT EXISTS idx_circuit_breaker_timestamp ON paper_trading_circuit_breaker_log (timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_circuit_breaker_trigger_type ON paper_trading_circuit_breaker_log (trigger_type);

-- ============================================================================
-- Insert Initial Data (Optional)
-- ============================================================================
-- Insert sample data for testing (remove in production)

-- Example: Successful trade
INSERT INTO paper_trading_predictions (
    symbol, ensemble_action, ensemble_signal, ensemble_confidence, disagreement_rate,
    dqn_signal, dqn_confidence, dqn_weight,
    ppo_signal, ppo_confidence, ppo_weight,
    executed, execution_price, position_size, position_value,
    entry_price, exit_price, pnl, pnl_percentage,
    baseline_action, baseline_pnl,
    prediction_latency_us
) VALUES (
    'ES.FUT', 'BUY', 0.75, 0.85, 0.25,
    0.8, 0.9, 0.5,
    0.7, 0.8, 0.5,
    TRUE, 4500.00, 2, 9000.00,
    4500.00, 4515.00, 30.00, 0.33,
    'HOLD', 0.00,
    42
);

-- Grant permissions (adjust users as needed)
-- GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA public TO foxhunt_app;
-- GRANT SELECT ON ALL MATERIALIZED VIEWS IN SCHEMA public TO foxhunt_app;
-- GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO foxhunt_app;

-- ============================================================================
-- Schema Validation Queries
-- ============================================================================

-- Verify tables created
SELECT table_name, table_type
FROM information_schema.tables
WHERE table_schema = 'public'
  AND table_name LIKE 'paper_trading%'
ORDER BY table_name;

-- Verify indexes created
SELECT tablename, indexname, indexdef
FROM pg_indexes
WHERE schemaname = 'public'
  AND tablename LIKE 'paper_trading%'
ORDER BY tablename, indexname;

-- Verify functions created
SELECT routine_name, routine_type
FROM information_schema.routines
WHERE routine_schema = 'public'
  AND (routine_name LIKE 'calculate_%' OR routine_name LIKE 'refresh_%')
ORDER BY routine_name;

COMMENT ON SCHEMA public IS 'Paper trading schema created: 2025-10-14';

-- ============================================================================
-- End of Schema
-- ============================================================================