foxhunt/migrations/002_up_create_risk_performance_tables.sql

-- Migration 002: Create risk management and performance tracking tables
-- This migration adds comprehensive risk monitoring and performance metrics capabilities

-- Risk metrics table - comprehensive risk tracking
CREATE TABLE IF NOT EXISTS risk_metrics (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    account_id VARCHAR(64), -- Account identifier
    metric_type VARCHAR(50) NOT NULL CHECK (metric_type IN (
        'exposure', 'var', 'drawdown', 'violation', 'concentration',
        'leverage', 'margin', 'volatility', 'beta', 'correlation'
    )),
    symbol VARCHAR(32), -- NULL for portfolio-level metrics
    value DECIMAL(20, 8) NOT NULL, -- Metric value with high precision
    threshold DECIMAL(20, 8), -- Risk threshold
    severity VARCHAR(20) NOT NULL DEFAULT 'low' CHECK (severity IN ('low', 'medium', 'high', 'critical')),
    description TEXT NOT NULL, -- Human-readable description
    timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    metadata JSONB -- Additional risk data
);

-- Risk violations table - audit trail of risk breaches
CREATE TABLE IF NOT EXISTS risk_violations (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    account_id VARCHAR(64),
    violation_type VARCHAR(50) NOT NULL,
    symbol VARCHAR(32),
    threshold_value DECIMAL(20, 8) NOT NULL,
    actual_value DECIMAL(20, 8) NOT NULL,
    severity VARCHAR(20) NOT NULL CHECK (severity IN ('warning', 'error', 'critical')),
    description TEXT NOT NULL,
    action_taken VARCHAR(100), -- Action taken in response
    resolved_at TIMESTAMP WITH TIME ZONE, -- When violation was resolved
    timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    metadata JSONB
);

-- Daily statistics table - comprehensive daily trading metrics
CREATE TABLE IF NOT EXISTS daily_stats (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    account_id VARCHAR(64),
    date DATE NOT NULL, -- Trading date
    total_trades INTEGER NOT NULL DEFAULT 0,
    total_volume BIGINT NOT NULL DEFAULT 0,
    gross_pnl BIGINT NOT NULL DEFAULT 0, -- Gross P&L in fixed-point cents
    net_pnl BIGINT NOT NULL DEFAULT 0, -- Net P&L after fees in fixed-point cents
    fees_paid BIGINT NOT NULL DEFAULT 0, -- Total fees paid in fixed-point cents
    winning_trades INTEGER NOT NULL DEFAULT 0,
    losing_trades INTEGER NOT NULL DEFAULT 0,
    largest_win BIGINT NOT NULL DEFAULT 0, -- Largest winning trade
    largest_loss BIGINT NOT NULL DEFAULT 0, -- Largest losing trade (negative)
    max_drawdown DECIMAL(10, 4) NOT NULL DEFAULT 0, -- Maximum drawdown percentage
    max_position_size BIGINT NOT NULL DEFAULT 0, -- Maximum position size held
    avg_trade_size BIGINT NOT NULL DEFAULT 0, -- Average trade size
    sharpe_ratio DECIMAL(10, 4), -- Sharpe ratio
    win_rate DECIMAL(5, 4), -- Win rate percentage (0-1)
    profit_factor DECIMAL(10, 4), -- Profit factor
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    
    -- Ensure unique stats per account-date combination
    UNIQUE(account_id, date)
);

-- Performance metrics table - system and trading performance tracking
CREATE TABLE IF NOT EXISTS performance_metrics (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    metric_name VARCHAR(100) NOT NULL, -- e.g., "latency_p50", "latency_p95", "throughput"
    metric_value DECIMAL(20, 8) NOT NULL, -- Metric value
    unit VARCHAR(50) NOT NULL, -- "nanoseconds", "ops_per_second", "percent", etc.
    component VARCHAR(100) NOT NULL, -- "order_processing", "market_data", "risk_engine"
    timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
    tags JSONB, -- Additional tags for grouping/filtering
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
    
    -- Composite indexes will be created after table creation
);

-- Audit logs table - comprehensive audit trail for compliance
CREATE TABLE IF NOT EXISTS audit_logs (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    event_type VARCHAR(50) NOT NULL, -- "order_placed", "trade_executed", "position_updated"
    entity_type VARCHAR(50) NOT NULL, -- "order", "fill", "position", "risk_metric"
    entity_id UUID NOT NULL, -- ID of the affected entity
    account_id VARCHAR(64),
    user_id VARCHAR(64),
    action VARCHAR(50) NOT NULL, -- "create", "update", "delete", "execute"
    old_values JSONB, -- Previous state (for updates)
    new_values JSONB, -- New state
    timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
    source VARCHAR(100) NOT NULL, -- System component that generated the event
    correlation_id UUID, -- For tracking related events
    session_id VARCHAR(128), -- User session identifier
    ip_address INET, -- Client IP address
    user_agent TEXT, -- Client user agent
    metadata JSONB, -- Additional audit metadata
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);

-- Strategy performance table - track individual strategy performance
CREATE TABLE IF NOT EXISTS strategy_performance (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    strategy_name VARCHAR(100) NOT NULL,
    account_id VARCHAR(64),
    date DATE NOT NULL,
    trades_count INTEGER NOT NULL DEFAULT 0,
    total_pnl BIGINT NOT NULL DEFAULT 0, -- P&L in fixed-point cents
    win_rate DECIMAL(5, 4), -- Win rate (0-1)
    sharpe_ratio DECIMAL(10, 4),
    max_drawdown DECIMAL(10, 4),
    avg_trade_duration INTERVAL, -- Average time positions are held
    total_volume BIGINT NOT NULL DEFAULT 0,
    risk_adjusted_return DECIMAL(10, 4),
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    
    UNIQUE(strategy_name, account_id, date)
);

-- Symbol statistics table - per-symbol performance and characteristics
CREATE TABLE IF NOT EXISTS symbol_stats (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    symbol VARCHAR(32) NOT NULL,
    date DATE NOT NULL,
    open_price BIGINT, -- Opening price in fixed-point cents
    high_price BIGINT, -- High price
    low_price BIGINT, -- Low price
    close_price BIGINT, -- Closing price
    volume BIGINT NOT NULL DEFAULT 0,
    trade_count INTEGER NOT NULL DEFAULT 0,
    vwap BIGINT, -- Volume-weighted average price
    volatility DECIMAL(10, 6), -- Daily volatility
    beta DECIMAL(10, 4), -- Beta relative to market
    correlation_spy DECIMAL(10, 4), -- Correlation to SPY
    avg_spread BIGINT, -- Average bid-ask spread
    liquidity_score DECIMAL(5, 2), -- Liquidity scoring
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    
    UNIQUE(symbol, date)
);

-- Create composite indexes for risk_metrics table
CREATE INDEX IF NOT EXISTS idx_risk_metrics_composite ON risk_metrics(metric_type, severity, timestamp);
CREATE INDEX IF NOT EXISTS idx_risk_metrics_symbol_type ON risk_metrics(symbol, metric_type);
CREATE INDEX IF NOT EXISTS idx_risk_metrics_account_timestamp ON risk_metrics(account_id, timestamp);

-- Create composite indexes for performance_metrics table
CREATE INDEX IF NOT EXISTS idx_performance_metrics_component_name_timestamp ON performance_metrics(component, metric_name, timestamp);
CREATE INDEX IF NOT EXISTS idx_performance_metrics_timestamp ON performance_metrics(timestamp);

-- Create optimized indexes for performance queries

-- Risk metrics indexes
CREATE INDEX IF NOT EXISTS idx_risk_metrics_timestamp ON risk_metrics(timestamp);
CREATE INDEX IF NOT EXISTS idx_risk_metrics_account_id ON risk_metrics(account_id) WHERE account_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_risk_violations_timestamp ON risk_violations(timestamp);
CREATE INDEX IF NOT EXISTS idx_risk_violations_severity ON risk_violations(severity);

-- Daily stats indexes
CREATE INDEX IF NOT EXISTS idx_daily_stats_date ON daily_stats(date);
CREATE INDEX IF NOT EXISTS idx_daily_stats_account_date ON daily_stats(account_id, date);

-- Performance metrics indexes (optimized for time-series analysis)
CREATE INDEX IF NOT EXISTS idx_performance_metrics_name_timestamp ON performance_metrics(metric_name, timestamp);

-- Audit logs indexes (optimized for compliance queries)
CREATE INDEX IF NOT EXISTS idx_audit_logs_timestamp ON audit_logs(timestamp);
CREATE INDEX IF NOT EXISTS idx_audit_logs_entity ON audit_logs(entity_type, entity_id);
CREATE INDEX IF NOT EXISTS idx_audit_logs_account_id ON audit_logs(account_id) WHERE account_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_audit_logs_user_id ON audit_logs(user_id) WHERE user_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_audit_logs_correlation_id ON audit_logs(correlation_id) WHERE correlation_id IS NOT NULL;

-- Strategy performance indexes
CREATE INDEX IF NOT EXISTS idx_strategy_performance_name_date ON strategy_performance(strategy_name, date);
CREATE INDEX IF NOT EXISTS idx_strategy_performance_account_date ON strategy_performance(account_id, date);

-- Symbol stats indexes
CREATE INDEX IF NOT EXISTS idx_symbol_stats_symbol_date ON symbol_stats(symbol, date);
CREATE INDEX IF NOT EXISTS idx_symbol_stats_date ON symbol_stats(date);

-- Create triggers for automatic timestamp updates
CREATE TRIGGER trigger_daily_stats_updated_at
    BEFORE UPDATE ON daily_stats
    FOR EACH ROW
    EXECUTE FUNCTION update_updated_at_column();

CREATE TRIGGER trigger_strategy_performance_updated_at
    BEFORE UPDATE ON strategy_performance
    FOR EACH ROW
    EXECUTE FUNCTION update_updated_at_column();

-- Create materialized view for real-time risk dashboard
CREATE MATERIALIZED VIEW IF NOT EXISTS risk_dashboard AS
SELECT 
    account_id,
    metric_type,
    symbol,
    AVG(value) as avg_value,
    MAX(value) as max_value,
    MIN(value) as min_value,
    COUNT(*) as measurement_count,
    COUNT(*) FILTER (WHERE severity IN ('high', 'critical')) as high_risk_count,
    MAX(timestamp) as last_updated
FROM risk_metrics 
WHERE timestamp >= NOW() - INTERVAL '1 day'
GROUP BY account_id, metric_type, symbol;

-- Create unique index on risk dashboard materialized view
CREATE UNIQUE INDEX IF NOT EXISTS idx_risk_dashboard_unique 
ON risk_dashboard(account_id, metric_type, COALESCE(symbol, ''));

-- Create materialized view for performance summary
CREATE MATERIALIZED VIEW IF NOT EXISTS performance_summary AS
SELECT 
    component,
    metric_name,
    unit,
    AVG(metric_value) as avg_value,
    PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY metric_value) as p50_value,
    PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY metric_value) as p95_value,
    PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY metric_value) as p99_value,
    MAX(metric_value) as max_value,
    MIN(metric_value) as min_value,
    COUNT(*) as sample_count,
    MAX(timestamp) as last_updated
FROM performance_metrics 
WHERE timestamp >= NOW() - INTERVAL '1 hour'
GROUP BY component, metric_name, unit;

-- Create unique index on performance summary
CREATE UNIQUE INDEX IF NOT EXISTS idx_performance_summary_unique 
ON performance_summary(component, metric_name, unit);

-- Create functions for risk management

-- Function to calculate portfolio exposure
CREATE OR REPLACE FUNCTION calculate_portfolio_exposure(p_account_id VARCHAR(64) DEFAULT NULL)
RETURNS DECIMAL(20, 2) AS $$
DECLARE
    total_exposure DECIMAL(20, 2) := 0;
BEGIN
    SELECT COALESCE(SUM(ABS(quantity * last_price) / 100.0), 0)
    INTO total_exposure
    FROM positions 
    WHERE (p_account_id IS NULL OR account_id = p_account_id)
      AND quantity != 0;
    
    RETURN total_exposure;
END;
$$ LANGUAGE plpgsql;

-- Function to calculate daily P&L
CREATE OR REPLACE FUNCTION calculate_daily_pnl(p_date DATE, p_account_id VARCHAR(64) DEFAULT NULL)
RETURNS DECIMAL(20, 2) AS $$
DECLARE
    total_pnl DECIMAL(20, 2) := 0;
BEGIN
    SELECT COALESCE(SUM(
        (CASE WHEN f.side = 'buy' THEN -1 ELSE 1 END) * f.quantity * f.price / 100.0
    ), 0)
    INTO total_pnl
    FROM fills f
    WHERE DATE(f.execution_time) = p_date
      AND (p_account_id IS NULL OR EXISTS (
          SELECT 1 FROM orders o 
          WHERE o.id = f.order_id 
            AND o.account_id = p_account_id
      ));
    
    RETURN total_pnl;
END;
$$ LANGUAGE plpgsql;

-- Function to update daily statistics
CREATE OR REPLACE FUNCTION update_daily_stats(p_date DATE, p_account_id VARCHAR(64) DEFAULT NULL)
RETURNS VOID AS $$
DECLARE
    v_total_trades INTEGER;
    v_total_volume BIGINT;
    v_gross_pnl BIGINT;
    v_winning_trades INTEGER;
    v_losing_trades INTEGER;
    v_largest_win BIGINT;
    v_largest_loss BIGINT;
    v_avg_trade_size BIGINT;
    v_win_rate DECIMAL(5, 4);
    v_profit_factor DECIMAL(10, 4);
    v_gross_wins DECIMAL(20, 2);
    v_gross_losses DECIMAL(20, 2);
BEGIN
    -- Calculate basic stats
    SELECT 
        COUNT(*),
        SUM(quantity),
        SUM((CASE WHEN side = 'buy' THEN -1 ELSE 1 END) * quantity * price),
        AVG(quantity * price)
    INTO v_total_trades, v_total_volume, v_gross_pnl, v_avg_trade_size
    FROM fills f
    JOIN orders o ON f.order_id = o.id
    WHERE DATE(f.execution_time) = p_date
      AND (p_account_id IS NULL OR o.account_id = p_account_id);
    
    -- Calculate win/loss statistics
    SELECT 
        COUNT(*) FILTER (WHERE pnl > 0),
        COUNT(*) FILTER (WHERE pnl < 0),
        MAX(pnl),
        MIN(pnl),
        SUM(pnl) FILTER (WHERE pnl > 0),
        ABS(SUM(pnl) FILTER (WHERE pnl < 0))
    INTO v_winning_trades, v_losing_trades, v_largest_win, v_largest_loss, v_gross_wins, v_gross_losses
    FROM (
        SELECT (CASE WHEN f.side = 'buy' THEN -1 ELSE 1 END) * f.quantity * f.price as pnl
        FROM fills f
        JOIN orders o ON f.order_id = o.id
        WHERE DATE(f.execution_time) = p_date
          AND (p_account_id IS NULL OR o.account_id = p_account_id)
    ) trade_pnl;
    
    -- Calculate derived metrics
    v_win_rate := CASE 
        WHEN v_total_trades > 0 THEN v_winning_trades::DECIMAL / v_total_trades 
        ELSE 0 
    END;
    
    v_profit_factor := CASE 
        WHEN v_gross_losses > 0 THEN v_gross_wins / v_gross_losses 
        ELSE NULL 
    END;
    
    -- Insert or update daily stats
    INSERT INTO daily_stats (
        account_id, date, total_trades, total_volume, gross_pnl,
        winning_trades, losing_trades, largest_win, largest_loss,
        avg_trade_size, win_rate, profit_factor
    ) VALUES (
        p_account_id, p_date, COALESCE(v_total_trades, 0), COALESCE(v_total_volume, 0), COALESCE(v_gross_pnl, 0),
        COALESCE(v_winning_trades, 0), COALESCE(v_losing_trades, 0), COALESCE(v_largest_win, 0), COALESCE(v_largest_loss, 0),
        COALESCE(v_avg_trade_size, 0), v_win_rate, v_profit_factor
    )
    ON CONFLICT (account_id, date) 
    DO UPDATE SET
        total_trades = EXCLUDED.total_trades,
        total_volume = EXCLUDED.total_volume,
        gross_pnl = EXCLUDED.gross_pnl,
        winning_trades = EXCLUDED.winning_trades,
        losing_trades = EXCLUDED.losing_trades,
        largest_win = EXCLUDED.largest_win,
        largest_loss = EXCLUDED.largest_loss,
        avg_trade_size = EXCLUDED.avg_trade_size,
        win_rate = EXCLUDED.win_rate,
        profit_factor = EXCLUDED.profit_factor,
        updated_at = NOW();
END;
$$ LANGUAGE plpgsql;

-- Add comments for documentation
COMMENT ON TABLE risk_metrics IS 'Comprehensive risk metrics tracking with real-time monitoring';
COMMENT ON TABLE risk_violations IS 'Audit trail of risk limit breaches for compliance';
COMMENT ON TABLE daily_stats IS 'Daily trading statistics and performance metrics';
COMMENT ON TABLE performance_metrics IS 'System performance metrics for latency and throughput monitoring';
COMMENT ON TABLE audit_logs IS 'Complete audit trail for regulatory compliance';
COMMENT ON TABLE strategy_performance IS 'Individual strategy performance tracking';
COMMENT ON TABLE symbol_stats IS 'Per-symbol market characteristics and performance';

COMMENT ON FUNCTION calculate_portfolio_exposure IS 'Calculate total portfolio exposure in dollars';
COMMENT ON FUNCTION calculate_daily_pnl IS 'Calculate daily P&L for specified date and account';
COMMENT ON FUNCTION update_daily_stats IS 'Update daily statistics from fill data';