foxhunt/migrations/006_up_create_performance_indexes.sql

-- Performance-Optimized Indexes for HFT Trading System
-- ====================================================
-- Creates essential indexes for existing tables only

-- === ORDERS TABLE INDEXES ===
-- Critical path: Order lookups by client_order_id (most frequent)
CREATE INDEX IF NOT EXISTS idx_orders_client_order_id
ON orders (client_order_id) WHERE client_order_id IS NOT NULL;

-- Critical path: Order status queries for active orders  
CREATE INDEX IF NOT EXISTS idx_orders_status_created
ON orders (status, created_at DESC) 
WHERE status IN ('pending', 'partial');

-- Critical path: Symbol-based order queries with time
CREATE INDEX IF NOT EXISTS idx_orders_symbol_created
ON orders (symbol, created_at DESC);

-- Critical path: Account order history
CREATE INDEX IF NOT EXISTS idx_orders_account_created
ON orders (account_id, created_at DESC) WHERE account_id IS NOT NULL;

-- === FILLS TABLE INDEXES ===
-- Critical path: Order fill lookups
CREATE INDEX IF NOT EXISTS idx_fills_order_execution
ON fills (order_id, execution_time DESC);

-- Critical path: Symbol fill analysis
CREATE INDEX IF NOT EXISTS idx_fills_symbol_execution
ON fills (symbol, execution_time DESC);

-- Critical path: Recent fills
CREATE INDEX IF NOT EXISTS idx_fills_execution_time
ON fills (execution_time DESC);

-- === POSITIONS TABLE INDEXES ===
-- Critical path: Position lookups by account and symbol
CREATE INDEX IF NOT EXISTS idx_positions_account_symbol
ON positions (account_id, symbol) WHERE account_id IS NOT NULL;

-- Critical path: Non-zero positions only
CREATE INDEX IF NOT EXISTS idx_positions_active
ON positions (account_id, symbol)
WHERE quantity != 0 AND account_id IS NOT NULL;

-- === MARKET_DATA TABLE INDEXES ===
-- Note: market_data is partitioned, so we skip CONCURRENTLY
-- Critical path: Recent market data by symbol
CREATE INDEX IF NOT EXISTS idx_market_data_symbol_timestamp
ON market_data (symbol, timestamp DESC);

-- Critical path: Market data time range queries
CREATE INDEX IF NOT EXISTS idx_market_data_timestamp
ON market_data (timestamp DESC);

-- === BARS TABLE INDEXES ===
-- Critical path: Bar data by symbol and timeframe
CREATE INDEX IF NOT EXISTS idx_bars_symbol_timeframe_timestamp
ON bars (symbol, timeframe, timestamp DESC);

-- Critical path: Recent bars
CREATE INDEX IF NOT EXISTS idx_bars_timestamp
ON bars (timestamp DESC);

-- === RISK_METRICS TABLE INDEXES ===
-- These are already created in migration 2, adding complementary ones

-- Critical path: Recent risk metrics by severity
CREATE INDEX IF NOT EXISTS idx_risk_metrics_severity_timestamp
ON risk_metrics (severity, timestamp DESC) 
WHERE severity IN ('high', 'critical');

-- Critical path: Account risk monitoring
CREATE INDEX IF NOT EXISTS idx_risk_metrics_account_metric_timestamp
ON risk_metrics (account_id, metric_type, timestamp DESC)
WHERE account_id IS NOT NULL;

-- === AUDIT_LOGS TABLE INDEXES ===
-- Critical path: Recent audit queries
CREATE INDEX IF NOT EXISTS idx_audit_logs_timestamp_desc
ON audit_logs (timestamp DESC);

-- Critical path: Event type queries
CREATE INDEX IF NOT EXISTS idx_audit_logs_event_timestamp
ON audit_logs (event_type, timestamp DESC);

-- Critical path: Entity audit trail
CREATE INDEX IF NOT EXISTS idx_audit_logs_entity_timestamp
ON audit_logs (entity_type, entity_id, timestamp DESC);

-- === PERFORMANCE_METRICS TABLE INDEXES ===
-- These are already created in migration 2, adding complementary ones

-- Critical path: Metric analysis by component
CREATE INDEX IF NOT EXISTS idx_performance_metrics_component_timestamp
ON performance_metrics (component, timestamp DESC);

-- === PARTIAL INDEXES FOR HOT DATA ===
-- Only index recent orders (recent data only)
CREATE INDEX IF NOT EXISTS idx_orders_recent
ON orders (symbol, created_at DESC, status);

-- Only index recent fills
CREATE INDEX IF NOT EXISTS idx_fills_recent
ON fills (symbol, execution_time DESC);

-- === EXPRESSION INDEXES ===
-- Index for order value calculations (quantity * price)
CREATE INDEX IF NOT EXISTS idx_orders_value
ON orders ((quantity * price))
WHERE status IN ('pending', 'partial') AND price IS NOT NULL;

-- Index for remaining quantity calculations
CREATE INDEX IF NOT EXISTS idx_orders_remaining_qty
ON orders ((quantity - filled_quantity))
WHERE status = 'partial';

-- === GIN INDEXES FOR JSONB DATA ===
-- Enable fast queries on JSONB metadata where it exists
CREATE INDEX IF NOT EXISTS idx_orders_metadata_gin
ON orders USING GIN (metadata) WHERE metadata IS NOT NULL;

CREATE INDEX IF NOT EXISTS idx_fills_metadata_gin
ON fills USING GIN (metadata) WHERE metadata IS NOT NULL;

-- === UPDATE STATISTICS ===
-- Update statistics for better query planning
ANALYZE orders;
ANALYZE fills;
ANALYZE positions;
ANALYZE market_data;
ANALYZE bars;
ANALYZE risk_metrics;
ANALYZE performance_metrics;
ANALYZE audit_logs;

-- === INDEX MONITORING FUNCTIONS ===
-- Create function to monitor index usage
CREATE OR REPLACE FUNCTION get_index_usage_stats()
RETURNS TABLE (
    schemaname TEXT,
    tablename TEXT,
    indexname TEXT,
    idx_tup_read BIGINT,
    idx_tup_fetch BIGINT,
    usage_ratio NUMERIC
) 
LANGUAGE SQL AS $$
    SELECT 
        s.schemaname,
        s.relname as tablename,
        s.indexrelname as indexname,
        s.idx_tup_read,
        s.idx_tup_fetch,
        CASE WHEN s.idx_tup_read = 0 
             THEN 0 
             ELSE ROUND(s.idx_tup_fetch::NUMERIC / s.idx_tup_read * 100, 2) 
        END as usage_ratio
    FROM pg_stat_user_indexes s
    WHERE s.schemaname = 'public'
    ORDER BY s.idx_tup_read DESC;
$$;

-- === INDEX SIZE MONITORING ===
-- Create function to monitor index sizes
CREATE OR REPLACE FUNCTION get_index_sizes()
RETURNS TABLE (
    tablename TEXT,
    indexname TEXT,
    index_size TEXT,
    index_size_bytes BIGINT
)
LANGUAGE SQL AS $$
    SELECT 
        t.relname as tablename,
        i.relname as indexname,
        pg_size_pretty(pg_relation_size(i.oid)) as index_size,
        pg_relation_size(i.oid) as index_size_bytes
    FROM pg_class i
    JOIN pg_index ix ON i.oid = ix.indexrelid
    JOIN pg_class t ON ix.indrelid = t.oid
    JOIN pg_namespace n ON t.relnamespace = n.oid
    WHERE i.relkind = 'i'
    AND n.nspname = 'public'
    ORDER BY pg_relation_size(i.oid) DESC;
$$;

-- Performance-optimized indexes created successfully!
-- Essential indexes for HFT workloads:
-- - Orders: client_order_id, status, symbol, account lookups
-- - Fills: order_id, symbol, execution_time lookups  
-- - Positions: account/symbol combinations, active positions
-- - Market Data: symbol/timestamp combinations
-- - Risk Metrics: severity and account-based monitoring
-- - Audit Logs: timestamp and entity-based queries
-- - Partial indexes for hot data (last 7 days)
-- - Expression indexes for calculated values
-- - GIN indexes for JSONB metadata
-- 
-- Monitoring functions:
-- - get_index_usage_stats(): Monitor index usage patterns
-- - get_index_sizes(): Monitor index storage requirements