foxhunt/database/migrations/016_ml_training_data_tables.sql

-- ML Training Data Tables
-- Phase 2: Historical data storage for ML model training
--
-- This migration creates tables to store market data for ML training:
-- - order_book_snapshots: Level 2 order book data
-- - trade_executions: Trade history for feature extraction
-- - market_events: External events (news, earnings, etc.)
-- - ml_feature_cache: Cached computed features

-- Order book snapshots for microstructure analysis
CREATE TABLE IF NOT EXISTS order_book_snapshots (
    id BIGSERIAL PRIMARY KEY,
    timestamp TIMESTAMPTZ NOT NULL,
    symbol VARCHAR(50) NOT NULL,

    -- Best bid/ask (Level 1)
    best_bid DECIMAL(18,8) NOT NULL,
    best_ask DECIMAL(18,8) NOT NULL,
    bid_volume DECIMAL(18,8) NOT NULL,
    ask_volume DECIMAL(18,8) NOT NULL,

    -- Microstructure features
    spread_bps INTEGER NOT NULL,
    mid_price DECIMAL(18,8) NOT NULL,
    imbalance DOUBLE PRECISION NOT NULL, -- -1.0 to 1.0

    -- Level 2 data (top 5 levels)
    bid_levels JSONB, -- [{price, volume}, ...]
    ask_levels JSONB, -- [{price, volume}, ...]

    -- Metadata
    exchange VARCHAR(50),
    data_quality INTEGER DEFAULT 100, -- 0-100 quality score

    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Trade executions for volume analysis and price discovery
CREATE TABLE IF NOT EXISTS trade_executions (
    id BIGSERIAL PRIMARY KEY,
    timestamp TIMESTAMPTZ NOT NULL,
    symbol VARCHAR(50) NOT NULL,

    -- Trade details
    price DECIMAL(18,8) NOT NULL,
    quantity DECIMAL(18,8) NOT NULL,
    side VARCHAR(10) NOT NULL, -- 'buy' or 'sell'

    -- Trade identification
    trade_id VARCHAR(100),
    exchange VARCHAR(50),

    -- Derived features
    vwap DECIMAL(18,8),
    trade_intensity DOUBLE PRECISION, -- trades per second
    aggressive_flag BOOLEAN, -- true if crossing spread

    -- Metadata
    data_quality INTEGER DEFAULT 100,

    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Market events for regime detection and impact analysis
CREATE TABLE IF NOT EXISTS market_events (
    id BIGSERIAL PRIMARY KEY,
    timestamp TIMESTAMPTZ NOT NULL,

    -- Event classification
    event_type VARCHAR(50) NOT NULL, -- 'news', 'earnings', 'economic_data', 'halt', 'circuit_breaker'
    symbol VARCHAR(50), -- NULL for market-wide events

    -- Event details
    title TEXT,
    description TEXT,
    source VARCHAR(100),

    -- Impact assessment
    impact_score DOUBLE PRECISION, -- 0.0-1.0 magnitude
    sentiment DOUBLE PRECISION, -- -1.0 to 1.0

    -- Structured data
    metadata JSONB DEFAULT '{}',

    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Cached computed features to speed up training
CREATE TABLE IF NOT EXISTS ml_feature_cache (
    id BIGSERIAL PRIMARY KEY,
    timestamp TIMESTAMPTZ NOT NULL,
    symbol VARCHAR(50) NOT NULL,

    -- Feature version for cache invalidation
    feature_version VARCHAR(50) NOT NULL,

    -- Technical indicators (JSONB for flexibility)
    technical_indicators JSONB DEFAULT '{}', -- {rsi: 0.65, macd: 0.02, ...}

    -- Microstructure features
    microstructure_features JSONB DEFAULT '{}', -- {spread_bps: 10, imbalance: 0.2, ...}

    -- Risk metrics
    risk_metrics JSONB DEFAULT '{}', -- {var_5pct: -0.02, sharpe: 1.5, ...}

    -- Raw data reference
    order_book_snapshot_id BIGINT REFERENCES order_book_snapshots(id),

    created_at TIMESTAMPTZ DEFAULT NOW(),

    UNIQUE(timestamp, symbol, feature_version)
);

-- Performance indexes for time-range queries
CREATE INDEX IF NOT EXISTS idx_order_book_snapshots_timestamp_symbol
    ON order_book_snapshots(timestamp DESC, symbol);
CREATE INDEX IF NOT EXISTS idx_order_book_snapshots_symbol
    ON order_book_snapshots(symbol);
CREATE INDEX IF NOT EXISTS idx_order_book_snapshots_timestamp
    ON order_book_snapshots(timestamp DESC);

CREATE INDEX IF NOT EXISTS idx_trade_executions_timestamp_symbol
    ON trade_executions(timestamp DESC, symbol);
CREATE INDEX IF NOT EXISTS idx_trade_executions_symbol
    ON trade_executions(symbol);
CREATE INDEX IF NOT EXISTS idx_trade_executions_timestamp
    ON trade_executions(timestamp DESC);

CREATE INDEX IF NOT EXISTS idx_market_events_timestamp
    ON market_events(timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_market_events_symbol
    ON market_events(symbol);
CREATE INDEX IF NOT EXISTS idx_market_events_type
    ON market_events(event_type);

CREATE INDEX IF NOT EXISTS idx_ml_feature_cache_timestamp_symbol
    ON ml_feature_cache(timestamp DESC, symbol);
CREATE INDEX IF NOT EXISTS idx_ml_feature_cache_version
    ON ml_feature_cache(feature_version);

-- Add table statistics tracking
CREATE INDEX IF NOT EXISTS idx_order_book_snapshots_created_at
    ON order_book_snapshots(created_at);
CREATE INDEX IF NOT EXISTS idx_trade_executions_created_at
    ON trade_executions(created_at);

-- Comments for documentation
COMMENT ON TABLE order_book_snapshots IS 'Level 2 order book snapshots for microstructure analysis and ML training';
COMMENT ON TABLE trade_executions IS 'Historical trade executions for volume analysis and price discovery';
COMMENT ON TABLE market_events IS 'External market events for regime detection and sentiment analysis';
COMMENT ON TABLE ml_feature_cache IS 'Cached computed features to accelerate ML training data loading';

COMMENT ON COLUMN order_book_snapshots.spread_bps IS 'Bid-ask spread in basis points (bps)';
COMMENT ON COLUMN order_book_snapshots.imbalance IS 'Order book imbalance: (bid_vol - ask_vol) / (bid_vol + ask_vol)';
COMMENT ON COLUMN order_book_snapshots.data_quality IS 'Data quality score 0-100, used for filtering suspicious data';

COMMENT ON COLUMN trade_executions.aggressive_flag IS 'True if trade crossed the spread (market order), false if added liquidity';
COMMENT ON COLUMN trade_executions.trade_intensity IS 'Recent trade rate (trades per second) for momentum detection';

COMMENT ON COLUMN market_events.impact_score IS 'Estimated market impact magnitude 0.0-1.0';
COMMENT ON COLUMN market_events.sentiment IS 'Event sentiment: -1.0 (very negative) to +1.0 (very positive)';