diff --git a/migrations/001_trading_events.sql b/migrations/001_trading_events.sql index 241612030..deb294846 100644 --- a/migrations/001_trading_events.sql +++ b/migrations/001_trading_events.sql @@ -59,7 +59,7 @@ CREATE TYPE time_in_force AS ENUM ('day', 'gtc', 'ioc', 'fok', 'gtd'); -- ================================================================================================ CREATE TABLE trading_events ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), event_id BIGSERIAL NOT NULL, -- Sequential event ID for ordering correlation_id UUID NOT NULL, -- Links related events @@ -100,18 +100,36 @@ CREATE TABLE trading_events ( parent_hash VARCHAR(64), -- Hash of previous related event -- Partition key for performance - event_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(event_timestamp / 1000000000.0))) STORED, + event_date DATE NOT NULL, -- Constraints CONSTRAINT chk_timestamps CHECK ( received_timestamp >= event_timestamp AND processing_timestamp >= received_timestamp - ) + ), + + -- Composite primary key including partition column + PRIMARY KEY (id, event_date) ) PARTITION BY RANGE (event_date); -- Create table comment COMMENT ON TABLE trading_events IS 'Immutable event store for all trading activities with nanosecond precision and compliance features'; +-- Trigger function to set event_date from event_timestamp (nanoseconds) +CREATE OR REPLACE FUNCTION set_trading_event_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.event_date := DATE(TO_TIMESTAMP(NEW.event_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +-- Trigger to auto-populate event_date on insert +CREATE TRIGGER tg_set_trading_event_date + BEFORE INSERT ON trading_events + FOR EACH ROW + EXECUTE FUNCTION set_trading_event_date(); + -- ================================================================================================ -- ORDERS TABLE (CURRENT STATE) -- Mutable state table for current order status (derived from events) @@ -132,7 +150,7 @@ CREATE TABLE orders ( -- Quantities (in base units, scaled for precision) quantity BIGINT NOT NULL CHECK (quantity > 0), filled_quantity BIGINT NOT NULL DEFAULT 0 CHECK (filled_quantity >= 0), - remaining_quantity BIGINT GENERATED ALWAYS AS (quantity - filled_quantity) STORED, + remaining_quantity BIGINT NOT NULL DEFAULT 0, -- Pricing (in cents or smallest currency unit) limit_price BIGINT, -- NULL for market orders @@ -175,6 +193,21 @@ CREATE TABLE orders ( ) ); +-- Trigger function to calculate remaining_quantity +CREATE OR REPLACE FUNCTION set_order_remaining_quantity() +RETURNS TRIGGER AS $$ +BEGIN + NEW.remaining_quantity := NEW.quantity - NEW.filled_quantity; + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +-- Trigger to auto-populate remaining_quantity on insert/update +CREATE TRIGGER tg_set_order_remaining_quantity + BEFORE INSERT OR UPDATE OF quantity, filled_quantity ON orders + FOR EACH ROW + EXECUTE FUNCTION set_order_remaining_quantity(); + -- ================================================================================================ -- FILLS TABLE (TRADE EXECUTIONS) -- Immutable record of trade executions @@ -247,7 +280,7 @@ CREATE TABLE positions ( -- Market data last_price BIGINT NOT NULL DEFAULT 0, -- Last known market price - market_value BIGINT GENERATED ALWAYS AS (ABS(quantity) * last_price) STORED, + market_value BIGINT NOT NULL DEFAULT 0, -- Risk metrics var_1d BIGINT, -- 1-day Value at Risk @@ -261,7 +294,7 @@ CREATE TABLE positions ( -- Position limits max_position BIGINT, -- Maximum allowed position size - current_exposure BIGINT GENERATED ALWAYS AS (ABS(quantity * last_price)) STORED, + current_exposure BIGINT NOT NULL DEFAULT 0, -- Audit version INTEGER NOT NULL DEFAULT 1, -- Optimistic locking version @@ -275,6 +308,22 @@ CREATE TABLE positions ( ) ); +-- Trigger function to calculate market_value and current_exposure +CREATE OR REPLACE FUNCTION set_position_calculated_fields() +RETURNS TRIGGER AS $$ +BEGIN + NEW.market_value := ABS(NEW.quantity) * NEW.last_price; + NEW.current_exposure := ABS(NEW.quantity * NEW.last_price); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +-- Trigger to auto-populate calculated fields on insert/update +CREATE TRIGGER tg_set_position_calculated_fields + BEFORE INSERT OR UPDATE OF quantity, last_price ON positions + FOR EACH ROW + EXECUTE FUNCTION set_position_calculated_fields(); + -- ================================================================================================ -- HIGH-PERFORMANCE INDEXES -- Optimized for HFT query patterns and real-time operations diff --git a/migrations/001_up_create_core_tables.sql b/migrations/001_up_create_core_tables.sql deleted file mode 100644 index eebd07cae..000000000 --- a/migrations/001_up_create_core_tables.sql +++ /dev/null @@ -1,272 +0,0 @@ --- Migration 001: Create core trading tables for HFT system --- This migration establishes the foundational tables for orders, fills, positions, and market data - --- Enable required extensions -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -CREATE EXTENSION IF NOT EXISTS "pg_stat_statements"; - --- Orders table - core trading orders with optimized indexing -CREATE TABLE IF NOT EXISTS orders ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - symbol VARCHAR(32) NOT NULL, - side VARCHAR(10) NOT NULL CHECK (side IN ('buy', 'sell')), - order_type VARCHAR(20) NOT NULL CHECK (order_type IN ('market', 'limit', 'stop', 'stop_limit')), - quantity BIGINT NOT NULL CHECK (quantity > 0), - price BIGINT, -- Fixed-point price in cents, nullable for market orders - filled_quantity BIGINT NOT NULL DEFAULT 0 CHECK (filled_quantity >= 0), - status VARCHAR(20) NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'partial', 'filled', 'cancelled', 'expired')), - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - expires_at TIMESTAMP WITH TIME ZONE, - client_order_id VARCHAR(128), -- Client-provided identifier - account_id VARCHAR(64), -- Account identifier - metadata JSONB -- Additional order metadata -); - --- Fills table - trade executions with foreign key to orders -CREATE TABLE IF NOT EXISTS fills ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - order_id UUID NOT NULL REFERENCES orders(id) ON DELETE CASCADE, - symbol VARCHAR(32) NOT NULL, - side VARCHAR(10) NOT NULL CHECK (side IN ('buy', 'sell')), - quantity BIGINT NOT NULL CHECK (quantity > 0), - price BIGINT NOT NULL CHECK (price > 0), -- Execution price in fixed-point cents - fee BIGINT, -- Trading fee in fixed-point cents - fee_currency VARCHAR(10), -- Fee currency - execution_time TIMESTAMP WITH TIME ZONE NOT NULL, - venue VARCHAR(64), -- Execution venue - execution_id VARCHAR(128), -- Venue-specific execution ID - is_maker BOOLEAN, -- Maker/taker classification - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -- Additional fill metadata -); - --- Positions table - current holdings by symbol and account -CREATE TABLE IF NOT EXISTS positions ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - symbol VARCHAR(32) NOT NULL, - account_id VARCHAR(64), -- Account identifier - quantity BIGINT NOT NULL DEFAULT 0, -- Signed quantity (positive = long, negative = short) - avg_price BIGINT NOT NULL DEFAULT 0, -- Average entry price in fixed-point cents - market_value BIGINT NOT NULL DEFAULT 0, -- Current market value in fixed-point cents - unrealized_pnl BIGINT NOT NULL DEFAULT 0, -- Unrealized P&L in fixed-point cents - realized_pnl BIGINT NOT NULL DEFAULT 0, -- Realized P&L in fixed-point cents - total_cost BIGINT NOT NULL DEFAULT 0, -- Total cost basis in fixed-point cents - last_price BIGINT NOT NULL DEFAULT 0, -- Last known market price - trade_count INTEGER NOT NULL DEFAULT 0, -- Number of trades that created this position - first_trade_time TIMESTAMP WITH TIME ZONE, -- Time of first trade - last_updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB, -- Additional position metadata - - -- Ensure unique position per symbol-account combination - UNIQUE(symbol, account_id) -); - --- Market data table - high-frequency tick data with partitioning support -CREATE TABLE IF NOT EXISTS market_data ( - id UUID NOT NULL DEFAULT uuid_generate_v4(), - symbol VARCHAR(32) NOT NULL, - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, -- Market timestamp - received_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), -- When we received the data - bid BIGINT, -- Best bid price in fixed-point cents - ask BIGINT, -- Best ask price in fixed-point cents - last BIGINT, -- Last trade price in fixed-point cents - volume BIGINT, -- Volume - bid_size BIGINT, -- Best bid size - ask_size BIGINT, -- Best ask size - trade_count INTEGER, -- Number of trades - vwap BIGINT, -- Volume weighted average price - open BIGINT, -- Opening price - high BIGINT, -- High price - low BIGINT, -- Low price - close BIGINT, -- Closing price - data_type VARCHAR(20) NOT NULL DEFAULT 'tick' CHECK (data_type IN ('tick', 'quote', 'trade', 'bar')), - source VARCHAR(64) NOT NULL, -- Data provider - metadata JSONB, -- Additional market data - - PRIMARY KEY (id, timestamp) -- Composite primary key for partitioning -) PARTITION BY RANGE (timestamp); - --- Create initial partition for market data (current month) -DO $$ -DECLARE - partition_start DATE := date_trunc('month', CURRENT_DATE); - partition_end DATE := partition_start + INTERVAL '1 month'; - partition_name TEXT := 'market_data_' || to_char(partition_start, 'YYYY_MM'); -BEGIN - EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF market_data - FOR VALUES FROM (%L) TO (%L)', - partition_name, partition_start, partition_end); -END $$; - --- Bars table - aggregated OHLCV data -CREATE TABLE IF NOT EXISTS bars ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - symbol VARCHAR(32) NOT NULL, - timeframe VARCHAR(10) NOT NULL, -- "1m", "5m", "1h", "1d", etc. - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, -- Bar start time - open BIGINT NOT NULL, -- Opening price in fixed-point cents - high BIGINT NOT NULL, -- High price in fixed-point cents - low BIGINT NOT NULL, -- Low price in fixed-point cents - close BIGINT NOT NULL, -- Closing price in fixed-point cents - volume BIGINT NOT NULL DEFAULT 0, -- Volume - trade_count INTEGER NOT NULL DEFAULT 0, -- Number of trades - vwap BIGINT, -- Volume weighted average price - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - -- Ensure unique bar per symbol-timeframe-timestamp combination - UNIQUE(symbol, timeframe, timestamp) -); - --- Create high-performance indexes for HFT queries - --- Orders table indexes (optimized for order management) -CREATE INDEX IF NOT EXISTS idx_orders_symbol_status ON orders(symbol, status); -CREATE INDEX IF NOT EXISTS idx_orders_created_at ON orders(created_at); -CREATE INDEX IF NOT EXISTS idx_orders_account_id ON orders(account_id) WHERE account_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_orders_client_order_id ON orders(client_order_id) WHERE client_order_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_orders_symbol_status_created ON orders(symbol, status, created_at); - --- Fills table indexes (optimized for execution tracking) -CREATE INDEX IF NOT EXISTS idx_fills_order_id ON fills(order_id); -CREATE INDEX IF NOT EXISTS idx_fills_symbol_execution_time ON fills(symbol, execution_time); -CREATE INDEX IF NOT EXISTS idx_fills_execution_time ON fills(execution_time); -CREATE INDEX IF NOT EXISTS idx_fills_venue ON fills(venue) WHERE venue IS NOT NULL; - --- Positions table indexes (optimized for position tracking) -CREATE INDEX IF NOT EXISTS idx_positions_symbol ON positions(symbol); -CREATE INDEX IF NOT EXISTS idx_positions_account_id ON positions(account_id) WHERE account_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_positions_last_updated ON positions(last_updated); - --- Market data table indexes (optimized for time-series queries) -CREATE INDEX IF NOT EXISTS idx_market_data_symbol_timestamp ON market_data(symbol, timestamp); -CREATE INDEX IF NOT EXISTS idx_market_data_timestamp ON market_data(timestamp); -CREATE INDEX IF NOT EXISTS idx_market_data_source ON market_data(source); -CREATE INDEX IF NOT EXISTS idx_market_data_received_at ON market_data(received_at); - --- Bars table indexes (optimized for chart data queries) -CREATE INDEX IF NOT EXISTS idx_bars_symbol_timeframe_timestamp ON bars(symbol, timeframe, timestamp); -CREATE INDEX IF NOT EXISTS idx_bars_timestamp ON bars(timestamp); - --- Create functions for automatic timestamp updates -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create triggers for automatic timestamp updates -CREATE TRIGGER trigger_orders_updated_at - BEFORE UPDATE ON orders - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_positions_updated_at - BEFORE UPDATE ON positions - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - --- Create function to automatically create market data partitions -CREATE OR REPLACE FUNCTION create_market_data_partition_if_not_exists(target_date DATE) -RETURNS VOID AS $$ -DECLARE - partition_start DATE := date_trunc('month', target_date); - partition_end DATE := partition_start + INTERVAL '1 month'; - partition_name TEXT := 'market_data_' || to_char(partition_start, 'YYYY_MM'); -BEGIN - -- Check if partition exists - IF NOT EXISTS ( - SELECT 1 FROM information_schema.tables - WHERE table_name = partition_name - ) THEN - EXECUTE format('CREATE TABLE %I PARTITION OF market_data - FOR VALUES FROM (%L) TO (%L)', - partition_name, partition_start, partition_end); - - -- Add indexes to the new partition - EXECUTE format('CREATE INDEX %I ON %I(symbol, timestamp)', - 'idx_' || partition_name || '_symbol_timestamp', partition_name); - EXECUTE format('CREATE INDEX %I ON %I(timestamp)', - 'idx_' || partition_name || '_timestamp', partition_name); - END IF; -END; -$$ LANGUAGE plpgsql; - --- Create function to validate order constraints -CREATE OR REPLACE FUNCTION validate_order_constraints() -RETURNS TRIGGER AS $$ -BEGIN - -- Validate that limit orders have a price - IF NEW.order_type = 'limit' AND NEW.price IS NULL THEN - RAISE EXCEPTION 'Limit orders must have a price'; - END IF; - - -- Validate that filled quantity doesn't exceed order quantity - IF NEW.filled_quantity > NEW.quantity THEN - RAISE EXCEPTION 'Filled quantity cannot exceed order quantity'; - END IF; - - -- Update status based on filled quantity - IF NEW.filled_quantity = 0 THEN - NEW.status = 'pending'; - ELSIF NEW.filled_quantity = NEW.quantity THEN - NEW.status = 'filled'; - ELSIF NEW.filled_quantity < NEW.quantity THEN - NEW.status = 'partial'; - END IF; - - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create trigger for order validation -CREATE TRIGGER trigger_validate_orders - BEFORE INSERT OR UPDATE ON orders - FOR EACH ROW - EXECUTE FUNCTION validate_order_constraints(); - --- Create materialized view for fast position summaries -CREATE MATERIALIZED VIEW IF NOT EXISTS position_summaries AS -SELECT - symbol, - account_id, - SUM(quantity) as total_quantity, - COUNT(*) as position_count, - SUM(unrealized_pnl) as total_unrealized_pnl, - SUM(realized_pnl) as total_realized_pnl, - AVG(avg_price) as weighted_avg_price, - MAX(last_updated) as last_updated -FROM positions -WHERE quantity != 0 -GROUP BY symbol, account_id; - --- Create unique index on the materialized view -CREATE UNIQUE INDEX IF NOT EXISTS idx_position_summaries_symbol_account -ON position_summaries(symbol, account_id); - --- Create function to refresh position summaries -CREATE OR REPLACE FUNCTION refresh_position_summaries() -RETURNS VOID AS $$ -BEGIN - REFRESH MATERIALIZED VIEW CONCURRENTLY position_summaries; -END; -$$ LANGUAGE plpgsql; - --- Add comments for documentation -COMMENT ON TABLE orders IS 'Core trading orders with ACID compliance'; -COMMENT ON TABLE fills IS 'Trade executions linked to orders'; -COMMENT ON TABLE positions IS 'Current holdings by symbol and account'; -COMMENT ON TABLE market_data IS 'High-frequency tick data with automatic partitioning'; -COMMENT ON TABLE bars IS 'Aggregated OHLCV bars for charting'; - -COMMENT ON COLUMN orders.price IS 'Price in fixed-point cents (divide by 100 for dollars)'; -COMMENT ON COLUMN orders.quantity IS 'Order quantity in shares/units'; -COMMENT ON COLUMN fills.price IS 'Execution price in fixed-point cents'; -COMMENT ON COLUMN positions.quantity IS 'Signed quantity: positive=long, negative=short'; - --- Grant appropriate permissions (adjust as needed for your setup) --- GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO foxhunt_app; --- GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO foxhunt_app; \ No newline at end of file diff --git a/migrations/002_risk_events.sql b/migrations/002_risk_events.sql index df173f8e0..cf996d0fc 100644 --- a/migrations/002_risk_events.sql +++ b/migrations/002_risk_events.sql @@ -78,7 +78,7 @@ CREATE TYPE risk_metric_type AS ENUM ( -- ================================================================================================ CREATE TABLE risk_events ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), event_id BIGSERIAL NOT NULL, correlation_id UUID NOT NULL, @@ -132,9 +132,10 @@ CREATE TABLE risk_events ( metadata JSONB, -- Partition key - event_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(event_timestamp / 1000000000.0))) STORED, + event_date DATE NOT NULL, -- Constraints + PRIMARY KEY (id, event_date), CONSTRAINT chk_risk_timestamps CHECK ( detected_timestamp >= event_timestamp AND (acknowledged_timestamp IS NULL OR acknowledged_timestamp >= detected_timestamp) AND @@ -151,7 +152,7 @@ CREATE TABLE risk_events ( -- ================================================================================================ CREATE TABLE risk_metrics ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), metric_name risk_metric_type NOT NULL, -- Scope identifiers @@ -193,9 +194,10 @@ CREATE TABLE risk_metrics ( calculation_details JSONB, -- Model parameters and inputs -- Partition key - metric_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(calculation_timestamp / 1000000000.0))) STORED, + metric_date DATE NOT NULL, -- Constraints + PRIMARY KEY (id, metric_date), CONSTRAINT chk_confidence_level CHECK (confidence_level > 0 AND confidence_level <= 1), CONSTRAINT chk_time_horizons CHECK ( time_horizon_days IS NULL OR time_horizon_days > 0 @@ -257,8 +259,7 @@ CREATE TABLE risk_limits ( limit_details JSONB, -- Additional limit parameters override_permissions TEXT[], -- Who can override this limit - -- Constraints - CONSTRAINT uk_risk_limits_unique UNIQUE (limit_type, scope_level, COALESCE(account_id, ''), COALESCE(strategy_id, ''), COALESCE(symbol, '')), + -- Constraints (unique constraint moved to expression index below) CONSTRAINT chk_threshold_order CHECK ( warning_threshold IS NULL OR breach_threshold IS NULL OR warning_threshold <= breach_threshold ), @@ -270,6 +271,15 @@ CREATE TABLE risk_limits ( ) ); +-- Expression index for risk_limits uniqueness +CREATE UNIQUE INDEX uk_risk_limits_unique ON risk_limits ( + limit_type, + scope_level, + COALESCE(account_id, ''), + COALESCE(strategy_id, ''), + COALESCE(symbol, '') +); + -- ================================================================================================ -- STRESS TEST SCENARIOS TABLE -- Predefined stress test scenarios and results @@ -308,7 +318,7 @@ CREATE TABLE stress_test_scenarios ( -- ================================================================================================ CREATE TABLE stress_test_results ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), scenario_id UUID NOT NULL REFERENCES stress_test_scenarios(id), execution_id UUID NOT NULL, -- Groups results from same execution @@ -344,7 +354,10 @@ CREATE TABLE stress_test_results ( calculation_time_ms INTEGER, -- How long the calculation took -- Partition key - execution_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(execution_timestamp / 1000000000.0))) STORED + execution_date DATE NOT NULL, + + -- Constraints + PRIMARY KEY (id, execution_date) ) PARTITION BY RANGE (execution_date); -- ================================================================================================ @@ -449,6 +462,52 @@ CREATE INDEX idx_stress_test_results_execution ON stress_test_results USING BTRE CREATE INDEX idx_stress_test_results_scenario ON stress_test_results USING BTREE (scenario_id, execution_timestamp); CREATE INDEX idx_stress_test_results_account ON stress_test_results USING BTREE (account_id, execution_timestamp) WHERE account_id IS NOT NULL; +-- ================================================================================================ +-- TRIGGER FUNCTIONS FOR GENERATED COLUMNS (converted from GENERATED ALWAYS) +-- ================================================================================================ + +-- Set event_date for risk_events +CREATE OR REPLACE FUNCTION set_risk_event_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.event_date := DATE(TO_TIMESTAMP(NEW.event_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE TRIGGER tg_set_risk_event_date + BEFORE INSERT OR UPDATE ON risk_events + FOR EACH ROW + EXECUTE FUNCTION set_risk_event_date(); + +-- Set metric_date for risk_metrics +CREATE OR REPLACE FUNCTION set_risk_metric_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.metric_date := DATE(TO_TIMESTAMP(NEW.calculation_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE TRIGGER tg_set_risk_metric_date + BEFORE INSERT OR UPDATE ON risk_metrics + FOR EACH ROW + EXECUTE FUNCTION set_risk_metric_date(); + +-- Set execution_date for stress_test_results +CREATE OR REPLACE FUNCTION set_stress_test_execution_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.execution_date := DATE(TO_TIMESTAMP(NEW.execution_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE TRIGGER tg_set_stress_test_execution_date + BEFORE INSERT OR UPDATE ON stress_test_results + FOR EACH ROW + EXECUTE FUNCTION set_stress_test_execution_date(); + -- ================================================================================================ -- AUTOMATIC PARTITIONING -- ================================================================================================ @@ -523,12 +582,30 @@ BEGIN -- Create risk_metrics partitions for current and next 2 months FOR i IN 0..2 LOOP - PERFORM create_risk_metrics_partition(CURRENT_DATE + (i || ' months')::INTERVAL); + PERFORM create_risk_metrics_partition((CURRENT_DATE + (i || ' months')::INTERVAL)::DATE); END LOOP; - -- Create stress_test_results partitions + -- Create stress_test_results partitions (daily, same as risk_events) FOR i IN 0..7 LOOP - PERFORM create_trading_events_partition(CURRENT_DATE + i); -- Reuse function with same logic + -- Create stress_test_results partitions using similar logic + DECLARE + partition_name TEXT; + start_date DATE; + end_date DATE; + BEGIN + start_date := CURRENT_DATE + i; + end_date := start_date + INTERVAL '1 day'; + partition_name := 'stress_test_results_' || to_char(start_date, 'YYYY_MM_DD'); + + IF NOT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_name = partition_name + ) THEN + EXECUTE format('CREATE TABLE %I PARTITION OF stress_test_results + FOR VALUES FROM (%L) TO (%L)', + partition_name, start_date, end_date); + END IF; + END; END LOOP; END $$; @@ -604,31 +681,31 @@ BEGIN IF NEW.value > COALESCE(applicable_limit.emergency_threshold, applicable_limit.breach_threshold) THEN breach_detected := TRUE; severity_level := 'emergency'; - event_type_val := CASE NEW.metric_name - WHEN 'var_1d', 'var_10d' THEN 'var_breach' - WHEN 'exposure_gross', 'exposure_net' THEN 'exposure_limit_breach' - WHEN 'leverage_ratio' THEN 'leverage_excess' - WHEN 'concentration_single', 'concentration_sector' THEN 'concentration_risk' + event_type_val := CASE + WHEN NEW.metric_name IN ('var_1d', 'var_10d') THEN 'var_breach' + WHEN NEW.metric_name IN ('exposure_gross', 'exposure_net') THEN 'exposure_limit_breach' + WHEN NEW.metric_name = 'leverage_ratio' THEN 'leverage_excess' + WHEN NEW.metric_name IN ('concentration_single', 'concentration_sector') THEN 'concentration_risk' ELSE 'stress_test_failure' END; ELSIF NEW.value > applicable_limit.breach_threshold THEN breach_detected := TRUE; severity_level := 'critical'; - event_type_val := CASE NEW.metric_name - WHEN 'var_1d', 'var_10d' THEN 'var_breach' - WHEN 'exposure_gross', 'exposure_net' THEN 'exposure_limit_breach' - WHEN 'leverage_ratio' THEN 'leverage_excess' - WHEN 'concentration_single', 'concentration_sector' THEN 'concentration_risk' + event_type_val := CASE + WHEN NEW.metric_name IN ('var_1d', 'var_10d') THEN 'var_breach' + WHEN NEW.metric_name IN ('exposure_gross', 'exposure_net') THEN 'exposure_limit_breach' + WHEN NEW.metric_name = 'leverage_ratio' THEN 'leverage_excess' + WHEN NEW.metric_name IN ('concentration_single', 'concentration_sector') THEN 'concentration_risk' ELSE 'stress_test_failure' END; ELSIF NEW.value > COALESCE(applicable_limit.warning_threshold, applicable_limit.breach_threshold * 0.8) THEN breach_detected := TRUE; severity_level := 'medium'; - event_type_val := CASE NEW.metric_name - WHEN 'var_1d', 'var_10d' THEN 'var_breach' - WHEN 'exposure_gross', 'exposure_net' THEN 'exposure_limit_breach' - WHEN 'leverage_ratio' THEN 'leverage_excess' - WHEN 'concentration_single', 'concentration_sector' THEN 'concentration_risk' + event_type_val := CASE + WHEN NEW.metric_name IN ('var_1d', 'var_10d') THEN 'var_breach' + WHEN NEW.metric_name IN ('exposure_gross', 'exposure_net') THEN 'exposure_limit_breach' + WHEN NEW.metric_name = 'leverage_ratio' THEN 'leverage_excess' + WHEN NEW.metric_name IN ('concentration_single', 'concentration_sector') THEN 'concentration_risk' ELSE 'stress_test_failure' END; END IF; @@ -763,7 +840,7 @@ $$ LANGUAGE plpgsql; -- Function to get active risk alerts CREATE OR REPLACE FUNCTION get_active_risk_alerts( - p_severity risk_severity[] DEFAULT ARRAY['high', 'critical', 'emergency'] + p_severity risk_severity[] DEFAULT ARRAY['high'::risk_severity, 'critical'::risk_severity, 'emergency'::risk_severity] ) RETURNS TABLE ( event_id UUID, event_type risk_event_type, diff --git a/migrations/002_up_create_risk_performance_tables.sql b/migrations/002_up_create_risk_performance_tables.sql deleted file mode 100644 index abc951c7c..000000000 --- a/migrations/002_up_create_risk_performance_tables.sql +++ /dev/null @@ -1,369 +0,0 @@ --- Migration 002: Create risk management and performance tracking tables --- This migration adds comprehensive risk monitoring and performance metrics capabilities - --- Risk metrics table - comprehensive risk tracking -CREATE TABLE IF NOT EXISTS risk_metrics ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - account_id VARCHAR(64), -- Account identifier - metric_type VARCHAR(50) NOT NULL CHECK (metric_type IN ( - 'exposure', 'var', 'drawdown', 'violation', 'concentration', - 'leverage', 'margin', 'volatility', 'beta', 'correlation' - )), - symbol VARCHAR(32), -- NULL for portfolio-level metrics - value DECIMAL(20, 8) NOT NULL, -- Metric value with high precision - threshold DECIMAL(20, 8), -- Risk threshold - severity VARCHAR(20) NOT NULL DEFAULT 'low' CHECK (severity IN ('low', 'medium', 'high', 'critical')), - description TEXT NOT NULL, -- Human-readable description - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -- Additional risk data -); - --- Risk violations table - audit trail of risk breaches -CREATE TABLE IF NOT EXISTS risk_violations ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - account_id VARCHAR(64), - violation_type VARCHAR(50) NOT NULL, - symbol VARCHAR(32), - threshold_value DECIMAL(20, 8) NOT NULL, - actual_value DECIMAL(20, 8) NOT NULL, - severity VARCHAR(20) NOT NULL CHECK (severity IN ('warning', 'error', 'critical')), - description TEXT NOT NULL, - action_taken VARCHAR(100), -- Action taken in response - resolved_at TIMESTAMP WITH TIME ZONE, -- When violation was resolved - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -); - --- Daily statistics table - comprehensive daily trading metrics -CREATE TABLE IF NOT EXISTS daily_stats ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - account_id VARCHAR(64), - date DATE NOT NULL, -- Trading date - total_trades INTEGER NOT NULL DEFAULT 0, - total_volume BIGINT NOT NULL DEFAULT 0, - gross_pnl BIGINT NOT NULL DEFAULT 0, -- Gross P&L in fixed-point cents - net_pnl BIGINT NOT NULL DEFAULT 0, -- Net P&L after fees in fixed-point cents - fees_paid BIGINT NOT NULL DEFAULT 0, -- Total fees paid in fixed-point cents - winning_trades INTEGER NOT NULL DEFAULT 0, - losing_trades INTEGER NOT NULL DEFAULT 0, - largest_win BIGINT NOT NULL DEFAULT 0, -- Largest winning trade - largest_loss BIGINT NOT NULL DEFAULT 0, -- Largest losing trade (negative) - max_drawdown DECIMAL(10, 4) NOT NULL DEFAULT 0, -- Maximum drawdown percentage - max_position_size BIGINT NOT NULL DEFAULT 0, -- Maximum position size held - avg_trade_size BIGINT NOT NULL DEFAULT 0, -- Average trade size - sharpe_ratio DECIMAL(10, 4), -- Sharpe ratio - win_rate DECIMAL(5, 4), -- Win rate percentage (0-1) - profit_factor DECIMAL(10, 4), -- Profit factor - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - -- Ensure unique stats per account-date combination - UNIQUE(account_id, date) -); - --- Performance metrics table - system and trading performance tracking -CREATE TABLE IF NOT EXISTS performance_metrics ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - metric_name VARCHAR(100) NOT NULL, -- e.g., "latency_p50", "latency_p95", "throughput" - metric_value DECIMAL(20, 8) NOT NULL, -- Metric value - unit VARCHAR(50) NOT NULL, -- "nanoseconds", "ops_per_second", "percent", etc. - component VARCHAR(100) NOT NULL, -- "order_processing", "market_data", "risk_engine" - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - tags JSONB, -- Additional tags for grouping/filtering - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() - - -- Composite indexes will be created after table creation -); - --- Audit logs table - comprehensive audit trail for compliance -CREATE TABLE IF NOT EXISTS audit_logs ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - event_type VARCHAR(50) NOT NULL, -- "order_placed", "trade_executed", "position_updated" - entity_type VARCHAR(50) NOT NULL, -- "order", "fill", "position", "risk_metric" - entity_id UUID NOT NULL, -- ID of the affected entity - account_id VARCHAR(64), - user_id VARCHAR(64), - action VARCHAR(50) NOT NULL, -- "create", "update", "delete", "execute" - old_values JSONB, -- Previous state (for updates) - new_values JSONB, -- New state - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - source VARCHAR(100) NOT NULL, -- System component that generated the event - correlation_id UUID, -- For tracking related events - session_id VARCHAR(128), -- User session identifier - ip_address INET, -- Client IP address - user_agent TEXT, -- Client user agent - metadata JSONB, -- Additional audit metadata - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); - --- Strategy performance table - track individual strategy performance -CREATE TABLE IF NOT EXISTS strategy_performance ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - strategy_name VARCHAR(100) NOT NULL, - account_id VARCHAR(64), - date DATE NOT NULL, - trades_count INTEGER NOT NULL DEFAULT 0, - total_pnl BIGINT NOT NULL DEFAULT 0, -- P&L in fixed-point cents - win_rate DECIMAL(5, 4), -- Win rate (0-1) - sharpe_ratio DECIMAL(10, 4), - max_drawdown DECIMAL(10, 4), - avg_trade_duration INTERVAL, -- Average time positions are held - total_volume BIGINT NOT NULL DEFAULT 0, - risk_adjusted_return DECIMAL(10, 4), - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - UNIQUE(strategy_name, account_id, date) -); - --- Symbol statistics table - per-symbol performance and characteristics -CREATE TABLE IF NOT EXISTS symbol_stats ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - symbol VARCHAR(32) NOT NULL, - date DATE NOT NULL, - open_price BIGINT, -- Opening price in fixed-point cents - high_price BIGINT, -- High price - low_price BIGINT, -- Low price - close_price BIGINT, -- Closing price - volume BIGINT NOT NULL DEFAULT 0, - trade_count INTEGER NOT NULL DEFAULT 0, - vwap BIGINT, -- Volume-weighted average price - volatility DECIMAL(10, 6), -- Daily volatility - beta DECIMAL(10, 4), -- Beta relative to market - correlation_spy DECIMAL(10, 4), -- Correlation to SPY - avg_spread BIGINT, -- Average bid-ask spread - liquidity_score DECIMAL(5, 2), -- Liquidity scoring - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - UNIQUE(symbol, date) -); - --- Create composite indexes for risk_metrics table -CREATE INDEX IF NOT EXISTS idx_risk_metrics_composite ON risk_metrics(metric_type, severity, timestamp); -CREATE INDEX IF NOT EXISTS idx_risk_metrics_symbol_type ON risk_metrics(symbol, metric_type); -CREATE INDEX IF NOT EXISTS idx_risk_metrics_account_timestamp ON risk_metrics(account_id, timestamp); - --- Create composite indexes for performance_metrics table -CREATE INDEX IF NOT EXISTS idx_performance_metrics_component_name_timestamp ON performance_metrics(component, metric_name, timestamp); -CREATE INDEX IF NOT EXISTS idx_performance_metrics_timestamp ON performance_metrics(timestamp); - --- Create optimized indexes for performance queries - --- Risk metrics indexes -CREATE INDEX IF NOT EXISTS idx_risk_metrics_timestamp ON risk_metrics(timestamp); -CREATE INDEX IF NOT EXISTS idx_risk_metrics_account_id ON risk_metrics(account_id) WHERE account_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_risk_violations_timestamp ON risk_violations(timestamp); -CREATE INDEX IF NOT EXISTS idx_risk_violations_severity ON risk_violations(severity); - --- Daily stats indexes -CREATE INDEX IF NOT EXISTS idx_daily_stats_date ON daily_stats(date); -CREATE INDEX IF NOT EXISTS idx_daily_stats_account_date ON daily_stats(account_id, date); - --- Performance metrics indexes (optimized for time-series analysis) -CREATE INDEX IF NOT EXISTS idx_performance_metrics_name_timestamp ON performance_metrics(metric_name, timestamp); - --- Audit logs indexes (optimized for compliance queries) -CREATE INDEX IF NOT EXISTS idx_audit_logs_timestamp ON audit_logs(timestamp); -CREATE INDEX IF NOT EXISTS idx_audit_logs_entity ON audit_logs(entity_type, entity_id); -CREATE INDEX IF NOT EXISTS idx_audit_logs_account_id ON audit_logs(account_id) WHERE account_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_audit_logs_user_id ON audit_logs(user_id) WHERE user_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_audit_logs_correlation_id ON audit_logs(correlation_id) WHERE correlation_id IS NOT NULL; - --- Strategy performance indexes -CREATE INDEX IF NOT EXISTS idx_strategy_performance_name_date ON strategy_performance(strategy_name, date); -CREATE INDEX IF NOT EXISTS idx_strategy_performance_account_date ON strategy_performance(account_id, date); - --- Symbol stats indexes -CREATE INDEX IF NOT EXISTS idx_symbol_stats_symbol_date ON symbol_stats(symbol, date); -CREATE INDEX IF NOT EXISTS idx_symbol_stats_date ON symbol_stats(date); - --- Create triggers for automatic timestamp updates -CREATE TRIGGER trigger_daily_stats_updated_at - BEFORE UPDATE ON daily_stats - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_strategy_performance_updated_at - BEFORE UPDATE ON strategy_performance - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - --- Create materialized view for real-time risk dashboard -CREATE MATERIALIZED VIEW IF NOT EXISTS risk_dashboard AS -SELECT - account_id, - metric_type, - symbol, - AVG(value) as avg_value, - MAX(value) as max_value, - MIN(value) as min_value, - COUNT(*) as measurement_count, - COUNT(*) FILTER (WHERE severity IN ('high', 'critical')) as high_risk_count, - MAX(timestamp) as last_updated -FROM risk_metrics -WHERE timestamp >= NOW() - INTERVAL '1 day' -GROUP BY account_id, metric_type, symbol; - --- Create unique index on risk dashboard materialized view -CREATE UNIQUE INDEX IF NOT EXISTS idx_risk_dashboard_unique -ON risk_dashboard(account_id, metric_type, COALESCE(symbol, '')); - --- Create materialized view for performance summary -CREATE MATERIALIZED VIEW IF NOT EXISTS performance_summary AS -SELECT - component, - metric_name, - unit, - AVG(metric_value) as avg_value, - PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY metric_value) as p50_value, - PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY metric_value) as p95_value, - PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY metric_value) as p99_value, - MAX(metric_value) as max_value, - MIN(metric_value) as min_value, - COUNT(*) as sample_count, - MAX(timestamp) as last_updated -FROM performance_metrics -WHERE timestamp >= NOW() - INTERVAL '1 hour' -GROUP BY component, metric_name, unit; - --- Create unique index on performance summary -CREATE UNIQUE INDEX IF NOT EXISTS idx_performance_summary_unique -ON performance_summary(component, metric_name, unit); - --- Create functions for risk management - --- Function to calculate portfolio exposure -CREATE OR REPLACE FUNCTION calculate_portfolio_exposure(p_account_id VARCHAR(64) DEFAULT NULL) -RETURNS DECIMAL(20, 2) AS $$ -DECLARE - total_exposure DECIMAL(20, 2) := 0; -BEGIN - SELECT COALESCE(SUM(ABS(quantity * last_price) / 100.0), 0) - INTO total_exposure - FROM positions - WHERE (p_account_id IS NULL OR account_id = p_account_id) - AND quantity != 0; - - RETURN total_exposure; -END; -$$ LANGUAGE plpgsql; - --- Function to calculate daily P&L -CREATE OR REPLACE FUNCTION calculate_daily_pnl(p_date DATE, p_account_id VARCHAR(64) DEFAULT NULL) -RETURNS DECIMAL(20, 2) AS $$ -DECLARE - total_pnl DECIMAL(20, 2) := 0; -BEGIN - SELECT COALESCE(SUM( - (CASE WHEN f.side = 'buy' THEN -1 ELSE 1 END) * f.quantity * f.price / 100.0 - ), 0) - INTO total_pnl - FROM fills f - WHERE DATE(f.execution_time) = p_date - AND (p_account_id IS NULL OR EXISTS ( - SELECT 1 FROM orders o - WHERE o.id = f.order_id - AND o.account_id = p_account_id - )); - - RETURN total_pnl; -END; -$$ LANGUAGE plpgsql; - --- Function to update daily statistics -CREATE OR REPLACE FUNCTION update_daily_stats(p_date DATE, p_account_id VARCHAR(64) DEFAULT NULL) -RETURNS VOID AS $$ -DECLARE - v_total_trades INTEGER; - v_total_volume BIGINT; - v_gross_pnl BIGINT; - v_winning_trades INTEGER; - v_losing_trades INTEGER; - v_largest_win BIGINT; - v_largest_loss BIGINT; - v_avg_trade_size BIGINT; - v_win_rate DECIMAL(5, 4); - v_profit_factor DECIMAL(10, 4); - v_gross_wins DECIMAL(20, 2); - v_gross_losses DECIMAL(20, 2); -BEGIN - -- Calculate basic stats - SELECT - COUNT(*), - SUM(quantity), - SUM((CASE WHEN side = 'buy' THEN -1 ELSE 1 END) * quantity * price), - AVG(quantity * price) - INTO v_total_trades, v_total_volume, v_gross_pnl, v_avg_trade_size - FROM fills f - JOIN orders o ON f.order_id = o.id - WHERE DATE(f.execution_time) = p_date - AND (p_account_id IS NULL OR o.account_id = p_account_id); - - -- Calculate win/loss statistics - SELECT - COUNT(*) FILTER (WHERE pnl > 0), - COUNT(*) FILTER (WHERE pnl < 0), - MAX(pnl), - MIN(pnl), - SUM(pnl) FILTER (WHERE pnl > 0), - ABS(SUM(pnl) FILTER (WHERE pnl < 0)) - INTO v_winning_trades, v_losing_trades, v_largest_win, v_largest_loss, v_gross_wins, v_gross_losses - FROM ( - SELECT (CASE WHEN f.side = 'buy' THEN -1 ELSE 1 END) * f.quantity * f.price as pnl - FROM fills f - JOIN orders o ON f.order_id = o.id - WHERE DATE(f.execution_time) = p_date - AND (p_account_id IS NULL OR o.account_id = p_account_id) - ) trade_pnl; - - -- Calculate derived metrics - v_win_rate := CASE - WHEN v_total_trades > 0 THEN v_winning_trades::DECIMAL / v_total_trades - ELSE 0 - END; - - v_profit_factor := CASE - WHEN v_gross_losses > 0 THEN v_gross_wins / v_gross_losses - ELSE NULL - END; - - -- Insert or update daily stats - INSERT INTO daily_stats ( - account_id, date, total_trades, total_volume, gross_pnl, - winning_trades, losing_trades, largest_win, largest_loss, - avg_trade_size, win_rate, profit_factor - ) VALUES ( - p_account_id, p_date, COALESCE(v_total_trades, 0), COALESCE(v_total_volume, 0), COALESCE(v_gross_pnl, 0), - COALESCE(v_winning_trades, 0), COALESCE(v_losing_trades, 0), COALESCE(v_largest_win, 0), COALESCE(v_largest_loss, 0), - COALESCE(v_avg_trade_size, 0), v_win_rate, v_profit_factor - ) - ON CONFLICT (account_id, date) - DO UPDATE SET - total_trades = EXCLUDED.total_trades, - total_volume = EXCLUDED.total_volume, - gross_pnl = EXCLUDED.gross_pnl, - winning_trades = EXCLUDED.winning_trades, - losing_trades = EXCLUDED.losing_trades, - largest_win = EXCLUDED.largest_win, - largest_loss = EXCLUDED.largest_loss, - avg_trade_size = EXCLUDED.avg_trade_size, - win_rate = EXCLUDED.win_rate, - profit_factor = EXCLUDED.profit_factor, - updated_at = NOW(); -END; -$$ LANGUAGE plpgsql; - --- Add comments for documentation -COMMENT ON TABLE risk_metrics IS 'Comprehensive risk metrics tracking with real-time monitoring'; -COMMENT ON TABLE risk_violations IS 'Audit trail of risk limit breaches for compliance'; -COMMENT ON TABLE daily_stats IS 'Daily trading statistics and performance metrics'; -COMMENT ON TABLE performance_metrics IS 'System performance metrics for latency and throughput monitoring'; -COMMENT ON TABLE audit_logs IS 'Complete audit trail for regulatory compliance'; -COMMENT ON TABLE strategy_performance IS 'Individual strategy performance tracking'; -COMMENT ON TABLE symbol_stats IS 'Per-symbol market characteristics and performance'; - -COMMENT ON FUNCTION calculate_portfolio_exposure IS 'Calculate total portfolio exposure in dollars'; -COMMENT ON FUNCTION calculate_daily_pnl IS 'Calculate daily P&L for specified date and account'; -COMMENT ON FUNCTION update_daily_stats IS 'Update daily statistics from fill data'; \ No newline at end of file diff --git a/migrations/003_audit_system.sql b/migrations/003_audit_system.sql index 5a7b4e957..4f0a15a8c 100644 --- a/migrations/003_audit_system.sql +++ b/migrations/003_audit_system.sql @@ -105,7 +105,7 @@ CREATE TYPE system_component AS ENUM ( -- ================================================================================================ CREATE TABLE audit_log ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), audit_id BIGSERIAL NOT NULL, -- Sequential audit ID for ordering correlation_id UUID, -- Links related audit entries @@ -183,9 +183,10 @@ CREATE TABLE audit_log ( stack_trace TEXT, -- Partition key for performance - audit_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(event_timestamp / 1000000000.0))) STORED, + audit_date DATE NOT NULL, -- Constraints + PRIMARY KEY (id, audit_date), CONSTRAINT chk_audit_timestamps CHECK ( recorded_timestamp >= event_timestamp AND processing_timestamp >= recorded_timestamp @@ -200,7 +201,7 @@ CREATE TABLE audit_log ( -- ================================================================================================ CREATE TABLE ml_events ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), model_id VARCHAR(200) NOT NULL, -- Unique model identifier model_version VARCHAR(50) NOT NULL, @@ -270,7 +271,10 @@ CREATE TABLE ml_events ( custom_metrics JSONB, -- Domain-specific metrics -- Partition key - event_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(event_timestamp / 1000000000.0))) STORED + event_date DATE NOT NULL, + + -- Constraints + PRIMARY KEY (id, event_date) ) PARTITION BY RANGE (event_date); -- ================================================================================================ @@ -279,7 +283,7 @@ CREATE TABLE ml_events ( -- ================================================================================================ CREATE TABLE system_events ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), event_timestamp ns_timestamp NOT NULL, -- Event classification @@ -336,7 +340,10 @@ CREATE TABLE system_events ( namespace VARCHAR(100), -- Kubernetes namespace -- Partition key - event_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(event_timestamp / 1000000000.0))) STORED + event_date DATE NOT NULL, + + -- Constraints + PRIMARY KEY (id, event_date) ) PARTITION BY RANGE (event_date); -- ================================================================================================ @@ -345,7 +352,7 @@ CREATE TABLE system_events ( -- ================================================================================================ CREATE TABLE change_tracking ( -- Primary identifiers - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + id UUID DEFAULT uuid_generate_v4(), change_timestamp ns_timestamp NOT NULL, -- Change context @@ -377,7 +384,10 @@ CREATE TABLE change_tracking ( checksum VARCHAR(64) NOT NULL, -- Integrity check -- Partition key - change_date DATE GENERATED ALWAYS AS (DATE(TO_TIMESTAMP(change_timestamp / 1000000000.0))) STORED + change_date DATE NOT NULL, + + -- Constraints + PRIMARY KEY (id, change_date) ) PARTITION BY RANGE (change_date); -- ================================================================================================ @@ -387,7 +397,7 @@ CREATE TABLE change_tracking ( CREATE TABLE compliance_annotations ( -- Primary identifiers id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - audit_log_id UUID NOT NULL REFERENCES audit_log(id), + audit_log_id UUID NOT NULL, -- FK removed: audit_log has composite PK (id, audit_date) -- Compliance framework regulation_name VARCHAR(100) NOT NULL, -- MiFID II, GDPR, SOX, etc. @@ -779,6 +789,66 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- ================================================================================================ +-- TRIGGER FUNCTIONS FOR GENERATED COLUMNS (converted from GENERATED ALWAYS) +-- ================================================================================================ + +-- Set audit_date for audit_log +CREATE OR REPLACE FUNCTION set_audit_log_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.audit_date := DATE(TO_TIMESTAMP(NEW.event_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE TRIGGER tg_set_audit_log_date + BEFORE INSERT OR UPDATE ON audit_log + FOR EACH ROW + EXECUTE FUNCTION set_audit_log_date(); + +-- Set event_date for ml_events +CREATE OR REPLACE FUNCTION set_ml_event_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.event_date := DATE(TO_TIMESTAMP(NEW.event_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE TRIGGER tg_set_ml_event_date + BEFORE INSERT OR UPDATE ON ml_events + FOR EACH ROW + EXECUTE FUNCTION set_ml_event_date(); + +-- Set event_date for system_events +CREATE OR REPLACE FUNCTION set_system_event_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.event_date := DATE(TO_TIMESTAMP(NEW.event_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE TRIGGER tg_set_system_event_date + BEFORE INSERT OR UPDATE ON system_events + FOR EACH ROW + EXECUTE FUNCTION set_system_event_date(); + +-- Set change_date for change_tracking +CREATE OR REPLACE FUNCTION set_change_tracking_date() +RETURNS TRIGGER AS $$ +BEGIN + NEW.change_date := DATE(TO_TIMESTAMP(NEW.change_timestamp / 1000000000.0)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE TRIGGER tg_set_change_tracking_date + BEFORE INSERT OR UPDATE ON change_tracking + FOR EACH ROW + EXECUTE FUNCTION set_change_tracking_date(); + -- ================================================================================================ -- CREATE TRIGGERS FOR CHANGE TRACKING -- ================================================================================================ diff --git a/migrations/003_up_create_wal_checkpoints.sql b/migrations/003_up_create_wal_checkpoints.sql deleted file mode 100644 index de5b076c5..000000000 --- a/migrations/003_up_create_wal_checkpoints.sql +++ /dev/null @@ -1,198 +0,0 @@ --- WAL checkpoint table for write-ahead logging and recovery -CREATE TABLE IF NOT EXISTS wal_checkpoints ( - id UUID PRIMARY KEY, - sequence_number BIGINT NOT NULL UNIQUE, - checkpoint_timestamp TIMESTAMPTZ NOT NULL, - database_state_hash TEXT NOT NULL, - entries_count BIGINT NOT NULL DEFAULT 0, - file_path TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- Index for efficient checkpoint queries -CREATE INDEX IF NOT EXISTS idx_wal_checkpoints_sequence ON wal_checkpoints(sequence_number DESC); -CREATE INDEX IF NOT EXISTS idx_wal_checkpoints_timestamp ON wal_checkpoints(checkpoint_timestamp DESC); - --- Add sequence number to audit_logs for WAL ordering -ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS sequence_number BIGINT; - --- Create sequence for audit log entries if not exists -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_sequences WHERE sequencename = 'audit_logs_sequence_seq') THEN - CREATE SEQUENCE audit_logs_sequence_seq START 1; - END IF; -END $$; - --- Set default for sequence number -ALTER TABLE audit_logs ALTER COLUMN sequence_number SET DEFAULT nextval('audit_logs_sequence_seq'); - --- Update existing audit_logs entries to have sequence numbers -DO $$ -BEGIN - IF EXISTS (SELECT 1 FROM audit_logs WHERE sequence_number IS NULL LIMIT 1) THEN - WITH ordered_logs AS ( - SELECT id, ROW_NUMBER() OVER (ORDER BY created_at ASC) as seq_num - FROM audit_logs - WHERE sequence_number IS NULL - ) - UPDATE audit_logs - SET sequence_number = ordered_logs.seq_num - FROM ordered_logs - WHERE audit_logs.id = ordered_logs.id; - - -- Update sequence to continue from max value - PERFORM setval('audit_logs_sequence_seq', (SELECT COALESCE(MAX(sequence_number), 0) FROM audit_logs)); - END IF; -END $$; - --- Make sequence_number NOT NULL after populating existing data -ALTER TABLE audit_logs ALTER COLUMN sequence_number SET NOT NULL; - --- Create unique index on sequence number -CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_logs_sequence_unique ON audit_logs(sequence_number); - --- Additional indexes for WAL operations -CREATE INDEX IF NOT EXISTS idx_audit_logs_session_sequence ON audit_logs(session_id, sequence_number); - --- Function to get next WAL sequence number (atomic) -CREATE OR REPLACE FUNCTION get_next_wal_sequence() -RETURNS BIGINT -LANGUAGE plpgsql -AS $$ -DECLARE - next_seq BIGINT; -BEGIN - SELECT nextval('audit_logs_sequence_seq') INTO next_seq; - RETURN next_seq; -END; -$$; - --- Function to create WAL checkpoint -CREATE OR REPLACE FUNCTION create_wal_checkpoint( - p_checkpoint_id UUID, - p_database_state_hash TEXT, - p_file_path TEXT -) -RETURNS TABLE( - checkpoint_id UUID, - sequence_number BIGINT, - checkpoint_timestamp TIMESTAMPTZ, - entries_count BIGINT -) -LANGUAGE plpgsql -AS $$ -DECLARE - current_sequence BIGINT; - current_count BIGINT; - checkpoint_time TIMESTAMPTZ := NOW(); -BEGIN - -- Get current WAL position - SELECT COALESCE(MAX(audit_logs.sequence_number), 0) INTO current_sequence FROM audit_logs; - SELECT COUNT(*) INTO current_count FROM audit_logs; - - -- Insert checkpoint - INSERT INTO wal_checkpoints ( - id, sequence_number, checkpoint_timestamp, database_state_hash, - entries_count, file_path, created_at - ) VALUES ( - p_checkpoint_id, current_sequence, checkpoint_time, - p_database_state_hash, current_count, p_file_path, checkpoint_time - ); - - -- Return checkpoint info - RETURN QUERY SELECT - p_checkpoint_id, - current_sequence, - checkpoint_time, - current_count; -END; -$$; - --- Function to verify WAL integrity -CREATE OR REPLACE FUNCTION verify_wal_integrity( - p_start_sequence BIGINT, - p_end_sequence BIGINT -) -RETURNS TABLE( - sequence_number BIGINT, - is_valid BOOLEAN, - expected_checksum TEXT, - actual_checksum TEXT -) -LANGUAGE plpgsql -AS $$ -BEGIN - RETURN QUERY - SELECT - al.sequence_number, - TRUE as is_valid, -- Simplified integrity check - '' as expected_checksum, - '' as actual_checksum - FROM audit_logs al - WHERE al.sequence_number >= p_start_sequence - AND al.sequence_number <= p_end_sequence - ORDER BY al.sequence_number; -END; -$$; - --- Function to cleanup old WAL entries before checkpoint -CREATE OR REPLACE FUNCTION cleanup_wal_before_checkpoint(p_checkpoint_id UUID) -RETURNS BIGINT -LANGUAGE plpgsql -AS $$ -DECLARE - checkpoint_sequence BIGINT; - deleted_count BIGINT; -BEGIN - -- Get checkpoint sequence number - SELECT wc.sequence_number INTO checkpoint_sequence - FROM wal_checkpoints wc - WHERE wc.id = p_checkpoint_id; - - IF checkpoint_sequence IS NULL THEN - RAISE EXCEPTION 'Checkpoint not found: %', p_checkpoint_id; - END IF; - - -- Delete audit logs before checkpoint, keeping some safety margin - DELETE FROM audit_logs - WHERE sequence_number < (checkpoint_sequence - 1000); -- Keep 1000 entries as safety margin - - GET DIAGNOSTICS deleted_count = ROW_COUNT; - - RETURN deleted_count; -END; -$$; - --- WAL statistics view for monitoring -CREATE OR REPLACE VIEW wal_statistics AS -SELECT - COUNT(*) as total_entries, - MIN(timestamp) as oldest_entry_timestamp, - MAX(timestamp) as newest_entry_timestamp, - MIN(sequence_number) as min_sequence, - MAX(sequence_number) as max_sequence, - pg_size_pretty(pg_total_relation_size('audit_logs')) as table_size, - (SELECT COUNT(*) FROM wal_checkpoints) as checkpoint_count, - (SELECT MAX(sequence_number) FROM wal_checkpoints) as last_checkpoint_sequence, - CASE - WHEN MAX(timestamp) > MIN(timestamp) - THEN COUNT(*)::FLOAT / EXTRACT(epoch FROM (MAX(timestamp) - MIN(timestamp))) - ELSE 0 - END as avg_entries_per_second -FROM audit_logs; - --- Grant permissions for trading engine user -DO $$ -BEGIN - IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'foxhunt_trader') THEN - GRANT SELECT, INSERT ON wal_checkpoints TO foxhunt_trader; - GRANT SELECT, INSERT, UPDATE ON audit_logs TO foxhunt_trader; - GRANT USAGE ON SEQUENCE audit_logs_sequence_seq TO foxhunt_trader; - GRANT SELECT ON wal_statistics TO foxhunt_trader; - GRANT EXECUTE ON FUNCTION get_next_wal_sequence() TO foxhunt_trader; - GRANT EXECUTE ON FUNCTION create_wal_checkpoint(UUID, TEXT, TEXT) TO foxhunt_trader; - GRANT EXECUTE ON FUNCTION verify_wal_integrity(BIGINT, BIGINT) TO foxhunt_trader; - GRANT EXECUTE ON FUNCTION cleanup_wal_before_checkpoint(UUID) TO foxhunt_trader; - END IF; -END $$; \ No newline at end of file diff --git a/migrations/004_compliance_views.sql b/migrations/004_compliance_views.sql index a88dd35c2..322c3b50e 100644 --- a/migrations/004_compliance_views.sql +++ b/migrations/004_compliance_views.sql @@ -376,8 +376,9 @@ SELECT string_agg(DISTINCT al.action, ', ') as processing_purposes, -- Data minimization assessment - COUNT(DISTINCT jsonb_object_keys(al.event_data)) as data_fields_processed, - AVG(jsonb_array_length(COALESCE(al.affected_fields, '[]'::text[]))) as avg_fields_per_operation + -- Note: Using jsonb field count estimation (not exact unique keys across group) + AVG((SELECT COUNT(*) FROM jsonb_each(al.event_data))::numeric) as avg_data_fields_processed, + AVG(array_length(COALESCE(al.affected_fields, ARRAY[]::text[]), 1)) as avg_fields_per_operation FROM audit_log al WHERE al.event_timestamp >= EXTRACT(EPOCH FROM (CURRENT_DATE - INTERVAL '12 months')) * 1000000000 @@ -556,7 +557,7 @@ SELECT -- Risk and compliance alerts (SELECT COUNT(*) FROM risk_events WHERE event_timestamp >= EXTRACT(EPOCH FROM CURRENT_DATE) * 1000000000 AND severity IN ('critical', 'emergency')) as critical_risk_events, (SELECT COUNT(*) FROM audit_log WHERE event_timestamp >= EXTRACT(EPOCH FROM CURRENT_DATE) * 1000000000 AND severity = 'error') as system_errors, - (SELECT COUNT(*) FROM audit_log WHERE event_timestamp >= EXTRACT(EPOCH FROM CURRENT_DATE) * 1000000000 AND event_type = 'compliance_violation') as compliance_violations, + (SELECT COUNT(*) FROM audit_log WHERE event_timestamp >= EXTRACT(EPOCH FROM CURRENT_DATE) * 1000000000 AND event_type = 'compliance_check_failed') as compliance_violations, -- Regulatory reporting status (SELECT COUNT(*) FROM report_generation_log WHERE generation_started_at >= CURRENT_DATE AND generation_status = 'completed') as reports_completed_today, diff --git a/migrations/004_up_create_user_management.sql b/migrations/004_up_create_user_management.sql deleted file mode 100644 index 7260229f9..000000000 --- a/migrations/004_up_create_user_management.sql +++ /dev/null @@ -1,509 +0,0 @@ --- Migration 004: User Management and Authentication --- This migration creates comprehensive user management for HFT trading systems - --- Enable required extensions -CREATE EXTENSION IF NOT EXISTS "pgcrypto"; - --- Users table - comprehensive user management -CREATE TABLE IF NOT EXISTS users ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - username VARCHAR(64) NOT NULL UNIQUE, - email VARCHAR(255) NOT NULL UNIQUE, - password_hash TEXT NOT NULL, - salt TEXT NOT NULL, - first_name VARCHAR(100), - last_name VARCHAR(100), - phone VARCHAR(20), - time_zone VARCHAR(50) DEFAULT 'UTC', - language VARCHAR(10) DEFAULT 'en', - status VARCHAR(20) NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'inactive', 'suspended', 'locked')), - role VARCHAR(50) NOT NULL DEFAULT 'trader' CHECK (role IN ('admin', 'trader', 'risk_manager', 'analyst', 'readonly')), - last_login TIMESTAMP WITH TIME ZONE, - failed_login_attempts INTEGER NOT NULL DEFAULT 0, - locked_until TIMESTAMP WITH TIME ZONE, - password_changed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - password_expires_at TIMESTAMP WITH TIME ZONE, - two_factor_enabled BOOLEAN NOT NULL DEFAULT false, - two_factor_secret TEXT, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - created_by UUID REFERENCES users(id), - updated_by UUID REFERENCES users(id), - metadata JSONB -); - --- Sessions table - track user sessions for security -CREATE TABLE IF NOT EXISTS user_sessions ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - session_token TEXT NOT NULL UNIQUE, - refresh_token TEXT, - ip_address INET NOT NULL, - user_agent TEXT, - location JSONB, - is_active BOOLEAN NOT NULL DEFAULT true, - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - last_used_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -); - --- API Keys table - for programmatic access -CREATE TABLE IF NOT EXISTS api_keys ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - key_name VARCHAR(100) NOT NULL, - key_hash TEXT NOT NULL UNIQUE, - key_prefix VARCHAR(20) NOT NULL, - permissions JSONB NOT NULL DEFAULT '[]'::jsonb, - rate_limit INTEGER DEFAULT 1000, -- requests per minute - is_active BOOLEAN NOT NULL DEFAULT true, - last_used_at TIMESTAMP WITH TIME ZONE, - usage_count BIGINT NOT NULL DEFAULT 0, - expires_at TIMESTAMP WITH TIME ZONE, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -); - --- Accounts table - trading accounts linked to users -CREATE TABLE IF NOT EXISTS accounts ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - account_number VARCHAR(64) NOT NULL UNIQUE, - account_name VARCHAR(100) NOT NULL, - user_id UUID NOT NULL REFERENCES users(id), - account_type VARCHAR(20) NOT NULL DEFAULT 'individual' CHECK (account_type IN ('individual', 'corporate', 'institutional', 'demo')), - base_currency VARCHAR(10) NOT NULL DEFAULT 'USD', - initial_balance DECIMAL(20, 8) NOT NULL DEFAULT 0, - current_balance DECIMAL(20, 8) NOT NULL DEFAULT 0, - available_balance DECIMAL(20, 8) NOT NULL DEFAULT 0, - margin_balance DECIMAL(20, 8) NOT NULL DEFAULT 0, - equity DECIMAL(20, 8) NOT NULL DEFAULT 0, - free_margin DECIMAL(20, 8) NOT NULL DEFAULT 0, - margin_level DECIMAL(10, 4) NOT NULL DEFAULT 0, -- Margin level percentage - leverage DECIMAL(10, 2) NOT NULL DEFAULT 1.00, - max_leverage DECIMAL(10, 2) NOT NULL DEFAULT 100.00, - status VARCHAR(20) NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'inactive', 'suspended', 'closed')), - risk_profile VARCHAR(20) NOT NULL DEFAULT 'medium' CHECK (risk_profile IN ('conservative', 'medium', 'aggressive', 'high_frequency')), - broker VARCHAR(100), - broker_account_id VARCHAR(100), - is_demo BOOLEAN NOT NULL DEFAULT false, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -); - --- Account permissions - granular access control -CREATE TABLE IF NOT EXISTS account_permissions ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - account_id UUID NOT NULL REFERENCES accounts(id) ON DELETE CASCADE, - permission_type VARCHAR(50) NOT NULL, -- 'read', 'trade', 'admin', 'risk_override' - granted_by UUID NOT NULL REFERENCES users(id), - granted_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - expires_at TIMESTAMP WITH TIME ZONE, - is_active BOOLEAN NOT NULL DEFAULT true, - metadata JSONB, - - UNIQUE(user_id, account_id, permission_type) -); - --- Brokers table - external broker connections -CREATE TABLE IF NOT EXISTS brokers ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - name VARCHAR(100) NOT NULL UNIQUE, - broker_type VARCHAR(50) NOT NULL, -- 'mt4', 'mt5', 'ctrader', 'fix', 'rest' - api_endpoint TEXT, - fix_settings JSONB, - connection_settings JSONB NOT NULL DEFAULT '{}'::jsonb, - credentials_encrypted TEXT, - is_active BOOLEAN NOT NULL DEFAULT true, - is_demo BOOLEAN NOT NULL DEFAULT false, - supported_symbols TEXT[], -- Array of supported symbols - commission_settings JSONB, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -); - --- Broker connections - track live connections -CREATE TABLE IF NOT EXISTS broker_connections ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - broker_id UUID NOT NULL REFERENCES brokers(id) ON DELETE CASCADE, - account_id UUID NOT NULL REFERENCES accounts(id) ON DELETE CASCADE, - connection_id VARCHAR(100) NOT NULL, - status VARCHAR(20) NOT NULL DEFAULT 'disconnected' CHECK (status IN ('connected', 'connecting', 'disconnected', 'error')), - last_heartbeat TIMESTAMP WITH TIME ZONE, - latency_ms INTEGER, - error_message TEXT, - connection_attempts INTEGER NOT NULL DEFAULT 0, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB, - - UNIQUE(broker_id, account_id) -); - --- Compliance profiles - regulatory requirements -CREATE TABLE IF NOT EXISTS compliance_profiles ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - profile_name VARCHAR(100) NOT NULL UNIQUE, - jurisdiction VARCHAR(10) NOT NULL, -- 'US', 'EU', 'UK', 'APAC' - regulations JSONB NOT NULL DEFAULT '{}'::jsonb, - requirements JSONB NOT NULL DEFAULT '{}'::jsonb, - reporting_requirements JSONB NOT NULL DEFAULT '{}'::jsonb, - retention_periods JSONB NOT NULL DEFAULT '{}'::jsonb, - is_active BOOLEAN NOT NULL DEFAULT true, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -); - --- User compliance assignments -CREATE TABLE IF NOT EXISTS user_compliance ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - compliance_profile_id UUID NOT NULL REFERENCES compliance_profiles(id), - assigned_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - assigned_by UUID NOT NULL REFERENCES users(id), - is_active BOOLEAN NOT NULL DEFAULT true, - metadata JSONB, - - UNIQUE(user_id, compliance_profile_id) -); - --- Create optimized indexes for HFT performance - --- Users table indexes -CREATE INDEX IF NOT EXISTS idx_users_username ON users(username); -CREATE INDEX IF NOT EXISTS idx_users_email ON users(email); -CREATE INDEX IF NOT EXISTS idx_users_status ON users(status); -CREATE INDEX IF NOT EXISTS idx_users_role ON users(role); -CREATE INDEX IF NOT EXISTS idx_users_last_login ON users(last_login); - --- Sessions table indexes -CREATE INDEX IF NOT EXISTS idx_user_sessions_user_id ON user_sessions(user_id); -CREATE INDEX IF NOT EXISTS idx_user_sessions_token ON user_sessions(session_token); -CREATE INDEX IF NOT EXISTS idx_user_sessions_active ON user_sessions(is_active, expires_at); -CREATE INDEX IF NOT EXISTS idx_user_sessions_ip_address ON user_sessions(ip_address); - --- API Keys table indexes -CREATE INDEX IF NOT EXISTS idx_api_keys_user_id ON api_keys(user_id); -CREATE INDEX IF NOT EXISTS idx_api_keys_prefix ON api_keys(key_prefix); -CREATE INDEX IF NOT EXISTS idx_api_keys_active ON api_keys(is_active, expires_at); - --- Accounts table indexes -CREATE INDEX IF NOT EXISTS idx_accounts_user_id ON accounts(user_id); -CREATE INDEX IF NOT EXISTS idx_accounts_number ON accounts(account_number); -CREATE INDEX IF NOT EXISTS idx_accounts_status ON accounts(status); -CREATE INDEX IF NOT EXISTS idx_accounts_type ON accounts(account_type); -CREATE INDEX IF NOT EXISTS idx_accounts_broker ON accounts(broker); - --- Account permissions indexes -CREATE INDEX IF NOT EXISTS idx_account_permissions_user_account ON account_permissions(user_id, account_id); -CREATE INDEX IF NOT EXISTS idx_account_permissions_type ON account_permissions(permission_type); -CREATE INDEX IF NOT EXISTS idx_account_permissions_active ON account_permissions(is_active, expires_at); - --- Brokers table indexes -CREATE INDEX IF NOT EXISTS idx_brokers_name ON brokers(name); -CREATE INDEX IF NOT EXISTS idx_brokers_type ON brokers(broker_type); -CREATE INDEX IF NOT EXISTS idx_brokers_active ON brokers(is_active); - --- Broker connections indexes -CREATE INDEX IF NOT EXISTS idx_broker_connections_broker_account ON broker_connections(broker_id, account_id); -CREATE INDEX IF NOT EXISTS idx_broker_connections_status ON broker_connections(status); -CREATE INDEX IF NOT EXISTS idx_broker_connections_heartbeat ON broker_connections(last_heartbeat); - --- Create triggers for automatic updates -CREATE TRIGGER trigger_users_updated_at - BEFORE UPDATE ON users - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_accounts_updated_at - BEFORE UPDATE ON accounts - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_brokers_updated_at - BEFORE UPDATE ON brokers - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_broker_connections_updated_at - BEFORE UPDATE ON broker_connections - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - --- Create functions for user management - --- Function to create new user with encrypted password -CREATE OR REPLACE FUNCTION create_user( - p_username VARCHAR(64), - p_email VARCHAR(255), - p_password TEXT, - p_first_name VARCHAR(100) DEFAULT NULL, - p_last_name VARCHAR(100) DEFAULT NULL, - p_role VARCHAR(50) DEFAULT 'trader', - p_created_by UUID DEFAULT NULL -) -RETURNS UUID AS $$ -DECLARE - v_user_id UUID; - v_salt TEXT; - v_password_hash TEXT; -BEGIN - -- Generate salt and hash password - v_salt := encode(gen_random_bytes(32), 'hex'); - v_password_hash := crypt(p_password || v_salt, gen_salt('bf', 12)); - - -- Insert user - INSERT INTO users ( - username, email, password_hash, salt, first_name, last_name, - role, created_by, password_changed_at - ) VALUES ( - p_username, p_email, v_password_hash, v_salt, p_first_name, p_last_name, - p_role, p_created_by, NOW() - ) RETURNING id INTO v_user_id; - - -- Create audit log entry - INSERT INTO audit_logs ( - event_type, entity_type, entity_id, user_id, action, - new_values, timestamp, source - ) VALUES ( - 'user_created', 'user', v_user_id, p_created_by, 'create', - jsonb_build_object('username', p_username, 'email', p_email, 'role', p_role), - NOW(), 'user_management' - ); - - RETURN v_user_id; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Function to authenticate user -CREATE OR REPLACE FUNCTION authenticate_user( - p_username VARCHAR(64), - p_password TEXT, - p_ip_address INET DEFAULT NULL, - p_user_agent TEXT DEFAULT NULL -) -RETURNS TABLE( - user_id UUID, - session_token TEXT, - expires_at TIMESTAMP WITH TIME ZONE, - role VARCHAR(50), - status VARCHAR(20) -) AS $$ -DECLARE - v_user_record RECORD; - v_session_token TEXT; - v_expires_at TIMESTAMP WITH TIME ZONE; -BEGIN - -- Get user record - SELECT u.id, u.username, u.password_hash, u.salt, u.status, u.role, u.failed_login_attempts, u.locked_until - INTO v_user_record - FROM users u - WHERE u.username = p_username OR u.email = p_username; - - -- Check if user exists - IF v_user_record.id IS NULL THEN - RAISE EXCEPTION 'Invalid credentials'; - END IF; - - -- Check if account is locked - IF v_user_record.locked_until IS NOT NULL AND v_user_record.locked_until > NOW() THEN - RAISE EXCEPTION 'Account is locked until %', v_user_record.locked_until; - END IF; - - -- Check if account is active - IF v_user_record.status != 'active' THEN - RAISE EXCEPTION 'Account is not active'; - END IF; - - -- Verify password - IF NOT (v_user_record.password_hash = crypt(p_password || v_user_record.salt, v_user_record.password_hash)) THEN - -- Increment failed login attempts - UPDATE users - SET failed_login_attempts = failed_login_attempts + 1, - locked_until = CASE - WHEN failed_login_attempts >= 4 THEN NOW() + INTERVAL '30 minutes' - ELSE NULL - END - WHERE id = v_user_record.id; - - RAISE EXCEPTION 'Invalid credentials'; - END IF; - - -- Reset failed login attempts and update last login - UPDATE users - SET failed_login_attempts = 0, - locked_until = NULL, - last_login = NOW() - WHERE id = v_user_record.id; - - -- Generate session token - v_session_token := encode(gen_random_bytes(64), 'hex'); - v_expires_at := NOW() + INTERVAL '8 hours'; - - -- Create session - INSERT INTO user_sessions ( - user_id, session_token, ip_address, user_agent, expires_at, last_used_at - ) VALUES ( - v_user_record.id, v_session_token, p_ip_address, p_user_agent, v_expires_at, NOW() - ); - - -- Log successful login - INSERT INTO audit_logs ( - event_type, entity_type, entity_id, user_id, action, - new_values, timestamp, source, ip_address, user_agent - ) VALUES ( - 'user_login', 'user', v_user_record.id, v_user_record.id, 'login', - jsonb_build_object('ip_address', p_ip_address::TEXT), - NOW(), 'authentication', p_ip_address, p_user_agent - ); - - -- Return session info - RETURN QUERY SELECT - v_user_record.id, - v_session_token, - v_expires_at, - v_user_record.role, - v_user_record.status; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Function to validate session -CREATE OR REPLACE FUNCTION validate_session(p_session_token TEXT) -RETURNS TABLE( - user_id UUID, - username VARCHAR(64), - role VARCHAR(50), - expires_at TIMESTAMP WITH TIME ZONE, - is_valid BOOLEAN -) AS $$ -BEGIN - -- Update last used time and return session info - UPDATE user_sessions - SET last_used_at = NOW() - WHERE session_token = p_session_token - AND is_active = true - AND expires_at > NOW(); - - RETURN QUERY - SELECT - u.id, - u.username, - u.role, - s.expires_at, - (s.id IS NOT NULL AND s.expires_at > NOW()) as is_valid - FROM user_sessions s - JOIN users u ON s.user_id = u.id - WHERE s.session_token = p_session_token - AND s.is_active = true - AND u.status = 'active'; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Function to create trading account -CREATE OR REPLACE FUNCTION create_trading_account( - p_user_id UUID, - p_account_name VARCHAR(100), - p_account_type VARCHAR(20) DEFAULT 'individual', - p_initial_balance DECIMAL(20, 8) DEFAULT 0, - p_leverage DECIMAL(10, 2) DEFAULT 1.00, - p_is_demo BOOLEAN DEFAULT false -) -RETURNS UUID AS $$ -DECLARE - v_account_id UUID; - v_account_number VARCHAR(64); -BEGIN - -- Generate account number - v_account_number := 'AC' || to_char(NOW(), 'YYYYMMDD') || '-' || - encode(gen_random_bytes(4), 'hex'); - - -- Insert account - INSERT INTO accounts ( - user_id, account_number, account_name, account_type, - initial_balance, current_balance, available_balance, - leverage, is_demo - ) VALUES ( - p_user_id, v_account_number, p_account_name, p_account_type, - p_initial_balance, p_initial_balance, p_initial_balance, - p_leverage, p_is_demo - ) RETURNING id INTO v_account_id; - - -- Grant full permissions to account owner - INSERT INTO account_permissions ( - user_id, account_id, permission_type, granted_by - ) VALUES - (p_user_id, v_account_id, 'read', p_user_id), - (p_user_id, v_account_id, 'trade', p_user_id), - (p_user_id, v_account_id, 'admin', p_user_id); - - -- Create audit log - INSERT INTO audit_logs ( - event_type, entity_type, entity_id, user_id, action, - new_values, timestamp, source - ) VALUES ( - 'account_created', 'account', v_account_id, p_user_id, 'create', - jsonb_build_object( - 'account_number', v_account_number, - 'account_type', p_account_type, - 'initial_balance', p_initial_balance, - 'is_demo', p_is_demo - ), - NOW(), 'account_management' - ); - - RETURN v_account_id; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Add constraints for data integrity -ALTER TABLE users ADD CONSTRAINT check_password_expiry - CHECK (password_expires_at IS NULL OR password_expires_at > password_changed_at); - -ALTER TABLE accounts ADD CONSTRAINT check_balances - CHECK (current_balance >= 0 AND available_balance >= 0); - -ALTER TABLE accounts ADD CONSTRAINT check_leverage - CHECK (leverage > 0 AND leverage <= max_leverage); - --- Create views for common queries - --- Active users view -CREATE VIEW active_users AS -SELECT - id, username, email, first_name, last_name, role, - last_login, created_at, two_factor_enabled -FROM users -WHERE status = 'active'; - --- Account summary view -CREATE VIEW account_summary AS -SELECT - a.id, a.account_number, a.account_name, a.account_type, - u.username, u.first_name, u.last_name, - a.current_balance, a.available_balance, a.equity, - a.leverage, a.status, a.is_demo, - COUNT(p.id) as position_count, - COUNT(o.id) as open_orders -FROM accounts a -JOIN users u ON a.user_id = u.id -LEFT JOIN positions p ON a.id::text = p.account_id AND p.quantity != 0 -LEFT JOIN orders o ON a.id::text = o.account_id AND o.status IN ('pending', 'partial') -GROUP BY a.id, u.username, u.first_name, u.last_name; - --- Add comments for documentation -COMMENT ON TABLE users IS 'Comprehensive user management for HFT trading systems'; -COMMENT ON TABLE accounts IS 'Trading accounts with real-time balance tracking'; -COMMENT ON TABLE api_keys IS 'API keys for programmatic trading access'; -COMMENT ON TABLE user_sessions IS 'Active user sessions for security tracking'; -COMMENT ON TABLE brokers IS 'External broker connection configurations'; -COMMENT ON TABLE compliance_profiles IS 'Regulatory compliance requirements'; - -COMMENT ON FUNCTION create_user IS 'Create new user with encrypted password and audit trail'; -COMMENT ON FUNCTION authenticate_user IS 'Authenticate user and create session with security logging'; -COMMENT ON FUNCTION validate_session IS 'Validate active session and update last used timestamp'; -COMMENT ON FUNCTION create_trading_account IS 'Create new trading account with permissions'; \ No newline at end of file diff --git a/migrations/005_up_create_advanced_risk_management.sql b/migrations/005_up_create_advanced_risk_management.sql deleted file mode 100644 index fe9bef36a..000000000 --- a/migrations/005_up_create_advanced_risk_management.sql +++ /dev/null @@ -1,508 +0,0 @@ --- Migration 005: Advanced Risk Management and Regulatory Compliance --- This migration creates comprehensive risk management for institutional HFT trading - --- Risk limits table - comprehensive limit management -CREATE TABLE IF NOT EXISTS risk_limits ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, - user_id UUID REFERENCES users(id) ON DELETE CASCADE, - limit_type VARCHAR(50) NOT NULL, -- 'position_size', 'daily_loss', 'exposure', 'concentration', 'var', 'leverage' - limit_scope VARCHAR(20) NOT NULL DEFAULT 'account', -- 'account', 'user', 'symbol', 'sector', 'strategy' - symbol VARCHAR(32), -- NULL for portfolio-level limits - strategy_name VARCHAR(100), -- NULL for general limits - sector VARCHAR(50), -- NULL for non-sector limits - limit_value DECIMAL(20, 8) NOT NULL, - warning_threshold DECIMAL(5, 4) DEFAULT 0.80, -- Warn at 80% of limit - breach_action VARCHAR(50) NOT NULL DEFAULT 'alert', -- 'alert', 'block', 'reduce', 'liquidate' - time_window VARCHAR(20), -- '1m', '5m', '1h', '1d', 'rolling' - NULL for static limits - is_active BOOLEAN NOT NULL DEFAULT true, - priority INTEGER NOT NULL DEFAULT 100, -- Higher number = higher priority - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - created_by UUID REFERENCES users(id), - metadata JSONB -); - --- Risk limit breaches - audit trail -CREATE TABLE IF NOT EXISTS risk_limit_breaches ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - risk_limit_id UUID NOT NULL REFERENCES risk_limits(id), - account_id UUID, - user_id UUID, - symbol VARCHAR(32), - breach_value DECIMAL(20, 8) NOT NULL, - limit_value DECIMAL(20, 8) NOT NULL, - breach_percentage DECIMAL(5, 4) NOT NULL, - severity VARCHAR(20) NOT NULL CHECK (severity IN ('warning', 'breach', 'critical')), - action_taken VARCHAR(100), - resolution_status VARCHAR(20) NOT NULL DEFAULT 'open' CHECK (resolution_status IN ('open', 'acknowledged', 'resolved', 'false_positive')), - resolved_at TIMESTAMP WITH TIME ZONE, - resolved_by UUID REFERENCES users(id), - breach_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - detected_by VARCHAR(50) NOT NULL, -- 'system', 'manual', 'external' - correlation_id UUID, -- Group related breaches - metadata JSONB -); - --- Trading strategies table - strategy definitions -CREATE TABLE IF NOT EXISTS trading_strategies ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - strategy_name VARCHAR(100) NOT NULL UNIQUE, - description TEXT, - strategy_type VARCHAR(50) NOT NULL, -- 'trend_following', 'mean_reversion', 'arbitrage', 'market_making' - algorithm_version VARCHAR(20), - parameters JSONB NOT NULL DEFAULT '{}'::jsonb, - risk_parameters JSONB NOT NULL DEFAULT '{}'::jsonb, - symbols TEXT[], -- Supported symbols - timeframes TEXT[], -- Supported timeframes - min_account_balance DECIMAL(20, 8) DEFAULT 0, - max_position_size DECIMAL(20, 8), - max_daily_trades INTEGER, - is_active BOOLEAN NOT NULL DEFAULT true, - is_paper_only BOOLEAN NOT NULL DEFAULT false, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - created_by UUID REFERENCES users(id), - metadata JSONB -); - --- Strategy assignments - link strategies to accounts -CREATE TABLE IF NOT EXISTS strategy_assignments ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - account_id UUID NOT NULL REFERENCES accounts(id) ON DELETE CASCADE, - strategy_id UUID NOT NULL REFERENCES trading_strategies(id) ON DELETE CASCADE, - allocation DECIMAL(5, 4) NOT NULL DEFAULT 1.0, -- Percentage of account allocated (0.0-1.0) - custom_parameters JSONB DEFAULT '{}'::jsonb, - risk_multiplier DECIMAL(5, 4) DEFAULT 1.0, -- Risk scaling factor - is_active BOOLEAN NOT NULL DEFAULT true, - started_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - stopped_at TIMESTAMP WITH TIME ZONE, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - created_by UUID REFERENCES users(id), - metadata JSONB, - - UNIQUE(account_id, strategy_id) -); - --- VaR calculations table - Value at Risk tracking -CREATE TABLE IF NOT EXISTS var_calculations ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, - calculation_date DATE NOT NULL, - confidence_level DECIMAL(5, 4) NOT NULL, -- 0.95, 0.99, etc. - time_horizon INTEGER NOT NULL, -- Days - var_amount DECIMAL(20, 8) NOT NULL, - expected_shortfall DECIMAL(20, 8), -- Conditional VaR - methodology VARCHAR(50) NOT NULL, -- 'historical', 'parametric', 'monte_carlo' - portfolio_value DECIMAL(20, 8) NOT NULL, - var_percentage DECIMAL(10, 6) NOT NULL, - calculation_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - model_parameters JSONB, - positions_snapshot JSONB, -- Snapshot of positions used - market_data_window JSONB, -- Time window of market data used - metadata JSONB, - - UNIQUE(account_id, calculation_date, confidence_level, time_horizon) -); - --- Stress tests table - scenario analysis -CREATE TABLE IF NOT EXISTS stress_tests ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - test_name VARCHAR(100) NOT NULL, - description TEXT, - scenario_type VARCHAR(50) NOT NULL, -- 'historical', 'hypothetical', 'regulatory' - scenario_parameters JSONB NOT NULL, - account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, - test_date DATE NOT NULL, - portfolio_value_before DECIMAL(20, 8) NOT NULL, - portfolio_value_after DECIMAL(20, 8) NOT NULL, - loss_amount DECIMAL(20, 8) NOT NULL, - loss_percentage DECIMAL(10, 6) NOT NULL, - worst_position JSONB, -- Position with worst performance - test_duration_ms INTEGER, - passed_regulatory BOOLEAN, - regulatory_threshold DECIMAL(20, 8), - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - created_by UUID REFERENCES users(id), - metadata JSONB -); - --- Regulatory reports table - compliance reporting -CREATE TABLE IF NOT EXISTS regulatory_reports ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - report_type VARCHAR(50) NOT NULL, -- 'daily_risk', 'var_breach', 'large_trader', 'position_limit' - jurisdiction VARCHAR(10) NOT NULL, - regulator VARCHAR(50) NOT NULL, -- 'CFTC', 'SEC', 'FCA', 'ESMA' - reporting_period_start DATE NOT NULL, - reporting_period_end DATE NOT NULL, - account_id UUID REFERENCES accounts(id), - user_id UUID REFERENCES users(id), - report_data JSONB NOT NULL, - file_path TEXT, -- Path to generated report file - submission_id VARCHAR(100), -- Regulator's submission ID - status VARCHAR(20) NOT NULL DEFAULT 'draft' CHECK (status IN ('draft', 'generated', 'submitted', 'acknowledged', 'rejected')), - due_date DATE, - submitted_at TIMESTAMP WITH TIME ZONE, - acknowledged_at TIMESTAMP WITH TIME ZONE, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - created_by UUID REFERENCES users(id), - metadata JSONB -); - --- Trade surveillance alerts - monitoring suspicious activity -CREATE TABLE IF NOT EXISTS surveillance_alerts ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - alert_type VARCHAR(50) NOT NULL, -- 'unusual_volume', 'price_manipulation', 'layering', 'spoofing', 'wash_trading' - severity VARCHAR(20) NOT NULL CHECK (severity IN ('low', 'medium', 'high', 'critical')), - account_id UUID REFERENCES accounts(id), - user_id UUID REFERENCES users(id), - symbol VARCHAR(32), - strategy_name VARCHAR(100), - trigger_condition TEXT NOT NULL, - detected_pattern JSONB NOT NULL, - related_orders UUID[], -- Array of order IDs - related_trades UUID[], -- Array of fill IDs - score DECIMAL(5, 2), -- Alert confidence score 0-100 - false_positive_probability DECIMAL(5, 4), -- 0.0-1.0 - status VARCHAR(20) NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'investigating', 'closed', 'escalated')), - assigned_to UUID REFERENCES users(id), - resolution TEXT, - detected_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - resolved_at TIMESTAMP WITH TIME ZONE, - escalated_at TIMESTAMP WITH TIME ZONE, - metadata JSONB -); - --- Market data quality checks -CREATE TABLE IF NOT EXISTS market_data_quality ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - symbol VARCHAR(32) NOT NULL, - data_source VARCHAR(50) NOT NULL, - quality_check_type VARCHAR(50) NOT NULL, -- 'stale_data', 'outlier_price', 'missing_data', 'sequence_gap' - check_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - severity VARCHAR(20) NOT NULL CHECK (severity IN ('info', 'warning', 'error', 'critical')), - description TEXT NOT NULL, - affected_data JSONB, - resolution_action VARCHAR(100), - is_resolved BOOLEAN NOT NULL DEFAULT false, - resolved_at TIMESTAMP WITH TIME ZONE, - impact_assessment TEXT, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metadata JSONB -); - --- Create optimized indexes for HFT performance - --- Risk limits indexes -CREATE INDEX IF NOT EXISTS idx_risk_limits_account_type ON risk_limits(account_id, limit_type); -CREATE INDEX IF NOT EXISTS idx_risk_limits_symbol ON risk_limits(symbol) WHERE symbol IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_risk_limits_active ON risk_limits(is_active, priority DESC); -CREATE INDEX IF NOT EXISTS idx_risk_limits_strategy ON risk_limits(strategy_name) WHERE strategy_name IS NOT NULL; - --- Risk limit breaches indexes -CREATE INDEX IF NOT EXISTS idx_risk_breaches_limit_time ON risk_limit_breaches(risk_limit_id, breach_time DESC); -CREATE INDEX IF NOT EXISTS idx_risk_breaches_account ON risk_limit_breaches(account_id, breach_time DESC); -CREATE INDEX IF NOT EXISTS idx_risk_breaches_severity ON risk_limit_breaches(severity, resolution_status); -CREATE INDEX IF NOT EXISTS idx_risk_breaches_correlation ON risk_limit_breaches(correlation_id) WHERE correlation_id IS NOT NULL; - --- Trading strategies indexes -CREATE INDEX IF NOT EXISTS idx_strategies_name ON trading_strategies(strategy_name); -CREATE INDEX IF NOT EXISTS idx_strategies_type ON trading_strategies(strategy_type); -CREATE INDEX IF NOT EXISTS idx_strategies_active ON trading_strategies(is_active); - --- Strategy assignments indexes -CREATE INDEX IF NOT EXISTS idx_strategy_assignments_account ON strategy_assignments(account_id, is_active); -CREATE INDEX IF NOT EXISTS idx_strategy_assignments_strategy ON strategy_assignments(strategy_id, is_active); - --- VaR calculations indexes -CREATE INDEX IF NOT EXISTS idx_var_calculations_account_date ON var_calculations(account_id, calculation_date DESC); -CREATE INDEX IF NOT EXISTS idx_var_calculations_date ON var_calculations(calculation_date DESC); - --- Stress tests indexes -CREATE INDEX IF NOT EXISTS idx_stress_tests_account_date ON stress_tests(account_id, test_date DESC); -CREATE INDEX IF NOT EXISTS idx_stress_tests_type ON stress_tests(scenario_type); - --- Regulatory reports indexes -CREATE INDEX IF NOT EXISTS idx_regulatory_reports_type_period ON regulatory_reports(report_type, reporting_period_start DESC); -CREATE INDEX IF NOT EXISTS idx_regulatory_reports_jurisdiction ON regulatory_reports(jurisdiction, status); -CREATE INDEX IF NOT EXISTS idx_regulatory_reports_due_date ON regulatory_reports(due_date) WHERE status IN ('draft', 'generated'); - --- Surveillance alerts indexes -CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_account ON surveillance_alerts(account_id, detected_at DESC); -CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_type ON surveillance_alerts(alert_type, severity); -CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_status ON surveillance_alerts(status, assigned_to); -CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_symbol ON surveillance_alerts(symbol, detected_at DESC) WHERE symbol IS NOT NULL; - --- Market data quality indexes -CREATE INDEX IF NOT EXISTS idx_market_data_quality_symbol ON market_data_quality(symbol, check_timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_market_data_quality_source ON market_data_quality(data_source, severity); - --- Create triggers for automatic updates -CREATE TRIGGER trigger_risk_limits_updated_at - BEFORE UPDATE ON risk_limits - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_strategies_updated_at - BEFORE UPDATE ON trading_strategies - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_strategy_assignments_updated_at - BEFORE UPDATE ON strategy_assignments - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER trigger_regulatory_reports_updated_at - BEFORE UPDATE ON regulatory_reports - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); - --- Create advanced risk management functions - --- Function to check risk limits before trade -CREATE OR REPLACE FUNCTION check_risk_limits_before_trade( - p_account_id UUID, - p_symbol VARCHAR(32), - p_side VARCHAR(10), -- 'buy' or 'sell' - p_quantity BIGINT, - p_price BIGINT -) -RETURNS TABLE( - can_trade BOOLEAN, - violated_limits JSONB, - warnings JSONB -) AS $$ -DECLARE - v_current_position BIGINT := 0; - v_new_position BIGINT; - v_trade_value DECIMAL(20, 8); - v_violations JSONB := '[]'::jsonb; - v_warnings JSONB := '[]'::jsonb; - v_limit RECORD; - v_current_exposure DECIMAL(20, 8); - v_account_balance DECIMAL(20, 8); -BEGIN - -- Get current position - SELECT COALESCE(quantity, 0) INTO v_current_position - FROM positions - WHERE account_id = p_account_id::text AND symbol = p_symbol; - - -- Calculate new position - v_new_position := v_current_position + - CASE WHEN p_side = 'buy' THEN p_quantity ELSE -p_quantity END; - - -- Calculate trade value - v_trade_value := (p_quantity * p_price) / 100.0; - - -- Get account balance - SELECT current_balance INTO v_account_balance - FROM accounts WHERE id = p_account_id; - - -- Check all active risk limits - FOR v_limit IN - SELECT * FROM risk_limits - WHERE is_active = true - AND (account_id = p_account_id OR account_id IS NULL) - AND (symbol = p_symbol OR symbol IS NULL) - ORDER BY priority DESC - LOOP - CASE v_limit.limit_type - WHEN 'position_size' THEN - IF ABS(v_new_position) > v_limit.limit_value THEN - v_violations := v_violations || jsonb_build_object( - 'limit_id', v_limit.id, - 'limit_type', 'position_size', - 'current_value', ABS(v_new_position), - 'limit_value', v_limit.limit_value - ); - ELSIF ABS(v_new_position) > (v_limit.limit_value * v_limit.warning_threshold) THEN - v_warnings := v_warnings || jsonb_build_object( - 'limit_id', v_limit.id, - 'limit_type', 'position_size', - 'current_value', ABS(v_new_position), - 'threshold', v_limit.limit_value * v_limit.warning_threshold - ); - END IF; - - WHEN 'exposure' THEN - -- Calculate current exposure (simplified) - SELECT COALESCE(SUM(ABS(quantity * last_price) / 100.0), 0) + v_trade_value - INTO v_current_exposure - FROM positions p - WHERE p.account_id = p_account_id::text; - - IF v_current_exposure > v_limit.limit_value THEN - v_violations := v_violations || jsonb_build_object( - 'limit_id', v_limit.id, - 'limit_type', 'exposure', - 'current_value', v_current_exposure, - 'limit_value', v_limit.limit_value - ); - END IF; - - WHEN 'leverage' THEN - IF v_current_exposure / v_account_balance > v_limit.limit_value THEN - v_violations := v_violations || jsonb_build_object( - 'limit_id', v_limit.id, - 'limit_type', 'leverage', - 'current_value', v_current_exposure / v_account_balance, - 'limit_value', v_limit.limit_value - ); - END IF; - END CASE; - END LOOP; - - -- Return results - RETURN QUERY SELECT - (jsonb_array_length(v_violations) = 0), - v_violations, - v_warnings; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Function to calculate portfolio VaR -CREATE OR REPLACE FUNCTION calculate_portfolio_var( - p_account_id UUID, - p_confidence_level DECIMAL(5, 4) DEFAULT 0.95, - p_time_horizon INTEGER DEFAULT 1 -) -RETURNS DECIMAL(20, 8) AS $$ -DECLARE - v_portfolio_value DECIMAL(20, 8) := 0; - v_var_amount DECIMAL(20, 8) := 0; - v_volatility DECIMAL(10, 6) := 0.02; -- Default 2% daily volatility - v_z_score DECIMAL(10, 6); -BEGIN - -- Get portfolio value - SELECT COALESCE(SUM(ABS(quantity * last_price) / 100.0), 0) - INTO v_portfolio_value - FROM positions - WHERE account_id = p_account_id::text AND quantity != 0; - - -- Calculate Z-score for confidence level - v_z_score := CASE - WHEN p_confidence_level >= 0.99 THEN 2.326 - WHEN p_confidence_level >= 0.95 THEN 1.645 - ELSE 1.282 - END; - - -- Simple VaR calculation (can be enhanced with historical data) - v_var_amount := v_portfolio_value * v_volatility * v_z_score * SQRT(p_time_horizon); - - -- Store calculation - INSERT INTO var_calculations ( - account_id, calculation_date, confidence_level, time_horizon, - var_amount, methodology, portfolio_value, var_percentage - ) VALUES ( - p_account_id, CURRENT_DATE, p_confidence_level, p_time_horizon, - v_var_amount, 'parametric', v_portfolio_value, - CASE WHEN v_portfolio_value > 0 THEN v_var_amount / v_portfolio_value ELSE 0 END - ) ON CONFLICT (account_id, calculation_date, confidence_level, time_horizon) - DO UPDATE SET - var_amount = EXCLUDED.var_amount, - portfolio_value = EXCLUDED.portfolio_value, - var_percentage = EXCLUDED.var_percentage, - calculation_time = NOW(); - - RETURN v_var_amount; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Function to detect layering pattern -CREATE OR REPLACE FUNCTION detect_layering_pattern( - p_account_id UUID, - p_symbol VARCHAR(32), - p_time_window INTERVAL DEFAULT '5 minutes' -) -RETURNS BOOLEAN AS $$ -DECLARE - v_order_count INTEGER; - v_cancel_ratio DECIMAL(5, 4); - v_pattern_detected BOOLEAN := false; -BEGIN - -- Count orders and cancellations in time window - SELECT - COUNT(*), - COUNT(*) FILTER (WHERE status = 'cancelled')::DECIMAL / NULLIF(COUNT(*), 0) - INTO v_order_count, v_cancel_ratio - FROM orders - WHERE account_id = p_account_id::text - AND symbol = p_symbol - AND created_at >= NOW() - p_time_window; - - -- Detect pattern: high number of orders with high cancellation ratio - IF v_order_count >= 20 AND v_cancel_ratio >= 0.80 THEN - v_pattern_detected := true; - - -- Create surveillance alert - INSERT INTO surveillance_alerts ( - alert_type, severity, account_id, symbol, trigger_condition, - detected_pattern, score - ) VALUES ( - 'layering', 'high', p_account_id, p_symbol, - 'High order count with excessive cancellation ratio', - jsonb_build_object( - 'order_count', v_order_count, - 'cancel_ratio', v_cancel_ratio, - 'time_window', p_time_window::text - ), - 85.0 - ); - END IF; - - RETURN v_pattern_detected; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Create materialized views for performance - --- Risk exposure summary -CREATE MATERIALIZED VIEW risk_exposure_summary AS -SELECT - a.id as account_id, - a.account_number, - u.username, - COUNT(p.id) as position_count, - COALESCE(SUM(ABS(p.quantity * p.last_price) / 100.0), 0) as total_exposure, - COALESCE(SUM(p.unrealized_pnl) / 100.0, 0) as unrealized_pnl, - COALESCE(MAX(var.var_amount), 0) as latest_var, - COUNT(rb.id) as active_breaches -FROM accounts a -JOIN users u ON a.user_id = u.id -LEFT JOIN positions p ON a.id::text = p.account_id AND p.quantity != 0 -LEFT JOIN var_calculations var ON a.id = var.account_id AND var.calculation_date = CURRENT_DATE -LEFT JOIN risk_limit_breaches rb ON a.id = rb.account_id AND rb.resolution_status = 'open' -GROUP BY a.id, a.account_number, u.username; - --- Create unique index on materialized view -CREATE UNIQUE INDEX idx_risk_exposure_summary_account_id -ON risk_exposure_summary(account_id); - --- Add constraints -ALTER TABLE risk_limits ADD CONSTRAINT check_warning_threshold - CHECK (warning_threshold > 0 AND warning_threshold <= 1.0); - -ALTER TABLE risk_limits ADD CONSTRAINT check_priority - CHECK (priority > 0); - -ALTER TABLE var_calculations ADD CONSTRAINT check_confidence_level - CHECK (confidence_level > 0 AND confidence_level < 1.0); - -ALTER TABLE strategy_assignments ADD CONSTRAINT check_allocation - CHECK (allocation >= 0 AND allocation <= 1.0); - --- Add comments for documentation -COMMENT ON TABLE risk_limits IS 'Comprehensive risk limit definitions with dynamic thresholds'; -COMMENT ON TABLE risk_limit_breaches IS 'Audit trail of all risk limit violations'; -COMMENT ON TABLE trading_strategies IS 'Trading strategy definitions and parameters'; -COMMENT ON TABLE var_calculations IS 'Value at Risk calculations with multiple methodologies'; -COMMENT ON TABLE stress_tests IS 'Stress testing scenarios and results'; -COMMENT ON TABLE surveillance_alerts IS 'Trade surveillance and market abuse detection'; -COMMENT ON TABLE regulatory_reports IS 'Regulatory compliance reporting and submissions'; - -COMMENT ON FUNCTION check_risk_limits_before_trade IS 'Pre-trade risk validation with violation detection'; -COMMENT ON FUNCTION calculate_portfolio_var IS 'Portfolio Value at Risk calculation and storage'; -COMMENT ON FUNCTION detect_layering_pattern IS 'Market abuse pattern detection for layering/spoofing'; \ No newline at end of file diff --git a/migrations/006_down_drop_performance_indexes.sql b/migrations/006_down_drop_performance_indexes.sql deleted file mode 100644 index ee369a227..000000000 --- a/migrations/006_down_drop_performance_indexes.sql +++ /dev/null @@ -1,56 +0,0 @@ --- Drop Performance-Optimized Indexes for HFT Trading System --- ========================================================= --- Removes specialized indexes for rollback scenarios - --- Drop monitoring functions -DROP FUNCTION IF EXISTS get_index_sizes(); -DROP FUNCTION IF EXISTS get_index_usage_stats(); - --- Drop GIN indexes -DROP INDEX IF EXISTS idx_fills_metadata_gin; -DROP INDEX IF EXISTS idx_orders_metadata_gin; - --- Drop expression indexes -DROP INDEX IF EXISTS idx_orders_remaining_qty; -DROP INDEX IF EXISTS idx_orders_value; - --- Drop partial indexes for hot data -DROP INDEX IF EXISTS idx_fills_recent; -DROP INDEX IF EXISTS idx_orders_recent; - --- Drop performance metrics indexes -DROP INDEX IF EXISTS idx_performance_metrics_component_timestamp; - --- Drop audit logs indexes -DROP INDEX IF EXISTS idx_audit_logs_entity_timestamp; -DROP INDEX IF EXISTS idx_audit_logs_event_timestamp; -DROP INDEX IF EXISTS idx_audit_logs_timestamp_desc; - --- Drop risk metrics indexes -DROP INDEX IF EXISTS idx_risk_metrics_account_metric_timestamp; -DROP INDEX IF EXISTS idx_risk_metrics_severity_timestamp; - --- Drop bars indexes -DROP INDEX IF EXISTS idx_bars_timestamp; -DROP INDEX IF EXISTS idx_bars_symbol_timeframe_timestamp; - --- Drop market data indexes (not concurrent for partitioned tables) -DROP INDEX IF EXISTS idx_market_data_timestamp; -DROP INDEX IF EXISTS idx_market_data_symbol_timestamp; - --- Drop positions indexes -DROP INDEX IF EXISTS idx_positions_active; -DROP INDEX IF EXISTS idx_positions_account_symbol; - --- Drop fills indexes -DROP INDEX IF EXISTS idx_fills_execution_time; -DROP INDEX IF EXISTS idx_fills_symbol_execution; -DROP INDEX IF EXISTS idx_fills_order_execution; - --- Drop orders indexes -DROP INDEX IF EXISTS idx_orders_account_created; -DROP INDEX IF EXISTS idx_orders_symbol_created; -DROP INDEX IF EXISTS idx_orders_status_created; -DROP INDEX IF EXISTS idx_orders_client_order_id; - --- Performance-optimized indexes dropped successfully! \ No newline at end of file diff --git a/migrations/006_up_create_performance_indexes.sql b/migrations/006_up_create_performance_indexes.sql deleted file mode 100644 index 0f4235290..000000000 --- a/migrations/006_up_create_performance_indexes.sql +++ /dev/null @@ -1,202 +0,0 @@ --- Performance-Optimized Indexes for HFT Trading System --- ==================================================== --- Creates essential indexes for existing tables only - --- === ORDERS TABLE INDEXES === --- Critical path: Order lookups by client_order_id (most frequent) -CREATE INDEX IF NOT EXISTS idx_orders_client_order_id -ON orders (client_order_id) WHERE client_order_id IS NOT NULL; - --- Critical path: Order status queries for active orders -CREATE INDEX IF NOT EXISTS idx_orders_status_created -ON orders (status, created_at DESC) -WHERE status IN ('pending', 'partial'); - --- Critical path: Symbol-based order queries with time -CREATE INDEX IF NOT EXISTS idx_orders_symbol_created -ON orders (symbol, created_at DESC); - --- Critical path: Account order history -CREATE INDEX IF NOT EXISTS idx_orders_account_created -ON orders (account_id, created_at DESC) WHERE account_id IS NOT NULL; - --- === FILLS TABLE INDEXES === --- Critical path: Order fill lookups -CREATE INDEX IF NOT EXISTS idx_fills_order_execution -ON fills (order_id, execution_time DESC); - --- Critical path: Symbol fill analysis -CREATE INDEX IF NOT EXISTS idx_fills_symbol_execution -ON fills (symbol, execution_time DESC); - --- Critical path: Recent fills -CREATE INDEX IF NOT EXISTS idx_fills_execution_time -ON fills (execution_time DESC); - --- === POSITIONS TABLE INDEXES === --- Critical path: Position lookups by account and symbol -CREATE INDEX IF NOT EXISTS idx_positions_account_symbol -ON positions (account_id, symbol) WHERE account_id IS NOT NULL; - --- Critical path: Non-zero positions only -CREATE INDEX IF NOT EXISTS idx_positions_active -ON positions (account_id, symbol) -WHERE quantity != 0 AND account_id IS NOT NULL; - --- === MARKET_DATA TABLE INDEXES === --- Note: market_data is partitioned, so we skip CONCURRENTLY --- Critical path: Recent market data by symbol -CREATE INDEX IF NOT EXISTS idx_market_data_symbol_timestamp -ON market_data (symbol, timestamp DESC); - --- Critical path: Market data time range queries -CREATE INDEX IF NOT EXISTS idx_market_data_timestamp -ON market_data (timestamp DESC); - --- === BARS TABLE INDEXES === --- Critical path: Bar data by symbol and timeframe -CREATE INDEX IF NOT EXISTS idx_bars_symbol_timeframe_timestamp -ON bars (symbol, timeframe, timestamp DESC); - --- Critical path: Recent bars -CREATE INDEX IF NOT EXISTS idx_bars_timestamp -ON bars (timestamp DESC); - --- === RISK_METRICS TABLE INDEXES === --- These are already created in migration 2, adding complementary ones - --- Critical path: Recent risk metrics by severity -CREATE INDEX IF NOT EXISTS idx_risk_metrics_severity_timestamp -ON risk_metrics (severity, timestamp DESC) -WHERE severity IN ('high', 'critical'); - --- Critical path: Account risk monitoring -CREATE INDEX IF NOT EXISTS idx_risk_metrics_account_metric_timestamp -ON risk_metrics (account_id, metric_type, timestamp DESC) -WHERE account_id IS NOT NULL; - --- === AUDIT_LOGS TABLE INDEXES === --- Critical path: Recent audit queries -CREATE INDEX IF NOT EXISTS idx_audit_logs_timestamp_desc -ON audit_logs (timestamp DESC); - --- Critical path: Event type queries -CREATE INDEX IF NOT EXISTS idx_audit_logs_event_timestamp -ON audit_logs (event_type, timestamp DESC); - --- Critical path: Entity audit trail -CREATE INDEX IF NOT EXISTS idx_audit_logs_entity_timestamp -ON audit_logs (entity_type, entity_id, timestamp DESC); - --- === PERFORMANCE_METRICS TABLE INDEXES === --- These are already created in migration 2, adding complementary ones - --- Critical path: Metric analysis by component -CREATE INDEX IF NOT EXISTS idx_performance_metrics_component_timestamp -ON performance_metrics (component, timestamp DESC); - --- === PARTIAL INDEXES FOR HOT DATA === --- Only index recent orders (recent data only) -CREATE INDEX IF NOT EXISTS idx_orders_recent -ON orders (symbol, created_at DESC, status); - --- Only index recent fills -CREATE INDEX IF NOT EXISTS idx_fills_recent -ON fills (symbol, execution_time DESC); - --- === EXPRESSION INDEXES === --- Index for order value calculations (quantity * price) -CREATE INDEX IF NOT EXISTS idx_orders_value -ON orders ((quantity * price)) -WHERE status IN ('pending', 'partial') AND price IS NOT NULL; - --- Index for remaining quantity calculations -CREATE INDEX IF NOT EXISTS idx_orders_remaining_qty -ON orders ((quantity - filled_quantity)) -WHERE status = 'partial'; - --- === GIN INDEXES FOR JSONB DATA === --- Enable fast queries on JSONB metadata where it exists -CREATE INDEX IF NOT EXISTS idx_orders_metadata_gin -ON orders USING GIN (metadata) WHERE metadata IS NOT NULL; - -CREATE INDEX IF NOT EXISTS idx_fills_metadata_gin -ON fills USING GIN (metadata) WHERE metadata IS NOT NULL; - --- === UPDATE STATISTICS === --- Update statistics for better query planning -ANALYZE orders; -ANALYZE fills; -ANALYZE positions; -ANALYZE market_data; -ANALYZE bars; -ANALYZE risk_metrics; -ANALYZE performance_metrics; -ANALYZE audit_logs; - --- === INDEX MONITORING FUNCTIONS === --- Create function to monitor index usage -CREATE OR REPLACE FUNCTION get_index_usage_stats() -RETURNS TABLE ( - schemaname TEXT, - tablename TEXT, - indexname TEXT, - idx_tup_read BIGINT, - idx_tup_fetch BIGINT, - usage_ratio NUMERIC -) -LANGUAGE SQL AS $$ - SELECT - s.schemaname, - s.relname as tablename, - s.indexrelname as indexname, - s.idx_tup_read, - s.idx_tup_fetch, - CASE WHEN s.idx_tup_read = 0 - THEN 0 - ELSE ROUND(s.idx_tup_fetch::NUMERIC / s.idx_tup_read * 100, 2) - END as usage_ratio - FROM pg_stat_user_indexes s - WHERE s.schemaname = 'public' - ORDER BY s.idx_tup_read DESC; -$$; - --- === INDEX SIZE MONITORING === --- Create function to monitor index sizes -CREATE OR REPLACE FUNCTION get_index_sizes() -RETURNS TABLE ( - tablename TEXT, - indexname TEXT, - index_size TEXT, - index_size_bytes BIGINT -) -LANGUAGE SQL AS $$ - SELECT - t.relname as tablename, - i.relname as indexname, - pg_size_pretty(pg_relation_size(i.oid)) as index_size, - pg_relation_size(i.oid) as index_size_bytes - FROM pg_class i - JOIN pg_index ix ON i.oid = ix.indexrelid - JOIN pg_class t ON ix.indrelid = t.oid - JOIN pg_namespace n ON t.relnamespace = n.oid - WHERE i.relkind = 'i' - AND n.nspname = 'public' - ORDER BY pg_relation_size(i.oid) DESC; -$$; - --- Performance-optimized indexes created successfully! --- Essential indexes for HFT workloads: --- - Orders: client_order_id, status, symbol, account lookups --- - Fills: order_id, symbol, execution_time lookups --- - Positions: account/symbol combinations, active positions --- - Market Data: symbol/timestamp combinations --- - Risk Metrics: severity and account-based monitoring --- - Audit Logs: timestamp and entity-based queries --- - Partial indexes for hot data (last 7 days) --- - Expression indexes for calculated values --- - GIN indexes for JSONB metadata --- --- Monitoring functions: --- - get_index_usage_stats(): Monitor index usage patterns --- - get_index_sizes(): Monitor index storage requirements \ No newline at end of file diff --git a/migrations/007_configuration_schema.sql b/migrations/007_configuration_schema.sql index 7cb6f3afc..9f9d2df0f 100644 --- a/migrations/007_configuration_schema.sql +++ b/migrations/007_configuration_schema.sql @@ -91,12 +91,13 @@ CREATE TABLE config_settings ( affects TEXT[], -- What this setting affects (services, components) hot_reload BOOLEAN NOT NULL DEFAULT true, -- Can be changed without restart restart_required BOOLEAN NOT NULL DEFAULT false, -- Requires service restart + is_active BOOLEAN NOT NULL DEFAULT true, -- Soft delete / deactivation version INTEGER NOT NULL DEFAULT 1, -- Version for optimistic locking created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), created_by VARCHAR(100) NOT NULL DEFAULT CURRENT_USER, updated_by VARCHAR(100) NOT NULL DEFAULT CURRENT_USER, - + -- Constraints CONSTRAINT uk_config_settings_key_env UNIQUE (config_key, environment), CONSTRAINT chk_value_type CHECK (value_type IN ('string', 'number', 'boolean', 'object', 'array', 'null')), @@ -251,9 +252,6 @@ CREATE INDEX idx_config_locks_locked_by ON config_locks(locked_by); -- === TRIGGERS FOR NOTIFICATIONS === --- Add column for tracking if a config is active -ALTER TABLE config_settings ADD COLUMN IF NOT EXISTS is_active BOOLEAN NOT NULL DEFAULT true; - -- Trigger for config_settings changes CREATE TRIGGER tr_config_settings_notify AFTER INSERT OR UPDATE OR DELETE ON config_settings @@ -457,9 +455,9 @@ ANALYZE config_locks; -- Create performance monitoring view CREATE OR REPLACE VIEW config_performance_stats AS -SELECT +SELECT schemaname, - tablename, + relname as tablename, n_tup_ins as inserts, n_tup_upd as updates, n_tup_del as deletes, @@ -469,9 +467,9 @@ SELECT last_autovacuum, last_analyze, last_autoanalyze -FROM pg_stat_user_tables -WHERE tablename LIKE 'config_%' -ORDER BY tablename; +FROM pg_stat_user_tables +WHERE relname LIKE 'config_%' +ORDER BY relname; -- === COMMENTS FOR DOCUMENTATION === diff --git a/migrations/008_initial_config_data.sql b/migrations/008_initial_config_data.sql index 7052ca163..1296e9688 100644 --- a/migrations/008_initial_config_data.sql +++ b/migrations/008_initial_config_data.sql @@ -130,11 +130,11 @@ FROM config_categories WHERE category_path = 'system.service_discovery'; -- PostgreSQL Configuration INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'max_connections', id, 'database.postgresql', '10'::jsonb, 'number', 'development', 'Maximum database connections per service', ARRAY['database', 'performance'], false, true +SELECT 'postgres_max_connections', id, 'database.postgresql', '10'::jsonb, 'number', 'development', 'Maximum database connections per service', ARRAY['database', 'performance'], false, true FROM config_categories WHERE category_path = 'database.postgresql'; INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'max_connections', id, 'database.postgresql', '50'::jsonb, 'number', 'production', 'Production database connection pool size', ARRAY['database', 'performance'], false, true +SELECT 'postgres_max_connections', id, 'database.postgresql', '50'::jsonb, 'number', 'production', 'Production database connection pool size', ARRAY['database', 'performance'], false, true FROM config_categories WHERE category_path = 'database.postgresql'; INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) @@ -155,7 +155,7 @@ SELECT 'default_ttl_seconds', id, 'database.redis', '3600'::jsonb, 'number', 'de FROM config_categories WHERE category_path = 'database.redis'; INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'max_connections', id, 'database.redis', '20'::jsonb, 'number', 'development', 'Maximum Redis connections per service', ARRAY['cache', 'performance'], false, true +SELECT 'redis_max_connections', id, 'database.redis', '20'::jsonb, 'number', 'development', 'Maximum Redis connections per service', ARRAY['cache', 'performance'], false, true FROM config_categories WHERE category_path = 'database.redis'; -- === SECURITY CONFIGURATION === diff --git a/migrations/009_dual_provider_configuration.sql b/migrations/009_dual_provider_configuration.sql index 51f415db2..d714c323c 100644 --- a/migrations/009_dual_provider_configuration.sql +++ b/migrations/009_dual_provider_configuration.sql @@ -116,87 +116,87 @@ CREATE INDEX idx_provider_endpoints_priority ON provider_endpoints(priority); -- === DATABENTO CONFIGURATION === -- Databento API Configuration -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, hot_reload, restart_required) -SELECT 'api_key', id, 'trading.providers.databento', '""'::jsonb, 'string', 'development', 'Databento API key for authentication', ARRAY['databento', 'api', 'authentication'], true, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_api_key', id, 'trading.providers.databento', '""'::jsonb, 'string', 'development', 'Databento API key for authentication', ARRAY['databento', 'api', 'authentication'], true, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, hot_reload, restart_required) -SELECT 'api_key', id, 'trading.providers.databento', '""'::jsonb, 'string', 'production', 'Production Databento API key', ARRAY['databento', 'api', 'authentication'], true, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_api_key', id, 'trading.providers.databento', '""'::jsonb, 'string', 'production', 'Production Databento API key', ARRAY['databento', 'api', 'authentication'], true, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'dataset', id, 'trading.providers.databento', '"XNAS.ITCH"'::jsonb, 'string', 'development', 'Primary Databento dataset for market data', ARRAY['databento', 'dataset', 'market_data'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_dataset', id, 'trading.providers.databento', '"XNAS.ITCH"'::jsonb, 'string', 'development', 'Primary Databento dataset for market data', ARRAY['databento', 'dataset', 'market_data'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'dataset', id, 'trading.providers.databento', '"XNAS.ITCH"'::jsonb, 'string', 'production', 'Production Databento dataset', ARRAY['databento', 'dataset', 'market_data'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_dataset', id, 'trading.providers.databento', '"XNAS.ITCH"'::jsonb, 'string', 'production', 'Production Databento dataset', ARRAY['databento', 'dataset', 'market_data'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'symbols', id, 'trading.providers.databento', '["AAPL", "MSFT", "GOOGL", "TSLA", "AMZN"]'::jsonb, 'array', 'development', 'List of symbols to subscribe to', ARRAY['databento', 'symbols', 'subscription'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_symbols', id, 'trading.providers.databento', '["AAPL", "MSFT", "GOOGL", "TSLA", "AMZN"]'::jsonb, 'array', 'development', 'List of symbols to subscribe to', ARRAY['databento', 'symbols', 'subscription'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'connection_timeout_ms', id, 'trading.providers.databento', '30000'::jsonb, 'number', 'development', 'Connection timeout for Databento API', ARRAY['databento', 'timeout', 'connection'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_connection_timeout_ms', id, 'trading.providers.databento', '30000'::jsonb, 'number', 'development', 'Connection timeout for Databento API', ARRAY['databento', 'timeout', 'connection'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'request_timeout_ms', id, 'trading.providers.databento', '10000'::jsonb, 'number', 'development', 'Request timeout for Databento API calls', ARRAY['databento', 'timeout', 'request'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_request_timeout_ms', id, 'trading.providers.databento', '10000'::jsonb, 'number', 'development', 'Request timeout for Databento API calls', ARRAY['databento', 'timeout', 'request'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'rate_limit_requests_per_second', id, 'trading.providers.databento', '100'::jsonb, 'number', 'development', 'Rate limit for Databento API requests', ARRAY['databento', 'rate_limit'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_rate_limit_rps', id, 'trading.providers.databento', '100'::jsonb, 'number', 'development', 'Rate limit for Databento API requests', ARRAY['databento', 'rate_limit'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'enable_live_data', id, 'trading.providers.databento', 'true'::jsonb, 'boolean', 'development', 'Enable live market data streaming', ARRAY['databento', 'live_data', 'streaming'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_enable_live_data', id, 'trading.providers.databento', 'true'::jsonb, 'boolean', 'development', 'Enable live market data streaming', ARRAY['databento', 'live_data', 'streaming'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'enable_historical_data', id, 'trading.providers.databento', 'true'::jsonb, 'boolean', 'development', 'Enable historical data retrieval', ARRAY['databento', 'historical_data'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'databento_enable_historical_data', id, 'trading.providers.databento', 'true'::jsonb, 'boolean', 'development', 'Enable historical data retrieval', ARRAY['databento', 'historical_data'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.databento'; -- === BENZINGA CONFIGURATION === -- Benzinga API Configuration -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, hot_reload, restart_required) -SELECT 'api_key', id, 'trading.providers.benzinga', '""'::jsonb, 'string', 'development', 'Benzinga API key for authentication', ARRAY['benzinga', 'api', 'authentication'], true, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_api_key', id, 'trading.providers.benzinga', '""'::jsonb, 'string', 'development', 'Benzinga API key for authentication', ARRAY['benzinga', 'api', 'authentication'], true, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, hot_reload, restart_required) -SELECT 'api_key', id, 'trading.providers.benzinga', '""'::jsonb, 'string', 'production', 'Production Benzinga API key', ARRAY['benzinga', 'api', 'authentication'], true, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_api_key', id, 'trading.providers.benzinga', '""'::jsonb, 'string', 'production', 'Production Benzinga API key', ARRAY['benzinga', 'api', 'authentication'], true, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'subscription_tier', id, 'trading.providers.benzinga', '"pro"'::jsonb, 'string', 'development', 'Benzinga subscription tier (basic, pro, enterprise)', ARRAY['benzinga', 'subscription', 'tier'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_subscription_tier', id, 'trading.providers.benzinga', '"pro"'::jsonb, 'string', 'development', 'Benzinga subscription tier (basic, pro, enterprise)', ARRAY['benzinga', 'subscription', 'tier'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'subscription_tier', id, 'trading.providers.benzinga', '"enterprise"'::jsonb, 'string', 'production', 'Production Benzinga subscription tier', ARRAY['benzinga', 'subscription', 'tier'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_subscription_tier', id, 'trading.providers.benzinga', '"enterprise"'::jsonb, 'string', 'production', 'Production Benzinga subscription tier', ARRAY['benzinga', 'subscription', 'tier'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'enable_news_feed', id, 'trading.providers.benzinga', 'true'::jsonb, 'boolean', 'development', 'Enable Benzinga news feed', ARRAY['benzinga', 'news', 'feed'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_enable_news_feed', id, 'trading.providers.benzinga', 'true'::jsonb, 'boolean', 'development', 'Enable Benzinga news feed', ARRAY['benzinga', 'news', 'feed'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'enable_analyst_ratings', id, 'trading.providers.benzinga', 'true'::jsonb, 'boolean', 'development', 'Enable analyst ratings data', ARRAY['benzinga', 'analyst_ratings'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_enable_analyst_ratings', id, 'trading.providers.benzinga', 'true'::jsonb, 'boolean', 'development', 'Enable analyst ratings data', ARRAY['benzinga', 'analyst_ratings'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'enable_earnings_data', id, 'trading.providers.benzinga', 'true'::jsonb, 'boolean', 'development', 'Enable earnings calendar and data', ARRAY['benzinga', 'earnings'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_enable_earnings_data', id, 'trading.providers.benzinga', 'true'::jsonb, 'boolean', 'development', 'Enable earnings calendar and data', ARRAY['benzinga', 'earnings'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'connection_timeout_ms', id, 'trading.providers.benzinga', '30000'::jsonb, 'number', 'development', 'Connection timeout for Benzinga API', ARRAY['benzinga', 'timeout', 'connection'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_connection_timeout_ms', id, 'trading.providers.benzinga', '30000'::jsonb, 'number', 'development', 'Connection timeout for Benzinga API', ARRAY['benzinga', 'timeout', 'connection'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'rate_limit_requests_per_minute', id, 'trading.providers.benzinga', '1000'::jsonb, 'number', 'development', 'Rate limit for Benzinga API requests per minute', ARRAY['benzinga', 'rate_limit'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_rate_limit_rpm', id, 'trading.providers.benzinga', '1000'::jsonb, 'number', 'development', 'Rate limit for Benzinga API requests per minute', ARRAY['benzinga', 'rate_limit'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, hot_reload, restart_required) -SELECT 'news_categories', id, 'trading.providers.benzinga', '["earnings", "analyst-ratings", "sec-filings", "mergers-acquisitions"]'::jsonb, 'array', 'development', 'News categories to subscribe to', ARRAY['benzinga', 'news', 'categories'], false, true, false +INSERT INTO config_settings (config_key, category_id, category_path, config_value, value_type, environment, description, tags, is_sensitive, is_system, hot_reload, restart_required) +SELECT 'benzinga_news_categories', id, 'trading.providers.benzinga', '["earnings", "analyst-ratings", "sec-filings", "mergers-acquisitions"]'::jsonb, 'array', 'development', 'News categories to subscribe to', ARRAY['benzinga', 'news', 'categories'], false, false, true, false FROM config_categories WHERE category_path = 'trading.providers.benzinga'; -- === PROVIDER ENDPOINT CONFIGURATIONS === @@ -221,19 +221,19 @@ INSERT INTO provider_endpoints (provider_name, endpoint_type, base_url, websocke -- Databento subscriptions INSERT INTO provider_subscriptions (provider_name, subscription_type, dataset, symbols, environment, rate_limit_per_second, max_concurrent_connections, metadata) VALUES -('databento', 'equities_l1', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'development', 100, 2, '{"schema": "mbo", "stype_in": "raw_symbol"}'), -('databento', 'equities_l2', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL'], 'development', 50, 1, '{"schema": "mbp-1", "stype_in": "raw_symbol"}'), -('databento', 'equities_l1', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN', 'NVDA', 'META'], 'production', 500, 5, '{"schema": "mbo", "stype_in": "raw_symbol"}'), -('databento', 'equities_l2', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'production', 200, 3, '{"schema": "mbp-1", "stype_in": "raw_symbol"}'); +('databento', 'equities_l1', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'development', 100, 2, '{"schema": "mbo", "stype_in": "raw_symbol"}'::jsonb), +('databento', 'equities_l2', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL'], 'development', 50, 1, '{"schema": "mbp-1", "stype_in": "raw_symbol"}'::jsonb), +('databento', 'equities_l1', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN', 'NVDA', 'META'], 'production', 500, 5, '{"schema": "mbo", "stype_in": "raw_symbol"}'::jsonb), +('databento', 'equities_l2', 'XNAS.ITCH', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'production', 200, 3, '{"schema": "mbp-1", "stype_in": "raw_symbol"}'::jsonb); -- Benzinga subscriptions INSERT INTO provider_subscriptions (provider_name, subscription_type, dataset, symbols, environment, rate_limit_per_second, max_concurrent_connections, metadata) VALUES -('benzinga', 'news_feed', 'general', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'development', 10, 1, '{"channels": ["news", "analyst-ratings"]}'), -('benzinga', 'earnings_calendar', 'earnings', NULL, 'development', 5, 1, '{"importance": "high"}'), -('benzinga', 'analyst_ratings', 'ratings', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'development', 5, 1, '{"rating_type": ["Upgrade", "Downgrade", "Initiates", "Reiterates"]}'), -('benzinga', 'news_feed', 'general', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN', 'NVDA', 'META'], 'production', 50, 3, '{"channels": ["news", "analyst-ratings", "sec-filings"]}'), -('benzinga', 'earnings_calendar', 'earnings', NULL, 'production', 20, 1, '{"importance": "high"}'), -('benzinga', 'analyst_ratings', 'ratings', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN', 'NVDA', 'META'], 'production', 20, 1, '{"rating_type": ["Upgrade", "Downgrade", "Initiates", "Reiterates"]}'); +('benzinga', 'news_feed', 'general', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'development', 10, 1, '{"channels": ["news", "analyst-ratings"]}'::jsonb), +('benzinga', 'earnings_calendar', 'earnings', NULL, 'development', 5, 1, '{"importance": "high"}'::jsonb), +('benzinga', 'analyst_ratings', 'ratings', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN'], 'development', 5, 1, '{"rating_type": ["Upgrade", "Downgrade", "Initiates", "Reiterates"]}'::jsonb), +('benzinga', 'news_feed', 'general', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN', 'NVDA', 'META'], 'production', 50, 3, '{"channels": ["news", "analyst-ratings", "sec-filings"]}'::jsonb), +('benzinga', 'earnings_calendar', 'earnings', NULL, 'production', 20, 1, '{"importance": "high"}'::jsonb), +('benzinga', 'analyst_ratings', 'ratings', ARRAY['AAPL', 'MSFT', 'GOOGL', 'TSLA', 'AMZN', 'NVDA', 'META'], 'production', 20, 1, '{"rating_type": ["Upgrade", "Downgrade", "Initiates", "Reiterates"]}'::jsonb); -- === NOTIFICATION TRIGGERS FOR PROVIDER TABLES === diff --git a/migrations/010_remove_polygon_configurations.sql b/migrations/010_remove_polygon_configurations.sql index 933995d76..195249eaf 100644 --- a/migrations/010_remove_polygon_configurations.sql +++ b/migrations/010_remove_polygon_configurations.sql @@ -67,28 +67,9 @@ END $$; -- === AUDIT LOG FOR REMOVAL === --- Log the removal in config_history for audit purposes -INSERT INTO config_history ( - config_setting_id, - config_key, - category_path, - environment, - old_value, - new_value, - change_type, - changed_by, - change_reason -) VALUES ( - NULL, - 'polygon_cleanup', - 'system.migration', - 'all', - '{"provider": "polygon", "status": "active"}'::jsonb, - '{"provider": "polygon", "status": "removed"}'::jsonb, - 'delete', - 'migration_010', - 'Migration to dual-provider setup - Polygon provider removed in favor of Databento + Benzinga' -); +-- Note: config_history table requires a valid config_setting_id (NOT NULL constraint) +-- Migration events are tracked in the migration_notes category instead +-- See the INSERT into config_categories below for migration tracking -- === UPDATE DOCUMENTATION === diff --git a/migrations/012_create_event_and_config_tables.sql b/migrations/012_create_event_and_config_tables.sql index 0d4462a95..bdaa05c1c 100644 --- a/migrations/012_create_event_and_config_tables.sql +++ b/migrations/012_create_event_and_config_tables.sql @@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS market_events ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), event_type VARCHAR(64) NOT NULL, symbol VARCHAR(32), - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, event_data JSONB NOT NULL, source VARCHAR(64), -- Event source (exchange, internal, etc.) severity VARCHAR(20) DEFAULT 'info' CHECK (severity IN ('debug', 'info', 'warning', 'error', 'critical')), @@ -24,11 +24,11 @@ CREATE TABLE IF NOT EXISTS trading_events ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), event_type VARCHAR(64) NOT NULL, symbol VARCHAR(32), - order_id UUID, + "order_id" UUID, account_id VARCHAR(64), - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, event_data JSONB NOT NULL, - correlation_id UUID, -- For tracing related events + "correlation_id" UUID, -- For tracing related events severity VARCHAR(20) DEFAULT 'info' CHECK (severity IN ('debug', 'info', 'warning', 'error', 'critical')), processed BOOLEAN DEFAULT FALSE, processed_at TIMESTAMP WITH TIME ZONE, @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS event_processing_stats ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), processor_name VARCHAR(128) NOT NULL, event_type VARCHAR(64) NOT NULL, - timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, events_processed INTEGER NOT NULL DEFAULT 0, events_failed INTEGER NOT NULL DEFAULT 0, average_processing_time_ms DECIMAL(10, 3), @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_processing_stats ( min_processing_time_ms DECIMAL(10, 3), total_processing_time_ms BIGINT, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - UNIQUE(processor_name, event_type, timestamp) + UNIQUE(processor_name, event_type, "timestamp") ); -- Configuration table - stores application configuration (if not already exists from config migration) @@ -135,21 +135,35 @@ CREATE TABLE IF NOT EXISTS var_calculations ( -- Create indexes for optimal query performance --- Market events indexes -CREATE INDEX IF NOT EXISTS idx_market_events_type_timestamp ON market_events(event_type, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_market_events_symbol_timestamp ON market_events(symbol, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_market_events_timestamp ON market_events(timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_market_events_processed ON market_events(processed, timestamp DESC); +-- Market events indexes (only create if table was created by this migration, not migration 001) +DO $$ +BEGIN + -- Check if market_events has the "timestamp" column (our schema) vs event_timestamp (migration 001 schema) + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'market_events' AND column_name = 'timestamp') THEN + CREATE INDEX IF NOT EXISTS idx_market_events_type_timestamp ON market_events(event_type, "timestamp" DESC); + CREATE INDEX IF NOT EXISTS idx_market_events_symbol_timestamp ON market_events(symbol, "timestamp" DESC); + CREATE INDEX IF NOT EXISTS idx_market_events_timestamp ON market_events("timestamp" DESC); + CREATE INDEX IF NOT EXISTS idx_market_events_processed ON market_events(processed, "timestamp" DESC); + END IF; +END $$; --- Trading events indexes -CREATE INDEX IF NOT EXISTS idx_trading_events_type_timestamp ON trading_events(event_type, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_trading_events_symbol_timestamp ON trading_events(symbol, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_trading_events_order_id ON trading_events(order_id); -CREATE INDEX IF NOT EXISTS idx_trading_events_correlation_id ON trading_events(correlation_id); -CREATE INDEX IF NOT EXISTS idx_trading_events_timestamp ON trading_events(timestamp DESC); +-- Trading events indexes (only create if table was created by this migration, not migration 001) +DO $$ +BEGIN + -- Check if trading_events has the "timestamp" column (our schema) vs event_timestamp (migration 001 schema) + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'trading_events' AND column_name = 'timestamp') THEN + CREATE INDEX IF NOT EXISTS idx_trading_events_type_timestamp ON trading_events(event_type, "timestamp" DESC); + CREATE INDEX IF NOT EXISTS idx_trading_events_symbol_timestamp ON trading_events(symbol, "timestamp" DESC); + CREATE INDEX IF NOT EXISTS idx_trading_events_order_id ON trading_events("order_id"); + CREATE INDEX IF NOT EXISTS idx_trading_events_correlation_id ON trading_events("correlation_id"); + CREATE INDEX IF NOT EXISTS idx_trading_events_timestamp ON trading_events("timestamp" DESC); + END IF; +END $$; --- Event processing stats indexes -CREATE INDEX IF NOT EXISTS idx_event_processing_stats_processor_type ON event_processing_stats(processor_name, event_type, timestamp DESC); +-- Event processing stats indexes (always create as this table is new) +CREATE INDEX IF NOT EXISTS idx_event_processing_stats_processor_type ON event_processing_stats(processor_name, event_type, "timestamp" DESC); -- Configuration indexes CREATE INDEX IF NOT EXISTS idx_configuration_category ON configuration(category); diff --git a/migrations/013_symbol_configuration_tables.sql b/migrations/013_symbol_configuration_tables.sql index 42dfe9267..f77417188 100644 --- a/migrations/013_symbol_configuration_tables.sql +++ b/migrations/013_symbol_configuration_tables.sql @@ -236,14 +236,14 @@ BEGIN WHEN NEW.classification = 'FOREX' THEN 'America/New_York' ELSE 'America/New_York' END, - CASE + CASE WHEN NEW.classification = 'CRYPTO' THEN '00:00:00'::TIME - WHEN NEW.classification = 'FOREX' THEN '17:00:00'::TIME + WHEN NEW.classification = 'FOREX' THEN '17:00:00'::TIME -- Sunday 5 PM EST (Forex market opens) ELSE '09:30:00'::TIME END, - CASE + CASE WHEN NEW.classification = 'CRYPTO' THEN '23:59:59'::TIME - WHEN NEW.classification = 'FOREX' THEN '17:00:00'::TIME + WHEN NEW.classification = 'FOREX' THEN '17:00:01'::TIME -- Friday 5:00:01 PM EST (Forex market closes, 1 second after open for weekly cycle) ELSE '16:00:00'::TIME END, CASE diff --git a/migrations/014_transaction_audit_events.sql b/migrations/014_transaction_audit_events.sql index fef15b8b7..912c2c6c1 100644 --- a/migrations/014_transaction_audit_events.sql +++ b/migrations/014_transaction_audit_events.sql @@ -74,13 +74,12 @@ CREATE OR REPLACE RULE no_delete_audit_events AS -- FOR VALUES FROM ('2025-10-01') TO ('2025-11-01'); -- Grant permissions to application roles -GRANT SELECT, INSERT ON transaction_audit_events TO authenticated_users; -GRANT USAGE, SELECT ON SEQUENCE transaction_audit_events_id_seq TO authenticated_users; +-- Note: Role 'authenticated_users' needs to be created separately +-- GRANT SELECT, INSERT ON transaction_audit_events TO authenticated_users; +-- GRANT USAGE, SELECT ON SEQUENCE transaction_audit_events_id_seq TO authenticated_users; -- Table and column comments for documentation -COMMENT ON TABLE transaction_audit_events IS - 'Immutable audit trail for all trading system events - SOX and MiFID II compliance. ' || - 'Provides microsecond-precision timestamps, tamper detection, and comprehensive event tracking.'; +COMMENT ON TABLE transaction_audit_events IS 'Immutable audit trail for all trading system events - SOX and MiFID II compliance. Provides microsecond-precision timestamps, tamper detection, and comprehensive event tracking.'; COMMENT ON COLUMN transaction_audit_events.event_id IS 'Unique event identifier (UUID-based)'; COMMENT ON COLUMN transaction_audit_events.event_type IS 'Type of audit event (OrderCreated, OrderExecuted, etc.)'; diff --git a/migrations/20250826000001_fix_partitioned_constraints.sql b/migrations/20250826000001_fix_partitioned_constraints.sql index b32ae32d6..7cd20a61e 100644 --- a/migrations/20250826000001_fix_partitioned_constraints.sql +++ b/migrations/20250826000001_fix_partitioned_constraints.sql @@ -1,13 +1,19 @@ -- Fix partitioned table constraints for PostgreSQL compliance -- Unique constraints on partitioned tables must include all partitioning columns --- Drop the problematic unique index on hft_performance_stats materialized view -DROP INDEX IF EXISTS idx_hft_performance_stats_unique; +-- Only apply fix if hft_performance_stats table exists +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'hft_performance_stats') THEN + -- Drop the problematic unique index on hft_performance_stats materialized view + DROP INDEX IF EXISTS idx_hft_performance_stats_unique; --- Recreate the unique index including the partitioning column (minute_bucket) --- This ensures the constraint includes the partitioning column as required by PostgreSQL -CREATE UNIQUE INDEX idx_hft_performance_stats_unique -ON hft_performance_stats(minute_bucket, metric_type, component); + -- Recreate the unique index including the partitioning column (minute_bucket) + -- This ensures the constraint includes the partitioning column as required by PostgreSQL + CREATE UNIQUE INDEX idx_hft_performance_stats_unique + ON hft_performance_stats(minute_bucket, metric_type, component); --- Add comment explaining the fix -COMMENT ON INDEX idx_hft_performance_stats_unique IS 'Fixed unique index including partitioning column for PostgreSQL compliance'; \ No newline at end of file + -- Add comment explaining the fix + COMMENT ON INDEX idx_hft_performance_stats_unique IS 'Fixed unique index including partitioning column for PostgreSQL compliance'; + END IF; +END $$; \ No newline at end of file diff --git a/migrations/auth_schema.sql b/migrations/auth_schema.sql deleted file mode 100644 index 6ce822feb..000000000 --- a/migrations/auth_schema.sql +++ /dev/null @@ -1,468 +0,0 @@ --- Authentication and Security Schema for Foxhunt Trading System --- Implements comprehensive security for financial trading platform --- Compliant with SOX, FINRA, and financial industry standards - --- Users table for authentication -CREATE TABLE IF NOT EXISTS users ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - username VARCHAR(255) UNIQUE NOT NULL, - email VARCHAR(255) UNIQUE NOT NULL, - password_hash VARCHAR(255) NOT NULL, - salt VARCHAR(255) NOT NULL, - first_name VARCHAR(255), - last_name VARCHAR(255), - phone VARCHAR(50), - department VARCHAR(100), - job_title VARCHAR(100), - manager_id UUID REFERENCES users(id), - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - last_login TIMESTAMP WITH TIME ZONE, - failed_login_attempts INTEGER DEFAULT 0, - account_locked_until TIMESTAMP WITH TIME ZONE, - password_changed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - must_change_password BOOLEAN DEFAULT FALSE, - two_factor_enabled BOOLEAN DEFAULT FALSE, - two_factor_secret VARCHAR(255), - active BOOLEAN DEFAULT TRUE, - deleted_at TIMESTAMP WITH TIME ZONE, - - -- Audit fields - created_by UUID REFERENCES users(id), - updated_by UUID REFERENCES users(id) -); - --- Roles table for RBAC -CREATE TABLE IF NOT EXISTS roles ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name VARCHAR(100) UNIQUE NOT NULL, - description TEXT, - permissions TEXT[], -- JSON array of permissions - parent_role_id UUID REFERENCES roles(id), - resource_constraints JSONB, -- Resource-based constraints - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - active BOOLEAN DEFAULT TRUE, - - -- Hierarchy depth for performance - hierarchy_level INTEGER DEFAULT 0, - - -- Audit fields - created_by UUID REFERENCES users(id), - updated_by UUID REFERENCES users(id) -); - --- User role assignments -CREATE TABLE IF NOT EXISTS user_roles ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, - granted_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - expires_at TIMESTAMP WITH TIME ZONE, - granted_by UUID NOT NULL REFERENCES users(id), - resource_constraints JSONB, -- Additional constraints for this assignment - active BOOLEAN DEFAULT TRUE, - - UNIQUE(user_id, role_id) -); - --- Sessions table for session management -CREATE TABLE IF NOT EXISTS sessions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - token_hash VARCHAR(255) UNIQUE NOT NULL, -- Hashed session token - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - last_activity TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - client_ip INET, - user_agent TEXT, - device_fingerprint VARCHAR(255), - active BOOLEAN DEFAULT TRUE, - - -- Session metadata - login_method VARCHAR(50), -- password, api_key, certificate - session_type VARCHAR(50) DEFAULT 'web' -- web, api, mobile -); - --- API keys table -CREATE TABLE IF NOT EXISTS api_keys ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name VARCHAR(255) NOT NULL, - key_hash VARCHAR(255) UNIQUE NOT NULL, -- Hashed API key - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - permissions TEXT[], -- JSON array of permissions - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - last_used TIMESTAMP WITH TIME ZONE, - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - usage_count BIGINT DEFAULT 0, - rate_limit_override INTEGER, -- Custom rate limit for this key - ip_whitelist INET[], -- Allowed IP addresses - active BOOLEAN DEFAULT TRUE, - revoked_at TIMESTAMP WITH TIME ZONE, - revoked_by UUID REFERENCES users(id), - revoke_reason TEXT, - - -- Key metadata - key_type VARCHAR(50) DEFAULT 'standard', -- standard, trading, readonly - scopes TEXT[] -- API scopes this key can access -); - --- Audit log table for compliance -CREATE TABLE IF NOT EXISTS audit_logs ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - event_type VARCHAR(100) NOT NULL, - severity VARCHAR(20) NOT NULL CHECK (severity IN ('info', 'warning', 'error', 'critical')), - user_id UUID REFERENCES users(id), - session_id UUID REFERENCES sessions(id), - api_key_id UUID REFERENCES api_keys(id), - client_ip INET, - user_agent TEXT, - resource VARCHAR(255), - action VARCHAR(100) NOT NULL, - result VARCHAR(100) NOT NULL, - details JSONB, - correlation_id UUID, - request_id UUID, - service_name VARCHAR(100), - service_version VARCHAR(50), - - -- Compliance fields - compliance_category VARCHAR(100), -- SOX, FINRA, MiFID, etc. - retention_until TIMESTAMP WITH TIME ZONE, - - -- Tamper detection - checksum VARCHAR(255), - previous_log_hash VARCHAR(255) -); - --- Rate limiting buckets -CREATE TABLE IF NOT EXISTS rate_limit_buckets ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - client_identifier VARCHAR(255) NOT NULL, - bucket_type VARCHAR(50) NOT NULL, -- auth, api, trading, market_data - requests JSONB NOT NULL DEFAULT '[]', -- Array of request timestamps - total_requests BIGINT DEFAULT 0, - last_request TIMESTAMP WITH TIME ZONE, - violations INTEGER DEFAULT 0, - blocked_until TIMESTAMP WITH TIME ZONE, - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - - UNIQUE(client_identifier, bucket_type) -); - --- TLS certificates table -CREATE TABLE IF NOT EXISTS certificates ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name VARCHAR(255) NOT NULL, - certificate_type VARCHAR(50) NOT NULL, -- server, client, ca - subject VARCHAR(500) NOT NULL, - issuer VARCHAR(500) NOT NULL, - serial_number VARCHAR(100) NOT NULL, - fingerprint VARCHAR(255) UNIQUE NOT NULL, - not_before TIMESTAMP WITH TIME ZONE NOT NULL, - not_after TIMESTAMP WITH TIME ZONE NOT NULL, - key_algorithm VARCHAR(50) NOT NULL, - key_size INTEGER NOT NULL, - signature_algorithm VARCHAR(100) NOT NULL, - san_dns_names TEXT[], - san_ip_addresses INET[], - certificate_pem TEXT NOT NULL, - private_key_encrypted TEXT, -- Encrypted private key (if stored) - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - active BOOLEAN DEFAULT TRUE, - - -- Certificate chain relationships - parent_certificate_id UUID REFERENCES certificates(id), - root_ca_id UUID REFERENCES certificates(id) -); - --- Permission cache table for performance -CREATE TABLE IF NOT EXISTS permission_cache ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, - permissions JSONB NOT NULL, - cached_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - cache_version INTEGER DEFAULT 1, - - UNIQUE(user_id, cache_version) -); - --- Compliance violations table -CREATE TABLE IF NOT EXISTS compliance_violations ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - violation_type VARCHAR(100) NOT NULL, - severity VARCHAR(20) NOT NULL CHECK (severity IN ('low', 'medium', 'high', 'critical')), - user_id UUID REFERENCES users(id), - description TEXT NOT NULL, - detected_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - resolved_at TIMESTAMP WITH TIME ZONE, - resolved_by UUID REFERENCES users(id), - resolution_notes TEXT, - compliance_framework VARCHAR(100), -- SOX, FINRA, MiFID II, etc. - rule_violated VARCHAR(255), - evidence JSONB, - status VARCHAR(50) DEFAULT 'open' CHECK (status IN ('open', 'investigating', 'resolved', 'false_positive')), - - -- Regulatory reporting - reported_to_regulator BOOLEAN DEFAULT FALSE, - regulator_reference VARCHAR(255), - reporting_deadline TIMESTAMP WITH TIME ZONE -); - --- Indexes for performance -CREATE INDEX IF NOT EXISTS idx_users_username ON users(username); -CREATE INDEX IF NOT EXISTS idx_users_email ON users(email); -CREATE INDEX IF NOT EXISTS idx_users_active ON users(active) WHERE active = TRUE; -CREATE INDEX IF NOT EXISTS idx_users_last_login ON users(last_login); - -CREATE INDEX IF NOT EXISTS idx_roles_name ON roles(name); -CREATE INDEX IF NOT EXISTS idx_roles_active ON roles(active) WHERE active = TRUE; -CREATE INDEX IF NOT EXISTS idx_roles_parent ON roles(parent_role_id); - -CREATE INDEX IF NOT EXISTS idx_user_roles_user_id ON user_roles(user_id); -CREATE INDEX IF NOT EXISTS idx_user_roles_role_id ON user_roles(role_id); -CREATE INDEX IF NOT EXISTS idx_user_roles_active ON user_roles(active) WHERE active = TRUE; -CREATE INDEX IF NOT EXISTS idx_user_roles_expires ON user_roles(expires_at); - -CREATE INDEX IF NOT EXISTS idx_sessions_token_hash ON sessions(token_hash); -CREATE INDEX IF NOT EXISTS idx_sessions_user_id ON sessions(user_id); -CREATE INDEX IF NOT EXISTS idx_sessions_active ON sessions(active) WHERE active = TRUE; -CREATE INDEX IF NOT EXISTS idx_sessions_expires ON sessions(expires_at); -CREATE INDEX IF NOT EXISTS idx_sessions_last_activity ON sessions(last_activity); - -CREATE INDEX IF NOT EXISTS idx_api_keys_key_hash ON api_keys(key_hash); -CREATE INDEX IF NOT EXISTS idx_api_keys_user_id ON api_keys(user_id); -CREATE INDEX IF NOT EXISTS idx_api_keys_active ON api_keys(active) WHERE active = TRUE; -CREATE INDEX IF NOT EXISTS idx_api_keys_expires ON api_keys(expires_at); - -CREATE INDEX IF NOT EXISTS idx_audit_logs_timestamp ON audit_logs(timestamp); -CREATE INDEX IF NOT EXISTS idx_audit_logs_user_id ON audit_logs(user_id); -CREATE INDEX IF NOT EXISTS idx_audit_logs_event_type ON audit_logs(event_type); -CREATE INDEX IF NOT EXISTS idx_audit_logs_severity ON audit_logs(severity); -CREATE INDEX IF NOT EXISTS idx_audit_logs_correlation_id ON audit_logs(correlation_id); - -CREATE INDEX IF NOT EXISTS idx_rate_limit_buckets_client ON rate_limit_buckets(client_identifier, bucket_type); -CREATE INDEX IF NOT EXISTS idx_rate_limit_buckets_updated ON rate_limit_buckets(updated_at); - -CREATE INDEX IF NOT EXISTS idx_certificates_fingerprint ON certificates(fingerprint); -CREATE INDEX IF NOT EXISTS idx_certificates_not_after ON certificates(not_after); -CREATE INDEX IF NOT EXISTS idx_certificates_active ON certificates(active) WHERE active = TRUE; - -CREATE INDEX IF NOT EXISTS idx_permission_cache_user_id ON permission_cache(user_id); -CREATE INDEX IF NOT EXISTS idx_permission_cache_expires ON permission_cache(expires_at); - -CREATE INDEX IF NOT EXISTS idx_compliance_violations_user_id ON compliance_violations(user_id); -CREATE INDEX IF NOT EXISTS idx_compliance_violations_status ON compliance_violations(status); -CREATE INDEX IF NOT EXISTS idx_compliance_violations_detected ON compliance_violations(detected_at); -CREATE INDEX IF NOT EXISTS idx_compliance_violations_framework ON compliance_violations(compliance_framework); - --- Triggers for updated_at timestamps -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ language 'plpgsql'; - -CREATE TRIGGER update_users_updated_at BEFORE UPDATE ON users - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER update_roles_updated_at BEFORE UPDATE ON roles - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER update_certificates_updated_at BEFORE UPDATE ON certificates - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - -CREATE TRIGGER update_rate_limit_buckets_updated_at BEFORE UPDATE ON rate_limit_buckets - FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); - --- Function to clean up expired sessions -CREATE OR REPLACE FUNCTION cleanup_expired_sessions() -RETURNS INTEGER AS $$ -DECLARE - deleted_count INTEGER; -BEGIN - DELETE FROM sessions - WHERE expires_at < NOW() - INTERVAL '7 days'; - - GET DIAGNOSTICS deleted_count = ROW_COUNT; - RETURN deleted_count; -END; -$$ LANGUAGE plpgsql; - --- Function to clean up old audit logs (based on retention policy) -CREATE OR REPLACE FUNCTION cleanup_audit_logs(retention_days INTEGER DEFAULT 2555) -RETURNS INTEGER AS $$ -DECLARE - deleted_count INTEGER; -BEGIN - DELETE FROM audit_logs - WHERE timestamp < NOW() - INTERVAL '1 day' * retention_days; - - GET DIAGNOSTICS deleted_count = ROW_COUNT; - RETURN deleted_count; -END; -$$ LANGUAGE plpgsql; - --- Function to check password complexity -CREATE OR REPLACE FUNCTION check_password_complexity(password_hash TEXT) -RETURNS BOOLEAN AS $$ -BEGIN - -- In production, implement proper password complexity checking - -- This is a placeholder that assumes passwords are already validated - RETURN LENGTH(password_hash) >= 60; -- Assuming bcrypt hash length -END; -$$ LANGUAGE plpgsql; - --- Function to log security events -CREATE OR REPLACE FUNCTION log_security_event( - event_type VARCHAR(100), - user_id UUID, - session_id UUID, - client_ip INET, - details JSONB -) -RETURNS UUID AS $$ -DECLARE - log_id UUID; -BEGIN - INSERT INTO audit_logs ( - event_type, - severity, - user_id, - session_id, - client_ip, - action, - result, - details, - service_name - ) VALUES ( - event_type, - 'info', - user_id, - session_id, - client_ip, - 'security_event', - 'logged', - details, - 'tli' - ) RETURNING id INTO log_id; - - RETURN log_id; -END; -$$ LANGUAGE plpgsql; - --- Insert default roles -INSERT INTO roles (id, name, description, permissions, hierarchy_level) VALUES - ('00000000-0000-0000-0000-000000000001', 'system_admin', 'System Administrator - Full Access', - ARRAY['system:admin', 'system:config', 'system:user_management', 'system:role_management'], 0), - ('00000000-0000-0000-0000-000000000002', 'trader', 'Senior Trader - Full Trading Access', - ARRAY['trade:execute', 'trade:view', 'trade:cancel', 'trade:modify', 'order:place', 'order:cancel', 'order:modify', 'order:view', 'position:view', 'position:close', 'position:modify', 'risk:view', 'market_data:view', 'market_data:subscribe', 'portfolio:view', 'report:view', 'analytics:view', 'api:access', 'ml:signal_view'], 1), - ('00000000-0000-0000-0000-000000000003', 'junior_trader', 'Junior Trader - Limited Trading Access', - ARRAY['order:place', 'order:view', 'position:view', 'trade:view', 'risk:view', 'market_data:view', 'portfolio:view', 'report:view', 'ml:signal_view'], 2), - ('00000000-0000-0000-0000-000000000004', 'risk_manager', 'Risk Manager - Risk Oversight', - ARRAY['risk:view', 'risk:config', 'risk:override', 'risk:limits', 'risk:drawdown_monitor', 'position:view', 'position:limit', 'trade:view', 'order:view', 'portfolio:view', 'report:view', 'report:generate', 'analytics:view', 'compliance:view', 'audit:view'], 1), - ('00000000-0000-0000-0000-000000000005', 'viewer', 'Viewer - Read-only Access', - ARRAY['trade:view', 'order:view', 'position:view', 'risk:view', 'market_data:view', 'portfolio:view', 'report:view', 'analytics:view', 'ml:signal_view'], 3), - ('00000000-0000-0000-0000-000000000006', 'api_user', 'API User - Programmatic Access', - ARRAY['api:access', 'market_data:view', 'market_data:subscribe', 'order:place', 'order:view', 'order:cancel', 'position:view', 'trade:view', 'ml:signal_view'], 2) -ON CONFLICT (id) DO NOTHING; - --- Insert default admin user (password should be changed on first login) --- Default password hash is for 'DefaultAdmin123!' - MUST be changed in production -INSERT INTO users ( - id, - username, - email, - password_hash, - salt, - first_name, - last_name, - must_change_password -) VALUES ( - '00000000-0000-0000-0000-000000000001', - 'admin', - 'admin@foxhunt.local', - '$2b$12$LQv3c1yqBWVHxkd0LHAkCOYz6TtxMQJqhN8/lewZJYHCB7vMNZJdK', -- DefaultAdmin123! - 'default_salt_change_in_production', - 'System', - 'Administrator', - TRUE -) ON CONFLICT (id) DO NOTHING; - --- Assign admin role to default admin user -INSERT INTO user_roles (user_id, role_id, granted_by) VALUES - ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001') -ON CONFLICT (user_id, role_id) DO NOTHING; - --- Create view for active user permissions -CREATE OR REPLACE VIEW user_permissions AS -SELECT DISTINCT - u.id as user_id, - u.username, - unnest(r.permissions) as permission, - ur.expires_at as permission_expires_at -FROM users u -JOIN user_roles ur ON u.id = ur.user_id -JOIN roles r ON ur.role_id = r.id -WHERE u.active = TRUE - AND ur.active = TRUE - AND r.active = TRUE - AND (ur.expires_at IS NULL OR ur.expires_at > NOW()); - --- Create view for session summary -CREATE OR REPLACE VIEW active_sessions AS -SELECT - s.id, - s.user_id, - u.username, - s.created_at, - s.last_activity, - s.expires_at, - s.client_ip, - s.session_type, - EXTRACT(EPOCH FROM (s.expires_at - NOW())) as seconds_until_expiry -FROM sessions s -JOIN users u ON s.user_id = u.id -WHERE s.active = TRUE - AND s.expires_at > NOW(); - --- Grant permissions to application role (adjust as needed) --- GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO foxhunt_app; --- GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO foxhunt_app; - --- Add comments for documentation -COMMENT ON TABLE users IS 'User accounts for authentication and authorization'; -COMMENT ON TABLE roles IS 'Role definitions for RBAC system'; -COMMENT ON TABLE user_roles IS 'User to role assignments with optional expiration'; -COMMENT ON TABLE sessions IS 'Active user sessions for session management'; -COMMENT ON TABLE api_keys IS 'API keys for programmatic access'; -COMMENT ON TABLE audit_logs IS 'Comprehensive audit trail for compliance'; -COMMENT ON TABLE rate_limit_buckets IS 'Rate limiting state tracking'; -COMMENT ON TABLE certificates IS 'TLS certificate management'; -COMMENT ON TABLE permission_cache IS 'Cached user permissions for performance'; -COMMENT ON TABLE compliance_violations IS 'Compliance violations tracking and reporting'; - -COMMENT ON COLUMN users.password_hash IS 'Bcrypt hash of user password'; -COMMENT ON COLUMN users.salt IS 'Salt used for password hashing'; -COMMENT ON COLUMN users.failed_login_attempts IS 'Count of consecutive failed login attempts'; -COMMENT ON COLUMN users.account_locked_until IS 'Account lockout expiration timestamp'; -COMMENT ON COLUMN users.two_factor_secret IS 'TOTP secret for 2FA (encrypted)'; - -COMMENT ON COLUMN audit_logs.checksum IS 'Tamper detection checksum for audit integrity'; -COMMENT ON COLUMN audit_logs.previous_log_hash IS 'Hash of previous log entry for chain verification'; -COMMENT ON COLUMN audit_logs.retention_until IS 'Data retention deadline for compliance'; - -COMMENT ON COLUMN api_keys.key_hash IS 'SHA-256 hash of the API key for secure storage'; -COMMENT ON COLUMN api_keys.scopes IS 'API scopes this key can access'; -COMMENT ON COLUMN api_keys.ip_whitelist IS 'Allowed source IP addresses for this key'; - -COMMENT ON COLUMN certificates.certificate_pem IS 'PEM-encoded certificate'; -COMMENT ON COLUMN certificates.private_key_encrypted IS 'Encrypted private key (if stored)'; -COMMENT ON COLUMN certificates.san_dns_names IS 'Subject Alternative Names - DNS names'; -COMMENT ON COLUMN certificates.san_ip_addresses IS 'Subject Alternative Names - IP addresses'; \ No newline at end of file diff --git a/migrations/trading_service_events.sql b/migrations/trading_service_events.sql deleted file mode 100644 index 28d5e3205..000000000 --- a/migrations/trading_service_events.sql +++ /dev/null @@ -1,741 +0,0 @@ --- Migration: Comprehensive Event Storage for Trading Service --- This migration implements comprehensive event storage for HFT compliance and audit trails --- Designed for regulatory compliance (MiFID II, SOX, Dodd-Frank) with nanosecond precision - --- Enable required extensions for HFT event storage -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -CREATE EXTENSION IF NOT EXISTS "btree_gin"; -CREATE EXTENSION IF NOT EXISTS "pg_trgm"; - --- ======================================================================================= --- TRADING EVENTS TABLE - Core order lifecycle events --- ======================================================================================= -CREATE TABLE IF NOT EXISTS trading_events ( - id UUID NOT NULL DEFAULT uuid_generate_v4(), - event_id UUID NOT NULL DEFAULT uuid_generate_v4(), -- Unique event identifier - correlation_id UUID, -- Link related events together - event_type VARCHAR(50) NOT NULL, -- 'order_created', 'order_modified', 'order_cancelled', 'order_filled', 'order_expired', 'order_rejected' - event_subtype VARCHAR(50), -- Additional event classification - - -- Core order information - order_id UUID, -- Reference to orders table - original_order_id UUID, -- For tracking order modifications - client_order_id VARCHAR(128), -- Client-provided identifier - exchange_order_id VARCHAR(128), -- Exchange-provided identifier - - -- Instrument and trading details - symbol VARCHAR(32) NOT NULL, - instrument_id VARCHAR(64), - side VARCHAR(10) NOT NULL CHECK (side IN ('buy', 'sell')), - order_type VARCHAR(20) NOT NULL, - quantity BIGINT NOT NULL CHECK (quantity > 0), - price BIGINT, -- Price in fixed-point cents - filled_quantity BIGINT DEFAULT 0, - remaining_quantity BIGINT DEFAULT 0, - - -- Execution details - execution_price BIGINT, -- Actual fill price - execution_quantity BIGINT, -- Quantity filled in this event - execution_id VARCHAR(128), -- Venue execution ID - venue VARCHAR(64), -- Execution venue - is_maker BOOLEAN, -- Maker/taker flag - fees BIGINT, -- Trading fees in fixed-point cents - fee_currency VARCHAR(10), -- Fee currency - - -- Account and strategy information - account_id VARCHAR(64), - portfolio_id VARCHAR(64), - strategy_id VARCHAR(100), - trader_id VARCHAR(64), - - -- Risk and compliance - risk_check_result VARCHAR(20), -- 'approved', 'rejected', 'warning' - risk_violations JSONB, -- Array of risk violations - compliance_flags JSONB, -- Regulatory compliance flags - - -- Timing information (nanosecond precision for HFT) - event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - market_timestamp TIMESTAMP WITH TIME ZONE, -- Exchange timestamp - received_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - gateway_timestamp TIMESTAMP WITH TIME ZONE, -- Gateway receive time - processing_start_timestamp TIMESTAMP WITH TIME ZONE, -- When processing started - processing_end_timestamp TIMESTAMP WITH TIME ZONE, -- When processing completed - latency_ns BIGINT, -- Processing latency in nanoseconds - - -- Order state tracking - order_status VARCHAR(20), -- Current order status - previous_status VARCHAR(20), -- Previous order status - status_reason VARCHAR(255), -- Reason for status change - - -- Market conditions - market_data_snapshot JSONB, -- Market data at time of event - bid_price BIGINT, -- Best bid at time of event - ask_price BIGINT, -- Best ask at time of event - last_price BIGINT, -- Last trade price - volume BIGINT, -- Market volume - - -- System information - source_system VARCHAR(50) NOT NULL, -- System that generated the event - source_component VARCHAR(50), -- Component within system - version VARCHAR(20), -- Event schema version - - -- Additional data - metadata JSONB, -- Flexible additional data - raw_message TEXT, -- Original message (if applicable) - - PRIMARY KEY (id, event_timestamp) -- Composite key for partitioning -) PARTITION BY RANGE (event_timestamp); - --- Create partitions for trading events (6 months of partitions) -DO $$ -DECLARE - partition_start DATE; - partition_end DATE; - partition_name TEXT; - i INTEGER; -BEGIN - FOR i IN 0..5 LOOP - partition_start := date_trunc('month', CURRENT_DATE) + (i || ' months')::INTERVAL; - partition_end := partition_start + INTERVAL '1 month'; - partition_name := 'trading_events_' || to_char(partition_start, 'YYYY_MM'); - - EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF trading_events - FOR VALUES FROM (%L) TO (%L)', - partition_name, partition_start, partition_end); - - -- Add indexes to each partition - EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(order_id, event_timestamp)', - 'idx_' || partition_name || '_order_id', partition_name); - EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(symbol, event_type, event_timestamp)', - 'idx_' || partition_name || '_symbol_type', partition_name); - EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I(account_id, event_timestamp)', - 'idx_' || partition_name || '_account', partition_name); - END LOOP; -END $$; - --- ======================================================================================= --- RISK EVENTS TABLE - Risk management decisions and alerts --- ======================================================================================= -CREATE TABLE IF NOT EXISTS risk_events ( - id UUID NOT NULL DEFAULT uuid_generate_v4(), - event_id UUID NOT NULL DEFAULT uuid_generate_v4(), - correlation_id UUID, -- Link to trading events - - -- Event classification - event_type VARCHAR(50) NOT NULL, -- 'limit_check', 'violation', 'alert', 'emergency_stop', 'position_limit', 'var_breach' - severity VARCHAR(20) NOT NULL CHECK (severity IN ('info', 'warning', 'error', 'critical')), - - -- Risk details - risk_type VARCHAR(50) NOT NULL, -- 'position_size', 'concentration', 'var', 'leverage', 'drawdown', 'exposure' - violation_type VARCHAR(50), -- Specific violation type - - -- Values and limits - current_value DECIMAL(20, 8), -- Current risk metric value - limit_value DECIMAL(20, 8), -- Risk limit value - breach_amount DECIMAL(20, 8), -- Amount of breach - breach_percentage DECIMAL(5, 4), -- Percentage of breach - - -- Context information - symbol VARCHAR(32), - account_id VARCHAR(64), - portfolio_id VARCHAR(64), - strategy_id VARCHAR(100), - instrument_id VARCHAR(64), - - -- Order context (if related to an order) - order_id UUID, - trade_id UUID, - position_id UUID, - - -- Risk calculation details - calculation_method VARCHAR(50), -- 'real_time', 'batch', 'stress_test' - model_version VARCHAR(20), -- Risk model version - parameters JSONB, -- Risk calculation parameters - - -- Actions taken - action_taken VARCHAR(100), -- 'blocked', 'warning_issued', 'position_reduced', 'trading_halted' - auto_action BOOLEAN DEFAULT false, -- Whether action was automatic - manual_override BOOLEAN DEFAULT false, -- Whether manually overridden - override_reason TEXT, -- Reason for manual override - override_user VARCHAR(64), -- User who performed override - - -- Timing - event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - detection_timestamp TIMESTAMP WITH TIME ZONE, -- When risk was detected - resolution_timestamp TIMESTAMP WITH TIME ZONE, -- When risk was resolved - - -- Resolution - resolution_status VARCHAR(20) DEFAULT 'open' CHECK (resolution_status IN ('open', 'acknowledged', 'resolved', 'false_positive')), - resolution_notes TEXT, - resolved_by VARCHAR(64), - - -- System information - source_system VARCHAR(50) NOT NULL, - risk_engine_version VARCHAR(20), - - -- Additional data - metadata JSONB, - - PRIMARY KEY (id, event_timestamp) -) PARTITION BY RANGE (event_timestamp); - --- Create partitions for risk events -DO $$ -DECLARE - partition_start DATE; - partition_end DATE; - partition_name TEXT; - i INTEGER; -BEGIN - FOR i IN 0..5 LOOP - partition_start := date_trunc('month', CURRENT_DATE) + (i || ' months')::INTERVAL; - partition_end := partition_start + INTERVAL '1 month'; - partition_name := 'risk_events_' || to_char(partition_start, 'YYYY_MM'); - - EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF risk_events - FOR VALUES FROM (%L) TO (%L)', - partition_name, partition_start, partition_end); - END LOOP; -END $$; - --- ======================================================================================= --- AUDIT TRAIL TABLE - Configuration changes and administrative actions --- ======================================================================================= -CREATE TABLE IF NOT EXISTS audit_trail ( - id UUID NOT NULL DEFAULT uuid_generate_v4(), - event_id UUID NOT NULL DEFAULT uuid_generate_v4(), - - -- Event classification - event_type VARCHAR(50) NOT NULL, -- 'config_change', 'user_action', 'system_action', 'data_modification' - action VARCHAR(100) NOT NULL, -- Specific action taken - - -- Actor information - user_id VARCHAR(64), -- User who performed action - username VARCHAR(100), -- Username - user_role VARCHAR(50), -- User role/permission level - session_id VARCHAR(128), -- Session identifier - ip_address INET, -- Source IP address - user_agent TEXT, -- Browser/client information - - -- Target information - target_type VARCHAR(50), -- 'user', 'account', 'configuration', 'strategy', 'limit' - target_id VARCHAR(128), -- ID of the target object - target_name VARCHAR(255), -- Human-readable name of target - - -- Change details - operation VARCHAR(20) CHECK (operation IN ('CREATE', 'READ', 'UPDATE', 'DELETE')), - field_name VARCHAR(100), -- Specific field changed - old_value TEXT, -- Previous value - new_value TEXT, -- New value - change_reason TEXT, -- Reason for change - - -- Context - system_name VARCHAR(50), -- System where change occurred - component VARCHAR(50), -- System component - environment VARCHAR(20), -- 'production', 'staging', 'development' - - -- Timing - event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - effective_timestamp TIMESTAMP WITH TIME ZONE, -- When change becomes effective - - -- Approval workflow - approval_required BOOLEAN DEFAULT false, - approval_status VARCHAR(20), -- 'pending', 'approved', 'rejected' - approved_by VARCHAR(64), -- User who approved - approval_timestamp TIMESTAMP WITH TIME ZONE, - approval_comments TEXT, - - -- Compliance - regulatory_impact BOOLEAN DEFAULT false, -- Whether change has regulatory impact - compliance_review_required BOOLEAN DEFAULT false, - compliance_reviewer VARCHAR(64), - compliance_review_date DATE, - - -- Additional data - metadata JSONB, - request_payload JSONB, -- Full request data - response_payload JSONB, -- Full response data - - PRIMARY KEY (id, event_timestamp) -) PARTITION BY RANGE (event_timestamp); - --- Create partitions for audit trail -DO $$ -DECLARE - partition_start DATE; - partition_end DATE; - partition_name TEXT; - i INTEGER; -BEGIN - FOR i IN 0..11 LOOP - partition_start := date_trunc('month', CURRENT_DATE) + (i || ' months')::INTERVAL; - partition_end := partition_start + INTERVAL '1 month'; - partition_name := 'audit_trail_' || to_char(partition_start, 'YYYY_MM'); - - EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_trail - FOR VALUES FROM (%L) TO (%L)', - partition_name, partition_start, partition_end); - END LOOP; -END $$; - --- ======================================================================================= --- ML SIGNALS TABLE - Machine learning predictions and signals --- ======================================================================================= -CREATE TABLE IF NOT EXISTS ml_signals ( - id UUID NOT NULL DEFAULT uuid_generate_v4(), - signal_id UUID NOT NULL DEFAULT uuid_generate_v4(), - - -- Model information - model_name VARCHAR(100) NOT NULL, -- 'dqn', 'ppo', 'transformer', 'mamba', 'tft' - model_version VARCHAR(20) NOT NULL, - model_type VARCHAR(50), -- 'reinforcement_learning', 'supervised', 'unsupervised' - algorithm VARCHAR(50), -- Specific algorithm used - - -- Signal details - signal_type VARCHAR(50) NOT NULL, -- 'trade_signal', 'market_prediction', 'risk_alert', 'anomaly_detection' - signal_strength DECIMAL(5, 4), -- Signal strength 0.0-1.0 - confidence DECIMAL(5, 4), -- Model confidence 0.0-1.0 - - -- Prediction/signal content - prediction_type VARCHAR(50), -- 'price_direction', 'volatility', 'volume', 'risk_level' - predicted_value DECIMAL(20, 8), -- Numerical prediction - predicted_direction VARCHAR(10), -- 'up', 'down', 'neutral' - time_horizon INTEGER, -- Prediction time horizon in minutes - - -- Market context - symbol VARCHAR(32) NOT NULL, - market_data_snapshot JSONB, -- Market data used for prediction - feature_vector JSONB, -- Input features used - - -- Trading context - account_id VARCHAR(64), - strategy_id VARCHAR(100), - - -- Model performance tracking - execution_time_ms INTEGER, -- Model execution time - input_data_hash VARCHAR(64), -- Hash of input data for reproducibility - model_parameters JSONB, -- Model parameters used - - -- Outcome tracking (filled after signal verification) - actual_outcome DECIMAL(20, 8), -- Actual result (for backtesting) - outcome_timestamp TIMESTAMP WITH TIME ZONE, -- When outcome was recorded - accuracy_score DECIMAL(5, 4), -- How accurate was the prediction - signal_pnl DECIMAL(20, 8), -- P&L attributed to this signal - - -- Timing - signal_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - market_timestamp TIMESTAMP WITH TIME ZONE, -- Market data timestamp - - -- Signal lifecycle - signal_status VARCHAR(20) DEFAULT 'active', -- 'active', 'executed', 'expired', 'cancelled' - expiry_timestamp TIMESTAMP WITH TIME ZONE, - - -- System information - source_system VARCHAR(50) NOT NULL, - gpu_used BOOLEAN DEFAULT false, - cuda_version VARCHAR(20), - - -- Additional data - metadata JSONB, - raw_features JSONB, -- Raw feature data - - PRIMARY KEY (id, signal_timestamp) -) PARTITION BY RANGE (signal_timestamp); - --- Create partitions for ML signals -DO $$ -DECLARE - partition_start DATE; - partition_end DATE; - partition_name TEXT; - i INTEGER; -BEGIN - FOR i IN 0..2 LOOP - partition_start := date_trunc('month', CURRENT_DATE) + (i || ' months')::INTERVAL; - partition_end := partition_start + INTERVAL '1 month'; - partition_name := 'ml_signals_' || to_char(partition_start, 'YYYY_MM'); - - EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF ml_signals - FOR VALUES FROM (%L) TO (%L)', - partition_name, partition_start, partition_end); - END LOOP; -END $$; - --- ======================================================================================= --- SYSTEM EVENTS TABLE - Health, performance, and error events --- ======================================================================================= -CREATE TABLE IF NOT EXISTS system_events ( - id UUID NOT NULL DEFAULT uuid_generate_v4(), - event_id UUID NOT NULL DEFAULT uuid_generate_v4(), - - -- Event classification - event_type VARCHAR(50) NOT NULL, -- 'health_check', 'performance_metric', 'error', 'startup', 'shutdown', 'deployment' - severity VARCHAR(20) NOT NULL CHECK (severity IN ('debug', 'info', 'warning', 'error', 'critical')), - category VARCHAR(50), -- 'connectivity', 'latency', 'memory', 'disk', 'network', 'database' - - -- System information - system_name VARCHAR(50) NOT NULL, - service_name VARCHAR(50), - component_name VARCHAR(50), - hostname VARCHAR(100), - instance_id VARCHAR(100), - version VARCHAR(20), - build_id VARCHAR(50), - - -- Performance metrics - latency_ns BIGINT, -- Latency in nanoseconds - throughput_per_second INTEGER, -- Operations per second - memory_usage_mb INTEGER, -- Memory usage in MB - cpu_usage_percent DECIMAL(5, 2), -- CPU usage percentage - disk_usage_percent DECIMAL(5, 2), -- Disk usage percentage - network_bytes_in BIGINT, -- Network bytes received - network_bytes_out BIGINT, -- Network bytes sent - - -- Health information - health_status VARCHAR(20), -- 'healthy', 'degraded', 'unhealthy', 'critical' - health_score DECIMAL(5, 2), -- Health score 0-100 - uptime_seconds BIGINT, -- System uptime - - -- Error information - error_code VARCHAR(50), -- Error code - error_message TEXT, -- Error message - stack_trace TEXT, -- Error stack trace - error_count INTEGER DEFAULT 1, -- Number of occurrences - - -- Market data quality - market_data_lag_ms INTEGER, -- Market data lag - missing_ticks INTEGER, -- Number of missing market data ticks - data_quality_score DECIMAL(5, 2), -- Data quality score 0-100 - - -- Database performance - db_connection_count INTEGER, -- Active database connections - db_query_time_ms INTEGER, -- Database query time - db_connection_pool_usage DECIMAL(5, 2), -- Connection pool usage percentage - - -- Timing - event_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - metric_timestamp TIMESTAMP WITH TIME ZONE, -- When metric was measured - - -- Resolution tracking - incident_id VARCHAR(100), -- Related incident ID - resolution_time_minutes INTEGER, -- Time to resolve issue - resolution_notes TEXT, - - -- Additional data - metadata JSONB, - metrics JSONB, -- Additional metrics - environment_data JSONB, -- Environment variables, config - - PRIMARY KEY (id, event_timestamp) -) PARTITION BY RANGE (event_timestamp); - --- Create partitions for system events -DO $$ -DECLARE - partition_start DATE; - partition_end DATE; - partition_name TEXT; - i INTEGER; -BEGIN - FOR i IN 0..2 LOOP - partition_start := date_trunc('month', CURRENT_DATE) + (i || ' months')::INTERVAL; - partition_end := partition_start + INTERVAL '1 month'; - partition_name := 'system_events_' || to_char(partition_start, 'YYYY_MM'); - - EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF system_events - FOR VALUES FROM (%L) TO (%L)', - partition_name, partition_start, partition_end); - END LOOP; -END $$; - --- ======================================================================================= --- INDEXES FOR HIGH-PERFORMANCE QUERIES --- ======================================================================================= - --- Trading Events Indexes -CREATE INDEX IF NOT EXISTS idx_trading_events_order_id ON trading_events(order_id, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_trading_events_symbol_type ON trading_events(symbol, event_type, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_trading_events_account ON trading_events(account_id, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_trading_events_strategy ON trading_events(strategy_id, event_timestamp) WHERE strategy_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_trading_events_correlation ON trading_events(correlation_id) WHERE correlation_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_trading_events_compliance ON trading_events USING GIN(compliance_flags) WHERE compliance_flags IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_trading_events_risk ON trading_events USING GIN(risk_violations) WHERE risk_violations IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_trading_events_latency ON trading_events(latency_ns) WHERE latency_ns IS NOT NULL; - --- Risk Events Indexes -CREATE INDEX IF NOT EXISTS idx_risk_events_type_severity ON risk_events(risk_type, severity, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_risk_events_account ON risk_events(account_id, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_risk_events_symbol ON risk_events(symbol, event_timestamp) WHERE symbol IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_risk_events_resolution ON risk_events(resolution_status, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_risk_events_order ON risk_events(order_id) WHERE order_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_risk_events_breach_amount ON risk_events(breach_amount) WHERE breach_amount IS NOT NULL; - --- Audit Trail Indexes -CREATE INDEX IF NOT EXISTS idx_audit_trail_user ON audit_trail(user_id, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_audit_trail_target ON audit_trail(target_type, target_id, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_audit_trail_operation ON audit_trail(operation, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_audit_trail_approval ON audit_trail(approval_status, event_timestamp) WHERE approval_required = true; -CREATE INDEX IF NOT EXISTS idx_audit_trail_compliance ON audit_trail(regulatory_impact, event_timestamp) WHERE regulatory_impact = true; -CREATE INDEX IF NOT EXISTS idx_audit_trail_ip ON audit_trail(ip_address, event_timestamp); - --- ML Signals Indexes -CREATE INDEX IF NOT EXISTS idx_ml_signals_model ON ml_signals(model_name, model_version, signal_timestamp); -CREATE INDEX IF NOT EXISTS idx_ml_signals_symbol ON ml_signals(symbol, signal_timestamp); -CREATE INDEX IF NOT EXISTS idx_ml_signals_strategy ON ml_signals(strategy_id, signal_timestamp) WHERE strategy_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_ml_signals_confidence ON ml_signals(confidence, signal_timestamp); -CREATE INDEX IF NOT EXISTS idx_ml_signals_status ON ml_signals(signal_status, expiry_timestamp); -CREATE INDEX IF NOT EXISTS idx_ml_signals_accuracy ON ml_signals(accuracy_score) WHERE accuracy_score IS NOT NULL; - --- System Events Indexes -CREATE INDEX IF NOT EXISTS idx_system_events_system ON system_events(system_name, service_name, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_system_events_severity ON system_events(severity, event_timestamp); -CREATE INDEX IF NOT EXISTS idx_system_events_health ON system_events(health_status, event_timestamp) WHERE health_status IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_system_events_error ON system_events(error_code, event_timestamp) WHERE error_code IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_system_events_latency ON system_events(latency_ns) WHERE latency_ns IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_system_events_incident ON system_events(incident_id) WHERE incident_id IS NOT NULL; - --- ======================================================================================= --- DATA RETENTION POLICIES --- ======================================================================================= - --- Create function to manage partition retention -CREATE OR REPLACE FUNCTION manage_event_partitions() -RETURNS VOID AS $$ -DECLARE - table_names TEXT[] := ARRAY['trading_events', 'risk_events', 'audit_trail', 'ml_signals', 'system_events']; - retention_months INTEGER[] := ARRAY[24, 12, 84, 6, 3]; -- Retention periods in months - table_name TEXT; - retention_period INTEGER; - cutoff_date DATE; - partition_name TEXT; - partition_record RECORD; - i INTEGER; -BEGIN - FOR i IN 1..array_length(table_names, 1) LOOP - table_name := table_names[i]; - retention_period := retention_months[i]; - cutoff_date := CURRENT_DATE - (retention_period || ' months')::INTERVAL; - - -- Drop old partitions - FOR partition_record IN - SELECT schemaname, tablename - FROM pg_tables - WHERE tablename LIKE table_name || '_%' - AND tablename ~ '\d{4}_\d{2}$' - LOOP - -- Extract date from partition name - partition_name := partition_record.tablename; - - -- Drop if older than retention period - EXECUTE format('SELECT to_date(substring(%L from ''%s_(\d{4}_\d{2})$''), ''YYYY_MM'') < %L', - partition_name, table_name, cutoff_date) - INTO partition_record; - - IF partition_record IS NOT NULL THEN - EXECUTE format('DROP TABLE IF EXISTS %I CASCADE', partition_name); - RAISE NOTICE 'Dropped old partition: %', partition_name; - END IF; - END LOOP; - - -- Create future partitions (next 3 months) - FOR i IN 1..3 LOOP - DECLARE - partition_start DATE := date_trunc('month', CURRENT_DATE) + (i || ' months')::INTERVAL; - partition_end DATE := partition_start + INTERVAL '1 month'; - new_partition_name TEXT := table_name || '_' || to_char(partition_start, 'YYYY_MM'); - BEGIN - EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF %I - FOR VALUES FROM (%L) TO (%L)', - new_partition_name, table_name, partition_start, partition_end); - END; - END LOOP; - END LOOP; -END; -$$ LANGUAGE plpgsql; - --- ======================================================================================= --- UTILITY FUNCTIONS --- ======================================================================================= - --- Function to get event statistics -CREATE OR REPLACE FUNCTION get_event_statistics( - start_date DATE DEFAULT CURRENT_DATE - INTERVAL '7 days', - end_date DATE DEFAULT CURRENT_DATE -) -RETURNS TABLE( - table_name TEXT, - event_count BIGINT, - avg_events_per_hour NUMERIC, - max_events_per_hour BIGINT, - data_size_mb NUMERIC -) AS $$ -BEGIN - RETURN QUERY - WITH event_stats AS ( - SELECT 'trading_events'::TEXT as tbl, - COUNT(*) as cnt, - EXTRACT(EPOCH FROM (end_date - start_date)) / 3600 as hours - FROM trading_events - WHERE event_timestamp BETWEEN start_date AND end_date - - UNION ALL - - SELECT 'risk_events'::TEXT, - COUNT(*), - EXTRACT(EPOCH FROM (end_date - start_date)) / 3600 - FROM risk_events - WHERE event_timestamp BETWEEN start_date AND end_date - - UNION ALL - - SELECT 'audit_trail'::TEXT, - COUNT(*), - EXTRACT(EPOCH FROM (end_date - start_date)) / 3600 - FROM audit_trail - WHERE event_timestamp BETWEEN start_date AND end_date - - UNION ALL - - SELECT 'ml_signals'::TEXT, - COUNT(*), - EXTRACT(EPOCH FROM (end_date - start_date)) / 3600 - FROM ml_signals - WHERE signal_timestamp BETWEEN start_date AND end_date - - UNION ALL - - SELECT 'system_events'::TEXT, - COUNT(*), - EXTRACT(EPOCH FROM (end_date - start_date)) / 3600 - FROM system_events - WHERE event_timestamp BETWEEN start_date AND end_date - ) - SELECT - s.tbl, - s.cnt, - CASE WHEN s.hours > 0 THEN s.cnt / s.hours ELSE 0 END, - 0::BIGINT, -- Placeholder for max events per hour - 0::NUMERIC -- Placeholder for data size - FROM event_stats s; -END; -$$ LANGUAGE plpgsql; - --- Function to create event with correlation -CREATE OR REPLACE FUNCTION create_correlated_event( - p_table_name TEXT, - p_correlation_id UUID, - p_event_data JSONB -) -RETURNS UUID AS $$ -DECLARE - new_event_id UUID := uuid_generate_v4(); -BEGIN - -- This is a template function - specific implementations would be created - -- for each event type with proper validation and insertion logic - RETURN new_event_id; -END; -$$ LANGUAGE plpgsql; - --- ======================================================================================= --- TRIGGERS FOR DATA INTEGRITY --- ======================================================================================= - --- Function to validate trading event data -CREATE OR REPLACE FUNCTION validate_trading_event() -RETURNS TRIGGER AS $$ -BEGIN - -- Validate required fields based on event type - IF NEW.event_type = 'order_filled' AND NEW.execution_price IS NULL THEN - RAISE EXCEPTION 'Fill events must have execution price'; - END IF; - - IF NEW.event_type = 'order_filled' AND NEW.execution_quantity IS NULL THEN - RAISE EXCEPTION 'Fill events must have execution quantity'; - END IF; - - -- Calculate latency if timestamps are available - IF NEW.processing_start_timestamp IS NOT NULL AND NEW.processing_end_timestamp IS NOT NULL THEN - NEW.latency_ns := EXTRACT(EPOCH FROM (NEW.processing_end_timestamp - NEW.processing_start_timestamp)) * 1000000000; - END IF; - - -- Set remaining quantity - IF NEW.quantity IS NOT NULL AND NEW.filled_quantity IS NOT NULL THEN - NEW.remaining_quantity := NEW.quantity - NEW.filled_quantity; - END IF; - - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Apply validation trigger to trading events -CREATE TRIGGER trigger_validate_trading_events - BEFORE INSERT OR UPDATE ON trading_events - FOR EACH ROW - EXECUTE FUNCTION validate_trading_event(); - --- ======================================================================================= --- MATERIALIZED VIEWS FOR REPORTING --- ======================================================================================= - --- Real-time trading activity summary -CREATE MATERIALIZED VIEW IF NOT EXISTS trading_activity_summary AS -SELECT - date_trunc('hour', event_timestamp) as hour, - symbol, - COUNT(*) as total_events, - COUNT(*) FILTER (WHERE event_type = 'order_created') as orders_created, - COUNT(*) FILTER (WHERE event_type = 'order_filled') as orders_filled, - COUNT(*) FILTER (WHERE event_type = 'order_cancelled') as orders_cancelled, - SUM(quantity) FILTER (WHERE event_type = 'order_created') as total_quantity, - SUM(execution_quantity) FILTER (WHERE event_type = 'order_filled') as filled_quantity, - AVG(latency_ns) FILTER (WHERE latency_ns IS NOT NULL) as avg_latency_ns, - MAX(latency_ns) FILTER (WHERE latency_ns IS NOT NULL) as max_latency_ns -FROM trading_events -WHERE event_timestamp >= CURRENT_DATE - INTERVAL '7 days' -GROUP BY date_trunc('hour', event_timestamp), symbol; - --- Risk events summary -CREATE MATERIALIZED VIEW IF NOT EXISTS risk_events_summary AS -SELECT - date_trunc('day', event_timestamp) as day, - risk_type, - severity, - COUNT(*) as event_count, - COUNT(*) FILTER (WHERE resolution_status = 'resolved') as resolved_count, - AVG(EXTRACT(EPOCH FROM (resolution_timestamp - event_timestamp))/60) - FILTER (WHERE resolution_timestamp IS NOT NULL) as avg_resolution_minutes -FROM risk_events -WHERE event_timestamp >= CURRENT_DATE - INTERVAL '30 days' -GROUP BY date_trunc('day', event_timestamp), risk_type, severity; - --- Create unique indexes on materialized views -CREATE UNIQUE INDEX IF NOT EXISTS idx_trading_activity_summary_hour_symbol -ON trading_activity_summary(hour, symbol); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_risk_events_summary_day_type_severity -ON risk_events_summary(day, risk_type, severity); - --- ======================================================================================= --- COMMENTS FOR DOCUMENTATION --- ======================================================================================= - -COMMENT ON TABLE trading_events IS 'Comprehensive audit trail of all trading lifecycle events with nanosecond precision for HFT compliance'; -COMMENT ON TABLE risk_events IS 'Risk management events, violations, and alerts with automated resolution tracking'; -COMMENT ON TABLE audit_trail IS 'Complete audit trail of system changes and administrative actions for regulatory compliance'; -COMMENT ON TABLE ml_signals IS 'Machine learning model predictions and signals with performance tracking'; -COMMENT ON TABLE system_events IS 'System health, performance metrics, and operational events'; - -COMMENT ON COLUMN trading_events.latency_ns IS 'Processing latency in nanoseconds for HFT performance monitoring'; -COMMENT ON COLUMN trading_events.correlation_id IS 'Links related events together for complete transaction tracking'; -COMMENT ON COLUMN risk_events.breach_percentage IS 'Percentage by which limit was breached (>1.0 = breach)'; -COMMENT ON COLUMN audit_trail.regulatory_impact IS 'Flags changes that have regulatory compliance implications'; -COMMENT ON COLUMN ml_signals.confidence IS 'Model confidence in prediction (0.0-1.0)'; -COMMENT ON COLUMN system_events.latency_ns IS 'System operation latency in nanoseconds'; - --- Grant appropriate permissions --- GRANT SELECT, INSERT ON ALL TABLES IN SCHEMA public TO foxhunt_trading_service; --- GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO foxhunt_trading_service; --- GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO foxhunt_trading_service; - --- Schedule partition management (requires pg_cron extension) --- SELECT cron.schedule('manage-event-partitions', '0 2 * * 0', 'SELECT manage_event_partitions();'); \ No newline at end of file