-- Migration 005: Advanced Risk Management and Regulatory Compliance -- This migration creates comprehensive risk management for institutional HFT trading -- Risk limits table - comprehensive limit management CREATE TABLE IF NOT EXISTS risk_limits ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, user_id UUID REFERENCES users(id) ON DELETE CASCADE, limit_type VARCHAR(50) NOT NULL, -- 'position_size', 'daily_loss', 'exposure', 'concentration', 'var', 'leverage' limit_scope VARCHAR(20) NOT NULL DEFAULT 'account', -- 'account', 'user', 'symbol', 'sector', 'strategy' symbol VARCHAR(32), -- NULL for portfolio-level limits strategy_name VARCHAR(100), -- NULL for general limits sector VARCHAR(50), -- NULL for non-sector limits limit_value DECIMAL(20, 8) NOT NULL, warning_threshold DECIMAL(5, 4) DEFAULT 0.80, -- Warn at 80% of limit breach_action VARCHAR(50) NOT NULL DEFAULT 'alert', -- 'alert', 'block', 'reduce', 'liquidate' time_window VARCHAR(20), -- '1m', '5m', '1h', '1d', 'rolling' - NULL for static limits is_active BOOLEAN NOT NULL DEFAULT true, priority INTEGER NOT NULL DEFAULT 100, -- Higher number = higher priority created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), created_by UUID REFERENCES users(id), metadata JSONB ); -- Risk limit breaches - audit trail CREATE TABLE IF NOT EXISTS risk_limit_breaches ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), risk_limit_id UUID NOT NULL REFERENCES risk_limits(id), account_id UUID, user_id UUID, symbol VARCHAR(32), breach_value DECIMAL(20, 8) NOT NULL, limit_value DECIMAL(20, 8) NOT NULL, breach_percentage DECIMAL(5, 4) NOT NULL, severity VARCHAR(20) NOT NULL CHECK (severity IN ('warning', 'breach', 'critical')), action_taken VARCHAR(100), resolution_status VARCHAR(20) NOT NULL DEFAULT 'open' CHECK (resolution_status IN ('open', 'acknowledged', 'resolved', 'false_positive')), resolved_at TIMESTAMP WITH TIME ZONE, resolved_by UUID REFERENCES users(id), breach_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), detected_by VARCHAR(50) NOT NULL, -- 'system', 'manual', 'external' correlation_id UUID, -- Group related breaches metadata JSONB ); -- Trading strategies table - strategy definitions CREATE TABLE IF NOT EXISTS trading_strategies ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), strategy_name VARCHAR(100) NOT NULL UNIQUE, description TEXT, strategy_type VARCHAR(50) NOT NULL, -- 'trend_following', 'mean_reversion', 'arbitrage', 'market_making' algorithm_version VARCHAR(20), parameters JSONB NOT NULL DEFAULT '{}'::jsonb, risk_parameters JSONB NOT NULL DEFAULT '{}'::jsonb, symbols TEXT[], -- Supported symbols timeframes TEXT[], -- Supported timeframes min_account_balance DECIMAL(20, 8) DEFAULT 0, max_position_size DECIMAL(20, 8), max_daily_trades INTEGER, is_active BOOLEAN NOT NULL DEFAULT true, is_paper_only BOOLEAN NOT NULL DEFAULT false, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), created_by UUID REFERENCES users(id), metadata JSONB ); -- Strategy assignments - link strategies to accounts CREATE TABLE IF NOT EXISTS strategy_assignments ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), account_id UUID NOT NULL REFERENCES accounts(id) ON DELETE CASCADE, strategy_id UUID NOT NULL REFERENCES trading_strategies(id) ON DELETE CASCADE, allocation DECIMAL(5, 4) NOT NULL DEFAULT 1.0, -- Percentage of account allocated (0.0-1.0) custom_parameters JSONB DEFAULT '{}'::jsonb, risk_multiplier DECIMAL(5, 4) DEFAULT 1.0, -- Risk scaling factor is_active BOOLEAN NOT NULL DEFAULT true, started_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), stopped_at TIMESTAMP WITH TIME ZONE, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), created_by UUID REFERENCES users(id), metadata JSONB, UNIQUE(account_id, strategy_id) ); -- VaR calculations table - Value at Risk tracking CREATE TABLE IF NOT EXISTS var_calculations ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, calculation_date DATE NOT NULL, confidence_level DECIMAL(5, 4) NOT NULL, -- 0.95, 0.99, etc. time_horizon INTEGER NOT NULL, -- Days var_amount DECIMAL(20, 8) NOT NULL, expected_shortfall DECIMAL(20, 8), -- Conditional VaR methodology VARCHAR(50) NOT NULL, -- 'historical', 'parametric', 'monte_carlo' portfolio_value DECIMAL(20, 8) NOT NULL, var_percentage DECIMAL(10, 6) NOT NULL, calculation_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), model_parameters JSONB, positions_snapshot JSONB, -- Snapshot of positions used market_data_window JSONB, -- Time window of market data used metadata JSONB, UNIQUE(account_id, calculation_date, confidence_level, time_horizon) ); -- Stress tests table - scenario analysis CREATE TABLE IF NOT EXISTS stress_tests ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), test_name VARCHAR(100) NOT NULL, description TEXT, scenario_type VARCHAR(50) NOT NULL, -- 'historical', 'hypothetical', 'regulatory' scenario_parameters JSONB NOT NULL, account_id UUID REFERENCES accounts(id) ON DELETE CASCADE, test_date DATE NOT NULL, portfolio_value_before DECIMAL(20, 8) NOT NULL, portfolio_value_after DECIMAL(20, 8) NOT NULL, loss_amount DECIMAL(20, 8) NOT NULL, loss_percentage DECIMAL(10, 6) NOT NULL, worst_position JSONB, -- Position with worst performance test_duration_ms INTEGER, passed_regulatory BOOLEAN, regulatory_threshold DECIMAL(20, 8), created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), created_by UUID REFERENCES users(id), metadata JSONB ); -- Regulatory reports table - compliance reporting CREATE TABLE IF NOT EXISTS regulatory_reports ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), report_type VARCHAR(50) NOT NULL, -- 'daily_risk', 'var_breach', 'large_trader', 'position_limit' jurisdiction VARCHAR(10) NOT NULL, regulator VARCHAR(50) NOT NULL, -- 'CFTC', 'SEC', 'FCA', 'ESMA' reporting_period_start DATE NOT NULL, reporting_period_end DATE NOT NULL, account_id UUID REFERENCES accounts(id), user_id UUID REFERENCES users(id), report_data JSONB NOT NULL, file_path TEXT, -- Path to generated report file submission_id VARCHAR(100), -- Regulator's submission ID status VARCHAR(20) NOT NULL DEFAULT 'draft' CHECK (status IN ('draft', 'generated', 'submitted', 'acknowledged', 'rejected')), due_date DATE, submitted_at TIMESTAMP WITH TIME ZONE, acknowledged_at TIMESTAMP WITH TIME ZONE, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), created_by UUID REFERENCES users(id), metadata JSONB ); -- Trade surveillance alerts - monitoring suspicious activity CREATE TABLE IF NOT EXISTS surveillance_alerts ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), alert_type VARCHAR(50) NOT NULL, -- 'unusual_volume', 'price_manipulation', 'layering', 'spoofing', 'wash_trading' severity VARCHAR(20) NOT NULL CHECK (severity IN ('low', 'medium', 'high', 'critical')), account_id UUID REFERENCES accounts(id), user_id UUID REFERENCES users(id), symbol VARCHAR(32), strategy_name VARCHAR(100), trigger_condition TEXT NOT NULL, detected_pattern JSONB NOT NULL, related_orders UUID[], -- Array of order IDs related_trades UUID[], -- Array of fill IDs score DECIMAL(5, 2), -- Alert confidence score 0-100 false_positive_probability DECIMAL(5, 4), -- 0.0-1.0 status VARCHAR(20) NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'investigating', 'closed', 'escalated')), assigned_to UUID REFERENCES users(id), resolution TEXT, detected_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), resolved_at TIMESTAMP WITH TIME ZONE, escalated_at TIMESTAMP WITH TIME ZONE, metadata JSONB ); -- Market data quality checks CREATE TABLE IF NOT EXISTS market_data_quality ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), symbol VARCHAR(32) NOT NULL, data_source VARCHAR(50) NOT NULL, quality_check_type VARCHAR(50) NOT NULL, -- 'stale_data', 'outlier_price', 'missing_data', 'sequence_gap' check_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, severity VARCHAR(20) NOT NULL CHECK (severity IN ('info', 'warning', 'error', 'critical')), description TEXT NOT NULL, affected_data JSONB, resolution_action VARCHAR(100), is_resolved BOOLEAN NOT NULL DEFAULT false, resolved_at TIMESTAMP WITH TIME ZONE, impact_assessment TEXT, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), metadata JSONB ); -- Create optimized indexes for HFT performance -- Risk limits indexes CREATE INDEX IF NOT EXISTS idx_risk_limits_account_type ON risk_limits(account_id, limit_type); CREATE INDEX IF NOT EXISTS idx_risk_limits_symbol ON risk_limits(symbol) WHERE symbol IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_risk_limits_active ON risk_limits(is_active, priority DESC); CREATE INDEX IF NOT EXISTS idx_risk_limits_strategy ON risk_limits(strategy_name) WHERE strategy_name IS NOT NULL; -- Risk limit breaches indexes CREATE INDEX IF NOT EXISTS idx_risk_breaches_limit_time ON risk_limit_breaches(risk_limit_id, breach_time DESC); CREATE INDEX IF NOT EXISTS idx_risk_breaches_account ON risk_limit_breaches(account_id, breach_time DESC); CREATE INDEX IF NOT EXISTS idx_risk_breaches_severity ON risk_limit_breaches(severity, resolution_status); CREATE INDEX IF NOT EXISTS idx_risk_breaches_correlation ON risk_limit_breaches(correlation_id) WHERE correlation_id IS NOT NULL; -- Trading strategies indexes CREATE INDEX IF NOT EXISTS idx_strategies_name ON trading_strategies(strategy_name); CREATE INDEX IF NOT EXISTS idx_strategies_type ON trading_strategies(strategy_type); CREATE INDEX IF NOT EXISTS idx_strategies_active ON trading_strategies(is_active); -- Strategy assignments indexes CREATE INDEX IF NOT EXISTS idx_strategy_assignments_account ON strategy_assignments(account_id, is_active); CREATE INDEX IF NOT EXISTS idx_strategy_assignments_strategy ON strategy_assignments(strategy_id, is_active); -- VaR calculations indexes CREATE INDEX IF NOT EXISTS idx_var_calculations_account_date ON var_calculations(account_id, calculation_date DESC); CREATE INDEX IF NOT EXISTS idx_var_calculations_date ON var_calculations(calculation_date DESC); -- Stress tests indexes CREATE INDEX IF NOT EXISTS idx_stress_tests_account_date ON stress_tests(account_id, test_date DESC); CREATE INDEX IF NOT EXISTS idx_stress_tests_type ON stress_tests(scenario_type); -- Regulatory reports indexes CREATE INDEX IF NOT EXISTS idx_regulatory_reports_type_period ON regulatory_reports(report_type, reporting_period_start DESC); CREATE INDEX IF NOT EXISTS idx_regulatory_reports_jurisdiction ON regulatory_reports(jurisdiction, status); CREATE INDEX IF NOT EXISTS idx_regulatory_reports_due_date ON regulatory_reports(due_date) WHERE status IN ('draft', 'generated'); -- Surveillance alerts indexes CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_account ON surveillance_alerts(account_id, detected_at DESC); CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_type ON surveillance_alerts(alert_type, severity); CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_status ON surveillance_alerts(status, assigned_to); CREATE INDEX IF NOT EXISTS idx_surveillance_alerts_symbol ON surveillance_alerts(symbol, detected_at DESC) WHERE symbol IS NOT NULL; -- Market data quality indexes CREATE INDEX IF NOT EXISTS idx_market_data_quality_symbol ON market_data_quality(symbol, check_timestamp DESC); CREATE INDEX IF NOT EXISTS idx_market_data_quality_source ON market_data_quality(data_source, severity); -- Create triggers for automatic updates CREATE TRIGGER trigger_risk_limits_updated_at BEFORE UPDATE ON risk_limits FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); CREATE TRIGGER trigger_strategies_updated_at BEFORE UPDATE ON trading_strategies FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); CREATE TRIGGER trigger_strategy_assignments_updated_at BEFORE UPDATE ON strategy_assignments FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); CREATE TRIGGER trigger_regulatory_reports_updated_at BEFORE UPDATE ON regulatory_reports FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); -- Create advanced risk management functions -- Function to check risk limits before trade CREATE OR REPLACE FUNCTION check_risk_limits_before_trade( p_account_id UUID, p_symbol VARCHAR(32), p_side VARCHAR(10), -- 'buy' or 'sell' p_quantity BIGINT, p_price BIGINT ) RETURNS TABLE( can_trade BOOLEAN, violated_limits JSONB, warnings JSONB ) AS $$ DECLARE v_current_position BIGINT := 0; v_new_position BIGINT; v_trade_value DECIMAL(20, 8); v_violations JSONB := '[]'::jsonb; v_warnings JSONB := '[]'::jsonb; v_limit RECORD; v_current_exposure DECIMAL(20, 8); v_account_balance DECIMAL(20, 8); BEGIN -- Get current position SELECT COALESCE(quantity, 0) INTO v_current_position FROM positions WHERE account_id = p_account_id::text AND symbol = p_symbol; -- Calculate new position v_new_position := v_current_position + CASE WHEN p_side = 'buy' THEN p_quantity ELSE -p_quantity END; -- Calculate trade value v_trade_value := (p_quantity * p_price) / 100.0; -- Get account balance SELECT current_balance INTO v_account_balance FROM accounts WHERE id = p_account_id; -- Check all active risk limits FOR v_limit IN SELECT * FROM risk_limits WHERE is_active = true AND (account_id = p_account_id OR account_id IS NULL) AND (symbol = p_symbol OR symbol IS NULL) ORDER BY priority DESC LOOP CASE v_limit.limit_type WHEN 'position_size' THEN IF ABS(v_new_position) > v_limit.limit_value THEN v_violations := v_violations || jsonb_build_object( 'limit_id', v_limit.id, 'limit_type', 'position_size', 'current_value', ABS(v_new_position), 'limit_value', v_limit.limit_value ); ELSIF ABS(v_new_position) > (v_limit.limit_value * v_limit.warning_threshold) THEN v_warnings := v_warnings || jsonb_build_object( 'limit_id', v_limit.id, 'limit_type', 'position_size', 'current_value', ABS(v_new_position), 'threshold', v_limit.limit_value * v_limit.warning_threshold ); END IF; WHEN 'exposure' THEN -- Calculate current exposure (simplified) SELECT COALESCE(SUM(ABS(quantity * last_price) / 100.0), 0) + v_trade_value INTO v_current_exposure FROM positions p WHERE p.account_id = p_account_id::text; IF v_current_exposure > v_limit.limit_value THEN v_violations := v_violations || jsonb_build_object( 'limit_id', v_limit.id, 'limit_type', 'exposure', 'current_value', v_current_exposure, 'limit_value', v_limit.limit_value ); END IF; WHEN 'leverage' THEN IF v_current_exposure / v_account_balance > v_limit.limit_value THEN v_violations := v_violations || jsonb_build_object( 'limit_id', v_limit.id, 'limit_type', 'leverage', 'current_value', v_current_exposure / v_account_balance, 'limit_value', v_limit.limit_value ); END IF; END CASE; END LOOP; -- Return results RETURN QUERY SELECT (jsonb_array_length(v_violations) = 0), v_violations, v_warnings; END; $$ LANGUAGE plpgsql SECURITY DEFINER; -- Function to calculate portfolio VaR CREATE OR REPLACE FUNCTION calculate_portfolio_var( p_account_id UUID, p_confidence_level DECIMAL(5, 4) DEFAULT 0.95, p_time_horizon INTEGER DEFAULT 1 ) RETURNS DECIMAL(20, 8) AS $$ DECLARE v_portfolio_value DECIMAL(20, 8) := 0; v_var_amount DECIMAL(20, 8) := 0; v_volatility DECIMAL(10, 6) := 0.02; -- Default 2% daily volatility v_z_score DECIMAL(10, 6); BEGIN -- Get portfolio value SELECT COALESCE(SUM(ABS(quantity * last_price) / 100.0), 0) INTO v_portfolio_value FROM positions WHERE account_id = p_account_id::text AND quantity != 0; -- Calculate Z-score for confidence level v_z_score := CASE WHEN p_confidence_level >= 0.99 THEN 2.326 WHEN p_confidence_level >= 0.95 THEN 1.645 ELSE 1.282 END; -- Simple VaR calculation (can be enhanced with historical data) v_var_amount := v_portfolio_value * v_volatility * v_z_score * SQRT(p_time_horizon); -- Store calculation INSERT INTO var_calculations ( account_id, calculation_date, confidence_level, time_horizon, var_amount, methodology, portfolio_value, var_percentage ) VALUES ( p_account_id, CURRENT_DATE, p_confidence_level, p_time_horizon, v_var_amount, 'parametric', v_portfolio_value, CASE WHEN v_portfolio_value > 0 THEN v_var_amount / v_portfolio_value ELSE 0 END ) ON CONFLICT (account_id, calculation_date, confidence_level, time_horizon) DO UPDATE SET var_amount = EXCLUDED.var_amount, portfolio_value = EXCLUDED.portfolio_value, var_percentage = EXCLUDED.var_percentage, calculation_time = NOW(); RETURN v_var_amount; END; $$ LANGUAGE plpgsql SECURITY DEFINER; -- Function to detect layering pattern CREATE OR REPLACE FUNCTION detect_layering_pattern( p_account_id UUID, p_symbol VARCHAR(32), p_time_window INTERVAL DEFAULT '5 minutes' ) RETURNS BOOLEAN AS $$ DECLARE v_order_count INTEGER; v_cancel_ratio DECIMAL(5, 4); v_pattern_detected BOOLEAN := false; BEGIN -- Count orders and cancellations in time window SELECT COUNT(*), COUNT(*) FILTER (WHERE status = 'cancelled')::DECIMAL / NULLIF(COUNT(*), 0) INTO v_order_count, v_cancel_ratio FROM orders WHERE account_id = p_account_id::text AND symbol = p_symbol AND created_at >= NOW() - p_time_window; -- Detect pattern: high number of orders with high cancellation ratio IF v_order_count >= 20 AND v_cancel_ratio >= 0.80 THEN v_pattern_detected := true; -- Create surveillance alert INSERT INTO surveillance_alerts ( alert_type, severity, account_id, symbol, trigger_condition, detected_pattern, score ) VALUES ( 'layering', 'high', p_account_id, p_symbol, 'High order count with excessive cancellation ratio', jsonb_build_object( 'order_count', v_order_count, 'cancel_ratio', v_cancel_ratio, 'time_window', p_time_window::text ), 85.0 ); END IF; RETURN v_pattern_detected; END; $$ LANGUAGE plpgsql SECURITY DEFINER; -- Create materialized views for performance -- Risk exposure summary CREATE MATERIALIZED VIEW risk_exposure_summary AS SELECT a.id as account_id, a.account_number, u.username, COUNT(p.id) as position_count, COALESCE(SUM(ABS(p.quantity * p.last_price) / 100.0), 0) as total_exposure, COALESCE(SUM(p.unrealized_pnl) / 100.0, 0) as unrealized_pnl, COALESCE(MAX(var.var_amount), 0) as latest_var, COUNT(rb.id) as active_breaches FROM accounts a JOIN users u ON a.user_id = u.id LEFT JOIN positions p ON a.id::text = p.account_id AND p.quantity != 0 LEFT JOIN var_calculations var ON a.id = var.account_id AND var.calculation_date = CURRENT_DATE LEFT JOIN risk_limit_breaches rb ON a.id = rb.account_id AND rb.resolution_status = 'open' GROUP BY a.id, a.account_number, u.username; -- Create unique index on materialized view CREATE UNIQUE INDEX idx_risk_exposure_summary_account_id ON risk_exposure_summary(account_id); -- Add constraints ALTER TABLE risk_limits ADD CONSTRAINT check_warning_threshold CHECK (warning_threshold > 0 AND warning_threshold <= 1.0); ALTER TABLE risk_limits ADD CONSTRAINT check_priority CHECK (priority > 0); ALTER TABLE var_calculations ADD CONSTRAINT check_confidence_level CHECK (confidence_level > 0 AND confidence_level < 1.0); ALTER TABLE strategy_assignments ADD CONSTRAINT check_allocation CHECK (allocation >= 0 AND allocation <= 1.0); -- Add comments for documentation COMMENT ON TABLE risk_limits IS 'Comprehensive risk limit definitions with dynamic thresholds'; COMMENT ON TABLE risk_limit_breaches IS 'Audit trail of all risk limit violations'; COMMENT ON TABLE trading_strategies IS 'Trading strategy definitions and parameters'; COMMENT ON TABLE var_calculations IS 'Value at Risk calculations with multiple methodologies'; COMMENT ON TABLE stress_tests IS 'Stress testing scenarios and results'; COMMENT ON TABLE surveillance_alerts IS 'Trade surveillance and market abuse detection'; COMMENT ON TABLE regulatory_reports IS 'Regulatory compliance reporting and submissions'; COMMENT ON FUNCTION check_risk_limits_before_trade IS 'Pre-trade risk validation with violation detection'; COMMENT ON FUNCTION calculate_portfolio_var IS 'Portfolio Value at Risk calculation and storage'; COMMENT ON FUNCTION detect_layering_pattern IS 'Market abuse pattern detection for layering/spoofing';