foxhunt/migrations/trading_service_events_implementation.md

Trading Service Event Storage Implementation Guide

Overview

This document provides implementation guidance for using the comprehensive event storage system designed for the Trading Service. The system stores ALL trading events in PostgreSQL for compliance and audit trails, supporting regulatory requirements like MiFID II, SOX, and Dodd-Frank.

Table Structure Summary

1. trading_events - Core Trading Lifecycle Events

• Purpose: All order lifecycle events (created, modified, filled, cancelled)
• Retention: 24 months (regulatory requirement)
• Partitioning: Monthly partitions for performance
• Key Features: Nanosecond latency tracking, correlation IDs, risk validation results

2. risk_events - Risk Management Events

• Purpose: Risk violations, alerts, emergency actions
• Retention: 12 months
• Key Features: Breach amounts, resolution tracking, automatic actions

3. audit_trail - Administrative Actions

• Purpose: Configuration changes, user actions, approvals
• Retention: 84 months (7 years for SOX compliance)
• Key Features: Before/after values, approval workflows, IP tracking

4. ml_signals - ML Model Predictions

• Purpose: AI/ML model outputs and performance tracking
• Retention: 6 months
• Key Features: Model versions, confidence scores, outcome tracking

5. system_events - Health and Performance

• Purpose: System monitoring, errors, performance metrics
• Retention: 3 months
• Key Features: Latency metrics, health scores, incident tracking

Implementation Examples

1. Recording Trading Events

-- Example: Record order creation event
INSERT INTO trading_events (
    event_type,
    correlation_id,
    order_id,
    client_order_id,
    symbol,
    side,
    order_type,
    quantity,
    price,
    account_id,
    portfolio_id,
    strategy_id,
    risk_check_result,
    risk_violations,
    source_system,
    market_data_snapshot,
    processing_start_timestamp,
    processing_end_timestamp
) VALUES (
    'order_created',
    '123e4567-e89b-12d3-a456-426614174000'::UUID,
    '987fcdeb-51a2-43d7-a456-426614174000'::UUID,
    'CLIENT_ORDER_123',
    'AAPL',
    'buy',
    'limit',
    100,
    15000, -- $150.00 in cents
    'ACC_001',
    'PORT_001',
    'STRATEGY_MOMENTUM',
    'approved',
    '[]'::JSONB,
    'trading_engine',
    '{"bid": 14995, "ask": 15005, "last": 15000}'::JSONB,
    NOW() - INTERVAL '50 microseconds',
    NOW()
);

-- Example: Record order fill event
INSERT INTO trading_events (
    event_type,
    correlation_id,
    order_id,
    symbol,
    side,
    order_type,
    quantity,
    filled_quantity,
    execution_price,
    execution_quantity,
    execution_id,
    venue,
    is_maker,
    fees,
    account_id,
    source_system
) VALUES (
    'order_filled',
    '123e4567-e89b-12d3-a456-426614174000'::UUID,
    '987fcdeb-51a2-43d7-a456-426614174000'::UUID,
    'AAPL',
    'buy',
    'limit',
    100,
    50, -- Partial fill
    14998, -- $149.98
    50,
    'NASDAQ_12345',
    'NASDAQ',
    true,
    25, -- $0.25 fee
    'ACC_001',
    'execution_engine'
);

2. Recording Risk Events

-- Example: Record position limit breach
INSERT INTO risk_events (
    event_type,
    severity,
    risk_type,
    violation_type,
    current_value,
    limit_value,
    breach_amount,
    breach_percentage,
    symbol,
    account_id,
    order_id,
    action_taken,
    auto_action,
    source_system
) VALUES (
    'violation',
    'error',
    'position_size',
    'PositionSizeExceeded',
    1200.00,
    1000.00,
    200.00,
    0.20, -- 20% breach
    'AAPL',
    'ACC_001',
    '987fcdeb-51a2-43d7-a456-426614174000'::UUID,
    'order_rejected',
    true,
    'risk_engine'
);

-- Example: VaR breach alert
INSERT INTO risk_events (
    event_type,
    severity,
    risk_type,
    current_value,
    limit_value,
    portfolio_id,
    calculation_method,
    action_taken,
    source_system,
    metadata
) VALUES (
    'alert',
    'warning',
    'var',
    95000.00, -- $95k VaR
    100000.00, -- $100k limit
    'PORT_001',
    'monte_carlo',
    'warning_issued',
    'var_calculator',
    '{"confidence_level": 0.95, "time_horizon": 1}'::JSONB
);

3. Recording Audit Trail Events

-- Example: Configuration change
INSERT INTO audit_trail (
    event_type,
    action,
    user_id,
    username,
    user_role,
    ip_address,
    target_type,
    target_id,
    target_name,
    operation,
    field_name,
    old_value,
    new_value,
    change_reason,
    system_name,
    regulatory_impact
) VALUES (
    'config_change',
    'update_risk_limit',
    'user_123',
    'risk_manager',
    'RISK_MANAGER',
    '192.168.1.100'::INET,
    'risk_limit',
    'LIMIT_POSITION_AAPL',
    'AAPL Position Limit',
    'UPDATE',
    'limit_value',
    '1000',
    '1200',
    'Increased limit due to volatility decrease',
    'risk_management_ui',
    true
);

-- Example: Emergency stop action
INSERT INTO audit_trail (
    event_type,
    action,
    user_id,
    username,
    target_type,
    operation,
    change_reason,
    system_name,
    metadata
) VALUES (
    'system_action',
    'emergency_stop_triggered',
    'system',
    'automated_risk_system',
    'trading_engine',
    'UPDATE',
    'Automatic stop due to drawdown breach',
    'risk_engine',
    '{"drawdown_pct": 15.5, "limit_pct": 15.0}'::JSONB
);

4. Recording ML Signals

-- Example: DQN trading signal
INSERT INTO ml_signals (
    model_name,
    model_version,
    model_type,
    signal_type,
    signal_strength,
    confidence,
    prediction_type,
    predicted_direction,
    time_horizon,
    symbol,
    strategy_id,
    execution_time_ms,
    source_system,
    market_data_snapshot,
    feature_vector
) VALUES (
    'dqn_trader',
    'v2.1.0',
    'reinforcement_learning',
    'trade_signal',
    0.85,
    0.92,
    'price_direction',
    'up',
    30, -- 30 minute horizon
    'AAPL',
    'STRATEGY_DQN',
    45, -- 45ms execution time
    'ml_inference_engine',
    '{"bid": 14995, "ask": 15005, "volume": 50000}'::JSONB,
    '{"rsi": 45.2, "macd": 0.15, "volume_ratio": 1.2}'::JSONB
);

-- Example: Transformer price prediction
INSERT INTO ml_signals (
    model_name,
    model_version,
    signal_type,
    predicted_value,
    confidence,
    symbol,
    execution_time_ms,
    gpu_used,
    source_system
) VALUES (
    'transformer_predictor',
    'v1.5.2',
    'market_prediction',
    15125, -- Predicted price $151.25
    0.78,
    'AAPL',
    125, -- 125ms with GPU
    true,
    'gpu_inference_cluster'
);

5. Recording System Events

-- Example: Performance metric
INSERT INTO system_events (
    event_type,
    severity,
    category,
    system_name,
    service_name,
    latency_ns,
    throughput_per_second,
    memory_usage_mb,
    cpu_usage_percent,
    health_status,
    health_score
) VALUES (
    'performance_metric',
    'info',
    'latency',
    'trading_engine',
    'order_processor',
    14000, -- 14μs latency
    50000, -- 50k orders/second
    2048, -- 2GB memory
    65.5,
    'healthy',
    95.2
);

-- Example: Error event
INSERT INTO system_events (
    event_type,
    severity,
    category,
    system_name,
    service_name,
    error_code,
    error_message,
    error_count,
    incident_id
) VALUES (
    'error',
    'error',
    'connectivity',
    'market_data_feed',
    'polygon_connector',
    'CONN_TIMEOUT',
    'Connection timeout to Polygon.io websocket',
    1,
    'INC_20250923_001'
);

Query Examples

1. Order Lifecycle Tracking

-- Get complete lifecycle of an order
SELECT
    event_timestamp,
    event_type,
    order_status,
    quantity,
    filled_quantity,
    remaining_quantity,
    execution_price,
    latency_ns / 1000000.0 AS latency_ms
FROM trading_events
WHERE order_id = '987fcdeb-51a2-43d7-a456-426614174000'
ORDER BY event_timestamp;

-- Get correlated events for a transaction
SELECT
    te.event_type,
    te.symbol,
    te.quantity,
    re.risk_type,
    re.severity
FROM trading_events te
LEFT JOIN risk_events re ON te.correlation_id = re.correlation_id
WHERE te.correlation_id = '123e4567-e89b-12d3-a456-426614174000'
ORDER BY te.event_timestamp;

2. Risk Analysis

-- Daily risk violations by type
SELECT
    DATE(event_timestamp) AS date,
    risk_type,
    severity,
    COUNT(*) AS violation_count,
    AVG(breach_percentage) AS avg_breach_pct
FROM risk_events
WHERE event_timestamp >= CURRENT_DATE - INTERVAL '30 days'
    AND event_type = 'violation'
GROUP BY DATE(event_timestamp), risk_type, severity
ORDER BY date, violation_count DESC;

-- Active unresolved risk events
SELECT
    event_timestamp,
    risk_type,
    severity,
    symbol,
    account_id,
    current_value,
    limit_value,
    breach_amount
FROM risk_events
WHERE resolution_status = 'open'
    AND severity IN ('error', 'critical')
ORDER BY event_timestamp DESC;

3. Performance Analytics

-- Trading latency analysis
SELECT
    symbol,
    DATE_TRUNC('hour', event_timestamp) AS hour,
    COUNT(*) AS order_count,
    AVG(latency_ns) / 1000000.0 AS avg_latency_ms,
    MAX(latency_ns) / 1000000.0 AS max_latency_ms,
    PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY latency_ns) / 1000000.0 AS p95_latency_ms
FROM trading_events
WHERE event_type = 'order_created'
    AND latency_ns IS NOT NULL
    AND event_timestamp >= CURRENT_DATE - INTERVAL '24 hours'
GROUP BY symbol, DATE_TRUNC('hour', event_timestamp)
ORDER BY hour DESC, avg_latency_ms DESC;

-- System health monitoring
SELECT
    system_name,
    service_name,
    AVG(health_score) AS avg_health_score,
    AVG(latency_ns) / 1000000.0 AS avg_latency_ms,
    AVG(cpu_usage_percent) AS avg_cpu_usage,
    COUNT(*) FILTER (WHERE severity = 'error') AS error_count
FROM system_events
WHERE event_timestamp >= CURRENT_DATE - INTERVAL '24 hours'
GROUP BY system_name, service_name
ORDER BY avg_health_score ASC;

4. Compliance Reporting

-- Audit trail for regulatory review
SELECT
    event_timestamp,
    action,
    username,
    target_type,
    target_name,
    operation,
    old_value,
    new_value,
    change_reason,
    ip_address
FROM audit_trail
WHERE regulatory_impact = true
    AND event_timestamp >= CURRENT_DATE - INTERVAL '90 days'
ORDER BY event_timestamp DESC;

-- Configuration changes requiring approval
SELECT
    event_timestamp,
    action,
    username,
    target_name,
    approval_status,
    approved_by,
    approval_timestamp
FROM audit_trail
WHERE approval_required = true
    AND approval_status != 'approved'
ORDER BY event_timestamp DESC;

5. ML Model Performance

-- Model accuracy tracking
SELECT
    model_name,
    model_version,
    AVG(confidence) AS avg_confidence,
    AVG(accuracy_score) AS avg_accuracy,
    COUNT(*) AS prediction_count,
    COUNT(*) FILTER (WHERE accuracy_score >= 0.8) AS accurate_predictions
FROM ml_signals
WHERE accuracy_score IS NOT NULL
    AND signal_timestamp >= CURRENT_DATE - INTERVAL '7 days'
GROUP BY model_name, model_version
ORDER BY avg_accuracy DESC;

-- Signal performance by strategy
SELECT
    strategy_id,
    symbol,
    COUNT(*) AS signal_count,
    AVG(signal_pnl) AS avg_pnl,
    SUM(signal_pnl) AS total_pnl
FROM ml_signals
WHERE signal_pnl IS NOT NULL
    AND strategy_id IS NOT NULL
    AND signal_timestamp >= CURRENT_DATE - INTERVAL '30 days'
GROUP BY strategy_id, symbol
ORDER BY total_pnl DESC;

Best Practices

1. Event Correlation

• Always use correlation_id to link related events
• Generate unique correlation IDs for each trading flow
• Include correlation IDs in all related events (trading, risk, audit)

2. Latency Tracking

• Record processing timestamps at key points
• Calculate and store latency in nanoseconds
• Use for performance optimization and SLA monitoring

3. Data Retention

• Follow regulatory requirements for retention periods
• Use automated partition management
• Archive old data to cold storage as needed

4. Indexing Strategy

• Use time-based partitioning for all event tables
• Index on frequently queried columns (symbol, account_id, event_type)
• Consider partial indexes for optional fields

5. Compliance Considerations

• Mark regulatory-impact events in audit trail
• Ensure immutable event records (no updates/deletes)
• Include sufficient context for regulatory inquiries
• Track all configuration changes with before/after values

Monitoring and Alerting

Key Metrics to Monitor

1. Event Volume: Events per second by type
2. Latency: Processing latency distribution
3. Storage Growth: Disk usage and partition sizes
4. Data Quality: Missing events or data integrity issues
5. Compliance: Unresolved risk events and pending approvals

Recommended Alerts

• Risk events with severity 'critical' or 'error'
• Trading latency exceeding SLA thresholds
• Failed event insertions
• Partition creation failures
• Audit trail events requiring approval