foxhunt/crates/database/compliance_schemas.sql

-- FOXHUNT HFT TRADING SYSTEM - COMPLIANCE DATABASE SCHEMAS
-- Comprehensive database schema for regulatory compliance
-- Supports MiFID II, SOX, ISO 27001, MAR, and FIX Protocol requirements
-- Version: 1.0.0
-- Created: 2025-01-21

-- Enable required PostgreSQL extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "btree_gin";
CREATE EXTENSION IF NOT EXISTS "pg_trgm";

-- =============================================================================
-- CORE COMPLIANCE AUDIT TRAIL
-- =============================================================================

-- Main audit trail table for all compliance events
-- Supports nanosecond precision timestamps and immutable logging
CREATE TABLE compliance_audit_trail (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    sequence_number BIGSERIAL NOT NULL,

    -- Timestamp precision (nanosecond level for MiFID II compliance)
    timestamp_ns BIGINT NOT NULL,
    timestamp_utc TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),

    -- Event classification
    event_type VARCHAR(100) NOT NULL,
    event_category VARCHAR(50) NOT NULL
        CHECK (event_category IN ('ORDER', 'RISK', 'SYSTEM', 'SECURITY', 'COMPLIANCE')),
    event_subcategory VARCHAR(50),
    severity VARCHAR(20) NOT NULL
        CHECK (severity IN ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')),

    -- Actor information (who performed the action)
    user_id VARCHAR(100),
    session_id VARCHAR(100),
    source_ip INET,
    user_agent TEXT,
    authentication_method VARCHAR(50),

    -- Business context
    order_id VARCHAR(100),
    instrument_id VARCHAR(50),
    portfolio_id VARCHAR(50),
    strategy_id VARCHAR(50),
    client_id VARCHAR(100),
    counterparty_id VARCHAR(100),

    -- Financial details
    quantity DECIMAL(18,8),
    price DECIMAL(18,8),
    notional_value DECIMAL(18,2),
    currency VARCHAR(3),

    -- Event details
    description TEXT NOT NULL,
    event_data JSONB NOT NULL DEFAULT '{}',
    metadata JSONB DEFAULT '{}',

    -- Compliance specifics
    regulatory_references TEXT[] DEFAULT '{}',
    compliance_status VARCHAR(20)
        CHECK (compliance_status IN ('COMPLIANT', 'WARNING', 'VIOLATION', 'UNDER_REVIEW')),
    risk_score DECIMAL(10,4),

    -- Best execution analysis
    execution_venue VARCHAR(50),
    venue_analysis JSONB,

    -- Data integrity and immutability
    data_hash VARCHAR(64) NOT NULL,
    previous_hash VARCHAR(64),
    signature VARCHAR(512), -- Digital signature for non-repudiation

    -- Regulatory flags
    requires_reporting BOOLEAN DEFAULT FALSE,
    reporting_deadline TIMESTAMP WITH TIME ZONE,
    reported_at TIMESTAMP WITH TIME ZONE,

    -- Record keeping
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    archived_at TIMESTAMP WITH TIME ZONE,
    retention_until DATE NOT NULL DEFAULT (CURRENT_DATE + INTERVAL '7 years')
);

-- Optimized indexes for compliance queries and regulatory reporting
CREATE INDEX idx_audit_timestamp_ns ON compliance_audit_trail (timestamp_ns);
CREATE INDEX idx_audit_timestamp_utc ON compliance_audit_trail (timestamp_utc);
CREATE INDEX idx_audit_event_type ON compliance_audit_trail (event_type);
CREATE INDEX idx_audit_event_category ON compliance_audit_trail (event_category);
CREATE INDEX idx_audit_user_id ON compliance_audit_trail (user_id) WHERE user_id IS NOT NULL;
CREATE INDEX idx_audit_order_id ON compliance_audit_trail (order_id) WHERE order_id IS NOT NULL;
CREATE INDEX idx_audit_instrument_id ON compliance_audit_trail (instrument_id) WHERE instrument_id IS NOT NULL;
CREATE INDEX idx_audit_client_id ON compliance_audit_trail (client_id) WHERE client_id IS NOT NULL;
CREATE INDEX idx_audit_compliance_status ON compliance_audit_trail (compliance_status) WHERE compliance_status IS NOT NULL;
CREATE INDEX idx_audit_regulatory_refs ON compliance_audit_trail USING GIN(regulatory_references);
CREATE INDEX idx_audit_requires_reporting ON compliance_audit_trail (requires_reporting, reporting_deadline) WHERE requires_reporting = TRUE;
CREATE INDEX idx_audit_retention ON compliance_audit_trail (retention_until);
CREATE INDEX idx_audit_sequence ON compliance_audit_trail (sequence_number);

-- Ensure audit trail integrity
CREATE UNIQUE INDEX idx_audit_sequence_unique ON compliance_audit_trail (sequence_number);

-- =============================================================================
-- ORDER LIFECYCLE TRACKING (MiFID II Article 26)
-- =============================================================================

-- Comprehensive order tracking for transaction reporting compliance
CREATE TABLE order_lifecycle (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    order_id VARCHAR(100) NOT NULL UNIQUE,
    parent_order_id VARCHAR(100),
    original_order_id VARCHAR(100), -- For amendment chains

    -- Client and counterparty details
    client_id VARCHAR(100) NOT NULL,
    lei VARCHAR(20), -- Legal Entity Identifier (MiFID II requirement)
    client_classification VARCHAR(20)
        CHECK (client_classification IN ('RETAIL', 'PROFESSIONAL', 'ELIGIBLE_COUNTERPARTY')),

    -- Timestamps with nanosecond precision
    received_time_ns BIGINT NOT NULL,
    validated_time_ns BIGINT,
    routed_time_ns BIGINT,
    executed_time_ns BIGINT,
    reported_time_ns BIGINT,
    cancelled_time_ns BIGINT,

    -- Order details
    instrument_id VARCHAR(50) NOT NULL,
    instrument_type VARCHAR(20) NOT NULL,
    isin VARCHAR(12), -- International Securities Identification Number
    side VARCHAR(4) NOT NULL CHECK (side IN ('BUY', 'SELL')),
    order_type VARCHAR(20) NOT NULL,
    quantity DECIMAL(18,8) NOT NULL,
    price DECIMAL(18,8),
    currency VARCHAR(3) NOT NULL,

    -- Execution details
    executed_quantity DECIMAL(18,8) DEFAULT 0,
    remaining_quantity DECIMAL(18,8),
    average_price DECIMAL(18,8),
    last_execution_price DECIMAL(18,8),
    last_execution_quantity DECIMAL(18,8),

    -- Venue and routing
    execution_venue VARCHAR(50),
    systematic_internaliser BOOLEAN DEFAULT FALSE,
    venue_mic VARCHAR(4), -- Market Identifier Code

    -- Status tracking
    order_status VARCHAR(20) NOT NULL
        CHECK (order_status IN ('NEW', 'PARTIAL', 'FILLED', 'CANCELLED', 'REJECTED', 'EXPIRED')),
    reject_reason TEXT,
    reject_code VARCHAR(10),

    -- Risk and compliance validation
    pre_trade_validation JSONB DEFAULT '{}',
    risk_score DECIMAL(10,4),
    compliance_flags TEXT[] DEFAULT '{}',

    -- Best execution analysis
    venue_analysis JSONB,
    execution_quality_metrics JSONB,
    price_improvement DECIMAL(18,8),

    -- Regulatory requirements
    regulatory_flags TEXT[] DEFAULT '{}',
    reporting_required BOOLEAN DEFAULT TRUE,

    -- MiFID II specific fields
    mifid_transaction_id VARCHAR(100),
    transaction_reference VARCHAR(100),
    short_selling_indicator VARCHAR(10),
    commodity_derivative_indicator VARCHAR(10),
    securities_financing_indicator VARCHAR(10),

    -- Algorithm identification (for algorithmic trading)
    algorithm_indicator BOOLEAN DEFAULT FALSE,
    algorithm_id VARCHAR(50),

    -- Timing and latency metrics
    validation_latency_ns BIGINT,
    routing_latency_ns BIGINT,
    execution_latency_ns BIGINT,

    -- Record keeping
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    retention_until DATE NOT NULL DEFAULT (CURRENT_DATE + INTERVAL '5 years')
);

-- Performance and compliance indexes
CREATE INDEX idx_order_received_time ON order_lifecycle (received_time_ns);
CREATE INDEX idx_order_client_id ON order_lifecycle (client_id);
CREATE INDEX idx_order_instrument_id ON order_lifecycle (instrument_id);
CREATE INDEX idx_order_status ON order_lifecycle (order_status);
CREATE INDEX idx_order_execution_venue ON order_lifecycle (execution_venue) WHERE execution_venue IS NOT NULL;
CREATE INDEX idx_order_reporting_required ON order_lifecycle (reporting_required, received_time_ns) WHERE reporting_required = TRUE;
CREATE INDEX idx_order_compliance_flags ON order_lifecycle USING GIN(compliance_flags);
CREATE INDEX idx_order_mifid_transaction_id ON order_lifecycle (mifid_transaction_id) WHERE mifid_transaction_id IS NOT NULL;
CREATE INDEX idx_order_retention ON order_lifecycle (retention_until);

-- =============================================================================
-- RISK CONTROL EVENTS
-- =============================================================================

-- Risk control validations and violations for compliance monitoring
CREATE TABLE risk_control_events (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    timestamp_ns BIGINT NOT NULL,
    timestamp_utc TIMESTAMP WITH TIME ZONE DEFAULT NOW(),

    -- Event classification
    event_type VARCHAR(50) NOT NULL,
    control_type VARCHAR(50) NOT NULL,
    control_name VARCHAR(100) NOT NULL,

    -- Context
    order_id VARCHAR(100),
    portfolio_id VARCHAR(50),
    instrument_id VARCHAR(50),
    user_id VARCHAR(100),
    strategy_id VARCHAR(50),

    -- Risk assessment
    control_result VARCHAR(20) NOT NULL
        CHECK (control_result IN ('PASS', 'WARN', 'FAIL', 'BLOCK', 'OVERRIDE')),
    risk_value DECIMAL(18,8),
    risk_limit DECIMAL(18,8),
    breach_amount DECIMAL(18,8),
    breach_percentage DECIMAL(10,4),

    -- Risk metrics
    var_amount DECIMAL(18,2),
    expected_shortfall DECIMAL(18,2),
    position_delta DECIMAL(18,8),
    portfolio_exposure DECIMAL(18,2),

    -- Control details
    description TEXT NOT NULL,
    control_parameters JSONB DEFAULT '{}',
    calculation_details JSONB DEFAULT '{}',

    -- Actions and overrides
    action_taken VARCHAR(100),
    override_user VARCHAR(100),
    override_reason TEXT,
    override_timestamp TIMESTAMP WITH TIME ZONE,

    -- Regulatory context
    regulatory_rule VARCHAR(100),
    basel_capital_requirement DECIMAL(18,2),

    -- Record keeping
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    retention_until DATE NOT NULL DEFAULT (CURRENT_DATE + INTERVAL '7 years')
);

-- Indexes for risk monitoring and reporting
CREATE INDEX idx_risk_timestamp_ns ON risk_control_events (timestamp_ns);
CREATE INDEX idx_risk_control_type ON risk_control_events (control_type);
CREATE INDEX idx_risk_control_result ON risk_control_events (control_result);
CREATE INDEX idx_risk_order_id ON risk_control_events (order_id) WHERE order_id IS NOT NULL;
CREATE INDEX idx_risk_portfolio_id ON risk_control_events (portfolio_id) WHERE portfolio_id IS NOT NULL;
CREATE INDEX idx_risk_user_id ON risk_control_events (user_id) WHERE user_id IS NOT NULL;
CREATE INDEX idx_risk_breach_amount ON risk_control_events (breach_amount) WHERE breach_amount IS NOT NULL;

-- =============================================================================
-- REGULATORY REPORTING QUEUE
-- =============================================================================

-- Queue for regulatory transaction reporting
CREATE TABLE regulatory_reports (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    report_id VARCHAR(100) UNIQUE NOT NULL,

    -- Report classification
    report_type VARCHAR(50) NOT NULL
        CHECK (report_type IN ('MIFID_TRANSACTION', 'EMIR_DERIVATIVE', 'MAR_SUSPICIOUS', 'BEST_EXECUTION', 'POSITION_REPORT')),
    report_subtype VARCHAR(50),
    report_version INTEGER DEFAULT 1,

    -- Source data
    order_ids TEXT[] NOT NULL,
    transaction_data JSONB NOT NULL,
    source_system VARCHAR(50) DEFAULT 'FOXHUNT_HFT',

    -- Regulatory context
    regulator VARCHAR(50) NOT NULL,
    jurisdiction VARCHAR(10) NOT NULL,
    regulatory_reference VARCHAR(100),

    -- Timing requirements
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    reporting_deadline TIMESTAMP WITH TIME ZONE NOT NULL,
    submitted_at TIMESTAMP WITH TIME ZONE,
    acknowledgment_received_at TIMESTAMP WITH TIME ZONE,

    -- Status tracking
    report_status VARCHAR(20) DEFAULT 'PENDING'
        CHECK (report_status IN ('PENDING', 'VALIDATED', 'SENT', 'ACKNOWLEDGED', 'FAILED', 'CANCELLED')),

    -- Submission details
    submission_id VARCHAR(100),
    submission_method VARCHAR(50),
    submission_endpoint VARCHAR(200),

    -- Error handling
    error_details TEXT,
    retry_count INTEGER DEFAULT 0,
    max_retries INTEGER DEFAULT 3,
    next_retry_at TIMESTAMP WITH TIME ZONE,

    -- Validation
    validation_status VARCHAR(20) DEFAULT 'PENDING'
        CHECK (validation_status IN ('PENDING', 'VALID', 'INVALID', 'WARNING')),
    validation_errors JSONB DEFAULT '{}',
    validation_warnings JSONB DEFAULT '{}',

    -- File management
    report_file_path VARCHAR(500),
    acknowledgment_file_path VARCHAR(500),

    -- Record keeping
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    retention_until DATE NOT NULL DEFAULT (CURRENT_DATE + INTERVAL '7 years')
);

-- Indexes for regulatory reporting management
CREATE INDEX idx_regulatory_reports_deadline ON regulatory_reports (reporting_deadline, report_status) WHERE report_status IN ('PENDING', 'VALIDATED');
CREATE INDEX idx_regulatory_reports_status ON regulatory_reports (report_status, created_at);
CREATE INDEX idx_regulatory_reports_regulator ON regulatory_reports (regulator, report_type);
CREATE INDEX idx_regulatory_reports_retry ON regulatory_reports (next_retry_at) WHERE next_retry_at IS NOT NULL;

-- =============================================================================
-- KILL SWITCH AUDIT LOG
-- =============================================================================

-- Kill switch activation and deactivation audit trail
CREATE TABLE kill_switch_audit (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    event_id VARCHAR(100) UNIQUE NOT NULL,

    -- Timing
    timestamp_ns BIGINT NOT NULL,
    timestamp_utc TIMESTAMP WITH TIME ZONE DEFAULT NOW(),

    -- Kill switch details
    switch_scope VARCHAR(50) NOT NULL
        CHECK (switch_scope IN ('GLOBAL', 'PORTFOLIO', 'STRATEGY', 'INSTRUMENT', 'SYMBOL', 'ACCOUNT')),
    scope_identifier VARCHAR(100),

    -- Action details
    action VARCHAR(20) NOT NULL CHECK (action IN ('ACTIVATE', 'DEACTIVATE', 'TEST', 'CHECK')),
    trigger_type VARCHAR(50) NOT NULL
        CHECK (trigger_type IN ('MANUAL', 'AUTOMATIC', 'RISK_BREACH', 'SYSTEM_ERROR', 'REGULATORY')),

    -- Actor information
    user_id VARCHAR(100),
    system_component VARCHAR(100),

    -- Context
    reason TEXT NOT NULL,
    risk_score DECIMAL(10,4),
    breach_details JSONB DEFAULT '{}',

    -- Impact assessment
    orders_affected INTEGER DEFAULT 0,
    positions_affected INTEGER DEFAULT 0,
    notional_affected DECIMAL(18,2) DEFAULT 0,

    -- Response metrics
    activation_latency_ns BIGINT,
    propagation_time_ms INTEGER,
    acknowledgments_received INTEGER DEFAULT 0,

    -- Recovery details
    recovery_time TIMESTAMP WITH TIME ZONE,
    recovery_user VARCHAR(100),
    recovery_reason TEXT,

    -- Regulatory implications
    regulatory_notification_required BOOLEAN DEFAULT FALSE,
    regulatory_notification_sent BOOLEAN DEFAULT FALSE,

    -- Record keeping
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    retention_until DATE NOT NULL DEFAULT (CURRENT_DATE + INTERVAL '7 years')
);

-- Indexes for kill switch monitoring
CREATE INDEX idx_kill_switch_timestamp ON kill_switch_audit (timestamp_ns);
CREATE INDEX idx_kill_switch_scope ON kill_switch_audit (switch_scope, scope_identifier);
CREATE INDEX idx_kill_switch_action ON kill_switch_audit (action, trigger_type);
CREATE INDEX idx_kill_switch_user ON kill_switch_audit (user_id) WHERE user_id IS NOT NULL;

-- =============================================================================
-- MARKET SURVEILLANCE EVENTS
-- =============================================================================

-- Market abuse and suspicious activity monitoring
CREATE TABLE market_surveillance_events (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    alert_id VARCHAR(100) UNIQUE NOT NULL,

    -- Timing
    timestamp_ns BIGINT NOT NULL,
    timestamp_utc TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    detection_timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),

    -- Alert classification
    alert_type VARCHAR(50) NOT NULL
        CHECK (alert_type IN ('LAYERING', 'SPOOFING', 'WASH_TRADING', 'RAMPING', 'MARKING_CLOSE', 'INSIDER_TRADING', 'FRONT_RUNNING')),
    alert_subtype VARCHAR(50),
    severity VARCHAR(20) NOT NULL CHECK (severity IN ('LOW', 'MEDIUM', 'HIGH', 'CRITICAL')),

    -- Market context
    instrument_id VARCHAR(50) NOT NULL,
    market_segment VARCHAR(50),
    trading_venue VARCHAR(50),

    -- Pattern details
    pattern_description TEXT NOT NULL,
    detection_algorithm VARCHAR(100),
    confidence_score DECIMAL(5,4),

    -- Trade details
    order_ids TEXT[] DEFAULT '{}',
    trade_ids TEXT[] DEFAULT '{}',
    affected_orders INTEGER,
    total_quantity DECIMAL(18,8),
    total_notional DECIMAL(18,2),

    -- Actor information
    trader_id VARCHAR(100),
    client_id VARCHAR(100),
    strategy_id VARCHAR(50),

    -- Analysis data
    pattern_data JSONB DEFAULT '{}',
    market_impact_analysis JSONB DEFAULT '{}',

    -- Investigation status
    investigation_status VARCHAR(20) DEFAULT 'PENDING'
        CHECK (investigation_status IN ('PENDING', 'UNDER_REVIEW', 'ESCALATED', 'CLEARED', 'REPORTED')),
    assigned_analyst VARCHAR(100),

    -- Regulatory action
    reportable_to_regulator BOOLEAN DEFAULT FALSE,
    reported_to_regulator BOOLEAN DEFAULT FALSE,
    regulator_reference VARCHAR(100),
    report_submission_date TIMESTAMP WITH TIME ZONE,

    -- False positive tracking
    false_positive BOOLEAN DEFAULT NULL,
    false_positive_reason TEXT,
    false_positive_marked_by VARCHAR(100),
    false_positive_marked_at TIMESTAMP WITH TIME ZONE,

    -- Record keeping
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    retention_until DATE NOT NULL DEFAULT (CURRENT_DATE + INTERVAL '5 years')
);

-- Indexes for surveillance and investigation
CREATE INDEX idx_surveillance_timestamp ON market_surveillance_events (timestamp_ns);
CREATE INDEX idx_surveillance_alert_type ON market_surveillance_events (alert_type, severity);
CREATE INDEX idx_surveillance_instrument ON market_surveillance_events (instrument_id);
CREATE INDEX idx_surveillance_status ON market_surveillance_events (investigation_status);
CREATE INDEX idx_surveillance_trader ON market_surveillance_events (trader_id) WHERE trader_id IS NOT NULL;
CREATE INDEX idx_surveillance_reportable ON market_surveillance_events (reportable_to_regulator, reported_to_regulator);

-- =============================================================================
-- CLIENT CLASSIFICATION AND SUITABILITY
-- =============================================================================

-- Client classification for MiFID II compliance
CREATE TABLE client_classifications (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    client_id VARCHAR(100) NOT NULL,

    -- Classification details
    classification VARCHAR(30) NOT NULL
        CHECK (classification IN ('RETAIL_CLIENT', 'PROFESSIONAL_CLIENT', 'ELIGIBLE_COUNTERPARTY')),
    classification_date DATE NOT NULL,
    classification_basis TEXT NOT NULL,

    -- Professional client criteria
    professional_criteria JSONB DEFAULT '{}',
    opted_up_to_professional BOOLEAN DEFAULT FALSE,
    opt_up_date DATE,

    -- Eligible counterparty status
    eligible_counterparty_categories TEXT[] DEFAULT '{}',

    -- Risk assessment
    risk_tolerance VARCHAR(20) CHECK (risk_tolerance IN ('CONSERVATIVE', 'MODERATE', 'AGGRESSIVE', 'SPECULATIVE')),
    investment_objectives TEXT,
    investment_experience_years INTEGER,

    -- Financial information
    net_worth DECIMAL(18,2),
    annual_income DECIMAL(18,2),
    liquid_assets DECIMAL(18,2),

    -- Regulatory limits
    leverage_limit DECIMAL(10,4),
    position_limits JSONB DEFAULT '{}',

    -- Suitability assessment
    suitability_assessment_date DATE,
    suitability_status VARCHAR(20) CHECK (suitability_status IN ('SUITABLE', 'UNSUITABLE', 'PENDING', 'EXPIRED')),
    next_assessment_due DATE,

    -- Documentation
    classification_documents TEXT[] DEFAULT '{}',

    -- Audit trail
    classified_by VARCHAR(100) NOT NULL,
    approved_by VARCHAR(100),

    -- Record keeping
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    valid_until DATE,
    retention_until DATE NOT NULL DEFAULT (CURRENT_DATE + INTERVAL '5 years')
);

-- Indexes for client management
CREATE INDEX idx_client_class_client_id ON client_classifications (client_id, classification_date DESC);
CREATE INDEX idx_client_class_classification ON client_classifications (classification);
CREATE INDEX idx_client_class_assessment_due ON client_classifications (next_assessment_due) WHERE next_assessment_due IS NOT NULL;

-- =============================================================================
-- VIEWS FOR REGULATORY REPORTING
-- =============================================================================

-- MiFID II Transaction Reporting View
CREATE VIEW mifid_transaction_report AS
SELECT
    ol.mifid_transaction_id,
    ol.order_id,
    ol.client_id,
    ol.lei,
    ol.instrument_id,
    ol.isin,
    ol.side,
    ol.quantity,
    ol.price,
    ol.currency,
    ol.executed_quantity,
    ol.average_price,
    ol.execution_venue,
    ol.venue_mic,
    ol.received_time_ns,
    ol.executed_time_ns,
    cc.classification as client_classification,
    ol.short_selling_indicator,
    ol.algorithm_indicator,
    ol.algorithm_id
FROM order_lifecycle ol
LEFT JOIN client_classifications cc ON ol.client_id = cc.client_id
WHERE ol.reporting_required = TRUE
  AND ol.order_status IN ('FILLED', 'PARTIAL');

-- Risk Control Summary View
CREATE VIEW risk_control_summary AS
SELECT
    DATE(timestamp_utc) as report_date,
    control_type,
    control_result,
    COUNT(*) as event_count,
    COUNT(CASE WHEN control_result = 'FAIL' THEN 1 END) as failures,
    COUNT(CASE WHEN control_result = 'BLOCK' THEN 1 END) as blocks,
    COUNT(CASE WHEN control_result = 'OVERRIDE' THEN 1 END) as overrides,
    AVG(risk_value) as avg_risk_value,
    MAX(breach_amount) as max_breach_amount
FROM risk_control_events
GROUP BY DATE(timestamp_utc), control_type, control_result;

-- Surveillance Alert Summary View
CREATE VIEW surveillance_alert_summary AS
SELECT
    DATE(timestamp_utc) as report_date,
    alert_type,
    severity,
    investigation_status,
    COUNT(*) as alert_count,
    COUNT(CASE WHEN false_positive = TRUE THEN 1 END) as false_positives,
    COUNT(CASE WHEN reportable_to_regulator = TRUE THEN 1 END) as reportable_alerts,
    COUNT(CASE WHEN reported_to_regulator = TRUE THEN 1 END) as reported_alerts
FROM market_surveillance_events
GROUP BY DATE(timestamp_utc), alert_type, severity, investigation_status;

-- =============================================================================
-- COMPLIANCE FUNCTIONS
-- =============================================================================

-- Function to calculate audit trail hash chain
CREATE OR REPLACE FUNCTION calculate_audit_hash(
    p_event_data JSONB,
    p_previous_hash VARCHAR(64)
) RETURNS VARCHAR(64) AS $$
BEGIN
    RETURN encode(
        digest(
            CONCAT(
                p_previous_hash,
                p_event_data::text
            ),
            'sha256'
        ),
        'hex'
    );
END;
$$ LANGUAGE plpgsql IMMUTABLE;

-- Function to validate order against compliance rules
CREATE OR REPLACE FUNCTION validate_order_compliance(
    p_order_id VARCHAR(100),
    p_client_id VARCHAR(100),
    p_instrument_id VARCHAR(50),
    p_quantity DECIMAL(18,8),
    p_price DECIMAL(18,8)
) RETURNS JSONB AS $$
DECLARE
    v_result JSONB := '{"valid": true, "warnings": [], "violations": []}';
    v_client_class VARCHAR(30);
    v_risk_tolerance VARCHAR(20);
BEGIN
    -- Get client classification
    SELECT classification, risk_tolerance
    INTO v_client_class, v_risk_tolerance
    FROM client_classifications
    WHERE client_id = p_client_id
    ORDER BY classification_date DESC
    LIMIT 1;

    -- Add client-specific validations based on classification
    IF v_client_class = 'RETAIL_CLIENT' THEN
        -- Add retail client specific checks
        v_result := jsonb_set(v_result, '{client_checks}', '["retail_protection_applied"]');
    END IF;

    RETURN v_result;
END;
$$ LANGUAGE plpgsql;

-- =============================================================================
-- TRIGGERS FOR AUDIT TRAIL INTEGRITY
-- =============================================================================

-- Trigger to maintain hash chain in audit trail
CREATE OR REPLACE FUNCTION update_audit_hash() RETURNS TRIGGER AS $$
DECLARE
    v_previous_hash VARCHAR(64);
BEGIN
    -- Get the previous hash from the last audit entry
    SELECT data_hash INTO v_previous_hash
    FROM compliance_audit_trail
    ORDER BY sequence_number DESC
    LIMIT 1;

    -- Calculate hash for new entry
    NEW.data_hash := calculate_audit_hash(NEW.event_data, COALESCE(v_previous_hash, ''));
    NEW.previous_hash := v_previous_hash;

    RETURN NEW;
END;
$$ LANGUAGE plpgsql;

CREATE TRIGGER trigger_audit_hash
    BEFORE INSERT ON compliance_audit_trail
    FOR EACH ROW
    EXECUTE FUNCTION update_audit_hash();

-- Trigger to prevent modification of audit trail
CREATE OR REPLACE FUNCTION prevent_audit_modification() RETURNS TRIGGER AS $$
BEGIN
    RAISE EXCEPTION 'Audit trail records cannot be modified after creation';
    RETURN NULL;
END;
$$ LANGUAGE plpgsql;

CREATE TRIGGER trigger_prevent_audit_update
    BEFORE UPDATE ON compliance_audit_trail
    FOR EACH ROW
    EXECUTE FUNCTION prevent_audit_modification();

CREATE TRIGGER trigger_prevent_audit_delete
    BEFORE DELETE ON compliance_audit_trail
    FOR EACH ROW
    EXECUTE FUNCTION prevent_audit_modification();

-- =============================================================================
-- PARTITIONING FOR PERFORMANCE
-- =============================================================================

-- Partition audit trail by month for performance
-- This will be implemented based on data volume requirements

-- =============================================================================
-- GRANTS AND SECURITY
-- =============================================================================

-- Create compliance-specific roles
CREATE ROLE compliance_officer;
CREATE ROLE compliance_analyst;
CREATE ROLE compliance_auditor;
CREATE ROLE system_auditor;

-- Grant appropriate permissions
GRANT SELECT ON ALL TABLES IN SCHEMA public TO compliance_auditor;
GRANT SELECT, INSERT ON compliance_audit_trail TO compliance_officer;
GRANT SELECT, INSERT, UPDATE ON regulatory_reports TO compliance_officer;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO compliance_analyst;

-- Row-level security policies can be added here based on requirements

-- =============================================================================
-- PERFORMANCE OPTIMIZATION
-- =============================================================================

-- Additional indexes for high-frequency queries
CREATE INDEX CONCURRENTLY idx_audit_recent
    ON compliance_audit_trail (timestamp_utc DESC)
    WHERE timestamp_utc > (NOW() - INTERVAL '30 days');

CREATE INDEX CONCURRENTLY idx_order_recent
    ON order_lifecycle (received_time_ns DESC)
    WHERE created_at > (NOW() - INTERVAL '7 days');

-- =============================================================================
-- COMMENTS AND DOCUMENTATION
-- =============================================================================

COMMENT ON TABLE compliance_audit_trail IS 'Immutable audit trail for all compliance events with nanosecond precision timestamps';
COMMENT ON TABLE order_lifecycle IS 'Complete order lifecycle tracking for MiFID II transaction reporting compliance';
COMMENT ON TABLE risk_control_events IS 'Risk control validation events for pre-trade and post-trade monitoring';
COMMENT ON TABLE regulatory_reports IS 'Queue for regulatory transaction reporting with deadline management';
COMMENT ON TABLE kill_switch_audit IS 'Kill switch activation/deactivation audit trail for emergency response tracking';
COMMENT ON TABLE market_surveillance_events IS 'Market abuse surveillance alerts and investigation tracking';
COMMENT ON TABLE client_classifications IS 'Client classification and suitability assessments for MiFID II compliance';

-- Schema version tracking
CREATE TABLE schema_version (
    version VARCHAR(20) PRIMARY KEY,
    applied_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    description TEXT
);

INSERT INTO schema_version (version, description)
VALUES ('1.0.0', 'Initial compliance schema with MiFID II, SOX, and MAR support');