foxhunt/database/migrations/020_transaction_audit_events.sql

-- 020_transaction_audit_events.sql
-- Comprehensive Transaction Audit Events Table
-- SOX/MiFID II Compliance - Immutable Audit Trail
-- Created: 2025-10-03 (Wave 74 Agent 1)

-- Enable required extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

-- Comprehensive transaction audit events table
-- Designed for high-performance HFT audit logging with SOX/MiFID II compliance
CREATE TABLE transaction_audit_events (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    event_id VARCHAR(255) NOT NULL UNIQUE,

    -- Event classification
    event_type VARCHAR(50) NOT NULL,

    -- High-precision timestamps
    timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
    timestamp_nanos BIGINT NOT NULL,

    -- Transaction context
    transaction_id VARCHAR(255) NOT NULL,
    order_id VARCHAR(255) NOT NULL,

    -- Actor identification (user/system)
    actor VARCHAR(255) NOT NULL,
    session_id VARCHAR(255),
    client_ip VARCHAR(45), -- IPv4 or IPv6

    -- Event details (JSONB for flexibility)
    details JSONB NOT NULL,

    -- State tracking for modifications
    before_state JSONB,
    after_state JSONB,

    -- Compliance metadata
    compliance_tags TEXT[] NOT NULL DEFAULT '{}',
    risk_level VARCHAR(20) NOT NULL,

    -- Security and integrity
    digital_signature VARCHAR(512),
    checksum VARCHAR(64) NOT NULL,

    -- Automatic timestamp
    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),

    -- Constraints for data integrity
    CONSTRAINT valid_event_id CHECK (length(event_id) > 0),
    CONSTRAINT valid_transaction_id CHECK (length(transaction_id) > 0),
    CONSTRAINT valid_order_id CHECK (length(order_id) > 0),
    CONSTRAINT valid_actor CHECK (length(actor) > 0),
    CONSTRAINT valid_checksum CHECK (length(checksum) = 64),
    CONSTRAINT valid_risk_level CHECK (risk_level IN ('Low', 'Medium', 'High', 'Critical')),
    CONSTRAINT positive_timestamp_nanos CHECK (timestamp_nanos >= 0)
);

-- Performance indexes for high-frequency queries
CREATE INDEX idx_audit_events_timestamp ON transaction_audit_events(timestamp DESC);
CREATE INDEX idx_audit_events_transaction_id ON transaction_audit_events(transaction_id, timestamp DESC);
CREATE INDEX idx_audit_events_order_id ON transaction_audit_events(order_id, timestamp DESC);
CREATE INDEX idx_audit_events_actor ON transaction_audit_events(actor, timestamp DESC);
CREATE INDEX idx_audit_events_event_type ON transaction_audit_events(event_type, timestamp DESC);
CREATE INDEX idx_audit_events_risk_level ON transaction_audit_events(risk_level, timestamp DESC);
CREATE INDEX idx_audit_events_checksum ON transaction_audit_events(checksum);

-- GIN index for compliance tags array searches
CREATE INDEX idx_audit_events_compliance_tags ON transaction_audit_events USING GIN(compliance_tags);

-- BRIN index for time-series optimization (efficient for large datasets)
CREATE INDEX idx_audit_events_timestamp_brin ON transaction_audit_events USING BRIN(timestamp);

-- Partial index for high-risk events (faster filtering)
CREATE INDEX idx_audit_events_high_risk ON transaction_audit_events(timestamp DESC)
    WHERE risk_level IN ('High', 'Critical');

-- Table partitioning by date for performance (daily partitions)
-- This helps with query performance and data retention policies
-- Note: Implement partitioning strategy based on retention requirements

-- Row Level Security for compliance
ALTER TABLE transaction_audit_events ENABLE ROW LEVEL SECURITY;

-- RLS Policy: Users can only see their own audit events, unless they're admin/compliance
CREATE POLICY audit_events_user_policy ON transaction_audit_events
    FOR SELECT
    USING (
        actor = current_user
        OR has_role('admin')
        OR has_role('compliance_officer')
        OR has_role('risk_manager')
    );

-- RLS Policy: Only system can INSERT audit events (prevents tampering)
CREATE POLICY audit_events_insert_policy ON transaction_audit_events
    FOR INSERT
    WITH CHECK (has_role('admin') OR has_role('system'));

-- RLS Policy: NO UPDATE/DELETE allowed (immutability requirement)
-- Audit logs must be immutable for SOX/MiFID II compliance

-- Grant permissions
GRANT SELECT ON transaction_audit_events TO authenticated_users;
GRANT INSERT ON transaction_audit_events TO authenticated_users;

-- Prevent any UPDATE or DELETE operations (immutable audit trail)
REVOKE UPDATE, DELETE ON transaction_audit_events FROM authenticated_users;
REVOKE UPDATE, DELETE ON transaction_audit_events FROM PUBLIC;

-- Function to verify audit event integrity (checksum validation)
CREATE OR REPLACE FUNCTION verify_audit_event_integrity(p_event_id VARCHAR)
RETURNS BOOLEAN AS $$
DECLARE
    v_event RECORD;
    v_calculated_checksum VARCHAR(64);
    v_serialized TEXT;
BEGIN
    -- Fetch the event
    SELECT * INTO v_event
    FROM transaction_audit_events
    WHERE event_id = p_event_id;

    IF NOT FOUND THEN
        RAISE EXCEPTION 'Audit event not found: %', p_event_id;
    END IF;

    -- Recreate the serialized event (without checksum) for verification
    -- This is a simplified version - production should match exact serialization
    v_serialized := v_event.event_id || v_event.event_type ||
                   v_event.timestamp::text || v_event.transaction_id ||
                   v_event.order_id || v_event.actor;

    -- Calculate SHA-256 checksum
    v_calculated_checksum := encode(digest(v_serialized, 'sha256'), 'hex');

    -- Compare with stored checksum
    RETURN v_calculated_checksum = v_event.checksum;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;

-- Function to query audit events with validation
CREATE OR REPLACE FUNCTION query_audit_events(
    p_start_time TIMESTAMP WITH TIME ZONE,
    p_end_time TIMESTAMP WITH TIME ZONE,
    p_transaction_id VARCHAR DEFAULT NULL,
    p_order_id VARCHAR DEFAULT NULL,
    p_actor VARCHAR DEFAULT NULL,
    p_event_type VARCHAR DEFAULT NULL,
    p_risk_level VARCHAR DEFAULT NULL,
    p_limit INTEGER DEFAULT 1000,
    p_offset INTEGER DEFAULT 0
)
RETURNS TABLE (
    event_id VARCHAR,
    event_type VARCHAR,
    timestamp TIMESTAMP WITH TIME ZONE,
    transaction_id VARCHAR,
    order_id VARCHAR,
    actor VARCHAR,
    details JSONB,
    risk_level VARCHAR,
    checksum VARCHAR
) AS $$
BEGIN
    RETURN QUERY
    SELECT
        e.event_id,
        e.event_type,
        e.timestamp,
        e.transaction_id,
        e.order_id,
        e.actor,
        e.details,
        e.risk_level,
        e.checksum
    FROM transaction_audit_events e
    WHERE e.timestamp >= p_start_time
      AND e.timestamp <= p_end_time
      AND (p_transaction_id IS NULL OR e.transaction_id = p_transaction_id)
      AND (p_order_id IS NULL OR e.order_id = p_order_id)
      AND (p_actor IS NULL OR e.actor = p_actor)
      AND (p_event_type IS NULL OR e.event_type = p_event_type)
      AND (p_risk_level IS NULL OR e.risk_level = p_risk_level)
    ORDER BY e.timestamp DESC
    LIMIT p_limit
    OFFSET p_offset;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;

-- Function to get audit event statistics
CREATE OR REPLACE FUNCTION get_audit_event_statistics(
    p_start_time TIMESTAMP WITH TIME ZONE,
    p_end_time TIMESTAMP WITH TIME ZONE
)
RETURNS TABLE (
    total_events BIGINT,
    events_by_type JSONB,
    events_by_risk_level JSONB,
    unique_actors BIGINT,
    unique_transactions BIGINT
) AS $$
BEGIN
    RETURN QUERY
    SELECT
        COUNT(*) as total_events,
        jsonb_object_agg(event_type, type_count) as events_by_type,
        jsonb_object_agg(risk_level, risk_count) as events_by_risk_level,
        COUNT(DISTINCT actor) as unique_actors,
        COUNT(DISTINCT transaction_id) as unique_transactions
    FROM (
        SELECT
            e.event_type,
            e.risk_level,
            e.actor,
            e.transaction_id,
            COUNT(*) OVER (PARTITION BY e.event_type) as type_count,
            COUNT(*) OVER (PARTITION BY e.risk_level) as risk_count
        FROM transaction_audit_events e
        WHERE e.timestamp >= p_start_time
          AND e.timestamp <= p_end_time
    ) stats
    GROUP BY stats.event_type, stats.risk_level
    LIMIT 1;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;

-- Grant execute permissions on functions
GRANT EXECUTE ON FUNCTION verify_audit_event_integrity TO authenticated_users;
GRANT EXECUTE ON FUNCTION query_audit_events TO authenticated_users;
GRANT EXECUTE ON FUNCTION get_audit_event_statistics TO authenticated_users;

-- Comments for documentation
COMMENT ON TABLE transaction_audit_events IS
    'Comprehensive audit trail for all trading transactions. SOX/MiFID II compliant. Immutable - no updates/deletes allowed.';

COMMENT ON COLUMN transaction_audit_events.event_id IS
    'Unique identifier for this audit event (client-generated)';

COMMENT ON COLUMN transaction_audit_events.timestamp_nanos IS
    'High-precision nanosecond timestamp for HFT ordering';

COMMENT ON COLUMN transaction_audit_events.checksum IS
    'SHA-256 checksum for tamper detection';

COMMENT ON COLUMN transaction_audit_events.compliance_tags IS
    'Array of compliance framework tags (SOX, MIFID2, etc.)';

COMMENT ON FUNCTION verify_audit_event_integrity IS
    'Verifies the integrity of an audit event using its checksum';

COMMENT ON FUNCTION query_audit_events IS
    'Query audit events with flexible filtering and pagination';

COMMENT ON FUNCTION get_audit_event_statistics IS
    'Get aggregated statistics for audit events in a time range';

-- Performance optimization: Analyze table for query planner
ANALYZE transaction_audit_events;