foxhunt/database/migrations/021_archived_audit_events.sql

-- 021_archived_audit_events.sql
-- Archived Transaction Audit Events Table
-- SOX/MiFID II Compliance - Long-term Audit Storage
-- Created: 2025-10-03 (Wave 82 Agent 3)

-- Enable required extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

-- Archived audit events table (mirrors transaction_audit_events structure)
-- Used for long-term storage after retention period cleanup
CREATE TABLE archived_audit_events (
    -- Primary identification
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    event_id VARCHAR(255) NOT NULL,

    -- Event classification
    event_type VARCHAR(50) NOT NULL,

    -- High-precision timestamps
    timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
    timestamp_nanos BIGINT NOT NULL,

    -- Transaction context
    transaction_id VARCHAR(255) NOT NULL,
    order_id VARCHAR(255) NOT NULL,

    -- Actor identification (user/system)
    actor VARCHAR(255) NOT NULL,
    session_id VARCHAR(255),
    client_ip VARCHAR(45), -- IPv4 or IPv6

    -- Event details (JSONB for flexibility)
    details JSONB NOT NULL,

    -- State tracking for modifications
    before_state JSONB,
    after_state JSONB,

    -- Compliance metadata
    compliance_tags TEXT[] NOT NULL DEFAULT '{}',
    risk_level VARCHAR(20) NOT NULL,

    -- Security and integrity
    digital_signature VARCHAR(512),
    checksum VARCHAR(64) NOT NULL,

    -- Original creation time
    created_at TIMESTAMP WITH TIME ZONE NOT NULL,

    -- Archival metadata
    archived_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
    archived_by VARCHAR(255) DEFAULT 'system',

    -- Constraints for data integrity
    CONSTRAINT valid_archived_event_id CHECK (length(event_id) > 0),
    CONSTRAINT valid_archived_transaction_id CHECK (length(transaction_id) > 0),
    CONSTRAINT valid_archived_order_id CHECK (length(order_id) > 0),
    CONSTRAINT valid_archived_actor CHECK (length(actor) > 0),
    CONSTRAINT valid_archived_checksum CHECK (length(checksum) = 64),
    CONSTRAINT valid_archived_risk_level CHECK (risk_level IN ('Low', 'Medium', 'High', 'Critical')),
    CONSTRAINT positive_archived_timestamp_nanos CHECK (timestamp_nanos >= 0)
);

-- Performance indexes for archived data queries
CREATE INDEX idx_archived_audit_events_timestamp ON archived_audit_events(timestamp DESC);
CREATE INDEX idx_archived_audit_events_transaction_id ON archived_audit_events(transaction_id);
CREATE INDEX idx_archived_audit_events_order_id ON archived_audit_events(order_id);
CREATE INDEX idx_archived_audit_events_actor ON archived_audit_events(actor);
CREATE INDEX idx_archived_audit_events_event_type ON archived_audit_events(event_type);
CREATE INDEX idx_archived_audit_events_archived_at ON archived_audit_events(archived_at DESC);

-- GIN index for compliance tags
CREATE INDEX idx_archived_audit_events_compliance_tags ON archived_audit_events USING GIN(compliance_tags);

-- BRIN index for time-series optimization
CREATE INDEX idx_archived_audit_events_timestamp_brin ON archived_audit_events USING BRIN(timestamp);

-- Table partitioning by archive date (monthly partitions for long-term storage)
-- Note: Implement partitioning strategy based on archive retention requirements

-- Row Level Security for archived data
ALTER TABLE archived_audit_events ENABLE ROW LEVEL SECURITY;

-- RLS Policy: Read-only access for compliance/admin roles
CREATE POLICY archived_audit_events_read_policy ON archived_audit_events
    FOR SELECT
    USING (
        has_role('admin')
        OR has_role('compliance_officer')
        OR has_role('risk_manager')
        OR has_role('auditor')
    );

-- RLS Policy: Only system can INSERT archived events
CREATE POLICY archived_audit_events_insert_policy ON archived_audit_events
    FOR INSERT
    WITH CHECK (has_role('admin') OR has_role('system'));

-- NO UPDATE/DELETE allowed (immutable archive)
REVOKE UPDATE, DELETE ON archived_audit_events FROM PUBLIC;
REVOKE UPDATE, DELETE ON archived_audit_events FROM authenticated_users;

-- Grant permissions
GRANT SELECT ON archived_audit_events TO authenticated_users;
GRANT INSERT ON archived_audit_events TO authenticated_users;

-- Function to archive expired audit events
CREATE OR REPLACE FUNCTION archive_expired_audit_events(
    p_retention_days INTEGER DEFAULT 2555
)
RETURNS TABLE (
    archived_count BIGINT,
    deleted_count BIGINT
) AS $$
DECLARE
    v_cutoff_date TIMESTAMP WITH TIME ZONE;
    v_archived BIGINT;
    v_deleted BIGINT;
BEGIN
    -- Calculate cutoff date (default 7 years for SOX compliance)
    v_cutoff_date := NOW() - (p_retention_days || ' days')::INTERVAL;

    -- Archive events to archived_audit_events table (atomic transaction)
    WITH archived AS (
        INSERT INTO archived_audit_events (
            event_id, event_type, timestamp, timestamp_nanos,
            transaction_id, order_id, actor, session_id, client_ip,
            details, before_state, after_state,
            compliance_tags, risk_level, digital_signature, checksum,
            created_at, archived_at, archived_by
        )
        SELECT
            event_id, event_type, timestamp, timestamp_nanos,
            transaction_id, order_id, actor, session_id, client_ip,
            details, before_state, after_state,
            compliance_tags, risk_level, digital_signature, checksum,
            created_at, NOW(), 'system'
        FROM transaction_audit_events
        WHERE timestamp < v_cutoff_date
        RETURNING 1
    )
    SELECT COUNT(*) INTO v_archived FROM archived;

    -- Delete only after successful archive
    WITH deleted AS (
        DELETE FROM transaction_audit_events
        WHERE timestamp < v_cutoff_date
        RETURNING 1
    )
    SELECT COUNT(*) INTO v_deleted FROM deleted;

    RETURN QUERY SELECT v_archived, v_deleted;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;

-- Function to query archived audit events
CREATE OR REPLACE FUNCTION query_archived_audit_events(
    p_start_time TIMESTAMP WITH TIME ZONE,
    p_end_time TIMESTAMP WITH TIME ZONE,
    p_transaction_id VARCHAR DEFAULT NULL,
    p_order_id VARCHAR DEFAULT NULL,
    p_limit INTEGER DEFAULT 1000,
    p_offset INTEGER DEFAULT 0
)
RETURNS TABLE (
    event_id VARCHAR,
    event_type VARCHAR,
    timestamp TIMESTAMP WITH TIME ZONE,
    transaction_id VARCHAR,
    order_id VARCHAR,
    actor VARCHAR,
    details JSONB,
    risk_level VARCHAR,
    archived_at TIMESTAMP WITH TIME ZONE
) AS $$
BEGIN
    RETURN QUERY
    SELECT
        e.event_id,
        e.event_type,
        e.timestamp,
        e.transaction_id,
        e.order_id,
        e.actor,
        e.details,
        e.risk_level,
        e.archived_at
    FROM archived_audit_events e
    WHERE e.timestamp >= p_start_time
      AND e.timestamp <= p_end_time
      AND (p_transaction_id IS NULL OR e.transaction_id = p_transaction_id)
      AND (p_order_id IS NULL OR e.order_id = p_order_id)
    ORDER BY e.timestamp DESC
    LIMIT p_limit
    OFFSET p_offset;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;

-- Grant execute permissions on functions
GRANT EXECUTE ON FUNCTION archive_expired_audit_events TO authenticated_users;
GRANT EXECUTE ON FUNCTION query_archived_audit_events TO authenticated_users;

-- Comments for documentation
COMMENT ON TABLE archived_audit_events IS
    'Long-term archive for audit trails after retention period. SOX/MiFID II compliant. Immutable - no updates/deletes allowed.';

COMMENT ON COLUMN archived_audit_events.archived_at IS
    'Timestamp when this event was archived from the active audit trail';

COMMENT ON FUNCTION archive_expired_audit_events IS
    'Archives audit events older than retention period and deletes from active table. Uses transaction for atomicity.';

COMMENT ON FUNCTION query_archived_audit_events IS
    'Query archived audit events with flexible filtering and pagination';

-- Performance optimization: Analyze table for query planner
ANALYZE archived_audit_events;