foxhunt/docs/sox/AUDIT_TRAIL_QUERIES.md

Audit Trail Query Examples - Foxhunt HFT Trading System

Version: 1.0 Last Updated: 2025-10-07

---

Table of Contents

1. Query by Actor (WHO)
2. Query by Action (WHAT)
3. Query by Time Range (WHEN)
4. Query by Source (WHERE)
5. Query by Result
6. Complex Multi-Field Queries
7. Integrity Verification Queries
8. Compliance Reporting Queries
9. Performance Tips

---

Query by Actor (WHO)

Find All Actions by Specific User

-- All actions by trader "trader_001"
SELECT
    event_id,
    timestamp,
    event_type,
    transaction_id,
    order_id,
    details::jsonb->'symbol' as symbol,
    details::jsonb->'quantity' as quantity,
    details::jsonb->'price' as price,
    after_state
FROM transaction_audit_events
WHERE actor = 'trader_001'
ORDER BY timestamp DESC
LIMIT 100;

Find Users Who Modified Specific Resource

-- Find who modified order "ORD-12345"
SELECT DISTINCT
    actor,
    COUNT(*) as action_count,
    MIN(timestamp) as first_action,
    MAX(timestamp) as last_action
FROM transaction_audit_events
WHERE order_id = 'ORD-12345'
GROUP BY actor
ORDER BY action_count DESC;

Find Sessions for Specific User

-- All sessions for user in last 30 days
SELECT DISTINCT
    session_id,
    MIN(timestamp) as session_start,
    MAX(timestamp) as session_end,
    COUNT(*) as event_count,
    ARRAY_AGG(DISTINCT event_type) as action_types
FROM transaction_audit_events
WHERE actor = 'trader_001'
  AND timestamp >= NOW() - INTERVAL '30 days'
  AND session_id IS NOT NULL
GROUP BY session_id
ORDER BY session_start DESC;

Find Administrative Actions

-- All actions by users with admin roles
SELECT
    actor,
    event_type,
    timestamp,
    client_ip,
    details::jsonb->'metadata' as metadata,
    after_state
FROM transaction_audit_events
WHERE actor IN (
    SELECT user_id FROM user_roles
    WHERE role_name IN ('admin', 'system_admin', 'dba')
)
AND timestamp >= NOW() - INTERVAL '7 days'
ORDER BY timestamp DESC;

---

Query by Action (WHAT)

Find All Order Creations

-- All order creation events
SELECT
    event_id,
    timestamp,
    actor,
    transaction_id,
    order_id,
    details::jsonb->'symbol' as symbol,
    details::jsonb->'side' as side,
    details::jsonb->'quantity' as quantity,
    details::jsonb->'price' as price,
    details::jsonb->'order_type' as order_type
FROM transaction_audit_events
WHERE event_type = 'OrderCreated'
ORDER BY timestamp DESC
LIMIT 100;

Find All Configuration Changes

-- All configuration changes with before/after state
SELECT
    event_id,
    timestamp,
    actor,
    client_ip,
    details::jsonb->'metadata'->>'config_key' as config_key,
    before_state::jsonb as old_value,
    after_state::jsonb as new_value,
    details::jsonb->'metadata'->>'reason' as reason
FROM transaction_audit_events
WHERE event_type = 'SystemEvent'
  AND details::jsonb->'metadata'->>'config_key' IS NOT NULL
ORDER BY timestamp DESC;

Find All Position Modifications

-- All position changes (entries and exits)
SELECT
    event_id,
    timestamp,
    actor,
    details::jsonb->'symbol' as symbol,
    before_state::jsonb->'quantity' as old_quantity,
    after_state::jsonb->'quantity' as new_quantity,
    (after_state::jsonb->'quantity')::numeric - (before_state::jsonb->'quantity')::numeric as quantity_change,
    details::jsonb->'strategy_id' as strategy
FROM transaction_audit_events
WHERE event_type IN ('PositionOpened', 'PositionClosed', 'PositionModified')
ORDER BY timestamp DESC
LIMIT 100;

Find Risk Limit Breaches

-- All risk limit breaches
SELECT
    event_id,
    timestamp,
    actor,
    details::jsonb->'account_id' as account_id,
    details::jsonb->'metadata'->>'limit_type' as limit_type,
    details::jsonb->'metadata'->>'limit_value' as limit_value,
    details::jsonb->'metadata'->>'actual_value' as actual_value,
    details::jsonb->'metadata'->>'breach_percentage' as breach_percentage
FROM transaction_audit_events
WHERE event_type = 'RiskLimitBreach'
ORDER BY timestamp DESC;

---

Query by Time Range (WHEN)

Last 24 Hours of Activity

-- All events in last 24 hours
SELECT
    event_type,
    COUNT(*) as event_count,
    COUNT(DISTINCT actor) as unique_users
FROM transaction_audit_events
WHERE timestamp >= NOW() - INTERVAL '24 hours'
GROUP BY event_type
ORDER BY event_count DESC;

Trading Day Activity

-- All activity during trading hours (9:30 AM - 4:00 PM ET)
SELECT
    DATE(timestamp AT TIME ZONE 'America/New_York') as trading_day,
    event_type,
    COUNT(*) as event_count,
    COUNT(DISTINCT actor) as unique_users
FROM transaction_audit_events
WHERE EXTRACT(HOUR FROM timestamp AT TIME ZONE 'America/New_York') BETWEEN 9 AND 16
  AND EXTRACT(DOW FROM timestamp) BETWEEN 1 AND 5  -- Monday-Friday
  AND timestamp >= NOW() - INTERVAL '30 days'
GROUP BY trading_day, event_type
ORDER BY trading_day DESC, event_count DESC;

Month-End Close Activity

-- All events during month-end close period (last 3 days of month)
SELECT
    TO_CHAR(timestamp, 'YYYY-MM') as month,
    event_type,
    actor,
    COUNT(*) as event_count
FROM transaction_audit_events
WHERE EXTRACT(DAY FROM timestamp) >= (
    DATE_TRUNC('month', timestamp) + INTERVAL '1 month' - INTERVAL '3 days'
)::date
AND timestamp >= NOW() - INTERVAL '12 months'
GROUP BY month, event_type, actor
ORDER BY month DESC, event_count DESC;

After-Hours Access

-- All access outside trading hours (potential unauthorized activity)
SELECT
    actor,
    event_type,
    timestamp,
    client_ip,
    details::jsonb->'metadata' as metadata
FROM transaction_audit_events
WHERE (
    EXTRACT(HOUR FROM timestamp AT TIME ZONE 'America/New_York') < 9
    OR EXTRACT(HOUR FROM timestamp AT TIME ZONE 'America/New_York') >= 16
    OR EXTRACT(DOW FROM timestamp) IN (0, 6)  -- Saturday, Sunday
)
AND event_type IN ('OrderCreated', 'PositionModified', 'ConfigurationChanged')
AND timestamp >= NOW() - INTERVAL '7 days'
ORDER BY timestamp DESC;

---

Query by Source (WHERE)

Find All Access from Specific IP

-- All events from IP address "192.168.1.100"
SELECT
    timestamp,
    actor,
    event_type,
    session_id,
    details::jsonb->'metadata' as metadata
FROM transaction_audit_events
WHERE client_ip = '192.168.1.100'
ORDER BY timestamp DESC
LIMIT 100;

Find Suspicious IP Patterns

-- Users accessing from multiple IPs in short time window (potential account sharing)
WITH user_ip_changes AS (
    SELECT
        actor,
        session_id,
        client_ip,
        timestamp,
        LAG(client_ip) OVER (PARTITION BY actor ORDER BY timestamp) as prev_ip,
        LAG(timestamp) OVER (PARTITION BY actor ORDER BY timestamp) as prev_timestamp
    FROM transaction_audit_events
    WHERE timestamp >= NOW() - INTERVAL '24 hours'
      AND client_ip IS NOT NULL
)
SELECT
    actor,
    COUNT(*) as ip_changes,
    STRING_AGG(DISTINCT client_ip, ', ') as all_ips
FROM user_ip_changes
WHERE client_ip <> COALESCE(prev_ip, client_ip)
  AND (timestamp - prev_timestamp) < INTERVAL '30 minutes'
GROUP BY actor
HAVING COUNT(*) > 3  -- More than 3 IP changes in 24 hours
ORDER BY ip_changes DESC;

Find External Access (Outside Office Network)

-- All access from IPs outside corporate network (10.0.0.0/8, 192.168.0.0/16)
SELECT
    actor,
    client_ip,
    event_type,
    timestamp,
    details::jsonb->'metadata' as metadata
FROM transaction_audit_events
WHERE client_ip IS NOT NULL
  AND NOT (
    client_ip LIKE '10.%'
    OR client_ip LIKE '192.168.%'
    OR client_ip = '127.0.0.1'
  )
AND timestamp >= NOW() - INTERVAL '7 days'
ORDER BY timestamp DESC;

---

Query by Result

Find All Failed Actions

-- All failed events (errors, rejections)
SELECT
    event_type,
    actor,
    timestamp,
    order_id,
    details::jsonb->'metadata'->>'error_code' as error_code,
    details::jsonb->'metadata'->>'error_message' as error_message
FROM transaction_audit_events
WHERE details::jsonb->'metadata'->>'result' = 'FAILURE'
  OR details::jsonb->'metadata'->>'status' = 'REJECTED'
ORDER BY timestamp DESC
LIMIT 100;

Find High-Latency Events

-- Events with processing latency > 1ms (HFT threshold)
SELECT
    event_type,
    actor,
    timestamp,
    order_id,
    (details::jsonb->'performance_metrics'->>'processing_latency_ns')::bigint / 1000000.0 as latency_ms
FROM transaction_audit_events
WHERE (details::jsonb->'performance_metrics'->>'processing_latency_ns')::bigint > 1000000  -- 1ms in nanoseconds
ORDER BY latency_ms DESC
LIMIT 100;

Find Successful High-Risk Operations

-- High-risk operations that succeeded
SELECT
    event_type,
    actor,
    timestamp,
    risk_level,
    details::jsonb->'metadata' as metadata,
    after_state
FROM transaction_audit_events
WHERE risk_level IN ('High', 'Critical')
  AND details::jsonb->'metadata'->>'result' = 'SUCCESS'
ORDER BY timestamp DESC
LIMIT 100;

---

Complex Multi-Field Queries

Complete Order Lifecycle

-- Full lifecycle for order "ORD-12345"
WITH order_events AS (
    SELECT
        event_id,
        event_type,
        timestamp,
        actor,
        client_ip,
        details::jsonb->'quantity' as quantity,
        details::jsonb->'price' as price,
        after_state
    FROM transaction_audit_events
    WHERE order_id = 'ORD-12345'
    ORDER BY timestamp
)
SELECT
    event_type,
    timestamp,
    EXTRACT(EPOCH FROM (timestamp - LAG(timestamp) OVER (ORDER BY timestamp))) * 1000 as latency_ms,
    actor,
    quantity,
    price,
    after_state::jsonb->'status' as status
FROM order_events;

User Activity Summary (Last 30 Days)

-- Comprehensive user activity report
SELECT
    actor,
    COUNT(*) as total_actions,
    COUNT(DISTINCT DATE(timestamp)) as active_days,
    COUNT(DISTINCT session_id) as total_sessions,
    COUNT(DISTINCT client_ip) as unique_ips,
    STRING_AGG(DISTINCT event_type, ', ') as action_types,
    MIN(timestamp) as first_action,
    MAX(timestamp) as last_action,
    COUNT(CASE WHEN risk_level IN ('High', 'Critical') THEN 1 END) as high_risk_actions,
    COUNT(CASE WHEN details::jsonb->'metadata'->>'result' = 'FAILURE' THEN 1 END) as failed_actions
FROM transaction_audit_events
WHERE timestamp >= NOW() - INTERVAL '30 days'
GROUP BY actor
ORDER BY total_actions DESC;

Suspicious Activity Detection

-- Detect potential unauthorized or suspicious activity
WITH suspicious_patterns AS (
    -- Pattern 1: After-hours trading
    SELECT actor, 'after_hours_trading' as pattern_type, COUNT(*) as occurrences
    FROM transaction_audit_events
    WHERE event_type = 'OrderCreated'
      AND (EXTRACT(HOUR FROM timestamp AT TIME ZONE 'America/New_York') < 9
           OR EXTRACT(HOUR FROM timestamp AT TIME ZONE 'America/New_York') >= 16)
      AND timestamp >= NOW() - INTERVAL '7 days'
    GROUP BY actor
    HAVING COUNT(*) > 5

    UNION ALL

    -- Pattern 2: High failure rate
    SELECT actor, 'high_failure_rate' as pattern_type, COUNT(*) as occurrences
    FROM transaction_audit_events
    WHERE details::jsonb->'metadata'->>'result' = 'FAILURE'
      AND timestamp >= NOW() - INTERVAL '7 days'
    GROUP BY actor
    HAVING COUNT(*) > 20

    UNION ALL

    -- Pattern 3: Unusual IP access
    SELECT actor, 'unusual_ip_access' as pattern_type, COUNT(DISTINCT client_ip) as occurrences
    FROM transaction_audit_events
    WHERE client_ip IS NOT NULL
      AND timestamp >= NOW() - INTERVAL '7 days'
    GROUP BY actor
    HAVING COUNT(DISTINCT client_ip) > 5
)
SELECT
    actor,
    STRING_AGG(pattern_type || ' (' || occurrences || ')', ', ') as detected_patterns,
    SUM(occurrences) as total_suspicious_events
FROM suspicious_patterns
GROUP BY actor
ORDER BY total_suspicious_events DESC;

Position Reconciliation Report

-- Daily position reconciliation (compare audit trail vs. position table)
WITH audit_positions AS (
    SELECT
        details::jsonb->>'symbol' as symbol,
        details::jsonb->>'account_id' as account_id,
        SUM(
            CASE
                WHEN event_type = 'PositionOpened' THEN (details::jsonb->>'quantity')::numeric
                WHEN event_type = 'PositionClosed' THEN -(details::jsonb->>'quantity')::numeric
                ELSE 0
            END
        ) as audit_quantity
    FROM transaction_audit_events
    WHERE event_type IN ('PositionOpened', 'PositionClosed')
      AND DATE(timestamp) = CURRENT_DATE - INTERVAL '1 day'
    GROUP BY symbol, account_id
)
SELECT
    ap.symbol,
    ap.account_id,
    ap.audit_quantity as audit_trail_quantity,
    p.quantity as position_table_quantity,
    ap.audit_quantity - p.quantity as variance
FROM audit_positions ap
LEFT JOIN positions p ON ap.symbol = p.symbol AND ap.account_id = p.account_id
WHERE ABS(ap.audit_quantity - COALESCE(p.quantity, 0)) > 0.01  -- 0.01 tolerance for rounding
ORDER BY ABS(variance) DESC;

---

Integrity Verification Queries

Verify Checksums

-- Verify checksum integrity for sample of events
SELECT
    event_id,
    timestamp,
    actor,
    checksum,
    LENGTH(checksum) as checksum_length,
    CASE
        WHEN LENGTH(checksum) = 64 THEN 'Valid (SHA-256)'
        ELSE 'Invalid'
    END as checksum_validation
FROM transaction_audit_events
WHERE timestamp >= NOW() - INTERVAL '1 hour'
ORDER BY timestamp DESC
LIMIT 100;

Detect Tampered Records

-- Find events with missing or invalid checksums
SELECT
    event_id,
    timestamp,
    actor,
    event_type,
    checksum,
    CASE
        WHEN checksum IS NULL THEN 'Missing checksum'
        WHEN LENGTH(checksum) <> 64 THEN 'Invalid checksum length'
        WHEN checksum !~ '^[0-9a-f]{64}$' THEN 'Invalid checksum format'
        ELSE 'Valid'
    END as issue
FROM transaction_audit_events
WHERE checksum IS NULL
   OR LENGTH(checksum) <> 64
   OR checksum !~ '^[0-9a-f]{64}$'
ORDER BY timestamp DESC;

Verify Immutability (No Updates/Deletes)

-- Check for duplicate event_ids (shouldn't exist if immutable)
SELECT
    event_id,
    COUNT(*) as occurrences
FROM transaction_audit_events
GROUP BY event_id
HAVING COUNT(*) > 1;

Verify Retention Policy

-- Check if any events older than 7 years (2,555 days)
SELECT
    DATE(timestamp) as event_date,
    CURRENT_DATE - DATE(timestamp) as days_old,
    COUNT(*) as event_count
FROM transaction_audit_events
WHERE timestamp < NOW() - INTERVAL '2555 days'
GROUP BY DATE(timestamp)
ORDER BY event_date;

---

Compliance Reporting Queries

SOX Section 302: User Activity Report

-- Quarterly user activity report for Section 302 certification
SELECT
    actor,
    COUNT(*) as total_actions,
    COUNT(DISTINCT DATE(timestamp)) as active_days,
    COUNT(DISTINCT event_type) as action_types,
    COUNT(CASE WHEN risk_level = 'Critical' THEN 1 END) as critical_actions,
    COUNT(CASE WHEN details::jsonb->'metadata'->>'result' = 'FAILURE' THEN 1 END) as failed_actions,
    MAX(timestamp) as last_action
FROM transaction_audit_events
WHERE timestamp >= DATE_TRUNC('quarter', NOW() - INTERVAL '3 months')
  AND timestamp < DATE_TRUNC('quarter', NOW())
GROUP BY actor
ORDER BY total_actions DESC;

SOX Section 404: Control Testing Report

-- Sample transactions for control testing (25 random samples per quarter)
SELECT
    event_id,
    timestamp,
    actor,
    event_type,
    order_id,
    details::jsonb->'symbol' as symbol,
    details::jsonb->'quantity' as quantity,
    risk_level,
    compliance_tags
FROM transaction_audit_events
WHERE event_type IN ('OrderCreated', 'OrderExecuted', 'PositionModified')
  AND timestamp >= DATE_TRUNC('quarter', NOW() - INTERVAL '3 months')
  AND timestamp < DATE_TRUNC('quarter', NOW())
ORDER BY RANDOM()
LIMIT 25;

MiFID II: Best Execution Report

-- Best execution analysis for MiFID II compliance
SELECT
    details::jsonb->>'symbol' as symbol,
    details::jsonb->>'venue' as venue,
    COUNT(*) as execution_count,
    AVG((details::jsonb->>'price')::numeric) as avg_execution_price,
    AVG((details::jsonb->'performance_metrics'->>'processing_latency_ns')::bigint) / 1000000.0 as avg_latency_ms,
    COUNT(CASE WHEN compliance_tags && ARRAY['BEST_EXECUTION'] THEN 1 END) as best_execution_tagged
FROM transaction_audit_events
WHERE event_type = 'OrderExecuted'
  AND timestamp >= NOW() - INTERVAL '30 days'
GROUP BY symbol, venue
ORDER BY execution_count DESC;

Access Control: Quarterly Review Report

-- Quarterly access review: users who accessed critical functions
SELECT
    actor,
    STRING_AGG(DISTINCT event_type, ', ') as critical_actions,
    COUNT(*) as total_critical_actions,
    MIN(timestamp) as first_access,
    MAX(timestamp) as last_access,
    COUNT(DISTINCT DATE(timestamp)) as active_days
FROM transaction_audit_events
WHERE event_type IN (
    'ConfigurationChanged',
    'UserRoleModified',
    'PermissionGranted',
    'EmergencyOverride'
)
AND timestamp >= DATE_TRUNC('quarter', NOW() - INTERVAL '3 months')
AND timestamp < DATE_TRUNC('quarter', NOW())
GROUP BY actor
ORDER BY total_critical_actions DESC;

---

Performance Tips

Use Indexes Efficiently

-- Create composite indexes for common queries
CREATE INDEX idx_audit_actor_timestamp ON transaction_audit_events(actor, timestamp DESC);
CREATE INDEX idx_audit_event_type_timestamp ON transaction_audit_events(event_type, timestamp DESC);
CREATE INDEX idx_audit_order_id ON transaction_audit_events(order_id) WHERE order_id <> 'N/A';
CREATE INDEX idx_audit_session_id ON transaction_audit_events(session_id) WHERE session_id IS NOT NULL;
CREATE INDEX idx_audit_client_ip ON transaction_audit_events(client_ip) WHERE client_ip IS NOT NULL;

-- Create GIN index for JSONB metadata searches
CREATE INDEX idx_audit_details_gin ON transaction_audit_events USING GIN (details jsonb_path_ops);

Use Partitioning

-- Monthly partitioning (already implemented in TimescaleDB)
-- Queries automatically use partition pruning when filtering by timestamp

-- Example: Query only last 30 days (scans only 1 partition)
SELECT * FROM transaction_audit_events
WHERE timestamp >= NOW() - INTERVAL '30 days';

-- Example: Query specific month (scans only 1 partition)
SELECT * FROM transaction_audit_events
WHERE timestamp >= '2024-01-01' AND timestamp < '2024-02-01';

Limit Large Result Sets

-- Always use LIMIT for large tables
SELECT * FROM transaction_audit_events
ORDER BY timestamp DESC
LIMIT 1000;

-- Or use cursor for pagination
DECLARE audit_cursor CURSOR FOR
    SELECT * FROM transaction_audit_events
    ORDER BY timestamp DESC;
FETCH 100 FROM audit_cursor;

Use CTEs for Complex Queries

-- Use CTEs to break complex queries into steps (more readable, sometimes faster)
WITH recent_orders AS (
    SELECT * FROM transaction_audit_events
    WHERE event_type = 'OrderCreated'
      AND timestamp >= NOW() - INTERVAL '24 hours'
),
failed_orders AS (
    SELECT * FROM recent_orders
    WHERE details::jsonb->'metadata'->>'result' = 'FAILURE'
)
SELECT * FROM failed_orders
ORDER BY timestamp DESC;

---

Document Owner: Compliance Officer Approval: CTO, CFO Next Review Date: 2026-01-01