foxhunt/migrations/003_up_create_wal_checkpoints.sql

-- WAL checkpoint table for write-ahead logging and recovery
CREATE TABLE IF NOT EXISTS wal_checkpoints (
    id UUID PRIMARY KEY,
    sequence_number BIGINT NOT NULL UNIQUE,
    checkpoint_timestamp TIMESTAMPTZ NOT NULL,
    database_state_hash TEXT NOT NULL,
    entries_count BIGINT NOT NULL DEFAULT 0,
    file_path TEXT NOT NULL,
    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);

-- Index for efficient checkpoint queries
CREATE INDEX IF NOT EXISTS idx_wal_checkpoints_sequence ON wal_checkpoints(sequence_number DESC);
CREATE INDEX IF NOT EXISTS idx_wal_checkpoints_timestamp ON wal_checkpoints(checkpoint_timestamp DESC);

-- Add sequence number to audit_logs for WAL ordering
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS sequence_number BIGINT;

-- Create sequence for audit log entries if not exists
DO $$
BEGIN
    IF NOT EXISTS (SELECT 1 FROM pg_sequences WHERE sequencename = 'audit_logs_sequence_seq') THEN
        CREATE SEQUENCE audit_logs_sequence_seq START 1;
    END IF;
END $$;

-- Set default for sequence number
ALTER TABLE audit_logs ALTER COLUMN sequence_number SET DEFAULT nextval('audit_logs_sequence_seq');

-- Update existing audit_logs entries to have sequence numbers
DO $$
BEGIN
    IF EXISTS (SELECT 1 FROM audit_logs WHERE sequence_number IS NULL LIMIT 1) THEN
        WITH ordered_logs AS (
            SELECT id, ROW_NUMBER() OVER (ORDER BY created_at ASC) as seq_num
            FROM audit_logs 
            WHERE sequence_number IS NULL
        )
        UPDATE audit_logs 
        SET sequence_number = ordered_logs.seq_num 
        FROM ordered_logs 
        WHERE audit_logs.id = ordered_logs.id;
        
        -- Update sequence to continue from max value
        PERFORM setval('audit_logs_sequence_seq', (SELECT COALESCE(MAX(sequence_number), 0) FROM audit_logs));
    END IF;
END $$;

-- Make sequence_number NOT NULL after populating existing data
ALTER TABLE audit_logs ALTER COLUMN sequence_number SET NOT NULL;

-- Create unique index on sequence number
CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_logs_sequence_unique ON audit_logs(sequence_number);

-- Additional indexes for WAL operations
CREATE INDEX IF NOT EXISTS idx_audit_logs_session_sequence ON audit_logs(session_id, sequence_number);

-- Function to get next WAL sequence number (atomic)
CREATE OR REPLACE FUNCTION get_next_wal_sequence()
RETURNS BIGINT 
LANGUAGE plpgsql
AS $$
DECLARE
    next_seq BIGINT;
BEGIN
    SELECT nextval('audit_logs_sequence_seq') INTO next_seq;
    RETURN next_seq;
END;
$$;

-- Function to create WAL checkpoint
CREATE OR REPLACE FUNCTION create_wal_checkpoint(
    p_checkpoint_id UUID,
    p_database_state_hash TEXT,
    p_file_path TEXT
)
RETURNS TABLE(
    checkpoint_id UUID,
    sequence_number BIGINT,
    checkpoint_timestamp TIMESTAMPTZ,
    entries_count BIGINT
)
LANGUAGE plpgsql
AS $$
DECLARE
    current_sequence BIGINT;
    current_count BIGINT;
    checkpoint_time TIMESTAMPTZ := NOW();
BEGIN
    -- Get current WAL position
    SELECT COALESCE(MAX(audit_logs.sequence_number), 0) INTO current_sequence FROM audit_logs;
    SELECT COUNT(*) INTO current_count FROM audit_logs;
    
    -- Insert checkpoint
    INSERT INTO wal_checkpoints (
        id, sequence_number, checkpoint_timestamp, database_state_hash, 
        entries_count, file_path, created_at
    ) VALUES (
        p_checkpoint_id, current_sequence, checkpoint_time, 
        p_database_state_hash, current_count, p_file_path, checkpoint_time
    );
    
    -- Return checkpoint info
    RETURN QUERY SELECT 
        p_checkpoint_id, 
        current_sequence, 
        checkpoint_time, 
        current_count;
END;
$$;

-- Function to verify WAL integrity
CREATE OR REPLACE FUNCTION verify_wal_integrity(
    p_start_sequence BIGINT,
    p_end_sequence BIGINT
)
RETURNS TABLE(
    sequence_number BIGINT,
    is_valid BOOLEAN,
    expected_checksum TEXT,
    actual_checksum TEXT
)
LANGUAGE plpgsql
AS $$
BEGIN
    RETURN QUERY 
    SELECT 
        al.sequence_number,
        TRUE as is_valid, -- Simplified integrity check
        '' as expected_checksum,
        '' as actual_checksum
    FROM audit_logs al
    WHERE al.sequence_number >= p_start_sequence 
    AND al.sequence_number <= p_end_sequence
    ORDER BY al.sequence_number;
END;
$$;

-- Function to cleanup old WAL entries before checkpoint
CREATE OR REPLACE FUNCTION cleanup_wal_before_checkpoint(p_checkpoint_id UUID)
RETURNS BIGINT
LANGUAGE plpgsql
AS $$
DECLARE
    checkpoint_sequence BIGINT;
    deleted_count BIGINT;
BEGIN
    -- Get checkpoint sequence number
    SELECT wc.sequence_number INTO checkpoint_sequence 
    FROM wal_checkpoints wc 
    WHERE wc.id = p_checkpoint_id;
    
    IF checkpoint_sequence IS NULL THEN
        RAISE EXCEPTION 'Checkpoint not found: %', p_checkpoint_id;
    END IF;
    
    -- Delete audit logs before checkpoint, keeping some safety margin
    DELETE FROM audit_logs 
    WHERE sequence_number < (checkpoint_sequence - 1000);  -- Keep 1000 entries as safety margin
    
    GET DIAGNOSTICS deleted_count = ROW_COUNT;
    
    RETURN deleted_count;
END;
$$;

-- WAL statistics view for monitoring
CREATE OR REPLACE VIEW wal_statistics AS
SELECT 
    COUNT(*) as total_entries,
    MIN(timestamp) as oldest_entry_timestamp,
    MAX(timestamp) as newest_entry_timestamp,
    MIN(sequence_number) as min_sequence,
    MAX(sequence_number) as max_sequence,
    pg_size_pretty(pg_total_relation_size('audit_logs')) as table_size,
    (SELECT COUNT(*) FROM wal_checkpoints) as checkpoint_count,
    (SELECT MAX(sequence_number) FROM wal_checkpoints) as last_checkpoint_sequence,
    CASE 
        WHEN MAX(timestamp) > MIN(timestamp) 
        THEN COUNT(*)::FLOAT / EXTRACT(epoch FROM (MAX(timestamp) - MIN(timestamp))) 
        ELSE 0 
    END as avg_entries_per_second
FROM audit_logs;

-- Grant permissions for trading engine user
DO $$
BEGIN
    IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'foxhunt_trader') THEN
        GRANT SELECT, INSERT ON wal_checkpoints TO foxhunt_trader;
        GRANT SELECT, INSERT, UPDATE ON audit_logs TO foxhunt_trader;
        GRANT USAGE ON SEQUENCE audit_logs_sequence_seq TO foxhunt_trader;
        GRANT SELECT ON wal_statistics TO foxhunt_trader;
        GRANT EXECUTE ON FUNCTION get_next_wal_sequence() TO foxhunt_trader;
        GRANT EXECUTE ON FUNCTION create_wal_checkpoint(UUID, TEXT, TEXT) TO foxhunt_trader;
        GRANT EXECUTE ON FUNCTION verify_wal_integrity(BIGINT, BIGINT) TO foxhunt_trader;
        GRANT EXECUTE ON FUNCTION cleanup_wal_before_checkpoint(UUID) TO foxhunt_trader;
    END IF;
END $$;