foxhunt/scripts/audit_sp18_consumers.sh

#!/usr/bin/env bash
# scripts/audit_sp18_consumers.sh
#
# SP18 combined-scope (D-leg + B-leg) consumer audit.
#
# Purpose: surface every live reference to the SP13/SP16 Hold-cost-scale chain
# (D-leg) and the TD(λ) `q_next = rewards` self-bootstrap (B-leg) BEFORE atomic
# deletion. Per `feedback_no_partial_refactor`, deletion must be atomic across
# every consumer; an undetected consumer = SP17-style "missed quantile_q_select"
# regression.
#
# Modes:
#   audit_sp18_consumers.sh           -> print all sections (default)
#   audit_sp18_consumers.sh --check   -> diff against docs/sp18-wireup-audit.md
#                                        captured snapshot; exit 1 if drift
#                                        (used by pre-commit hook).
#
# Exit codes:
#   0  clean (no drift in --check mode, or default mode prints output)
#   1  --check found drift OR a required file is missing
set -euo pipefail

cd "$(git rev-parse --show-toplevel)"

CHECK_MODE=0
if [[ "${1-}" == "--check" ]]; then
    CHECK_MODE=1
fi

run_audit() {
    echo "=== D-leg: Slot 380 (HOLD_COST_INDEX) consumers ==="
    grep -rn "ISV_HOLD_COST_IDX\|HOLD_COST_INDEX\|isv\[380\]\|read_isv_signal_at(HOLD_COST_INDEX)\|read_isv_signal_at(380)" \
        crates/ bin/ services/ scripts/ docs/ \
        --include="*.rs" --include="*.cu" --include="*.cuh" --include="*.toml" --include="*.md" \
        2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: Slot 461 (HOLD_COST_SCALE_INDEX) consumers ==="
    grep -rn "ISV_HOLD_COST_SCALE_IDX\|HOLD_COST_SCALE_INDEX\|isv\[461\]" \
        crates/ bin/ services/ scripts/ docs/ \
        --include="*.rs" --include="*.cu" --include="*.cuh" --include="*.toml" --include="*.md" \
        2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: Slots [462..468) (HCS_* Welford) consumers ==="
    grep -rn "HCS_TARGET_MEAN\|HCS_TARGET_M2\|HCS_DIFF_MEAN\|HCS_DIFF_M2\|HCS_PREV_TARGET\|HCS_SAMPLE_COUNT\|isv\[46[234567]\]" \
        crates/ bin/ services/ scripts/ docs/ \
        --include="*.rs" --include="*.cu" --include="*.cuh" --include="*.toml" --include="*.md" \
        2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: hold_cost_scale_update_kernel references ==="
    grep -rn "hold_cost_scale_update\|HoldCostScaleUpdate\|launch_hold_cost_scale\|hold_cost_scale\b" \
        crates/ bin/ services/ scripts/ docs/ \
        --include="*.rs" --include="*.cu" --include="*.cuh" --include="*.toml" --include="*.md" --include="*.json" \
        2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: hold_rate_observer_kernel (RETAINED — diag chain stays per DD7=c) ==="
    grep -rn "hold_rate_observer\|HoldRateObserver\|HOLD_RATE_OBSERVED_EMA_INDEX\|HOLD_RATE_TARGET_INDEX\|HOLD_RATE_LR_INDEX\|isv\[380\]\|isv\[381\]\|isv\[382\]" \
        crates/ bin/ services/ scripts/ docs/ \
        --include="*.rs" --include="*.cu" --include="*.cuh" --include="*.toml" --include="*.md" \
        2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: Per-bar Hold-cost subtraction sites (3 lines we change in Phase 1.5) ==="
    grep -Hn "isv_signals_ptr\[ISV_HOLD_COST_IDX\]\|isv\[ISV_HOLD_COST_IDX\]\|hold_cost_scale" \
        crates/ml/src/cuda_pipeline/experience_kernels.cu 2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: state_layout.cuh mirror constants (HOLD_COST + HCS) ==="
    grep -Hn "HOLD_COST\|HCS_\|HOLD_COST_SCALE" \
        crates/ml/src/cuda_pipeline/state_layout.cuh 2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: state_reset_registry slot 461 entries ==="
    grep -Hn "HOLD_COST_SCALE\|sp16_phase2_hold_cost_scale\|sp16_t3_hcs_" \
        crates/ml/src/trainers/dqn/state_reset_registry.rs 2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== D-leg: Cubin manifest references in build.rs ==="
    grep -Hn "hold_cost_scale\|hold_rate_observer" \
        crates/ml/build.rs crates/ml-dqn/build.rs 2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== B-leg: td_lambda_kernel launch sites ==="
    grep -rn "td_lambda_kernel\b" \
        crates/ bin/ services/ \
        --include="*.rs" --include="*.cu" \
        2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== B-leg: q_next argument origin (the line ~4143 site) ==="
    grep -n "q_next_bootstrap\|q_next_target\|let q_next\b" \
        crates/ml/src/cuda_pipeline/gpu_experience_collector.rs 2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== B-leg: rewards_out consumers (downstream of TD(λ) output) ==="
    grep -n "rewards_out\b" \
        crates/ml/src/cuda_pipeline/gpu_experience_collector.rs 2>/dev/null \
        | grep -v "^.*://\|^.*: \*\|^.*: //" || echo "(no hits)"

    echo ""
    echo "=== B-leg: PER priority computation sites (verify B-DD4: NO migration needed) ==="
    grep -rn "update_priorities_from_td\|update_priorities_gpu\b" \
        crates/ bin/ services/ \
        --include="*.rs" \
        2>/dev/null || echo "(no hits)"

    echo ""
    echo "=== B-leg: Replay buffer schema — q_next storage check (verify NO migration) ==="
    grep -rn "q_next\|Q_next" \
        crates/ml/src/agents/ crates/ml/src/trainers/dqn/replay_buffer_type.rs crates/ml/src/dqn/gpu_replay_buffer.rs 2>/dev/null \
        --include="*.rs" \
        || echo "(no hits — confirms no q_next stored in buffer)"

    echo ""
    echo "=== B-leg: c51_loss_kernel target-Q origin (verify already correct at loss time) ==="
    grep -n "block_bellman_project_f\|target_probs\b" \
        crates/ml/src/cuda_pipeline/c51_loss_kernel.cu 2>/dev/null \
        | head -10 || echo "(no hits)"

    echo ""
    echo "=== B-leg: target_params_buf consumers (verify Polyak-tracked, available at collection time) ==="
    grep -n "target_params_buf\b" \
        crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs 2>/dev/null \
        | head -20 || echo "(no hits)"

    echo ""
    echo "=== B-leg: PopArt slot 63 references (verify B-DD11 reset is the only PopArt change) ==="
    grep -rn "PopArt\|POPART\|popart\|slot 63\|POPART_INDEX" \
        crates/ bin/ services/ \
        --include="*.rs" --include="*.cu" --include="*.cuh" \
        2>/dev/null \
        | head -20 || echo "(no hits)"

    echo ""
    echo "=== END ==="
}

# Reduce the full audit output to a per-section hit count + per-file hit
# count "fingerprint". Catches both new-consumer (file appears) and
# expanded-consumer (line count grows) drift while staying short enough
# to inline in the audit doc and human-review at commit time.
fingerprint_audit() {
    run_audit | awk '
    BEGIN { section="(none)"; section_n=0 }
    /^=== / {
        section=$0
        if (!(section in seen)) {
            section_order[section_n++] = section
            seen[section] = 1
        }
        counts[section] = (section in counts ? counts[section] : 0)
        next
    }
    /^[a-zA-Z]/ && /:/ {
        # Split file:line:content
        n = split($0, parts, ":")
        if (n >= 2) {
            file = parts[1]
            key = section "|" file
            if (!(key in file_seen)) {
                file_seen[key] = 1
                file_order[section, file_count[section]++] = file
            }
            file_counts[key]++
            counts[section]++
        }
    }
    END {
        for (i = 0; i < section_n; i++) {
            s = section_order[i]
            print s
            print "  total_hits=" counts[s]
            n = file_count[s]
            for (j = 0; j < n; j++) {
                f = file_order[s, j]
                print "  " f " " file_counts[s "|" f]
            }
        }
    }'
}

if [[ $CHECK_MODE -eq 1 ]]; then
    SNAPSHOT="docs/sp18-wireup-audit.md"
    if [[ ! -f "$SNAPSHOT" ]]; then
        echo "ERROR: --check requires snapshot at $SNAPSHOT" >&2
        exit 1
    fi
    # Extract the captured fingerprint from the doc between the two
    # markers below, and diff against current run.
    BEGIN_MARK='<!-- BEGIN audit_sp18_consumers.sh FINGERPRINT -->'
    END_MARK='<!-- END audit_sp18_consumers.sh FINGERPRINT -->'
    if ! grep -qF "$BEGIN_MARK" "$SNAPSHOT"; then
        echo "ERROR: $SNAPSHOT missing audit-fingerprint markers; cannot --check" >&2
        exit 1
    fi
    EXPECTED=$(awk -v b="$BEGIN_MARK" -v e="$END_MARK" '
        $0 == b { flag=1; next }
        $0 == e { flag=0 }
        flag && $0 != "```" { print }
    ' "$SNAPSHOT")
    ACTUAL=$(fingerprint_audit)
    if ! diff -u <(echo "$EXPECTED") <(echo "$ACTUAL") > /tmp/sp18_audit_drift.diff; then
        echo "SP18 consumer-audit drift detected:" >&2
        cat /tmp/sp18_audit_drift.diff >&2
        echo "" >&2
        echo "If this drift is intentional, regenerate the fingerprint:" >&2
        echo "  scripts/audit_sp18_consumers.sh --fingerprint" >&2
        echo "  # then update docs/sp18-wireup-audit.md between the marker tags" >&2
        exit 1
    fi
    exit 0
fi

if [[ "${1-}" == "--fingerprint" ]]; then
    fingerprint_audit
    exit 0
fi

run_audit