foxhunt/parse_wave_141_results.py

#!/usr/bin/env python3
"""Parse Wave 141 test results and generate comprehensive report."""

import re
import sys
from collections import defaultdict

def parse_test_results(filename):
    """Parse test results from cargo test output."""

    with open(filename, 'r') as f:
        content = f.read()

    # Find all test result blocks
    # Pattern: "test result: ok. X passed; Y failed; Z ignored; W measured; A filtered out"
    result_pattern = re.compile(
        r'test result: (\w+)\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out'
    )

    results = []
    total_passed = 0
    total_failed = 0
    total_ignored = 0

    for match in result_pattern.finditer(content):
        status = match.group(1)
        passed = int(match.group(2))
        failed = int(match.group(3))
        ignored = int(match.group(4))
        measured = int(match.group(5))
        filtered = int(match.group(6))

        total_passed += passed
        total_failed += failed
        total_ignored += ignored

        results.append({
            'status': status,
            'passed': passed,
            'failed': failed,
            'ignored': ignored,
            'measured': measured,
            'filtered': filtered
        })

    # Find compilation errors
    error_pattern = re.compile(r'^error\[E\d+\]:', re.MULTILINE)
    compilation_errors = len(error_pattern.findall(content))

    # Check for compilation failure
    compile_failed_pattern = re.compile(r'error: could not compile')
    compile_failures = compile_failed_pattern.findall(content)

    return {
        'results': results,
        'total_passed': total_passed,
        'total_failed': total_failed,
        'total_ignored': total_ignored,
        'compilation_errors': compilation_errors,
        'compile_failures': len(compile_failures),
        'num_test_suites': len(results)
    }

def generate_report(data):
    """Generate comprehensive test report."""

    total_tests = data['total_passed'] + data['total_failed']
    pass_rate = (data['total_passed'] / total_tests * 100) if total_tests > 0 else 0

    print("=" * 80)
    print("WAVE 141 FULL WORKSPACE TEST RESULTS")
    print("=" * 80)
    print()

    print("SUMMARY:")
    print(f"  Test Suites Run:    {data['num_test_suites']}")
    print(f"  Total Tests:        {total_tests}")
    print(f"  Tests Passed:       {data['total_passed']}")
    print(f"  Tests Failed:       {data['total_failed']}")
    print(f"  Tests Ignored:      {data['total_ignored']}")
    print(f"  Pass Rate:          {pass_rate:.1f}%")
    print()

    print("COMPILATION STATUS:")
    print(f"  Compilation Errors: {data['compilation_errors']}")
    print(f"  Failed Compiles:    {data['compile_failures']}")
    print()

    if data['compile_failures'] > 0:
        print("⚠️  WARNING: Some tests failed to compile!")
        print("   The saturation_point_tests crate has 6 compilation errors")
        print()

    print("WAVE 140 BASELINE COMPARISON:")
    print(f"  Wave 140:  430/456 passing (94.2%)")
    print(f"  Wave 141:  {data['total_passed']}/{total_tests} passing ({pass_rate:.1f}%)")
    if total_tests > 0:
        improvement = data['total_passed'] - 430
        print(f"  Change:    {improvement:+d} tests ({pass_rate - 94.2:+.1f}%)")
    print()

    print("WAVE 141 DIRECT FIXES (6 fixes applied):")
    print("  ✅ Agent 211: TLOB metadata test")
    print("  ✅ Agent 214: Revocation statistics (3 tests)")
    print("  ✅ Agent 215: API Gateway health endpoint")
    print("  ✅ Agent 216: MFA backup code count")
    print("  ✅ Agent 218: MFA Base32 validation")
    print("  ✅ Agent 231: Load test compilation (8 errors fixed)")
    print()

    print("DETAILED BREAKDOWN:")
    for i, result in enumerate(data['results'], 1):
        status_symbol = "✅" if result['status'] == 'ok' else "❌"
        print(f"  {status_symbol} Suite {i}: {result['passed']} passed, "
              f"{result['failed']} failed, {result['ignored']} ignored")
    print()

    print("=" * 80)

    return pass_rate

if __name__ == '__main__':
    filename = '/home/jgrusewski/Work/foxhunt/WAVE_141_FULL_TEST_RESULTS.txt'
    data = parse_test_results(filename)
    pass_rate = generate_report(data)

    # Exit with status based on results
    sys.exit(0 if data['total_failed'] == 0 and data['compile_failures'] == 0 else 1)