From a7973d4bf7aeb66eec4f3f043ec37485ce186f60 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 5 Oct 2025 22:23:43 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=8D=20Wave=20112:=20Python=20fix=20scr?= =?UTF-8?q?ipts=20for=20audit=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fix_all_audit_tests.py: Comprehensive audit test fixes - fix_async_audit_queue_tests.py: Async queue test fixes (v1) - fix_async_audit_queue_tests_v2.py: Async queue test fixes (v2) - fix_audit_compliance.py: Compliance test fixes --- scripts/fix_all_audit_tests.py | 170 ++++++++++++++++ scripts/fix_async_audit_queue_tests.py | 237 ++++++++++++++++++++++ scripts/fix_async_audit_queue_tests_v2.py | 165 +++++++++++++++ scripts/fix_audit_compliance.py | 67 ++++++ 4 files changed, 639 insertions(+) create mode 100644 scripts/fix_all_audit_tests.py create mode 100755 scripts/fix_async_audit_queue_tests.py create mode 100755 scripts/fix_async_audit_queue_tests_v2.py create mode 100644 scripts/fix_audit_compliance.py diff --git a/scripts/fix_all_audit_tests.py b/scripts/fix_all_audit_tests.py new file mode 100644 index 000000000..8c30017d4 --- /dev/null +++ b/scripts/fix_all_audit_tests.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Comprehensive fix for all audit trail test files. +Fixes the AsyncAuditQueue and AuditTrailEngine API changes from Wave 107. +""" + +import re +import sys +from pathlib import Path + +# Files to fix +TEST_FILES = [ + "trading_engine/tests/async_audit_queue_tests.rs", + "trading_engine/tests/audit_trail_persistence_test.rs", + "trading_engine/tests/audit_persistence_tests.rs", + "trading_engine/tests/audit_persistence_comprehensive.rs", + "trading_engine/tests/audit_retention_tests.rs", +] + +def fix_audit_trail_persistence_test(content): + """Fix audit_trail_persistence_test.rs specific issues""" + + # Pattern 1: Fix AuditTrailEngine::new() - old API without await + # OLD: let audit_engine = AuditTrailEngine::new(audit_config); + # NEW: let wal_path = ...; let audit_engine = AuditTrailEngine::new(audit_config, postgres_pool.clone(), wal_path).await.unwrap(); + + lines = content.split('\n') + fixed_lines = [] + + for i, line in enumerate(lines): + # Fix AuditTrailEngine::new() calls without .await + if 'let audit_engine = AuditTrailEngine::new(audit_config);' in line: + indent = len(line) - len(line.lstrip()) + spaces = ' ' * indent + fixed_lines.append(f'{spaces}let wal_path = std::env::temp_dir().join(format!("audit_wal_{{}}.log", uuid::Uuid::new_v4()));') + fixed_lines.append(f'{spaces}let audit_engine = AuditTrailEngine::new(audit_config, postgres_pool.clone(), wal_path).await.unwrap();') + continue + + # Remove .set_postgres_pool() calls (no longer needed) + if '.set_postgres_pool(' in line and 'audit_engine' in line: + continue + + # Fix AuditTrailConfig - remove old fields + if 'real_time_persistence:' in line: + continue + + fixed_lines.append(line) + + return '\n'.join(fixed_lines) + +def fix_async_audit_queue_tests(content): + """Fix async_audit_queue_tests.rs specific issues""" + + lines = content.split('\n') + fixed_lines = [] + + for i, line in enumerate(lines): + # Fix AsyncAuditQueue::new() - now async and returns Result + # OLD: let queue = Arc::new(AsyncAuditQueue::new(wal_path.clone())); + # NEW: let queue = AsyncAuditQueue::new(wal_path.clone(), pool.clone(), 100, 100).await.unwrap(); + if 'Arc::new(AsyncAuditQueue::new(' in line and 'wal_path' in line: + indent = len(line) - len(line.lstrip()) + spaces = ' ' * indent + # Extract variable name + var_match = re.search(r'let (\w+)', line) + if var_match: + var_name = var_match.group(1) + fixed_lines.append(f'{spaces}let {var_name} = AsyncAuditQueue::new(wal_path.clone(), pool.clone(), 100, 100).await.unwrap();') + continue + + # Fix submit calls that need to be await + if '.submit(' in line and 'queue' in line and not '.await' in line and not 'expect' in line: + line = line.replace('.submit(', '.submit(').replace(')?;', ').await?;').replace(');', ').await.unwrap();') + + fixed_lines.append(line) + + return '\n'.join(fixed_lines) + +def fix_audit_config_fields(content): + """Remove deprecated AuditTrailConfig fields""" + + deprecated_fields = [ + 'enabled:', + 'compression_algorithm:', + 'encryption_algorithm:', + 'encryption_key:', + 'postgres_pool:', + 'file_path:', + 'enable_checksums:', + 'enable_tamper_detection:', + 'enable_best_execution_tracking:', + 'enable_mifid_reporting:', + 'compression_enabled:', + 'encryption_enabled:', + 'retention_days:', + 'real_time_persistence:', + ] + + lines = content.split('\n') + fixed_lines = [] + skip_next = False + + for i, line in enumerate(lines): + if skip_next: + skip_next = False + continue + + # Skip lines with deprecated fields + if any(field in line for field in deprecated_fields): + continue + + fixed_lines.append(line) + + return '\n'.join(fixed_lines) + +def fix_file(file_path): + """Fix a single test file""" + + with open(file_path, 'r') as f: + content = f.read() + + original_content = content + + # Apply fixes based on file + if 'audit_trail_persistence_test' in file_path: + content = fix_audit_trail_persistence_test(content) + elif 'async_audit_queue_tests' in file_path: + content = fix_async_audit_queue_tests(content) + + # Apply common fixes to all files + content = fix_audit_config_fields(content) + + # Fix PostgresPool::new() calls - should use PostgresPool::new(config).await + content = re.sub( + r'PostgresPool::new\(postgres_config, (\d+)\)', + r'PostgresPool::new(postgres_config)', + content + ) + + # Write back if changed + if content != original_content: + with open(file_path, 'w') as f: + f.write(content) + print(f"āœ… Fixed {file_path}") + return True + else: + print(f"āš ļø No changes in {file_path}") + return False + +def main(): + base_path = Path('/home/jgrusewski/Work/foxhunt') + + fixed_count = 0 + for test_file in TEST_FILES: + file_path = base_path / test_file + if file_path.exists(): + if fix_file(str(file_path)): + fixed_count += 1 + else: + print(f"āŒ File not found: {file_path}") + + print(f"\nšŸ“Š Summary: Fixed {fixed_count}/{len(TEST_FILES)} files") + + if fixed_count > 0: + sys.exit(0) + else: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/scripts/fix_async_audit_queue_tests.py b/scripts/fix_async_audit_queue_tests.py new file mode 100755 index 000000000..3e6b2ece3 --- /dev/null +++ b/scripts/fix_async_audit_queue_tests.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +""" +Migration script to fix AsyncAuditQueue test compilation errors. + +This script updates all test files to use the new 4-parameter async AsyncAuditQueue API: +- Old: Arc::new(AsyncAuditQueue::new(wal_path)) +- New: AsyncAuditQueue::new(wal_path, pool, batch_size, flush_interval).await? + +Changes applied: +1. Add .await? to AsyncAuditQueue::new() calls +2. Add 3 new parameters (pool, batch_size, flush_interval) +3. Remove Arc::new() wrapping (returns Arc internally) +4. Fix function signatures to be async +5. Fix AuditTrailEngine::new() calls (add pool and wal_path parameters) +""" + +import re +import sys +from pathlib import Path + +def fix_async_audit_queue_new(content: str) -> str: + """Fix AsyncAuditQueue::new() calls to use new 4-parameter async API.""" + + # Pattern 1: Arc::new(AsyncAuditQueue::new(wal_path)) + # Replace with: AsyncAuditQueue::new(wal_path, Arc::clone(&pool), 100, 100).await? + pattern1 = r'Arc::new\(AsyncAuditQueue::new\(([^)]+)\)\)' + replacement1 = r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")' + content = re.sub(pattern1, replacement1, content) + + # Pattern 2: AsyncAuditQueue::new(wal_path) without Arc::new + # Replace with: AsyncAuditQueue::new(wal_path, Arc::clone(&pool), 100, 100).await? + pattern2 = r'AsyncAuditQueue::new\(([^,)]+)\)(?!\s*\.await)' + replacement2 = r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")' + content = re.sub(pattern2, replacement2, content) + + return content + +def fix_audit_trail_engine_new(content: str) -> str: + """Fix AuditTrailEngine::new() calls to use new 3-parameter async API.""" + + # Pattern: AuditTrailEngine::new(config) -> AuditTrailEngine::new(config, pool, wal_path).await? + pattern = r'AuditTrailEngine::new\(([^)]+)\)(?!\s*\.await)' + + def replacement(match): + config = match.group(1).strip() + # If only config, add pool and wal_path + if ',' not in config: + return f'AuditTrailEngine::new({config}, Arc::clone(&pool), wal_path.clone()).await.expect("Failed to create AuditTrailEngine")' + return match.group(0) + + content = re.sub(pattern, replacement, content) + return content + +def add_async_to_test_functions(content: str) -> str: + """Add async to test functions that use .await.""" + + # Pattern: #[test] or #[tokio::test] followed by fn function_name() + # If function body contains .await, ensure #[tokio::test] + lines = content.split('\n') + result = [] + i = 0 + + while i < len(lines): + line = lines[i] + + # Check if this is a test function + if '#[test]' in line or '#[tokio::test]' in line: + # Look ahead to find the function body + fn_start = i + 1 + while fn_start < len(lines) and 'fn ' not in lines[fn_start]: + fn_start += 1 + + if fn_start < len(lines): + # Check if function uses .await + fn_line = lines[fn_start] + + # Find function body end + brace_count = 0 + fn_body_lines = [] + j = fn_start + started = False + + while j < len(lines): + if '{' in lines[j]: + started = True + brace_count += lines[j].count('{') + if '}' in lines[j]: + brace_count -= lines[j].count('}') + + fn_body_lines.append(lines[j]) + j += 1 + + if started and brace_count == 0: + break + + fn_body = '\n'.join(fn_body_lines) + + # If function uses .await, ensure it's async and uses #[tokio::test] + if '.await' in fn_body: + # Replace #[test] with #[tokio::test] + if '#[test]' in line: + line = line.replace('#[test]', '#[tokio::test]') + + # Make function async if not already + if 'async fn ' not in fn_line: + lines[fn_start] = fn_line.replace('fn ', 'async fn ') + + result.append(line) + i += 1 + + return '\n'.join(result) + +def add_pool_setup(content: str) -> str: + """Add PostgreSQL pool setup to test functions that need it.""" + + lines = content.split('\n') + result = [] + + for i, line in enumerate(lines): + result.append(line) + + # If this is a test function that uses AsyncAuditQueue or AuditTrailEngine + if ('async fn test_' in line or 'async fn bench_' in line) and i + 1 < len(lines): + # Look ahead for AsyncAuditQueue or AuditTrailEngine usage + body_start = i + 1 + while body_start < len(lines) and '{' not in lines[body_start]: + body_start += 1 + + if body_start < len(lines): + # Check next 50 lines for usage + check_lines = '\n'.join(lines[body_start:min(body_start + 50, len(lines))]) + + if ('AsyncAuditQueue::new' in check_lines or 'AuditTrailEngine::new' in check_lines) and 'setup_test_postgres_pool' not in check_lines: + # Add pool setup after opening brace + result.append(' // Setup PostgreSQL pool for audit persistence') + result.append(' let pool = setup_test_postgres_pool().await;') + result.append(' let wal_path = std::env::temp_dir().join(format!("foxhunt_audit_test_{}.wal", uuid::Uuid::new_v4()));') + result.append('') + + return '\n'.join(result) + +def fix_audit_config_fields(content: str) -> str: + """Fix AuditTrailConfig field names that have changed.""" + + # Remove old fields that no longer exist + old_fields = [ + r'\s*enabled:\s*true,?\n', + r'\s*compression_algorithm:\s*CompressionAlgorithm::\w+,?\n', + r'\s*encryption_algorithm:\s*EncryptionAlgorithm::\w+,?\n', + r'\s*encryption_key:\s*vec!\[.*?\],?\n', + r'\s*postgres_pool:\s*\w+,?\n', + r'\s*file_path:\s*\w+,?\n', + r'\s*enable_checksums:\s*\w+,?\n', + r'\s*enable_tamper_detection:\s*\w+,?\n', + r'\s*enable_best_execution_tracking:\s*\w+,?\n', + ] + + for pattern in old_fields: + content = re.sub(pattern, '', content) + + # Fix enum variant names + content = content.replace('EncryptionAlgorithm::Aes256Gcm', 'EncryptionAlgorithm::AES256GCM') + + return content + +def fix_postgres_pool_new(content: str) -> str: + """Fix PostgresPool::new() calls - it takes only 1 argument (connection string).""" + + # Pattern: PostgresPool::new(&database_url, pool_size) + # Replace with: PostgresPool::connect(&database_url).await? + pattern = r'PostgresPool::new\(([^,)]+)(?:,\s*\d+)?\)' + replacement = r'PostgresPool::connect(\1).await.expect("Failed to connect to PostgreSQL")' + content = re.sub(pattern, replacement, content) + + return content + +def process_file(file_path: Path) -> bool: + """Process a single test file.""" + + print(f"Processing {file_path}...") + + try: + content = file_path.read_text() + original = content + + # Apply all fixes + content = fix_async_audit_queue_new(content) + content = fix_audit_trail_engine_new(content) + content = add_async_to_test_functions(content) + content = add_pool_setup(content) + content = fix_audit_config_fields(content) + content = fix_postgres_pool_new(content) + + # Only write if changed + if content != original: + file_path.write_text(content) + print(f" āœ… Updated {file_path.name}") + return True + else: + print(f" ā­ļø No changes needed for {file_path.name}") + return False + + except Exception as e: + print(f" āŒ Error processing {file_path}: {e}") + return False + +def main(): + """Main entry point.""" + + # Find all test files in trading_engine/tests/ + test_dir = Path(__file__).parent.parent / 'trading_engine' / 'tests' + + if not test_dir.exists(): + print(f"Error: Test directory not found: {test_dir}") + sys.exit(1) + + test_files = list(test_dir.glob('*.rs')) + + if not test_files: + print(f"Error: No test files found in {test_dir}") + sys.exit(1) + + print(f"Found {len(test_files)} test files to process\n") + + updated_count = 0 + for test_file in sorted(test_files): + if process_file(test_file): + updated_count += 1 + + print(f"\n{'='*60}") + print(f"Summary: Updated {updated_count} of {len(test_files)} files") + print(f"{'='*60}") + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/fix_async_audit_queue_tests_v2.py b/scripts/fix_async_audit_queue_tests_v2.py new file mode 100755 index 000000000..949f66e69 --- /dev/null +++ b/scripts/fix_async_audit_queue_tests_v2.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +""" +Fixed migration script for AsyncAuditQueue API changes. +""" + +import re +import sys +from pathlib import Path + +def fix_file_content(content: str) -> str: + """Apply all fixes to file content.""" + + # Fix 1: AsyncAuditQueue::new() - add parameters and .await + # Pattern: AsyncAuditQueue::new(wal_path_expr) + # Replace: AsyncAuditQueue::new(wal_path_expr, Arc::clone(&pool), 100, 100).await.expect("...") + content = re.sub( + r'AsyncAuditQueue::new\(([^)]+)\)(?!\s*\.await)', + r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")', + content + ) + + # Fix 2: Remove Arc::new() wrapper from AsyncAuditQueue (it returns Arc now) + content = re.sub( + r'Arc::new\(AsyncAuditQueue::new\(([^)]+), Arc::clone\(&pool\), 100, 100\)\.await\.expect\("Failed to create AsyncAuditQueue"\)\)', + r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")', + content + ) + + # Fix 3: AuditTrailEngine::new() - add pool and wal_path parameters + # Only fix if it doesn't already have .await + content = re.sub( + r'AuditTrailEngine::new\(([^,)]+)\)(?!\s*\.await)', + r'AuditTrailEngine::new(\1, Arc::clone(&pool), wal_path.clone()).await.expect("Failed to create AuditTrailEngine")', + content + ) + + # Fix 4: PostgresPool::new() with 2 args -> PostgresPool::connect() with 1 arg + content = re.sub( + r'PostgresPool::new\(([^,)]+),\s*\d+\)', + r'PostgresPool::connect(\1).await.expect("Failed to connect to PostgreSQL")', + content + ) + + # Fix 5: Add missing pool setup at start of test functions if needed + # This is done more carefully per-function + lines = content.split('\n') + result = [] + i = 0 + + while i < len(lines): + line = lines[i] + result.append(line) + + # Check if this is an async test function + if 'async fn test_' in line or 'async fn bench_' in line: + # Look for opening brace + j = i + 1 + while j < len(lines) and '{' not in lines[j]: + result.append(lines[j]) + j += 1 + + if j < len(lines): + result.append(lines[j]) # Add the line with { + + # Check if next few lines already have pool setup + has_pool = False + for k in range(j + 1, min(j + 5, len(lines))): + if 'let pool =' in lines[k] or 'setup_test_postgres_pool' in lines[k]: + has_pool = True + break + + # Check if function uses AsyncAuditQueue or AuditTrailEngine + needs_pool = False + for k in range(j + 1, min(j + 30, len(lines))): + if 'AsyncAuditQueue::new' in lines[k] or 'AuditTrailEngine::new' in lines[k]: + needs_pool = True + break + + # Add pool setup if needed + if needs_pool and not has_pool: + result.append(' // Setup PostgreSQL pool for audit persistence') + result.append(' let pool = setup_test_postgres_pool().await;') + result.append(' let wal_path = std::env::temp_dir().join(format!("foxhunt_audit_test_{}.wal", uuid::Uuid::new_v4()));') + result.append('') + + i = j + + i += 1 + + content = '\n'.join(result) + + # Fix 6: Remove old AuditTrailConfig fields + old_fields_patterns = [ + r'\s+enabled:\s*\w+,?\s*\n', + r'\s+compression_algorithm:\s*CompressionAlgorithm::\w+,?\s*\n', + r'\s+encryption_algorithm:\s*EncryptionAlgorithm::\w+,?\s*\n', + r'\s+encryption_key:\s*vec!\[.*?\],?\s*\n', + r'\s+postgres_pool:\s*[^,\n]+,?\s*\n', + r'\s+file_path:\s*[^,\n]+,?\s*\n', + r'\s+enable_checksums:\s*\w+,?\s*\n', + r'\s+enable_tamper_detection:\s*\w+,?\s*\n', + r'\s+enable_best_execution_tracking:\s*\w+,?\s*\n', + r'\s+enable_mifid_reporting:\s*\w+,?\s*\n', + ] + + for pattern in old_fields_patterns: + content = re.sub(pattern, '', content) + + # Fix 7: Fix enum variants + content = content.replace('EncryptionAlgorithm::Aes256Gcm', 'EncryptionAlgorithm::AES256GCM') + + # Fix 8: Remove duplicate PostgresPool::connect().await.expect().await patterns + content = re.sub( + r'PostgresPool::connect\(([^)]+)\)\.await\.expect\("([^"]+)"\)\.await', + r'PostgresPool::connect(\1).await.expect("\2")', + content + ) + + return content + +def process_file(file_path: Path) -> bool: + """Process a single test file.""" + print(f"Processing {file_path.name}...") + + try: + original = file_path.read_text() + fixed = fix_file_content(original) + + if fixed != original: + file_path.write_text(fixed) + print(f" āœ… Fixed {file_path.name}") + return True + else: + print(f" ā­ļø No changes for {file_path.name}") + return False + + except Exception as e: + print(f" āŒ Error: {e}") + return False + +def main(): + test_dir = Path(__file__).parent.parent / 'trading_engine' / 'tests' + + if not test_dir.exists(): + print(f"Error: {test_dir} not found") + return 1 + + test_files = sorted(test_dir.glob('*.rs')) + + if not test_files: + print(f"Error: No test files in {test_dir}") + return 1 + + print(f"Found {len(test_files)} test files\n") + + updated = sum(1 for f in test_files if process_file(f)) + + print(f"\n{'='*60}") + print(f"Updated {updated}/{len(test_files)} files") + print(f"{'='*60}") + + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/fix_audit_compliance.py b/scripts/fix_audit_compliance.py new file mode 100644 index 000000000..40492e0cf --- /dev/null +++ b/scripts/fix_audit_compliance.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +""" +Fix audit_compliance.rs by updating all AuditTrailEngine::new() calls. +This script performs surgical fixes to update the API from: + AuditTrailEngine::new(config) +to: + AuditTrailEngine::new(config, pool, wal_path).await.unwrap() +""" + +import re +import sys + +def fix_audit_compliance(file_path): + with open(file_path, 'r') as f: + content = f.read() + + original_content = content + + # Pattern 1: Find test functions and add wal_path after pg_pool setup + # We'll insert WAL path right before AuditTrailEngine::new() calls + + # Find all AuditTrailEngine::new(config) patterns + pattern = r'let audit_engine = AuditTrailEngine::new\(config\)\.await\.unwrap\(\);' + + def replacement(match): + # Extract indentation + return '''let wal_path = std::env::temp_dir().join(format!("audit_wal_{}.log", uuid::Uuid::new_v4())); + let audit_engine = AuditTrailEngine::new(config, pg_pool.clone().unwrap(), wal_path).await.unwrap();''' + + # First, let's find all occurrences and their line positions + lines = content.split('\n') + fixed_lines = [] + + for i, line in enumerate(lines): + if 'let audit_engine = AuditTrailEngine::new(config).await.unwrap();' in line: + # Get indentation + indent = len(line) - len(line.lstrip()) + spaces = ' ' * indent + + # Add wal_path line before audit_engine + fixed_lines.append(f'{spaces}let wal_path = std::env::temp_dir().join(format!("audit_wal_{{}}.log", uuid::Uuid::new_v4()));') + # Fix the audit_engine line + fixed_lines.append(f'{spaces}let audit_engine = AuditTrailEngine::new(config, pg_pool.clone().unwrap(), wal_path).await.unwrap();') + else: + fixed_lines.append(line) + + content = '\n'.join(fixed_lines) + + # Count changes + changes = content.count('pg_pool.clone().unwrap(), wal_path') + + # Write back + if content != original_content: + with open(file_path, 'w') as f: + f.write(content) + print(f"āœ… Fixed {changes} AuditTrailEngine::new() calls in {file_path}") + return True + else: + print(f"āš ļø No changes needed in {file_path}") + return False + +if __name__ == '__main__': + file_path = '/home/jgrusewski/Work/foxhunt/trading_engine/tests/audit_compliance.rs' + if fix_audit_compliance(file_path): + sys.exit(0) + else: + sys.exit(1)