🐍 Wave 112: Python fix scripts for audit tests

- fix_all_audit_tests.py: Comprehensive audit test fixes
- fix_async_audit_queue_tests.py: Async queue test fixes (v1)
- fix_async_audit_queue_tests_v2.py: Async queue test fixes (v2)
- fix_audit_compliance.py: Compliance test fixes
This commit is contained in:
jgrusewski
2025-10-05 22:23:43 +02:00
parent 0d9f890aa6
commit a7973d4bf7
4 changed files with 639 additions and 0 deletions

View File

@@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""
Comprehensive fix for all audit trail test files.
Fixes the AsyncAuditQueue and AuditTrailEngine API changes from Wave 107.
"""
import re
import sys
from pathlib import Path
# Files to fix
TEST_FILES = [
"trading_engine/tests/async_audit_queue_tests.rs",
"trading_engine/tests/audit_trail_persistence_test.rs",
"trading_engine/tests/audit_persistence_tests.rs",
"trading_engine/tests/audit_persistence_comprehensive.rs",
"trading_engine/tests/audit_retention_tests.rs",
]
def fix_audit_trail_persistence_test(content):
"""Fix audit_trail_persistence_test.rs specific issues"""
# Pattern 1: Fix AuditTrailEngine::new() - old API without await
# OLD: let audit_engine = AuditTrailEngine::new(audit_config);
# NEW: let wal_path = ...; let audit_engine = AuditTrailEngine::new(audit_config, postgres_pool.clone(), wal_path).await.unwrap();
lines = content.split('\n')
fixed_lines = []
for i, line in enumerate(lines):
# Fix AuditTrailEngine::new() calls without .await
if 'let audit_engine = AuditTrailEngine::new(audit_config);' in line:
indent = len(line) - len(line.lstrip())
spaces = ' ' * indent
fixed_lines.append(f'{spaces}let wal_path = std::env::temp_dir().join(format!("audit_wal_{{}}.log", uuid::Uuid::new_v4()));')
fixed_lines.append(f'{spaces}let audit_engine = AuditTrailEngine::new(audit_config, postgres_pool.clone(), wal_path).await.unwrap();')
continue
# Remove .set_postgres_pool() calls (no longer needed)
if '.set_postgres_pool(' in line and 'audit_engine' in line:
continue
# Fix AuditTrailConfig - remove old fields
if 'real_time_persistence:' in line:
continue
fixed_lines.append(line)
return '\n'.join(fixed_lines)
def fix_async_audit_queue_tests(content):
"""Fix async_audit_queue_tests.rs specific issues"""
lines = content.split('\n')
fixed_lines = []
for i, line in enumerate(lines):
# Fix AsyncAuditQueue::new() - now async and returns Result
# OLD: let queue = Arc::new(AsyncAuditQueue::new(wal_path.clone()));
# NEW: let queue = AsyncAuditQueue::new(wal_path.clone(), pool.clone(), 100, 100).await.unwrap();
if 'Arc::new(AsyncAuditQueue::new(' in line and 'wal_path' in line:
indent = len(line) - len(line.lstrip())
spaces = ' ' * indent
# Extract variable name
var_match = re.search(r'let (\w+)', line)
if var_match:
var_name = var_match.group(1)
fixed_lines.append(f'{spaces}let {var_name} = AsyncAuditQueue::new(wal_path.clone(), pool.clone(), 100, 100).await.unwrap();')
continue
# Fix submit calls that need to be await
if '.submit(' in line and 'queue' in line and not '.await' in line and not 'expect' in line:
line = line.replace('.submit(', '.submit(').replace(')?;', ').await?;').replace(');', ').await.unwrap();')
fixed_lines.append(line)
return '\n'.join(fixed_lines)
def fix_audit_config_fields(content):
"""Remove deprecated AuditTrailConfig fields"""
deprecated_fields = [
'enabled:',
'compression_algorithm:',
'encryption_algorithm:',
'encryption_key:',
'postgres_pool:',
'file_path:',
'enable_checksums:',
'enable_tamper_detection:',
'enable_best_execution_tracking:',
'enable_mifid_reporting:',
'compression_enabled:',
'encryption_enabled:',
'retention_days:',
'real_time_persistence:',
]
lines = content.split('\n')
fixed_lines = []
skip_next = False
for i, line in enumerate(lines):
if skip_next:
skip_next = False
continue
# Skip lines with deprecated fields
if any(field in line for field in deprecated_fields):
continue
fixed_lines.append(line)
return '\n'.join(fixed_lines)
def fix_file(file_path):
"""Fix a single test file"""
with open(file_path, 'r') as f:
content = f.read()
original_content = content
# Apply fixes based on file
if 'audit_trail_persistence_test' in file_path:
content = fix_audit_trail_persistence_test(content)
elif 'async_audit_queue_tests' in file_path:
content = fix_async_audit_queue_tests(content)
# Apply common fixes to all files
content = fix_audit_config_fields(content)
# Fix PostgresPool::new() calls - should use PostgresPool::new(config).await
content = re.sub(
r'PostgresPool::new\(postgres_config, (\d+)\)',
r'PostgresPool::new(postgres_config)',
content
)
# Write back if changed
if content != original_content:
with open(file_path, 'w') as f:
f.write(content)
print(f"✅ Fixed {file_path}")
return True
else:
print(f"⚠️ No changes in {file_path}")
return False
def main():
base_path = Path('/home/jgrusewski/Work/foxhunt')
fixed_count = 0
for test_file in TEST_FILES:
file_path = base_path / test_file
if file_path.exists():
if fix_file(str(file_path)):
fixed_count += 1
else:
print(f"❌ File not found: {file_path}")
print(f"\n📊 Summary: Fixed {fixed_count}/{len(TEST_FILES)} files")
if fixed_count > 0:
sys.exit(0)
else:
sys.exit(1)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,237 @@
#!/usr/bin/env python3
"""
Migration script to fix AsyncAuditQueue test compilation errors.
This script updates all test files to use the new 4-parameter async AsyncAuditQueue API:
- Old: Arc::new(AsyncAuditQueue::new(wal_path))
- New: AsyncAuditQueue::new(wal_path, pool, batch_size, flush_interval).await?
Changes applied:
1. Add .await? to AsyncAuditQueue::new() calls
2. Add 3 new parameters (pool, batch_size, flush_interval)
3. Remove Arc::new() wrapping (returns Arc internally)
4. Fix function signatures to be async
5. Fix AuditTrailEngine::new() calls (add pool and wal_path parameters)
"""
import re
import sys
from pathlib import Path
def fix_async_audit_queue_new(content: str) -> str:
"""Fix AsyncAuditQueue::new() calls to use new 4-parameter async API."""
# Pattern 1: Arc::new(AsyncAuditQueue::new(wal_path))
# Replace with: AsyncAuditQueue::new(wal_path, Arc::clone(&pool), 100, 100).await?
pattern1 = r'Arc::new\(AsyncAuditQueue::new\(([^)]+)\)\)'
replacement1 = r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")'
content = re.sub(pattern1, replacement1, content)
# Pattern 2: AsyncAuditQueue::new(wal_path) without Arc::new
# Replace with: AsyncAuditQueue::new(wal_path, Arc::clone(&pool), 100, 100).await?
pattern2 = r'AsyncAuditQueue::new\(([^,)]+)\)(?!\s*\.await)'
replacement2 = r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")'
content = re.sub(pattern2, replacement2, content)
return content
def fix_audit_trail_engine_new(content: str) -> str:
"""Fix AuditTrailEngine::new() calls to use new 3-parameter async API."""
# Pattern: AuditTrailEngine::new(config) -> AuditTrailEngine::new(config, pool, wal_path).await?
pattern = r'AuditTrailEngine::new\(([^)]+)\)(?!\s*\.await)'
def replacement(match):
config = match.group(1).strip()
# If only config, add pool and wal_path
if ',' not in config:
return f'AuditTrailEngine::new({config}, Arc::clone(&pool), wal_path.clone()).await.expect("Failed to create AuditTrailEngine")'
return match.group(0)
content = re.sub(pattern, replacement, content)
return content
def add_async_to_test_functions(content: str) -> str:
"""Add async to test functions that use .await."""
# Pattern: #[test] or #[tokio::test] followed by fn function_name()
# If function body contains .await, ensure #[tokio::test]
lines = content.split('\n')
result = []
i = 0
while i < len(lines):
line = lines[i]
# Check if this is a test function
if '#[test]' in line or '#[tokio::test]' in line:
# Look ahead to find the function body
fn_start = i + 1
while fn_start < len(lines) and 'fn ' not in lines[fn_start]:
fn_start += 1
if fn_start < len(lines):
# Check if function uses .await
fn_line = lines[fn_start]
# Find function body end
brace_count = 0
fn_body_lines = []
j = fn_start
started = False
while j < len(lines):
if '{' in lines[j]:
started = True
brace_count += lines[j].count('{')
if '}' in lines[j]:
brace_count -= lines[j].count('}')
fn_body_lines.append(lines[j])
j += 1
if started and brace_count == 0:
break
fn_body = '\n'.join(fn_body_lines)
# If function uses .await, ensure it's async and uses #[tokio::test]
if '.await' in fn_body:
# Replace #[test] with #[tokio::test]
if '#[test]' in line:
line = line.replace('#[test]', '#[tokio::test]')
# Make function async if not already
if 'async fn ' not in fn_line:
lines[fn_start] = fn_line.replace('fn ', 'async fn ')
result.append(line)
i += 1
return '\n'.join(result)
def add_pool_setup(content: str) -> str:
"""Add PostgreSQL pool setup to test functions that need it."""
lines = content.split('\n')
result = []
for i, line in enumerate(lines):
result.append(line)
# If this is a test function that uses AsyncAuditQueue or AuditTrailEngine
if ('async fn test_' in line or 'async fn bench_' in line) and i + 1 < len(lines):
# Look ahead for AsyncAuditQueue or AuditTrailEngine usage
body_start = i + 1
while body_start < len(lines) and '{' not in lines[body_start]:
body_start += 1
if body_start < len(lines):
# Check next 50 lines for usage
check_lines = '\n'.join(lines[body_start:min(body_start + 50, len(lines))])
if ('AsyncAuditQueue::new' in check_lines or 'AuditTrailEngine::new' in check_lines) and 'setup_test_postgres_pool' not in check_lines:
# Add pool setup after opening brace
result.append(' // Setup PostgreSQL pool for audit persistence')
result.append(' let pool = setup_test_postgres_pool().await;')
result.append(' let wal_path = std::env::temp_dir().join(format!("foxhunt_audit_test_{}.wal", uuid::Uuid::new_v4()));')
result.append('')
return '\n'.join(result)
def fix_audit_config_fields(content: str) -> str:
"""Fix AuditTrailConfig field names that have changed."""
# Remove old fields that no longer exist
old_fields = [
r'\s*enabled:\s*true,?\n',
r'\s*compression_algorithm:\s*CompressionAlgorithm::\w+,?\n',
r'\s*encryption_algorithm:\s*EncryptionAlgorithm::\w+,?\n',
r'\s*encryption_key:\s*vec!\[.*?\],?\n',
r'\s*postgres_pool:\s*\w+,?\n',
r'\s*file_path:\s*\w+,?\n',
r'\s*enable_checksums:\s*\w+,?\n',
r'\s*enable_tamper_detection:\s*\w+,?\n',
r'\s*enable_best_execution_tracking:\s*\w+,?\n',
]
for pattern in old_fields:
content = re.sub(pattern, '', content)
# Fix enum variant names
content = content.replace('EncryptionAlgorithm::Aes256Gcm', 'EncryptionAlgorithm::AES256GCM')
return content
def fix_postgres_pool_new(content: str) -> str:
"""Fix PostgresPool::new() calls - it takes only 1 argument (connection string)."""
# Pattern: PostgresPool::new(&database_url, pool_size)
# Replace with: PostgresPool::connect(&database_url).await?
pattern = r'PostgresPool::new\(([^,)]+)(?:,\s*\d+)?\)'
replacement = r'PostgresPool::connect(\1).await.expect("Failed to connect to PostgreSQL")'
content = re.sub(pattern, replacement, content)
return content
def process_file(file_path: Path) -> bool:
"""Process a single test file."""
print(f"Processing {file_path}...")
try:
content = file_path.read_text()
original = content
# Apply all fixes
content = fix_async_audit_queue_new(content)
content = fix_audit_trail_engine_new(content)
content = add_async_to_test_functions(content)
content = add_pool_setup(content)
content = fix_audit_config_fields(content)
content = fix_postgres_pool_new(content)
# Only write if changed
if content != original:
file_path.write_text(content)
print(f" ✅ Updated {file_path.name}")
return True
else:
print(f" ⏭️ No changes needed for {file_path.name}")
return False
except Exception as e:
print(f" ❌ Error processing {file_path}: {e}")
return False
def main():
"""Main entry point."""
# Find all test files in trading_engine/tests/
test_dir = Path(__file__).parent.parent / 'trading_engine' / 'tests'
if not test_dir.exists():
print(f"Error: Test directory not found: {test_dir}")
sys.exit(1)
test_files = list(test_dir.glob('*.rs'))
if not test_files:
print(f"Error: No test files found in {test_dir}")
sys.exit(1)
print(f"Found {len(test_files)} test files to process\n")
updated_count = 0
for test_file in sorted(test_files):
if process_file(test_file):
updated_count += 1
print(f"\n{'='*60}")
print(f"Summary: Updated {updated_count} of {len(test_files)} files")
print(f"{'='*60}")
return 0
if __name__ == '__main__':
sys.exit(main())

View File

@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""
Fixed migration script for AsyncAuditQueue API changes.
"""
import re
import sys
from pathlib import Path
def fix_file_content(content: str) -> str:
"""Apply all fixes to file content."""
# Fix 1: AsyncAuditQueue::new() - add parameters and .await
# Pattern: AsyncAuditQueue::new(wal_path_expr)
# Replace: AsyncAuditQueue::new(wal_path_expr, Arc::clone(&pool), 100, 100).await.expect("...")
content = re.sub(
r'AsyncAuditQueue::new\(([^)]+)\)(?!\s*\.await)',
r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")',
content
)
# Fix 2: Remove Arc::new() wrapper from AsyncAuditQueue (it returns Arc now)
content = re.sub(
r'Arc::new\(AsyncAuditQueue::new\(([^)]+), Arc::clone\(&pool\), 100, 100\)\.await\.expect\("Failed to create AsyncAuditQueue"\)\)',
r'AsyncAuditQueue::new(\1, Arc::clone(&pool), 100, 100).await.expect("Failed to create AsyncAuditQueue")',
content
)
# Fix 3: AuditTrailEngine::new() - add pool and wal_path parameters
# Only fix if it doesn't already have .await
content = re.sub(
r'AuditTrailEngine::new\(([^,)]+)\)(?!\s*\.await)',
r'AuditTrailEngine::new(\1, Arc::clone(&pool), wal_path.clone()).await.expect("Failed to create AuditTrailEngine")',
content
)
# Fix 4: PostgresPool::new() with 2 args -> PostgresPool::connect() with 1 arg
content = re.sub(
r'PostgresPool::new\(([^,)]+),\s*\d+\)',
r'PostgresPool::connect(\1).await.expect("Failed to connect to PostgreSQL")',
content
)
# Fix 5: Add missing pool setup at start of test functions if needed
# This is done more carefully per-function
lines = content.split('\n')
result = []
i = 0
while i < len(lines):
line = lines[i]
result.append(line)
# Check if this is an async test function
if 'async fn test_' in line or 'async fn bench_' in line:
# Look for opening brace
j = i + 1
while j < len(lines) and '{' not in lines[j]:
result.append(lines[j])
j += 1
if j < len(lines):
result.append(lines[j]) # Add the line with {
# Check if next few lines already have pool setup
has_pool = False
for k in range(j + 1, min(j + 5, len(lines))):
if 'let pool =' in lines[k] or 'setup_test_postgres_pool' in lines[k]:
has_pool = True
break
# Check if function uses AsyncAuditQueue or AuditTrailEngine
needs_pool = False
for k in range(j + 1, min(j + 30, len(lines))):
if 'AsyncAuditQueue::new' in lines[k] or 'AuditTrailEngine::new' in lines[k]:
needs_pool = True
break
# Add pool setup if needed
if needs_pool and not has_pool:
result.append(' // Setup PostgreSQL pool for audit persistence')
result.append(' let pool = setup_test_postgres_pool().await;')
result.append(' let wal_path = std::env::temp_dir().join(format!("foxhunt_audit_test_{}.wal", uuid::Uuid::new_v4()));')
result.append('')
i = j
i += 1
content = '\n'.join(result)
# Fix 6: Remove old AuditTrailConfig fields
old_fields_patterns = [
r'\s+enabled:\s*\w+,?\s*\n',
r'\s+compression_algorithm:\s*CompressionAlgorithm::\w+,?\s*\n',
r'\s+encryption_algorithm:\s*EncryptionAlgorithm::\w+,?\s*\n',
r'\s+encryption_key:\s*vec!\[.*?\],?\s*\n',
r'\s+postgres_pool:\s*[^,\n]+,?\s*\n',
r'\s+file_path:\s*[^,\n]+,?\s*\n',
r'\s+enable_checksums:\s*\w+,?\s*\n',
r'\s+enable_tamper_detection:\s*\w+,?\s*\n',
r'\s+enable_best_execution_tracking:\s*\w+,?\s*\n',
r'\s+enable_mifid_reporting:\s*\w+,?\s*\n',
]
for pattern in old_fields_patterns:
content = re.sub(pattern, '', content)
# Fix 7: Fix enum variants
content = content.replace('EncryptionAlgorithm::Aes256Gcm', 'EncryptionAlgorithm::AES256GCM')
# Fix 8: Remove duplicate PostgresPool::connect().await.expect().await patterns
content = re.sub(
r'PostgresPool::connect\(([^)]+)\)\.await\.expect\("([^"]+)"\)\.await',
r'PostgresPool::connect(\1).await.expect("\2")',
content
)
return content
def process_file(file_path: Path) -> bool:
"""Process a single test file."""
print(f"Processing {file_path.name}...")
try:
original = file_path.read_text()
fixed = fix_file_content(original)
if fixed != original:
file_path.write_text(fixed)
print(f" ✅ Fixed {file_path.name}")
return True
else:
print(f" ⏭️ No changes for {file_path.name}")
return False
except Exception as e:
print(f" ❌ Error: {e}")
return False
def main():
test_dir = Path(__file__).parent.parent / 'trading_engine' / 'tests'
if not test_dir.exists():
print(f"Error: {test_dir} not found")
return 1
test_files = sorted(test_dir.glob('*.rs'))
if not test_files:
print(f"Error: No test files in {test_dir}")
return 1
print(f"Found {len(test_files)} test files\n")
updated = sum(1 for f in test_files if process_file(f))
print(f"\n{'='*60}")
print(f"Updated {updated}/{len(test_files)} files")
print(f"{'='*60}")
return 0
if __name__ == '__main__':
sys.exit(main())

View File

@@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""
Fix audit_compliance.rs by updating all AuditTrailEngine::new() calls.
This script performs surgical fixes to update the API from:
AuditTrailEngine::new(config)
to:
AuditTrailEngine::new(config, pool, wal_path).await.unwrap()
"""
import re
import sys
def fix_audit_compliance(file_path):
with open(file_path, 'r') as f:
content = f.read()
original_content = content
# Pattern 1: Find test functions and add wal_path after pg_pool setup
# We'll insert WAL path right before AuditTrailEngine::new() calls
# Find all AuditTrailEngine::new(config) patterns
pattern = r'let audit_engine = AuditTrailEngine::new\(config\)\.await\.unwrap\(\);'
def replacement(match):
# Extract indentation
return '''let wal_path = std::env::temp_dir().join(format!("audit_wal_{}.log", uuid::Uuid::new_v4()));
let audit_engine = AuditTrailEngine::new(config, pg_pool.clone().unwrap(), wal_path).await.unwrap();'''
# First, let's find all occurrences and their line positions
lines = content.split('\n')
fixed_lines = []
for i, line in enumerate(lines):
if 'let audit_engine = AuditTrailEngine::new(config).await.unwrap();' in line:
# Get indentation
indent = len(line) - len(line.lstrip())
spaces = ' ' * indent
# Add wal_path line before audit_engine
fixed_lines.append(f'{spaces}let wal_path = std::env::temp_dir().join(format!("audit_wal_{{}}.log", uuid::Uuid::new_v4()));')
# Fix the audit_engine line
fixed_lines.append(f'{spaces}let audit_engine = AuditTrailEngine::new(config, pg_pool.clone().unwrap(), wal_path).await.unwrap();')
else:
fixed_lines.append(line)
content = '\n'.join(fixed_lines)
# Count changes
changes = content.count('pg_pool.clone().unwrap(), wal_path')
# Write back
if content != original_content:
with open(file_path, 'w') as f:
f.write(content)
print(f"✅ Fixed {changes} AuditTrailEngine::new() calls in {file_path}")
return True
else:
print(f"⚠️ No changes needed in {file_path}")
return False
if __name__ == '__main__':
file_path = '/home/jgrusewski/Work/foxhunt/trading_engine/tests/audit_compliance.rs'
if fix_audit_compliance(file_path):
sys.exit(0)
else:
sys.exit(1)