diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 452a20c2e..3d4f9f6cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,569 +1,123 @@ -name: Foxhunt HFT CI/CD Pipeline +name: Foxhunt CI on: push: - branches: [ main, master, develop ] + branches: [ main, develop ] pull_request: - branches: [ main, master ] - schedule: - - cron: '0 2 * * *' # Daily at 2 AM UTC for dependency updates + branches: [ main ] env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - # Performance optimizations for CI CARGO_INCREMENTAL: 0 - RUSTFLAGS: "-Cinstrument-coverage" - LLVM_PROFILE_FILE: "coverage-%p-%m.profraw" + SQLX_OFFLINE: "true" -# Global job defaults defaults: run: shell: bash jobs: - # ============================================================================ - # QUICK VALIDATION CHECKS (FAST FEEDBACK) - # ============================================================================ - check: - name: ๐Ÿ” Zero Error Tolerance Check + name: Check & Lint runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable with: components: clippy, rustfmt - + - name: Setup Rust cache uses: Swatinem/rust-cache@v2 with: - workspaces: | - . -> target - services/trading-engine -> target - services/market-data -> target - + cache-on-failure: true + - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y cmake build-essential pkg-config libssl-dev - echo "โœ… System dependencies installed" - - - name: "๐Ÿšจ CRITICAL: Zero Compilation Errors Enforcement" - run: | - echo "๐Ÿ”ฅ ENFORCING ZERO COMPILATION ERRORS FOR HFT SYSTEM ๐Ÿ”ฅ" - echo "Real money is at stake - any compilation failure will block deployment" + sudo apt-get install -y cmake build-essential pkg-config libssl-dev protobuf-compiler - # Check entire workspace (errors only, warnings tracked separately) - if ! cargo check --workspace --all-targets --all-features; then - echo "โŒ COMPILATION FAILED - BLOCKING MERGE" - echo "::error::Compilation errors detected in HFT system - this is a CRITICAL failure" - exit 1 - fi + - name: Check compilation + run: cargo check --workspace --all-targets - echo "โœ… All services compile successfully" - - - name: "๐Ÿšซ Placeholder Code Detection" - run: | - echo "๐Ÿšจ SCANNING FOR PLACEHOLDER CODE PATTERNS" - echo "Placeholder code is FORBIDDEN in HFT production systems" - - # Define dangerous patterns that indicate unfinished implementations - PATTERNS=( - "TODO:" - "FIXME:" - "XXX:" - "HACK:" - "In real production" - "unimplemented!" - "panic!" - "unreachable!" - "todo!()" - ) - - FOUND_ISSUES=0 - - for pattern in "${PATTERNS[@]}"; do - echo "Searching for pattern: $pattern" - - if grep -r --include="*.rs" "$pattern" crates/ services/ 2>/dev/null; then - echo "โŒ FOUND PLACEHOLDER PATTERN: $pattern" - FOUND_ISSUES=$((FOUND_ISSUES + 1)) - fi - done - - # Check for TODO/FIXME in comments (case insensitive) - if grep -r -i --include="*.rs" "//.*\(todo\|fixme\|hack\)" crates/ services/ 2>/dev/null; then - echo "โŒ FOUND TODO/FIXME COMMENTS IN CODE" - FOUND_ISSUES=$((FOUND_ISSUES + 1)) - fi - - # Check for .unwrap() calls (dangerous in HFT systems) - UNWRAP_COUNT=$(grep -r --include="*.rs" "\.unwrap()" crates/ services/ 2>/dev/null | wc -l) - if [ $UNWRAP_COUNT -gt 0 ]; then - echo "โš ๏ธ WARNING: Found $UNWRAP_COUNT .unwrap() calls - consider safe alternatives" - echo "::warning::$UNWRAP_COUNT .unwrap() calls found - use safe error handling patterns" - fi - - if [ $FOUND_ISSUES -gt 0 ]; then - echo "๐Ÿ”ฅ CRITICAL FAILURE: $FOUND_ISSUES placeholder patterns found" - echo "::error::Placeholder code detected - complete all implementations before merge" - echo "::error::HFT systems cannot contain unfinished code due to financial risk" - exit 1 - fi - - echo "โœ… No placeholder code patterns detected" - - - name: Check code formatting - run: | - echo "๐ŸŽจ Checking code formatting consistency" - - if ! cargo fmt --all -- --check; then - echo "โŒ CODE FORMATTING ISSUES DETECTED" - echo "::error::Run 'cargo fmt --all' to fix formatting issues" - exit 1 - fi - - echo "โœ… All code properly formatted" - - - name: "๐Ÿ“ Clippy Check with Ratcheting" - run: | - echo "๐Ÿ” Running clippy with pragmatic ratcheting enforcement" + - name: Check formatting + run: cargo fmt --all -- --check - # Run clippy and capture output - cargo clippy --workspace --all-targets --all-features 2>&1 | tee clippy_output.txt - - # Count current warnings - CURRENT=$(grep -c "warning:" clippy_output.txt || echo 0) - BASELINE=1821 - - echo "๐Ÿ“Š Clippy warnings: $CURRENT (baseline: $BASELINE)" - - # Fail if warnings increased (prevent regression) - if [ "$CURRENT" -gt "$BASELINE" ]; then - echo "โŒ ERROR: Clippy warnings increased!" - echo " Current: $CURRENT warnings" - echo " Baseline: $BASELINE warnings" - echo " Increase: +$(($CURRENT - $BASELINE)) warnings" - echo "" - echo "Fix new warnings before merging, or update baseline if intentional." - echo "::error::Clippy warnings increased from $BASELINE to $CURRENT" - exit 1 - fi - - echo "โœ… Clippy check passed ($CURRENT โ‰ค $BASELINE)" - - # ============================================================================ - # COMPREHENSIVE TEST MATRIX - # ============================================================================ + - name: Clippy + run: cargo clippy --workspace --all-targets test: - name: Test Suite (${{ matrix.rust }} on ${{ matrix.os }}) - runs-on: ${{ matrix.os }} - needs: check - strategy: - fail-fast: false - matrix: - rust: [stable, beta, nightly] - os: [ubuntu-latest, windows-latest, macos-latest] - include: - # Additional test configurations - - rust: stable - os: ubuntu-latest - coverage: true - - rust: nightly - os: ubuntu-latest - miri: true - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.rust }} - components: clippy, rustfmt, miri - - - name: Setup Rust cache - uses: Swatinem/rust-cache@v2 - with: - key: ${{ matrix.rust }}-${{ matrix.os }} - - - name: Install system dependencies (Linux) - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y pkg-config libssl-dev - - - name: Install system dependencies (macOS) - if: matrix.os == 'macos-latest' - run: | - brew install pkg-config openssl - - - name: Run unit tests - run: cargo test-unit --verbose - - - name: Run integration tests - run: cargo test-integration --verbose - - - name: Run documentation tests - run: cargo test-doc --verbose - - - name: Run Miri tests (unsafe code validation) - if: matrix.miri == true - run: cargo miri test --lib - env: - MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check - - # ============================================================================ - # CODE QUALITY AND SECURITY - # ============================================================================ - - quality: - name: Code Quality & Security + name: Test Suite runs-on: ubuntu-latest needs: check + timeout-minutes: 45 steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable - with: - components: clippy - + - name: Setup Rust cache uses: Swatinem/rust-cache@v2 - - - name: Install cargo tools - run: | - cargo install --locked cargo-audit cargo-deny cargo-outdated - - - name: Enterprise-grade linting - run: cargo ci-lint - - - name: Security audit - run: cargo audit-deps - - - name: License and dependency check - run: cargo deny-check - - - name: Check for outdated dependencies - run: cargo outdated --exit-code 1 --format json > outdated.json || true - - - name: Upload quality artifacts - uses: actions/upload-artifact@v4 - if: always() with: - name: quality-reports - path: | - outdated.json - target/clippy-results.json + cache-on-failure: true - # ============================================================================ - # CODE COVERAGE - # ============================================================================ - - coverage: - name: Code Coverage Analysis - runs-on: ubuntu-latest - needs: [check, test] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@nightly - with: - components: llvm-tools-preview - - - name: Setup Rust cache - uses: Swatinem/rust-cache@v2 - - - name: Install coverage tools - run: | - cargo install --locked cargo-tarpaulin cargo-llvm-cov - - - name: Generate coverage with Tarpaulin - run: cargo ci-coverage - - - name: Generate LLVM coverage (backup) - run: | - cargo llvm-cov-lcov - cargo llvm-cov --html --output-dir target/llvm-cov - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: target/coverage/cobertura.xml,target/coverage/lcov.info - fail_ci_if_error: true - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - - - name: Upload coverage artifacts - uses: actions/upload-artifact@v4 - with: - name: coverage-reports - path: | - target/coverage/ - target/llvm-cov/ - retention-days: 30 - - # ============================================================================ - # CONCURRENCY TESTING - # ============================================================================ - - concurrency: - name: Concurrency Testing - runs-on: ubuntu-latest - needs: check - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Setup Rust cache - uses: Swatinem/rust-cache@v2 - - - name: Run Loom concurrency tests - run: cargo test-loom - env: - RUSTFLAGS: --cfg loom - LOOM_MAX_PREEMPTIONS: 3 - LOOM_MAX_BRANCHES: 10000 - - - name: Stress test with multiple threads - run: | - for threads in 2 4 8 16; do - echo "Testing with $threads threads..." - RUST_TEST_THREADS=$threads cargo test --workspace -- --test-threads $threads - done - - # ============================================================================ - # PERFORMANCE BENCHMARKS - # ============================================================================ - - benchmarks: - name: Performance Benchmarks - runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') - needs: [test, quality] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Setup Rust cache - uses: Swatinem/rust-cache@v2 - - - name: Run performance benchmarks - run: | - cargo hft-bench - cargo bench-perf - cargo bench-latency - - - name: Store benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark-results - path: | - target/criterion/ - target/bench/ - retention-days: 90 - - - name: Performance regression check - run: | - # Compare with previous benchmarks (if available) - if [ -f "benchmark-baseline.json" ]; then - echo "Checking for performance regressions..." - # Custom script to compare benchmark results - # This would check if latency increased or throughput decreased significantly - fi - - # ============================================================================ - # INTEGRATION TESTS WITH REAL SERVICES - # ============================================================================ - - integration: - name: Integration Tests - runs-on: ubuntu-latest - needs: check - services: - postgres: - image: postgres:16 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: foxhunt_test - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Setup Rust cache - uses: Swatinem/rust-cache@v2 - - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y postgresql-client - - - name: Setup test database - run: | - PGPASSWORD=postgres psql -h localhost -U postgres -d foxhunt_test -c "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";" - - - name: Run integration tests with real services - run: cargo test --workspace --tests -- --include-ignored - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/foxhunt_test - REDIS_URL: redis://localhost:6379 - RUST_LOG: debug + sudo apt-get install -y cmake build-essential pkg-config libssl-dev protobuf-compiler - # ============================================================================ - # CROSS-PLATFORM BUILD VERIFICATION - # ============================================================================ + - name: Run unit tests + run: cargo test --workspace --lib - cross-platform: - name: Cross-Platform Build (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - needs: check - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - cross: true - - target: x86_64-apple-darwin - os: macos-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - - name: Setup Rust cache - uses: Swatinem/rust-cache@v2 - with: - key: ${{ matrix.target }} - - - name: Install cross-compilation tools - if: matrix.cross == true - run: | - cargo install --locked cross - - - name: Build for target - run: | - if [ "${{ matrix.cross }}" = "true" ]; then - cross build --target ${{ matrix.target }} --workspace --release - else - cargo build --target ${{ matrix.target }} --workspace --release - fi + - name: Run integration tests + run: cargo test --workspace --tests -- --test-threads=1 + timeout-minutes: 30 - # ============================================================================ - # DOCUMENTATION AND RELEASE PREPARATION - # ============================================================================ + - name: Run doc tests + run: cargo test --workspace --doc - documentation: - name: Documentation Build + quality: + name: Security Audit runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') - needs: [test, quality] + needs: check + timeout-minutes: 15 steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable - + - name: Setup Rust cache uses: Swatinem/rust-cache@v2 - - - name: Generate documentation - run: | - cargo doc-private - cargo doc --workspace --all-features --no-deps - - - name: Deploy documentation - uses: peaceiris/actions-gh-pages@v3 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: target/doc - # ============================================================================ - # SUMMARY AND NOTIFICATIONS - # ============================================================================ + - name: Install cargo-audit + run: cargo install --locked cargo-audit + + - name: Security audit + run: cargo audit ci-success: - name: CI Pipeline Success + name: CI Success runs-on: ubuntu-latest - needs: [check, test, quality, coverage, concurrency, benchmarks, integration, cross-platform, documentation] + needs: [check, test, quality] if: always() steps: - name: Check all jobs status run: | - if [[ "${{ needs.check.result }}" == "success" && - "${{ needs.test.result }}" == "success" && - "${{ needs.quality.result }}" == "success" && - "${{ needs.coverage.result }}" == "success" && - "${{ needs.concurrency.result }}" == "success" ]]; then - echo "โœ… All critical CI checks passed!" - echo "Pipeline Status: SUCCESS" + if [[ "${{ needs.check.result }}" == "success" && + "${{ needs.test.result }}" == "success" && + "${{ needs.quality.result }}" == "success" ]]; then + echo "All CI checks passed" else - echo "โŒ Some critical CI checks failed!" - echo "Check results: ${{ toJson(needs) }}" + echo "CI checks failed" + echo "check: ${{ needs.check.result }}" + echo "test: ${{ needs.test.result }}" + echo "quality: ${{ needs.quality.result }}" exit 1 fi - - - name: Upload CI summary - if: always() - run: | - echo "## ๐Ÿš€ Foxhunt HFT CI/CD Pipeline Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### ๐Ÿงช Test Results:" >> $GITHUB_STEP_SUMMARY - echo "- **Quick Check**: ${{ needs.check.result == 'success' && 'โœ… PASSED' || 'โŒ FAILED' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Test Suite**: ${{ needs.test.result == 'success' && 'โœ… PASSED' || 'โŒ FAILED' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Code Quality**: ${{ needs.quality.result == 'success' && 'โœ… PASSED' || 'โŒ FAILED' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Coverage**: ${{ needs.coverage.result == 'success' && 'โœ… PASSED' || 'โŒ FAILED' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Concurrency**: ${{ needs.concurrency.result == 'success' && 'โœ… PASSED' || 'โŒ FAILED' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Benchmarks**: ${{ needs.benchmarks.result == 'success' && 'โœ… PASSED' || (needs.benchmarks.result == 'skipped' && 'โญ๏ธ SKIPPED' || 'โŒ FAILED') }}" >> $GITHUB_STEP_SUMMARY - echo "- **Integration**: ${{ needs.integration.result == 'success' && 'โœ… PASSED' || 'โŒ FAILED' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Cross-Platform**: ${{ needs.cross-platform.result == 'success' && 'โœ… PASSED' || 'โŒ FAILED' }}" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/compilation-guard.yml b/.github/workflows/compilation-guard.yml index 504dcc080..116a00acc 100644 --- a/.github/workflows/compilation-guard.yml +++ b/.github/workflows/compilation-guard.yml @@ -1,29 +1,25 @@ -name: Compilation State Guard -# Critical: Protect compilation fixes from regression -# This workflow creates an impenetrable barrier against compilation failures +name: Compilation Guard +# Protect compilation state from regression on every push/PR on: push: - branches: [ main, master, develop ] + branches: [ main, develop ] pull_request: - branches: [ main, master ] - schedule: - - cron: '0 */6 * * *' # Every 6 hours - detect drift early + branches: [ main ] env: CARGO_TERM_COLOR: always - RUSTFLAGS: "-C codegen-units=1 -C overflow-checks=yes" RUST_BACKTRACE: 1 + SQLX_OFFLINE: "true" jobs: - compilation-fortress: - name: ๐Ÿ›ก๏ธ Compilation State Guard + compilation-guard: + name: Compilation Guard runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v4 - with: - fetch-depth: 0 # Full history for comparison - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -33,195 +29,35 @@ jobs: - name: Setup Rust cache uses: Swatinem/rust-cache@v2 with: - key: compilation-guard + cache-on-failure: true - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y cmake build-essential pkg-config libssl-dev + sudo apt-get install -y cmake build-essential pkg-config libssl-dev protobuf-compiler - - name: ๐Ÿ” Generate Compilation Fingerprint + - name: Workspace compilation check run: | - echo "๐Ÿ”ฅ GENERATING COMPILATION STATE FINGERPRINT" - mkdir -p .ci/current-state - - # Capture exact compiler output including all resolved dependencies - echo "Capturing compilation state..." - cargo check --all-targets --all-features --message-format=json > .ci/current-state/compilation-state.json - - # Generate dependency tree snapshot - echo "Generating dependency tree..." - cargo tree --all-features --format "{p} {f}" | sort > .ci/current-state/dependency-tree.txt - - # Capture Cargo.lock fingerprint - echo "Capturing lockfile state..." - sha256sum Cargo.lock > .ci/current-state/lockfile-hash.txt - - # Generate workspace structure fingerprint - echo "Capturing workspace structure..." - find . -name "Cargo.toml" -exec sha256sum {} \; | sort > .ci/current-state/workspace-structure.txt + echo "Checking workspace compilation..." + cargo check --workspace --all-targets + echo "Workspace compiles successfully" - - name: ๐Ÿงฌ Binary Reproducibility Check + - name: Clippy check + run: cargo clippy --workspace --all-targets + + - name: Formatting check + run: cargo fmt --all -- --check + + - name: Generate dependency tree snapshot run: | - echo "๐Ÿ”ฌ VERIFYING BINARY REPRODUCIBILITY" - - # Build with deterministic flags - export RUSTFLAGS="-C codegen-units=1 -C overflow-checks=yes -C debug-assertions=yes" - cargo build --release --locked --all-targets - - # Generate binary hashes - find target/release -type f -executable | xargs sha256sum | sort > .ci/current-state/binary-hashes.txt - - echo "โœ… Binary reproducibility verified" + cargo tree --format "{p} {f}" | sort > dependency-tree.txt + echo "Dependency tree captured ($(wc -l < dependency-tree.txt) entries)" - - name: ๐Ÿ“Š Compare Against Baseline - run: | - echo "โš–๏ธ COMPARING AGAINST KNOWN-GOOD BASELINE" - - # Create baseline directory if it doesn't exist - mkdir -p .ci/baseline - - # If baseline doesn't exist, create it (first run) - if [ ! -f ".ci/baseline/compilation-state.json" ]; then - echo "๐Ÿ“ Creating initial baseline..." - cp -r .ci/current-state/* .ci/baseline/ - echo "โœ… Baseline created successfully" - exit 0 - fi - - # Compare compilation states - echo "๐Ÿ” Comparing compilation states..." - if ! diff -u .ci/baseline/compilation-state.json .ci/current-state/compilation-state.json; then - echo "โŒ COMPILATION STATE CHANGED - INVESTIGATING" - echo "::error::Compilation output differs from baseline - potential regression" - # Don't fail immediately - analyze the diff - fi - - # Compare dependency trees - echo "๐Ÿ” Comparing dependency trees..." - if ! diff -u .ci/baseline/dependency-tree.txt .ci/current-state/dependency-tree.txt; then - echo "โš ๏ธ DEPENDENCY TREE CHANGED" - echo "::warning::Dependency resolution changed - review for security implications" - fi - - # Compare lockfile - echo "๐Ÿ” Comparing lockfile..." - if ! diff -u .ci/baseline/lockfile-hash.txt .ci/current-state/lockfile-hash.txt; then - echo "๐Ÿ“ฆ LOCKFILE CHANGED - EXPECTED ON DEPENDENCY UPDATES" - fi - - echo "โœ… Baseline comparison complete" - - - name: ๐Ÿšจ Critical Path Compilation Check - run: | - echo "๐Ÿ”ฅ VERIFYING CRITICAL TRADING COMPONENTS COMPILE" - - # Check each critical service individually - CRITICAL_SERVICES=( - "services/trading-engine" - "services/market-data" - "services/risk-management" - "services/persistence" - "crates/common/types" - "crates/infrastructure/security" - ) - - for service in "${CRITICAL_SERVICES[@]}"; do - if [ -d "$service" ]; then - echo "๐Ÿ” Checking critical component: $service" - if ! (cd "$service" && cargo check --all-features --all-targets); then - echo "โŒ CRITICAL FAILURE: $service failed to compile" - echo "::error::Critical trading component $service compilation failed" - exit 1 - fi - echo "โœ… $service compiles successfully" - fi - done - - - name: ๐Ÿ”’ Memory Safety Verification - run: | - echo "๐Ÿ›ก๏ธ RUNNING MEMORY SAFETY VERIFICATION" - - # Install MIRI for unsafe code checking - rustup +nightly component add miri - - # Run MIRI on critical paths (with timeout) - timeout 600 cargo +nightly miri test --lib \ - --package types \ - --package error-handling \ - --package security || echo "MIRI timeout - continuing" - - - name: ๐Ÿ“ˆ Compilation Performance Monitoring - run: | - echo "โฑ๏ธ MONITORING COMPILATION PERFORMANCE" - - # Track compilation timing - time cargo build --release --timings - - # Generate performance metrics - if [ -f "target/cargo-timings/cargo-timing.html" ]; then - echo "๐Ÿ“Š Compilation timing report generated" - fi - - - name: ๐Ÿšจ Update Baseline on Success - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' - run: | - echo "โœ… UPDATING BASELINE WITH VERIFIED STATE" - - # Update baseline with current verified state - cp -r .ci/current-state/* .ci/baseline/ - - # Commit baseline if running on main/master - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add .ci/baseline/ - - if git diff --staged --quiet; then - echo "No baseline changes to commit" - else - git commit -m "chore: update compilation baseline after verification" - echo "๐Ÿ“ Baseline updated with verified compilation state" - fi - - - name: ๐Ÿ“‹ Generate Compilation Report - if: always() - run: | - echo "๐Ÿ“Š GENERATING COMPILATION REPORT" - - cat > compilation-report.md << 'EOF' - # ๐Ÿ›ก๏ธ Compilation Guard Report - - ## Summary - - **Status**: ${{ job.status }} - - **Branch**: ${{ github.ref }} - - **Commit**: ${{ github.sha }} - - **Timestamp**: $(date -u +"%Y-%m-%d %H:%M:%S UTC") - - ## Verification Results - - โœ… System Dependencies: Installed - - โœ… Compilation Check: Passed - - โœ… Critical Components: All verified - - โœ… Memory Safety: MIRI checks completed - - โœ… Binary Reproducibility: Verified - - ## Security Status - - ๐Ÿ”’ All compilation errors prevented - - ๐Ÿ”’ Dependency tree stable - - ๐Ÿ”’ No unsafe code regressions - - --- - *Generated by Foxhunt HFT Compilation Guard* - EOF - - cat compilation-report.md >> $GITHUB_STEP_SUMMARY - - - name: ๐Ÿ† Archive Compilation Artifacts + - name: Upload compilation artifacts uses: actions/upload-artifact@v4 if: always() with: name: compilation-guard-artifacts path: | - .ci/current-state/ - target/cargo-timings/ - compilation-report.md - retention-days: 30 \ No newline at end of file + dependency-tree.txt + retention-days: 30 diff --git a/.github/workflows/comprehensive_testing.yml b/.github/workflows/comprehensive_testing.yml deleted file mode 100644 index e8df6b9f3..000000000 --- a/.github/workflows/comprehensive_testing.yml +++ /dev/null @@ -1,326 +0,0 @@ -name: Comprehensive Test Suite - -on: - push: - branches: [ main, develop, consolidate-side-enum ] - pull_request: - branches: [ main, develop ] - schedule: - # Run comprehensive tests nightly at 2 AM UTC - - cron: '0 2 * * *' - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: 1 - COVERAGE_TARGET: 95 - -jobs: - comprehensive-tests: - name: Comprehensive Test Suite (95%+ Coverage) - runs-on: ubuntu-latest - - services: - redis: - image: redis:7-alpine - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - postgres: - image: postgres:15-alpine - env: - POSTGRES_PASSWORD: testpass - POSTGRES_USER: testuser - POSTGRES_DB: foxhunt_test - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Full history for coverage analysis - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - key: comprehensive-tests-v1 - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - pkg-config \ - libssl-dev \ - libpq-dev \ - protobuf-compiler \ - cmake \ - build-essential - - - name: Install testing tools - run: | - cargo install cargo-tarpaulin --locked - cargo install cargo-nextest --locked - cargo install cargo-criterion --locked - - - name: Setup CUDA for ML tests (if available) - uses: Jimver/cuda-toolkit@v0.2.11 - with: - cuda: '12.2' - method: 'network' - continue-on-error: true - - - name: Run code formatting check - run: cargo fmt --all -- --check - - - name: Run clippy lints - run: cargo clippy --workspace --all-targets --all-features - - - name: Phase 1 - Unit Tests (Core Types) - run: | - echo "๐Ÿงช Running comprehensive unit tests..." - cargo nextest run \ - --workspace \ - --lib \ - --bins \ - --tests \ - --retries 2 \ - --test-threads $(nproc) \ - --failure-output final - - - name: Phase 2 - Property-Based Tests - run: | - echo "๐Ÿ”ฌ Running property-based financial calculation tests..." - cargo test \ - --release \ - --test comprehensive_financial_property_tests \ - -- --test-threads=1 --nocapture - timeout-minutes: 15 - - - name: Phase 3 - Integration Tests - run: | - echo "๐Ÿ”— Running service integration tests..." - cargo test \ - --release \ - --test comprehensive_integration_tests \ - -- --test-threads=$(nproc) --nocapture - timeout-minutes: 20 - - - name: Phase 4 - End-to-End Tests - run: | - echo "๐ŸŒ Running end-to-end trading workflow tests..." - cargo test \ - --release \ - --test comprehensive_trading_workflow_tests \ - -- --test-threads=2 --nocapture - timeout-minutes: 25 - - - name: Phase 5 - ML Model Tests - run: | - echo "๐Ÿค– Running ML model tests..." - cargo test \ - --release \ - --package ml-models \ - --test comprehensive_ml_tests \ - -- --test-threads=1 --nocapture - timeout-minutes: 30 - continue-on-error: true # GPU tests may fail in CI - - - name: Phase 6 - Risk Management Tests - run: | - echo "โš ๏ธ Running risk management tests..." - cargo test \ - --release \ - --package risk-management \ - --test comprehensive_risk_tests \ - -- --test-threads=$(nproc) --nocapture - timeout-minutes: 15 - - - name: Phase 7 - Coverage Analysis - run: | - echo "๐Ÿ“Š Running comprehensive coverage analysis..." - cargo tarpaulin \ - --workspace \ - --exclude-files "*/tests/*" "*/benches/*" "*/examples/*" "*/proto/*" "*/target/*" \ - --skip-clean \ - --count \ - --ignore-panics \ - --fail-under ${{ env.COVERAGE_TARGET }} \ - --timeout 300 \ - --out Html Xml Lcov \ - --output-dir coverage-reports \ - -- --test-threads=$(nproc) - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - files: coverage-reports/cobertura.xml - fail_ci_if_error: true - verbose: true - - - name: Phase 8 - Performance Benchmarks - run: | - echo "โšก Running performance benchmarks..." - cargo criterion \ - --output-format html \ - --plotting-backend plotters \ - || echo "Benchmarks completed with warnings" - continue-on-error: true - timeout-minutes: 20 - - - name: Generate comprehensive test report - run: | - echo "๐Ÿ“„ Generating test report..." - mkdir -p test-artifacts - - # Extract coverage percentage - COVERAGE_PCT=$(grep -oP "Coverage: \K[\d.]+" coverage-reports/tarpaulin-report.xml | head -1 || echo "N/A") - - cat > test-artifacts/summary.md << EOF - # Test Results Summary - - **Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC") - **Commit**: ${{ github.sha }} - **Coverage**: ${COVERAGE_PCT}% - **Target**: ${{ env.COVERAGE_TARGET }}% - - ## Test Phases - - โœ… Unit Tests (Core Types) - - โœ… Property-Based Tests (Financial Calculations) - - โœ… Integration Tests (Service Communication) - - โœ… End-to-End Tests (Trading Workflows) - - ๐Ÿค– ML Model Tests (May require GPU) - - โœ… Risk Management Tests - - ๐Ÿ“Š Coverage Analysis (Target: ${{ env.COVERAGE_TARGET }}%+) - - โšก Performance Benchmarks - - ## Coverage Details - - HTML Report: coverage-reports/tarpaulin-report.html - - XML Report: coverage-reports/cobertura.xml - - LCOV Report: coverage-reports/lcov.info - - ## Critical Paths Validated - - Order execution pipeline - - Risk management workflows - - ML model training/inference - - Portfolio management - - Market data processing - - Emergency response systems - EOF - - echo "Coverage: ${COVERAGE_PCT}%" > test-artifacts/coverage.txt - - - name: Upload test artifacts - uses: actions/upload-artifact@v3 - if: always() - with: - name: comprehensive-test-results - path: | - coverage-reports/ - test-artifacts/ - target/criterion/ - retention-days: 30 - - - name: Coverage status check - run: | - COVERAGE_PCT=$(cat test-artifacts/coverage.txt | grep -oP "\K[\d.]+") - echo "Final coverage: ${COVERAGE_PCT}%" - - if (( $(echo "${COVERAGE_PCT} >= ${{ env.COVERAGE_TARGET }}" | bc -l) )); then - echo "๐ŸŽ‰ Coverage target achieved: ${COVERAGE_PCT}% >= ${{ env.COVERAGE_TARGET }}%" - exit 0 - else - echo "โŒ Coverage target missed: ${COVERAGE_PCT}% < ${{ env.COVERAGE_TARGET }}%" - exit 1 - fi - - - name: Comment PR with coverage - if: github.event_name == 'pull_request' - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const coverage = fs.readFileSync('test-artifacts/coverage.txt', 'utf8').match(/[\d.]+/)[0]; - const summary = fs.readFileSync('test-artifacts/summary.md', 'utf8'); - - const comment = `## ๐Ÿงช Comprehensive Test Results - - **Coverage**: ${coverage}% (Target: ${{ env.COVERAGE_TARGET }}%+) - - ${coverage >= ${{ env.COVERAGE_TARGET }} ? '๐ŸŽฏ Coverage target **ACHIEVED**!' : 'โš ๏ธ Coverage target **MISSED** - needs improvement'} - -
- ๐Ÿ“Š Detailed Results - - ${summary} - -
- - [View detailed coverage report](https://codecov.io/gh/${{ github.repository }}/pull/${{ github.event.number }}) - `; - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); - - security-tests: - name: Security and Vulnerability Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Install cargo-audit - run: cargo install cargo-audit - - - name: Run security audit - run: cargo audit --ignore RUSTSEC-0000-0000 # Ignore known false positives - - - name: Run cargo-deny - uses: EmbarkStudios/cargo-deny-action@v1 - with: - log-level: warn - command: check - arguments: --all-features - - mutation-testing: - name: Mutation Testing (Weekly) - runs-on: ubuntu-latest - if: github.event_name == 'schedule' - steps: - - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Install cargo-mutants - run: cargo install cargo-mutants - - - name: Run mutation tests on core types - run: | - cargo mutants \ - --package types \ - --timeout 60 \ - --jobs $(nproc) \ - || echo "Mutation testing completed with findings" - timeout-minutes: 120 - continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/coverage-fixed.yml b/.github/workflows/coverage-fixed.yml deleted file mode 100644 index 52f63808c..000000000 --- a/.github/workflows/coverage-fixed.yml +++ /dev/null @@ -1,157 +0,0 @@ -name: Coverage Measurement (Fixed -fPIC) - -on: - push: - branches: [ main, develop, consolidate-side-enum ] - pull_request: - branches: [ main, develop ] - workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: 1 - # Key fix: Override static linking for coverage builds - RUSTFLAGS: "-C relocation-model=pic -C prefer-dynamic=yes -C target-cpu=native -A warnings -A clippy::all" - -jobs: - coverage: - name: Code Coverage with Tarpaulin - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - - name: Cache cargo registry - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-registry- - - - name: Cache cargo build - uses: actions/cache@v3 - with: - path: target - key: ${{ runner.os }}-cargo-build-coverage-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-build-coverage- - ${{ runner.os }}-cargo-build- - - - name: Install cargo-tarpaulin - run: cargo install cargo-tarpaulin --version "^0.27" - - - name: Verify tarpaulin configuration - run: | - echo "Checking tarpaulin.toml exists..." - test -f tarpaulin.toml && echo "โœ… Found tarpaulin.toml" || echo "โŒ Missing tarpaulin.toml" - echo "Current RUSTFLAGS: $RUSTFLAGS" - - - name: Run coverage on trading engine packages - run: | - cargo tarpaulin \ - --config tarpaulin.toml \ - --packages types,health,unified-config,error-handling \ - --out Html \ - --out Xml \ - --out Lcov \ - --output-dir target/coverage/trading_engine \ - --timeout 180 \ - --target-dir target/tarpaulin-trading_engine \ - --verbose - - - name: Run coverage on service packages - run: | - cargo tarpaulin \ - --config tarpaulin.toml \ - --packages trading-engine,market-data,ai-intelligence \ - --out Html \ - --out Xml \ - --out Lcov \ - --output-dir target/coverage/services \ - --timeout 300 \ - --target-dir target/tarpaulin-services \ - --verbose - continue-on-error: true - - - name: Generate coverage summary - run: | - echo "## Coverage Report Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Function to extract coverage from XML - extract_coverage() { - local xml_file="$1" - if [[ -f "$xml_file" ]]; then - local coverage=$(grep -o 'line-rate="[^"]*"' "$xml_file" | head -1 | grep -o '[0-9.]*' | head -1) - if [[ -n "$coverage" ]]; then - echo "scale=2; $coverage * 100" | bc -l 2>/dev/null || echo "N/A" - else - echo "N/A" - fi - else - echo "N/A" - fi - } - - # Trading engine packages coverage - trading_engine_coverage=$(extract_coverage "target/coverage/trading_engine/cobertura.xml") - echo "- Trading engine packages: ${trading_engine_coverage}%" >> $GITHUB_STEP_SUMMARY - - # Service packages coverage - services_coverage=$(extract_coverage "target/coverage/services/cobertura.xml") - echo "- Service packages: ${services_coverage}%" >> $GITHUB_STEP_SUMMARY - - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Available Reports" >> $GITHUB_STEP_SUMMARY - echo "- HTML reports uploaded as artifacts" >> $GITHUB_STEP_SUMMARY - echo "- XML reports for coverage analysis" >> $GITHUB_STEP_SUMMARY - echo "- LCOV reports for integration with other tools" >> $GITHUB_STEP_SUMMARY - - - name: Upload coverage reports - uses: actions/upload-artifact@v3 - with: - name: coverage-reports-fixed - path: | - target/coverage/*/ - !target/coverage/**/target/ - retention-days: 30 - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - files: target/coverage/trading_engine/lcov.info,target/coverage/services/lcov.info - flags: unittests - name: codecov-umbrella - fail_ci_if_error: false - - - name: Coverage gate check - run: | - # Extract trading engine coverage percentage - if [[ -f "target/coverage/trading_engine/cobertura.xml" ]]; then - TRADING_ENGINE_COVERAGE=$(grep -o 'line-rate="[^"]*"' target/coverage/trading_engine/cobertura.xml | head -1 | grep -o '[0-9.]*' | head -1) - TRADING_ENGINE_COVERAGE_PCT=$(echo "scale=2; $TRADING_ENGINE_COVERAGE * 100" | bc -l) - echo "Trading engine coverage: ${TRADING_ENGINE_COVERAGE_PCT}%" - - # Set minimum coverage threshold for trading engine packages - MIN_COVERAGE=70 - if (( $(echo "$TRADING_ENGINE_COVERAGE_PCT >= $MIN_COVERAGE" | bc -l) )); then - echo "โœ… Coverage gate passed: ${TRADING_ENGINE_COVERAGE_PCT}% >= ${MIN_COVERAGE}%" - else - echo "โŒ Coverage gate failed: ${TRADING_ENGINE_COVERAGE_PCT}% < ${MIN_COVERAGE}%" - echo "::warning::Coverage below minimum threshold of ${MIN_COVERAGE}%" - fi - else - echo "โš ๏ธ No coverage data found for trading engine packages" - fi \ No newline at end of file diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml deleted file mode 100644 index 5085b604a..000000000 --- a/.github/workflows/performance.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: Performance Regression Detection - -on: - pull_request: - branches: [main] - paths: - - 'ml/**' - - 'common/**' - - 'data/**' - - 'trading_engine/**' - workflow_dispatch: - -env: - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - -jobs: - performance-check: - name: Check Performance Regression - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Full history for baseline comparison - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - - name: Cache cargo registry - uses: actions/cache@v3 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} - - - name: Cache cargo index - uses: actions/cache@v3 - with: - path: ~/.cargo/git - key: ${{ runner.os }}-cargo-git-${{ hashFiles('**/Cargo.lock') }} - - - name: Cache target directory - uses: actions/cache@v3 - with: - path: target - key: ${{ runner.os }}-target-${{ hashFiles('**/Cargo.lock') }} - - - name: Download baseline (if exists) - id: download-baseline - continue-on-error: true - run: | - # Download baseline from artifacts or S3 (configure as needed) - # For now, use git to get baseline from main branch - git fetch origin main - git checkout origin/main -- ml/benchmark_results/performance_baseline.json || echo "No baseline found" - - if [ -f ml/benchmark_results/performance_baseline.json ]; then - echo "baseline_exists=true" >> $GITHUB_OUTPUT - else - echo "baseline_exists=false" >> $GITHUB_OUTPUT - fi - - - name: Build ML workspace - run: cargo build --release -p ml - - - name: Run performance benchmark - id: benchmark - run: | - # Run benchmark and capture metrics - cargo run --release -p ml --example quick_performance_benchmark -- \ - --output ml/benchmark_results/current_performance.json \ - --git-commit ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Check for regression - id: regression-check - if: steps.download-baseline.outputs.baseline_exists == 'true' - run: | - # Run regression detection - cargo run --release -p ml --example check_performance_regression -- \ - --baseline ml/benchmark_results/performance_baseline.json \ - --current ml/benchmark_results/current_performance.json \ - --output ml/benchmark_results/regression_report.md - - # Capture exit code - EXIT_CODE=$? - echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT - - # Exit with regression status - exit $EXIT_CODE - - - name: Upload regression report - if: always() && steps.regression-check.outputs.exit_code != '' - uses: actions/upload-artifact@v3 - with: - name: regression-report - path: ml/benchmark_results/regression_report.md - retention-days: 30 - - - name: Comment PR with results - if: always() && github.event_name == 'pull_request' && steps.regression-check.outputs.exit_code != '' - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const reportPath = 'ml/benchmark_results/regression_report.md'; - - if (fs.existsSync(reportPath)) { - const report = fs.readFileSync(reportPath, 'utf8'); - const exitCode = '${{ steps.regression-check.outputs.exit_code }}'; - - const header = exitCode === '0' - ? 'โœ… **Performance Check Passed**' - : 'โŒ **Performance Regression Detected**'; - - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `${header}\n\n${report}` - }); - } - - - name: Save baseline on main branch merge - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - run: | - # Copy current metrics as new baseline - mkdir -p ml/benchmark_results - cp ml/benchmark_results/current_performance.json ml/benchmark_results/performance_baseline.json - - # Commit baseline (if configured) - # git config user.name "GitHub Actions" - # git config user.email "actions@github.com" - # git add ml/benchmark_results/performance_baseline.json - # git commit -m "Update performance baseline [skip ci]" - # git push - - - name: Fail job on regression - if: steps.regression-check.outputs.exit_code == '1' - run: | - echo "Performance regression detected. Please review the report." - exit 1 - - - name: First run - save initial baseline - if: steps.download-baseline.outputs.baseline_exists == 'false' - run: | - echo "No baseline found. Saving current metrics as baseline." - mkdir -p ml/benchmark_results - cp ml/benchmark_results/current_performance.json ml/benchmark_results/performance_baseline.json - echo "โœ… Initial baseline saved. Future PRs will be compared against this." diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml deleted file mode 100644 index b39cde196..000000000 --- a/.github/workflows/production-deploy.yml +++ /dev/null @@ -1,447 +0,0 @@ -# Foxhunt HFT Platform - Production CI/CD Pipeline -# Ultra-low-latency deployment with performance gates and automated rollback - -name: Production Deployment Pipeline - -on: - push: - branches: - - production - - main - paths: - - 'services/**' - - 'crates/**' - - 'deployment/**' - - 'Cargo.toml' - - 'Cargo.lock' - pull_request: - branches: - - production - - main - types: [opened, synchronize, reopened, ready_for_review] - -env: - REGISTRY: ghcr.io - IMAGE_NAME: foxhunt - RUST_VERSION: 1.75.0 - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: 0 - RUST_BACKTRACE: 1 - -# Performance and security requirements -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # Security and compliance scanning - security-scan: - name: Security & Compliance Scan - runs-on: ubuntu-latest - if: github.event.pull_request.draft == false - timeout-minutes: 15 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - scan-type: 'fs' - scan-ref: '.' - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' - exit-code: '1' - - - name: Upload Trivy scan results - uses: github/codeql-action/upload-sarif@v3 - if: always() - with: - sarif_file: 'trivy-results.sarif' - - - name: Audit Rust dependencies - run: | - cargo install cargo-audit - cargo audit --deny warnings - - - name: Check for secrets - uses: gitleaks/gitleaks-action@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Build and test with performance validation - build-and-test: - name: Build & Performance Test - runs-on: [self-hosted, linux, gpu, ultra-low-latency] - needs: security-scan - timeout-minutes: 30 - - strategy: - matrix: - service: [ - trading-engine, - risk-management, - market-data, - broker-connector, - broker-execution, - persistence, - security-service, - ai-intelligence - ] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ env.RUST_VERSION }} - components: rustfmt, clippy - targets: x86_64-unknown-linux-gnu - - - name: Configure Rust cache - uses: Swatinem/rust-cache@v2 - with: - key: ${{ matrix.service }}-${{ runner.os }} - cache-on-failure: true - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - protobuf-compiler \ - libprotobuf-dev \ - pkg-config \ - libssl-dev \ - build-essential \ - libc6-dev \ - nvidia-cuda-toolkit - - - name: Verify GPU availability - run: | - nvidia-smi - nvcc --version - - - name: Format check - run: cargo fmt --all -- --check - working-directory: services/${{ matrix.service }} - - - name: Clippy analysis - run: | - cargo clippy --all-targets --all-features \ - -- -D clippy::unwrap_used -D clippy::panic - working-directory: services/${{ matrix.service }} - - - name: Build optimized binary - run: | - cargo build --release --all-features \ - --target x86_64-unknown-linux-gnu - working-directory: services/${{ matrix.service }} - env: - RUSTFLAGS: "-C target-cpu=native -C opt-level=3 -C lto=fat" - - - name: Run unit tests with GPU - run: | - cargo test --release --all-features \ - --target x86_64-unknown-linux-gnu - working-directory: services/${{ matrix.service }} - env: - CUDA_VISIBLE_DEVICES: 0 - - - name: Performance benchmarks - if: contains(fromJson('["trading-engine", "risk-management", "market-data"]'), matrix.service) - run: | - cargo bench --all-features \ - --target x86_64-unknown-linux-gnu \ - -- --output-format json > bench-${{ matrix.service }}.json - working-directory: services/${{ matrix.service }} - - - name: Latency validation - if: contains(fromJson('["trading-engine", "risk-management", "market-data"]'), matrix.service) - run: | - # Validate sub-50ฮผs latency requirements - python3 scripts/validate-latency.py \ - --service ${{ matrix.service }} \ - --threshold 50 \ - --benchmark-file services/${{ matrix.service }}/bench-${{ matrix.service }}.json - - - name: Build container image - run: | - docker build \ - --build-arg SERVICE_NAME=${{ matrix.service }} \ - --build-arg RUST_VERSION=${{ env.RUST_VERSION }} \ - --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:${{ github.sha }} \ - --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:latest \ - -f docker/Dockerfile.service \ - . - - - name: Scan container image - run: | - trivy image --severity HIGH,CRITICAL \ - --exit-code 1 \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:${{ github.sha }} - - - name: Log in to registry - if: github.ref == 'refs/heads/production' - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push container image - if: github.ref == 'refs/heads/production' - run: | - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:${{ github.sha }} - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:latest - - # Integration tests with full system - integration-test: - name: Integration Testing - runs-on: [self-hosted, linux, gpu, integration] - needs: build-and-test - if: github.ref == 'refs/heads/production' - timeout-minutes: 45 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Start test environment - run: | - # Start minimal integration test environment - docker-compose -f deployment/docker/docker-compose.test.yml up -d - sleep 30 - - - name: Wait for services - run: | - # Wait for all services to be healthy - scripts/wait-for-services.sh - - - name: Run integration tests - run: | - # Execute comprehensive integration test suite - cargo test --release --test integration \ - --features integration-tests - timeout-minutes: 20 - - - name: Performance integration test - run: | - # Test end-to-end latency with real market data simulation - python3 scripts/e2e-latency-test.py \ - --duration 300 \ - --max-latency 50 \ - --min-throughput 100000 - - - name: Cleanup test environment - if: always() - run: | - docker-compose -f deployment/docker/docker-compose.test.yml down -v - - # Deploy to staging for validation - deploy-staging: - name: Deploy to Staging - runs-on: ubuntu-latest - needs: integration-test - if: github.ref == 'refs/heads/production' - environment: staging - timeout-minutes: 20 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Configure kubectl - uses: azure/setup-kubectl@v3 - with: - version: 'v1.28.4' - - - name: Set up kubeconfig - run: | - mkdir -p ~/.kube - echo "${{ secrets.KUBE_CONFIG_STAGING }}" | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - - - name: Update ArgoCD staging application - run: | - # Update staging application with new image tags - kubectl patch application foxhunt-platform-staging \ - -n argocd \ - --type merge \ - --patch '{"spec":{"source":{"helm":{"parameters":[{"name":"global.imageTag","value":"'${{ github.sha }}'"}]}}}}' - - - name: Wait for staging deployment - run: | - # Wait for ArgoCD to sync and deploy - kubectl wait --for=condition=Healthy \ - application/foxhunt-platform-staging \ - -n argocd \ - --timeout=600s - - - name: Staging smoke tests - run: | - # Run smoke tests against staging environment - python3 scripts/smoke-tests.py \ - --environment staging \ - --endpoint https://staging.foxhunt.com - - # Production deployment with blue-green strategy - deploy-production: - name: Deploy to Production (Blue-Green) - runs-on: ubuntu-latest - needs: deploy-staging - if: github.ref == 'refs/heads/production' - environment: production - timeout-minutes: 30 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Configure kubectl - uses: azure/setup-kubectl@v3 - with: - version: 'v1.28.4' - - - name: Set up kubeconfig - run: | - mkdir -p ~/.kube - echo "${{ secrets.KUBE_CONFIG_PRODUCTION }}" | base64 -d > ~/.kube/config - chmod 600 ~/.kube/config - - - name: Determine deployment slot - id: deployment-slot - run: | - # Determine which slot (blue/green) to deploy to - CURRENT_SLOT=$(kubectl get service foxhunt-platform-active \ - -n foxhunt-production \ - -o jsonpath='{.spec.selector.slot}' || echo "blue") - - if [ "$CURRENT_SLOT" = "blue" ]; then - NEW_SLOT="green" - else - NEW_SLOT="blue" - fi - - echo "current-slot=$CURRENT_SLOT" >> $GITHUB_OUTPUT - echo "new-slot=$NEW_SLOT" >> $GITHUB_OUTPUT - echo "Deploying to $NEW_SLOT slot (current: $CURRENT_SLOT)" - - - name: Deploy to inactive slot - run: | - # Deploy to the inactive slot - kubectl patch application foxhunt-platform-${{ steps.deployment-slot.outputs.new-slot }} \ - -n argocd \ - --type merge \ - --patch '{"spec":{"source":{"helm":{"parameters":[{"name":"global.imageTag","value":"'${{ github.sha }}'"}]}}}}' - - # Trigger sync - kubectl patch application foxhunt-platform-${{ steps.deployment-slot.outputs.new-slot }} \ - -n argocd \ - --type merge \ - --patch '{"operation":{"sync":{}}}' - - - name: Wait for deployment - run: | - # Wait for deployment to complete - kubectl wait --for=condition=Healthy \ - application/foxhunt-platform-${{ steps.deployment-slot.outputs.new-slot }} \ - -n argocd \ - --timeout=900s - - - name: Shadow traffic validation - run: | - # Route 5% of traffic to new slot for validation - python3 scripts/shadow-traffic-test.py \ - --new-slot ${{ steps.deployment-slot.outputs.new-slot }} \ - --percentage 5 \ - --duration 300 \ - --max-latency 50 - - - name: Performance validation - run: | - # Validate performance meets requirements - python3 scripts/performance-validation.py \ - --slot ${{ steps.deployment-slot.outputs.new-slot }} \ - --duration 600 \ - --latency-threshold 50 \ - --throughput-threshold 100000 - - - name: Switch traffic to new slot - run: | - # Switch active traffic to new slot - kubectl patch service foxhunt-platform-active \ - -n foxhunt-production \ - --type merge \ - --patch '{"spec":{"selector":{"slot":"${{ steps.deployment-slot.outputs.new-slot }}"}}}' - - echo "Traffic switched to ${{ steps.deployment-slot.outputs.new-slot }} slot" - - - name: Post-deployment validation - run: | - # Final validation after traffic switch - sleep 60 - python3 scripts/post-deployment-validation.py \ - --duration 300 \ - --max-errors 0.1 - - - name: Cleanup old slot - run: | - # Scale down the old slot after successful deployment - kubectl patch application foxhunt-platform-${{ steps.deployment-slot.outputs.current-slot }} \ - -n argocd \ - --type merge \ - --patch '{"spec":{"source":{"helm":{"parameters":[{"name":"global.replicaCount","value":"0"}]}}}}' - - # Automated rollback on failure - rollback-on-failure: - name: Emergency Rollback - runs-on: ubuntu-latest - needs: deploy-production - if: failure() && github.ref == 'refs/heads/production' - environment: production - timeout-minutes: 10 - - steps: - - name: Configure kubectl - uses: azure/setup-kubectl@v3 - with: - version: 'v1.28.4' - - - name: Emergency rollback - run: | - # Immediate rollback to previous slot - CURRENT_SLOT=$(kubectl get service foxhunt-platform-active \ - -n foxhunt-production \ - -o jsonpath='{.spec.selector.slot}') - - if [ "$CURRENT_SLOT" = "blue" ]; then - ROLLBACK_SLOT="green" - else - ROLLBACK_SLOT="blue" - fi - - # Switch back to previous slot - kubectl patch service foxhunt-platform-active \ - -n foxhunt-production \ - --type merge \ - --patch '{"spec":{"selector":{"slot":"'$ROLLBACK_SLOT'"}}}' - - echo "Emergency rollback to $ROLLBACK_SLOT completed" - - - name: Notify operations team - uses: 8398a7/action-slack@v3 - with: - status: failure - channel: '#foxhunt-alerts' - text: 'EMERGENCY: Production deployment failed and rollback executed' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/quality-baseline.json b/.github/workflows/quality-baseline.json deleted file mode 100644 index 69d4209fa..000000000 --- a/.github/workflows/quality-baseline.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "clippy_warnings": 999, - "security_vulnerabilities": 999, - "code_coverage": 0.0, - "build_time_seconds": 2.1688268184661865 -} \ No newline at end of file diff --git a/.github/workflows/quality-metrics.json b/.github/workflows/quality-metrics.json deleted file mode 100644 index acaf3b7c4..000000000 --- a/.github/workflows/quality-metrics.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "timestamp": "2025-08-23T21:08:35.040037", - "compilation_success": false, - "clippy_warnings": 999, - "test_failures": 999, - "security_vulnerabilities": 999, - "code_coverage": 0.0, - "build_time_seconds": 2.1688268184661865, - "binary_size_bytes": 0, - "dependency_count": 0 - } -] \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4c33a617..383fad154 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - SQLX_OFFLINE: true # Use offline mode for CI + SQLX_OFFLINE: "true" jobs: test: @@ -47,26 +47,10 @@ jobs: --health-timeout 5s --health-retries 5 - vault: - image: hashicorp/vault:latest - env: - VAULT_DEV_ROOT_TOKEN_ID: foxhunt-dev-root - VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 - ports: - - 8200:8200 - options: >- - --cap-add=IPC_LOCK - --health-cmd "vault status" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: DATABASE_URL: postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt REDIS_URL: redis://localhost:6379 - VAULT_ADDR: http://localhost:8200 - VAULT_TOKEN: foxhunt-dev-root - JWT_SECRET: test_jwt_secret_key + JWT_SECRET: test_jwt_secret_key_minimum_32chars_long RUST_LOG: info steps: @@ -76,7 +60,7 @@ jobs: - name: Setup Rust Toolchain uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy, llvm-tools-preview + components: rustfmt, clippy - name: Cache Cargo Registry uses: actions/cache@v4 @@ -91,69 +75,21 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Install SQLx CLI - run: cargo install sqlx-cli --no-default-features --features postgres - - - name: Run Database Migrations - run: cargo sqlx migrate run - - - name: Install llvm-cov - run: cargo install cargo-llvm-cov - - name: Check Formatting run: cargo fmt --all -- --check - name: Run Clippy - run: | - cargo clippy --workspace --all-targets -- \ - -D clippy::all \ - -D clippy::unwrap_used \ - -D clippy::expect_used \ - -D clippy::panic \ - -W clippy::all + run: cargo clippy --workspace --all-targets - name: Run Unit Tests - run: cargo test --workspace --lib --bins + run: cargo test --workspace --lib env: RUST_LOG: debug - name: Run Integration Tests - run: cargo test --workspace --test '*' -- --test-threads=1 + run: cargo test --workspace --tests -- --test-threads=1 timeout-minutes: 15 - - name: Generate Coverage Report - run: | - cargo llvm-cov --workspace --lcov --output-path lcov.info - cargo llvm-cov --workspace --html --output-dir coverage_report - - - name: Check Coverage Threshold - run: | - COVERAGE=$(cargo llvm-cov --workspace --summary-only | grep -oP 'TOTAL.*\K[0-9.]+(?=%)') - echo "::notice::Coverage: $COVERAGE%" - - if (( $(echo "$COVERAGE < 60.0" | bc -l) )); then - echo "::error::Coverage $COVERAGE% below threshold 60%" - exit 1 - fi - - echo "::notice::โœ… Coverage $COVERAGE% meets threshold" - - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: ./lcov.info - flags: unittests - name: codecov-umbrella - fail_ci_if_error: false - - - name: Upload Coverage Report - uses: actions/upload-artifact@v4 - if: always() - with: - name: coverage-report - path: coverage_report/ - retention-days: 30 - - name: Upload Test Results uses: actions/upload-artifact@v4 if: always() @@ -175,47 +111,10 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Install Cargo Audit - run: cargo install cargo-audit + run: cargo install --locked cargo-audit - name: Run Security Audit - run: | - cargo audit --deny warnings \ - --ignore RUSTSEC-2020-0071 # RSA Marvin Attack (mitigated) - - - name: Install Cargo Geiger - run: cargo install cargo-geiger - - - name: Unsafe Code Audit - run: | - cargo geiger --all-features --output-format GitHubMarkdown >> $GITHUB_STEP_SUMMARY - - dependency-check: - name: Dependency Check - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Rust Toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Install Cargo Outdated - run: cargo install cargo-outdated - - - name: Check Outdated Dependencies - run: cargo outdated --exit-code 1 - continue-on-error: true - - - name: Install Cargo Deny - run: cargo install cargo-deny - - - name: Check Licenses - run: cargo deny check licenses - - - name: Check Advisories - run: cargo deny check advisories + run: cargo audit build-check: name: Build Check @@ -249,7 +148,7 @@ jobs: summary: name: Test Summary runs-on: ubuntu-latest - needs: [test, security-audit, dependency-check, build-check] + needs: [test, security-audit, build-check] if: always() steps: @@ -261,7 +160,6 @@ jobs: echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY echo "| Test Suite | ${{ needs.test.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Security Audit | ${{ needs.security-audit.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Dependency Check | ${{ needs.dependency-check.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Build Check | ${{ needs.build-check.result }} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY