foxhunt/.github/workflows/production-deploy.yml

# Foxhunt HFT Platform - Production CI/CD Pipeline
# Ultra-low-latency deployment with performance gates and automated rollback

name: Production Deployment Pipeline

on:
  push:
    branches:
      - production
      - main
    paths:
      - 'services/**'
      - 'crates/**'
      - 'deployment/**'
      - 'Cargo.toml'
      - 'Cargo.lock'
  pull_request:
    branches:
      - production
      - main
    types: [opened, synchronize, reopened, ready_for_review]

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: foxhunt
  RUST_VERSION: 1.75.0
  CARGO_TERM_COLOR: always
  CARGO_INCREMENTAL: 0
  RUST_BACKTRACE: 1

# Performance and security requirements
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Security and compliance scanning
  security-scan:
    name: Security & Compliance Scan
    runs-on: ubuntu-latest
    if: github.event.pull_request.draft == false
    timeout-minutes: 15
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        token: ${{ secrets.GITHUB_TOKEN }}

    - name: Run Trivy vulnerability scanner
      uses: aquasecurity/trivy-action@master
      with:
        scan-type: 'fs'
        scan-ref: '.'
        format: 'sarif'
        output: 'trivy-results.sarif'
        severity: 'CRITICAL,HIGH'
        exit-code: '1'

    - name: Upload Trivy scan results
      uses: github/codeql-action/upload-sarif@v3
      if: always()
      with:
        sarif_file: 'trivy-results.sarif'

    - name: Audit Rust dependencies
      run: |
        cargo install cargo-audit
        cargo audit --deny warnings

    - name: Check for secrets
      uses: gitleaks/gitleaks-action@v2
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  # Build and test with performance validation
  build-and-test:
    name: Build & Performance Test
    runs-on: [self-hosted, linux, gpu, ultra-low-latency]
    needs: security-scan
    timeout-minutes: 30
    
    strategy:
      matrix:
        service: [
          trading-engine,
          risk-management,
          market-data,
          broker-connector,
          broker-execution,
          persistence,
          security-service,
          ai-intelligence
        ]
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set up Rust toolchain
      uses: dtolnay/rust-toolchain@stable
      with:
        toolchain: ${{ env.RUST_VERSION }}
        components: rustfmt, clippy
        targets: x86_64-unknown-linux-gnu

    - name: Configure Rust cache
      uses: Swatinem/rust-cache@v2
      with:
        key: ${{ matrix.service }}-${{ runner.os }}
        cache-on-failure: true

    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y \
          protobuf-compiler \
          libprotobuf-dev \
          pkg-config \
          libssl-dev \
          build-essential \
          libc6-dev \
          nvidia-cuda-toolkit

    - name: Verify GPU availability
      run: |
        nvidia-smi
        nvcc --version

    - name: Format check
      run: cargo fmt --all -- --check
      working-directory: services/${{ matrix.service }}

    - name: Clippy analysis
      run: |
        cargo clippy --all-targets --all-features \
          -- -D warnings -D clippy::unwrap_used -D clippy::panic
      working-directory: services/${{ matrix.service }}

    - name: Build optimized binary
      run: |
        cargo build --release --all-features \
          --target x86_64-unknown-linux-gnu
      working-directory: services/${{ matrix.service }}
      env:
        RUSTFLAGS: "-C target-cpu=native -C opt-level=3 -C lto=fat"

    - name: Run unit tests with GPU
      run: |
        cargo test --release --all-features \
          --target x86_64-unknown-linux-gnu
      working-directory: services/${{ matrix.service }}
      env:
        CUDA_VISIBLE_DEVICES: 0

    - name: Performance benchmarks
      if: contains(fromJson('["trading-engine", "risk-management", "market-data"]'), matrix.service)
      run: |
        cargo bench --all-features \
          --target x86_64-unknown-linux-gnu \
          -- --output-format json > bench-${{ matrix.service }}.json
      working-directory: services/${{ matrix.service }}

    - name: Latency validation
      if: contains(fromJson('["trading-engine", "risk-management", "market-data"]'), matrix.service)
      run: |
        # Validate sub-50μs latency requirements
        python3 scripts/validate-latency.py \
          --service ${{ matrix.service }} \
          --threshold 50 \
          --benchmark-file services/${{ matrix.service }}/bench-${{ matrix.service }}.json

    - name: Build container image
      run: |
        docker build \
          --build-arg SERVICE_NAME=${{ matrix.service }} \
          --build-arg RUST_VERSION=${{ env.RUST_VERSION }} \
          --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:${{ github.sha }} \
          --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:latest \
          -f docker/Dockerfile.service \
          .

    - name: Scan container image
      run: |
        trivy image --severity HIGH,CRITICAL \
          --exit-code 1 \
          ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:${{ github.sha }}

    - name: Log in to registry
      if: github.ref == 'refs/heads/production'
      uses: docker/login-action@v3
      with:
        registry: ${{ env.REGISTRY }}
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Push container image
      if: github.ref == 'refs/heads/production'
      run: |
        docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:${{ github.sha }}
        docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.service }}:latest

  # Integration tests with full system
  integration-test:
    name: Integration Testing
    runs-on: [self-hosted, linux, gpu, integration]
    needs: build-and-test
    if: github.ref == 'refs/heads/production'
    timeout-minutes: 45
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Start test environment
      run: |
        # Start minimal integration test environment
        docker-compose -f deployment/docker/docker-compose.test.yml up -d
        sleep 30

    - name: Wait for services
      run: |
        # Wait for all services to be healthy
        scripts/wait-for-services.sh

    - name: Run integration tests
      run: |
        # Execute comprehensive integration test suite
        cargo test --release --test integration \
          --features integration-tests
      timeout-minutes: 20

    - name: Performance integration test
      run: |
        # Test end-to-end latency with real market data simulation
        python3 scripts/e2e-latency-test.py \
          --duration 300 \
          --max-latency 50 \
          --min-throughput 100000

    - name: Cleanup test environment
      if: always()
      run: |
        docker-compose -f deployment/docker/docker-compose.test.yml down -v

  # Deploy to staging for validation
  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    needs: integration-test
    if: github.ref == 'refs/heads/production'
    environment: staging
    timeout-minutes: 20
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Configure kubectl
      uses: azure/setup-kubectl@v3
      with:
        version: 'v1.28.4'

    - name: Set up kubeconfig
      run: |
        mkdir -p ~/.kube
        echo "${{ secrets.KUBE_CONFIG_STAGING }}" | base64 -d > ~/.kube/config
        chmod 600 ~/.kube/config

    - name: Update ArgoCD staging application
      run: |
        # Update staging application with new image tags
        kubectl patch application foxhunt-platform-staging \
          -n argocd \
          --type merge \
          --patch '{"spec":{"source":{"helm":{"parameters":[{"name":"global.imageTag","value":"'${{ github.sha }}'"}]}}}}'

    - name: Wait for staging deployment
      run: |
        # Wait for ArgoCD to sync and deploy
        kubectl wait --for=condition=Healthy \
          application/foxhunt-platform-staging \
          -n argocd \
          --timeout=600s

    - name: Staging smoke tests
      run: |
        # Run smoke tests against staging environment
        python3 scripts/smoke-tests.py \
          --environment staging \
          --endpoint https://staging.foxhunt.com

  # Production deployment with blue-green strategy
  deploy-production:
    name: Deploy to Production (Blue-Green)
    runs-on: ubuntu-latest
    needs: deploy-staging
    if: github.ref == 'refs/heads/production'
    environment: production
    timeout-minutes: 30
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Configure kubectl
      uses: azure/setup-kubectl@v3
      with:
        version: 'v1.28.4'

    - name: Set up kubeconfig
      run: |
        mkdir -p ~/.kube
        echo "${{ secrets.KUBE_CONFIG_PRODUCTION }}" | base64 -d > ~/.kube/config
        chmod 600 ~/.kube/config

    - name: Determine deployment slot
      id: deployment-slot
      run: |
        # Determine which slot (blue/green) to deploy to
        CURRENT_SLOT=$(kubectl get service foxhunt-platform-active \
          -n foxhunt-production \
          -o jsonpath='{.spec.selector.slot}' || echo "blue")
        
        if [ "$CURRENT_SLOT" = "blue" ]; then
          NEW_SLOT="green"
        else
          NEW_SLOT="blue"
        fi
        
        echo "current-slot=$CURRENT_SLOT" >> $GITHUB_OUTPUT
        echo "new-slot=$NEW_SLOT" >> $GITHUB_OUTPUT
        echo "Deploying to $NEW_SLOT slot (current: $CURRENT_SLOT)"

    - name: Deploy to inactive slot
      run: |
        # Deploy to the inactive slot
        kubectl patch application foxhunt-platform-${{ steps.deployment-slot.outputs.new-slot }} \
          -n argocd \
          --type merge \
          --patch '{"spec":{"source":{"helm":{"parameters":[{"name":"global.imageTag","value":"'${{ github.sha }}'"}]}}}}'
        
        # Trigger sync
        kubectl patch application foxhunt-platform-${{ steps.deployment-slot.outputs.new-slot }} \
          -n argocd \
          --type merge \
          --patch '{"operation":{"sync":{}}}'

    - name: Wait for deployment
      run: |
        # Wait for deployment to complete
        kubectl wait --for=condition=Healthy \
          application/foxhunt-platform-${{ steps.deployment-slot.outputs.new-slot }} \
          -n argocd \
          --timeout=900s

    - name: Shadow traffic validation
      run: |
        # Route 5% of traffic to new slot for validation
        python3 scripts/shadow-traffic-test.py \
          --new-slot ${{ steps.deployment-slot.outputs.new-slot }} \
          --percentage 5 \
          --duration 300 \
          --max-latency 50

    - name: Performance validation
      run: |
        # Validate performance meets requirements
        python3 scripts/performance-validation.py \
          --slot ${{ steps.deployment-slot.outputs.new-slot }} \
          --duration 600 \
          --latency-threshold 50 \
          --throughput-threshold 100000

    - name: Switch traffic to new slot
      run: |
        # Switch active traffic to new slot
        kubectl patch service foxhunt-platform-active \
          -n foxhunt-production \
          --type merge \
          --patch '{"spec":{"selector":{"slot":"${{ steps.deployment-slot.outputs.new-slot }}"}}}'
        
        echo "Traffic switched to ${{ steps.deployment-slot.outputs.new-slot }} slot"

    - name: Post-deployment validation
      run: |
        # Final validation after traffic switch
        sleep 60
        python3 scripts/post-deployment-validation.py \
          --duration 300 \
          --max-errors 0.1

    - name: Cleanup old slot
      run: |
        # Scale down the old slot after successful deployment
        kubectl patch application foxhunt-platform-${{ steps.deployment-slot.outputs.current-slot }} \
          -n argocd \
          --type merge \
          --patch '{"spec":{"source":{"helm":{"parameters":[{"name":"global.replicaCount","value":"0"}]}}}}'

  # Automated rollback on failure
  rollback-on-failure:
    name: Emergency Rollback
    runs-on: ubuntu-latest
    needs: deploy-production
    if: failure() && github.ref == 'refs/heads/production'
    environment: production
    timeout-minutes: 10
    
    steps:
    - name: Configure kubectl
      uses: azure/setup-kubectl@v3
      with:
        version: 'v1.28.4'

    - name: Emergency rollback
      run: |
        # Immediate rollback to previous slot
        CURRENT_SLOT=$(kubectl get service foxhunt-platform-active \
          -n foxhunt-production \
          -o jsonpath='{.spec.selector.slot}')
        
        if [ "$CURRENT_SLOT" = "blue" ]; then
          ROLLBACK_SLOT="green"
        else
          ROLLBACK_SLOT="blue"
        fi
        
        # Switch back to previous slot
        kubectl patch service foxhunt-platform-active \
          -n foxhunt-production \
          --type merge \
          --patch '{"spec":{"selector":{"slot":"'$ROLLBACK_SLOT'"}}}'
        
        echo "Emergency rollback to $ROLLBACK_SLOT completed"

    - name: Notify operations team
      uses: 8398a7/action-slack@v3
      with:
        status: failure
        channel: '#foxhunt-alerts'
        text: 'EMERGENCY: Production deployment failed and rollback executed'
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}