foxhunt/.github/workflows/production-deployment.yml

# Foxhunt HFT Trading System - Production Deployment Pipeline
# Automated CI/CD with security scanning, performance validation, and blue-green deployment

name: Production Deployment Pipeline

on:
  push:
    branches:
      - main
      - release/*
    tags:
      - 'v*.*.*'
  pull_request:
    branches:
      - main
    types: [opened, synchronize, reopened]

env:
  REGISTRY: ghcr.io
  ECR_REGISTRY: 123456789.dkr.ecr.us-east-1.amazonaws.com
  CLUSTER_NAME: foxhunt-eks-production
  REGION: us-east-1

jobs:
  # Security and Quality Gates
  security-scan:
    name: Security Scan
    runs-on: ubuntu-latest
    permissions:
      contents: read
      security-events: write
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    
    - name: Set up Rust toolchain
      uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
        toolchain: stable
        components: clippy, rustfmt
    
    - name: Rust security audit
      uses: rustsec/audit-check@v1.4.1
      with:
        token: ${{ secrets.GITHUB_TOKEN }}
    
    - name: Run Clippy security lints
      run: |
        cargo clippy --all-targets --all-features -- -D warnings -W clippy::all
    
    - name: Code format check
      run: |
        cargo fmt --all -- --check
    
    - name: Dependency vulnerability scan
      run: |
        cargo audit --db advisory-db --deny warnings
    
    - name: SARIF upload
      uses: github/codeql-action/upload-sarif@v3
      if: always()
      with:
        sarif_file: results.sarif

  # Build and Test
  build-and-test:
    name: Build and Test
    runs-on: ubuntu-latest
    needs: security-scan
    
    strategy:
      matrix:
        service: [trading-engine, market-data, persistence, ai-intelligence, broker-connector, integration-hub, data-aggregator]
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Rust toolchain
      uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
        toolchain: stable
    
    - name: Cache dependencies
      uses: Swatinem/rust-cache@v2
      with:
        cache-on-failure: true
    
    - name: Build service
      run: |
        cd services/${{ matrix.service }}
        cargo build --release
    
    - name: Run unit tests
      run: |
        cd services/${{ matrix.service }}
        cargo test --release -- --test-threads=1
    
    - name: Run integration tests
      run: |
        cd services/${{ matrix.service }}
        cargo test --release --test integration_tests
    
    - name: Performance benchmarks
      run: |
        cd services/${{ matrix.service }}
        cargo bench --no-run
    
    - name: Upload build artifacts
      uses: actions/upload-artifact@v4
      with:
        name: ${{ matrix.service }}-binary
        path: services/${{ matrix.service }}/target/release/${{ matrix.service }}
        retention-days: 7

  # Container Build and Security Scan
  container-build:
    name: Container Build & Scan
    runs-on: ubuntu-latest
    needs: build-and-test
    if: github.event_name != 'pull_request'
    
    strategy:
      matrix:
        service: [trading-engine, market-data, persistence, ai-intelligence, broker-connector, integration-hub, data-aggregator]
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.REGION }}
    
    - name: Login to ECR
      uses: aws-actions/amazon-ecr-login@v2
    
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
    
    - name: Extract metadata
      id: meta
      uses: docker/metadata-action@v5
      with:
        images: |
          ${{ env.ECR_REGISTRY }}/foxhunt/${{ matrix.service }}
        tags: |
          type=ref,event=branch
          type=ref,event=pr
          type=semver,pattern={{version}}
          type=semver,pattern={{major}}.{{minor}}
          type=sha,prefix={{branch}}-
          type=raw,value=latest,enable={{is_default_branch}}
    
    - name: Build and push container
      uses: docker/build-push-action@v5
      with:
        context: .
        file: services/${{ matrix.service }}/Dockerfile
        target: production
        push: true
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        cache-from: type=gha
        cache-to: type=gha,mode=max
        platforms: linux/amd64
        build-args: |
          SERVICE_NAME=${{ matrix.service }}
          BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
          VCS_REF=${{ github.sha }}
          VERSION=${{ steps.meta.outputs.version }}
    
    - name: Container security scan
      uses: aquasecurity/trivy-action@master
      with:
        image-ref: ${{ env.ECR_REGISTRY }}/foxhunt/${{ matrix.service }}:${{ steps.meta.outputs.version }}
        format: 'sarif'
        output: 'trivy-results-${{ matrix.service }}.sarif'
    
    - name: Upload Trivy scan results
      uses: github/codeql-action/upload-sarif@v3
      with:
        sarif_file: 'trivy-results-${{ matrix.service }}.sarif'
    
    - name: Fail on critical vulnerabilities
      uses: aquasecurity/trivy-action@master
      with:
        image-ref: ${{ env.ECR_REGISTRY }}/foxhunt/${{ matrix.service }}:${{ steps.meta.outputs.version }}
        format: 'json'
        exit-code: '1'
        ignore-unfixed: true
        severity: 'CRITICAL,HIGH'

  # Performance Validation
  performance-validation:
    name: Performance Validation
    runs-on: ubuntu-latest
    needs: container-build
    if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.REGION }}
    
    - name: Update kubeconfig
      run: |
        aws eks update-kubeconfig --region ${{ env.REGION }} --name ${{ env.CLUSTER_NAME }}
    
    - name: Deploy to staging environment
      run: |
        # Deploy to staging namespace for performance testing
        kubectl apply -f deployment/kubernetes/ -n foxhunt-staging
        kubectl wait --for=condition=available deployment --all -n foxhunt-staging --timeout=300s
    
    - name: Run performance tests
      run: |
        # Run comprehensive performance validation
        kubectl apply -f - <<EOF
        apiVersion: batch/v1
        kind: Job
        metadata:
          name: performance-validation-$(date +%s)
          namespace: foxhunt-staging
        spec:
          template:
            spec:
              containers:
              - name: performance-test
                image: ${{ env.ECR_REGISTRY }}/foxhunt/performance-test:latest
                command: ["/bin/sh"]
                args:
                - -c
                - |
                  echo "Running comprehensive performance validation..."
                  
                  # Test latency requirements (< 5ms p99)
                  ./performance-test --target=trading-engine:50051 --duration=60s --threads=10 --latency-threshold=5ms
                  
                  # Test throughput requirements (> 10k TPS)
                  ./performance-test --target=trading-engine:50051 --duration=60s --threads=50 --throughput-threshold=10000
                  
                  # Memory and CPU validation
                  kubectl top pods -n foxhunt-staging --no-headers | awk '{if($3 > "2Gi" || $4 > "4000m") exit 1}'
                  
                  echo "Performance validation completed successfully"
              restartPolicy: Never
          backoffLimit: 3
        EOF
        
        # Wait for performance test completion
        kubectl wait --for=condition=complete job --all -n foxhunt-staging --timeout=300s
    
    - name: Cleanup staging environment
      if: always()
      run: |
        kubectl delete namespace foxhunt-staging --ignore-not-found=true

  # Blue-Green Production Deployment
  production-deployment:
    name: Production Deployment
    runs-on: ubuntu-latest
    needs: [performance-validation]
    if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
    environment:
      name: production
      url: https://trading.foxhunt.com
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.REGION }}
    
    - name: Update kubeconfig
      run: |
        aws eks update-kubeconfig --region ${{ env.REGION }} --name ${{ env.CLUSTER_NAME }}
    
    - name: Extract image tag
      id: image-tag
      run: |
        if [[ "${{ github.ref }}" == refs/tags/* ]]; then
          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
        else
          echo "tag=${{ github.sha }}" >> $GITHUB_OUTPUT
        fi
    
    - name: Blue-Green Deployment
      run: |
        # Use our blue-green deployment script
        chmod +x deployment/scripts/blue-green-deploy.sh
        ./deployment/scripts/blue-green-deploy.sh ${{ steps.image-tag.outputs.tag }}
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
    
    - name: Update GitOps repository
      run: |
        # Update ArgoCD configuration with new image tag
        git config --global user.name "GitHub Actions"
        git config --global user.email "actions@github.com"
        
        # Clone infrastructure repository
        git clone https://github.com/foxhunt-hft/infrastructure.git
        cd infrastructure
        
        # Update image tags in values files
        sed -i "s/tag: .*/tag: ${{ steps.image-tag.outputs.tag }}/g" deployment/kubernetes/values-production.yaml
        
        # Commit and push changes
        git add deployment/kubernetes/values-production.yaml
        git commit -m "feat: Update production deployment to ${{ steps.image-tag.outputs.tag }}"
        git push origin main
      env:
        GITHUB_TOKEN: ${{ secrets.GITOPS_TOKEN }}
    
    - name: Notify deployment success
      if: success()
      run: |
        curl -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
        -H 'Content-type: application/json' \
        --data '{
          "text": "✅ Foxhunt HFT Production Deployment Successful",
          "blocks": [
            {
              "type": "section",
              "text": {
                "type": "mrkdwn",
                "text": "*Foxhunt HFT Production Deployment* :rocket:\n*Status:* Success\n*Version:* `${{ steps.image-tag.outputs.tag }}`\n*Environment:* Production\n*Deployed by:* ${{ github.actor }}"
              }
            },
            {
              "type": "context",
              "elements": [
                {
                  "type": "mrkdwn",
                  "text": "Commit: <https://github.com/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
                }
              ]
            }
          ]
        }'
    
    - name: Notify deployment failure
      if: failure()
      run: |
        curl -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
        -H 'Content-type: application/json' \
        --data '{
          "text": "🚨 Foxhunt HFT Production Deployment Failed",
          "blocks": [
            {
              "type": "section",
              "text": {
                "type": "mrkdwn",
                "text": "*Foxhunt HFT Production Deployment* :x:\n*Status:* Failed\n*Version:* `${{ steps.image-tag.outputs.tag }}`\n*Environment:* Production\n*Failed step:* ${{ job.status }}"
              }
            }
          ]
        }'

  # Rollback capability
  rollback:
    name: Emergency Rollback
    runs-on: ubuntu-latest
    if: failure() && github.event_name != 'pull_request'
    needs: [production-deployment]
    environment:
      name: production-rollback
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.REGION }}
    
    - name: Update kubeconfig
      run: |
        aws eks update-kubeconfig --region ${{ env.REGION }} --name ${{ env.CLUSTER_NAME }}
    
    - name: Emergency rollback
      run: |
        # Get the previously active color and switch back
        current_color=$(kubectl get service trading-engine-active -n foxhunt-trading -o jsonpath='{.spec.selector.version}')
        rollback_color=$([ "$current_color" == "blue" ] && echo "green" || echo "blue")
        
        echo "Rolling back from $current_color to $rollback_color"
        
        # Switch traffic back
        kubectl patch service trading-engine-active -n foxhunt-trading \
          -p "{\"spec\":{\"selector\":{\"version\":\"$rollback_color\"}}}"
        
        echo "Emergency rollback completed"