foxhunt/.github/workflows/deploy.yml

name: Deployment Pipeline

on:
  workflow_dispatch:
    inputs:
      environment:
        description: 'Environment to deploy to'
        required: true
        type: choice
        options:
          - dev
          - staging
          - production
      version:
        description: 'Docker image tag (e.g., main-abc123 or v1.2.3)'
        required: true
        type: string
      reason:
        description: 'Deployment reason'
        required: true
        type: string

env:
  REGISTRY: ghcr.io
  IMAGE_PREFIX: ghcr.io/${{ github.repository_owner }}

jobs:
  validate-deployment:
    name: Validate Deployment
    runs-on: ubuntu-latest
    timeout-minutes: 5

    outputs:
      approved: ${{ steps.validate.outputs.approved }}

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Validate Version Tag
        id: validate
        run: |
          VERSION="${{ github.event.inputs.version }}"
          ENVIRONMENT="${{ github.event.inputs.environment }}"

          echo "Deploying version: $VERSION to $ENVIRONMENT"

          # Verify image exists in registry
          docker manifest inspect ${{ env.IMAGE_PREFIX }}/trading_service:$VERSION > /dev/null 2>&1

          if [ $? -ne 0 ]; then
            echo "::error::Image tag $VERSION not found in registry"
            exit 1
          fi

          echo "✅ Image tag $VERSION verified"
          echo "approved=true" >> $GITHUB_OUTPUT

  request-approval:
    name: Request Deployment Approval
    runs-on: ubuntu-latest
    needs: validate-deployment
    if: github.event.inputs.environment == 'staging' || github.event.inputs.environment == 'production'
    timeout-minutes: 1440  # 24 hours

    steps:
      - name: Request Staging Approval
        if: github.event.inputs.environment == 'staging'
        uses: trstringer/manual-approval@v1
        with:
          secret: ${{ secrets.GITHUB_TOKEN }}
          approvers: team-leads
          minimum-approvals: 1
          issue-title: "Deploy to Staging: ${{ github.event.inputs.version }}"
          issue-body: |
            **Environment**: Staging
            **Version**: ${{ github.event.inputs.version }}
            **Reason**: ${{ github.event.inputs.reason }}
            **Triggered by**: ${{ github.actor }}
            **Timestamp**: ${{ github.event.head_commit.timestamp }}

            Please review and approve this deployment.

      - name: Request Production Approval
        if: github.event.inputs.environment == 'production'
        uses: trstringer/manual-approval@v1
        with:
          secret: ${{ secrets.GITHUB_TOKEN }}
          approvers: team-leads,devops-team
          minimum-approvals: 2
          issue-title: "🚀 Deploy to Production: ${{ github.event.inputs.version }}"
          issue-body: |
            **Environment**: Production
            **Version**: ${{ github.event.inputs.version }}
            **Reason**: ${{ github.event.inputs.reason }}
            **Triggered by**: ${{ github.actor }}
            **Timestamp**: ${{ github.event.head_commit.timestamp }}

            ⚠️ **PRODUCTION DEPLOYMENT** - Requires 2 approvals

            Please review the following before approving:
            - [ ] Staging deployment successful in last 24h
            - [ ] All tests passing
            - [ ] Security scan clean
            - [ ] Rollback plan documented
            - [ ] On-call engineer notified

  deploy:
    name: Deploy to ${{ github.event.inputs.environment }}
    runs-on: ubuntu-latest
    needs: [validate-deployment, request-approval]
    if: always() && needs.validate-deployment.outputs.approved == 'true'
    timeout-minutes: 30

    environment:
      name: ${{ github.event.inputs.environment }}
      url: https://${{ github.event.inputs.environment }}.foxhunt.io

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Setup kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: 'v1.28.0'

      - name: Configure kubectl
        run: |
          aws eks update-kubeconfig \
            --region us-east-1 \
            --name foxhunt-${{ github.event.inputs.environment }}

      - name: Update Kubernetes Manifests
        run: |
          cd k8s/overlays/${{ github.event.inputs.environment }}

          # Update image tags for all services
          kustomize edit set image \
            api-gateway=${{ env.IMAGE_PREFIX }}/api_gateway:${{ github.event.inputs.version }} \
            trading-service=${{ env.IMAGE_PREFIX }}/trading_service:${{ github.event.inputs.version }} \
            backtesting-service=${{ env.IMAGE_PREFIX }}/backtesting_service:${{ github.event.inputs.version }} \
            ml-training-service=${{ env.IMAGE_PREFIX }}/ml_training_service:${{ github.event.inputs.version }}

      - name: Run Database Migrations
        if: github.event.inputs.environment == 'production'
        run: |
          # Port-forward to PostgreSQL
          kubectl port-forward -n foxhunt-${{ github.event.inputs.environment }} \
            svc/postgres 5432:5432 &

          sleep 5

          # Run migrations
          cargo sqlx migrate run \
            --database-url postgresql://foxhunt:${{ secrets.DB_PASSWORD }}@localhost:5432/foxhunt

      - name: Deploy to Kubernetes
        run: |
          kubectl apply -k k8s/overlays/${{ github.event.inputs.environment }}

      - name: Wait for Rollout
        run: |
          kubectl rollout status deployment/api-gateway \
            -n foxhunt-${{ github.event.inputs.environment }} \
            --timeout=10m

          kubectl rollout status deployment/trading-service \
            -n foxhunt-${{ github.event.inputs.environment }} \
            --timeout=10m

          kubectl rollout status deployment/backtesting-service \
            -n foxhunt-${{ github.event.inputs.environment }} \
            --timeout=10m

          kubectl rollout status deployment/ml-training-service \
            -n foxhunt-${{ github.event.inputs.environment }} \
            --timeout=10m

      - name: Run Health Checks
        run: |
          NAMESPACE=foxhunt-${{ github.event.inputs.environment }}
          MAX_RETRIES=30
          RETRY_DELAY=10

          for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do
            echo "Checking health of $SERVICE..."

            for i in $(seq 1 $MAX_RETRIES); do
              POD=$(kubectl get pod -n $NAMESPACE -l app=$SERVICE -o jsonpath='{.items[0].metadata.name}')

              if kubectl exec -n $NAMESPACE $POD -- grpc_health_probe -addr=:50051; then
                echo "✅ $SERVICE health check passed (attempt $i/$MAX_RETRIES)"
                break
              fi

              if [ $i -eq $MAX_RETRIES ]; then
                echo "❌ $SERVICE health check failed after $MAX_RETRIES attempts"
                exit 1
              fi

              echo "⏳ Health check failed (attempt $i/$MAX_RETRIES), retrying in ${RETRY_DELAY}s..."
              sleep $RETRY_DELAY
            done
          done

      - name: Run Smoke Tests
        run: |
          ENVIRONMENT=${{ github.event.inputs.environment }}

          # API Gateway smoke test
          curl -f https://$ENVIRONMENT.foxhunt.io/health || exit 1

          # Trading Service smoke test
          grpcurl -plaintext $ENVIRONMENT.foxhunt.io:50052 grpc.health.v1.Health/Check

      - name: Monitor Metrics (Production)
        if: github.event.inputs.environment == 'production'
        run: |
          # Wait 5 minutes and check Prometheus alerts
          echo "Monitoring metrics for 5 minutes..."
          sleep 300

          # Query Prometheus for firing alerts
          ALERTS=$(curl -s http://prometheus.foxhunt.io/api/v1/alerts | jq '.data.alerts | length')

          if [ "$ALERTS" -gt 0 ]; then
            echo "⚠️ Warning: $ALERTS Prometheus alerts firing"
            curl -s http://prometheus.foxhunt.io/api/v1/alerts | jq '.data.alerts'
          else
            echo "✅ No Prometheus alerts firing"
          fi

      - name: Notify Slack (Success)
        if: success()
        run: |
          curl -X POST ${{ secrets.SLACK_WEBHOOK }} -H 'Content-Type: application/json' -d '{
            "text": "✅ Deployment Successful",
            "attachments": [{
              "color": "good",
              "fields": [
                {"title": "Environment", "value": "${{ github.event.inputs.environment }}", "short": true},
                {"title": "Version", "value": "${{ github.event.inputs.version }}", "short": true},
                {"title": "Reason", "value": "${{ github.event.inputs.reason }}", "short": false},
                {"title": "Deployed by", "value": "${{ github.actor }}", "short": true}
              ]
            }]
          }'

  rollback:
    name: Rollback Deployment
    runs-on: ubuntu-latest
    needs: deploy
    if: failure()
    timeout-minutes: 10

    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Setup kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: 'v1.28.0'

      - name: Configure kubectl
        run: |
          aws eks update-kubeconfig \
            --region us-east-1 \
            --name foxhunt-${{ github.event.inputs.environment }}

      - name: Rollback Deployments
        run: |
          NAMESPACE=foxhunt-${{ github.event.inputs.environment }}

          echo "🔄 Initiating rollback..."

          kubectl rollout undo deployment/api-gateway -n $NAMESPACE
          kubectl rollout undo deployment/trading-service -n $NAMESPACE
          kubectl rollout undo deployment/backtesting-service -n $NAMESPACE
          kubectl rollout undo deployment/ml-training-service -n $NAMESPACE

          echo "⏳ Waiting for rollback to complete..."

          kubectl rollout status deployment/api-gateway -n $NAMESPACE --timeout=5m
          kubectl rollout status deployment/trading-service -n $NAMESPACE --timeout=5m
          kubectl rollout status deployment/backtesting-service -n $NAMESPACE --timeout=5m
          kubectl rollout status deployment/ml-training-service -n $NAMESPACE --timeout=5m

          echo "✅ Rollback completed"

      - name: Verify Rollback
        run: |
          NAMESPACE=foxhunt-${{ github.event.inputs.environment }}

          for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do
            POD=$(kubectl get pod -n $NAMESPACE -l app=$SERVICE -o jsonpath='{.items[0].metadata.name}')
            kubectl exec -n $NAMESPACE $POD -- grpc_health_probe -addr=:50051
          done

      - name: Notify Slack (Rollback)
        run: |
          curl -X POST ${{ secrets.SLACK_WEBHOOK }} -H 'Content-Type: application/json' -d '{
            "text": "⚠️ Deployment Failed - Rollback Completed",
            "attachments": [{
              "color": "danger",
              "fields": [
                {"title": "Environment", "value": "${{ github.event.inputs.environment }}", "short": true},
                {"title": "Version", "value": "${{ github.event.inputs.version }}", "short": true},
                {"title": "Reason", "value": "Deployment failed health checks", "short": false},
                {"title": "Triggered by", "value": "${{ github.actor }}", "short": true}
              ]
            }]
          }'

  create-release-tag:
    name: Create Release Tag (Production)
    runs-on: ubuntu-latest
    needs: deploy
    if: github.event.inputs.environment == 'production' && success()
    timeout-minutes: 5

    permissions:
      contents: write

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Create Release
        uses: softprops/action-gh-release@v1
        with:
          tag_name: ${{ github.event.inputs.version }}
          name: Release ${{ github.event.inputs.version }}
          body: |
            ## Release ${{ github.event.inputs.version }}

            **Deployed to Production**: ${{ github.event.head_commit.timestamp }}
            **Deployment Reason**: ${{ github.event.inputs.reason }}
            **Deployed by**: ${{ github.actor }}

            ### Services Deployed
            - API Gateway: `${{ env.IMAGE_PREFIX }}/api_gateway:${{ github.event.inputs.version }}`
            - Trading Service: `${{ env.IMAGE_PREFIX }}/trading_service:${{ github.event.inputs.version }}`
            - Backtesting Service: `${{ env.IMAGE_PREFIX }}/backtesting_service:${{ github.event.inputs.version }}`
            - ML Training Service: `${{ env.IMAGE_PREFIX }}/ml_training_service:${{ github.event.inputs.version }}`

            See [CHANGELOG.md](CHANGELOG.md) for details.
          draft: false
          prerelease: false
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}