name: Deployment Pipeline on: workflow_dispatch: inputs: environment: description: 'Environment to deploy to' required: true type: choice options: - dev - staging - production version: description: 'Docker image tag (e.g., main-abc123 or v1.2.3)' required: true type: string reason: description: 'Deployment reason' required: true type: string env: REGISTRY: ghcr.io IMAGE_PREFIX: ghcr.io/${{ github.repository_owner }} jobs: validate-deployment: name: Validate Deployment runs-on: ubuntu-latest timeout-minutes: 5 outputs: approved: ${{ steps.validate.outputs.approved }} steps: - name: Checkout Code uses: actions/checkout@v4 - name: Validate Version Tag id: validate run: | VERSION="${{ github.event.inputs.version }}" ENVIRONMENT="${{ github.event.inputs.environment }}" echo "Deploying version: $VERSION to $ENVIRONMENT" # Verify image exists in registry docker manifest inspect ${{ env.IMAGE_PREFIX }}/trading_service:$VERSION > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "::error::Image tag $VERSION not found in registry" exit 1 fi echo "✅ Image tag $VERSION verified" echo "approved=true" >> $GITHUB_OUTPUT request-approval: name: Request Deployment Approval runs-on: ubuntu-latest needs: validate-deployment if: github.event.inputs.environment == 'staging' || github.event.inputs.environment == 'production' timeout-minutes: 1440 # 24 hours steps: - name: Request Staging Approval if: github.event.inputs.environment == 'staging' uses: trstringer/manual-approval@v1 with: secret: ${{ secrets.GITHUB_TOKEN }} approvers: team-leads minimum-approvals: 1 issue-title: "Deploy to Staging: ${{ github.event.inputs.version }}" issue-body: | **Environment**: Staging **Version**: ${{ github.event.inputs.version }} **Reason**: ${{ github.event.inputs.reason }} **Triggered by**: ${{ github.actor }} **Timestamp**: ${{ github.event.head_commit.timestamp }} Please review and approve this deployment. - name: Request Production Approval if: github.event.inputs.environment == 'production' uses: trstringer/manual-approval@v1 with: secret: ${{ secrets.GITHUB_TOKEN }} approvers: team-leads,devops-team minimum-approvals: 2 issue-title: "🚀 Deploy to Production: ${{ github.event.inputs.version }}" issue-body: | **Environment**: Production **Version**: ${{ github.event.inputs.version }} **Reason**: ${{ github.event.inputs.reason }} **Triggered by**: ${{ github.actor }} **Timestamp**: ${{ github.event.head_commit.timestamp }} ⚠️ **PRODUCTION DEPLOYMENT** - Requires 2 approvals Please review the following before approving: - [ ] Staging deployment successful in last 24h - [ ] All tests passing - [ ] Security scan clean - [ ] Rollback plan documented - [ ] On-call engineer notified deploy: name: Deploy to ${{ github.event.inputs.environment }} runs-on: ubuntu-latest needs: [validate-deployment, request-approval] if: always() && needs.validate-deployment.outputs.approved == 'true' timeout-minutes: 30 environment: name: ${{ github.event.inputs.environment }} url: https://${{ github.event.inputs.environment }}.foxhunt.io steps: - name: Checkout Code uses: actions/checkout@v4 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: us-east-1 - name: Setup kubectl uses: azure/setup-kubectl@v3 with: version: 'v1.28.0' - name: Configure kubectl run: | aws eks update-kubeconfig \ --region us-east-1 \ --name foxhunt-${{ github.event.inputs.environment }} - name: Update Kubernetes Manifests run: | cd k8s/overlays/${{ github.event.inputs.environment }} # Update image tags for all services kustomize edit set image \ api-gateway=${{ env.IMAGE_PREFIX }}/api_gateway:${{ github.event.inputs.version }} \ trading-service=${{ env.IMAGE_PREFIX }}/trading_service:${{ github.event.inputs.version }} \ backtesting-service=${{ env.IMAGE_PREFIX }}/backtesting_service:${{ github.event.inputs.version }} \ ml-training-service=${{ env.IMAGE_PREFIX }}/ml_training_service:${{ github.event.inputs.version }} - name: Run Database Migrations if: github.event.inputs.environment == 'production' run: | # Port-forward to PostgreSQL kubectl port-forward -n foxhunt-${{ github.event.inputs.environment }} \ svc/postgres 5432:5432 & sleep 5 # Run migrations cargo sqlx migrate run \ --database-url postgresql://foxhunt:${{ secrets.DB_PASSWORD }}@localhost:5432/foxhunt - name: Deploy to Kubernetes run: | kubectl apply -k k8s/overlays/${{ github.event.inputs.environment }} - name: Wait for Rollout run: | kubectl rollout status deployment/api-gateway \ -n foxhunt-${{ github.event.inputs.environment }} \ --timeout=10m kubectl rollout status deployment/trading-service \ -n foxhunt-${{ github.event.inputs.environment }} \ --timeout=10m kubectl rollout status deployment/backtesting-service \ -n foxhunt-${{ github.event.inputs.environment }} \ --timeout=10m kubectl rollout status deployment/ml-training-service \ -n foxhunt-${{ github.event.inputs.environment }} \ --timeout=10m - name: Run Health Checks run: | NAMESPACE=foxhunt-${{ github.event.inputs.environment }} MAX_RETRIES=30 RETRY_DELAY=10 for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do echo "Checking health of $SERVICE..." for i in $(seq 1 $MAX_RETRIES); do POD=$(kubectl get pod -n $NAMESPACE -l app=$SERVICE -o jsonpath='{.items[0].metadata.name}') if kubectl exec -n $NAMESPACE $POD -- grpc_health_probe -addr=:50051; then echo "✅ $SERVICE health check passed (attempt $i/$MAX_RETRIES)" break fi if [ $i -eq $MAX_RETRIES ]; then echo "❌ $SERVICE health check failed after $MAX_RETRIES attempts" exit 1 fi echo "⏳ Health check failed (attempt $i/$MAX_RETRIES), retrying in ${RETRY_DELAY}s..." sleep $RETRY_DELAY done done - name: Run Smoke Tests run: | ENVIRONMENT=${{ github.event.inputs.environment }} # API Gateway smoke test curl -f https://$ENVIRONMENT.foxhunt.io/health || exit 1 # Trading Service smoke test grpcurl -plaintext $ENVIRONMENT.foxhunt.io:50052 grpc.health.v1.Health/Check - name: Monitor Metrics (Production) if: github.event.inputs.environment == 'production' run: | # Wait 5 minutes and check Prometheus alerts echo "Monitoring metrics for 5 minutes..." sleep 300 # Query Prometheus for firing alerts ALERTS=$(curl -s http://prometheus.foxhunt.io/api/v1/alerts | jq '.data.alerts | length') if [ "$ALERTS" -gt 0 ]; then echo "⚠️ Warning: $ALERTS Prometheus alerts firing" curl -s http://prometheus.foxhunt.io/api/v1/alerts | jq '.data.alerts' else echo "✅ No Prometheus alerts firing" fi - name: Notify Slack (Success) if: success() run: | curl -X POST ${{ secrets.SLACK_WEBHOOK }} -H 'Content-Type: application/json' -d '{ "text": "✅ Deployment Successful", "attachments": [{ "color": "good", "fields": [ {"title": "Environment", "value": "${{ github.event.inputs.environment }}", "short": true}, {"title": "Version", "value": "${{ github.event.inputs.version }}", "short": true}, {"title": "Reason", "value": "${{ github.event.inputs.reason }}", "short": false}, {"title": "Deployed by", "value": "${{ github.actor }}", "short": true} ] }] }' rollback: name: Rollback Deployment runs-on: ubuntu-latest needs: deploy if: failure() timeout-minutes: 10 steps: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: us-east-1 - name: Setup kubectl uses: azure/setup-kubectl@v3 with: version: 'v1.28.0' - name: Configure kubectl run: | aws eks update-kubeconfig \ --region us-east-1 \ --name foxhunt-${{ github.event.inputs.environment }} - name: Rollback Deployments run: | NAMESPACE=foxhunt-${{ github.event.inputs.environment }} echo "🔄 Initiating rollback..." kubectl rollout undo deployment/api-gateway -n $NAMESPACE kubectl rollout undo deployment/trading-service -n $NAMESPACE kubectl rollout undo deployment/backtesting-service -n $NAMESPACE kubectl rollout undo deployment/ml-training-service -n $NAMESPACE echo "⏳ Waiting for rollback to complete..." kubectl rollout status deployment/api-gateway -n $NAMESPACE --timeout=5m kubectl rollout status deployment/trading-service -n $NAMESPACE --timeout=5m kubectl rollout status deployment/backtesting-service -n $NAMESPACE --timeout=5m kubectl rollout status deployment/ml-training-service -n $NAMESPACE --timeout=5m echo "✅ Rollback completed" - name: Verify Rollback run: | NAMESPACE=foxhunt-${{ github.event.inputs.environment }} for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do POD=$(kubectl get pod -n $NAMESPACE -l app=$SERVICE -o jsonpath='{.items[0].metadata.name}') kubectl exec -n $NAMESPACE $POD -- grpc_health_probe -addr=:50051 done - name: Notify Slack (Rollback) run: | curl -X POST ${{ secrets.SLACK_WEBHOOK }} -H 'Content-Type: application/json' -d '{ "text": "⚠️ Deployment Failed - Rollback Completed", "attachments": [{ "color": "danger", "fields": [ {"title": "Environment", "value": "${{ github.event.inputs.environment }}", "short": true}, {"title": "Version", "value": "${{ github.event.inputs.version }}", "short": true}, {"title": "Reason", "value": "Deployment failed health checks", "short": false}, {"title": "Triggered by", "value": "${{ github.actor }}", "short": true} ] }] }' create-release-tag: name: Create Release Tag (Production) runs-on: ubuntu-latest needs: deploy if: github.event.inputs.environment == 'production' && success() timeout-minutes: 5 permissions: contents: write steps: - name: Checkout Code uses: actions/checkout@v4 - name: Create Release uses: softprops/action-gh-release@v1 with: tag_name: ${{ github.event.inputs.version }} name: Release ${{ github.event.inputs.version }} body: | ## Release ${{ github.event.inputs.version }} **Deployed to Production**: ${{ github.event.head_commit.timestamp }} **Deployment Reason**: ${{ github.event.inputs.reason }} **Deployed by**: ${{ github.actor }} ### Services Deployed - API Gateway: `${{ env.IMAGE_PREFIX }}/api_gateway:${{ github.event.inputs.version }}` - Trading Service: `${{ env.IMAGE_PREFIX }}/trading_service:${{ github.event.inputs.version }}` - Backtesting Service: `${{ env.IMAGE_PREFIX }}/backtesting_service:${{ github.event.inputs.version }}` - ML Training Service: `${{ env.IMAGE_PREFIX }}/ml_training_service:${{ github.event.inputs.version }}` See [CHANGELOG.md](CHANGELOG.md) for details. draft: false prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}