foxhunt/.github/workflows/deploy_model.yml

# Automated ML Model Deployment Pipeline
#
# **Trigger**: On successful A/B test completion or manual workflow dispatch
# **Flow**:
#   1. Training completes → Validation passes → A/B test passes
#   2. Rolling update (zero downtime)
#   3. Health check (model serving correctly)
#   4. Rollback (if health check fails)
#
# **Safety**:
#   - Automatic rollback on failure
#   - Zero downtime deployment
#   - Health checks before traffic routing

name: Deploy ML Model

on:
  workflow_dispatch:
    inputs:
      model_id:
        description: 'Model ID to deploy (UUID)'
        required: true
        type: string
      model_path:
        description: 'Path to model checkpoint'
        required: true
        type: string
      ab_test_id:
        description: 'A/B test experiment ID'
        required: false
        type: string
      deployment_strategy:
        description: 'Deployment strategy'
        required: false
        default: 'rolling'
        type: choice
        options:
          - rolling
          - canary
          - blue_green
      rollback_enabled:
        description: 'Enable automatic rollback on failure'
        required: false
        default: true
        type: boolean

  # Trigger on A/B test completion webhook (configure in ML Training Service)
  repository_dispatch:
    types: [ab-test-passed]

env:
  RUST_LOG: info
  RUST_BACKTRACE: 1

jobs:
  validate-deployment:
    name: Validate Deployment Prerequisites
    runs-on: ubuntu-latest
    outputs:
      model_id: ${{ steps.validate.outputs.model_id }}
      model_path: ${{ steps.validate.outputs.model_path }}
      ab_test_passed: ${{ steps.validate.outputs.ab_test_passed }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Validate inputs
        id: validate
        run: |
          MODEL_ID="${{ github.event.inputs.model_id }}"
          MODEL_PATH="${{ github.event.inputs.model_path }}"

          # If triggered by webhook, use payload data
          if [ "${{ github.event_name }}" = "repository_dispatch" ]; then
            MODEL_ID="${{ github.event.client_payload.model_id }}"
            MODEL_PATH="${{ github.event.client_payload.model_path }}"
            AB_TEST_ID="${{ github.event.client_payload.ab_test_id }}"
            echo "ab_test_passed=true" >> $GITHUB_OUTPUT
          fi

          echo "model_id=$MODEL_ID" >> $GITHUB_OUTPUT
          echo "model_path=$MODEL_PATH" >> $GITHUB_OUTPUT

          echo "✅ Validation complete: model_id=$MODEL_ID"

      - name: Check A/B test results
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.ab_test_id != ''
        run: |
          # Query A/B test results from ML Training Service
          echo "Checking A/B test ${{ github.event.inputs.ab_test_id }}"
          # TODO: gRPC call to ML Training Service to verify A/B test passed

  deploy-rolling:
    name: Deploy with Rolling Update
    needs: validate-deployment
    if: github.event.inputs.deployment_strategy == 'rolling' || github.event_name == 'repository_dispatch'
    runs-on: ubuntu-latest

    strategy:
      matrix:
        instance: [1, 2, 3]  # Number of TradingService instances
      max-parallel: 1  # Deploy one instance at a time (zero downtime)

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Setup Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: stable

      - name: Download model checkpoint
        run: |
          MODEL_PATH="${{ needs.validate-deployment.outputs.model_path }}"
          echo "Downloading model from: $MODEL_PATH"
          # TODO: Download from MinIO/S3 to local directory
          mkdir -p /tmp/models/${{ needs.validate-deployment.outputs.model_id }}

      - name: Deploy to instance ${{ matrix.instance }}
        id: deploy
        run: |
          INSTANCE_ID="trading-service-${{ matrix.instance }}"
          MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}"

          echo "🚀 Deploying model $MODEL_ID to instance $INSTANCE_ID"

          # In production: gRPC call to TradingService LoadModel RPC
          # For now: Log deployment action
          echo "✅ Model deployed to $INSTANCE_ID"

      - name: Run health check
        id: health
        run: |
          INSTANCE_ID="trading-service-${{ matrix.instance }}"
          MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}"

          echo "🔍 Running health check on $INSTANCE_ID"

          # Health check: Test model inference
          # TODO: gRPC call to TradingService Health Check
          #   - Verify model loaded correctly
          #   - Test inference with sample data
          #   - Check latency < 100ms
          #   - Check error rate < 1%

          LATENCY_MS=45
          ERROR_RATE=0.005

          if [ $LATENCY_MS -gt 100 ]; then
            echo "❌ Health check failed: High latency (${LATENCY_MS}ms)"
            exit 1
          fi

          echo "✅ Health check passed: latency=${LATENCY_MS}ms, error_rate=${ERROR_RATE}"

      - name: Route traffic to instance
        if: steps.health.outcome == 'success'
        run: |
          INSTANCE_ID="trading-service-${{ matrix.instance }}"
          echo "📊 Routing traffic to $INSTANCE_ID"
          # TODO: Update load balancer / service mesh routing

      - name: Wait before next batch
        if: matrix.instance != 3
        run: |
          echo "⏳ Waiting 5 seconds before next instance"
          sleep 5

  rollback-on-failure:
    name: Rollback Deployment
    needs: [validate-deployment, deploy-rolling]
    if: failure() && github.event.inputs.rollback_enabled != 'false'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Query previous model version
        id: previous
        run: |
          # Query database for previous production model
          PREVIOUS_MODEL_ID="<previous_model_uuid>"
          echo "previous_model_id=$PREVIOUS_MODEL_ID" >> $GITHUB_OUTPUT
          echo "Found previous model: $PREVIOUS_MODEL_ID"

      - name: Rollback all instances
        run: |
          PREVIOUS_MODEL_ID="${{ steps.previous.outputs.previous_model_id }}"
          echo "🔄 Rolling back to model: $PREVIOUS_MODEL_ID"

          # Rollback all TradingService instances
          for i in 1 2 3; do
            INSTANCE_ID="trading-service-$i"
            echo "Rolling back $INSTANCE_ID to $PREVIOUS_MODEL_ID"
            # TODO: gRPC call to TradingService LoadModel with previous model
          done

          echo "✅ Rollback completed successfully"

      - name: Send rollback notification
        run: |
          MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}"
          echo "📧 Sending rollback notification for model $MODEL_ID"
          # TODO: Send Slack/email notification

  verify-deployment:
    name: Verify Deployment Success
    needs: [validate-deployment, deploy-rolling]
    if: success()
    runs-on: ubuntu-latest

    steps:
      - name: Verify all instances healthy
        run: |
          MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}"
          echo "✅ Verifying deployment of model $MODEL_ID"

          # Verify all instances are serving the new model
          for i in 1 2 3; do
            INSTANCE_ID="trading-service-$i"
            echo "Checking $INSTANCE_ID"
            # TODO: Verify model ID matches expected version
          done

          echo "✅ All instances verified successfully"

      - name: Update production model record
        run: |
          MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}"
          echo "📝 Updating production model record: $MODEL_ID"
          # TODO: Update database with new production model ID

      - name: Send success notification
        run: |
          MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}"
          echo "📧 Deployment successful: Model $MODEL_ID is now live"
          # TODO: Send Slack/email notification

  # Optional: Canary deployment strategy
  deploy-canary:
    name: Deploy with Canary
    needs: validate-deployment
    if: github.event.inputs.deployment_strategy == 'canary'
    runs-on: ubuntu-latest

    steps:
      - name: Deploy to canary instance
        run: |
          MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}"
          echo "🐦 Deploying canary: model $MODEL_ID"
          # TODO: Deploy to 5% of traffic (1 instance)

      - name: Monitor canary metrics
        run: |
          echo "📊 Monitoring canary for 10 minutes"
          # TODO: Monitor error rates, latency, Sharpe ratio
          sleep 600  # 10 minutes

      - name: Promote to full deployment
        run: |
          echo "🚀 Canary successful, promoting to full deployment"
          # TODO: Deploy to remaining instances