# Automated ML Model Deployment Pipeline # # **Trigger**: On successful A/B test completion or manual workflow dispatch # **Flow**: # 1. Training completes → Validation passes → A/B test passes # 2. Rolling update (zero downtime) # 3. Health check (model serving correctly) # 4. Rollback (if health check fails) # # **Safety**: # - Automatic rollback on failure # - Zero downtime deployment # - Health checks before traffic routing name: Deploy ML Model on: workflow_dispatch: inputs: model_id: description: 'Model ID to deploy (UUID)' required: true type: string model_path: description: 'Path to model checkpoint' required: true type: string ab_test_id: description: 'A/B test experiment ID' required: false type: string deployment_strategy: description: 'Deployment strategy' required: false default: 'rolling' type: choice options: - rolling - canary - blue_green rollback_enabled: description: 'Enable automatic rollback on failure' required: false default: true type: boolean # Trigger on A/B test completion webhook (configure in ML Training Service) repository_dispatch: types: [ab-test-passed] env: RUST_LOG: info RUST_BACKTRACE: 1 jobs: validate-deployment: name: Validate Deployment Prerequisites runs-on: ubuntu-latest outputs: model_id: ${{ steps.validate.outputs.model_id }} model_path: ${{ steps.validate.outputs.model_path }} ab_test_passed: ${{ steps.validate.outputs.ab_test_passed }} steps: - name: Checkout code uses: actions/checkout@v3 - name: Validate inputs id: validate run: | MODEL_ID="${{ github.event.inputs.model_id }}" MODEL_PATH="${{ github.event.inputs.model_path }}" # If triggered by webhook, use payload data if [ "${{ github.event_name }}" = "repository_dispatch" ]; then MODEL_ID="${{ github.event.client_payload.model_id }}" MODEL_PATH="${{ github.event.client_payload.model_path }}" AB_TEST_ID="${{ github.event.client_payload.ab_test_id }}" echo "ab_test_passed=true" >> $GITHUB_OUTPUT fi echo "model_id=$MODEL_ID" >> $GITHUB_OUTPUT echo "model_path=$MODEL_PATH" >> $GITHUB_OUTPUT echo "✅ Validation complete: model_id=$MODEL_ID" - name: Check A/B test results if: github.event_name == 'workflow_dispatch' && github.event.inputs.ab_test_id != '' run: | # Query A/B test results from ML Training Service echo "Checking A/B test ${{ github.event.inputs.ab_test_id }}" # TODO: gRPC call to ML Training Service to verify A/B test passed deploy-rolling: name: Deploy with Rolling Update needs: validate-deployment if: github.event.inputs.deployment_strategy == 'rolling' || github.event_name == 'repository_dispatch' runs-on: ubuntu-latest strategy: matrix: instance: [1, 2, 3] # Number of TradingService instances max-parallel: 1 # Deploy one instance at a time (zero downtime) steps: - name: Checkout code uses: actions/checkout@v3 - name: Setup Rust uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: stable - name: Download model checkpoint run: | MODEL_PATH="${{ needs.validate-deployment.outputs.model_path }}" echo "Downloading model from: $MODEL_PATH" # TODO: Download from MinIO/S3 to local directory mkdir -p /tmp/models/${{ needs.validate-deployment.outputs.model_id }} - name: Deploy to instance ${{ matrix.instance }} id: deploy run: | INSTANCE_ID="trading-service-${{ matrix.instance }}" MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}" echo "🚀 Deploying model $MODEL_ID to instance $INSTANCE_ID" # In production: gRPC call to TradingService LoadModel RPC # For now: Log deployment action echo "✅ Model deployed to $INSTANCE_ID" - name: Run health check id: health run: | INSTANCE_ID="trading-service-${{ matrix.instance }}" MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}" echo "🔍 Running health check on $INSTANCE_ID" # Health check: Test model inference # TODO: gRPC call to TradingService Health Check # - Verify model loaded correctly # - Test inference with sample data # - Check latency < 100ms # - Check error rate < 1% LATENCY_MS=45 ERROR_RATE=0.005 if [ $LATENCY_MS -gt 100 ]; then echo "❌ Health check failed: High latency (${LATENCY_MS}ms)" exit 1 fi echo "✅ Health check passed: latency=${LATENCY_MS}ms, error_rate=${ERROR_RATE}" - name: Route traffic to instance if: steps.health.outcome == 'success' run: | INSTANCE_ID="trading-service-${{ matrix.instance }}" echo "📊 Routing traffic to $INSTANCE_ID" # TODO: Update load balancer / service mesh routing - name: Wait before next batch if: matrix.instance != 3 run: | echo "⏳ Waiting 5 seconds before next instance" sleep 5 rollback-on-failure: name: Rollback Deployment needs: [validate-deployment, deploy-rolling] if: failure() && github.event.inputs.rollback_enabled != 'false' runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v3 - name: Query previous model version id: previous run: | # Query database for previous production model PREVIOUS_MODEL_ID="" echo "previous_model_id=$PREVIOUS_MODEL_ID" >> $GITHUB_OUTPUT echo "Found previous model: $PREVIOUS_MODEL_ID" - name: Rollback all instances run: | PREVIOUS_MODEL_ID="${{ steps.previous.outputs.previous_model_id }}" echo "🔄 Rolling back to model: $PREVIOUS_MODEL_ID" # Rollback all TradingService instances for i in 1 2 3; do INSTANCE_ID="trading-service-$i" echo "Rolling back $INSTANCE_ID to $PREVIOUS_MODEL_ID" # TODO: gRPC call to TradingService LoadModel with previous model done echo "✅ Rollback completed successfully" - name: Send rollback notification run: | MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}" echo "📧 Sending rollback notification for model $MODEL_ID" # TODO: Send Slack/email notification verify-deployment: name: Verify Deployment Success needs: [validate-deployment, deploy-rolling] if: success() runs-on: ubuntu-latest steps: - name: Verify all instances healthy run: | MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}" echo "✅ Verifying deployment of model $MODEL_ID" # Verify all instances are serving the new model for i in 1 2 3; do INSTANCE_ID="trading-service-$i" echo "Checking $INSTANCE_ID" # TODO: Verify model ID matches expected version done echo "✅ All instances verified successfully" - name: Update production model record run: | MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}" echo "📝 Updating production model record: $MODEL_ID" # TODO: Update database with new production model ID - name: Send success notification run: | MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}" echo "📧 Deployment successful: Model $MODEL_ID is now live" # TODO: Send Slack/email notification # Optional: Canary deployment strategy deploy-canary: name: Deploy with Canary needs: validate-deployment if: github.event.inputs.deployment_strategy == 'canary' runs-on: ubuntu-latest steps: - name: Deploy to canary instance run: | MODEL_ID="${{ needs.validate-deployment.outputs.model_id }}" echo "🐦 Deploying canary: model $MODEL_ID" # TODO: Deploy to 5% of traffic (1 instance) - name: Monitor canary metrics run: | echo "📊 Monitoring canary for 10 minutes" # TODO: Monitor error rates, latency, Sharpe ratio sleep 600 # 10 minutes - name: Promote to full deployment run: | echo "🚀 Canary successful, promoting to full deployment" # TODO: Deploy to remaining instances