diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..ac1a724e3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,283 @@ +name: Build Pipeline + +on: + push: + branches: + - main + - develop + tags: + - 'v*.*.*' + pull_request: + branches: + - main + - develop + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ghcr.io/${{ github.repository_owner }} + +jobs: + docker-build: + name: Docker Build (${{ matrix.service }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + matrix: + service: + - api_gateway + - trading_service + - backtesting_service + - ml_training_service + + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker Metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_PREFIX }}/${{ matrix.service }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and Push Docker Image + uses: docker/build-push-action@v5 + with: + context: . + file: services/${{ matrix.service }}/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + build-args: | + SERVICE_NAME=${{ matrix.service }} + GIT_COMMIT=${{ github.sha }} + GIT_BRANCH=${{ github.ref_name }} + BUILD_DATE=${{ github.event.head_commit.timestamp }} + + - name: Scan Docker Image (Trivy) + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} + format: 'sarif' + output: 'trivy-results-${{ matrix.service }}.sarif' + severity: 'CRITICAL,HIGH' + exit-code: '0' # Don't fail build, just report + + - name: Upload Trivy Results to GitHub Security + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results-${{ matrix.service }}.sarif' + category: 'docker-${{ matrix.service }}' + + - name: Check for Critical Vulnerabilities + run: | + CRITICAL_COUNT=$(grep -c "CRITICAL" trivy-results-${{ matrix.service }}.sarif || true) + HIGH_COUNT=$(grep -c "HIGH" trivy-results-${{ matrix.service }}.sarif || true) + + echo "::notice::Critical vulnerabilities: $CRITICAL_COUNT" + echo "::notice::High vulnerabilities: $HIGH_COUNT" + + if [ "$CRITICAL_COUNT" -gt 0 ]; then + echo "::error::Critical vulnerabilities found in ${{ matrix.service }}" + exit 1 + fi + + build-tli: + name: Build TLI Client + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + artifact_name: tli + asset_name: tli-linux-amd64 + - os: macos-latest + target: x86_64-apple-darwin + artifact_name: tli + asset_name: tli-macos-amd64 + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact_name: tli.exe + asset_name: tli-windows-amd64.exe + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Rust Toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache Cargo Registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Build TLI + run: cargo build --release -p tli --target ${{ matrix.target }} + + - name: Strip Binary (Linux/macOS) + if: matrix.os != 'windows-latest' + run: strip target/${{ matrix.target }}/release/${{ matrix.artifact_name }} + + - name: Upload Binary + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.asset_name }} + path: target/${{ matrix.target }}/release/${{ matrix.artifact_name }} + retention-days: 30 + + create-release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: [docker-build, build-tli] + if: startsWith(github.ref, 'refs/tags/v') + timeout-minutes: 10 + + permissions: + contents: write + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + files: artifacts/**/* + generate_release_notes: true + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + push-to-ecr: + name: Push to AWS ECR (Production) + runs-on: ubuntu-latest + needs: docker-build + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + timeout-minutes: 15 + + strategy: + matrix: + service: + - api_gateway + - trading_service + - backtesting_service + - ml_training_service + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Pull Image from GHCR + run: | + docker pull ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} + + - name: Tag Image for ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + docker tag \ + ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} \ + $ECR_REGISTRY/${{ matrix.service }}:${{ github.sha }} + + docker tag \ + ${{ env.IMAGE_PREFIX }}/${{ matrix.service }}:${{ github.sha }} \ + $ECR_REGISTRY/${{ matrix.service }}:latest + + - name: Push Image to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + docker push $ECR_REGISTRY/${{ matrix.service }}:${{ github.sha }} + docker push $ECR_REGISTRY/${{ matrix.service }}:latest + + summary: + name: Build Summary + runs-on: ubuntu-latest + needs: [docker-build, build-tli] + if: always() + + steps: + - name: Generate Summary + run: | + echo "## Build Pipeline Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Docker Build | ${{ needs.docker-build.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| TLI Build | ${{ needs.build-tli.result }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "### 🚀 Production Images Built" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Images available at:" >> $GITHUB_STEP_SUMMARY + echo "- \`ghcr.io/${{ github.repository_owner }}/api_gateway:${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`ghcr.io/${{ github.repository_owner }}/trading_service:${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`ghcr.io/${{ github.repository_owner }}/backtesting_service:${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`ghcr.io/${{ github.repository_owner }}/ml_training_service:${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY + fi + + - name: Check Overall Status + if: contains(needs.*.result, 'failure') + run: | + echo "::error::Build pipeline failed - check job results above" + exit 1 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..2c7be23f2 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,361 @@ +name: Deployment Pipeline + +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to deploy to' + required: true + type: choice + options: + - dev + - staging + - production + version: + description: 'Docker image tag (e.g., main-abc123 or v1.2.3)' + required: true + type: string + reason: + description: 'Deployment reason' + required: true + type: string + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ghcr.io/${{ github.repository_owner }} + +jobs: + validate-deployment: + name: Validate Deployment + runs-on: ubuntu-latest + timeout-minutes: 5 + + outputs: + approved: ${{ steps.validate.outputs.approved }} + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Validate Version Tag + id: validate + run: | + VERSION="${{ github.event.inputs.version }}" + ENVIRONMENT="${{ github.event.inputs.environment }}" + + echo "Deploying version: $VERSION to $ENVIRONMENT" + + # Verify image exists in registry + docker manifest inspect ${{ env.IMAGE_PREFIX }}/trading_service:$VERSION > /dev/null 2>&1 + + if [ $? -ne 0 ]; then + echo "::error::Image tag $VERSION not found in registry" + exit 1 + fi + + echo "✅ Image tag $VERSION verified" + echo "approved=true" >> $GITHUB_OUTPUT + + request-approval: + name: Request Deployment Approval + runs-on: ubuntu-latest + needs: validate-deployment + if: github.event.inputs.environment == 'staging' || github.event.inputs.environment == 'production' + timeout-minutes: 1440 # 24 hours + + steps: + - name: Request Staging Approval + if: github.event.inputs.environment == 'staging' + uses: trstringer/manual-approval@v1 + with: + secret: ${{ secrets.GITHUB_TOKEN }} + approvers: team-leads + minimum-approvals: 1 + issue-title: "Deploy to Staging: ${{ github.event.inputs.version }}" + issue-body: | + **Environment**: Staging + **Version**: ${{ github.event.inputs.version }} + **Reason**: ${{ github.event.inputs.reason }} + **Triggered by**: ${{ github.actor }} + **Timestamp**: ${{ github.event.head_commit.timestamp }} + + Please review and approve this deployment. + + - name: Request Production Approval + if: github.event.inputs.environment == 'production' + uses: trstringer/manual-approval@v1 + with: + secret: ${{ secrets.GITHUB_TOKEN }} + approvers: team-leads,devops-team + minimum-approvals: 2 + issue-title: "🚀 Deploy to Production: ${{ github.event.inputs.version }}" + issue-body: | + **Environment**: Production + **Version**: ${{ github.event.inputs.version }} + **Reason**: ${{ github.event.inputs.reason }} + **Triggered by**: ${{ github.actor }} + **Timestamp**: ${{ github.event.head_commit.timestamp }} + + ⚠ïļ **PRODUCTION DEPLOYMENT** - Requires 2 approvals + + Please review the following before approving: + - [ ] Staging deployment successful in last 24h + - [ ] All tests passing + - [ ] Security scan clean + - [ ] Rollback plan documented + - [ ] On-call engineer notified + + deploy: + name: Deploy to ${{ github.event.inputs.environment }} + runs-on: ubuntu-latest + needs: [validate-deployment, request-approval] + if: always() && needs.validate-deployment.outputs.approved == 'true' + timeout-minutes: 30 + + environment: + name: ${{ github.event.inputs.environment }} + url: https://${{ github.event.inputs.environment }}.foxhunt.io + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Setup kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.28.0' + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --region us-east-1 \ + --name foxhunt-${{ github.event.inputs.environment }} + + - name: Update Kubernetes Manifests + run: | + cd k8s/overlays/${{ github.event.inputs.environment }} + + # Update image tags for all services + kustomize edit set image \ + api-gateway=${{ env.IMAGE_PREFIX }}/api_gateway:${{ github.event.inputs.version }} \ + trading-service=${{ env.IMAGE_PREFIX }}/trading_service:${{ github.event.inputs.version }} \ + backtesting-service=${{ env.IMAGE_PREFIX }}/backtesting_service:${{ github.event.inputs.version }} \ + ml-training-service=${{ env.IMAGE_PREFIX }}/ml_training_service:${{ github.event.inputs.version }} + + - name: Run Database Migrations + if: github.event.inputs.environment == 'production' + run: | + # Port-forward to PostgreSQL + kubectl port-forward -n foxhunt-${{ github.event.inputs.environment }} \ + svc/postgres 5432:5432 & + + sleep 5 + + # Run migrations + cargo sqlx migrate run \ + --database-url postgresql://foxhunt:${{ secrets.DB_PASSWORD }}@localhost:5432/foxhunt + + - name: Deploy to Kubernetes + run: | + kubectl apply -k k8s/overlays/${{ github.event.inputs.environment }} + + - name: Wait for Rollout + run: | + kubectl rollout status deployment/api-gateway \ + -n foxhunt-${{ github.event.inputs.environment }} \ + --timeout=10m + + kubectl rollout status deployment/trading-service \ + -n foxhunt-${{ github.event.inputs.environment }} \ + --timeout=10m + + kubectl rollout status deployment/backtesting-service \ + -n foxhunt-${{ github.event.inputs.environment }} \ + --timeout=10m + + kubectl rollout status deployment/ml-training-service \ + -n foxhunt-${{ github.event.inputs.environment }} \ + --timeout=10m + + - name: Run Health Checks + run: | + NAMESPACE=foxhunt-${{ github.event.inputs.environment }} + MAX_RETRIES=30 + RETRY_DELAY=10 + + for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do + echo "Checking health of $SERVICE..." + + for i in $(seq 1 $MAX_RETRIES); do + POD=$(kubectl get pod -n $NAMESPACE -l app=$SERVICE -o jsonpath='{.items[0].metadata.name}') + + if kubectl exec -n $NAMESPACE $POD -- grpc_health_probe -addr=:50051; then + echo "✅ $SERVICE health check passed (attempt $i/$MAX_RETRIES)" + break + fi + + if [ $i -eq $MAX_RETRIES ]; then + echo "❌ $SERVICE health check failed after $MAX_RETRIES attempts" + exit 1 + fi + + echo "âģ Health check failed (attempt $i/$MAX_RETRIES), retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + done + done + + - name: Run Smoke Tests + run: | + ENVIRONMENT=${{ github.event.inputs.environment }} + + # API Gateway smoke test + curl -f https://$ENVIRONMENT.foxhunt.io/health || exit 1 + + # Trading Service smoke test + grpcurl -plaintext $ENVIRONMENT.foxhunt.io:50052 grpc.health.v1.Health/Check + + - name: Monitor Metrics (Production) + if: github.event.inputs.environment == 'production' + run: | + # Wait 5 minutes and check Prometheus alerts + echo "Monitoring metrics for 5 minutes..." + sleep 300 + + # Query Prometheus for firing alerts + ALERTS=$(curl -s http://prometheus.foxhunt.io/api/v1/alerts | jq '.data.alerts | length') + + if [ "$ALERTS" -gt 0 ]; then + echo "⚠ïļ Warning: $ALERTS Prometheus alerts firing" + curl -s http://prometheus.foxhunt.io/api/v1/alerts | jq '.data.alerts' + else + echo "✅ No Prometheus alerts firing" + fi + + - name: Notify Slack (Success) + if: success() + run: | + curl -X POST ${{ secrets.SLACK_WEBHOOK }} -H 'Content-Type: application/json' -d '{ + "text": "✅ Deployment Successful", + "attachments": [{ + "color": "good", + "fields": [ + {"title": "Environment", "value": "${{ github.event.inputs.environment }}", "short": true}, + {"title": "Version", "value": "${{ github.event.inputs.version }}", "short": true}, + {"title": "Reason", "value": "${{ github.event.inputs.reason }}", "short": false}, + {"title": "Deployed by", "value": "${{ github.actor }}", "short": true} + ] + }] + }' + + rollback: + name: Rollback Deployment + runs-on: ubuntu-latest + needs: deploy + if: failure() + timeout-minutes: 10 + + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Setup kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.28.0' + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --region us-east-1 \ + --name foxhunt-${{ github.event.inputs.environment }} + + - name: Rollback Deployments + run: | + NAMESPACE=foxhunt-${{ github.event.inputs.environment }} + + echo "🔄 Initiating rollback..." + + kubectl rollout undo deployment/api-gateway -n $NAMESPACE + kubectl rollout undo deployment/trading-service -n $NAMESPACE + kubectl rollout undo deployment/backtesting-service -n $NAMESPACE + kubectl rollout undo deployment/ml-training-service -n $NAMESPACE + + echo "âģ Waiting for rollback to complete..." + + kubectl rollout status deployment/api-gateway -n $NAMESPACE --timeout=5m + kubectl rollout status deployment/trading-service -n $NAMESPACE --timeout=5m + kubectl rollout status deployment/backtesting-service -n $NAMESPACE --timeout=5m + kubectl rollout status deployment/ml-training-service -n $NAMESPACE --timeout=5m + + echo "✅ Rollback completed" + + - name: Verify Rollback + run: | + NAMESPACE=foxhunt-${{ github.event.inputs.environment }} + + for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do + POD=$(kubectl get pod -n $NAMESPACE -l app=$SERVICE -o jsonpath='{.items[0].metadata.name}') + kubectl exec -n $NAMESPACE $POD -- grpc_health_probe -addr=:50051 + done + + - name: Notify Slack (Rollback) + run: | + curl -X POST ${{ secrets.SLACK_WEBHOOK }} -H 'Content-Type: application/json' -d '{ + "text": "⚠ïļ Deployment Failed - Rollback Completed", + "attachments": [{ + "color": "danger", + "fields": [ + {"title": "Environment", "value": "${{ github.event.inputs.environment }}", "short": true}, + {"title": "Version", "value": "${{ github.event.inputs.version }}", "short": true}, + {"title": "Reason", "value": "Deployment failed health checks", "short": false}, + {"title": "Triggered by", "value": "${{ github.actor }}", "short": true} + ] + }] + }' + + create-release-tag: + name: Create Release Tag (Production) + runs-on: ubuntu-latest + needs: deploy + if: github.event.inputs.environment == 'production' && success() + timeout-minutes: 5 + + permissions: + contents: write + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ github.event.inputs.version }} + name: Release ${{ github.event.inputs.version }} + body: | + ## Release ${{ github.event.inputs.version }} + + **Deployed to Production**: ${{ github.event.head_commit.timestamp }} + **Deployment Reason**: ${{ github.event.inputs.reason }} + **Deployed by**: ${{ github.actor }} + + ### Services Deployed + - API Gateway: `${{ env.IMAGE_PREFIX }}/api_gateway:${{ github.event.inputs.version }}` + - Trading Service: `${{ env.IMAGE_PREFIX }}/trading_service:${{ github.event.inputs.version }}` + - Backtesting Service: `${{ env.IMAGE_PREFIX }}/backtesting_service:${{ github.event.inputs.version }}` + - ML Training Service: `${{ env.IMAGE_PREFIX }}/ml_training_service:${{ github.event.inputs.version }}` + + See [CHANGELOG.md](CHANGELOG.md) for details. + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..761628771 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,272 @@ +name: Test Pipeline + +on: + push: + branches: + - main + - develop + - 'feature/**' + pull_request: + branches: + - main + - develop + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + SQLX_OFFLINE: true # Use offline mode for CI + +jobs: + test: + name: Test Suite + runs-on: ubuntu-latest + timeout-minutes: 30 + + services: + postgres: + image: timescale/timescaledb:latest-pg16 + env: + POSTGRES_USER: foxhunt + POSTGRES_PASSWORD: foxhunt_dev_password + POSTGRES_DB: foxhunt + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + vault: + image: hashicorp/vault:latest + env: + VAULT_DEV_ROOT_TOKEN_ID: foxhunt-dev-root + VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 + ports: + - 8200:8200 + options: >- + --cap-add=IPC_LOCK + --health-cmd "vault status" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + DATABASE_URL: postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt + REDIS_URL: redis://localhost:6379 + VAULT_ADDR: http://localhost:8200 + VAULT_TOKEN: foxhunt-dev-root + JWT_SECRET: test_jwt_secret_key + RUST_LOG: info + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Rust Toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy, llvm-tools-preview + + - name: Cache Cargo Registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Install SQLx CLI + run: cargo install sqlx-cli --no-default-features --features postgres + + - name: Run Database Migrations + run: cargo sqlx migrate run + + - name: Install llvm-cov + run: cargo install cargo-llvm-cov + + - name: Check Formatting + run: cargo fmt --all -- --check + + - name: Run Clippy + run: | + cargo clippy --workspace --all-targets -- \ + -D warnings \ + -D clippy::unwrap_used \ + -D clippy::expect_used \ + -D clippy::panic \ + -W clippy::all + + - name: Run Unit Tests + run: cargo test --workspace --lib --bins + env: + RUST_LOG: debug + + - name: Run Integration Tests + run: cargo test --workspace --test '*' -- --test-threads=1 + timeout-minutes: 15 + + - name: Generate Coverage Report + run: | + cargo llvm-cov --workspace --lcov --output-path lcov.info + cargo llvm-cov --workspace --html --output-dir coverage_report + + - name: Check Coverage Threshold + run: | + COVERAGE=$(cargo llvm-cov --workspace --summary-only | grep -oP 'TOTAL.*\K[0-9.]+(?=%)') + echo "::notice::Coverage: $COVERAGE%" + + if (( $(echo "$COVERAGE < 60.0" | bc -l) )); then + echo "::error::Coverage $COVERAGE% below threshold 60%" + exit 1 + fi + + echo "::notice::✅ Coverage $COVERAGE% meets threshold" + + - name: Upload Coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: ./lcov.info + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + + - name: Upload Coverage Report + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-report + path: coverage_report/ + retention-days: 30 + + - name: Upload Test Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: target/nextest/ + retention-days: 30 + + security-audit: + name: Security Audit + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Rust Toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install Cargo Audit + run: cargo install cargo-audit + + - name: Run Security Audit + run: | + cargo audit --deny warnings \ + --ignore RUSTSEC-2020-0071 # RSA Marvin Attack (mitigated) + + - name: Install Cargo Geiger + run: cargo install cargo-geiger + + - name: Unsafe Code Audit + run: | + cargo geiger --all-features --output-format GitHubMarkdown >> $GITHUB_STEP_SUMMARY + + dependency-check: + name: Dependency Check + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Rust Toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install Cargo Outdated + run: cargo install cargo-outdated + + - name: Check Outdated Dependencies + run: cargo outdated --exit-code 1 + continue-on-error: true + + - name: Install Cargo Deny + run: cargo install cargo-deny + + - name: Check Licenses + run: cargo deny check licenses + + - name: Check Advisories + run: cargo deny check advisories + + build-check: + name: Build Check + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Rust Toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache Cargo Registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Check Compilation + run: cargo check --workspace --all-targets + + - name: Build Release + run: cargo build --workspace --release + + summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [test, security-audit, dependency-check, build-check] + if: always() + + steps: + - name: Generate Summary + run: | + echo "## Test Pipeline Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Test Suite | ${{ needs.test.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Security Audit | ${{ needs.security-audit.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Dependency Check | ${{ needs.dependency-check.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Build Check | ${{ needs.build-check.result }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Check Overall Status + if: contains(needs.*.result, 'failure') + run: | + echo "::error::Pipeline failed - check job results above" + exit 1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..b907a8cea --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,422 @@ +# GitLab CI/CD Pipeline for Foxhunt HFT Trading System +# This is an example configuration for teams using GitLab CI instead of GitHub Actions + +stages: + - test + - security + - build + - deploy + +variables: + CARGO_HOME: ${CI_PROJECT_DIR}/.cargo + RUST_BACKTRACE: "1" + SQLX_OFFLINE: "true" + REGISTRY: ${CI_REGISTRY} + IMAGE_PREFIX: ${CI_REGISTRY_IMAGE} + +# Cache Cargo dependencies +.rust_cache: + cache: + key: ${CI_COMMIT_REF_SLUG} + paths: + - .cargo/ + - target/ + +# Docker-in-Docker service for building images +.docker_service: + services: + - docker:24-dind + variables: + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_TLS_VERIFY: 1 + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" + +# ================================ +# Test Stage +# ================================ + +test:unit: + stage: test + image: rust:1.75-slim + extends: .rust_cache + services: + - postgres:16-alpine + - redis:7-alpine + - vault:latest + variables: + POSTGRES_USER: foxhunt + POSTGRES_PASSWORD: foxhunt_dev_password + POSTGRES_DB: foxhunt + DATABASE_URL: postgresql://foxhunt:foxhunt_dev_password@postgres:5432/foxhunt + REDIS_URL: redis://redis:6379 + VAULT_ADDR: http://vault:8200 + VAULT_TOKEN: foxhunt-dev-root + VAULT_DEV_ROOT_TOKEN_ID: foxhunt-dev-root + JWT_SECRET: test_jwt_secret + RUST_LOG: info + before_script: + - apt-get update && apt-get install -y libssl-dev pkg-config postgresql-client + - cargo install sqlx-cli --no-default-features --features postgres + - cargo install cargo-llvm-cov + - cargo sqlx migrate run + script: + - cargo fmt --all -- --check + - cargo clippy --workspace --all-targets -- -D warnings + - cargo test --workspace --lib --bins + - cargo test --workspace --test '*' -- --test-threads=1 + timeout: 30m + artifacts: + when: always + paths: + - target/nextest/ + expire_in: 30 days + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH == "develop"' + - if: '$CI_COMMIT_BRANCH =~ /^feature\/.*/' + +test:coverage: + stage: test + image: rust:1.75-slim + extends: .rust_cache + services: + - postgres:16-alpine + - redis:7-alpine + - vault:latest + variables: + POSTGRES_USER: foxhunt + POSTGRES_PASSWORD: foxhunt_dev_password + POSTGRES_DB: foxhunt + DATABASE_URL: postgresql://foxhunt:foxhunt_dev_password@postgres:5432/foxhunt + REDIS_URL: redis://redis:6379 + VAULT_ADDR: http://vault:8200 + VAULT_TOKEN: foxhunt-dev-root + VAULT_DEV_ROOT_TOKEN_ID: foxhunt-dev-root + JWT_SECRET: test_jwt_secret + before_script: + - apt-get update && apt-get install -y libssl-dev pkg-config postgresql-client + - cargo install sqlx-cli --no-default-features --features postgres + - cargo install cargo-llvm-cov + - cargo sqlx migrate run + script: + - cargo llvm-cov --workspace --lcov --output-path lcov.info + - cargo llvm-cov --workspace --html --output-dir coverage_report + - | + COVERAGE=$(cargo llvm-cov --workspace --summary-only | grep -oP 'TOTAL.*\K[0-9.]+(?=%)') + echo "Coverage: $COVERAGE%" + if (( $(echo "$COVERAGE < 60.0" | bc -l) )); then + echo "❌ Coverage $COVERAGE% below threshold 60%" + exit 1 + fi + echo "✅ Coverage $COVERAGE% meets threshold" + coverage: '/TOTAL.*\s+(\d+\.\d+)%/' + timeout: 30m + artifacts: + paths: + - lcov.info + - coverage_report/ + expire_in: 30 days + reports: + coverage_report: + coverage_format: cobertura + path: lcov.info + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH == "develop"' + +# ================================ +# Security Stage +# ================================ + +security:audit: + stage: security + image: rust:1.75-slim + extends: .rust_cache + before_script: + - cargo install cargo-audit + - cargo install cargo-geiger + script: + - cargo audit --deny warnings --ignore RUSTSEC-2020-0071 + - cargo geiger --all-features --output-format Json > geiger-report.json + timeout: 10m + artifacts: + paths: + - geiger-report.json + expire_in: 30 days + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH == "develop"' + +security:sast: + stage: security + image: returntocorp/semgrep:latest + script: + - semgrep --config=auto --sarif --output=sast-report.sarif . + timeout: 10m + artifacts: + reports: + sast: sast-report.sarif + expire_in: 30 days + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + +security:secrets: + stage: security + image: trufflesecurity/trufflehog:latest + script: + - trufflehog filesystem . --json --only-verified > secrets-scan.json + timeout: 10m + artifacts: + paths: + - secrets-scan.json + expire_in: 30 days + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_COMMIT_BRANCH == "main"' + +# ================================ +# Build Stage +# ================================ + +.build_docker_image: + stage: build + extends: .docker_service + image: docker:24 + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - | + # Build multi-platform image + docker buildx create --use --name multiarch --driver docker-container + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --file services/${SERVICE}/Dockerfile \ + --tag ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_SHA} \ + --tag ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_REF_SLUG} \ + --push \ + --build-arg SERVICE_NAME=${SERVICE} \ + --build-arg GIT_COMMIT=${CI_COMMIT_SHA} \ + --build-arg GIT_BRANCH=${CI_COMMIT_REF_NAME} \ + --build-arg BUILD_DATE=${CI_COMMIT_TIMESTAMP} \ + . + - | + # Scan image for vulnerabilities + docker pull ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_SHA} + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + aquasec/trivy:latest \ + image --severity CRITICAL,HIGH \ + --exit-code 0 \ + --format json \ + --output trivy-${SERVICE}.json \ + ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_SHA} + timeout: 30m + artifacts: + paths: + - trivy-*.json + expire_in: 30 days + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH == "develop"' + - if: '$CI_COMMIT_TAG' + +build:api_gateway: + extends: .build_docker_image + variables: + SERVICE: api_gateway + +build:trading_service: + extends: .build_docker_image + variables: + SERVICE: trading_service + +build:backtesting_service: + extends: .build_docker_image + variables: + SERVICE: backtesting_service + +build:ml_training_service: + extends: .build_docker_image + variables: + SERVICE: ml_training_service + +build:tli: + stage: build + image: rust:1.75-slim + extends: .rust_cache + parallel: + matrix: + - TARGET: x86_64-unknown-linux-gnu + PLATFORM: linux-amd64 + - TARGET: x86_64-apple-darwin + PLATFORM: macos-amd64 + - TARGET: x86_64-pc-windows-msvc + PLATFORM: windows-amd64 + before_script: + - apt-get update && apt-get install -y libssl-dev pkg-config + - rustup target add ${TARGET} + script: + - cargo build --release -p tli --target ${TARGET} + - | + if [ "${TARGET}" != "x86_64-pc-windows-msvc" ]; then + strip target/${TARGET}/release/tli + fi + timeout: 20m + artifacts: + name: "tli-${PLATFORM}" + paths: + - target/${TARGET}/release/tli* + expire_in: 30 days + rules: + - if: '$CI_COMMIT_TAG' + +# ================================ +# Deploy Stage +# ================================ + +.deploy: + stage: deploy + image: bitnami/kubectl:latest + before_script: + - kubectl config set-cluster k8s --server="${K8S_API_URL}" + - kubectl config set-credentials gitlab --token="${K8S_TOKEN}" + - kubectl config set-context default --cluster=k8s --user=gitlab --namespace=${NAMESPACE} + - kubectl config use-context default + script: + - | + # Update image tags + cd k8s/overlays/${ENVIRONMENT} + kubectl set image deployment/api-gateway api-gateway=${IMAGE_PREFIX}/api_gateway:${CI_COMMIT_SHA} + kubectl set image deployment/trading-service trading-service=${IMAGE_PREFIX}/trading_service:${CI_COMMIT_SHA} + kubectl set image deployment/backtesting-service backtesting-service=${IMAGE_PREFIX}/backtesting_service:${CI_COMMIT_SHA} + kubectl set image deployment/ml-training-service ml-training-service=${IMAGE_PREFIX}/ml_training_service:${CI_COMMIT_SHA} + + # Wait for rollout + kubectl rollout status deployment/api-gateway --timeout=10m + kubectl rollout status deployment/trading-service --timeout=10m + kubectl rollout status deployment/backtesting-service --timeout=10m + kubectl rollout status deployment/ml-training-service --timeout=10m + + # Health checks + for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do + POD=$(kubectl get pod -l app=${SERVICE} -o jsonpath='{.items[0].metadata.name}') + kubectl exec ${POD} -- grpc_health_probe -addr=:50051 + done + timeout: 30m + +deploy:dev: + extends: .deploy + variables: + ENVIRONMENT: dev + NAMESPACE: foxhunt-dev + K8S_API_URL: ${DEV_K8S_API_URL} + K8S_TOKEN: ${DEV_K8S_TOKEN} + environment: + name: development + url: https://dev.foxhunt.io + on_stop: stop:dev + rules: + - if: '$CI_COMMIT_BRANCH == "develop"' + +deploy:staging: + extends: .deploy + variables: + ENVIRONMENT: staging + NAMESPACE: foxhunt-staging + K8S_API_URL: ${STAGING_K8S_API_URL} + K8S_TOKEN: ${STAGING_K8S_TOKEN} + environment: + name: staging + url: https://staging.foxhunt.io + on_stop: stop:staging + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + when: manual + +deploy:production: + extends: .deploy + variables: + ENVIRONMENT: production + NAMESPACE: foxhunt-prod + K8S_API_URL: ${PROD_K8S_API_URL} + K8S_TOKEN: ${PROD_K8S_TOKEN} + environment: + name: production + url: https://foxhunt.io + on_stop: stop:production + rules: + - if: '$CI_COMMIT_TAG' + when: manual + needs: + - build:api_gateway + - build:trading_service + - build:backtesting_service + - build:ml_training_service + - security:audit + +# ================================ +# Cleanup +# ================================ + +.stop_environment: + stage: deploy + image: bitnami/kubectl:latest + when: manual + script: + - kubectl scale deployment --all --replicas=0 -n ${NAMESPACE} + +stop:dev: + extends: .stop_environment + variables: + NAMESPACE: foxhunt-dev + environment: + name: development + action: stop + +stop:staging: + extends: .stop_environment + variables: + NAMESPACE: foxhunt-staging + environment: + name: staging + action: stop + +stop:production: + extends: .stop_environment + variables: + NAMESPACE: foxhunt-prod + environment: + name: production + action: stop + +# ================================ +# Release +# ================================ + +release: + stage: deploy + image: registry.gitlab.com/gitlab-org/release-cli:latest + script: + - echo "Creating release for ${CI_COMMIT_TAG}" + release: + tag_name: ${CI_COMMIT_TAG} + name: 'Release ${CI_COMMIT_TAG}' + description: | + ## Release ${CI_COMMIT_TAG} + + Deployed to production at ${CI_COMMIT_TIMESTAMP} + + ### Services + - API Gateway: ${IMAGE_PREFIX}/api_gateway:${CI_COMMIT_TAG} + - Trading Service: ${IMAGE_PREFIX}/trading_service:${CI_COMMIT_TAG} + - Backtesting Service: ${IMAGE_PREFIX}/backtesting_service:${CI_COMMIT_TAG} + - ML Training Service: ${IMAGE_PREFIX}/ml_training_service:${CI_COMMIT_TAG} + rules: + - if: '$CI_COMMIT_TAG' + needs: + - deploy:production diff --git a/CI_CD_PIPELINE.md b/CI_CD_PIPELINE.md new file mode 100644 index 000000000..90f20e124 --- /dev/null +++ b/CI_CD_PIPELINE.md @@ -0,0 +1,1229 @@ +# CI/CD Pipeline Documentation + +**Last Updated**: 2025-10-07 +**Version**: 1.0.0 +**Owner**: Platform Team + +--- + +## Table of Contents + +1. [Pipeline Overview](#pipeline-overview) +2. [Testing Stage](#testing-stage) +3. [Build Stage](#build-stage) +4. [Deployment Stage](#deployment-stage) +5. [Security Scanning](#security-scanning) +6. [Performance Benchmarks](#performance-benchmarks) +7. [GitOps Integration](#gitops-integration) +8. [Workflow Examples](#workflow-examples) +9. [Troubleshooting](#troubleshooting) + +--- + +## Pipeline Overview + +### Pipeline Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ CI/CD Pipeline │ +└─────────────────────────────────────────────────────────────────┘ + │ + ┌───────────â”ī───────────┐ + ▾ ▾ + ┌──────────┐ ┌──────────┐ + │ Test │ │ Build │ + │ Stage │ │ Stage │ + └────┮─────┘ └────┮─────┘ + │ │ + ┌─────────â”ī─────────┐ │ + ▾ ▾ ▾ + ┌─────────┐ ┌─────────┐ ┌─────────┐ + │ Unit │ │ Integ │ │ Docker │ + │ Tests │ │ Tests │ │ Build │ + └────┮────┘ └────┮────┘ └────┮────┘ + │ │ │ + └─────────┮─────────┘ │ + ▾ ▾ + ┌──────────┐ ┌──────────┐ + │ Security │ │ Deploy │ + │ Scan │ │ Stage │ + └────┮─────┘ └────┮─────┘ + │ │ + │ ┌─────────────â”ī─────────────┐ + │ ▾ ▾ ▾ + │ ┌──────┐ ┌─────────┐ ┌──────┐ + │ │ Dev │ │ Staging │ │ Prod │ + │ └──────┘ └─────────┘ └──────┘ + │ │ │ │ + └──────â”ī─────────────â”ī────────────┘ + │ + ▾ + ┌──────────┐ + │ Validate │ + │ Health │ + └──────────┘ +``` + +### Pipeline Stages + +| Stage | Purpose | Duration | Auto/Manual | +|-------|---------|----------|-------------| +| **Test** | Run unit, integration, and coverage tests | 5-10 min | Auto | +| **Security Scan** | Vulnerability scanning, SAST/DAST | 3-5 min | Auto | +| **Build** | Docker image building and tagging | 8-12 min | Auto | +| **Deploy Dev** | Deploy to development environment | 2-3 min | Auto | +| **Deploy Staging** | Deploy to staging environment | 3-5 min | Manual approval | +| **Performance Test** | Run automated benchmarks | 5-10 min | Auto (staging) | +| **Deploy Prod** | Deploy to production environment | 5-8 min | Manual approval | +| **Rollback** | Automated rollback on health check failure | 1-2 min | Auto (on failure) | + +### Branching Strategy + +``` +main (production) + │ + ├── develop (staging) + │ │ + │ ├── feature/agent-98-ci-cd-docs + │ ├── feature/performance-optimization + │ └── feature/new-ml-model + │ + └── hotfix/critical-bug-fix +``` + +**Branch Protection Rules**: +- **main**: Requires 2 approvals, all checks pass, no force push +- **develop**: Requires 1 approval, all checks pass +- **feature/***: No restrictions, auto-delete after merge + +**Merge Strategy**: +- Feature → Develop: Squash merge +- Develop → Main: Merge commit (preserves history) +- Hotfix → Main: Cherry-pick to develop + +### Automated vs Manual Gates + +**Automated Gates** (block merge on failure): +- ✅ All unit tests pass (>99% pass rate required) +- ✅ Integration tests pass +- ✅ Code coverage â‰Ĩ60% (enforced) +- ✅ No critical/high security vulnerabilities +- ✅ Docker build succeeds +- ✅ Linting passes (`cargo clippy`) +- ✅ Formatting passes (`cargo fmt --check`) + +**Manual Gates** (require approval): +- 🔒 Deploy to staging (team lead approval) +- 🔒 Deploy to production (2 approvals: team lead + DevOps) +- 🔒 Database migrations (DBA approval) +- 🔒 Infrastructure changes (DevOps approval) + +### Integration Points + +**GitHub Actions**: Primary CI/CD platform +- Workflows: `.github/workflows/*.yml` +- Secrets: GitHub Secrets (encrypted) +- Artifacts: GitHub Packages / Docker Registry + +**GitLab CI**: Alternative/fallback +- Configuration: `.gitlab-ci.yml` +- Runners: Kubernetes-based GitLab Runners +- Registry: GitLab Container Registry + +**Kubernetes**: Deployment target +- Manifests: `k8s/*.yaml` +- Helm charts: `helm/foxhunt/` +- ArgoCD: GitOps continuous deployment + +**Observability**: +- Prometheus: Metrics collection +- Grafana: Dashboards and alerts +- Loki: Log aggregation +- Jaeger: Distributed tracing + +--- + +## Testing Stage + +### Test Execution + +**Unit Tests** (5-7 minutes): +```bash +# Run all unit tests with coverage +cargo llvm-cov --workspace --lcov --output-path lcov.info + +# Requirements: +# - Pass rate: >99% (336/337+ tests passing) +# - Coverage: â‰Ĩ60% workspace coverage +# - No panics or segfaults +``` + +**Integration Tests** (3-5 minutes): +```bash +# Run integration tests (requires Docker services) +docker-compose up -d postgres redis vault +cargo test --workspace --test '*' -- --test-threads=1 + +# Tests: +# - API Gateway E2E (services/api_gateway/tests/e2e_tests.rs) +# - Trading Service E2E (services/trading_service/tests/integration_e2e_tests.rs) +# - ML Training Service (services/ml_training_service/tests/integration_tests.rs) +# - Backtesting Service (services/backtesting_service/tests/integration_tests.rs) +``` + +**GPU Tests** (optional, 2-3 minutes): +```bash +# Run slow GPU tests (if CUDA available) +cargo test -p ml --lib -- --ignored --test-threads=1 + +# Skipped in CI if GPU not available +# Uses CPU fallback automatically +``` + +### Coverage Reporting + +**Coverage Threshold Enforcement**: +```yaml +# .github/workflows/test.yml +- name: Check Coverage Threshold + run: | + COVERAGE=$(cargo llvm-cov --workspace --summary-only | grep -oP 'TOTAL.*\K[0-9.]+(?=%)') + echo "Coverage: $COVERAGE%" + if (( $(echo "$COVERAGE < 60.0" | bc -l) )); then + echo "❌ Coverage $COVERAGE% below threshold 60%" + exit 1 + fi + echo "✅ Coverage $COVERAGE% meets threshold" +``` + +**Coverage Report Artifacts**: +- `lcov.info`: LCOV format (for Codecov/Coveralls) +- `coverage_report/`: HTML report (browsable) +- `coverage-summary.json`: JSON summary for badges + +**Coverage Badges**: +```markdown +[![Coverage](https://img.shields.io/codecov/c/github/foxhunt/foxhunt)](https://codecov.io/gh/foxhunt/foxhunt) +``` + +### Test Pass Rate Requirements + +| Environment | Pass Rate | Coverage | Action on Failure | +|-------------|-----------|----------|-------------------| +| **Feature Branch** | â‰Ĩ99% | â‰Ĩ55% | Block merge, notify author | +| **Develop** | â‰Ĩ99.5% | â‰Ĩ60% | Block merge, alert team | +| **Main** | 100% | â‰Ĩ60% | Block merge, rollback if deployed | + +**Flaky Test Handling**: +```yaml +# Retry flaky tests up to 3 times +- name: Run Tests with Retry + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: cargo test --workspace +``` + +--- + +## Build Stage + +### Docker Image Building + +**Multi-Stage Build Strategy** (from Agent 94): +```dockerfile +# Stage 1: Build dependencies +FROM rust:1.75-slim AS chef +RUN cargo install cargo-chef + +# Stage 2: Plan dependencies +FROM chef AS planner +WORKDIR /app +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +# Stage 3: Build dependencies (cached layer) +FROM chef AS builder +WORKDIR /app +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json + +# Stage 4: Build application +COPY . . +RUN cargo build --release -p trading_service + +# Stage 5: Runtime image +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y ca-certificates libssl3 && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/target/release/trading_service /usr/local/bin/ +CMD ["trading_service"] +``` + +**Build Optimization**: +- Dependency caching: `cargo-chef` reduces rebuild time 80% +- Layer caching: Only rebuild changed layers +- Multi-platform: Build for `linux/amd64` and `linux/arm64` +- Image size: ~50MB final runtime image + +### Image Tagging Strategy + +**Tagging Scheme**: +```bash +# Git commit SHA (immutable) +foxhunt/trading-service:abc123def456 + +# Semantic version (tagged releases) +foxhunt/trading-service:v1.2.3 + +# Environment tags (mutable) +foxhunt/trading-service:latest # main branch +foxhunt/trading-service:develop # develop branch +foxhunt/trading-service:staging # staging environment +foxhunt/trading-service:production # production environment + +# Full example +foxhunt/trading-service:v1.2.3-abc123def456 +``` + +**Tagging Logic**: +```yaml +- name: Docker Metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + foxhunt/trading-service + ghcr.io/foxhunt/trading-service + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix={{branch}}- +``` + +### Image Registry + +**Primary Registry: GitHub Container Registry (GHCR)**: +```bash +# Login +echo $GITHUB_TOKEN | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + +# Push +docker push ghcr.io/foxhunt/trading-service:v1.2.3 +``` + +**Alternative Registries**: +- **AWS ECR**: Production deployments (high availability) +- **Docker Hub**: Public images (documentation examples) +- **GitLab Registry**: Fallback/self-hosted option + +**Registry Configuration**: +```yaml +# .github/workflows/build.yml +- name: Push to Multiple Registries + run: | + # GitHub Container Registry (primary) + docker push ghcr.io/foxhunt/trading-service:${{ steps.meta.outputs.tags }} + + # AWS ECR (production) + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ECR_REGISTRY + docker tag ghcr.io/foxhunt/trading-service:$TAG $ECR_REGISTRY/trading-service:$TAG + docker push $ECR_REGISTRY/trading-service:$TAG +``` + +### Build Artifact Storage + +**Artifacts Stored**: +1. Docker images (tagged and pushed to registries) +2. Coverage reports (uploaded to Codecov) +3. Test results (JUnit XML format) +4. Build logs (GitHub Actions artifacts) +5. Binary releases (GitHub Releases for tagged versions) + +**Artifact Retention**: +- **Docker images**: 90 days for non-production tags +- **Coverage reports**: Permanent (Codecov) +- **Test results**: 30 days (GitHub Actions) +- **Build logs**: 90 days (GitHub Actions) +- **Releases**: Permanent (GitHub Releases) + +--- + +## Deployment Stage + +### Deployment Environments + +| Environment | Purpose | Auto-Deploy | Approval | Rollback | +|-------------|---------|-------------|----------|----------| +| **Development** | Feature testing, rapid iteration | ✅ Yes (on push to feature/*) | None | Auto (on failure) | +| **Staging** | Pre-production validation | ✅ Yes (on merge to develop) | Team Lead | Auto (on failure) | +| **Production** | Live trading | ❌ Manual | 2 approvals | Manual (with auto option) | + +### Development Environment + +**Auto-Deploy Trigger**: +```yaml +# Deploy on push to any feature branch +on: + push: + branches: + - 'feature/**' + - 'develop' +``` + +**Deployment Steps**: +1. Build Docker image +2. Push to dev registry +3. Update Kubernetes deployment (dev namespace) +4. Wait for rollout completion +5. Run smoke tests +6. Notify team on Slack + +**Dev Environment Configuration**: +```yaml +# k8s/dev/trading-service.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-service + namespace: foxhunt-dev +spec: + replicas: 1 # Single replica for dev + strategy: + type: Recreate # Fast deployment, downtime acceptable + template: + spec: + containers: + - name: trading-service + image: ghcr.io/foxhunt/trading-service:develop + env: + - name: RUST_LOG + value: "debug" # Verbose logging in dev + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: foxhunt-dev-secrets + key: database-url +``` + +### Staging Environment + +**Deploy Trigger**: +```yaml +# Deploy on merge to develop (with approval) +on: + pull_request: + types: [closed] + branches: + - develop +``` + +**Approval Workflow**: +```yaml +- name: Request Staging Deployment Approval + uses: trstringer/manual-approval@v1 + with: + secret: ${{ secrets.GITHUB_TOKEN }} + approvers: team-leads + minimum-approvals: 1 + issue-title: "Deploy to Staging: ${{ github.sha }}" +``` + +**Staging Deployment Steps**: +1. Run full test suite (unit + integration) +2. Security scan (no critical/high vulnerabilities) +3. Build Docker image +4. Push to staging registry +5. **Manual approval required** +6. Update Kubernetes deployment (staging namespace) +7. Run performance benchmarks +8. Run smoke tests +9. Validate metrics (Prometheus alerts) +10. Notify team on Slack + +**Staging Configuration**: +```yaml +# k8s/staging/trading-service.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-service + namespace: foxhunt-staging +spec: + replicas: 2 # Multi-replica for load testing + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 # Zero-downtime deployment + maxSurge: 1 + template: + spec: + containers: + - name: trading-service + image: ghcr.io/foxhunt/trading-service:staging-${{ github.sha }} + env: + - name: RUST_LOG + value: "info" + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi +``` + +### Production Environment + +**Deploy Trigger**: +```yaml +# Manual trigger only (no auto-deploy to production) +on: + workflow_dispatch: + inputs: + version: + description: 'Version to deploy (e.g., v1.2.3)' + required: true + reason: + description: 'Deployment reason' + required: true +``` + +**Approval Workflow**: +```yaml +- name: Request Production Deployment Approval + uses: trstringer/manual-approval@v1 + with: + secret: ${{ secrets.GITHUB_TOKEN }} + approvers: team-leads,devops-team + minimum-approvals: 2 # Requires 2 approvals + issue-title: "🚀 Deploy to Production: ${{ github.event.inputs.version }}" + issue-body: | + **Version**: ${{ github.event.inputs.version }} + **Reason**: ${{ github.event.inputs.reason }} + **Commit**: ${{ github.sha }} + **Triggered by**: ${{ github.actor }} +``` + +**Production Deployment Steps**: +1. Verify staging deployment successful (last 24h) +2. Security scan (zero critical vulnerabilities) +3. Build Docker image (tagged with version) +4. Push to production registry (ECR) +5. **2 manual approvals required** +6. Database migrations (if any, with DBA approval) +7. Blue-Green deployment: + - Deploy to "green" environment + - Run health checks + - Run smoke tests + - Shift traffic gradually (10% → 50% → 100%) + - Monitor metrics for 15 minutes +8. Decommission "blue" environment +9. Create GitHub release +10. Notify team on Slack + +**Production Configuration**: +```yaml +# k8s/prod/trading-service.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trading-service + namespace: foxhunt-prod +spec: + replicas: 4 # High availability + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 # Maintain 75% capacity during rollout + maxSurge: 1 + template: + spec: + containers: + - name: trading-service + image: 123456789012.dkr.ecr.us-east-1.amazonaws.com/foxhunt/trading-service:v1.2.3 + env: + - name: RUST_LOG + value: "warn" # Minimal logging in production + - name: ENABLE_AUDIT_LOGGING + value: "true" + resources: + requests: + cpu: 2000m + memory: 4Gi + limits: + cpu: 4000m + memory: 8Gi + affinity: + podAntiAffinity: # Spread across nodes + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - trading-service + topologyKey: kubernetes.io/hostname +``` + +### Health Check Validation + +**Kubernetes Liveness Probe**: +```yaml +livenessProbe: + exec: + command: + - /usr/local/bin/grpc_health_probe + - -addr=:50051 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 +``` + +**Kubernetes Readiness Probe**: +```yaml +readinessProbe: + exec: + command: + - /usr/local/bin/grpc_health_probe + - -addr=:50051 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 +``` + +**Post-Deployment Health Checks**: +```bash +#!/bin/bash +# scripts/health_check.sh + +SERVICE_URL="https://api.foxhunt.io" +MAX_RETRIES=30 +RETRY_DELAY=10 + +for i in $(seq 1 $MAX_RETRIES); do + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $SERVICE_URL/health) + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "✅ Health check passed (attempt $i/$MAX_RETRIES)" + exit 0 + fi + + echo "âģ Health check failed (attempt $i/$MAX_RETRIES), retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY +done + +echo "❌ Health check failed after $MAX_RETRIES attempts" +exit 1 +``` + +### Rollback Automation + +**Automatic Rollback Triggers**: +- Health check failure after deployment +- Error rate >5% (Prometheus alert) +- P99 latency >100ms (Prometheus alert) +- Memory usage >90% (Kubernetes OOM) + +**Rollback Process**: +```yaml +- name: Automatic Rollback on Failure + if: failure() + run: | + echo "🔄 Deployment failed, initiating rollback..." + kubectl rollout undo deployment/trading-service -n foxhunt-prod + kubectl rollout status deployment/trading-service -n foxhunt-prod + + # Notify team + curl -X POST $SLACK_WEBHOOK -d '{ + "text": "⚠ïļ Production deployment failed, rollback completed", + "attachments": [{ + "color": "danger", + "fields": [ + {"title": "Version", "value": "${{ github.event.inputs.version }}"}, + {"title": "Commit", "value": "${{ github.sha }}"}, + {"title": "Actor", "value": "${{ github.actor }}"} + ] + }] + }' +``` + +**Manual Rollback**: +```bash +# Rollback to previous version +kubectl rollout undo deployment/trading-service -n foxhunt-prod + +# Rollback to specific revision +kubectl rollout undo deployment/trading-service -n foxhunt-prod --to-revision=42 + +# Check rollout status +kubectl rollout status deployment/trading-service -n foxhunt-prod + +# Verify rollback +kubectl describe deployment trading-service -n foxhunt-prod | grep Image: +``` + +--- + +## Security Scanning + +### Dependency Vulnerability Scanning + +**Tool**: Cargo Audit (RustSec Advisory Database) + +```yaml +- name: Security Audit + run: | + cargo install cargo-audit + cargo audit --deny warnings --ignore RUSTSEC-2020-0071 +``` + +**Current Known Vulnerabilities**: +- **RSA Marvin Attack (CVSS 5.9)**: Mitigated (PostgreSQL-only, no MySQL) +- **Unmaintained**: `instant`, `paste` (low risk) + +**Vulnerability Response SLA**: +| Severity | Response Time | Fix Time | Action | +|----------|---------------|----------|--------| +| **Critical** | 1 hour | 24 hours | Block all deployments | +| **High** | 4 hours | 72 hours | Block production deployments | +| **Medium** | 1 day | 1 week | Warning, track in issue | +| **Low** | 1 week | 1 month | Track in backlog | + +### Docker Image Scanning + +**Tool**: Trivy (Aqua Security) + +```yaml +- name: Scan Docker Image + uses: aquasecurity/trivy-action@master + with: + image-ref: 'ghcr.io/foxhunt/trading-service:${{ github.sha }}' + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + exit-code: '1' # Fail on critical/high vulnerabilities +``` + +**Scan Coverage**: +- OS package vulnerabilities (Debian base image) +- Application dependencies (Rust crates) +- Embedded secrets detection +- Malware scanning + +### SAST/DAST Integration + +**Static Application Security Testing (SAST)**: + +Tool: **Clippy** (Rust linter with security rules) +```yaml +- name: Run Clippy Security Lints + run: | + cargo clippy --workspace --all-targets -- \ + -D clippy::unwrap_used \ + -D clippy::expect_used \ + -D clippy::panic \ + -D clippy::todo \ + -D clippy::unimplemented \ + -W clippy::all +``` + +Tool: **Cargo Geiger** (Unsafe code detection) +```yaml +- name: Unsafe Code Audit + run: | + cargo install cargo-geiger + cargo geiger --all-features --output-format GitHubMarkdown >> $GITHUB_STEP_SUMMARY +``` + +**Dynamic Application Security Testing (DAST)**: + +Tool: **OWASP ZAP** (API security testing) +```yaml +- name: DAST Scan + uses: zaproxy/action-api-scan@v0.4.0 + with: + target: 'https://staging.foxhunt.io' + rules_file_name: '.zap/rules.tsv' + fail_action: true +``` + +**DAST Test Coverage**: +- API authentication bypass +- SQL injection (gRPC parameters) +- XSS in error messages +- CSRF protection +- Rate limiting validation + +### Secret Detection + +**Tool**: TruffleHog (Git secret scanning) + +```yaml +- name: Scan for Secrets + uses: trufflesecurity/trufflehog@main + with: + path: ./ + base: ${{ github.event.repository.default_branch }} + head: HEAD + extra_args: --only-verified +``` + +**Prevented Secrets**: +- API keys (AWS, Databento, Alpaca) +- Database credentials +- JWT secrets +- Private keys (RSA, ED25519) +- OAuth tokens + +**Secret Management**: +- Development: Environment variables (`.env` gitignored) +- Staging/Production: HashiCorp Vault +- CI/CD: GitHub Secrets (encrypted at rest) + +--- + +## Performance Benchmarks + +### Automated Benchmark Execution + +**Criterion Benchmarks** (latency measurement): +```yaml +- name: Run Criterion Benchmarks + run: | + cargo bench --workspace --bench e2e_latency -- --save-baseline ${{ github.sha }} + cargo bench --workspace --bench order_matching -- --save-baseline ${{ github.sha }} +``` + +**Benchmark Suites**: +1. **E2E Latency** (`trading_engine/benches/e2e_latency.rs`): + - Order submission → execution latency + - Target: <100Ξs p99 + +2. **Order Matching** (`trading_engine/benches/order_matching.rs`): + - Matching engine throughput + - Target: 50K+ orders/sec + +3. **Lock-Free Queue** (`trading_engine/benches/lockfree_queue.rs`): + - Queue enqueue/dequeue latency + - Target: <1Ξs p50 + +4. **ML Inference** (`services/trading_service/tests/performance_benchmarks.rs`): + - MAMBA-2, DQN, PPO, TFT inference latency + - Target: <10ms p99 (GPU), <50ms (CPU) + +### Performance Regression Detection + +**Baseline Comparison**: +```yaml +- name: Compare Benchmarks + run: | + cargo bench --workspace -- --baseline ${{ github.event.before }} + + # Fail if >10% regression + python scripts/compare_benchmarks.py \ + --baseline ${{ github.event.before }} \ + --current ${{ github.sha }} \ + --threshold 0.10 +``` + +**Regression Thresholds**: +| Metric | Threshold | Action | +|--------|-----------|--------| +| **Latency** | +10% | Block merge, alert team | +| **Throughput** | -10% | Block merge, alert team | +| **Memory** | +20% | Warning, track in issue | +| **CPU** | +15% | Warning, track in issue | + +### Benchmark Reporting + +**Report Format**: +```markdown +## Benchmark Results + +| Benchmark | Baseline | Current | Change | Status | +|-----------|----------|---------|--------|--------| +| E2E Latency (p99) | 87Ξs | 92Ξs | +5.7% | ✅ PASS | +| Order Matching | 52K ops/s | 51K ops/s | -1.9% | ✅ PASS | +| ML Inference (p99) | 8.2ms | 7.9ms | -3.7% | ✅ PASS | + +**Detailed Results**: [View Full Report](https://github.com/foxhunt/foxhunt/actions/runs/123456) +``` + +**Storage**: +- Criterion output: `target/criterion/` +- Historical data: GitHub Actions artifacts (90 days) +- Long-term tracking: InfluxDB (permanent) + +**Visualization**: +```yaml +- name: Publish Benchmark Results + run: | + # Upload to InfluxDB + python scripts/upload_benchmarks.py \ + --file target/criterion/e2e_latency/base/estimates.json \ + --influx-url $INFLUX_URL \ + --token $INFLUX_TOKEN +``` + +--- + +## GitOps Integration + +### Infrastructure as Code (Terraform) + +**Directory Structure**: +``` +terraform/ +├── environments/ +│ ├── dev/ +│ │ ├── main.tf +│ │ ├── variables.tf +│ │ └── terraform.tfvars +│ ├── staging/ +│ └── production/ +├── modules/ +│ ├── eks/ # Kubernetes cluster +│ ├── rds/ # PostgreSQL database +│ ├── elasticache/ # Redis cache +│ └── s3/ # Object storage +└── global/ + ├── iam/ # IAM roles and policies + └── route53/ # DNS configuration +``` + +**Terraform Workflow**: +```yaml +- name: Terraform Plan + run: | + cd terraform/environments/${{ inputs.environment }} + terraform init + terraform plan -out=tfplan + +- name: Terraform Apply + if: github.event.inputs.approve == 'true' + run: | + cd terraform/environments/${{ inputs.environment }} + terraform apply tfplan +``` + +**State Management**: +- Backend: S3 + DynamoDB (state locking) +- Versioning: Enabled (rollback capability) +- Encryption: AES-256 at rest + +### Kubernetes Manifests + +**Directory Structure**: +``` +k8s/ +├── base/ # Kustomize base +│ ├── api-gateway/ +│ ├── trading-service/ +│ ├── backtesting-service/ +│ └── ml-training-service/ +├── overlays/ +│ ├── dev/ # Dev environment overrides +│ ├── staging/ # Staging overrides +│ └── production/ # Production overrides +└── helm/ + └── foxhunt/ # Helm chart + ├── Chart.yaml + ├── values.yaml + └── templates/ +``` + +**Kustomize Example**: +```yaml +# k8s/base/trading-service/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - deployment.yaml + - service.yaml + - servicemonitor.yaml + - hpa.yaml + +configMapGenerator: + - name: trading-service-config + files: + - config.toml + +secretGenerator: + - name: trading-service-secrets + envs: + - secrets.env +``` + +**Environment Overlays**: +```yaml +# k8s/overlays/production/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: + - ../../base/trading-service + +patchesStrategicMerge: + - replica-patch.yaml # Increase replicas to 4 + - resource-patch.yaml # Increase CPU/memory limits +``` + +### Configuration Management + +**ConfigMap Hot-Reload** (from CLAUDE.md): +```yaml +# k8s/base/trading-service/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + metadata: + annotations: + # Force pod restart on ConfigMap change + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + containers: + - name: trading-service + env: + - name: CONFIG_RELOAD_ENABLED + value: "true" + - name: CONFIG_RELOAD_INTERVAL_SECS + value: "60" + volumeMounts: + - name: config + mountPath: /etc/foxhunt + readOnly: true + volumes: + - name: config + configMap: + name: trading-service-config +``` + +**Secret Management (External Secrets Operator)**: +```yaml +# k8s/base/trading-service/external-secret.yaml +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: trading-service-secrets +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-backend + kind: SecretStore + target: + name: trading-service-secrets + creationPolicy: Owner + data: + - secretKey: database-url + remoteRef: + key: foxhunt/production/database + property: url + - secretKey: jwt-secret + remoteRef: + key: foxhunt/production/auth + property: jwt_secret +``` + +### GitOps Workflows + +**ArgoCD Application**: +```yaml +# argocd/trading-service.yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: trading-service + namespace: argocd +spec: + project: foxhunt + source: + repoURL: https://github.com/foxhunt/foxhunt + targetRevision: main + path: k8s/overlays/production + destination: + server: https://kubernetes.default.svc + namespace: foxhunt-prod + syncPolicy: + automated: + prune: true # Delete removed resources + selfHeal: true # Auto-sync on drift + allowEmpty: false + syncOptions: + - CreateNamespace=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m +``` + +**GitOps Flow**: +``` +1. Developer merges PR to main +2. GitHub Actions builds Docker image +3. GitHub Actions updates k8s/overlays/production/kustomization.yaml +4. ArgoCD detects change (poll interval: 3 minutes) +5. ArgoCD syncs new image to production +6. Kubernetes performs rolling update +7. ArgoCD reports sync status to Slack +``` + +**Deployment Notifications**: +```yaml +# argocd/notifications.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: argocd-notifications-cm +data: + trigger.on-deployed: | + - when: app.status.operationState.phase in ['Succeeded'] + send: [slack-deployment-success] + + template.slack-deployment-success: | + message: | + ✅ Application {{.app.metadata.name}} deployed successfully! + Version: {{.app.status.sync.revision}} + Environment: {{.app.spec.destination.namespace}} +``` + +--- + +## Workflow Examples + +See the following workflow files: +- `.github/workflows/test.yml` - Test pipeline +- `.github/workflows/build.yml` - Docker build pipeline +- `.github/workflows/deploy.yml` - Deployment pipeline +- `.gitlab-ci.yml` - GitLab CI example + +--- + +## Troubleshooting + +### Common Issues + +**1. Coverage Threshold Failure** +``` +Error: Coverage 58.3% below threshold 60% +``` + +**Solution**: +```bash +# Identify uncovered areas +cargo llvm-cov --html --output-dir coverage_report +open coverage_report/index.html + +# Add tests for uncovered code +cargo test -p --lib -- --nocapture +``` + +**2. Docker Build Failure** +``` +Error: failed to solve: executor failed running [/bin/sh -c cargo build --release] +``` + +**Solution**: +```bash +# Check Dockerfile syntax +docker build --no-cache -t test . + +# Verify dependencies +cargo check --workspace + +# Clear cache and rebuild +docker system prune -a +``` + +**3. Health Check Timeout** +``` +Error: Health check failed after 30 retries +``` + +**Solution**: +```bash +# Check pod logs +kubectl logs -n foxhunt-prod deployment/trading-service --tail=100 + +# Check pod status +kubectl describe pod -n foxhunt-prod -l app=trading-service + +# Manually test health check +kubectl exec -it -n foxhunt-prod -- grpc_health_probe -addr=:50051 +``` + +**4. Rollback Failed** +``` +Error: No previous revision available +``` + +**Solution**: +```bash +# Check rollout history +kubectl rollout history deployment/trading-service -n foxhunt-prod + +# Manually rollback to specific revision +kubectl rollout undo deployment/trading-service -n foxhunt-prod --to-revision=42 +``` + +### Pipeline Debugging + +**Enable Debug Logging**: +```yaml +# .github/workflows/test.yml +env: + RUST_LOG: debug + RUST_BACKTRACE: 1 + ACTIONS_RUNNER_DEBUG: true + ACTIONS_STEP_DEBUG: true +``` + +**SSH into GitHub Actions Runner**: +```yaml +- name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: failure() +``` + +**Local Pipeline Simulation**: +```bash +# Install act (GitHub Actions local runner) +brew install act + +# Run workflow locally +act -j test --secret-file .env.secrets + +# Debug specific step +act -j test --step "Run Tests" -v +``` + +### Contact and Support + +**Team Contacts**: +- DevOps Lead: @devops-lead (Slack) +- Platform Team: #platform-team (Slack) +- On-Call: @oncall (PagerDuty) + +**Documentation**: +- Architecture: `CLAUDE.md` +- Testing: `TESTING_PLAN.md` +- Deployment: `DEPLOYMENT_GUIDE.md` + +**Incident Response**: +- Critical: Page on-call engineer (PagerDuty) +- High: Alert in #incidents channel +- Medium: Create issue, assign to team +- Low: Track in backlog + +--- + +**Last Updated**: 2025-10-07 +**Maintained By**: Platform Team +**Review Cycle**: Quarterly