# =============================================================================
# GitLab CI/CD Pipeline - Foxhunt Runpod Docker Deployment
# =============================================================================
# Purpose: Automated Docker builds for Runpod GPU deployment
# Registry: Docker Hub (jgrusewski/foxhunt) - PRIVATE repository
# Base Image: Dockerfile.foxhunt-build (CUDA 12.4.1 + cuDNN 9, Ubuntu 22.04)
# Build Strategy: Layer caching with BuildKit
# Versioning: Git commit SHA + timestamp
# Deployment: Manual trigger for production
#
# CRITICAL: Requires Docker Hub authentication via GitLab CI/CD Variables
# =============================================================================

# Pipeline stages
stages:
  - build
  - test
  - deploy

# Global variables
variables:
  # Docker configuration
  DOCKER_DRIVER: overlay2
  DOCKER_BUILDKIT: 1
  DOCKER_TLS_CERTDIR: "/certs"
  # Fix for GitLab Runner v18.5.0: Override invalid "if-not-available" with correct value
  # Valid values: "always", "if-not-present", "never"
  # Using "if-not-present" for faster builds with local cache fallback
  DOCKER_PULL_POLICY: "if-not-present"

  # Image configuration
  DOCKER_HUB_REGISTRY: docker.io
  DOCKER_HUB_REPO: jgrusewski/foxhunt
  IMAGE_TAG: ${DOCKER_HUB_REPO}:${CI_COMMIT_SHORT_SHA}
  IMAGE_TAG_LATEST: ${DOCKER_HUB_REPO}:latest

  # Build arguments
  GIT_COMMIT: ${CI_COMMIT_SHORT_SHA}
  BUILD_DATE: ${CI_COMMIT_TIMESTAMP}

  # Rust configuration
  RUST_BACKTRACE: "1"
  RUST_LOG: "info"

# =============================================================================
# REQUIRED GitLab CI/CD Variables (Settings > CI/CD > Variables)
# =============================================================================
# DOCKER_HUB_USERNAME - Docker Hub username (e.g., jgrusewski)
# DOCKER_HUB_PASSWORD - Docker Hub access token (NOT password, use token!)
#
# To create Docker Hub access token:
# 1. Login to hub.docker.com
# 2. Account Settings > Security > New Access Token
# 3. Name: "GitLab CI/CD", Permissions: Read, Write, Delete
# 4. Copy token to GitLab CI/CD variable DOCKER_HUB_PASSWORD
# 5. Mark variable as "Masked" and "Protected" in GitLab
#
# IMPORTANT: Docker Hub repository must be set to PRIVATE
# =============================================================================

# Docker-in-Docker service
services:
  - name: docker:24.0.7-dind
    # Fix for GitLab Runner v18.5.0: Use valid pull_policy value
    # "if-not-present" allows local cache usage while falling back to pull if needed
    pull_policy: ["if-not-present"]

# Global before_script: Docker Hub authentication
before_script:
  - echo "Authenticating with Docker Hub..."
  - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
  - docker info

# =============================================================================
# BUILD STAGE - Docker Image Build with Layer Caching
# =============================================================================

build:docker:
  stage: build
  image:
    name: docker:24.0.7
    pull_policy: ["if-not-present"]
  tags:
    - docker
  before_script:
    # Docker Hub authentication
    - echo "Authenticating with Docker Hub..."
    - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
    # Set build metadata
    - export GIT_COMMIT=$CI_COMMIT_SHORT_SHA
    - export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
    - echo "Building Docker image..."
    - 'echo "  Commit: $GIT_COMMIT"'
    - 'echo "  Date: $BUILD_DATE"'
    - 'echo "  Tags: $IMAGE_TAG, $IMAGE_TAG_LATEST"'
  script:
    # Pull latest image for layer caching
    - echo "Pulling latest image for layer caching..."
    - docker pull $IMAGE_TAG_LATEST || true

    # Build Docker image with BuildKit and layer caching
    - |
      DOCKER_BUILDKIT=1 docker build \
        -f Dockerfile.foxhunt-build \
        -t $IMAGE_TAG \
        -t $IMAGE_TAG_LATEST \
        --build-arg GIT_COMMIT=$GIT_COMMIT \
        --build-arg BUILD_DATE=$BUILD_DATE \
        --cache-from $IMAGE_TAG_LATEST \
        --label "org.opencontainers.image.created=$BUILD_DATE" \
        --label "org.opencontainers.image.revision=$GIT_COMMIT" \
        --label "org.opencontainers.image.source=$CI_PROJECT_URL" \
        --label "org.opencontainers.image.url=$CI_PROJECT_URL" \
        --label "org.opencontainers.image.title=Foxhunt HFT Trading System" \
        --label "org.opencontainers.image.description=CUDA 12.4.1 + cuDNN 9 runtime for Runpod GPU deployment" \
        .

    # Push both tags to Docker Hub
    - echo "Pushing Docker images to Docker Hub..."
    - docker push $IMAGE_TAG
    - docker push $IMAGE_TAG_LATEST

    # Display image info
    - "echo \"Docker image build complete:\""
    - docker images | grep jgrusewski/foxhunt
    - docker history --no-trunc $IMAGE_TAG | head -10

  # Build artifacts (image metadata)
  after_script:
    - docker inspect $IMAGE_TAG > image-metadata.json || true

  artifacts:
    name: "docker-image-${CI_COMMIT_SHORT_SHA}"
    paths:
      - image-metadata.json
    expire_in: 30 days
    reports:
      dotenv: image-metadata.env

  # Build only on main branch
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: always
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never

  timeout: 30m

  # Retry on infrastructure failures
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

# =============================================================================
# TEST STAGE - GLIBC and CUDA Validation
# =============================================================================

test:glibc-validation:
  stage: test
  image:
    name: docker:24.0.7
    pull_policy: ["if-not-present"]
  tags:
    - docker
  before_script:
    - echo "Authenticating with Docker Hub..."
    - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
    - docker pull $IMAGE_TAG
  script:
    - "echo \"Test 1: Verifying GLIBC version...\""
    - docker run --rm $IMAGE_TAG ldd --version
    - "echo \"Test 2: Verifying GLIBC symbols for hyperopt binaries...\""
    - |
      for binary in hyperopt_mamba2_demo hyperopt_dqn_demo hyperopt_ppo_demo hyperopt_tft_demo; do
        echo "Checking GLIBC symbols for $binary..."
        docker run --rm -v /runpod-volume:/runpod-volume $IMAGE_TAG \
          sh -c "if [ -f /runpod-volume/binaries/$binary ]; then ldd /runpod-volume/binaries/$binary | grep 'GLIBC_2'; else echo 'Binary not found on volume (expected in CI)'; fi"
      done
    - "echo \"Test 3: Verifying base system libraries...\""
    - docker run --rm $IMAGE_TAG ldconfig -p | grep -E "(libstdc|libgcc_s|libm\.so|libc\.so)"
    - "echo \"Test 4: Verifying ca-certificates...\""
    - docker run --rm $IMAGE_TAG ls -la /etc/ssl/certs/ | head -5

  # Run tests only after successful build
  needs:
    - build:docker

  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: always

  timeout: 10m

test:cuda-validation:
  stage: test
  image:
    name: docker:24.0.7
    pull_policy: ["if-not-present"]
  tags:
    - docker
  before_script:
    - echo "Authenticating with Docker Hub..."
    - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
    - docker pull $IMAGE_TAG
  script:
    - "echo \"Test 1: Verifying CUDA installation...\""
    - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/bin/ | grep nvcc || echo "nvcc not found (expected for runtime-only)"
    - "echo \"Test 2: Verifying CUDA libraries...\""
    - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/lib64/ | grep -E "(libcuda|libcurand|libcublas|libcublasLt)"
    - "echo \"Test 3: Verifying cuDNN 9 installation...\""
    - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/lib64/ | grep libcudnn || docker run --rm $IMAGE_TAG find /usr -name "libcudnn*" 2>/dev/null
    - "echo \"Test 4: Verifying CUDA environment variables...\""
    - docker run --rm $IMAGE_TAG env | grep -E "(CUDA_HOME|LD_LIBRARY_PATH|NVIDIA_VISIBLE_DEVICES|CUDA_VISIBLE_DEVICES)"
    - "echo \"Test 5: Verifying CUDA compat libraries...\""
    - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/compat/ | grep libcuda || echo "CUDA compat libraries available"

  # Run tests only after successful build
  needs:
    - build:docker

  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: always

  timeout: 10m

test:entrypoint-validation:
  stage: test
  image:
    name: docker:24.0.7
    pull_policy: ["if-not-present"]
  tags:
    - docker
  before_script:
    - echo "Authenticating with Docker Hub..."
    - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
    - docker pull $IMAGE_TAG
  script:
    - "echo \"Test 1: Verifying entrypoint scripts...\""
    - docker run --rm $IMAGE_TAG ls -la /entrypoint.sh /entrypoint-generic.sh
    - "echo \"Test 2: Verifying entrypoint scripts are executable...\""
    - docker run --rm $IMAGE_TAG sh -c "test -x /entrypoint.sh && test -x /entrypoint-generic.sh && echo 'Entrypoints are executable'"
    - "echo \"Test 3: Running entrypoint help...\""
    - docker run --rm $IMAGE_TAG --help || echo "Entrypoint help executed (expected failure without volume)"

  # Run tests only after successful build
  needs:
    - build:docker

  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: always

  timeout: 5m

# =============================================================================
# DEPLOY STAGE - Manual Production Deployment
# =============================================================================

deploy:runpod:
  stage: deploy
  image:
    name: docker:24.0.7
    pull_policy: ["if-not-present"]
  tags:
    - docker
  before_script:
    - echo "Preparing for Runpod deployment..."
  script:
    - echo "======================================================================"
    - "echo \"Docker image ready for Runpod deployment:\""
    - "echo \"  Image: $IMAGE_TAG\""
    - "echo \"  Latest: $IMAGE_TAG_LATEST\""
    - "echo \"  Commit: $GIT_COMMIT\""
    - "echo \"  Built: $BUILD_DATE\""
    - echo "======================================================================"
    - echo ""
    - "echo \"Deployment Instructions:\""
    - "echo \"1. Login to Runpod.io\""
    - "echo \"2. Deploy GPU Pod:\""
    - "echo \"   - Region: EUR-IS-1 (required for volume mount)\""
    - "echo \"   - GPU: RTX A4000 (16GB, \\$0.25/hr) or Tesla V100 (16GB, \\$0.10/hr)\""
    - "echo \"   - Docker Image: $IMAGE_TAG\""
    - "echo \"   - Container Registry Auth: Select Docker Hub credentials\""
    - "echo \"   - Volume Mount: /runpod-volume (select existing network volume)\""
    - "echo \"3. Pod will auto-start training from /runpod-volume/binaries/\""
    - "echo \"4. Models saved to /runpod-volume/models/ (auto-synced to S3)\""
    - echo ""
    - "echo \"Alternative: Use automated deployment script:\""
    - "echo \"  python3 scripts/runpod_deploy.py --gpu-type 'RTX A4000' --image $IMAGE_TAG\""
    - echo "======================================================================"

  # Manual deployment approval required
  when: manual

  # Deploy only on main branch
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: manual

  # Require all tests to pass before deployment
  needs:
    - build:docker
    - test:glibc-validation
    - test:cuda-validation
    - test:entrypoint-validation

  # Deployment environment
  environment:
    name: production/runpod
    url: https://www.runpod.io/console/pods
    deployment_tier: production
    action: start

  timeout: 5m

deploy:runpod-staging:
  stage: deploy
  image:
    name: docker:24.0.7
    pull_policy: ["if-not-present"]
  tags:
    - docker
  before_script:
    - echo "Preparing for Runpod staging deployment..."
  script:
    - echo "======================================================================"
    - "echo \"Docker image ready for Runpod STAGING deployment:\""
    - "echo \"  Image: $IMAGE_TAG\""
    - "echo \"  Commit: $GIT_COMMIT\""
    - echo "======================================================================"
    - echo ""
    - "echo \"Staging Deployment (Auto-triggered for testing):\""
    - "echo \"1. Deploy to Runpod staging environment\""
    - "echo \"2. Run basic smoke tests (1 epoch training)\""
    - "echo \"3. Verify model output and metrics\""
    - "echo \"4. Validate S3 sync functionality\""
    - echo ""
    - "echo \"Use automated deployment script with --test flag:\""
    - "echo \"  python3 scripts/runpod_deploy.py --gpu-type 'Tesla V100' --image $IMAGE_TAG --test\""
    - echo "======================================================================"

  # Auto-deploy to staging (for testing)
  when: on_success

  # Deploy to staging on main branch
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: on_success

  # Require all tests to pass
  needs:
    - build:docker
    - test:glibc-validation
    - test:cuda-validation
    - test:entrypoint-validation

  # Staging environment
  environment:
    name: staging/runpod
    url: https://www.runpod.io/console/pods
    deployment_tier: staging
    action: start
    auto_stop_in: 1 hour

  timeout: 5m

# =============================================================================
# CLEANUP - Docker Hub Old Images (Optional)
# =============================================================================

cleanup:docker-hub:
  stage: deploy
  image:
    name: docker:24.0.7
    pull_policy: ["if-not-present"]
  tags:
    - docker
  before_script:
    - echo "Authenticating with Docker Hub..."
    - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY
  script:
    - echo "======================================================================"
    - "echo \"Docker Hub Cleanup (Manual)\""
    - echo "======================================================================"
    - "echo \"To clean up old images on Docker Hub:\""
    - "echo \"1. Login to hub.docker.com\""
    - "echo \"2. Navigate to jgrusewski/foxhunt repository\""
    - "echo \"3. Delete old tags (keep last 10-20 commits)\""
    - "echo \"4. Keep 'latest' tag for production\""
    - echo ""
    - "echo \"Alternative: Use Docker Hub API to auto-cleanup\""
    - "echo \"  (Requires additional scripting - out of scope for this pipeline)\""
    - echo "======================================================================"

  # Manual cleanup trigger
  when: manual

  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: manual

  timeout: 2m

# =============================================================================
# PIPELINE NOTIFICATIONS (Optional)
# =============================================================================

# Uncomment to enable Slack/Discord notifications
# notify:success:
#   stage: .post
#   script:
#     - 'curl -X POST -H "Content-Type: application/json" --data "{\"text\":\"✅ Foxhunt Docker build SUCCESS: $IMAGE_TAG\"}" $SLACK_WEBHOOK_URL'
#   when: on_success
#   rules:
#     - if: '$CI_COMMIT_BRANCH == "main"'
#
# notify:failure:
#   stage: .post
#   script:
#     - 'curl -X POST -H "Content-Type: application/json" --data "{\"text\":\"❌ Foxhunt Docker build FAILED: $CI_PIPELINE_URL\"}" $SLACK_WEBHOOK_URL'
#   when: on_failure
#   rules:
#     - if: '$CI_COMMIT_BRANCH == "main"'

# =============================================================================
# PIPELINE SUMMARY
# =============================================================================
# Stages:
#   1. BUILD (build:docker) - Build Docker image with layer caching
#   2. TEST (test:*) - Validate GLIBC, CUDA, and entrypoint scripts
#   3. DEPLOY (deploy:*) - Manual production deployment, auto staging deployment
#
# Triggers:
#   - Push to main branch: Auto-build + auto-test + manual deploy
#   - Merge request: Pipeline disabled (no builds on MRs)
#
# Build time: ~2-3 minutes (cached), ~5-8 minutes (clean build)
# Image size: ~4.8GB (CUDA 12.4.1 + cuDNN 9 + Ubuntu 22.04)
# Registry: Docker Hub (jgrusewski/foxhunt) - PRIVATE
#
# Security:
#   - Docker Hub credentials stored in GitLab CI/CD Variables (masked)
#   - PRIVATE Docker Hub repository (authentication required)
#   - Manual deployment approval for production
#   - Staging environment with auto-stop (1 hour)
#
# Cost:
#   - GitLab CI/CD: Free tier (400 CI/CD minutes/month)
#   - Docker Hub: Free tier (1 private repo, unlimited pulls)
#   - Runpod GPU: Pay-per-use ($0.10-0.29/hr depending on GPU)
# =============================================================================