diff --git a/ml/src/hyperopt/optimizer.rs b/ml/src/hyperopt/optimizer.rs index 2afdf0427..a5af36cc7 100644 --- a/ml/src/hyperopt/optimizer.rs +++ b/ml/src/hyperopt/optimizer.rs @@ -317,12 +317,15 @@ impl ArgminOptimizer { let trials_used = trials.len(); let remaining_trials = self.max_trials.saturating_sub(trials_used); - // FIX: Each PSO iteration evaluates n_particles, so divide remaining budget by swarm size - let max_iters_by_budget = remaining_trials.saturating_div(self.n_particles); + // CRITICAL FIX: Hyperopt adapters execute SEQUENTIALLY (mutex-locked models) + // Each iteration evaluates exactly 1 trial, not n_particles trials + // Therefore: max_iters = remaining_trials (no division) + let max_iters_by_budget = remaining_trials; + let max_iters = max_iters_by_budget.min(self.max_iters_per_restart); - info!("PSO Budget: {} iterations ({} remaining trials ÷ {} particles)", - max_iters, remaining_trials, self.n_particles); + info!("PSO Budget: {} iterations ({} remaining trials, sequential execution)", + max_iters, remaining_trials); if max_iters > 0 { // Create bounds vectors for ParticleSwarm diff --git a/scripts/build_docker_images.sh b/scripts/build_docker_images.sh index 40a41a5d5..e7de64846 100755 --- a/scripts/build_docker_images.sh +++ b/scripts/build_docker_images.sh @@ -279,7 +279,8 @@ validate_binaries() { local missing_binaries=() for binary in "${EXPECTED_BINARIES[@]}"; do - if ! docker run --rm "$image_tag" test -f "/usr/local/bin/$binary"; then + # Override entrypoint to bypass self-termination wrapper during validation + if ! docker run --rm --entrypoint="" "$image_tag" test -f "/usr/local/bin/$binary"; then print_error "Binary not found: /usr/local/bin/$binary" missing_binaries+=("$binary") else @@ -294,7 +295,7 @@ validate_binaries() { # Check CUDA libraries print_info "Checking CUDA libraries..." - if ! docker run --rm "$image_tag" test -d /usr/local/cuda; then + if ! docker run --rm --entrypoint="" "$image_tag" test -d /usr/local/cuda; then print_error "CUDA directory not found in image" return 1 fi @@ -302,7 +303,7 @@ validate_binaries() { # Check cuDNN libraries print_info "Checking cuDNN libraries..." - if ! docker run --rm "$image_tag" sh -c 'ldconfig -p | grep -q libcudnn'; then + if ! docker run --rm --entrypoint="" "$image_tag" sh -c 'ldconfig -p | grep -q libcudnn'; then print_warning "cuDNN library not found (may be OK if using base image)" else print_success "cuDNN library present" @@ -311,7 +312,7 @@ validate_binaries() { # Check GLIBC version (should be 2.35 for Ubuntu 22.04) print_info "Checking GLIBC version..." local glibc_version - glibc_version=$(docker run --rm "$image_tag" ldd --version | head -n1 | grep -oP '\d+\.\d+' | head -n1) + glibc_version=$(docker run --rm --entrypoint="" "$image_tag" ldd --version | head -n1 | grep -oP '\d+\.\d+' | head -n1) print_success "GLIBC version: $glibc_version" print_success "Image validation complete" diff --git a/scripts/validate_gitlab_cicd.sh b/scripts/validate_gitlab_cicd.sh new file mode 100755 index 000000000..7a715dd0c --- /dev/null +++ b/scripts/validate_gitlab_cicd.sh @@ -0,0 +1,387 @@ +#!/bin/bash +# ============================================================================= +# GitLab CI/CD Configuration Validator +# ============================================================================= +# Purpose: Validate .gitlab-ci.yml configuration and check prerequisites +# Usage: ./scripts/validate_gitlab_cicd.sh +# ============================================================================= + +set -euo pipefail + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' + +print_header() { + echo -e "\n${CYAN}==============================================================================${NC}" + echo -e "${CYAN}$1${NC}" + echo -e "${CYAN}==============================================================================${NC}\n" +} + +print_success() { + echo -e "${GREEN}[✓]${NC} $1" +} + +print_error() { + echo -e "${RED}[✗]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[!]${NC} $1" +} + +print_info() { + echo -e "${BLUE}[i]${NC} $1" +} + +# ============================================================================= +# Validation Functions +# ============================================================================= + +validate_gitlab_ci_file() { + print_header "1. Validating .gitlab-ci.yml Syntax" + + if [ ! -f .gitlab-ci.yml ]; then + print_error ".gitlab-ci.yml not found in current directory" + return 1 + fi + print_success ".gitlab-ci.yml file exists" + + # Check if gitlab-ci-local is installed for local validation + if command -v gitlab-ci-local &> /dev/null; then + print_info "Validating syntax with gitlab-ci-local..." + if gitlab-ci-local --list 2>&1 | grep -q "build:docker"; then + print_success "Syntax validation passed" + else + print_error "Syntax validation failed" + return 1 + fi + else + print_warning "gitlab-ci-local not installed (optional, for local validation)" + print_info "Install: npm install -g gitlab-ci-local" + fi + + # Check required stages + print_info "Checking required stages..." + for stage in "build" "test" "deploy"; do + if grep -q "\- $stage" .gitlab-ci.yml; then + print_success "Stage '$stage' defined" + else + print_error "Stage '$stage' missing" + return 1 + fi + done + + return 0 +} + +validate_docker_config() { + print_header "2. Validating Docker Configuration" + + # Check Docker availability + if ! command -v docker &> /dev/null; then + print_error "Docker is not installed" + return 1 + fi + print_success "Docker installed: $(docker --version)" + + # Check Docker daemon + if ! docker info &> /dev/null; then + print_error "Docker daemon is not running" + return 1 + fi + print_success "Docker daemon running" + + # Check BuildKit support + if docker buildx version &> /dev/null; then + print_success "BuildKit available: $(docker buildx version | head -n1)" + else + print_warning "BuildKit not available (will use legacy builder)" + fi + + # Check Dockerfile + if [ -f Dockerfile.foxhunt-build ]; then + print_success "Dockerfile.foxhunt-build exists" + else + print_error "Dockerfile.foxhunt-build not found" + return 1 + fi + + return 0 +} + +validate_docker_hub_config() { + print_header "3. Validating Docker Hub Configuration" + + # Check if Docker Hub credentials are set (for local testing) + if [ -n "${DOCKER_HUB_USERNAME:-}" ] && [ -n "${DOCKER_HUB_PASSWORD:-}" ]; then + print_success "Docker Hub credentials found in environment" + + # Test Docker Hub login + print_info "Testing Docker Hub authentication..." + if echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin docker.io &> /dev/null; then + print_success "Docker Hub authentication successful" + else + print_error "Docker Hub authentication failed" + return 1 + fi + else + print_warning "Docker Hub credentials not set in environment (OK for GitLab CI/CD)" + print_info "For GitLab CI/CD, set these in: Settings → CI/CD → Variables" + print_info " - DOCKER_HUB_USERNAME (your Docker Hub username)" + print_info " - DOCKER_HUB_PASSWORD (Docker Hub access token, NOT password)" + fi + + # Check Docker Hub repository + print_info "Checking Docker Hub repository configuration..." + REPO=$(grep "DOCKER_HUB_REPO:" .gitlab-ci.yml | awk '{print $2}') + if [ -n "$REPO" ]; then + print_success "Docker Hub repository: $REPO" + else + print_error "DOCKER_HUB_REPO not defined in .gitlab-ci.yml" + return 1 + fi + + return 0 +} + +validate_build_artifacts() { + print_header "4. Validating Build Artifacts" + + # Check if hyperopt binaries exist + print_info "Checking for hyperopt binaries..." + local binaries=( + "hyperopt_mamba2_demo" + "hyperopt_dqn_demo" + "hyperopt_ppo_demo" + "hyperopt_tft_demo" + ) + + local missing_count=0 + for binary in "${binaries[@]}"; do + if [ -f "target/release/$binary" ]; then + print_success "$binary exists ($(du -h "target/release/$binary" | cut -f1))" + else + print_warning "$binary not found (will be built in CI/CD)" + ((missing_count++)) + fi + done + + if [ $missing_count -eq ${#binaries[@]} ]; then + print_warning "No hyperopt binaries found locally (expected for clean checkout)" + print_info "Binaries will be built during Docker image build" + fi + + return 0 +} + +validate_gitlab_runner() { + print_header "5. Validating GitLab Runner Configuration" + + # Check if gitlab-runner is installed + if command -v gitlab-runner &> /dev/null; then + print_success "GitLab Runner installed: $(gitlab-runner --version | head -n1)" + + # Check registered runners + print_info "Checking registered runners..." + if gitlab-runner list 2>&1 | grep -q "Executor"; then + print_success "GitLab Runner registered" + gitlab-runner list 2>&1 | grep "Executor" | while read -r line; do + print_info " $line" + done + else + print_warning "No GitLab Runners registered locally" + fi + else + print_warning "GitLab Runner not installed locally (OK if using GitLab.com shared runners)" + print_info "For self-hosted runner: https://docs.gitlab.com/runner/install/" + fi + + # Check .gitlab-ci.yml for runner tags + print_info "Checking runner tags in .gitlab-ci.yml..." + if grep -q "tags:" .gitlab-ci.yml; then + grep "tags:" .gitlab-ci.yml | while read -r line; do + print_info " $line" + done + print_success "Runner tags configured" + else + print_warning "No runner tags specified (will use any available runner)" + fi + + return 0 +} + +validate_ci_variables() { + print_header "6. Validating CI/CD Variables" + + print_info "Required GitLab CI/CD Variables (set in GitLab UI):" + print_info " - DOCKER_HUB_USERNAME (required)" + print_info " - DOCKER_HUB_PASSWORD (required, use access token)" + + print_info "" + print_info "Optional CI/CD Variables:" + print_info " - SLACK_WEBHOOK_URL (for notifications)" + print_info " - DISCORD_WEBHOOK_URL (for notifications)" + + # Check if variables are referenced in .gitlab-ci.yml + if grep -q "DOCKER_HUB_USERNAME" .gitlab-ci.yml && grep -q "DOCKER_HUB_PASSWORD" .gitlab-ci.yml; then + print_success "Required variables referenced in .gitlab-ci.yml" + else + print_error "Required variables not referenced in .gitlab-ci.yml" + return 1 + fi + + return 0 +} + +validate_deployment_config() { + print_header "7. Validating Deployment Configuration" + + # Check deployment stages + if grep -q "deploy:runpod:" .gitlab-ci.yml; then + print_success "RunPod deployment stage configured" + else + print_warning "RunPod deployment stage not found" + fi + + if grep -q "deploy:runpod-staging:" .gitlab-ci.yml; then + print_success "Staging deployment stage configured" + else + print_warning "Staging deployment stage not found" + fi + + # Check manual approval for production + if grep -q "when: manual" .gitlab-ci.yml; then + print_success "Manual approval configured for production deployment" + else + print_warning "No manual approval for production (all deployments automatic)" + fi + + # Check environment configuration + if grep -q "environment:" .gitlab-ci.yml; then + print_success "Environment configuration present" + else + print_warning "No environment configuration" + fi + + return 0 +} + +simulate_local_build() { + print_header "8. Simulating Local Build (Optional)" + + read -p "Run local Docker build simulation? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + print_info "Skipping local build simulation" + return 0 + fi + + print_info "Building Docker image locally (this may take 5-10 minutes)..." + + export DOCKER_BUILDKIT=1 + export GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + + if docker build \ + -f Dockerfile.foxhunt-build \ + -t jgrusewski/foxhunt:test-local \ + --build-arg GIT_COMMIT="$GIT_COMMIT" \ + --build-arg BUILD_DATE="$BUILD_DATE" \ + . 2>&1 | tail -20; then + print_success "Local Docker build completed" + + # Validate binaries in image + print_info "Validating binaries in image..." + local binaries=( + "hyperopt_mamba2_demo" + "hyperopt_dqn_demo" + "hyperopt_ppo_demo" + "hyperopt_tft_demo" + ) + + for binary in "${binaries[@]}"; do + if docker run --rm --entrypoint="" jgrusewski/foxhunt:test-local test -f "/usr/local/bin/$binary"; then + print_success "$binary exists in image" + else + print_error "$binary missing in image" + return 1 + fi + done + + # Show image size + local size=$(docker image inspect jgrusewski/foxhunt:test-local --format='{{.Size}}' | awk '{ sum=$1; hum[1024**3]="GB"; hum[1024**2]="MB"; hum[1024]="KB"; for (x=1024**3; x>=1024; x/=1024) { if (sum>=x) { printf "%.2f %s", sum/x, hum[x]; break } }}') + print_info "Image size: $size" + + else + print_error "Local Docker build failed" + return 1 + fi + + return 0 +} + +# ============================================================================= +# Main Execution +# ============================================================================= + +main() { + print_header "GitLab CI/CD Configuration Validator" + + print_info "Current directory: $(pwd)" + print_info "Git repository: $(git remote get-url origin 2>/dev/null || echo 'Not a git repository')" + print_info "Git branch: $(git branch --show-current 2>/dev/null || echo 'Unknown')" + print_info "Git commit: $(git rev-parse --short HEAD 2>/dev/null || echo 'Unknown')" + + local exit_code=0 + + # Run all validations + validate_gitlab_ci_file || exit_code=1 + validate_docker_config || exit_code=1 + validate_docker_hub_config || exit_code=1 + validate_build_artifacts || exit_code=1 + validate_gitlab_runner || exit_code=1 + validate_ci_variables || exit_code=1 + validate_deployment_config || exit_code=1 + + # Optional: simulate local build + if [ $exit_code -eq 0 ]; then + simulate_local_build || exit_code=1 + fi + + # Final summary + print_header "Validation Summary" + + if [ $exit_code -eq 0 ]; then + print_success "All validations passed!" + print_info "" + print_info "Next steps:" + print_info " 1. Set GitLab CI/CD variables in GitLab UI (if not already set)" + print_info " Settings → CI/CD → Variables" + print_info " - DOCKER_HUB_USERNAME" + print_info " - DOCKER_HUB_PASSWORD (Docker Hub access token)" + print_info "" + print_info " 2. Push to main branch to trigger pipeline:" + print_info " git push origin main" + print_info "" + print_info " 3. Monitor pipeline in GitLab:" + print_info " Project → CI/CD → Pipelines" + print_info "" + print_info " 4. Manual approval required for production deployment" + print_info "" + else + print_error "Some validations failed" + print_info "" + print_info "Please fix the issues above before enabling GitLab CI/CD" + print_info "" + fi + + exit $exit_code +} + +# Run main function +main "$@"