fix(hyperopt): Fix PSO budget calculation for sequential execution
PROBLEM: - PPO/DQN/TFT/MAMBA2 hyperopt stopped at 23/50 trials (46% completion) - Root cause: Optimizer incorrectly divided remaining trials by n_particles - Sequential execution (mutex-locked models) means 1 eval per iteration, not n_particles FIX: - Remove division by n_particles in PSO budget calculation - Each iteration now evaluates exactly 1 trial (sequential execution) - Expected: 3 initial + 47 PSO iterations = 50 trials total ✅ IMPACT: - All hyperopt runs will now complete full trial count - No performance impact (same execution pattern) - Fixes PPO, DQN, TFT, and MAMBA2 hyperopt early termination Files modified: - ml/src/hyperopt/optimizer.rs: Fix budget calculation (lines 320-328) - scripts/validate_gitlab_cicd.sh: Add CI/CD configuration validator - scripts/build_docker_images.sh: Fix entrypoint override for validation Testing: - Code compiles successfully (2m 27s build time) - GitLab CI/CD validator passes all checks - Will be validated in CI/CD pipeline 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -279,7 +279,8 @@ validate_binaries() {
|
||||
local missing_binaries=()
|
||||
|
||||
for binary in "${EXPECTED_BINARIES[@]}"; do
|
||||
if ! docker run --rm "$image_tag" test -f "/usr/local/bin/$binary"; then
|
||||
# Override entrypoint to bypass self-termination wrapper during validation
|
||||
if ! docker run --rm --entrypoint="" "$image_tag" test -f "/usr/local/bin/$binary"; then
|
||||
print_error "Binary not found: /usr/local/bin/$binary"
|
||||
missing_binaries+=("$binary")
|
||||
else
|
||||
@@ -294,7 +295,7 @@ validate_binaries() {
|
||||
|
||||
# Check CUDA libraries
|
||||
print_info "Checking CUDA libraries..."
|
||||
if ! docker run --rm "$image_tag" test -d /usr/local/cuda; then
|
||||
if ! docker run --rm --entrypoint="" "$image_tag" test -d /usr/local/cuda; then
|
||||
print_error "CUDA directory not found in image"
|
||||
return 1
|
||||
fi
|
||||
@@ -302,7 +303,7 @@ validate_binaries() {
|
||||
|
||||
# Check cuDNN libraries
|
||||
print_info "Checking cuDNN libraries..."
|
||||
if ! docker run --rm "$image_tag" sh -c 'ldconfig -p | grep -q libcudnn'; then
|
||||
if ! docker run --rm --entrypoint="" "$image_tag" sh -c 'ldconfig -p | grep -q libcudnn'; then
|
||||
print_warning "cuDNN library not found (may be OK if using base image)"
|
||||
else
|
||||
print_success "cuDNN library present"
|
||||
@@ -311,7 +312,7 @@ validate_binaries() {
|
||||
# Check GLIBC version (should be 2.35 for Ubuntu 22.04)
|
||||
print_info "Checking GLIBC version..."
|
||||
local glibc_version
|
||||
glibc_version=$(docker run --rm "$image_tag" ldd --version | head -n1 | grep -oP '\d+\.\d+' | head -n1)
|
||||
glibc_version=$(docker run --rm --entrypoint="" "$image_tag" ldd --version | head -n1 | grep -oP '\d+\.\d+' | head -n1)
|
||||
print_success "GLIBC version: $glibc_version"
|
||||
|
||||
print_success "Image validation complete"
|
||||
|
||||
387
scripts/validate_gitlab_cicd.sh
Executable file
387
scripts/validate_gitlab_cicd.sh
Executable file
@@ -0,0 +1,387 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# GitLab CI/CD Configuration Validator
|
||||
# =============================================================================
|
||||
# Purpose: Validate .gitlab-ci.yml configuration and check prerequisites
|
||||
# Usage: ./scripts/validate_gitlab_cicd.sh
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
|
||||
print_header() {
|
||||
echo -e "\n${CYAN}==============================================================================${NC}"
|
||||
echo -e "${CYAN}$1${NC}"
|
||||
echo -e "${CYAN}==============================================================================${NC}\n"
|
||||
}
|
||||
|
||||
print_success() {
|
||||
echo -e "${GREEN}[✓]${NC} $1"
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo -e "${RED}[✗]${NC} $1"
|
||||
}
|
||||
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}[!]${NC} $1"
|
||||
}
|
||||
|
||||
print_info() {
|
||||
echo -e "${BLUE}[i]${NC} $1"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Validation Functions
|
||||
# =============================================================================
|
||||
|
||||
validate_gitlab_ci_file() {
|
||||
print_header "1. Validating .gitlab-ci.yml Syntax"
|
||||
|
||||
if [ ! -f .gitlab-ci.yml ]; then
|
||||
print_error ".gitlab-ci.yml not found in current directory"
|
||||
return 1
|
||||
fi
|
||||
print_success ".gitlab-ci.yml file exists"
|
||||
|
||||
# Check if gitlab-ci-local is installed for local validation
|
||||
if command -v gitlab-ci-local &> /dev/null; then
|
||||
print_info "Validating syntax with gitlab-ci-local..."
|
||||
if gitlab-ci-local --list 2>&1 | grep -q "build:docker"; then
|
||||
print_success "Syntax validation passed"
|
||||
else
|
||||
print_error "Syntax validation failed"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
print_warning "gitlab-ci-local not installed (optional, for local validation)"
|
||||
print_info "Install: npm install -g gitlab-ci-local"
|
||||
fi
|
||||
|
||||
# Check required stages
|
||||
print_info "Checking required stages..."
|
||||
for stage in "build" "test" "deploy"; do
|
||||
if grep -q "\- $stage" .gitlab-ci.yml; then
|
||||
print_success "Stage '$stage' defined"
|
||||
else
|
||||
print_error "Stage '$stage' missing"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_docker_config() {
|
||||
print_header "2. Validating Docker Configuration"
|
||||
|
||||
# Check Docker availability
|
||||
if ! command -v docker &> /dev/null; then
|
||||
print_error "Docker is not installed"
|
||||
return 1
|
||||
fi
|
||||
print_success "Docker installed: $(docker --version)"
|
||||
|
||||
# Check Docker daemon
|
||||
if ! docker info &> /dev/null; then
|
||||
print_error "Docker daemon is not running"
|
||||
return 1
|
||||
fi
|
||||
print_success "Docker daemon running"
|
||||
|
||||
# Check BuildKit support
|
||||
if docker buildx version &> /dev/null; then
|
||||
print_success "BuildKit available: $(docker buildx version | head -n1)"
|
||||
else
|
||||
print_warning "BuildKit not available (will use legacy builder)"
|
||||
fi
|
||||
|
||||
# Check Dockerfile
|
||||
if [ -f Dockerfile.foxhunt-build ]; then
|
||||
print_success "Dockerfile.foxhunt-build exists"
|
||||
else
|
||||
print_error "Dockerfile.foxhunt-build not found"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_docker_hub_config() {
|
||||
print_header "3. Validating Docker Hub Configuration"
|
||||
|
||||
# Check if Docker Hub credentials are set (for local testing)
|
||||
if [ -n "${DOCKER_HUB_USERNAME:-}" ] && [ -n "${DOCKER_HUB_PASSWORD:-}" ]; then
|
||||
print_success "Docker Hub credentials found in environment"
|
||||
|
||||
# Test Docker Hub login
|
||||
print_info "Testing Docker Hub authentication..."
|
||||
if echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin docker.io &> /dev/null; then
|
||||
print_success "Docker Hub authentication successful"
|
||||
else
|
||||
print_error "Docker Hub authentication failed"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
print_warning "Docker Hub credentials not set in environment (OK for GitLab CI/CD)"
|
||||
print_info "For GitLab CI/CD, set these in: Settings → CI/CD → Variables"
|
||||
print_info " - DOCKER_HUB_USERNAME (your Docker Hub username)"
|
||||
print_info " - DOCKER_HUB_PASSWORD (Docker Hub access token, NOT password)"
|
||||
fi
|
||||
|
||||
# Check Docker Hub repository
|
||||
print_info "Checking Docker Hub repository configuration..."
|
||||
REPO=$(grep "DOCKER_HUB_REPO:" .gitlab-ci.yml | awk '{print $2}')
|
||||
if [ -n "$REPO" ]; then
|
||||
print_success "Docker Hub repository: $REPO"
|
||||
else
|
||||
print_error "DOCKER_HUB_REPO not defined in .gitlab-ci.yml"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_build_artifacts() {
|
||||
print_header "4. Validating Build Artifacts"
|
||||
|
||||
# Check if hyperopt binaries exist
|
||||
print_info "Checking for hyperopt binaries..."
|
||||
local binaries=(
|
||||
"hyperopt_mamba2_demo"
|
||||
"hyperopt_dqn_demo"
|
||||
"hyperopt_ppo_demo"
|
||||
"hyperopt_tft_demo"
|
||||
)
|
||||
|
||||
local missing_count=0
|
||||
for binary in "${binaries[@]}"; do
|
||||
if [ -f "target/release/$binary" ]; then
|
||||
print_success "$binary exists ($(du -h "target/release/$binary" | cut -f1))"
|
||||
else
|
||||
print_warning "$binary not found (will be built in CI/CD)"
|
||||
((missing_count++))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $missing_count -eq ${#binaries[@]} ]; then
|
||||
print_warning "No hyperopt binaries found locally (expected for clean checkout)"
|
||||
print_info "Binaries will be built during Docker image build"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_gitlab_runner() {
|
||||
print_header "5. Validating GitLab Runner Configuration"
|
||||
|
||||
# Check if gitlab-runner is installed
|
||||
if command -v gitlab-runner &> /dev/null; then
|
||||
print_success "GitLab Runner installed: $(gitlab-runner --version | head -n1)"
|
||||
|
||||
# Check registered runners
|
||||
print_info "Checking registered runners..."
|
||||
if gitlab-runner list 2>&1 | grep -q "Executor"; then
|
||||
print_success "GitLab Runner registered"
|
||||
gitlab-runner list 2>&1 | grep "Executor" | while read -r line; do
|
||||
print_info " $line"
|
||||
done
|
||||
else
|
||||
print_warning "No GitLab Runners registered locally"
|
||||
fi
|
||||
else
|
||||
print_warning "GitLab Runner not installed locally (OK if using GitLab.com shared runners)"
|
||||
print_info "For self-hosted runner: https://docs.gitlab.com/runner/install/"
|
||||
fi
|
||||
|
||||
# Check .gitlab-ci.yml for runner tags
|
||||
print_info "Checking runner tags in .gitlab-ci.yml..."
|
||||
if grep -q "tags:" .gitlab-ci.yml; then
|
||||
grep "tags:" .gitlab-ci.yml | while read -r line; do
|
||||
print_info " $line"
|
||||
done
|
||||
print_success "Runner tags configured"
|
||||
else
|
||||
print_warning "No runner tags specified (will use any available runner)"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_ci_variables() {
|
||||
print_header "6. Validating CI/CD Variables"
|
||||
|
||||
print_info "Required GitLab CI/CD Variables (set in GitLab UI):"
|
||||
print_info " - DOCKER_HUB_USERNAME (required)"
|
||||
print_info " - DOCKER_HUB_PASSWORD (required, use access token)"
|
||||
|
||||
print_info ""
|
||||
print_info "Optional CI/CD Variables:"
|
||||
print_info " - SLACK_WEBHOOK_URL (for notifications)"
|
||||
print_info " - DISCORD_WEBHOOK_URL (for notifications)"
|
||||
|
||||
# Check if variables are referenced in .gitlab-ci.yml
|
||||
if grep -q "DOCKER_HUB_USERNAME" .gitlab-ci.yml && grep -q "DOCKER_HUB_PASSWORD" .gitlab-ci.yml; then
|
||||
print_success "Required variables referenced in .gitlab-ci.yml"
|
||||
else
|
||||
print_error "Required variables not referenced in .gitlab-ci.yml"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_deployment_config() {
|
||||
print_header "7. Validating Deployment Configuration"
|
||||
|
||||
# Check deployment stages
|
||||
if grep -q "deploy:runpod:" .gitlab-ci.yml; then
|
||||
print_success "RunPod deployment stage configured"
|
||||
else
|
||||
print_warning "RunPod deployment stage not found"
|
||||
fi
|
||||
|
||||
if grep -q "deploy:runpod-staging:" .gitlab-ci.yml; then
|
||||
print_success "Staging deployment stage configured"
|
||||
else
|
||||
print_warning "Staging deployment stage not found"
|
||||
fi
|
||||
|
||||
# Check manual approval for production
|
||||
if grep -q "when: manual" .gitlab-ci.yml; then
|
||||
print_success "Manual approval configured for production deployment"
|
||||
else
|
||||
print_warning "No manual approval for production (all deployments automatic)"
|
||||
fi
|
||||
|
||||
# Check environment configuration
|
||||
if grep -q "environment:" .gitlab-ci.yml; then
|
||||
print_success "Environment configuration present"
|
||||
else
|
||||
print_warning "No environment configuration"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
simulate_local_build() {
|
||||
print_header "8. Simulating Local Build (Optional)"
|
||||
|
||||
read -p "Run local Docker build simulation? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
print_info "Skipping local build simulation"
|
||||
return 0
|
||||
fi
|
||||
|
||||
print_info "Building Docker image locally (this may take 5-10 minutes)..."
|
||||
|
||||
export DOCKER_BUILDKIT=1
|
||||
export GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
if docker build \
|
||||
-f Dockerfile.foxhunt-build \
|
||||
-t jgrusewski/foxhunt:test-local \
|
||||
--build-arg GIT_COMMIT="$GIT_COMMIT" \
|
||||
--build-arg BUILD_DATE="$BUILD_DATE" \
|
||||
. 2>&1 | tail -20; then
|
||||
print_success "Local Docker build completed"
|
||||
|
||||
# Validate binaries in image
|
||||
print_info "Validating binaries in image..."
|
||||
local binaries=(
|
||||
"hyperopt_mamba2_demo"
|
||||
"hyperopt_dqn_demo"
|
||||
"hyperopt_ppo_demo"
|
||||
"hyperopt_tft_demo"
|
||||
)
|
||||
|
||||
for binary in "${binaries[@]}"; do
|
||||
if docker run --rm --entrypoint="" jgrusewski/foxhunt:test-local test -f "/usr/local/bin/$binary"; then
|
||||
print_success "$binary exists in image"
|
||||
else
|
||||
print_error "$binary missing in image"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Show image size
|
||||
local size=$(docker image inspect jgrusewski/foxhunt:test-local --format='{{.Size}}' | awk '{ sum=$1; hum[1024**3]="GB"; hum[1024**2]="MB"; hum[1024]="KB"; for (x=1024**3; x>=1024; x/=1024) { if (sum>=x) { printf "%.2f %s", sum/x, hum[x]; break } }}')
|
||||
print_info "Image size: $size"
|
||||
|
||||
else
|
||||
print_error "Local Docker build failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Main Execution
|
||||
# =============================================================================
|
||||
|
||||
main() {
|
||||
print_header "GitLab CI/CD Configuration Validator"
|
||||
|
||||
print_info "Current directory: $(pwd)"
|
||||
print_info "Git repository: $(git remote get-url origin 2>/dev/null || echo 'Not a git repository')"
|
||||
print_info "Git branch: $(git branch --show-current 2>/dev/null || echo 'Unknown')"
|
||||
print_info "Git commit: $(git rev-parse --short HEAD 2>/dev/null || echo 'Unknown')"
|
||||
|
||||
local exit_code=0
|
||||
|
||||
# Run all validations
|
||||
validate_gitlab_ci_file || exit_code=1
|
||||
validate_docker_config || exit_code=1
|
||||
validate_docker_hub_config || exit_code=1
|
||||
validate_build_artifacts || exit_code=1
|
||||
validate_gitlab_runner || exit_code=1
|
||||
validate_ci_variables || exit_code=1
|
||||
validate_deployment_config || exit_code=1
|
||||
|
||||
# Optional: simulate local build
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
simulate_local_build || exit_code=1
|
||||
fi
|
||||
|
||||
# Final summary
|
||||
print_header "Validation Summary"
|
||||
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
print_success "All validations passed!"
|
||||
print_info ""
|
||||
print_info "Next steps:"
|
||||
print_info " 1. Set GitLab CI/CD variables in GitLab UI (if not already set)"
|
||||
print_info " Settings → CI/CD → Variables"
|
||||
print_info " - DOCKER_HUB_USERNAME"
|
||||
print_info " - DOCKER_HUB_PASSWORD (Docker Hub access token)"
|
||||
print_info ""
|
||||
print_info " 2. Push to main branch to trigger pipeline:"
|
||||
print_info " git push origin main"
|
||||
print_info ""
|
||||
print_info " 3. Monitor pipeline in GitLab:"
|
||||
print_info " Project → CI/CD → Pipelines"
|
||||
print_info ""
|
||||
print_info " 4. Manual approval required for production deployment"
|
||||
print_info ""
|
||||
else
|
||||
print_error "Some validations failed"
|
||||
print_info ""
|
||||
print_info "Please fix the issues above before enabling GitLab CI/CD"
|
||||
print_info ""
|
||||
fi
|
||||
|
||||
exit $exit_code
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user