foxhunt/scripts/check-gpu-availability.sh

#!/usr/bin/env bash
# check-gpu-availability.sh — Check Scaleway GPU availability and select fastest path
# Usage:
#   ./scripts/check-gpu-availability.sh           # show all + recommend fastest
#   ./scripts/check-gpu-availability.sh --best     # output only the best pool name (for CI)
#   ./scripts/check-gpu-availability.sh H100       # filter by type
#
# Exit codes: 0 = found available GPU, 1 = nothing available, 2 = error
set -euo pipefail

CLUSTER_ZONE="fr-par-2"
BEST_ONLY=false
FILTER=""

for arg in "$@"; do
    case "$arg" in
        --best) BEST_ONLY=true ;;
        *)      FILTER="$arg" ;;
    esac
done

# GPU types ranked by speed (fastest first)
# Format: scw_type|pool_name|label|gpu_count|approx_tflops
GPU_RANK=(
    "H100-SXM-8-80G|ci-training-h100-sxm8|H100 SXM 8x80GB|8|26400"
    "H100-SXM-4-80G|ci-training-h100-sxm4|H100 SXM 4x80GB|4|13200"
    "H100-SXM-2-80G|ci-training-h100-sxm|H100 SXM 2x80GB|2|6600"
    "H100-2-80G|ci-training-h100x2|H100 PCIe 2x80GB|2|5280"
    "H100-1-80G|ci-training-h100|H100 PCIe 1x80GB|1|2640"
    "L40S-1-48G|ci-training|L40S 1x48GB|1|733"
)

get_availability() {
    local type=$1
    local avail
    avail=$(scw instance server-type list zone="$CLUSTER_ZONE" 2>/dev/null | grep "^${type} " || true)
    if [[ -z "$avail" ]]; then
        echo "unavailable"
        return
    fi
    if echo "$avail" | grep -q "out of stock"; then
        echo "out_of_stock"
    elif echo "$avail" | grep -q "low stock"; then
        echo "low_stock"
    elif echo "$avail" | grep -q "available"; then
        echo "available"
    else
        echo "unknown"
    fi
}

get_pool_status() {
    local pool_name=$1
    local cluster_id
    cluster_id=$(scw k8s cluster list -o json 2>/dev/null | jq -r '.[0].id // empty')
    [[ -z "$cluster_id" ]] && echo "no_cluster" && return
    local status
    status=$(scw k8s pool list cluster-id="$cluster_id" -o json 2>/dev/null | \
        jq -r ".[] | select(.name==\"$pool_name\") | .status" 2>/dev/null)
    echo "${status:-no_pool}"
}

# Find the fastest available GPU
best_pool=""
best_label=""
best_type=""

if ! $BEST_ONLY; then
    echo "╔══════════════════════════════════════════════════════════════╗"
    echo "║          Scaleway GPU — Fastest Path Selection              ║"
    echo "╠══════════════════════════════════════════════════════════════╣"
    printf "║  Zone: %-52s║\n" "$CLUSTER_ZONE"
    printf "║  Time: %-52s║\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
    echo "╚══════════════════════════════════════════════════════════════╝"
    echo ""
    printf "  %-22s %-6s %-13s %-12s %s\n" "TYPE" "GPUs" "TFLOPS(FP16)" "STOCK" "POOL"
    echo "  ─────────────────────────────────────────────────────────────"
fi

for entry in "${GPU_RANK[@]}"; do
    IFS='|' read -r scw_type pool_name label gpu_count tflops <<< "$entry"

    # Apply filter
    if [[ -n "$FILTER" && "$scw_type" != *"$FILTER"* && "$label" != *"$FILTER"* ]]; then
        continue
    fi

    avail=$(get_availability "$scw_type")

    # Color and symbol
    symbol="✗"
    color="\033[31m"
    case "$avail" in
        available)    symbol="✓"; color="\033[32m" ;;
        low_stock)    symbol="~"; color="\033[33m" ;;
        out_of_stock) symbol="✗"; color="\033[31m" ;;
        *)            symbol="?"; color="\033[90m" ;;
    esac

    # Track best available
    if [[ -z "$best_pool" && ("$avail" == "available" || "$avail" == "low_stock") ]]; then
        best_pool="$pool_name"
        best_label="$label"
        best_type="$scw_type"
    fi

    if ! $BEST_ONLY; then
        printf "  %b%s %-21s\033[0m %-6s %-13s %b%-12s\033[0m %s\n" \
            "$color" "$symbol" "$scw_type" "$gpu_count" "$tflops" "$color" "$avail" "$pool_name"
    fi
done

if ! $BEST_ONLY; then
    echo ""

    # Kapsule pool status
    echo "  ─── Kapsule GPU Pools ───"
    CLUSTER_ID=$(scw k8s cluster list -o json 2>/dev/null | jq -r '.[0].id // empty')
    if [[ -n "$CLUSTER_ID" ]]; then
        scw k8s pool list cluster-id="$CLUSTER_ID" -o json 2>/dev/null | \
            jq -r '.[] | select(.node_type | test("h100|l40|l4_|gpu")) | "  \(.name): \(.node_type) size=\(.size)/\(.max_size) status=\(.status)"'
    fi
    echo ""

    # Active GPU nodes
    echo "  ─── Active GPU Nodes ───"
    gpu_nodes=$(kubectl get nodes -l k8s.scaleway.com/pool-name -o json 2>/dev/null | \
        jq -r '.items[] | select(.metadata.labels["k8s.scaleway.com/pool-name"] | test("training|gpu")) | "  \(.metadata.name) (\(.metadata.labels["k8s.scaleway.com/pool-name"]))"' 2>/dev/null || true)
    echo "${gpu_nodes:-  (none running)}"
    echo ""

    # Node errors
    if [[ -n "$CLUSTER_ID" ]]; then
        errors=$(scw k8s node list cluster-id="$CLUSTER_ID" -o json 2>/dev/null | \
            jq -r '.[] | select(.status=="creation_error") | "  ⚠ \(.name): \(.error_message)"' 2>/dev/null || true)
        if [[ -n "$errors" ]]; then
            echo "  ─── Provisioning Errors ───"
            echo "$errors"
            echo ""
        fi
    fi

    # Recommendation
    if [[ -n "$best_pool" ]]; then
        echo "  ══════════════════════════════════════════════════════════"
        echo "  FASTEST AVAILABLE: $best_label"
        echo "  Pool: $best_pool  (type: $best_type)"
        echo "  ══════════════════════════════════════════════════════════"
    else
        echo "  ⚠ No GPU instances available in $CLUSTER_ZONE"
    fi
fi

# --best mode: just output pool name for CI scripting
if $BEST_ONLY; then
    if [[ -n "$best_pool" ]]; then
        echo "$best_pool"
        exit 0
    else
        exit 1
    fi
fi

[[ -n "$best_pool" ]] && exit 0 || exit 1