fix(ml): cast Bellman target to F32 before TD error sub on BF16 GPUs
BUG #41 kept forward pass in F32 for autograd, but the target-side tensors (reward, gamma, done, next_q) were cast to BF16 via `dtype`. The `state_action_values.sub(&target_q_values)` then hit F32-vs-BF16 mismatch on Ampere+ GPUs, causing every training step to fail silently. Fix: `.to_dtype(state_action_values.dtype())` on the detached target. Safe because target is detached (no autograd graph to break). Also: H100 runner → SXM2 pool, GPU availability checker script. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -2130,7 +2130,9 @@ impl DQN {
|
||||
let gamma_next = (&gamma_tensor * &next_state_values)
|
||||
.map_err(|e| MLError::TrainingError(format!("Gamma multiplication failed: {}", e)))?;
|
||||
let discounted = (&gamma_next * ¬_done)?;
|
||||
let target_q_values = (&rewards_tensor + &discounted)?.detach(); // Stop gradient computation
|
||||
let target_q_values = (&rewards_tensor + &discounted)?
|
||||
.detach() // Stop gradient computation
|
||||
.to_dtype(state_action_values.dtype())?; // Match forward pass dtype (F32) for sub
|
||||
let diff = state_action_values.sub(&target_q_values)?;
|
||||
|
||||
// BUG #14 FIX: NaN detection — periodic to avoid GPU sync stall every step.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# GitLab Runner — H100 GPU training workloads (hyperopt, walk-forward)
|
||||
# Runs on ci-training-h100 pool (H100: 80GB VRAM)
|
||||
# Runs on ci-training-h100-sxm pool (H100 SXM2: 2x80GB VRAM, NVLink)
|
||||
# Falls back from ci-training-h100-sxm → ci-training-h100 via pool availability.
|
||||
# Mounts separate PVCs per GPU node to avoid RWO conflicts.
|
||||
|
||||
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
||||
@@ -34,10 +35,10 @@ runners:
|
||||
image = "rust:1.89-slim"
|
||||
privileged = false
|
||||
node_selector_overwrite_allowed = ".*"
|
||||
cpu_request_overwrite_max_allowed = "8000m"
|
||||
cpu_limit_overwrite_max_allowed = "8000m"
|
||||
memory_request_overwrite_max_allowed = "96Gi"
|
||||
memory_limit_overwrite_max_allowed = "96Gi"
|
||||
cpu_request_overwrite_max_allowed = "24000m"
|
||||
cpu_limit_overwrite_max_allowed = "24000m"
|
||||
memory_request_overwrite_max_allowed = "200Gi"
|
||||
memory_limit_overwrite_max_allowed = "200Gi"
|
||||
poll_timeout = 600
|
||||
runtime_class_name = "nvidia"
|
||||
pod_annotations_overwrite_allowed = ".*"
|
||||
@@ -52,19 +53,19 @@ runners:
|
||||
helper_memory_limit = "512Mi"
|
||||
image_pull_secrets = ["scw-registry", "gitlab-registry"]
|
||||
[runners.kubernetes.node_selector]
|
||||
"k8s.scaleway.com/pool-name" = "ci-training-h100"
|
||||
"k8s.scaleway.com/pool-name" = "ci-training-h100-sxm"
|
||||
[runners.kubernetes.node_tolerations]
|
||||
"nvidia.com/gpu" = "NoSchedule"
|
||||
"node.cilium.io/agent-not-ready" = "NoSchedule"
|
||||
[runners.kubernetes.pod_labels]
|
||||
"app.kubernetes.io/part-of" = "foxhunt-ci"
|
||||
# H100-specific PVCs (separate from L40S to avoid RWO conflicts)
|
||||
# H100-SXM specific PVCs (separate from L40S/H100-PCIe to avoid RWO conflicts)
|
||||
[[runners.kubernetes.volumes.pvc]]
|
||||
name = "training-data-h100-pvc"
|
||||
name = "training-data-h100-sxm-pvc"
|
||||
mount_path = "/mnt/training-data"
|
||||
read_only = true
|
||||
[[runners.kubernetes.volumes.pvc]]
|
||||
name = "sccache-h100-pvc"
|
||||
name = "sccache-h100-sxm-pvc"
|
||||
mount_path = "/mnt/sccache"
|
||||
read_only = false
|
||||
|
||||
|
||||
164
scripts/check-gpu-availability.sh
Executable file
164
scripts/check-gpu-availability.sh
Executable file
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env bash
|
||||
# check-gpu-availability.sh — Check Scaleway GPU availability and select fastest path
|
||||
# Usage:
|
||||
# ./scripts/check-gpu-availability.sh # show all + recommend fastest
|
||||
# ./scripts/check-gpu-availability.sh --best # output only the best pool name (for CI)
|
||||
# ./scripts/check-gpu-availability.sh H100 # filter by type
|
||||
#
|
||||
# Exit codes: 0 = found available GPU, 1 = nothing available, 2 = error
|
||||
set -euo pipefail
|
||||
|
||||
CLUSTER_ZONE="fr-par-2"
|
||||
BEST_ONLY=false
|
||||
FILTER=""
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--best) BEST_ONLY=true ;;
|
||||
*) FILTER="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# GPU types ranked by speed (fastest first)
|
||||
# Format: scw_type|pool_name|label|gpu_count|approx_tflops
|
||||
GPU_RANK=(
|
||||
"H100-SXM-8-80G|ci-training-h100-sxm8|H100 SXM 8x80GB|8|26400"
|
||||
"H100-SXM-4-80G|ci-training-h100-sxm4|H100 SXM 4x80GB|4|13200"
|
||||
"H100-SXM-2-80G|ci-training-h100-sxm|H100 SXM 2x80GB|2|6600"
|
||||
"H100-2-80G|ci-training-h100x2|H100 PCIe 2x80GB|2|5280"
|
||||
"H100-1-80G|ci-training-h100|H100 PCIe 1x80GB|1|2640"
|
||||
"L40S-1-48G|ci-training|L40S 1x48GB|1|733"
|
||||
)
|
||||
|
||||
get_availability() {
|
||||
local type=$1
|
||||
local avail
|
||||
avail=$(scw instance server-type list zone="$CLUSTER_ZONE" 2>/dev/null | grep "^${type} " || true)
|
||||
if [[ -z "$avail" ]]; then
|
||||
echo "unavailable"
|
||||
return
|
||||
fi
|
||||
if echo "$avail" | grep -q "out of stock"; then
|
||||
echo "out_of_stock"
|
||||
elif echo "$avail" | grep -q "low stock"; then
|
||||
echo "low_stock"
|
||||
elif echo "$avail" | grep -q "available"; then
|
||||
echo "available"
|
||||
else
|
||||
echo "unknown"
|
||||
fi
|
||||
}
|
||||
|
||||
get_pool_status() {
|
||||
local pool_name=$1
|
||||
local cluster_id
|
||||
cluster_id=$(scw k8s cluster list -o json 2>/dev/null | jq -r '.[0].id // empty')
|
||||
[[ -z "$cluster_id" ]] && echo "no_cluster" && return
|
||||
local status
|
||||
status=$(scw k8s pool list cluster-id="$cluster_id" -o json 2>/dev/null | \
|
||||
jq -r ".[] | select(.name==\"$pool_name\") | .status" 2>/dev/null)
|
||||
echo "${status:-no_pool}"
|
||||
}
|
||||
|
||||
# Find the fastest available GPU
|
||||
best_pool=""
|
||||
best_label=""
|
||||
best_type=""
|
||||
|
||||
if ! $BEST_ONLY; then
|
||||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||||
echo "║ Scaleway GPU — Fastest Path Selection ║"
|
||||
echo "╠══════════════════════════════════════════════════════════════╣"
|
||||
printf "║ Zone: %-52s║\n" "$CLUSTER_ZONE"
|
||||
printf "║ Time: %-52s║\n" "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
printf " %-22s %-6s %-13s %-12s %s\n" "TYPE" "GPUs" "TFLOPS(FP16)" "STOCK" "POOL"
|
||||
echo " ─────────────────────────────────────────────────────────────"
|
||||
fi
|
||||
|
||||
for entry in "${GPU_RANK[@]}"; do
|
||||
IFS='|' read -r scw_type pool_name label gpu_count tflops <<< "$entry"
|
||||
|
||||
# Apply filter
|
||||
if [[ -n "$FILTER" && "$scw_type" != *"$FILTER"* && "$label" != *"$FILTER"* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
avail=$(get_availability "$scw_type")
|
||||
|
||||
# Color and symbol
|
||||
symbol="✗"
|
||||
color="\033[31m"
|
||||
case "$avail" in
|
||||
available) symbol="✓"; color="\033[32m" ;;
|
||||
low_stock) symbol="~"; color="\033[33m" ;;
|
||||
out_of_stock) symbol="✗"; color="\033[31m" ;;
|
||||
*) symbol="?"; color="\033[90m" ;;
|
||||
esac
|
||||
|
||||
# Track best available
|
||||
if [[ -z "$best_pool" && ("$avail" == "available" || "$avail" == "low_stock") ]]; then
|
||||
best_pool="$pool_name"
|
||||
best_label="$label"
|
||||
best_type="$scw_type"
|
||||
fi
|
||||
|
||||
if ! $BEST_ONLY; then
|
||||
printf " %b%s %-21s\033[0m %-6s %-13s %b%-12s\033[0m %s\n" \
|
||||
"$color" "$symbol" "$scw_type" "$gpu_count" "$tflops" "$color" "$avail" "$pool_name"
|
||||
fi
|
||||
done
|
||||
|
||||
if ! $BEST_ONLY; then
|
||||
echo ""
|
||||
|
||||
# Kapsule pool status
|
||||
echo " ─── Kapsule GPU Pools ───"
|
||||
CLUSTER_ID=$(scw k8s cluster list -o json 2>/dev/null | jq -r '.[0].id // empty')
|
||||
if [[ -n "$CLUSTER_ID" ]]; then
|
||||
scw k8s pool list cluster-id="$CLUSTER_ID" -o json 2>/dev/null | \
|
||||
jq -r '.[] | select(.node_type | test("h100|l40|l4_|gpu")) | " \(.name): \(.node_type) size=\(.size)/\(.max_size) status=\(.status)"'
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Active GPU nodes
|
||||
echo " ─── Active GPU Nodes ───"
|
||||
gpu_nodes=$(kubectl get nodes -l k8s.scaleway.com/pool-name -o json 2>/dev/null | \
|
||||
jq -r '.items[] | select(.metadata.labels["k8s.scaleway.com/pool-name"] | test("training|gpu")) | " \(.metadata.name) (\(.metadata.labels["k8s.scaleway.com/pool-name"]))"' 2>/dev/null || true)
|
||||
echo "${gpu_nodes:- (none running)}"
|
||||
echo ""
|
||||
|
||||
# Node errors
|
||||
if [[ -n "$CLUSTER_ID" ]]; then
|
||||
errors=$(scw k8s node list cluster-id="$CLUSTER_ID" -o json 2>/dev/null | \
|
||||
jq -r '.[] | select(.status=="creation_error") | " ⚠ \(.name): \(.error_message)"' 2>/dev/null || true)
|
||||
if [[ -n "$errors" ]]; then
|
||||
echo " ─── Provisioning Errors ───"
|
||||
echo "$errors"
|
||||
echo ""
|
||||
fi
|
||||
fi
|
||||
|
||||
# Recommendation
|
||||
if [[ -n "$best_pool" ]]; then
|
||||
echo " ══════════════════════════════════════════════════════════"
|
||||
echo " FASTEST AVAILABLE: $best_label"
|
||||
echo " Pool: $best_pool (type: $best_type)"
|
||||
echo " ══════════════════════════════════════════════════════════"
|
||||
else
|
||||
echo " ⚠ No GPU instances available in $CLUSTER_ZONE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --best mode: just output pool name for CI scripting
|
||||
if $BEST_ONLY; then
|
||||
if [[ -n "$best_pool" ]]; then
|
||||
echo "$best_pool"
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ -n "$best_pool" ]] && exit 0 || exit 1
|
||||
Reference in New Issue
Block a user