diff --git a/infra/k8s/argo/alpha-perception-template.yaml b/infra/k8s/argo/alpha-perception-template.yaml index 1c346bb8a..524c4dbbd 100644 --- a/infra/k8s/argo/alpha-perception-template.yaml +++ b/infra/k8s/argo/alpha-perception-template.yaml @@ -81,29 +81,111 @@ spec: templates: # ── DAG ── # - # ensure-binary (CPU compile) and warmup-gpu (autoscaler trigger - # for the L40S pool) run in parallel. train only depends on - # ensure-binary — warmup-gpu is fire-and-forget; the autoscaler - # keeps the node warm for its scaledown grace window after the - # warmup pod exits, so train lands on a hot node without waiting - # for autoscaler provisioning. + # check-cache (tiny alpine pod on platform pool, ~3 sec) inspects + # the training-data PVC for /data/bin/$SHA/alpha_train. If present + # (cache hit), ensure-binary is SKIPPED via `when:` — the + # ~4.8GB ci-builder image pull + sccache compile cycle are avoided + # entirely. train still runs, sourcing the binary path from + # check-cache's SHA output. + # + # warmup-gpu fires unconditionally in parallel — triggers the L40S + # autoscaler so the node is warm by the time train needs it, + # whether or not we compile. - name: pipeline dag: tasks: + - name: check-cache + template: check-cache - name: ensure-binary template: ensure-binary + dependencies: [check-cache] + when: "{{tasks.check-cache.outputs.parameters.cache}} == miss" + arguments: + parameters: + - name: sha + value: "{{tasks.check-cache.outputs.parameters.sha}}" - name: warmup-gpu template: warmup-gpu - name: train template: train - dependencies: [ensure-binary] + dependencies: [check-cache, ensure-binary] arguments: parameters: - name: sha - value: "{{tasks.ensure-binary.outputs.parameters.sha}}" + value: "{{tasks.check-cache.outputs.parameters.sha}}" + + # ── check-cache: probe the training-data PVC for an existing binary ── + # + # Runs on the platform pool (always up, no autoscaler delay). Tiny + # alpine pod, ~3 sec end-to-end. Emits two output parameters: + # - sha = the short SHA used for binary cache keying + # - cache = "hit" if /data/bin/$SHA/alpha_train exists else "miss" + # + # Resolves `commit-sha=HEAD` by reading /tmp/head-sha from a + # ConfigMap... actually no, simpler: the submission script + # (scripts/argo-alpha-perception.sh) pre-resolves HEAD via local + # git so commit-sha is always a real SHA in the workflow. This + # pod just trims to short SHA and stats the binary. + - name: check-cache + outputs: + parameters: + - name: sha + valueFrom: + path: /tmp/sha + - name: cache + valueFrom: + path: /tmp/cache + nodeSelector: + k8s.scaleway.com/pool-name: platform + topology.kubernetes.io/zone: fr-par-2 + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: alpine:3.20 + command: ["/bin/sh", "-c"] + resources: + requests: + cpu: "50m" + memory: 32Mi + limits: + cpu: "100m" + memory: 64Mi + volumeMounts: + - name: training-data + mountPath: /data + readOnly: true + args: + - | + set -e + SHA="{{workflow.parameters.commit-sha}}" + if [ "$SHA" = "HEAD" ]; then + echo "ERROR: commit-sha cannot be HEAD inside the workflow." + echo " Resolve via the submission script (scripts/argo-alpha-perception.sh)." + exit 1 + fi + SHORT_SHA=$(echo "$SHA" | cut -c1-9) + echo "$SHORT_SHA" > /tmp/sha + + BIN="/data/bin/$SHORT_SHA/alpha_train" + if [ -x "$BIN" ]; then + SIZE=$(stat -c %s "$BIN") + echo "Cache HIT: $BIN ($SIZE bytes)" + echo "hit" > /tmp/cache + else + echo "Cache MISS: $BIN not present, ensure-binary will compile" + ls -lh "/data/bin/" 2>/dev/null | head -10 || echo " (no /data/bin/ directory yet)" + echo "miss" > /tmp/cache + fi # ── ensure-binary: compile `alpha_train` example via sccache ── + # Only runs when check-cache returned cache=miss. Outputs the SHA + # for consistency, but the DAG sources SHA from check-cache directly. - name: ensure-binary + inputs: + parameters: + - name: sha outputs: parameters: - name: sha diff --git a/scripts/argo-alpha-perception.sh b/scripts/argo-alpha-perception.sh index cd1205083..489dcc1b5 100755 --- a/scripts/argo-alpha-perception.sh +++ b/scripts/argo-alpha-perception.sh @@ -69,13 +69,29 @@ done case "$GPU_POOL" in ci-training-l40s) SM=89 ;; - ci-training-h100*|ci-training-h100x2|ci-training-h100-sxm) + ci-training-h100) SM=90 ;; *) echo "Unknown gpu-pool: $GPU_POOL" exit 1 ;; esac +# Resolve commit-sha=HEAD to an actual SHA locally before submission. +# The in-cluster check-cache pod uses this SHA to look up the binary +# cache without needing git inside the alpine pod. +if [ "$SHA" = "HEAD" ]; then + echo "Resolving HEAD for branch $BRANCH..." + if ! git rev-parse --git-dir >/dev/null 2>&1; then + echo "ERROR: not in a git repo; cannot resolve HEAD" + exit 1 + fi + git fetch --quiet origin "$BRANCH" 2>/dev/null || true + SHA=$(git rev-parse "origin/$BRANCH" 2>/dev/null \ + || git rev-parse "$BRANCH" 2>/dev/null \ + || git rev-parse HEAD) + echo " resolved: $SHA" +fi + echo "Submitting alpha-perception workflow..." echo " branch: $BRANCH" echo " sha: $SHA"