87a37ac020
TOML training profiles are include_str!() into the ml binary at compile time. Changes to config/training/ or config/gpu/ must trigger GPU test rebuild, otherwise the binary runs with stale embedded defaults. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
663 lines
23 KiB
YAML
663 lines
23 KiB
YAML
apiVersion: argoproj.io/v1alpha1
|
|
kind: WorkflowTemplate
|
|
metadata:
|
|
name: ci-pipeline
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: ci-pipeline
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
activeDeadlineSeconds: 7200
|
|
serviceAccountName: argo-workflow
|
|
entrypoint: pipeline
|
|
onExit: notify-result
|
|
podMetadata:
|
|
labels:
|
|
app.kubernetes.io/component: ci-pipeline
|
|
app.kubernetes.io/part-of: foxhunt
|
|
podGC:
|
|
strategy: OnPodCompletion
|
|
ttlStrategy:
|
|
secondsAfterCompletion: 3600
|
|
volumes:
|
|
- name: git-ssh-key
|
|
secret:
|
|
secretName: argo-git-ssh-key
|
|
defaultMode: 256
|
|
- name: registry-auth
|
|
secret:
|
|
secretName: gitlab-registry
|
|
optional: true
|
|
items:
|
|
- key: .dockerconfigjson
|
|
path: config.json
|
|
- name: cargo-target-cpu
|
|
persistentVolumeClaim:
|
|
claimName: cargo-target-cpu
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: HEAD
|
|
- name: commits-json
|
|
value: "[]"
|
|
templates:
|
|
# ── DAG: orchestrate pipeline ──
|
|
# Compile + deploy moved to compile-and-deploy-template.yaml (manual trigger).
|
|
# This pipeline handles infra, docker images, dashboard, and argo templates only.
|
|
- name: pipeline
|
|
dag:
|
|
tasks:
|
|
- name: detect-changes
|
|
template: detect-changes
|
|
|
|
- name: build-web-dashboard
|
|
depends: "detect-changes"
|
|
template: build-web-dashboard
|
|
when: "{{tasks.detect-changes.outputs.parameters.needs-dashboard}} == true"
|
|
|
|
- name: rebuild-ci-builder
|
|
depends: "detect-changes"
|
|
templateRef:
|
|
name: build-ci-image
|
|
template: build
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
- name: dockerfile
|
|
value: Dockerfile.ci-builder
|
|
- name: image-name
|
|
value: ci-builder
|
|
when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
|
|
|
|
- name: rebuild-ci-builder-cpu
|
|
depends: "detect-changes"
|
|
templateRef:
|
|
name: build-ci-image
|
|
template: build
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
- name: dockerfile
|
|
value: Dockerfile.ci-builder-cpu
|
|
- name: image-name
|
|
value: ci-builder-cpu
|
|
when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
|
|
|
|
- name: rebuild-runtime
|
|
depends: "detect-changes"
|
|
templateRef:
|
|
name: build-ci-image
|
|
template: build
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
- name: dockerfile
|
|
value: Dockerfile.foxhunt-runtime
|
|
- name: image-name
|
|
value: foxhunt-runtime
|
|
when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
|
|
|
|
- name: rebuild-training-runtime
|
|
depends: "detect-changes"
|
|
templateRef:
|
|
name: build-ci-image
|
|
template: build
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
- name: dockerfile
|
|
value: Dockerfile.foxhunt-training-runtime
|
|
- name: image-name
|
|
value: foxhunt-training-runtime
|
|
when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
|
|
|
|
- name: test-gate
|
|
depends: "detect-changes && (rebuild-ci-builder-cpu.Succeeded || rebuild-ci-builder-cpu.Skipped)"
|
|
template: test-gate
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
when: "{{tasks.detect-changes.outputs.parameters.needs-code}} == true"
|
|
|
|
- name: apply-argo-templates
|
|
depends: "detect-changes.Succeeded && (rebuild-ci-builder-cpu.Succeeded || rebuild-ci-builder-cpu.Skipped)"
|
|
template: apply-argo-templates
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
when: "{{tasks.detect-changes.outputs.parameters.needs-argo-templates}} == true"
|
|
|
|
- name: terragrunt-apply
|
|
depends: "detect-changes.Succeeded && (rebuild-ci-builder-cpu.Succeeded || rebuild-ci-builder-cpu.Skipped)"
|
|
template: terragrunt-apply
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
when: "{{tasks.detect-changes.outputs.parameters.needs-infra}} == true"
|
|
|
|
- name: gpu-test
|
|
depends: "detect-changes"
|
|
template: submit-gpu-test
|
|
when: "{{tasks.detect-changes.outputs.parameters.ml-changed}} == true"
|
|
|
|
# ── detect-changes: classify changed files to gate downstream steps ──
|
|
- name: detect-changes
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: COMMITS_JSON
|
|
value: "{{workflow.parameters.commits-json}}"
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
args:
|
|
- |
|
|
set -e
|
|
cat <<'SCRIPT' > /tmp/detect.sh
|
|
#!/bin/sh
|
|
set -e
|
|
|
|
# Extract changed file paths from webhook JSON using grep+sed (no jq dependency).
|
|
CHANGED_FILES=$(echo "$COMMITS_JSON" \
|
|
| grep -oE '"(added|modified|removed)":\[[^]]*\]' \
|
|
| sed 's/"added"://;s/"modified"://;s/"removed"://' \
|
|
| tr ',' '\n' | tr -d '[]"' | sed '/^$/d' | sort -u \
|
|
|| echo "")
|
|
|
|
if [ -z "$CHANGED_FILES" ]; then
|
|
echo "No changed files detected — triggering full rebuild"
|
|
CHANGED_FILES="Cargo.toml"
|
|
fi
|
|
|
|
echo "=== Changed files ==="
|
|
echo "$CHANGED_FILES"
|
|
echo "===================="
|
|
|
|
# --- Helper: check if any changed file matches a set of path prefixes ---
|
|
check_paths() {
|
|
local patterns="$1"
|
|
for file in $CHANGED_FILES; do
|
|
for pattern in $patterns; do
|
|
case "$file" in
|
|
${pattern}*) echo "true"; return ;;
|
|
esac
|
|
done
|
|
done
|
|
echo "false"
|
|
}
|
|
|
|
# --- Boolean gates for downstream DAG tasks ---
|
|
NEEDS_CODE=$(check_paths "Cargo.toml Cargo.lock crates/ services/ bin/fxt/")
|
|
NEEDS_DASHBOARD=$(check_paths "web-dashboard/")
|
|
DOCKER_IMAGES=$(check_paths "infra/docker/")
|
|
NEEDS_INFRA=$(check_paths "infra/live/ infra/modules/")
|
|
NEEDS_ARGO_TEMPLATES=$(check_paths "infra/k8s/argo/")
|
|
ML_CHANGED=$(check_paths "crates/ml crates/ml-dqn crates/ml-core crates/ml-regime crates/ml-features crates/ml-supervised crates/ml-ppo crates/ml-ensemble crates/ml-hyperopt crates/ml-labeling config/training config/gpu")
|
|
|
|
echo "=== Build decisions ==="
|
|
echo "needs-code: $NEEDS_CODE"
|
|
echo "needs-dashboard: $NEEDS_DASHBOARD"
|
|
echo "docker-images: $DOCKER_IMAGES"
|
|
echo "needs-infra: $NEEDS_INFRA"
|
|
echo "needs-argo: $NEEDS_ARGO_TEMPLATES"
|
|
echo "ml-changed: $ML_CHANGED"
|
|
echo "======================"
|
|
|
|
mkdir -p /tmp/outputs
|
|
echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code
|
|
echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard
|
|
echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images
|
|
echo -n "$NEEDS_INFRA" > /tmp/outputs/needs-infra
|
|
echo -n "$NEEDS_ARGO_TEMPLATES" > /tmp/outputs/needs-argo-templates
|
|
echo -n "$ML_CHANGED" > /tmp/outputs/ml-changed
|
|
SCRIPT
|
|
chmod +x /tmp/detect.sh
|
|
/tmp/detect.sh
|
|
outputs:
|
|
parameters:
|
|
- name: needs-code
|
|
valueFrom:
|
|
path: /tmp/outputs/needs-code
|
|
- name: needs-dashboard
|
|
valueFrom:
|
|
path: /tmp/outputs/needs-dashboard
|
|
- name: docker-images
|
|
valueFrom:
|
|
path: /tmp/outputs/docker-images
|
|
- name: needs-infra
|
|
valueFrom:
|
|
path: /tmp/outputs/needs-infra
|
|
- name: needs-argo-templates
|
|
valueFrom:
|
|
path: /tmp/outputs/needs-argo-templates
|
|
- name: ml-changed
|
|
valueFrom:
|
|
path: /tmp/outputs/ml-changed
|
|
|
|
# ── test-gate: clippy + cargo test quality gate ──
|
|
# Uses ci-builder (with CUDA toolkit) because cudarc 0.19 requires nvcc at build time.
|
|
- name: test-gate
|
|
inputs:
|
|
parameters:
|
|
- name: commit-sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
tolerations:
|
|
- key: k8s.scaleway.com/pool-name
|
|
operator: Equal
|
|
value: ci-compile-cpu
|
|
effect: NoSchedule
|
|
sidecars:
|
|
- name: redis
|
|
image: redis:7-alpine
|
|
command: ["redis-server", "--save", "", "--appendonly", "no"]
|
|
ports:
|
|
- containerPort: 6379
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 128Mi
|
|
readinessProbe:
|
|
tcpSocket:
|
|
port: 6379
|
|
initialDelaySeconds: 1
|
|
periodSeconds: 1
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: Always
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CARGO_TERM_COLOR
|
|
value: always
|
|
- name: CARGO_TARGET_DIR
|
|
value: /cargo-target
|
|
- name: CARGO_HOME
|
|
value: /cargo-target/cargo-home
|
|
- name: REDIS_URL
|
|
value: "redis://localhost:6379"
|
|
resources:
|
|
requests:
|
|
cpu: "14"
|
|
memory: 16Gi
|
|
limits:
|
|
cpu: "30"
|
|
memory: 32Gi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
- name: cargo-target-cpu
|
|
mountPath: /cargo-target
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{inputs.parameters.commit-sha}}"
|
|
|
|
# SSH setup
|
|
mkdir -p ~/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
|
chmod 600 ~/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
|
chmod 600 ~/.ssh/config
|
|
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
WORKSPACE="/cargo-target/src"
|
|
|
|
git config --global --add safe.directory "$WORKSPACE"
|
|
|
|
if [ -d "$WORKSPACE/.git" ]; then
|
|
cd "$WORKSPACE"
|
|
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
|
if [ "$CURRENT" = "$SHA" ]; then
|
|
echo "=== Already at $SHA ==="
|
|
else
|
|
git fetch origin
|
|
git checkout --force "$SHA"
|
|
git clean -fd
|
|
fi
|
|
else
|
|
git clone --filter=blob:none "$REPO" "$WORKSPACE"
|
|
cd "$WORKSPACE"
|
|
git checkout "$SHA"
|
|
fi
|
|
|
|
export PATH="${CARGO_HOME}/bin:${PATH}"
|
|
|
|
echo "=== Waiting for Redis sidecar ==="
|
|
for i in 1 2 3 4 5; do
|
|
if printf 'PING\r\n' | nc -w1 127.0.0.1 6379 2>/dev/null | grep -q PONG; then
|
|
echo "Redis ready"
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
echo "=== Running clippy (lib targets) ==="
|
|
cargo clippy --workspace --lib -- -D warnings 2>&1 | tee /cargo-target/clippy.log
|
|
|
|
echo "=== Running tests (lib only, no integration) ==="
|
|
set +e
|
|
cargo test --workspace --lib 2>&1 | tee /cargo-target/test-output.log
|
|
TEST_EXIT=$?
|
|
set -e
|
|
if [ "$TEST_EXIT" -ne 0 ]; then
|
|
echo "=== TEST FAILURES ==="
|
|
grep -A5 'FAILED\|panicked\|test result: FAILED\|error\[' /cargo-target/test-output.log || true
|
|
exit "$TEST_EXIT"
|
|
fi
|
|
|
|
echo "=== Test gate passed ==="
|
|
|
|
# ── build-web-dashboard: npm build + upload to MinIO ──
|
|
- name: build-web-dashboard
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: node:22-alpine
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: MINIO_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: access-key
|
|
- name: MINIO_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: secret-key
|
|
resources:
|
|
requests:
|
|
cpu: "1"
|
|
memory: 1Gi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 2Gi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{workflow.parameters.commit-sha}}"
|
|
|
|
# Git clone
|
|
apk add --no-cache git openssh rclone
|
|
mkdir -p ~/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
|
chmod 600 ~/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
|
chmod 600 ~/.ssh/config
|
|
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
WORKSPACE="/tmp/workspace"
|
|
git clone --no-checkout --filter=blob:none "$REPO" "$WORKSPACE"
|
|
cd "$WORKSPACE"
|
|
git checkout "$SHA"
|
|
|
|
cd "$WORKSPACE/web-dashboard"
|
|
echo "=== Building web dashboard ==="
|
|
npm ci
|
|
npm run build
|
|
|
|
echo "=== Uploading to MinIO ==="
|
|
rclone copy dist/ :s3:foxhunt-binaries/web-dashboard/ \
|
|
--s3-provider=Minio \
|
|
--s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
|
|
--s3-access-key-id="${MINIO_ACCESS_KEY}" \
|
|
--s3-secret-access-key="${MINIO_SECRET_KEY}" \
|
|
--s3-no-check-bucket
|
|
|
|
echo "=== Web dashboard build + upload done ==="
|
|
|
|
# ── terragrunt-apply: apply infra changes on main push ──
|
|
- name: terragrunt-apply
|
|
inputs:
|
|
parameters:
|
|
- name: commit-sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
serviceAccountName: argo-workflow
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: GITLAB_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
- name: TF_HTTP_USERNAME
|
|
value: root
|
|
- name: TF_HTTP_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
- name: SCW_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: scaleway-credentials
|
|
key: access-key
|
|
- name: SCW_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: scaleway-credentials
|
|
key: secret-key
|
|
- name: SCW_DEFAULT_PROJECT_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: scaleway-credentials
|
|
key: project-id
|
|
- name: GITLAB_TF_STATE_URL
|
|
value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
|
resources:
|
|
requests:
|
|
cpu: 200m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: "1"
|
|
memory: 512Mi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
args:
|
|
- |
|
|
set -e
|
|
|
|
# Clone repo
|
|
mkdir -p /root/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
|
|
chmod 600 /root/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
|
chmod 600 /root/.ssh/config
|
|
|
|
SHA="{{inputs.parameters.commit-sha}}"
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
git clone --no-checkout --filter=blob:none "$REPO" /workspace/src
|
|
cd /workspace/src
|
|
git checkout "$SHA"
|
|
echo "Checked out $(git rev-parse --short HEAD)"
|
|
|
|
tofu --version
|
|
terragrunt --version
|
|
|
|
# Apply each terragrunt module
|
|
for module in block-storage dns public-gateway kapsule; do
|
|
MODULE_DIR="infra/live/production/${module}"
|
|
if [ -d "$MODULE_DIR" ]; then
|
|
echo "=== Terragrunt plan: ${module} ==="
|
|
cd "/workspace/src/${MODULE_DIR}"
|
|
terragrunt init --non-interactive -reconfigure
|
|
OUTPUT=$(terragrunt plan --non-interactive -detailed-exitcode 2>&1) || EXITCODE=$?
|
|
EXITCODE=${EXITCODE:-0}
|
|
|
|
if [ "$EXITCODE" -eq 2 ]; then
|
|
echo "=== Terragrunt plan output: ${module} ==="
|
|
echo "$OUTPUT"
|
|
echo "=== Terragrunt apply: ${module} ==="
|
|
terragrunt apply --non-interactive -auto-approve
|
|
elif [ "$EXITCODE" -eq 0 ]; then
|
|
echo "=== No changes for ${module} ==="
|
|
else
|
|
echo "=== ERROR planning ${module} ==="
|
|
echo "$OUTPUT"
|
|
exit 1
|
|
fi
|
|
cd /workspace/src
|
|
fi
|
|
done
|
|
|
|
echo "=== Terragrunt apply complete ==="
|
|
|
|
# ── apply-argo-templates: self-apply WorkflowTemplates, EventSources, Sensors ──
|
|
- name: apply-argo-templates
|
|
inputs:
|
|
parameters:
|
|
- name: commit-sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
serviceAccountName: argo-workflow
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
|
|
command: ["/bin/sh", "-c"]
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
args:
|
|
- |
|
|
set -e
|
|
|
|
# Clone repo at commit
|
|
mkdir -p /root/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
|
|
chmod 600 /root/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
|
chmod 600 /root/.ssh/config
|
|
|
|
SHA="{{inputs.parameters.commit-sha}}"
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
git clone --no-checkout --filter=blob:none "$REPO" /workspace/src
|
|
cd /workspace/src
|
|
git checkout "$SHA"
|
|
echo "Checked out $(git rev-parse --short HEAD)"
|
|
|
|
# Apply all Argo WorkflowTemplates
|
|
for f in infra/k8s/argo/*-template.yaml; do
|
|
if [ -f "$f" ]; then
|
|
echo "=== Applying $(basename $f) ==="
|
|
kubectl -n foxhunt apply -f "$f"
|
|
fi
|
|
done
|
|
|
|
# Apply EventSources and Sensors
|
|
for f in infra/k8s/argo/events/*.yaml; do
|
|
if [ -f "$f" ]; then
|
|
echo "=== Applying $(basename $f) ==="
|
|
kubectl -n foxhunt apply -f "$f"
|
|
fi
|
|
done
|
|
|
|
# Apply RBAC and NetworkPolicies
|
|
kubectl -n foxhunt apply -f infra/k8s/argo/ci-deploy-rbac.yaml
|
|
kubectl -n foxhunt apply -f infra/k8s/argo/argo-workflow-netpol.yaml
|
|
|
|
echo "=== Argo templates applied ==="
|
|
|
|
# ── submit-gpu-test: launch GPU test workflow as a child workflow ──
|
|
- name: submit-gpu-test
|
|
# Argo executor (main + wait) needs 256Mi to track large child workflow status JSON.
|
|
# Default 64Mi causes OOMKilled when child workflow has many parameters.
|
|
podSpecPatch: '{"containers":[{"name":"main","resources":{"requests":{"memory":"128Mi"},"limits":{"memory":"256Mi"}}},{"name":"wait","resources":{"requests":{"memory":"128Mi"},"limits":{"memory":"256Mi"}}}]}'
|
|
resource:
|
|
action: create
|
|
setOwnerReference: true
|
|
successCondition: status.phase == Succeeded
|
|
failureCondition: status.phase in (Failed, Error)
|
|
manifest: |
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Workflow
|
|
metadata:
|
|
generateName: gpu-test-on-push-
|
|
namespace: foxhunt
|
|
spec:
|
|
workflowTemplateRef:
|
|
name: gpu-test-pipeline
|
|
arguments:
|
|
parameters:
|
|
- name: commit-ref
|
|
value: "{{workflow.parameters.commit-sha}}"
|
|
- name: models
|
|
value: "dqn,ppo,tft"
|
|
|
|
# ── notify-result: post workflow outcome to Mattermost ──
|
|
- name: notify-result
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: WEBHOOK_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: notification-webhook
|
|
key: webhook-url
|
|
optional: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
args:
|
|
- |
|
|
STATUS="{{workflow.status}}"
|
|
NAME="{{workflow.name}}"
|
|
|
|
if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then
|
|
echo "No webhook configured, skipping notification"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$STATUS" = "Succeeded" ]; then
|
|
EMOJI=":white_check_mark:"
|
|
else
|
|
EMOJI=":x:"
|
|
fi
|
|
|
|
PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **${NAME}** — ${STATUS} ({{workflow.duration}}s)\"}"
|
|
|
|
curl -sf -X POST -H 'Content-Type: application/json' \
|
|
-d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed"
|