2606506cd8
- argo-train.sh: --profile flag forces multi-seed render path so the nsys wrapper + foxhunt-training-artifacts upload step are visible in --dry-run YAML without cluster contact (test surface). - train-multi-seed-template.yaml: new `profile` parameter (default "false") gates the per-(seed, fold) `nsys profile --capture-range=cudaProfilerApi` wrapper and the `mc cp` upload to foxhunt-training-artifacts/profiles/<sha>/. mc binary fetched on-demand (ci-builder image lacks it). MinIO creds optional — upload warn-skips if absent. - Dockerfile.foxhunt-training-runtime: install nsight-systems-cli unpinned (pinning the stale 2024.4.1.61-1 from earlier plans breaks builds when apt index advances). - minio.yaml: add foxhunt-training-artifacts bucket to minio-init. - compare-nsys-profiles.py: V0 regression detector — compares cuda_gpu_kern_sum total_ns / epoch_count between two profiles; exits 1 on >20% slowdown. NVTX per-epoch ranges deferred to T5. - tests/test_nsys_harness.sh: dry-run grep test — verifies both required strings appear when --profile is set, and that the default (no --profile) path keeps profile=false in the rendered template. - dqn-wire-up-audit.md: Plan 5 Task 3 row added documenting the harness + the baseline-capture deferral to T5. Backward compat: test_multi_seed_harness.sh from P5T1 still PASS.
189 lines
5.2 KiB
YAML
189 lines
5.2 KiB
YAML
# MinIO — in-cluster S3 for compiled binaries and trained models
|
|
# Replaces Scaleway public S3 to keep proprietary code/IP in-cluster
|
|
# Plain HTTP only — internal cluster traffic, no TLS needed
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: minio-credentials
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
type: Opaque
|
|
stringData:
|
|
root-user: foxhunt-minio
|
|
root-password: CHANGE_ME_BEFORE_DEPLOY
|
|
# rclone-compatible keys (same values, different key names for initContainers)
|
|
access-key: foxhunt-minio
|
|
secret-key: CHANGE_ME_BEFORE_DEPLOY
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: minio-data-new
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: scw-bssd
|
|
resources:
|
|
requests:
|
|
storage: 150Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: minio
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate # single-node MinIO — no rolling update
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: minio
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
containers:
|
|
- name: minio
|
|
image: minio/minio:latest
|
|
args: ["server", "/data", "--console-address", ":9001"]
|
|
ports:
|
|
- containerPort: 9000
|
|
name: api
|
|
- containerPort: 9001
|
|
name: console
|
|
env:
|
|
- name: MINIO_ROOT_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-user
|
|
- name: MINIO_ROOT_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-password
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /minio/health/live
|
|
port: 9000
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /minio/health/live
|
|
port: 9000
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 15
|
|
failureThreshold: 5
|
|
resources:
|
|
requests:
|
|
cpu: 200m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: minio-data-new
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: minio
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
selector:
|
|
app.kubernetes.io/name: minio
|
|
ports:
|
|
- port: 9000
|
|
targetPort: 9000
|
|
name: api
|
|
- port: 9001
|
|
targetPort: 9001
|
|
name: console
|
|
---
|
|
# Create buckets on first deploy
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: minio-init-buckets
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
backoffLimit: 5
|
|
ttlSecondsAfterFinished: 300
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: minio-init
|
|
spec:
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: mc
|
|
image: minio/mc:latest
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
echo "Waiting for MinIO to be ready..."
|
|
until mc alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do
|
|
echo "MinIO not ready, retrying in 3s..."
|
|
sleep 3
|
|
done
|
|
echo "Creating buckets..."
|
|
mc mb --ignore-existing foxhunt/foxhunt-binaries
|
|
mc mb --ignore-existing foxhunt/foxhunt-models
|
|
mc mb --ignore-existing foxhunt/foxhunt-training-results
|
|
mc mb --ignore-existing foxhunt/argo-logs
|
|
mc mb --ignore-existing foxhunt/foxhunt-sccache
|
|
mc mb --ignore-existing foxhunt/foxhunt-training-data
|
|
# Plan 5 Task 3 (A.4.1): nsys profile artefacts bucket.
|
|
# Stores .nsys-rep files keyed by commit SHA under
|
|
# profiles/<short-sha>/profile-seed*-fold*-*.nsys-rep.
|
|
mc mb --ignore-existing foxhunt/foxhunt-training-artifacts
|
|
echo "Buckets created:"
|
|
mc ls foxhunt/
|
|
env:
|
|
- name: MINIO_ROOT_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-user
|
|
- name: MINIO_ROOT_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-password
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 64Mi
|