From 34df9adc8ffe53d7720b07b0f238c9ba0e8f2bfa Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 2 Apr 2026 16:03:30 +0200 Subject: [PATCH] fix: force line-buffered stdout in Argo training pod MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rust tracing with JSON subscriber in a container without TTY uses fully-buffered stdout — logs only flush on buffer full or process exit. This made H100 training appear stuck for 28+ minutes with no output. stdbuf -oL forces line buffering so epoch logs appear immediately. Co-Authored-By: Claude Opus 4.6 (1M context) --- infra/k8s/argo/compile-and-train-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml index 91df3217d..f75e679bf 100644 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ b/infra/k8s/argo/compile-and-train-template.yaml @@ -497,7 +497,7 @@ spec: echo "=== Training best: $MODEL ({{workflow.parameters.train-epochs}} epochs) ===" - ${BINARY} \ + stdbuf -oL ${BINARY} \ --model "$MODEL" \ --symbol {{workflow.parameters.symbol}} \ --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \