8d89fe80ff
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
70 lines
2.0 KiB
Bash
Executable File
70 lines
2.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# TFT Training Restart Script (Agent 117)
|
|
# After Agent 112 TLOB Decoder fix
|
|
|
|
set -e
|
|
|
|
echo "=== TFT Training Restart ==="
|
|
echo "Timestamp: $(date)"
|
|
echo "GPU Status:"
|
|
nvidia-smi --query-gpu=name,memory.total,memory.used,temperature.gpu,utilization.gpu --format=csv,noheader
|
|
echo ""
|
|
|
|
echo "=== Pre-flight Checks ==="
|
|
echo "Checking for running training processes..."
|
|
if ps aux | grep -E "(train_tft|train_dqn|train_ppo)" | grep -v grep; then
|
|
echo "WARNING: Found running training processes!"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Checking compilation status..."
|
|
cargo check -p ml 2>&1 | grep -E "(error|Finished)" | tail -5
|
|
|
|
echo "Checking output directory..."
|
|
mkdir -p ml/trained_models/production/tft_real_data
|
|
ls -la ml/trained_models/production/tft_real_data/
|
|
|
|
echo "Checking training data..."
|
|
echo "Available DBN files: $(find test_data/real/databento/ml_training -name '*.dbn' | wc -l)"
|
|
echo "Total size: $(du -sh test_data/real/databento/ml_training/)"
|
|
|
|
echo ""
|
|
echo "=== Starting TFT Training ==="
|
|
echo "Configuration:"
|
|
echo " - Epochs: 200"
|
|
echo " - Learning Rate: 0.001"
|
|
echo " - Batch Size: 32"
|
|
echo " - Lookback Window: 60"
|
|
echo " - Forecast Horizon: 10"
|
|
echo " - GPU: Enabled (CUDA_VISIBLE_DEVICES=0)"
|
|
echo " - Output: ml/trained_models/production/tft_real_data"
|
|
echo ""
|
|
|
|
# Log file
|
|
LOG_FILE="tft_training_$(date +%Y%m%d_%H%M%S).log"
|
|
echo "Logging to: $LOG_FILE"
|
|
echo ""
|
|
|
|
# Start training
|
|
RUST_LOG=info \
|
|
RUST_BACKTRACE=1 \
|
|
CUDA_VISIBLE_DEVICES=0 \
|
|
cargo run --release -p ml --example train_tft_dbn -- \
|
|
--data-path test_data/real/databento/ml_training \
|
|
--epochs 200 \
|
|
--learning-rate 0.001 \
|
|
--batch-size 32 \
|
|
--lookback-window 60 \
|
|
--forecast-horizon 10 \
|
|
--use-gpu \
|
|
--output-dir ml/trained_models/production/tft_real_data \
|
|
--early-stopping-patience 20 \
|
|
--early-stopping-threshold 0.0001 \
|
|
--verbose 2>&1 | tee "$LOG_FILE"
|
|
|
|
echo ""
|
|
echo "=== Training Complete ==="
|
|
echo "Final GPU status:"
|
|
nvidia-smi --query-gpu=memory.used,memory.total,temperature.gpu --format=csv,noheader
|
|
echo "Log saved to: $LOG_FILE"
|