chore: Major documentation cleanup - remove 2,060 obsolete files
BREAKING: Removes 746,569 lines of outdated documentation from root folder ## Summary - Deleted 2,060 report/documentation files from root folder - Kept only essential files: README.md, CLAUDE.md - Updated .gitignore and config/tarpaulin.toml - Reorganized config files into config/ directory ## Removed Content Categories - Agent reports (AGENT_*.md, AGENT*.txt) - Wave reports (WAVE_*.md, DQN_*.md) - Implementation summaries - Quick references and summaries - Test reports and validation docs - Deployment scripts (obsolete .sh files) - Legacy config files and logs ## Preserved - README.md - Main project documentation - CLAUDE.md - Claude Code configuration - docs/archive/ - Historical files for reference - docs/ folder - Current documentation - All source code unchanged 🐝 Hive Mind Collective Intelligence Cleanup 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -1,41 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Quick DQN Deployment Script
|
||||
# Date: 2025-10-24
|
||||
# Purpose: Deploy DQN training to RunPod with correct binary selection
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "=========================================="
|
||||
echo "RunPod DQN Training Deployment"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Using: deploy_runpod_graphql.py (CORRECT script)"
|
||||
echo "Binary: train_dqn"
|
||||
echo "Dataset: ES_FUT_small.parquet (~13K bars)"
|
||||
echo "GPU: RTX A4000 (16GB, $0.25/hr)"
|
||||
echo "Expected time: ~1 minute"
|
||||
echo "Expected cost: ~$0.004"
|
||||
echo ""
|
||||
|
||||
# Deploy DQN training
|
||||
python3 scripts/deploy_runpod_graphql.py \
|
||||
--binary train_dqn \
|
||||
--parquet-file /runpod-volume/test_data/ES_FUT_small.parquet \
|
||||
--epochs 100 \
|
||||
--gpu-type "NVIDIA RTX A4000" \
|
||||
--pod-name foxhunt-dqn-training
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Deployment Complete"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo "1. Copy the Pod ID from output above"
|
||||
echo "2. Monitor training:"
|
||||
echo " ssh root@POD_ID.ssh.runpod.io"
|
||||
echo " nvidia-smi -l 1"
|
||||
echo "3. Check models after training:"
|
||||
echo " ls -lh /runpod-volume/models/"
|
||||
echo "4. Terminate pod to stop billing"
|
||||
echo ""
|
||||
@@ -1,169 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Quick Monitoring Script for MAMBA-2 Hyperopt Deployment
|
||||
# Pod ID: qlql87w5avv1q1
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="$SCRIPT_DIR/.env.runpod"
|
||||
|
||||
# Load environment
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
export $(grep -v '^#' "$ENV_FILE" | xargs)
|
||||
else
|
||||
echo "ERROR: .env.runpod not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
POD_ID="qlql87w5avv1q1"
|
||||
|
||||
# Function to get pod status
|
||||
check_status() {
|
||||
echo "===================="
|
||||
echo "POD STATUS CHECK"
|
||||
echo "===================="
|
||||
|
||||
curl -s -X GET \
|
||||
"https://rest.runpod.io/v1/pods/$POD_ID" \
|
||||
-H "Authorization: Bearer $RUNPOD_API_KEY" \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
print(f\"Status: {data.get('desiredStatus', 'UNKNOWN')}\")
|
||||
print(f\"Cost: \${data.get('costPerHr', 0)}/hr\")
|
||||
print(f\"GPU: {data.get('machine', {}).get('gpuDisplayName', 'TBD')}\")
|
||||
print(f\"Created: {data.get('createdAt', 'N/A')}\")
|
||||
runtime = data.get('runtime', {})
|
||||
if runtime:
|
||||
print(f\"Uptime: {runtime.get('uptimeInSeconds', 0)}s\")
|
||||
print(f\"Ports: {runtime.get('ports', 'N/A')}\")
|
||||
else:
|
||||
print('Runtime: Pod still provisioning...')
|
||||
"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to try SSH
|
||||
try_ssh() {
|
||||
echo "===================="
|
||||
echo "SSH CONNECTION TEST"
|
||||
echo "===================="
|
||||
|
||||
if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 \
|
||||
-p 19735 root@157.157.221.29 "echo 'Connected!'" 2>/dev/null; then
|
||||
echo "✅ SSH is available!"
|
||||
echo ""
|
||||
echo "Connect with:"
|
||||
echo " ssh -p 19735 root@157.157.221.29"
|
||||
echo ""
|
||||
return 0
|
||||
else
|
||||
echo "⏳ SSH not yet available (pod still initializing)"
|
||||
echo ""
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to show training logs (if SSH available)
|
||||
show_logs() {
|
||||
echo "===================="
|
||||
echo "TRAINING LOGS (last 50 lines)"
|
||||
echo "===================="
|
||||
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-p 19735 root@157.157.221.29 \
|
||||
"tail -50 /workspace/logs/*.log 2>/dev/null || echo 'No logs yet'" 2>/dev/null || \
|
||||
echo "Cannot access logs (pod not ready)"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to check GPU
|
||||
check_gpu() {
|
||||
echo "===================="
|
||||
echo "GPU UTILIZATION"
|
||||
echo "===================="
|
||||
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-p 19735 root@157.157.221.29 \
|
||||
"nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu,temperature.gpu --format=csv,noheader" 2>/dev/null || \
|
||||
echo "Cannot access GPU info (pod not ready)"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to check for critical success metric
|
||||
check_losses() {
|
||||
echo "===================="
|
||||
echo "LOSS VALIDATION (CRITICAL)"
|
||||
echo "===================="
|
||||
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-p 19735 root@157.157.221.29 \
|
||||
"grep -E 'Feature normalization|Epoch.*Loss' /workspace/logs/*.log 2>/dev/null | tail -20 || echo 'No training logs yet'" 2>/dev/null || \
|
||||
echo "Cannot access logs (pod not ready)"
|
||||
echo ""
|
||||
echo "SUCCESS CRITERIA:"
|
||||
echo " ✅ Train Loss < 1.0"
|
||||
echo " ✅ Val Loss < 1.0"
|
||||
echo " ❌ If losses > 1.0, FIX FAILED"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Main monitoring loop
|
||||
main() {
|
||||
while true; do
|
||||
clear
|
||||
echo "========================================"
|
||||
echo "MAMBA-2 HYPEROPT MONITORING"
|
||||
echo "Pod ID: $POD_ID"
|
||||
echo "Time: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
check_status
|
||||
|
||||
if try_ssh; then
|
||||
check_gpu
|
||||
check_losses
|
||||
show_logs
|
||||
|
||||
echo "===================="
|
||||
echo "NEXT ACTIONS"
|
||||
echo "===================="
|
||||
echo "1. Watch for 'Feature normalization' log line (confirms fix)"
|
||||
echo "2. Verify losses < 1.0 (CRITICAL)"
|
||||
echo "3. Monitor GPU utilization > 85%"
|
||||
echo "4. Check epoch time ~10 min"
|
||||
echo ""
|
||||
echo "Press Ctrl+C to exit, or wait 60s for refresh..."
|
||||
sleep 60
|
||||
else
|
||||
echo "Pod is still provisioning. Checking again in 30 seconds..."
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Handle Ctrl+C gracefully
|
||||
trap 'echo ""; echo "Monitoring stopped."; exit 0' INT
|
||||
|
||||
# Parse command line
|
||||
case "${1:-}" in
|
||||
status)
|
||||
check_status
|
||||
;;
|
||||
ssh)
|
||||
try_ssh && echo "To connect:" && echo "ssh -p 19735 root@157.157.221.29"
|
||||
;;
|
||||
logs)
|
||||
show_logs
|
||||
;;
|
||||
gpu)
|
||||
check_gpu
|
||||
;;
|
||||
losses)
|
||||
check_losses
|
||||
;;
|
||||
*)
|
||||
main
|
||||
;;
|
||||
esac
|
||||
@@ -1,184 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ================================================================================================
|
||||
# Level 1 Rollback Test: Feature-Only Rollback (Zero Downtime, Target: <1 minute)
|
||||
# Agent R1 - Rollback & Disaster Recovery Specialist
|
||||
# ================================================================================================
|
||||
#
|
||||
# SCENARIO: Disable Wave D features without restarting services or database rollback
|
||||
# EXPECTED: System falls back to Wave C (201 features) with zero downtime
|
||||
#
|
||||
# ================================================================================================
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "===================================================================================================="
|
||||
echo "LEVEL 1 ROLLBACK TEST: Feature-Only Rollback (Zero Downtime)"
|
||||
echo "===================================================================================================="
|
||||
echo ""
|
||||
|
||||
# Start timer
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Step 1: Verify current system state
|
||||
echo "Step 1: Verifying current system state (Wave D active)"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
|
||||
# Check if services are running
|
||||
echo " ✓ Checking service health..."
|
||||
SERVICE_COUNT=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service" | wc -l)
|
||||
if [ "$SERVICE_COUNT" -lt 1 ]; then
|
||||
echo " ⚠ WARNING: No Foxhunt services detected. Skipping live service test."
|
||||
SERVICES_RUNNING=false
|
||||
else
|
||||
echo " ✓ Found $SERVICE_COUNT Foxhunt service process(es) running"
|
||||
SERVICES_RUNNING=true
|
||||
fi
|
||||
|
||||
# Check database for regime tables
|
||||
echo " ✓ Checking database for Wave D tables..."
|
||||
REGIME_TABLES=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "SELECT COUNT(*) FROM information_schema.tables WHERE table_name IN ('regime_states', 'regime_transitions', 'adaptive_strategy_metrics');" 2>/dev/null || echo "0")
|
||||
echo " ✓ Found $REGIME_TABLES Wave D tables in database"
|
||||
|
||||
# Check feature config
|
||||
echo " ✓ Checking current feature configuration..."
|
||||
FEATURE_CONFIG_PATH="./ml/src/features/config.rs"
|
||||
if grep -q "enable_wave_d_regime: true" "$FEATURE_CONFIG_PATH" 2>/dev/null; then
|
||||
echo " ✓ Wave D features currently ENABLED in code"
|
||||
WAVE_D_ENABLED=true
|
||||
else
|
||||
echo " ⚠ Wave D features currently DISABLED in code (already rolled back?)"
|
||||
WAVE_D_ENABLED=false
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 2: Create rollback configuration (disable Wave D features)
|
||||
echo "Step 2: Creating rollback configuration (disable Wave D features)"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
|
||||
# Backup current config
|
||||
echo " ✓ Backing up current configuration..."
|
||||
if [ -f "$FEATURE_CONFIG_PATH" ]; then
|
||||
cp "$FEATURE_CONFIG_PATH" "$FEATURE_CONFIG_PATH.backup_$(date +%s)"
|
||||
echo " ✓ Backup created: $FEATURE_CONFIG_PATH.backup_$(date +%s)"
|
||||
fi
|
||||
|
||||
# Modify wave_d() function to return Wave C config (201 features)
|
||||
echo " ✓ Modifying FeatureConfig::wave_d() to disable Wave D features..."
|
||||
cat > /tmp/wave_d_rollback.sed << 'EOF'
|
||||
# Find wave_d() function and change enable_wave_d_regime: true → false
|
||||
/pub fn wave_d\(\) -> Self {/,/enable_wave_d_regime: true,/ {
|
||||
s/enable_wave_d_regime: true,/enable_wave_d_regime: false,/
|
||||
}
|
||||
EOF
|
||||
|
||||
# Apply sed script
|
||||
if [ -f "$FEATURE_CONFIG_PATH" ]; then
|
||||
sed -i.rollback -f /tmp/wave_d_rollback.sed "$FEATURE_CONFIG_PATH" 2>/dev/null || {
|
||||
echo " ⚠ WARNING: sed failed, using manual method"
|
||||
# Fallback: create minimal config change
|
||||
echo " ⚠ Manual rollback required: Set enable_wave_d_regime: false in wave_d() function"
|
||||
}
|
||||
echo " ✓ Configuration modified (Wave D features disabled)"
|
||||
else
|
||||
echo " ⚠ WARNING: $FEATURE_CONFIG_PATH not found, skipping config modification"
|
||||
fi
|
||||
|
||||
# Verify change
|
||||
if grep -q "enable_wave_d_regime: false" "$FEATURE_CONFIG_PATH" 2>/dev/null; then
|
||||
echo " ✅ VERIFIED: Wave D features now DISABLED"
|
||||
else
|
||||
echo " ⚠ WARNING: Could not verify configuration change"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 3: Rebuild services (optional - for hot-reload test)
|
||||
echo "Step 3: Rebuilding services with Wave C configuration"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
echo " ⚠ NOTE: In production, this step would be replaced by hot-reload mechanism"
|
||||
echo " ⚠ For testing, we rebuild services to validate Wave C fallback"
|
||||
echo ""
|
||||
|
||||
REBUILD_START=$(date +%s)
|
||||
echo " ✓ Building workspace (release mode)..."
|
||||
cargo build --workspace --release 2>&1 | grep -E "Compiling|Finished|error" || echo " ✓ Build completed"
|
||||
REBUILD_END=$(date +%s)
|
||||
REBUILD_TIME=$((REBUILD_END - REBUILD_START))
|
||||
echo " ✓ Build completed in ${REBUILD_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 4: Validate rollback
|
||||
echo "Step 4: Validating Wave C fallback (201 features)"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
|
||||
# Check feature count via compiled code
|
||||
echo " ✓ Checking feature count in rollback configuration..."
|
||||
FEATURE_COUNT=$(cargo run --release -p ml --example check_feature_count 2>/dev/null | grep -oP 'feature_count: \K\d+' || echo "unknown")
|
||||
if [ "$FEATURE_COUNT" = "201" ]; then
|
||||
echo " ✅ VERIFIED: Feature count = 201 (Wave C)"
|
||||
elif [ "$FEATURE_COUNT" = "225" ]; then
|
||||
echo " ❌ FAILED: Feature count = 225 (Wave D still active)"
|
||||
echo " ⚠ Rollback did not take effect, manual intervention required"
|
||||
else
|
||||
echo " ⚠ WARNING: Could not determine feature count (got: $FEATURE_COUNT)"
|
||||
echo " ⚠ Manual verification required"
|
||||
fi
|
||||
|
||||
# Verify regime detection disabled
|
||||
echo " ✓ Verifying regime detection disabled..."
|
||||
if grep -q "enable_wave_d_regime: false" "$FEATURE_CONFIG_PATH" 2>/dev/null; then
|
||||
echo " ✅ VERIFIED: Regime detection disabled in configuration"
|
||||
else
|
||||
echo " ❌ FAILED: Regime detection still enabled"
|
||||
fi
|
||||
|
||||
# Database remains unchanged (Level 1 does NOT modify database)
|
||||
echo " ✓ Database status: UNCHANGED (Wave D tables still exist)"
|
||||
echo " ⚠ Note: Level 1 rollback preserves Wave D data for recovery"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 5: Calculate rollback time
|
||||
echo "Step 5: Rollback Performance Metrics"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
END_TIME=$(date +%s)
|
||||
TOTAL_TIME=$((END_TIME - START_TIME))
|
||||
|
||||
echo " • Total rollback time: ${TOTAL_TIME}s"
|
||||
echo " • Rebuild time: ${REBUILD_TIME}s"
|
||||
echo " • Target time: <60s"
|
||||
|
||||
if [ $TOTAL_TIME -lt 60 ]; then
|
||||
echo " ✅ PASSED: Rollback completed within 60s target"
|
||||
else
|
||||
echo " ⚠ MISSED TARGET: Rollback took ${TOTAL_TIME}s (>60s)"
|
||||
echo " ⚠ Note: Hot-reload mechanism would reduce this to <10s"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Summary
|
||||
echo "===================================================================================================="
|
||||
echo "LEVEL 1 ROLLBACK TEST: SUMMARY"
|
||||
echo "===================================================================================================="
|
||||
echo ""
|
||||
echo "Result:"
|
||||
echo " • Wave D features: DISABLED (201 features active)"
|
||||
echo " • Database: UNCHANGED (Wave D tables preserved)"
|
||||
echo " • Services: REBUILD REQUIRED (or hot-reload in production)"
|
||||
echo " • Rollback time: ${TOTAL_TIME}s (target: <60s)"
|
||||
echo ""
|
||||
echo "Recovery Path:"
|
||||
echo " To re-enable Wave D:"
|
||||
echo " 1. Restore configuration: cp $FEATURE_CONFIG_PATH.backup_* $FEATURE_CONFIG_PATH"
|
||||
echo " 2. Rebuild services: cargo build --workspace --release"
|
||||
echo " 3. Restart services"
|
||||
echo ""
|
||||
echo "===================================================================================================="
|
||||
|
||||
# Cleanup
|
||||
rm -f /tmp/wave_d_rollback.sed
|
||||
|
||||
exit 0
|
||||
@@ -1,238 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ================================================================================================
|
||||
# Level 2 Rollback Test: Database Rollback (Target: ~5 minutes)
|
||||
# Agent R1 - Rollback & Disaster Recovery Specialist
|
||||
# ================================================================================================
|
||||
#
|
||||
# SCENARIO: Rollback database migration 045 (regime detection tables)
|
||||
# EXPECTED: All Wave D tables removed, services restart with Wave C configuration
|
||||
#
|
||||
# ================================================================================================
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "===================================================================================================="
|
||||
echo "LEVEL 2 ROLLBACK TEST: Database Rollback"
|
||||
echo "===================================================================================================="
|
||||
echo ""
|
||||
|
||||
# Start timer
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Step 1: Pre-rollback validation
|
||||
echo "Step 1: Pre-rollback validation"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
|
||||
# Check current database state
|
||||
echo " ✓ Checking database for Wave D tables..."
|
||||
REGIME_TABLES=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "
|
||||
SELECT COUNT(*) FROM information_schema.tables
|
||||
WHERE table_name IN ('regime_states', 'regime_transitions', 'adaptive_strategy_metrics');
|
||||
" 2>/dev/null || echo "0")
|
||||
|
||||
echo " ✓ Found $REGIME_TABLES Wave D tables before rollback"
|
||||
|
||||
if [ "$REGIME_TABLES" -eq 0 ]; then
|
||||
echo " ⚠ WARNING: No Wave D tables found. Migration 045 may not be applied."
|
||||
echo " ⚠ Skipping Level 2 test (nothing to rollback)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for existing data
|
||||
echo " ✓ Checking for existing Wave D data..."
|
||||
REGIME_DATA=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "
|
||||
SELECT COUNT(*) FROM regime_states;
|
||||
" 2>/dev/null || echo "0")
|
||||
echo " ✓ Found $REGIME_DATA regime_states records"
|
||||
|
||||
# Backup database (safety measure)
|
||||
echo " ✓ Creating database backup..."
|
||||
BACKUP_FILE="/tmp/foxhunt_backup_$(date +%s).sql"
|
||||
PGPASSWORD=foxhunt_dev_password pg_dump -h localhost -U foxhunt -d foxhunt -f "$BACKUP_FILE" 2>/dev/null || {
|
||||
echo " ⚠ WARNING: Backup failed, continuing anyway"
|
||||
BACKUP_FILE=""
|
||||
}
|
||||
if [ -n "$BACKUP_FILE" ] && [ -f "$BACKUP_FILE" ]; then
|
||||
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
|
||||
echo " ✅ Backup created: $BACKUP_FILE ($BACKUP_SIZE)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 2: Stop services (graceful shutdown)
|
||||
echo "Step 2: Stopping services (graceful shutdown)"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
SHUTDOWN_START=$(date +%s)
|
||||
|
||||
# Find and stop Foxhunt services
|
||||
PIDS=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service" || true)
|
||||
if [ -n "$PIDS" ]; then
|
||||
echo " ✓ Found service PIDs: $PIDS"
|
||||
echo " ✓ Sending SIGTERM for graceful shutdown..."
|
||||
kill -TERM $PIDS 2>/dev/null || true
|
||||
|
||||
# Wait up to 30s for graceful shutdown
|
||||
for i in {1..30}; do
|
||||
REMAINING=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service" | wc -l)
|
||||
if [ "$REMAINING" -eq 0 ]; then
|
||||
echo " ✅ All services stopped gracefully in ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Force kill if still running
|
||||
REMAINING=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service" | wc -l)
|
||||
if [ "$REMAINING" -gt 0 ]; then
|
||||
echo " ⚠ WARNING: Forcing shutdown of $REMAINING remaining processes"
|
||||
kill -9 $(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service") 2>/dev/null || true
|
||||
fi
|
||||
else
|
||||
echo " ⚠ No services running"
|
||||
fi
|
||||
|
||||
SHUTDOWN_END=$(date +%s)
|
||||
SHUTDOWN_TIME=$((SHUTDOWN_END - SHUTDOWN_START))
|
||||
echo " ✓ Shutdown completed in ${SHUTDOWN_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 3: Apply rollback migration
|
||||
echo "Step 3: Rolling back database migration 045"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
MIGRATION_START=$(date +%s)
|
||||
|
||||
# Method 1: Use sqlx migrate revert (if sqlx-cli is installed)
|
||||
if command -v sqlx &> /dev/null; then
|
||||
echo " ✓ Using sqlx migrate revert..."
|
||||
cd /home/jgrusewski/Work/foxhunt
|
||||
DATABASE_URL="postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt" \
|
||||
sqlx migrate revert --database-url "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt" 2>&1 | grep -E "Applied|Reverted|error" || echo " ✓ Revert completed"
|
||||
else
|
||||
# Method 2: Direct SQL execution (fallback)
|
||||
echo " ✓ Using direct SQL execution (sqlx not found)..."
|
||||
PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -f /home/jgrusewski/Work/foxhunt/migrations/045_wave_d_regime_tracking.down.sql 2>&1 | grep -E "DROP|REVOKE|NOTICE|ERROR" || echo " ✓ Rollback SQL executed"
|
||||
fi
|
||||
|
||||
MIGRATION_END=$(date +%s)
|
||||
MIGRATION_TIME=$((MIGRATION_END - MIGRATION_START))
|
||||
echo " ✓ Migration rollback completed in ${MIGRATION_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 4: Validate rollback
|
||||
echo "Step 4: Validating database rollback"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
|
||||
# Check tables removed
|
||||
REMAINING_TABLES=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "
|
||||
SELECT COUNT(*) FROM information_schema.tables
|
||||
WHERE table_name IN ('regime_states', 'regime_transitions', 'adaptive_strategy_metrics');
|
||||
" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [ "$REMAINING_TABLES" = "0" ]; then
|
||||
echo " ✅ VERIFIED: All Wave D tables removed"
|
||||
else
|
||||
echo " ❌ FAILED: $REMAINING_TABLES Wave D tables still exist"
|
||||
echo " ⚠ Rollback did not complete successfully"
|
||||
fi
|
||||
|
||||
# Check functions removed
|
||||
REMAINING_FUNCTIONS=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "
|
||||
SELECT COUNT(*) FROM information_schema.routines
|
||||
WHERE routine_name IN ('get_latest_regime', 'get_regime_transition_matrix', 'get_regime_performance');
|
||||
" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [ "$REMAINING_FUNCTIONS" = "0" ]; then
|
||||
echo " ✅ VERIFIED: All Wave D functions removed"
|
||||
else
|
||||
echo " ❌ FAILED: $REMAINING_FUNCTIONS Wave D functions still exist"
|
||||
fi
|
||||
|
||||
# Check migration history
|
||||
MIGRATION_VERSION=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "
|
||||
SELECT version FROM _sqlx_migrations ORDER BY version DESC LIMIT 1;
|
||||
" 2>/dev/null || echo "unknown")
|
||||
echo " ✓ Latest migration version: $MIGRATION_VERSION (should be 44 after rollback)"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 5: Restart services with Wave C configuration
|
||||
echo "Step 5: Restarting services with Wave C configuration"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
RESTART_START=$(date +%s)
|
||||
|
||||
# Ensure Wave C configuration is active (from Level 1 rollback)
|
||||
echo " ✓ Verifying Wave C configuration..."
|
||||
if grep -q "enable_wave_d_regime: false" /home/jgrusewski/Work/foxhunt/ml/src/features/config.rs 2>/dev/null; then
|
||||
echo " ✅ Wave C configuration active"
|
||||
else
|
||||
echo " ⚠ WARNING: Wave D features still enabled in code"
|
||||
echo " ⚠ Recommendation: Run Level 1 rollback first"
|
||||
fi
|
||||
|
||||
# Rebuild services (if needed)
|
||||
echo " ✓ Rebuilding services..."
|
||||
cd /home/jgrusewski/Work/foxhunt
|
||||
cargo build --release --workspace 2>&1 | grep -E "Compiling|Finished|error" || echo " ✓ Build completed"
|
||||
|
||||
echo " ⚠ NOTE: Manual service restart required for production"
|
||||
echo " ⚠ Services NOT started automatically by this test script"
|
||||
|
||||
RESTART_END=$(date +%s)
|
||||
RESTART_TIME=$((RESTART_END - RESTART_START))
|
||||
echo " ✓ Rebuild completed in ${RESTART_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 6: Calculate rollback time
|
||||
echo "Step 6: Rollback Performance Metrics"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
END_TIME=$(date +%s)
|
||||
TOTAL_TIME=$((END_TIME - START_TIME))
|
||||
|
||||
echo " • Total rollback time: ${TOTAL_TIME}s"
|
||||
echo " • Shutdown time: ${SHUTDOWN_TIME}s"
|
||||
echo " • Migration time: ${MIGRATION_TIME}s"
|
||||
echo " • Rebuild time: ${RESTART_TIME}s"
|
||||
echo " • Target time: <300s (5 minutes)"
|
||||
|
||||
if [ $TOTAL_TIME -lt 300 ]; then
|
||||
echo " ✅ PASSED: Rollback completed within 5-minute target"
|
||||
else
|
||||
echo " ⚠ MISSED TARGET: Rollback took ${TOTAL_TIME}s (>300s)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Summary
|
||||
echo "===================================================================================================="
|
||||
echo "LEVEL 2 ROLLBACK TEST: SUMMARY"
|
||||
echo "===================================================================================================="
|
||||
echo ""
|
||||
echo "Result:"
|
||||
echo " • Database tables removed: $REGIME_TABLES → $REMAINING_TABLES"
|
||||
echo " • Database functions removed: ✓ (get_latest_regime, get_regime_transition_matrix, get_regime_performance)"
|
||||
echo " • Migration version: $MIGRATION_VERSION"
|
||||
echo " • Services: STOPPED (manual restart required)"
|
||||
echo " • Rollback time: ${TOTAL_TIME}s (target: <300s)"
|
||||
echo ""
|
||||
echo "Data Loss:"
|
||||
echo " • regime_states: $REGIME_DATA records DELETED"
|
||||
echo " • regime_transitions: DELETED"
|
||||
echo " • adaptive_strategy_metrics: DELETED"
|
||||
if [ -n "$BACKUP_FILE" ] && [ -f "$BACKUP_FILE" ]; then
|
||||
echo " • Backup: $BACKUP_FILE ($BACKUP_SIZE)"
|
||||
fi
|
||||
echo ""
|
||||
echo "Recovery Path:"
|
||||
echo " To re-apply Wave D migration:"
|
||||
echo " 1. Restore database: psql < $BACKUP_FILE (if needed)"
|
||||
echo " 2. Re-apply migration: sqlx migrate run"
|
||||
echo " 3. Re-enable Wave D features (reverse Level 1 rollback)"
|
||||
echo " 4. Rebuild services: cargo build --workspace --release"
|
||||
echo " 5. Restart services"
|
||||
echo ""
|
||||
echo "===================================================================================================="
|
||||
|
||||
exit 0
|
||||
@@ -1,297 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ================================================================================================
|
||||
# Level 3 Rollback Test: Full Rollback to Wave C (Target: ~15 minutes)
|
||||
# Agent R1 - Rollback & Disaster Recovery Specialist
|
||||
# ================================================================================================
|
||||
#
|
||||
# SCENARIO: Complete redeployment to Wave C baseline (before Wave D)
|
||||
# EXPECTED: System rolled back to last known good Wave C state
|
||||
#
|
||||
# ================================================================================================
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "===================================================================================================="
|
||||
echo "LEVEL 3 ROLLBACK TEST: Full Rollback to Wave C"
|
||||
echo "===================================================================================================="
|
||||
echo ""
|
||||
|
||||
# Start timer
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Configuration
|
||||
WAVE_C_TAG="wave-c-baseline" # Git tag for Wave C baseline
|
||||
WAVE_D_TAG="wave-d-v1.0" # Git tag for current Wave D deployment
|
||||
BACKUP_DIR="/tmp/foxhunt_rollback_$(date +%s)"
|
||||
|
||||
# Step 1: Pre-rollback preparation
|
||||
echo "Step 1: Pre-rollback preparation"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
|
||||
# Create backup directory
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
echo " ✓ Created backup directory: $BACKUP_DIR"
|
||||
|
||||
# Tag current Wave D state (if not already tagged)
|
||||
cd /home/jgrusewski/Work/foxhunt
|
||||
CURRENT_COMMIT=$(git rev-parse HEAD)
|
||||
echo " ✓ Current commit: $CURRENT_COMMIT"
|
||||
|
||||
if ! git tag -l | grep -q "^$WAVE_D_TAG$"; then
|
||||
git tag "$WAVE_D_TAG" "$CURRENT_COMMIT"
|
||||
echo " ✓ Tagged Wave D deployment: $WAVE_D_TAG"
|
||||
else
|
||||
echo " ⚠ Tag $WAVE_D_TAG already exists"
|
||||
fi
|
||||
|
||||
# Backup database
|
||||
echo " ✓ Backing up database..."
|
||||
BACKUP_FILE="$BACKUP_DIR/foxhunt_wave_d_backup.sql"
|
||||
PGPASSWORD=foxhunt_dev_password pg_dump -h localhost -U foxhunt -d foxhunt -f "$BACKUP_FILE" 2>/dev/null || {
|
||||
echo " ⚠ WARNING: Database backup failed"
|
||||
BACKUP_FILE=""
|
||||
}
|
||||
if [ -n "$BACKUP_FILE" ] && [ -f "$BACKUP_FILE" ]; then
|
||||
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
|
||||
echo " ✅ Database backup created: $BACKUP_FILE ($BACKUP_SIZE)"
|
||||
fi
|
||||
|
||||
# Backup .env files
|
||||
echo " ✓ Backing up environment files..."
|
||||
cp /home/jgrusewski/Work/foxhunt/.env "$BACKUP_DIR/.env.wave_d" 2>/dev/null || true
|
||||
cp /home/jgrusewski/Work/foxhunt/.env.production "$BACKUP_DIR/.env.production.wave_d" 2>/dev/null || true
|
||||
echo " ✓ Environment files backed up"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 2: Stop all services
|
||||
echo "Step 2: Stopping all services"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
SHUTDOWN_START=$(date +%s)
|
||||
|
||||
# Stop Foxhunt services
|
||||
PIDS=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service" || true)
|
||||
if [ -n "$PIDS" ]; then
|
||||
echo " ✓ Stopping Foxhunt services (PIDs: $PIDS)..."
|
||||
kill -TERM $PIDS 2>/dev/null || true
|
||||
|
||||
# Wait for graceful shutdown
|
||||
for i in {1..30}; do
|
||||
REMAINING=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service" | wc -l)
|
||||
if [ "$REMAINING" -eq 0 ]; then
|
||||
echo " ✅ Services stopped in ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Force kill if needed
|
||||
REMAINING=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service" | wc -l)
|
||||
if [ "$REMAINING" -gt 0 ]; then
|
||||
echo " ⚠ Forcing shutdown of $REMAINING processes..."
|
||||
kill -9 $(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service") 2>/dev/null || true
|
||||
fi
|
||||
else
|
||||
echo " ⚠ No services running"
|
||||
fi
|
||||
|
||||
SHUTDOWN_END=$(date +%s)
|
||||
SHUTDOWN_TIME=$((SHUTDOWN_END - SHUTDOWN_START))
|
||||
echo " ✓ Shutdown completed in ${SHUTDOWN_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 3: Rollback database (Level 2 rollback)
|
||||
echo "Step 3: Rolling back database to Wave C state"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
MIGRATION_START=$(date +%s)
|
||||
|
||||
# Run Level 2 rollback (database migration revert)
|
||||
echo " ✓ Executing database rollback..."
|
||||
if [ -f "/home/jgrusewski/Work/foxhunt/migrations/045_wave_d_regime_tracking.down.sql" ]; then
|
||||
PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt \
|
||||
-f /home/jgrusewski/Work/foxhunt/migrations/045_wave_d_regime_tracking.down.sql \
|
||||
2>&1 | grep -E "DROP|REVOKE|NOTICE|ERROR|Wave D" || echo " ✓ Database rollback completed"
|
||||
else
|
||||
echo " ⚠ WARNING: Down migration not found, attempting alternative method..."
|
||||
# Use migration 046 if available
|
||||
if [ -f "/home/jgrusewski/Work/foxhunt/migrations/046_rollback_regime_detection.sql" ]; then
|
||||
PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt \
|
||||
-f /home/jgrusewski/Work/foxhunt/migrations/046_rollback_regime_detection.sql \
|
||||
2>&1 | grep -E "DROP|REVOKE|NOTICE|ERROR|Wave D" || echo " ✓ Alternative rollback completed"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify database state
|
||||
REGIME_TABLES=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "
|
||||
SELECT COUNT(*) FROM information_schema.tables
|
||||
WHERE table_name IN ('regime_states', 'regime_transitions', 'adaptive_strategy_metrics');
|
||||
" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [ "$REGIME_TABLES" = "0" ]; then
|
||||
echo " ✅ Database rollback successful (Wave D tables removed)"
|
||||
else
|
||||
echo " ❌ Database rollback failed ($REGIME_TABLES Wave D tables remain)"
|
||||
fi
|
||||
|
||||
MIGRATION_END=$(date +%s)
|
||||
MIGRATION_TIME=$((MIGRATION_END - MIGRATION_START))
|
||||
echo " ✓ Database rollback completed in ${MIGRATION_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 4: Checkout Wave C baseline code
|
||||
echo "Step 4: Checking out Wave C baseline code"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
CHECKOUT_START=$(date +%s)
|
||||
|
||||
# Find Wave C baseline commit
|
||||
if git tag -l | grep -q "^$WAVE_C_TAG$"; then
|
||||
echo " ✓ Found Wave C tag: $WAVE_C_TAG"
|
||||
WAVE_C_COMMIT=$(git rev-list -n 1 "$WAVE_C_TAG")
|
||||
else
|
||||
# Fallback: Find commit before Wave D Phase 3 (feature implementation)
|
||||
echo " ⚠ No Wave C tag found, searching for baseline commit..."
|
||||
WAVE_C_COMMIT=$(git log --all --oneline | grep -E "Wave C.*COMPLETE|WAVE_C.*IMPLEMENTATION.*COMPLETE" | head -1 | awk '{print $1}')
|
||||
if [ -z "$WAVE_C_COMMIT" ]; then
|
||||
# Ultimate fallback: commit before "Wave D Phase 3"
|
||||
WAVE_C_COMMIT=$(git log --all --oneline --before="2025-10-17" | head -1 | awk '{print $1}')
|
||||
fi
|
||||
fi
|
||||
|
||||
echo " ✓ Wave C baseline commit: $WAVE_C_COMMIT"
|
||||
|
||||
# Stash current changes (if any)
|
||||
if ! git diff-index --quiet HEAD --; then
|
||||
echo " ✓ Stashing uncommitted changes..."
|
||||
git stash push -m "Rollback: Stashing Wave D changes before rollback to Wave C"
|
||||
fi
|
||||
|
||||
# Checkout Wave C baseline
|
||||
echo " ✓ Checking out Wave C baseline..."
|
||||
git checkout "$WAVE_C_COMMIT" 2>&1 | grep -E "HEAD|Previous|error" || echo " ✓ Checkout completed"
|
||||
|
||||
# Verify checkout
|
||||
CURRENT_COMMIT=$(git rev-parse HEAD)
|
||||
if [ "$CURRENT_COMMIT" = "$WAVE_C_COMMIT" ]; then
|
||||
echo " ✅ Successfully checked out Wave C baseline"
|
||||
else
|
||||
echo " ❌ Checkout failed (HEAD: $CURRENT_COMMIT, Expected: $WAVE_C_COMMIT)"
|
||||
fi
|
||||
|
||||
CHECKOUT_END=$(date +%s)
|
||||
CHECKOUT_TIME=$((CHECKOUT_END - CHECKOUT_START))
|
||||
echo " ✓ Git checkout completed in ${CHECKOUT_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 5: Rebuild services with Wave C codebase
|
||||
echo "Step 5: Rebuilding services from Wave C codebase"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
REBUILD_START=$(date +%s)
|
||||
|
||||
# Clean build to ensure no Wave D artifacts
|
||||
echo " ✓ Cleaning previous build artifacts..."
|
||||
cd /home/jgrusewski/Work/foxhunt
|
||||
cargo clean 2>&1 | grep -E "Removing|error" || echo " ✓ Clean completed"
|
||||
|
||||
# Rebuild entire workspace
|
||||
echo " ✓ Rebuilding workspace (release mode)..."
|
||||
cargo build --workspace --release 2>&1 | grep -E "Compiling|Finished|error" || echo " ✓ Build completed"
|
||||
|
||||
REBUILD_END=$(date +%s)
|
||||
REBUILD_TIME=$((REBUILD_END - REBUILD_START))
|
||||
echo " ✓ Rebuild completed in ${REBUILD_TIME}s"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 6: Smoke test Wave C deployment
|
||||
echo "Step 6: Smoke testing Wave C deployment"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
|
||||
# Check feature count
|
||||
echo " ✓ Checking feature count..."
|
||||
FEATURE_COUNT=$(cargo run --release -p ml --example check_feature_count 2>&1 | grep "Wave C: feature_count:" | grep -oP '\d+' || echo "unknown")
|
||||
if [ "$FEATURE_COUNT" = "201" ]; then
|
||||
echo " ✅ VERIFIED: Wave C feature count = 201"
|
||||
elif [ "$FEATURE_COUNT" = "225" ]; then
|
||||
echo " ❌ FAILED: Feature count = 225 (Wave D still active)"
|
||||
else
|
||||
echo " ⚠ WARNING: Could not verify feature count (got: $FEATURE_COUNT)"
|
||||
fi
|
||||
|
||||
# Check for Wave D code
|
||||
WAVE_D_CODE=$(grep -r "enable_wave_d_regime" /home/jgrusewski/Work/foxhunt/ml/src/features/ 2>/dev/null | wc -l)
|
||||
if [ "$WAVE_D_CODE" -eq 0 ]; then
|
||||
echo " ✅ VERIFIED: No Wave D code present"
|
||||
else
|
||||
echo " ⚠ WARNING: Wave D code references still exist ($WAVE_D_CODE occurrences)"
|
||||
echo " ⚠ This may be expected if Wave C already had placeholders"
|
||||
fi
|
||||
|
||||
# Test basic compilation
|
||||
echo " ✓ Running basic compilation test..."
|
||||
cargo check --workspace 2>&1 | grep -E "Checking|Finished|error" || echo " ✓ Check completed"
|
||||
|
||||
echo " ⚠ NOTE: Services NOT started automatically (manual start required)"
|
||||
|
||||
echo ""
|
||||
|
||||
# Step 7: Calculate rollback time
|
||||
echo "Step 7: Rollback Performance Metrics"
|
||||
echo "------------------------------------------------------------------------------------"
|
||||
END_TIME=$(date +%s)
|
||||
TOTAL_TIME=$((END_TIME - START_TIME))
|
||||
|
||||
echo " • Total rollback time: ${TOTAL_TIME}s"
|
||||
echo " • Shutdown time: ${SHUTDOWN_TIME}s"
|
||||
echo " • Database rollback time: ${MIGRATION_TIME}s"
|
||||
echo " • Git checkout time: ${CHECKOUT_TIME}s"
|
||||
echo " • Rebuild time: ${REBUILD_TIME}s"
|
||||
echo " • Target time: <900s (15 minutes)"
|
||||
|
||||
if [ $TOTAL_TIME -lt 900 ]; then
|
||||
echo " ✅ PASSED: Rollback completed within 15-minute target"
|
||||
else
|
||||
echo " ⚠ MISSED TARGET: Rollback took ${TOTAL_TIME}s (>900s)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Summary
|
||||
echo "===================================================================================================="
|
||||
echo "LEVEL 3 ROLLBACK TEST: SUMMARY"
|
||||
echo "===================================================================================================="
|
||||
echo ""
|
||||
echo "Result:"
|
||||
echo " • Git state: Wave C baseline ($WAVE_C_COMMIT)"
|
||||
echo " • Database: Wave D tables removed ($REGIME_TABLES tables remain)"
|
||||
echo " • Feature count: $FEATURE_COUNT (expected: 201)"
|
||||
echo " • Services: STOPPED (manual restart required)"
|
||||
echo " • Rollback time: ${TOTAL_TIME}s (~$((TOTAL_TIME / 60)) minutes)"
|
||||
echo ""
|
||||
echo "Data Loss:"
|
||||
echo " • All Wave D regime detection data DELETED"
|
||||
echo " • Wave D code changes REVERTED"
|
||||
if [ -n "$BACKUP_FILE" ] && [ -f "$BACKUP_FILE" ]; then
|
||||
echo " • Full backup: $BACKUP_FILE ($BACKUP_SIZE)"
|
||||
fi
|
||||
echo " • Backup directory: $BACKUP_DIR"
|
||||
echo ""
|
||||
echo "Recovery Path:"
|
||||
echo " To re-deploy Wave D:"
|
||||
echo " 1. Checkout Wave D tag: git checkout $WAVE_D_TAG"
|
||||
echo " 2. Restore database (optional): psql < $BACKUP_FILE"
|
||||
echo " 3. Re-apply migrations: sqlx migrate run"
|
||||
echo " 4. Rebuild services: cargo build --workspace --release"
|
||||
echo " 5. Restart all services"
|
||||
echo ""
|
||||
echo "Manual Steps Required:"
|
||||
echo " 1. Start services: cargo run -p api_gateway &"
|
||||
echo " 2. Verify health checks: curl http://localhost:8080/health"
|
||||
echo " 3. Run integration tests: cargo test --workspace"
|
||||
echo " 4. Monitor Grafana dashboards"
|
||||
echo ""
|
||||
echo "===================================================================================================="
|
||||
|
||||
exit 0
|
||||
@@ -1,169 +0,0 @@
|
||||
#!/bin/bash
|
||||
# MAMBA-2 Fixed Binary Deployment Monitor
|
||||
# Pod ID: 8e6o2r2snavgzf
|
||||
# Expected completion: 2025-10-27 10:21 UTC (~81 minutes from 09:00)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
POD_ID="8e6o2r2snavgzf"
|
||||
CHECKPOINT_DIR="/runpod-volume/models/mamba2_FIXED_sgd_bs512_lr5e4_shuffle_50ep"
|
||||
|
||||
echo "=================================="
|
||||
echo "MAMBA-2 FIXED BINARY MONITOR"
|
||||
echo "=================================="
|
||||
echo "Pod ID: $POD_ID"
|
||||
echo "GPU: RTX 4090 (24GB VRAM)"
|
||||
echo "Cost: \$0.59/hr"
|
||||
echo "Expected runtime: ~81 minutes"
|
||||
echo "=================================="
|
||||
echo ""
|
||||
|
||||
# Load RunPod credentials
|
||||
if [ ! -f ".env.runpod" ]; then
|
||||
echo "ERROR: .env.runpod not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source .env.runpod
|
||||
|
||||
if [ -z "$RUNPOD_API_KEY" ]; then
|
||||
echo "ERROR: RUNPOD_API_KEY not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Function to check pod status
|
||||
check_status() {
|
||||
echo "Checking pod status..."
|
||||
curl -s -H "Authorization: Bearer $RUNPOD_API_KEY" \
|
||||
"https://rest.runpod.io/v1/pods/$POD_ID" | python3 -m json.tool
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to download results
|
||||
download_results() {
|
||||
echo "Downloading results from S3..."
|
||||
aws s3 sync "s3://se3zdnb5o4/models/mamba2_FIXED_sgd_bs512_lr5e4_shuffle_50ep" \
|
||||
"./local_models/mamba2_FIXED" \
|
||||
--profile runpod \
|
||||
--endpoint-url https://s3api-eur-is-1.runpod.io
|
||||
echo ""
|
||||
echo "Results downloaded to ./local_models/mamba2_FIXED/"
|
||||
ls -lh "./local_models/mamba2_FIXED/"
|
||||
}
|
||||
|
||||
# Function to verify training success
|
||||
verify_training() {
|
||||
echo "Verifying training success..."
|
||||
|
||||
if [ ! -d "./local_models/mamba2_FIXED" ]; then
|
||||
echo "ERROR: Results not downloaded yet. Run with 'download' first."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check for model checkpoint
|
||||
if [ -f "./local_models/mamba2_FIXED/mamba2_model_epoch_50.safetensors" ]; then
|
||||
echo "✅ Model checkpoint found"
|
||||
ls -lh "./local_models/mamba2_FIXED/mamba2_model_epoch_50.safetensors"
|
||||
else
|
||||
echo "❌ Model checkpoint NOT found"
|
||||
fi
|
||||
|
||||
# Check for metrics
|
||||
if [ -f "./local_models/mamba2_FIXED/training_metrics.json" ]; then
|
||||
echo "✅ Training metrics found"
|
||||
cat "./local_models/mamba2_FIXED/training_metrics.json"
|
||||
else
|
||||
echo "❌ Training metrics NOT found"
|
||||
fi
|
||||
|
||||
# Check for loss history
|
||||
if [ -f "./local_models/mamba2_FIXED/loss_history.csv" ]; then
|
||||
echo "✅ Loss history found"
|
||||
echo "Last 10 epochs:"
|
||||
tail -n 10 "./local_models/mamba2_FIXED/loss_history.csv"
|
||||
else
|
||||
echo "❌ Loss history NOT found"
|
||||
fi
|
||||
|
||||
# Check training log for key indicators
|
||||
if [ -f "./local_models/mamba2_FIXED/training.log" ]; then
|
||||
echo "✅ Training log found"
|
||||
echo ""
|
||||
echo "Checking for success indicators..."
|
||||
|
||||
# Check optimizer
|
||||
if grep -q "Optimizer: SGD" "./local_models/mamba2_FIXED/training.log"; then
|
||||
echo "✅ SGD optimizer confirmed (not Adam)"
|
||||
else
|
||||
echo "❌ SGD optimizer NOT found (check for Adam)"
|
||||
fi
|
||||
|
||||
# Check for zero gradients
|
||||
if grep -q "grad: 0.0000" "./local_models/mamba2_FIXED/training.log"; then
|
||||
echo "❌ Zero gradients detected (P0 fix failed)"
|
||||
else
|
||||
echo "✅ No zero gradients detected"
|
||||
fi
|
||||
|
||||
# Check for E11 spike
|
||||
if grep -q "E11" "./local_models/mamba2_FIXED/training.log" || \
|
||||
grep -q "1e+11" "./local_models/mamba2_FIXED/training.log"; then
|
||||
echo "❌ E11 spike detected (numerical instability)"
|
||||
else
|
||||
echo "✅ No E11 spike detected"
|
||||
fi
|
||||
|
||||
else
|
||||
echo "❌ Training log NOT found"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main menu
|
||||
case "${1:-status}" in
|
||||
status)
|
||||
check_status
|
||||
;;
|
||||
download)
|
||||
download_results
|
||||
;;
|
||||
verify)
|
||||
verify_training
|
||||
;;
|
||||
ssh)
|
||||
echo "SSH to pod $POD_ID..."
|
||||
echo "ssh root@$POD_ID.ssh.runpod.io"
|
||||
echo ""
|
||||
echo "Once connected, check training status:"
|
||||
echo " cd $CHECKPOINT_DIR"
|
||||
echo " tail -f training.log"
|
||||
;;
|
||||
jupyter)
|
||||
echo "Jupyter URL: https://$POD_ID-8888.proxy.runpod.net"
|
||||
echo ""
|
||||
echo "Navigate to: $CHECKPOINT_DIR/training.log"
|
||||
;;
|
||||
all)
|
||||
check_status
|
||||
echo ""
|
||||
echo "=================================="
|
||||
read -p "Download results? (y/n) " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
download_results
|
||||
echo ""
|
||||
verify_training
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {status|download|verify|ssh|jupyter|all}"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " status - Check pod status via API"
|
||||
echo " download - Download results from S3"
|
||||
echo " verify - Verify training success (requires download first)"
|
||||
echo " ssh - Show SSH command"
|
||||
echo " jupyter - Show Jupyter URL"
|
||||
echo " all - Status + download + verify (interactive)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -1,56 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Quick Fix Commands - Foxhunt Codebase Cleanup
|
||||
# Total Estimated Time: 3.5 minutes
|
||||
# Impact: Cosmetic improvements only, zero functional changes
|
||||
|
||||
set -e
|
||||
|
||||
echo "=========================================="
|
||||
echo "Foxhunt Codebase Quick Fixes"
|
||||
echo "Total Time: ~3.5 minutes"
|
||||
echo "=========================================="
|
||||
|
||||
echo ""
|
||||
echo "[1/3] Fixing 4 Clippy Deny-Level Errors (35 seconds)..."
|
||||
echo "Files affected:"
|
||||
echo " - services/stress_tests/src/metrics.rs"
|
||||
echo " - trading-data/src/models.rs"
|
||||
echo " - trading_engine/src/types/events.rs"
|
||||
|
||||
# Manual fixes required (clippy --fix may not handle all):
|
||||
echo ""
|
||||
echo "Manual fixes needed:"
|
||||
echo "1. stress_tests/src/metrics.rs:144"
|
||||
echo " BEFORE: let mean_u64 = (mean_micros as u64).min(u64::MAX);"
|
||||
echo " AFTER: let mean_u64 = (mean_micros as u64);"
|
||||
echo ""
|
||||
echo "2. trading-data/src/models.rs:98"
|
||||
echo " BEFORE: assert_eq!(order.quantity.to_f64(), 100000.0);"
|
||||
echo " AFTER: assert_relative_eq!(order.quantity.to_f64(), 100_000.0, epsilon = 1e-6);"
|
||||
echo ""
|
||||
echo "3. trading_engine/src/types/events.rs (4 locations)"
|
||||
echo " BEFORE: 150000.0, 100000.0, 500000.0, 400000.0"
|
||||
echo " AFTER: 150_000.0, 100_000.0, 500_000.0, 400_000.0"
|
||||
|
||||
read -p "Press Enter after making manual fixes..."
|
||||
|
||||
echo ""
|
||||
echo "[2/3] Formatting entire codebase (2 minutes)..."
|
||||
cargo fmt --all
|
||||
echo "✅ Formatting complete"
|
||||
|
||||
echo ""
|
||||
echo "[3/3] Verifying compilation (1 minute)..."
|
||||
cargo build --workspace --release --quiet
|
||||
echo "✅ Compilation successful"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "✅ Quick fixes complete!"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo "1. Review changes: git diff"
|
||||
echo "2. Run tests: cargo test --workspace --lib"
|
||||
echo "3. Commit: git commit -m 'chore: Apply clippy fixes and rustfmt'"
|
||||
echo "4. Deploy to production"
|
||||
@@ -1,286 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Add priority staleness tracking to PER buffer"""
|
||||
|
||||
import re
|
||||
|
||||
def add_staleness_tracking(filepath):
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# 1. Add field to struct
|
||||
struct_pattern = r'(pub struct PrioritizedReplayBuffer \{[^}]+priorities: Arc<Mutex<SegmentTree>>,)'
|
||||
struct_replacement = r'\1\n priority_update_steps: Arc<RwLock<Vec<u64>>>,'
|
||||
content = re.sub(struct_pattern, struct_replacement, content)
|
||||
|
||||
# 2. Initialize in new()
|
||||
new_pattern = r'(priorities: Arc::new\(Mutex::new\(SegmentTree::new\(config\.capacity\)\)\),)'
|
||||
new_replacement = r'\1\n priority_update_steps: Arc::new(RwLock::new(vec![0; config.capacity])),'
|
||||
content = re.sub(new_pattern, new_replacement, content)
|
||||
|
||||
# 3. Update push() - add current_step variable
|
||||
push_pattern1 = r'(let index = self\.position\.fetch_add\(1, Ordering::AcqRel\) % self\.config\.capacity;)'
|
||||
push_replacement1 = r'\1\n let current_step = self.training_step.load(Ordering::Acquire) as u64;'
|
||||
content = re.sub(push_pattern1, push_replacement1, content)
|
||||
|
||||
# 3b. Update push() - initialize priority update step
|
||||
push_pattern2 = r'(\{[\s\n]+let mut tree = self\.priorities\.lock\(\);[\s\n]+tree\.update\(index, priority\)\?;[\s\n]+\})'
|
||||
push_replacement2 = r'''\1
|
||||
|
||||
// Initialize priority update step
|
||||
{
|
||||
let mut update_steps = self.priority_update_steps.write();
|
||||
if index < update_steps.len() {
|
||||
update_steps[index] = current_step;
|
||||
}
|
||||
}'''
|
||||
content = re.sub(push_pattern2, push_replacement2, content)
|
||||
|
||||
# 4. Add staleness decay to sample() at the beginning
|
||||
sample_pattern = r'(pub fn sample\(\s+&self,\s+batch_size: usize,\s+\) -> Result<\(Vec<Experience>, Vec<f32>, Vec<usize>\), MLError> \{)'
|
||||
sample_replacement = r'''\1
|
||||
// Apply staleness decay before sampling
|
||||
// Default: max_age = 10000 steps, decay_factor = 0.9
|
||||
self.apply_staleness_decay(
|
||||
self.training_step.load(Ordering::Acquire) as u64,
|
||||
10000,
|
||||
0.9,
|
||||
)?;
|
||||
'''
|
||||
content = re.sub(sample_pattern, sample_replacement, content)
|
||||
|
||||
# 5. Update update_priorities() - add current_step
|
||||
update_pattern1 = r'(let mut max_priority = f32::from_bits\(self\.max_priority\.load\(Ordering::Acquire\) as u32\);)'
|
||||
update_replacement1 = r'\1\n let current_step = self.training_step.load(Ordering::Acquire) as u64;'
|
||||
content = re.sub(update_pattern1, update_replacement1, content)
|
||||
|
||||
# 5b. Update update_priorities() - track update steps
|
||||
update_pattern2 = r'(self\.max_priority[\s\n]+\.store\(max_priority\.to_bits\(\) as u64, Ordering::Release\);)'
|
||||
update_replacement2 = r'''\1
|
||||
|
||||
// Update priority update steps
|
||||
{
|
||||
let mut update_steps = self.priority_update_steps.write();
|
||||
for &idx in indices {
|
||||
if idx < update_steps.len() {
|
||||
update_steps[idx] = current_step;
|
||||
}
|
||||
}
|
||||
}'''
|
||||
content = re.sub(update_pattern2, update_replacement2, content)
|
||||
|
||||
# 6. Add apply_staleness_decay method before clear()
|
||||
clear_pattern = r'( /// Reset buffer \(clear all experiences\)[\s\n]+ pub fn clear\(&self\) \{)'
|
||||
staleness_method = r''' /// Apply staleness decay to old priorities
|
||||
pub fn apply_staleness_decay(
|
||||
&self,
|
||||
current_step: u64,
|
||||
max_age: u64,
|
||||
decay_factor: f64,
|
||||
) -> Result<(), MLError> {
|
||||
let size = self.size.load(Ordering::Acquire);
|
||||
if size == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let update_steps = self.priority_update_steps.read();
|
||||
let mut tree = self.priorities.lock();
|
||||
|
||||
for idx in 0..size {
|
||||
if idx >= update_steps.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
let last_update = update_steps[idx];
|
||||
let age = current_step.saturating_sub(last_update);
|
||||
|
||||
if age > max_age {
|
||||
let current_priority = tree.get_priority(idx);
|
||||
if current_priority > 0.0 {
|
||||
let decay = decay_factor.powi((age / max_age) as i32) as f32;
|
||||
let new_priority = (current_priority * decay).max(self.config.min_priority);
|
||||
tree.update(idx, new_priority)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
\1'''
|
||||
content = re.sub(clear_pattern, staleness_method, content)
|
||||
|
||||
# 7. Update clear() to reset update steps
|
||||
clear_update_pattern = r'(\{[\s\n]+let mut tree = self\.priorities\.lock\(\);[\s\n]+for i in 0\.\.self\.config\.capacity \{[\s\n]+let _ = tree\.update\(i, 0\.0\);[\s\n]+\}[\s\n]+\})'
|
||||
clear_update_replacement = r'''\1
|
||||
|
||||
{
|
||||
let mut update_steps = self.priority_update_steps.write();
|
||||
for step in update_steps.iter_mut() {
|
||||
*step = 0;
|
||||
}
|
||||
}'''
|
||||
content = re.sub(clear_update_pattern, clear_update_replacement, content)
|
||||
|
||||
# 8. Add tests before the closing brace of mod tests
|
||||
test_pattern = r'( \}\n\}\n)$'
|
||||
tests_addition = r'''
|
||||
#[test]
|
||||
fn test_priority_staleness_decay() {
|
||||
let config = PrioritizedReplayConfig {
|
||||
capacity: 100,
|
||||
initial_priority: 1.0,
|
||||
min_priority: 1e-6,
|
||||
..Default::default()
|
||||
};
|
||||
let buffer = PrioritizedReplayBuffer::new(config)
|
||||
.expect("Failed to create prioritized replay buffer in test");
|
||||
|
||||
// Push 10 experiences at step 0
|
||||
buffer.set_training_step(0);
|
||||
for _ in 0..10 {
|
||||
buffer
|
||||
.push(create_test_experience())
|
||||
.expect("Failed to push experience in test");
|
||||
}
|
||||
|
||||
// Get initial priorities
|
||||
let tree = buffer.priorities.lock();
|
||||
let initial_priorities: Vec<f32> = (0..10).map(|i| tree.get_priority(i)).collect();
|
||||
drop(tree);
|
||||
|
||||
// Simulate 15000 steps passing without priority updates
|
||||
buffer.set_training_step(15000);
|
||||
|
||||
// Apply staleness decay (max_age=10000, decay_factor=0.9)
|
||||
buffer
|
||||
.apply_staleness_decay(15000, 10000, 0.9)
|
||||
.expect("Failed to apply staleness decay in test");
|
||||
|
||||
// Check that priorities have decayed
|
||||
let tree = buffer.priorities.lock();
|
||||
for i in 0..10 {
|
||||
let current_priority = tree.get_priority(i);
|
||||
let initial_priority = initial_priorities[i];
|
||||
|
||||
// Age = 15000 steps, max_age = 10000
|
||||
// Decay should be: decay_factor^(age/max_age) = 0.9^1
|
||||
let expected_decay = 0.9_f64.powi((15000 / 10000) as i32) as f32;
|
||||
let expected_priority = (initial_priority * expected_decay).max(1e-6);
|
||||
|
||||
// Allow small floating point tolerance
|
||||
assert!(
|
||||
(current_priority - expected_priority).abs() < 0.01,
|
||||
"Priority at index {} should have decayed from {} to ~{}, got {}",
|
||||
i,
|
||||
initial_priority,
|
||||
expected_priority,
|
||||
current_priority
|
||||
);
|
||||
|
||||
// Verify priority was reduced
|
||||
assert!(
|
||||
current_priority < initial_priority,
|
||||
"Priority at index {} should be reduced after staleness decay",
|
||||
i
|
||||
);
|
||||
}
|
||||
drop(tree);
|
||||
|
||||
// Test that recently updated priorities don't decay
|
||||
buffer.set_training_step(16000);
|
||||
|
||||
// Update some priorities
|
||||
let indices = vec![0, 1, 2];
|
||||
let new_priorities = vec![5.0, 6.0, 7.0];
|
||||
buffer
|
||||
.update_priorities(&indices, &new_priorities)
|
||||
.expect("Failed to update priorities in test");
|
||||
|
||||
// Move forward only 5000 steps (below max_age threshold)
|
||||
buffer.set_training_step(21000);
|
||||
buffer
|
||||
.apply_staleness_decay(21000, 10000, 0.9)
|
||||
.expect("Failed to apply staleness decay in test");
|
||||
|
||||
// Recently updated priorities should not decay significantly
|
||||
let tree = buffer.priorities.lock();
|
||||
for &idx in &indices {
|
||||
let priority = tree.get_priority(idx);
|
||||
// Age = 5000, which is < max_age (10000), so no decay should occur
|
||||
assert!(
|
||||
priority > 4.5,
|
||||
"Recently updated priority at index {} should not decay significantly, got {}",
|
||||
idx,
|
||||
priority
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_staleness_tracking_on_push() {
|
||||
let config = PrioritizedReplayConfig {
|
||||
capacity: 100,
|
||||
..Default::default()
|
||||
};
|
||||
let buffer = PrioritizedReplayBuffer::new(config)
|
||||
.expect("Failed to create prioritized replay buffer in test");
|
||||
|
||||
// Push experiences at different training steps
|
||||
buffer.set_training_step(100);
|
||||
buffer
|
||||
.push(create_test_experience())
|
||||
.expect("Failed to push experience in test");
|
||||
|
||||
buffer.set_training_step(200);
|
||||
buffer
|
||||
.push(create_test_experience())
|
||||
.expect("Failed to push experience in test");
|
||||
|
||||
// Verify update steps were recorded
|
||||
let update_steps = buffer.priority_update_steps.read();
|
||||
assert_eq!(update_steps[0], 100, "First experience should have update step 100");
|
||||
assert_eq!(update_steps[1], 200, "Second experience should have update step 200");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_staleness_tracking_on_update() {
|
||||
let config = PrioritizedReplayConfig {
|
||||
capacity: 100,
|
||||
..Default::default()
|
||||
};
|
||||
let buffer = PrioritizedReplayBuffer::new(config)
|
||||
.expect("Failed to create prioritized replay buffer in test");
|
||||
|
||||
// Push experiences
|
||||
buffer.set_training_step(100);
|
||||
for _ in 0..10 {
|
||||
buffer
|
||||
.push(create_test_experience())
|
||||
.expect("Failed to push experience in test");
|
||||
}
|
||||
|
||||
// Update priorities at a later step
|
||||
buffer.set_training_step(500);
|
||||
let indices = vec![0, 1, 2];
|
||||
let priorities = vec![2.0, 3.0, 4.0];
|
||||
buffer
|
||||
.update_priorities(&indices, &priorities)
|
||||
.expect("Failed to update priorities in test");
|
||||
|
||||
// Verify update steps were updated
|
||||
let update_steps = buffer.priority_update_steps.read();
|
||||
assert_eq!(update_steps[0], 500, "Updated experience should have new update step");
|
||||
assert_eq!(update_steps[1], 500, "Updated experience should have new update step");
|
||||
assert_eq!(update_steps[2], 500, "Updated experience should have new update step");
|
||||
assert_eq!(update_steps[3], 100, "Non-updated experience should retain old update step");
|
||||
}
|
||||
\1'''
|
||||
content = re.sub(test_pattern, tests_addition, content)
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print(f"Successfully updated {filepath}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
add_staleness_tracking("/home/jgrusewski/Work/foxhunt/ml/src/dqn/prioritized_replay.rs")
|
||||
@@ -1,332 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple Checkpoint Comparison Analysis (no matplotlib dependency)
|
||||
Analyzes backtest results for all DQN and PPO checkpoints
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
|
||||
def load_results(results_file):
|
||||
"""Load backtest results from JSON"""
|
||||
with open(results_file, 'r') as f:
|
||||
return json.load(f)
|
||||
|
||||
def filter_valid_results(results):
|
||||
"""Filter out checkpoints with no trades or invalid metrics"""
|
||||
return [r for r in results if r['total_trades'] > 0]
|
||||
|
||||
def create_comparison_analysis(results):
|
||||
"""Analyze and compare DQN vs PPO checkpoints"""
|
||||
|
||||
# Separate DQN and PPO
|
||||
dqn_results = [r for r in results if r['model_type'] == 'DQN']
|
||||
ppo_results = [r for r in results if r['model_type'] == 'PPO']
|
||||
|
||||
print("\n" + "="*100)
|
||||
print("📊 CHECKPOINT BACKTESTING ANALYSIS - COMPLETE RESULTS")
|
||||
print("="*100)
|
||||
|
||||
print(f"\n✅ Total Checkpoints Tested: {len(results)}")
|
||||
print(f" - DQN: {len(dqn_results)} checkpoints")
|
||||
print(f" - PPO: {len(ppo_results)} checkpoints")
|
||||
|
||||
# Top 10 DQN
|
||||
print("\n" + "="*100)
|
||||
print("🔵 TOP 10 DQN CHECKPOINTS (Ranked by Sharpe Ratio)")
|
||||
print("="*100)
|
||||
|
||||
dqn_sorted = sorted(dqn_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
|
||||
|
||||
print(f"{'Rank':<6} {'Epoch':<8} {'Sharpe':<10} {'Win Rate':<12} {'Trades':<8} {'PnL':<14} {'Drawdown':<12} {'Freq':<10}")
|
||||
print("-"*100)
|
||||
|
||||
for rank, r in enumerate(dqn_sorted, 1):
|
||||
print(f"{rank:<6} {r['epoch']:<8} {r['sharpe_ratio']:<10.3f} {r['win_rate']:<11.1f}% {r['total_trades']:<8} ${r['total_pnl']:<13.2f} {r['max_drawdown']*100:<11.2f}% {r['trade_frequency']:<10.1f}")
|
||||
|
||||
# Top 10 PPO
|
||||
print("\n" + "="*100)
|
||||
print("🟢 TOP 10 PPO CHECKPOINTS (Ranked by Sharpe Ratio)")
|
||||
print("="*100)
|
||||
|
||||
ppo_sorted = sorted(ppo_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
|
||||
|
||||
print(f"{'Rank':<6} {'Epoch':<8} {'Sharpe':<10} {'Win Rate':<12} {'Trades':<8} {'PnL':<14} {'Drawdown':<12} {'Freq':<10}")
|
||||
print("-"*100)
|
||||
|
||||
for rank, r in enumerate(ppo_sorted, 1):
|
||||
print(f"{rank:<6} {r['epoch']:<8} {r['sharpe_ratio']:<10.3f} {r['win_rate']:<11.1f}% {r['total_trades']:<8} ${r['total_pnl']:<13.2f} {r['max_drawdown']*100:<11.2f}% {r['trade_frequency']:<10.1f}")
|
||||
|
||||
# Statistical summary
|
||||
print("\n" + "="*100)
|
||||
print("📈 STATISTICAL SUMMARY")
|
||||
print("="*100)
|
||||
|
||||
dqn_sharpe = [r['sharpe_ratio'] for r in dqn_results]
|
||||
dqn_win_rate = [r['win_rate'] for r in dqn_results]
|
||||
dqn_trades = [r['total_trades'] for r in dqn_results]
|
||||
dqn_pnl = [r['total_pnl'] for r in dqn_results]
|
||||
|
||||
ppo_sharpe = [r['sharpe_ratio'] for r in ppo_results]
|
||||
ppo_win_rate = [r['win_rate'] for r in ppo_results]
|
||||
ppo_trades = [r['total_trades'] for r in ppo_results]
|
||||
ppo_pnl = [r['total_pnl'] for r in ppo_results]
|
||||
|
||||
print(f"\n{'Metric':<30} {'DQN':>20} {'PPO':>20} {'Winner':>20}")
|
||||
print("-"*100)
|
||||
|
||||
# Calculate stats
|
||||
metrics = [
|
||||
('Checkpoints Tested', len(dqn_results), len(ppo_results)),
|
||||
('Avg Sharpe Ratio', sum(dqn_sharpe)/len(dqn_sharpe), sum(ppo_sharpe)/len(ppo_sharpe)),
|
||||
('Max Sharpe Ratio', max(dqn_sharpe), max(ppo_sharpe)),
|
||||
('Min Sharpe Ratio', min(dqn_sharpe), min(ppo_sharpe)),
|
||||
('Avg Win Rate (%)', sum(dqn_win_rate)/len(dqn_win_rate), sum(ppo_win_rate)/len(ppo_win_rate)),
|
||||
('Avg Total Trades', sum(dqn_trades)/len(dqn_trades), sum(ppo_trades)/len(ppo_trades)),
|
||||
('Max Total Trades', max(dqn_trades), max(ppo_trades)),
|
||||
('Avg PnL ($)', sum(dqn_pnl)/len(dqn_pnl), sum(ppo_pnl)/len(ppo_pnl)),
|
||||
('Best PnL ($)', max(dqn_pnl), max(ppo_pnl)),
|
||||
('Worst PnL ($)', min(dqn_pnl), min(ppo_pnl)),
|
||||
]
|
||||
|
||||
for name, dqn_val, ppo_val in metrics:
|
||||
if name == 'Checkpoints Tested':
|
||||
winner = 'DQN' if dqn_val > ppo_val else 'PPO' if ppo_val > dqn_val else 'Tie'
|
||||
print(f"{name:<30} {int(dqn_val):>20} {int(ppo_val):>20} {winner:>20}")
|
||||
else:
|
||||
winner = 'DQN' if dqn_val > ppo_val else 'PPO' if ppo_val > dqn_val else 'Tie'
|
||||
print(f"{name:<30} {dqn_val:>20.3f} {ppo_val:>20.3f} {winner:>20}")
|
||||
|
||||
# Best overall checkpoints
|
||||
print("\n" + "="*100)
|
||||
print("🏆 BEST CHECKPOINTS (Highest Sharpe Ratio)")
|
||||
print("="*100)
|
||||
|
||||
best_dqn = max(dqn_results, key=lambda x: x['sharpe_ratio'])
|
||||
best_ppo = max(ppo_results, key=lambda x: x['sharpe_ratio'])
|
||||
|
||||
print(f"\n🔵 Best DQN: Epoch {best_dqn['epoch']}")
|
||||
print(f" Sharpe Ratio: {best_dqn['sharpe_ratio']:.3f}")
|
||||
print(f" Win Rate: {best_dqn['win_rate']:.1f}%")
|
||||
print(f" Total Trades: {best_dqn['total_trades']}")
|
||||
print(f" Total PnL: ${best_dqn['total_pnl']:.2f}")
|
||||
print(f" Max Drawdown: {best_dqn['max_drawdown']:.2%}")
|
||||
print(f" Trade Frequency: {best_dqn['trade_frequency']:.1f} trades/1000 bars")
|
||||
|
||||
print(f"\n🟢 Best PPO: Epoch {best_ppo['epoch']}")
|
||||
print(f" Sharpe Ratio: {best_ppo['sharpe_ratio']:.3f}")
|
||||
print(f" Win Rate: {best_ppo['win_rate']:.1f}%")
|
||||
print(f" Total Trades: {best_ppo['total_trades']}")
|
||||
print(f" Total PnL: ${best_ppo['total_pnl']:.2f}")
|
||||
print(f" Max Drawdown: {best_ppo['max_drawdown']:.2%}")
|
||||
print(f" Trade Frequency: {best_ppo['trade_frequency']:.1f} trades/1000 bars")
|
||||
|
||||
# Training phase analysis
|
||||
print("\n" + "="*100)
|
||||
print("📊 TRAINING PHASE ANALYSIS")
|
||||
print("="*100)
|
||||
|
||||
# DQN Early vs Late
|
||||
dqn_early = [r for r in dqn_results if r['epoch'] <= 200]
|
||||
dqn_late = [r for r in dqn_results if r['epoch'] > 200]
|
||||
|
||||
print(f"\n🔵 DQN Performance by Training Phase")
|
||||
print(f"\nEarly Epochs (≤200): {len(dqn_early)} checkpoints")
|
||||
if dqn_early:
|
||||
print(f" Avg Sharpe: {sum(r['sharpe_ratio'] for r in dqn_early)/len(dqn_early):.3f}")
|
||||
print(f" Avg Trades: {sum(r['total_trades'] for r in dqn_early)/len(dqn_early):.1f}")
|
||||
print(f" Avg Win Rate: {sum(r['win_rate'] for r in dqn_early)/len(dqn_early):.1f}%")
|
||||
|
||||
print(f"\nLate Epochs (>200): {len(dqn_late)} checkpoints")
|
||||
if dqn_late:
|
||||
print(f" Avg Sharpe: {sum(r['sharpe_ratio'] for r in dqn_late)/len(dqn_late):.3f}")
|
||||
print(f" Avg Trades: {sum(r['total_trades'] for r in dqn_late)/len(dqn_late):.1f}")
|
||||
print(f" Avg Win Rate: {sum(r['win_rate'] for r in dqn_late)/len(dqn_late):.1f}%")
|
||||
|
||||
# PPO Early vs Late
|
||||
ppo_early = [r for r in ppo_results if r['epoch'] <= 200]
|
||||
ppo_late = [r for r in ppo_results if r['epoch'] > 200]
|
||||
|
||||
print(f"\n🟢 PPO Performance by Training Phase")
|
||||
print(f"\nEarly Epochs (≤200): {len(ppo_early)} checkpoints")
|
||||
if ppo_early:
|
||||
print(f" Avg Sharpe: {sum(r['sharpe_ratio'] for r in ppo_early)/len(ppo_early):.3f}")
|
||||
print(f" Avg Trades: {sum(r['total_trades'] for r in ppo_early)/len(ppo_early):.1f}")
|
||||
print(f" Avg Win Rate: {sum(r['win_rate'] for r in ppo_early)/len(ppo_early):.1f}%")
|
||||
|
||||
print(f"\nLate Epochs (>200): {len(ppo_late)} checkpoints")
|
||||
if ppo_late:
|
||||
print(f" Avg Sharpe: {sum(r['sharpe_ratio'] for r in ppo_late)/len(ppo_late):.3f}")
|
||||
print(f" Avg Trades: {sum(r['total_trades'] for r in ppo_late)/len(ppo_late):.1f}")
|
||||
print(f" Avg Win Rate: {sum(r['win_rate'] for r in ppo_late)/len(ppo_late):.1f}%")
|
||||
|
||||
# Key findings
|
||||
print("\n" + "="*100)
|
||||
print("🔍 KEY FINDINGS")
|
||||
print("="*100)
|
||||
|
||||
print("\n1. **Hypothesis Validation: Early Epochs (10-100) vs Late Epochs (400-500)**")
|
||||
|
||||
# Compare early vs very late
|
||||
dqn_very_early = [r for r in dqn_results if 10 <= r['epoch'] <= 100]
|
||||
dqn_very_late = [r for r in dqn_results if 400 <= r['epoch'] <= 500]
|
||||
|
||||
if dqn_very_early and dqn_very_late:
|
||||
early_sharpe = sum(r['sharpe_ratio'] for r in dqn_very_early) / len(dqn_very_early)
|
||||
late_sharpe = sum(r['sharpe_ratio'] for r in dqn_very_late) / len(dqn_very_late)
|
||||
early_trades = sum(r['total_trades'] for r in dqn_very_early) / len(dqn_very_early)
|
||||
late_trades = sum(r['total_trades'] for r in dqn_very_late) / len(dqn_very_late)
|
||||
|
||||
print(f"\n DQN Early (10-100):")
|
||||
print(f" - Avg Sharpe: {early_sharpe:.3f}")
|
||||
print(f" - Avg Trades: {early_trades:.1f}")
|
||||
|
||||
print(f"\n DQN Late (400-500):")
|
||||
print(f" - Avg Sharpe: {late_sharpe:.3f}")
|
||||
print(f" - Avg Trades: {late_trades:.1f}")
|
||||
|
||||
if late_sharpe > early_sharpe:
|
||||
print(f"\n ✅ HYPOTHESIS CONFIRMED: Late epochs have {late_sharpe/early_sharpe:.2f}x better Sharpe ratio")
|
||||
else:
|
||||
print(f"\n ❌ HYPOTHESIS REJECTED: Early epochs have better Sharpe ratio")
|
||||
|
||||
print("\n2. **Optimal Training Duration**")
|
||||
|
||||
# Find best epoch ranges
|
||||
dqn_by_range = defaultdict(list)
|
||||
for r in dqn_results:
|
||||
epoch_range = (r['epoch'] // 100) * 100
|
||||
dqn_by_range[epoch_range].append(r)
|
||||
|
||||
print(f"\n DQN Best Performance by Epoch Range:")
|
||||
for epoch_range in sorted(dqn_by_range.keys()):
|
||||
checkpoints = dqn_by_range[epoch_range]
|
||||
avg_sharpe = sum(r['sharpe_ratio'] for r in checkpoints) / len(checkpoints)
|
||||
best = max(checkpoints, key=lambda x: x['sharpe_ratio'])
|
||||
print(f" Epochs {epoch_range}-{epoch_range+99}: Avg Sharpe {avg_sharpe:.3f}, Best: Epoch {best['epoch']} (Sharpe {best['sharpe_ratio']:.3f})")
|
||||
|
||||
ppo_by_range = defaultdict(list)
|
||||
for r in ppo_results:
|
||||
epoch_range = (r['epoch'] // 100) * 100
|
||||
ppo_by_range[epoch_range].append(r)
|
||||
|
||||
print(f"\n PPO Best Performance by Epoch Range:")
|
||||
for epoch_range in sorted(ppo_by_range.keys()):
|
||||
checkpoints = ppo_by_range[epoch_range]
|
||||
avg_sharpe = sum(r['sharpe_ratio'] for r in checkpoints) / len(checkpoints)
|
||||
best = max(checkpoints, key=lambda x: x['sharpe_ratio'])
|
||||
print(f" Epochs {epoch_range}-{epoch_range+99}: Avg Sharpe {avg_sharpe:.3f}, Best: Epoch {best['epoch']} (Sharpe {best['sharpe_ratio']:.3f})")
|
||||
|
||||
print("\n3. **DQN vs PPO Comparison**")
|
||||
|
||||
overall_best = max(dqn_results + ppo_results, key=lambda x: x['sharpe_ratio'])
|
||||
print(f"\n 🏆 Overall Winner: {overall_best['model_type']} Epoch {overall_best['epoch']}")
|
||||
print(f" Sharpe Ratio: {overall_best['sharpe_ratio']:.3f}")
|
||||
print(f" PnL: ${overall_best['total_pnl']:.2f}")
|
||||
|
||||
return dqn_results, ppo_results
|
||||
|
||||
def create_markdown_report(dqn_results, ppo_results, output_dir):
|
||||
"""Create comprehensive markdown report"""
|
||||
|
||||
dqn_sorted = sorted(dqn_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
|
||||
ppo_sorted = sorted(ppo_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
|
||||
|
||||
report = []
|
||||
report.append("# Checkpoint Backtesting Results")
|
||||
report.append(f"\n**Date**: 2025-10-14")
|
||||
report.append(f"**Total Checkpoints Tested**: {len(dqn_results) + len(ppo_results)}")
|
||||
report.append(f"**Data**: 6E.FUT (Euro FX Futures), 7,223 bars, 4 days")
|
||||
report.append("\n---\n")
|
||||
|
||||
# Executive Summary
|
||||
best_dqn = max(dqn_results, key=lambda x: x['sharpe_ratio'])
|
||||
best_ppo = max(ppo_results, key=lambda x: x['sharpe_ratio'])
|
||||
|
||||
report.append("## Executive Summary")
|
||||
report.append(f"\n### DQN Performance")
|
||||
report.append(f"- **Best Checkpoint**: Epoch {best_dqn['epoch']}")
|
||||
report.append(f"- **Best Sharpe Ratio**: {best_dqn['sharpe_ratio']:.3f}")
|
||||
report.append(f"- **Best PnL**: ${best_dqn['total_pnl']:.2f}")
|
||||
report.append(f"- **Win Rate**: {best_dqn['win_rate']:.1f}%")
|
||||
|
||||
report.append(f"\n### PPO Performance")
|
||||
report.append(f"- **Best Checkpoint**: Epoch {best_ppo['epoch']}")
|
||||
report.append(f"- **Best Sharpe Ratio**: {best_ppo['sharpe_ratio']:.3f}")
|
||||
report.append(f"- **Best PnL**: ${best_ppo['total_pnl']:.2f}")
|
||||
report.append(f"- **Win Rate**: {best_ppo['win_rate']:.1f}%")
|
||||
|
||||
# Top 10 DQN
|
||||
report.append("\n---\n")
|
||||
report.append("## Top 10 DQN Checkpoints")
|
||||
report.append("\n| Rank | Epoch | Sharpe | Win Rate | Trades | PnL | Drawdown | Trade Freq |")
|
||||
report.append("|------|-------|--------|----------|--------|-----|----------|------------|")
|
||||
|
||||
for rank, r in enumerate(dqn_sorted, 1):
|
||||
report.append(f"| {rank} | {r['epoch']} | {r['sharpe_ratio']:.3f} | {r['win_rate']:.1f}% | {r['total_trades']} | ${r['total_pnl']:.2f} | {r['max_drawdown']:.2%} | {r['trade_frequency']:.1f} |")
|
||||
|
||||
# Top 10 PPO
|
||||
report.append("\n---\n")
|
||||
report.append("## Top 10 PPO Checkpoints")
|
||||
report.append("\n| Rank | Epoch | Sharpe | Win Rate | Trades | PnL | Drawdown | Trade Freq |")
|
||||
report.append("|------|-------|--------|----------|--------|-----|----------|------------|")
|
||||
|
||||
for rank, r in enumerate(ppo_sorted, 1):
|
||||
report.append(f"| {rank} | {r['epoch']} | {r['sharpe_ratio']:.3f} | {r['win_rate']:.1f}% | {r['total_trades']} | ${r['total_pnl']:.2f} | {r['max_drawdown']:.2%} | {r['trade_frequency']:.1f} |")
|
||||
|
||||
# Production Recommendations
|
||||
report.append("\n---\n")
|
||||
report.append("## Production Deployment Recommendations")
|
||||
report.append(f"\n### Primary Recommendation: **{best_dqn['model_type']} Epoch {best_dqn['epoch']}**")
|
||||
report.append(f"- Sharpe Ratio: {best_dqn['sharpe_ratio']:.3f}")
|
||||
report.append(f"- Win Rate: {best_dqn['win_rate']:.1f}%")
|
||||
report.append(f"- Total PnL: ${best_dqn['total_pnl']:.2f}")
|
||||
report.append(f"- Max Drawdown: {best_dqn['max_drawdown']:.2%}")
|
||||
|
||||
report.append(f"\n### Alternative: **{best_ppo['model_type']} Epoch {best_ppo['epoch']}**")
|
||||
report.append(f"- Sharpe Ratio: {best_ppo['sharpe_ratio']:.3f}")
|
||||
report.append(f"- Win Rate: {best_ppo['win_rate']:.1f}%")
|
||||
report.append(f"- Total PnL: ${best_ppo['total_pnl']:.2f}")
|
||||
report.append(f"- Max Drawdown: {best_ppo['max_drawdown']:.2%}")
|
||||
|
||||
# Save report
|
||||
report_file = output_dir / 'CHECKPOINT_BACKTEST_REPORT.md'
|
||||
with open(report_file, 'w') as f:
|
||||
f.write('\n'.join(report))
|
||||
|
||||
print(f"\n📄 Markdown report saved to: {report_file}")
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python analyze_checkpoints_simple.py <results_json_file>")
|
||||
sys.exit(1)
|
||||
|
||||
results_file = Path(sys.argv[1])
|
||||
if not results_file.exists():
|
||||
print(f"Error: Results file not found: {results_file}")
|
||||
sys.exit(1)
|
||||
|
||||
# Create output directory
|
||||
output_dir = Path(__file__).parent.parent / 'results'
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Load results
|
||||
print(f"\n📖 Loading results from: {results_file}")
|
||||
results = load_results(results_file)
|
||||
|
||||
# Filter valid results
|
||||
valid_results = filter_valid_results(results)
|
||||
print(f"✅ Found {len(valid_results)} checkpoints with valid trades")
|
||||
|
||||
# Create comprehensive analysis
|
||||
dqn_results, ppo_results = create_comparison_analysis(valid_results)
|
||||
|
||||
# Create markdown report
|
||||
create_markdown_report(dqn_results, ppo_results, output_dir)
|
||||
|
||||
print("\n✅ Analysis complete!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,98 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Foxhunt Hyperopt Docker Build Script
|
||||
# Builds CUDA-enabled hyperparameter optimization binaries using cargo-chef
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
IMAGE_NAME="${IMAGE_NAME:-jgrusewski/foxhunt-hyperopt}"
|
||||
IMAGE_TAG="${IMAGE_TAG:-latest}"
|
||||
GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
DOCKERFILE="Dockerfile.foxhunt-build"
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Foxhunt Hyperopt Docker Build${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
echo "Image: ${IMAGE_NAME}:${IMAGE_TAG}"
|
||||
echo "Commit: ${GIT_COMMIT}"
|
||||
echo "Build Date: ${BUILD_DATE}"
|
||||
echo ""
|
||||
|
||||
# Check if Docker is running
|
||||
if ! docker info > /dev/null 2>&1; then
|
||||
echo -e "${RED}Error: Docker is not running${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if Dockerfile exists
|
||||
if [ ! -f "${DOCKERFILE}" ]; then
|
||||
echo -e "${RED}Error: ${DOCKERFILE} not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Enable BuildKit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
echo -e "${YELLOW}Building Docker image...${NC}"
|
||||
echo ""
|
||||
|
||||
# Build with cache and build args
|
||||
docker build \
|
||||
--file "${DOCKERFILE}" \
|
||||
--tag "${IMAGE_NAME}:${IMAGE_TAG}" \
|
||||
--tag "${IMAGE_NAME}:${GIT_COMMIT}" \
|
||||
--build-arg GIT_COMMIT="${GIT_COMMIT}" \
|
||||
--build-arg BUILD_DATE="${BUILD_DATE}" \
|
||||
--build-arg CUDA_VERSION="12.4.1" \
|
||||
--build-arg CUDNN_VERSION="9" \
|
||||
--progress=plain \
|
||||
.
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo ""
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Build successful!${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
echo "Image: ${IMAGE_NAME}:${IMAGE_TAG}"
|
||||
echo "Image: ${IMAGE_NAME}:${GIT_COMMIT}"
|
||||
echo ""
|
||||
|
||||
# Get image size
|
||||
IMAGE_SIZE=$(docker images "${IMAGE_NAME}:${IMAGE_TAG}" --format "{{.Size}}")
|
||||
echo "Image size: ${IMAGE_SIZE}"
|
||||
echo ""
|
||||
|
||||
# List binaries in image
|
||||
echo -e "${YELLOW}Verifying binaries...${NC}"
|
||||
docker run --rm "${IMAGE_NAME}:${IMAGE_TAG}" ls -lh /usr/local/bin/hyperopt_*
|
||||
echo ""
|
||||
|
||||
# Verify GLIBC version
|
||||
echo -e "${YELLOW}Verifying GLIBC version...${NC}"
|
||||
docker run --rm "${IMAGE_NAME}:${IMAGE_TAG}" ldd --version | head -n 1
|
||||
echo ""
|
||||
|
||||
echo -e "${GREEN}Next steps:${NC}"
|
||||
echo "1. Test locally:"
|
||||
echo " docker run --rm --gpus all ${IMAGE_NAME}:${IMAGE_TAG} hyperopt_dqn_demo --help"
|
||||
echo ""
|
||||
echo "2. Push to registry:"
|
||||
echo " docker push ${IMAGE_NAME}:${IMAGE_TAG}"
|
||||
echo " docker push ${IMAGE_NAME}:${GIT_COMMIT}"
|
||||
echo ""
|
||||
else
|
||||
echo ""
|
||||
echo -e "${RED}========================================${NC}"
|
||||
echo -e "${RED}Build failed!${NC}"
|
||||
echo -e "${RED}========================================${NC}"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
import runpod
|
||||
|
||||
# Load environment
|
||||
env_path = Path.cwd().parent / '.env.runpod'
|
||||
load_dotenv(env_path)
|
||||
|
||||
runpod.api_key = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
# Get all GPU types with any availability
|
||||
gpu_types = runpod.get_gpus()
|
||||
available_count = 0
|
||||
|
||||
print("ALL GPUs with ≥16GB VRAM and ANY availability:\n")
|
||||
for gpu in gpu_types:
|
||||
memory_gb = gpu.get('memoryInGb', 0)
|
||||
secure = gpu.get('secureCloud', 0)
|
||||
community = gpu.get('communityCloud', 0)
|
||||
total = secure + community
|
||||
name = gpu.get('displayName', 'Unknown')
|
||||
|
||||
if memory_gb >= 16 and total > 0:
|
||||
price_info = gpu.get('lowestPrice', {})
|
||||
price = price_info.get('uninterruptablePrice', 'N/A') if price_info else 'N/A'
|
||||
|
||||
print(f"{name}:")
|
||||
print(f" VRAM: {memory_gb}GB")
|
||||
print(f" Secure cloud: {secure}")
|
||||
print(f" Community cloud: {community}")
|
||||
print(f" Total: {total}")
|
||||
print(f" Price: ${price}/hr" if price != 'N/A' else f" Price: {price}")
|
||||
print()
|
||||
available_count += 1
|
||||
|
||||
if available_count == 0:
|
||||
print("NO GPUs with ≥16GB VRAM are available right now.")
|
||||
print("\nMost common GPUs with ≥16GB VRAM (showing current status):")
|
||||
|
||||
common_gpus = ['RTX 4090', 'RTX 4000 Ada', 'RTX 5090', 'RTX A4000', 'A100', 'H100']
|
||||
for target in common_gpus:
|
||||
for gpu in gpu_types:
|
||||
if target in gpu.get('displayName', ''):
|
||||
memory_gb = gpu.get('memoryInGb', 0)
|
||||
if memory_gb >= 16:
|
||||
secure = gpu.get('secureCloud', 0)
|
||||
community = gpu.get('communityCloud', 0)
|
||||
print(f" {gpu['displayName']}: {secure + community} available")
|
||||
break
|
||||
@@ -1,30 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
import runpod
|
||||
|
||||
# Load environment
|
||||
env_path = Path.cwd().parent / '.env.runpod'
|
||||
load_dotenv(env_path)
|
||||
|
||||
runpod.api_key = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
# Get GPU types and check specific GPUs
|
||||
gpu_types = runpod.get_gpus()
|
||||
target_gpus = ['RTX 4090', 'RTX 4000 Ada', 'RTX 5090']
|
||||
|
||||
for target in target_gpus:
|
||||
for gpu in gpu_types:
|
||||
if target in gpu.get('displayName', ''):
|
||||
print(f"\n{target} details:")
|
||||
print(f" secureCloud: {gpu.get('secureCloud', 0)}")
|
||||
print(f" communityCloud: {gpu.get('communityCloud', 0)}")
|
||||
print(f" memoryInGb: {gpu.get('memoryInGb', 0)}")
|
||||
price_info = gpu.get('lowestPrice', {})
|
||||
if price_info:
|
||||
print(f" uninterruptablePrice: {price_info.get('uninterruptablePrice', 'N/A')}")
|
||||
else:
|
||||
print(f" price: N/A")
|
||||
break
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Quick status check for MAMBA-2 hyperopt on RunPod
|
||||
# Usage: ./check_hyperopt_status.sh [pod_id]
|
||||
|
||||
POD_ID="${1:-nyt86m4i89106a}"
|
||||
ENV_FILE="/home/jgrusewski/Work/foxhunt/.env.runpod"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "ERROR: .env.runpod not found at $ENV_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "$ENV_FILE"
|
||||
|
||||
echo "======================================================================"
|
||||
echo "MAMBA-2 Hyperopt Status Check"
|
||||
echo "======================================================================"
|
||||
echo "Pod ID: $POD_ID"
|
||||
echo "Time: $(date)"
|
||||
echo ""
|
||||
|
||||
# Check pod status via REST API
|
||||
echo "Fetching pod info..."
|
||||
RESPONSE=$(curl -s -H "Authorization: Bearer $RUNPOD_API_KEY" \
|
||||
"https://rest.runpod.io/v1/pods/$POD_ID")
|
||||
|
||||
STATUS=$(echo "$RESPONSE" | jq -r '.desiredStatus // "UNKNOWN"')
|
||||
GPU=$(echo "$RESPONSE" | jq -r '.machine.gpuType.displayName // "N/A"')
|
||||
DATACENTER=$(echo "$RESPONSE" | jq -r '.machine.dataCenterId // "N/A"')
|
||||
COST=$(echo "$RESPONSE" | jq -r '.costPerHr // "N/A"')
|
||||
|
||||
echo "Status: $STATUS"
|
||||
echo "GPU: $GPU"
|
||||
echo "Datacenter: $DATACENTER"
|
||||
echo "Cost: \$${COST}/hr"
|
||||
echo ""
|
||||
|
||||
if [ "$STATUS" = "RUNNING" ]; then
|
||||
echo "✅ Pod is RUNNING"
|
||||
echo ""
|
||||
echo "Access logs at: https://www.runpod.io/console/pods"
|
||||
echo "SSH: ssh root@${POD_ID}.ssh.runpod.io"
|
||||
echo ""
|
||||
echo "Expected completion: ~60 minutes from start"
|
||||
echo "Estimated cost: ~\$0.42 total"
|
||||
echo ""
|
||||
echo "CRITICAL: Monitor first 10 minutes for CUDA OOM errors!"
|
||||
echo "If OOM occurs with RTX A4000, redeploy with --batch-size-max 96"
|
||||
elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "EXITED" ]; then
|
||||
echo "✅ Pod has TERMINATED (training likely complete)"
|
||||
echo ""
|
||||
echo "Download results:"
|
||||
echo " aws s3 ls s3://se3zdnb5o4/models/ \\"
|
||||
echo " --profile runpod \\"
|
||||
echo " --endpoint-url https://s3api-eur-is-1.runpod.io \\"
|
||||
echo " --recursive"
|
||||
else
|
||||
echo "⏳ Pod status: $STATUS"
|
||||
echo ""
|
||||
echo "Wait 2-3 minutes for initialization, then check again"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
@@ -1,154 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Analyze Rust workspace for unused dependencies.
|
||||
Checks if dependencies in Cargo.toml are actually used in source code.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
|
||||
def parse_cargo_toml(path):
|
||||
"""Parse dependencies from Cargo.toml"""
|
||||
deps = []
|
||||
try:
|
||||
with open(path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract dependencies section
|
||||
in_deps = False
|
||||
in_dev_deps = False
|
||||
for line in content.split('\n'):
|
||||
if line.strip().startswith('[dependencies]'):
|
||||
in_deps = True
|
||||
in_dev_deps = False
|
||||
continue
|
||||
elif line.strip().startswith('[dev-dependencies]'):
|
||||
in_deps = False
|
||||
in_dev_deps = True
|
||||
continue
|
||||
elif line.strip().startswith('['):
|
||||
in_deps = False
|
||||
in_dev_deps = False
|
||||
continue
|
||||
|
||||
if in_deps and line.strip() and not line.strip().startswith('#'):
|
||||
# Extract dependency name
|
||||
match = re.match(r'^([a-zA-Z0-9_-]+)\s*=', line)
|
||||
if match:
|
||||
dep_name = match.group(1).replace('-', '_')
|
||||
deps.append((dep_name, 'normal'))
|
||||
elif in_dev_deps and line.strip() and not line.strip().startswith('#'):
|
||||
match = re.match(r'^([a-zA-Z0-9_-]+)\s*=', line)
|
||||
if match:
|
||||
dep_name = match.group(1).replace('-', '_')
|
||||
deps.append((dep_name, 'dev'))
|
||||
except Exception as e:
|
||||
print(f"Error parsing {path}: {e}")
|
||||
|
||||
return deps
|
||||
|
||||
def check_dep_usage(crate_path, dep_name):
|
||||
"""Check if dependency is used in Rust source files"""
|
||||
src_path = os.path.join(crate_path, 'src')
|
||||
tests_path = os.path.join(crate_path, 'tests')
|
||||
benches_path = os.path.join(crate_path, 'benches')
|
||||
examples_path = os.path.join(crate_path, 'examples')
|
||||
|
||||
patterns = [
|
||||
f'use {dep_name}',
|
||||
f'use {dep_name}::',
|
||||
f'extern crate {dep_name}',
|
||||
f'{dep_name}::',
|
||||
]
|
||||
|
||||
for base_path in [src_path, tests_path, benches_path, examples_path]:
|
||||
if not os.path.exists(base_path):
|
||||
continue
|
||||
|
||||
for root, dirs, files in os.walk(base_path):
|
||||
# Skip target directory
|
||||
if 'target' in dirs:
|
||||
dirs.remove('target')
|
||||
|
||||
for file in files:
|
||||
if not file.endswith('.rs'):
|
||||
continue
|
||||
|
||||
file_path = os.path.join(root, file)
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
for pattern in patterns:
|
||||
if pattern in content:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
def analyze_workspace():
|
||||
"""Analyze all crates in workspace"""
|
||||
workspace_root = '/home/jgrusewski/Work/foxhunt'
|
||||
|
||||
# Find all Cargo.toml files (excluding target directories)
|
||||
cargo_tomls = []
|
||||
for root, dirs, files in os.walk(workspace_root):
|
||||
# Skip target and vendor directories
|
||||
if 'target' in dirs:
|
||||
dirs.remove('target')
|
||||
if '.git' in dirs:
|
||||
dirs.remove('.git')
|
||||
|
||||
if 'Cargo.toml' in files:
|
||||
cargo_path = os.path.join(root, 'Cargo.toml')
|
||||
# Skip if it's a workspace-only Cargo.toml
|
||||
with open(cargo_path, 'r') as f:
|
||||
content = f.read()
|
||||
if '[package]' in content:
|
||||
cargo_tomls.append(cargo_path)
|
||||
|
||||
results = {}
|
||||
|
||||
for cargo_toml in cargo_tomls[:15]: # Limit to first 15 for performance
|
||||
crate_path = os.path.dirname(cargo_toml)
|
||||
crate_name = os.path.basename(crate_path)
|
||||
|
||||
print(f"\nAnalyzing {crate_name}...")
|
||||
|
||||
deps = parse_cargo_toml(cargo_toml)
|
||||
unused = []
|
||||
|
||||
for dep_name, dep_type in deps:
|
||||
# Skip workspace crates
|
||||
if dep_name in ['trading_engine', 'risk', 'data', 'backtesting',
|
||||
'common', 'storage', 'ml', 'config', 'database',
|
||||
'market_data', 'tli', 'risk_data', 'trading_data',
|
||||
'adaptive_strategy', 'model_loader']:
|
||||
continue
|
||||
|
||||
# Check if dependency is used
|
||||
if not check_dep_usage(crate_path, dep_name):
|
||||
unused.append((dep_name, dep_type))
|
||||
|
||||
if unused:
|
||||
results[crate_name] = unused
|
||||
|
||||
return results
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("=== UNUSED DEPENDENCY ANALYSIS ===\n")
|
||||
results = analyze_workspace()
|
||||
|
||||
if not results:
|
||||
print("\n✅ No obvious unused dependencies found!")
|
||||
else:
|
||||
print("\n⚠️ POTENTIALLY UNUSED DEPENDENCIES:\n")
|
||||
for crate, unused in results.items():
|
||||
print(f"\n{crate}:")
|
||||
for dep, dep_type in unused:
|
||||
print(f" - {dep} ({dep_type})")
|
||||
|
||||
print("\n=== ANALYSIS COMPLETE ===")
|
||||
@@ -1,332 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Checkpoint Comparison Analysis
|
||||
Analyzes backtest results for all DQN and PPO checkpoints
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib
|
||||
matplotlib.use('Agg') # Non-interactive backend
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
def load_results(results_file):
|
||||
"""Load backtest results from JSON"""
|
||||
with open(results_file, 'r') as f:
|
||||
return json.load(f)
|
||||
|
||||
def filter_valid_results(results):
|
||||
"""Filter out checkpoints with no trades or invalid metrics"""
|
||||
return [r for r in results if r['total_trades'] > 0]
|
||||
|
||||
def create_comparison_table(results):
|
||||
"""Create markdown comparison table"""
|
||||
df = pd.DataFrame(results)
|
||||
|
||||
# Separate DQN and PPO
|
||||
dqn_df = df[df['model_type'] == 'DQN'].copy()
|
||||
ppo_df = df[df['model_type'] == 'PPO'].copy()
|
||||
|
||||
# Sort by Sharpe ratio
|
||||
dqn_df_sorted = dqn_df.sort_values('sharpe_ratio', ascending=False).head(10)
|
||||
ppo_df_sorted = ppo_df.sort_values('sharpe_ratio', ascending=False).head(10)
|
||||
|
||||
print("\n" + "="*90)
|
||||
print("🔵 TOP 10 DQN CHECKPOINTS (Ranked by Sharpe Ratio)")
|
||||
print("="*90)
|
||||
print(f"{'Epoch':<8} {'Trades':<8} {'Win Rate':<12} {'Sharpe':<10} {'PnL':<12} {'Drawdown':<12} {'Trade Freq':<12}")
|
||||
print("-"*90)
|
||||
|
||||
for _, row in dqn_df_sorted.iterrows():
|
||||
print(f"{row['epoch']:<8} {row['total_trades']:<8} {row['win_rate']:>10.1f}% {row['sharpe_ratio']:>10.3f} ${row['total_pnl']:>10.2f} {row['max_drawdown']:>11.2%} {row['trade_frequency']:>12.1f}")
|
||||
|
||||
print("\n" + "="*90)
|
||||
print("🟢 TOP 10 PPO CHECKPOINTS (Ranked by Sharpe Ratio)")
|
||||
print("="*90)
|
||||
print(f"{'Epoch':<8} {'Trades':<8} {'Win Rate':<12} {'Sharpe':<10} {'PnL':<12} {'Drawdown':<12} {'Trade Freq':<12}")
|
||||
print("-"*90)
|
||||
|
||||
for _, row in ppo_df_sorted.iterrows():
|
||||
print(f"{row['epoch']:<8} {row['total_trades']:<8} {row['win_rate']:>10.1f}% {row['sharpe_ratio']:>10.3f} ${row['total_pnl']:>10.2f} {row['max_drawdown']:>11.2%} {row['trade_frequency']:>12.1f}")
|
||||
|
||||
return dqn_df, ppo_df
|
||||
|
||||
def plot_epoch_vs_metrics(dqn_df, ppo_df, output_dir):
|
||||
"""Create plots comparing epoch vs various metrics"""
|
||||
|
||||
# Filter valid results (with trades)
|
||||
dqn_valid = dqn_df[dqn_df['total_trades'] > 0]
|
||||
ppo_valid = ppo_df[ppo_df['total_trades'] > 0]
|
||||
|
||||
# Create figure with 2x2 subplots
|
||||
fig, axes = plt.subplots(2, 2, figsize=(16, 12))
|
||||
fig.suptitle('DQN vs PPO: Checkpoint Performance Analysis', fontsize=16, fontweight='bold')
|
||||
|
||||
# 1. Sharpe Ratio vs Epoch
|
||||
ax1 = axes[0, 0]
|
||||
ax1.scatter(dqn_valid['epoch'], dqn_valid['sharpe_ratio'],
|
||||
alpha=0.6, s=100, c='blue', label='DQN', marker='o')
|
||||
ax1.scatter(ppo_valid['epoch'], ppo_valid['sharpe_ratio'],
|
||||
alpha=0.6, s=100, c='green', label='PPO', marker='s')
|
||||
ax1.axhline(y=0, color='red', linestyle='--', alpha=0.5, label='Break-even')
|
||||
ax1.set_xlabel('Epoch', fontsize=12)
|
||||
ax1.set_ylabel('Sharpe Ratio', fontsize=12)
|
||||
ax1.set_title('Sharpe Ratio vs Training Epoch', fontsize=14, fontweight='bold')
|
||||
ax1.legend()
|
||||
ax1.grid(True, alpha=0.3)
|
||||
|
||||
# 2. Trade Count vs Epoch
|
||||
ax2 = axes[0, 1]
|
||||
ax2.scatter(dqn_valid['epoch'], dqn_valid['total_trades'],
|
||||
alpha=0.6, s=100, c='blue', label='DQN', marker='o')
|
||||
ax2.scatter(ppo_valid['epoch'], ppo_valid['total_trades'],
|
||||
alpha=0.6, s=100, c='green', label='PPO', marker='s')
|
||||
ax2.set_xlabel('Epoch', fontsize=12)
|
||||
ax2.set_ylabel('Total Trades', fontsize=12)
|
||||
ax2.set_title('Trade Count vs Training Epoch', fontsize=14, fontweight='bold')
|
||||
ax2.legend()
|
||||
ax2.grid(True, alpha=0.3)
|
||||
|
||||
# 3. Win Rate vs Epoch
|
||||
ax3 = axes[1, 0]
|
||||
ax3.scatter(dqn_valid['epoch'], dqn_valid['win_rate'],
|
||||
alpha=0.6, s=100, c='blue', label='DQN', marker='o')
|
||||
ax3.scatter(ppo_valid['epoch'], ppo_valid['win_rate'],
|
||||
alpha=0.6, s=100, c='green', label='PPO', marker='s')
|
||||
ax3.axhline(y=50, color='red', linestyle='--', alpha=0.5, label='50% Win Rate')
|
||||
ax3.set_xlabel('Epoch', fontsize=12)
|
||||
ax3.set_ylabel('Win Rate (%)', fontsize=12)
|
||||
ax3.set_title('Win Rate vs Training Epoch', fontsize=14, fontweight='bold')
|
||||
ax3.legend()
|
||||
ax3.grid(True, alpha=0.3)
|
||||
|
||||
# 4. PnL vs Epoch
|
||||
ax4 = axes[1, 1]
|
||||
ax4.scatter(dqn_valid['epoch'], dqn_valid['total_pnl'],
|
||||
alpha=0.6, s=100, c='blue', label='DQN', marker='o')
|
||||
ax4.scatter(ppo_valid['epoch'], ppo_valid['total_pnl'],
|
||||
alpha=0.6, s=100, c='green', label='PPO', marker='s')
|
||||
ax4.axhline(y=0, color='red', linestyle='--', alpha=0.5, label='Break-even')
|
||||
ax4.set_xlabel('Epoch', fontsize=12)
|
||||
ax4.set_ylabel('Total PnL ($)', fontsize=12)
|
||||
ax4.set_title('Total PnL vs Training Epoch', fontsize=14, fontweight='bold')
|
||||
ax4.legend()
|
||||
ax4.grid(True, alpha=0.3)
|
||||
|
||||
plt.tight_layout()
|
||||
output_file = output_dir / 'checkpoint_comparison_plots.png'
|
||||
plt.savefig(output_file, dpi=300, bbox_inches='tight')
|
||||
print(f"\n📊 Saved comparison plots to: {output_file}")
|
||||
plt.close()
|
||||
|
||||
def plot_sharpe_distribution(dqn_df, ppo_df, output_dir):
|
||||
"""Create box plot comparing Sharpe ratio distributions"""
|
||||
|
||||
dqn_valid = dqn_df[dqn_df['total_trades'] > 0]['sharpe_ratio']
|
||||
ppo_valid = ppo_df[ppo_df['total_trades'] > 0]['sharpe_ratio']
|
||||
|
||||
fig, ax = plt.subplots(figsize=(10, 6))
|
||||
|
||||
# Create box plots
|
||||
box_data = [dqn_valid, ppo_valid]
|
||||
bp = ax.boxplot(box_data, labels=['DQN', 'PPO'], patch_artist=True,
|
||||
showmeans=True, meanline=True)
|
||||
|
||||
# Customize colors
|
||||
colors = ['lightblue', 'lightgreen']
|
||||
for patch, color in zip(bp['boxes'], colors):
|
||||
patch.set_facecolor(color)
|
||||
|
||||
ax.set_ylabel('Sharpe Ratio', fontsize=12)
|
||||
ax.set_title('Sharpe Ratio Distribution: DQN vs PPO', fontsize=14, fontweight='bold')
|
||||
ax.axhline(y=0, color='red', linestyle='--', alpha=0.5, label='Break-even')
|
||||
ax.grid(True, alpha=0.3)
|
||||
ax.legend()
|
||||
|
||||
plt.tight_layout()
|
||||
output_file = output_dir / 'sharpe_distribution.png'
|
||||
plt.savefig(output_file, dpi=300, bbox_inches='tight')
|
||||
print(f"📊 Saved Sharpe distribution plot to: {output_file}")
|
||||
plt.close()
|
||||
|
||||
def create_statistical_summary(dqn_df, ppo_df):
|
||||
"""Generate statistical summary"""
|
||||
|
||||
dqn_valid = dqn_df[dqn_df['total_trades'] > 0]
|
||||
ppo_valid = ppo_df[ppo_df['total_trades'] > 0]
|
||||
|
||||
print("\n" + "="*90)
|
||||
print("📈 STATISTICAL SUMMARY")
|
||||
print("="*90)
|
||||
|
||||
print(f"\n{'Metric':<25} {'DQN':>20} {'PPO':>20} {'Winner':>20}")
|
||||
print("-"*90)
|
||||
|
||||
metrics = [
|
||||
('Checkpoints with trades', len(dqn_valid), len(ppo_valid)),
|
||||
('Avg Sharpe Ratio', dqn_valid['sharpe_ratio'].mean(), ppo_valid['sharpe_ratio'].mean()),
|
||||
('Max Sharpe Ratio', dqn_valid['sharpe_ratio'].max(), ppo_valid['sharpe_ratio'].max()),
|
||||
('Avg Win Rate (%)', dqn_valid['win_rate'].mean(), ppo_valid['win_rate'].mean()),
|
||||
('Avg Total Trades', dqn_valid['total_trades'].mean(), ppo_valid['total_trades'].mean()),
|
||||
('Avg PnL ($)', dqn_valid['total_pnl'].mean(), ppo_valid['total_pnl'].mean()),
|
||||
('Best PnL ($)', dqn_valid['total_pnl'].max(), ppo_valid['total_pnl'].max()),
|
||||
]
|
||||
|
||||
for name, dqn_val, ppo_val in metrics:
|
||||
if name == 'Checkpoints with trades':
|
||||
winner = 'DQN' if dqn_val > ppo_val else 'PPO' if ppo_val > dqn_val else 'Tie'
|
||||
print(f"{name:<25} {int(dqn_val):>20} {int(ppo_val):>20} {winner:>20}")
|
||||
else:
|
||||
winner = 'DQN' if dqn_val > ppo_val else 'PPO' if ppo_val > dqn_val else 'Tie'
|
||||
print(f"{name:<25} {dqn_val:>20.3f} {ppo_val:>20.3f} {winner:>20}")
|
||||
|
||||
# Best overall checkpoints
|
||||
print("\n" + "="*90)
|
||||
print("🏆 BEST CHECKPOINTS")
|
||||
print("="*90)
|
||||
|
||||
best_dqn = dqn_valid.loc[dqn_valid['sharpe_ratio'].idxmax()]
|
||||
best_ppo = ppo_valid.loc[ppo_valid['sharpe_ratio'].idxmax()]
|
||||
|
||||
print(f"\nBest DQN: Epoch {best_dqn['epoch']}")
|
||||
print(f" Sharpe: {best_dqn['sharpe_ratio']:.3f}")
|
||||
print(f" Win Rate: {best_dqn['win_rate']:.1f}%")
|
||||
print(f" Trades: {best_dqn['total_trades']}")
|
||||
print(f" PnL: ${best_dqn['total_pnl']:.2f}")
|
||||
|
||||
print(f"\nBest PPO: Epoch {best_ppo['epoch']}")
|
||||
print(f" Sharpe: {best_ppo['sharpe_ratio']:.3f}")
|
||||
print(f" Win Rate: {best_ppo['win_rate']:.1f}%")
|
||||
print(f" Trades: {best_ppo['total_trades']}")
|
||||
print(f" PnL: ${best_ppo['total_pnl']:.2f}")
|
||||
|
||||
def create_markdown_report(dqn_df, ppo_df, output_dir):
|
||||
"""Create comprehensive markdown report"""
|
||||
|
||||
dqn_valid = dqn_df[dqn_df['total_trades'] > 0]
|
||||
ppo_valid = ppo_df[ppo_df['total_trades'] > 0]
|
||||
|
||||
# Get top 10 from each
|
||||
dqn_top10 = dqn_valid.sort_values('sharpe_ratio', ascending=False).head(10)
|
||||
ppo_top10 = ppo_valid.sort_values('sharpe_ratio', ascending=False).head(10)
|
||||
|
||||
report = []
|
||||
report.append("# Checkpoint Backtesting Results")
|
||||
report.append(f"\n**Date**: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
report.append(f"**Total Checkpoints Tested**: {len(dqn_df) + len(ppo_df)}")
|
||||
report.append(f"**Checkpoints with Valid Trades**: {len(dqn_valid) + len(ppo_valid)}")
|
||||
report.append("\n---\n")
|
||||
|
||||
# Executive Summary
|
||||
report.append("## Executive Summary")
|
||||
report.append(f"\n### DQN Performance")
|
||||
report.append(f"- **Best Checkpoint**: Epoch {dqn_valid['sharpe_ratio'].idxmax()}")
|
||||
report.append(f"- **Best Sharpe Ratio**: {dqn_valid['sharpe_ratio'].max():.3f}")
|
||||
report.append(f"- **Average Sharpe Ratio**: {dqn_valid['sharpe_ratio'].mean():.3f}")
|
||||
report.append(f"- **Best PnL**: ${dqn_valid['total_pnl'].max():.2f}")
|
||||
|
||||
report.append(f"\n### PPO Performance")
|
||||
report.append(f"- **Best Checkpoint**: Epoch {ppo_valid['sharpe_ratio'].idxmax()}")
|
||||
report.append(f"- **Best Sharpe Ratio**: {ppo_valid['sharpe_ratio'].max():.3f}")
|
||||
report.append(f"- **Average Sharpe Ratio**: {ppo_valid['sharpe_ratio'].mean():.3f}")
|
||||
report.append(f"- **Best PnL**: ${ppo_valid['total_pnl'].max():.2f}")
|
||||
|
||||
# Top 10 DQN
|
||||
report.append("\n---\n")
|
||||
report.append("## Top 10 DQN Checkpoints")
|
||||
report.append("\n| Rank | Epoch | Sharpe | Win Rate | Trades | PnL | Drawdown | Trade Freq |")
|
||||
report.append("|------|-------|--------|----------|--------|-----|----------|------------|")
|
||||
|
||||
for rank, (_, row) in enumerate(dqn_top10.iterrows(), 1):
|
||||
report.append(f"| {rank} | {row['epoch']} | {row['sharpe_ratio']:.3f} | {row['win_rate']:.1f}% | {row['total_trades']} | ${row['total_pnl']:.2f} | {row['max_drawdown']:.2%} | {row['trade_frequency']:.1f} |")
|
||||
|
||||
# Top 10 PPO
|
||||
report.append("\n---\n")
|
||||
report.append("## Top 10 PPO Checkpoints")
|
||||
report.append("\n| Rank | Epoch | Sharpe | Win Rate | Trades | PnL | Drawdown | Trade Freq |")
|
||||
report.append("|------|-------|--------|----------|--------|-----|----------|------------|")
|
||||
|
||||
for rank, (_, row) in enumerate(ppo_top10.iterrows(), 1):
|
||||
report.append(f"| {rank} | {row['epoch']} | {row['sharpe_ratio']:.3f} | {row['win_rate']:.1f}% | {row['total_trades']} | ${row['total_pnl']:.2f} | {row['max_drawdown']:.2%} | {row['trade_frequency']:.1f} |")
|
||||
|
||||
# Key Insights
|
||||
report.append("\n---\n")
|
||||
report.append("## Key Insights")
|
||||
|
||||
# Insight 1: Early vs Late epochs
|
||||
dqn_early = dqn_valid[dqn_valid['epoch'] <= 200]
|
||||
dqn_late = dqn_valid[dqn_valid['epoch'] > 200]
|
||||
|
||||
report.append("\n### Training Phase Analysis")
|
||||
report.append(f"\n**DQN Early Epochs (≤200)**:")
|
||||
report.append(f"- Average Sharpe: {dqn_early['sharpe_ratio'].mean():.3f}")
|
||||
report.append(f"- Average Trades: {dqn_early['total_trades'].mean():.1f}")
|
||||
report.append(f"- Average Win Rate: {dqn_early['win_rate'].mean():.1f}%")
|
||||
|
||||
report.append(f"\n**DQN Late Epochs (>200)**:")
|
||||
report.append(f"- Average Sharpe: {dqn_late['sharpe_ratio'].mean():.3f}")
|
||||
report.append(f"- Average Trades: {dqn_late['total_trades'].mean():.1f}")
|
||||
report.append(f"- Average Win Rate: {dqn_late['win_rate'].mean():.1f}%")
|
||||
|
||||
ppo_early = ppo_valid[ppo_valid['epoch'] <= 200]
|
||||
ppo_late = ppo_valid[ppo_valid['epoch'] > 200]
|
||||
|
||||
report.append(f"\n**PPO Early Epochs (≤200)**:")
|
||||
report.append(f"- Average Sharpe: {ppo_early['sharpe_ratio'].mean():.3f}")
|
||||
report.append(f"- Average Trades: {ppo_early['total_trades'].mean():.1f}")
|
||||
report.append(f"- Average Win Rate: {ppo_early['win_rate'].mean():.1f}%")
|
||||
|
||||
report.append(f"\n**PPO Late Epochs (>200)**:")
|
||||
report.append(f"- Average Sharpe: {ppo_late['sharpe_ratio'].mean():.3f}")
|
||||
report.append(f"- Average Trades: {ppo_late['total_trades'].mean():.1f}")
|
||||
report.append(f"- Average Win Rate: {ppo_late['win_rate'].mean():.1f}%")
|
||||
|
||||
# Save report
|
||||
report_file = output_dir / 'CHECKPOINT_BACKTEST_REPORT.md'
|
||||
with open(report_file, 'w') as f:
|
||||
f.write('\n'.join(report))
|
||||
|
||||
print(f"\n📄 Saved markdown report to: {report_file}")
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python compare_checkpoints.py <results_json_file>")
|
||||
sys.exit(1)
|
||||
|
||||
results_file = Path(sys.argv[1])
|
||||
if not results_file.exists():
|
||||
print(f"Error: Results file not found: {results_file}")
|
||||
sys.exit(1)
|
||||
|
||||
# Create output directory
|
||||
output_dir = Path(__file__).parent.parent / 'results'
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Load results
|
||||
print(f"📖 Loading results from: {results_file}")
|
||||
results = load_results(results_file)
|
||||
|
||||
# Filter valid results
|
||||
valid_results = filter_valid_results(results)
|
||||
print(f"✅ Found {len(valid_results)} checkpoints with valid trades")
|
||||
|
||||
# Create comparison table
|
||||
dqn_df, ppo_df = create_comparison_table(valid_results)
|
||||
|
||||
# Statistical summary
|
||||
create_statistical_summary(dqn_df, ppo_df)
|
||||
|
||||
# Create plots
|
||||
plot_epoch_vs_metrics(dqn_df, ppo_df, output_dir)
|
||||
plot_sharpe_distribution(dqn_df, ppo_df, output_dir)
|
||||
|
||||
# Create markdown report
|
||||
create_markdown_report(dqn_df, ppo_df, output_dir)
|
||||
|
||||
print("\n✅ Analysis complete!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Convert CSV OHLCV data to Parquet format matching ParquetMarketDataEvent schema.
|
||||
|
||||
This script transforms candlestick (OHLCV) data from CSV files into market data events
|
||||
that match the Foxhunt trading system's Parquet schema.
|
||||
"""
|
||||
|
||||
import polars as pl
|
||||
from datetime import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
def convert_csv_to_parquet(csv_path: str, parquet_path: str, symbol: str, venue: str = "yahoo_finance") -> dict:
|
||||
"""
|
||||
Convert CSV OHLCV data to Parquet format.
|
||||
|
||||
Args:
|
||||
csv_path: Path to input CSV file
|
||||
parquet_path: Path to output Parquet file
|
||||
symbol: Trading symbol (e.g., "BTC/USD")
|
||||
venue: Trading venue identifier
|
||||
|
||||
Returns:
|
||||
Dictionary with conversion statistics
|
||||
"""
|
||||
print(f"Converting {csv_path} to {parquet_path}...")
|
||||
|
||||
# Read CSV
|
||||
df = pl.read_csv(csv_path)
|
||||
print(f" Loaded {len(df)} rows from CSV")
|
||||
|
||||
# Convert timestamp to nanoseconds (Unix epoch)
|
||||
df = df.with_columns([
|
||||
pl.col('timestamp').str.strptime(pl.Datetime, "%Y-%m-%d %H:%M:%S")
|
||||
.dt.epoch(time_unit='ns').alias('timestamp_ns')
|
||||
])
|
||||
|
||||
# Create 4 events per candle: Open, High, Low, Close
|
||||
# We'll use the close price as the primary event and include OHLCV in metadata
|
||||
|
||||
events = []
|
||||
|
||||
for row in df.iter_rows(named=True):
|
||||
timestamp_ns = row['timestamp_ns']
|
||||
|
||||
# Create a Trade event using the close price as the representative price
|
||||
# In a real scenario, we'd have tick data, but this is a reasonable approximation
|
||||
event = {
|
||||
'timestamp_ns': timestamp_ns,
|
||||
'symbol': symbol,
|
||||
'venue': venue,
|
||||
'event_type': 'Trade', # Using Trade as the event type for OHLCV data
|
||||
'price': row['close'],
|
||||
'quantity': row['volume'],
|
||||
'latency_ns': None # No latency data for historical data
|
||||
}
|
||||
events.append(event)
|
||||
|
||||
# Create DataFrame from events
|
||||
events_df = pl.DataFrame(events)
|
||||
|
||||
# Add sequence numbers using row_index
|
||||
events_df = events_df.with_row_index(name='sequence')
|
||||
|
||||
# Ensure correct data types matching ParquetMarketDataEvent schema
|
||||
events_df = events_df.with_columns([
|
||||
pl.col('timestamp_ns').cast(pl.Int64),
|
||||
pl.col('symbol').cast(pl.Utf8),
|
||||
pl.col('venue').cast(pl.Utf8),
|
||||
pl.col('event_type').cast(pl.Utf8),
|
||||
pl.col('price').cast(pl.Float64),
|
||||
pl.col('quantity').cast(pl.Float64),
|
||||
pl.col('sequence').cast(pl.UInt64),
|
||||
pl.col('latency_ns').cast(pl.UInt64)
|
||||
])
|
||||
|
||||
# Write Parquet with Snappy compression
|
||||
events_df.write_parquet(parquet_path, compression='snappy')
|
||||
|
||||
# Get file sizes
|
||||
csv_size = Path(csv_path).stat().st_size / (1024 * 1024) # MB
|
||||
parquet_size = Path(parquet_path).stat().st_size / (1024 * 1024) # MB
|
||||
compression_ratio = csv_size / parquet_size if parquet_size > 0 else 0
|
||||
|
||||
stats = {
|
||||
'csv_path': csv_path,
|
||||
'parquet_path': parquet_path,
|
||||
'csv_size_mb': round(csv_size, 2),
|
||||
'parquet_size_mb': round(parquet_size, 2),
|
||||
'compression_ratio': round(compression_ratio, 2),
|
||||
'rows': len(events_df),
|
||||
'csv_rows': len(df),
|
||||
'validation': 'passed'
|
||||
}
|
||||
|
||||
print(f" Created {len(events_df)} events from {len(df)} candles")
|
||||
print(f" CSV size: {stats['csv_size_mb']:.2f} MB")
|
||||
print(f" Parquet size: {stats['parquet_size_mb']:.2f} MB")
|
||||
print(f" Compression ratio: {stats['compression_ratio']:.2f}x")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
"""Main conversion workflow."""
|
||||
base_dir = Path("/home/jgrusewski/Work/foxhunt/test_data/real")
|
||||
csv_dir = base_dir / "csv"
|
||||
parquet_dir = base_dir / "parquet"
|
||||
|
||||
# Ensure output directory exists
|
||||
parquet_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Define conversions
|
||||
conversions = [
|
||||
{
|
||||
'csv': str(csv_dir / "BTC-USD_30day_2024-09.csv"),
|
||||
'parquet': str(parquet_dir / "BTC-USD_30day_2024-09.parquet"),
|
||||
'symbol': 'BTC/USD'
|
||||
},
|
||||
{
|
||||
'csv': str(csv_dir / "ETH-USD_30day_2024-09.csv"),
|
||||
'parquet': str(parquet_dir / "ETH-USD_30day_2024-09.parquet"),
|
||||
'symbol': 'ETH/USD'
|
||||
}
|
||||
]
|
||||
|
||||
# Track results
|
||||
results = {
|
||||
'conversion_timestamp': datetime.now().astimezone().isoformat(),
|
||||
'conversions': {}
|
||||
}
|
||||
|
||||
# Convert each file
|
||||
for conv in conversions:
|
||||
try:
|
||||
stats = convert_csv_to_parquet(
|
||||
conv['csv'],
|
||||
conv['parquet'],
|
||||
conv['symbol']
|
||||
)
|
||||
results['conversions'][conv['symbol']] = stats
|
||||
print(f"✓ Successfully converted {conv['symbol']}\n")
|
||||
except Exception as e:
|
||||
print(f"✗ Failed to convert {conv['symbol']}: {e}\n")
|
||||
results['conversions'][conv['symbol']] = {
|
||||
'error': str(e),
|
||||
'validation': 'failed'
|
||||
}
|
||||
|
||||
# Save conversion report
|
||||
report_path = parquet_dir / "CONVERSION_REPORT.json"
|
||||
with open(report_path, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
print(f"\n✓ Conversion report saved to {report_path}")
|
||||
|
||||
# Print summary
|
||||
print("\n" + "="*60)
|
||||
print("CONVERSION SUMMARY")
|
||||
print("="*60)
|
||||
successful = sum(1 for c in results['conversions'].values() if c.get('validation') == 'passed')
|
||||
total = len(results['conversions'])
|
||||
print(f"Status: {successful}/{total} conversions successful")
|
||||
|
||||
for symbol, stats in results['conversions'].items():
|
||||
if stats.get('validation') == 'passed':
|
||||
print(f"\n{symbol}:")
|
||||
print(f" Rows: {stats['rows']:,}")
|
||||
print(f" Size: {stats['csv_size_mb']:.2f} MB → {stats['parquet_size_mb']:.2f} MB")
|
||||
print(f" Compression: {stats['compression_ratio']:.2f}x")
|
||||
|
||||
return 0 if successful == total else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,140 +0,0 @@
|
||||
#!/usr/bin/env cargo +nightly -Zscript
|
||||
//! Databento API Test & Balance Check Script
|
||||
//!
|
||||
//! Purpose: Verify API key, check credit balance, and test minimal data download
|
||||
//!
|
||||
//! Usage:
|
||||
//! cargo run --bin databento_test
|
||||
//!
|
||||
//! Requirements:
|
||||
//! - DATABENTO_API_KEY environment variable set
|
||||
//! - Network connectivity to Databento API
|
||||
//!
|
||||
//! Safety:
|
||||
//! - NO automatic downloads
|
||||
//! - Displays costs BEFORE any data download
|
||||
//! - Requires explicit confirmation
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::error::Error;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct AccountInfo {
|
||||
balance: f64,
|
||||
currency: String,
|
||||
organization: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct DatasetInfo {
|
||||
dataset: String,
|
||||
schema: String,
|
||||
cost_per_gb: f64,
|
||||
available_symbols: Vec<String>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn Error>> {
|
||||
println!("🔍 Databento API Test & Balance Checker");
|
||||
println!("========================================\n");
|
||||
|
||||
// 1. Check for API key
|
||||
let api_key = std::env::var("DATABENTO_API_KEY")
|
||||
.map_err(|_| "DATABENTO_API_KEY not set in environment")?;
|
||||
|
||||
println!("✅ API Key found: {}...{}",
|
||||
&api_key[..10],
|
||||
&api_key[api_key.len()-10..]);
|
||||
|
||||
// 2. Verify API key and get account info
|
||||
println!("\n📊 Checking account balance...");
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Note: Databento API endpoint for account info
|
||||
// This is a placeholder - actual endpoint needs to be verified from docs
|
||||
let account_url = "https://api.databento.com/v0/account";
|
||||
|
||||
let response = client
|
||||
.get(account_url)
|
||||
.header("Authorization", format!("Bearer {}", api_key))
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match response {
|
||||
Ok(resp) => {
|
||||
if resp.status().is_success() {
|
||||
println!("✅ API key is valid!");
|
||||
|
||||
// Try to parse account info
|
||||
if let Ok(text) = resp.text().await {
|
||||
println!("\n📋 Account Info:");
|
||||
println!("{}", text);
|
||||
}
|
||||
} else {
|
||||
let status = resp.status();
|
||||
let error_text = resp.text().await.unwrap_or_default();
|
||||
println!("❌ API key validation failed:");
|
||||
println!(" Status: {}", status);
|
||||
println!(" Error: {}", error_text);
|
||||
return Err(format!("Invalid API key: {}", status).into());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
println!("❌ Network error connecting to Databento API:");
|
||||
println!(" {}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Display pricing information
|
||||
println!("\n💰 Dataset Pricing Information:");
|
||||
println!("================================");
|
||||
println!("\nFrom Databento documentation:");
|
||||
println!("• $125 FREE credits for historical data");
|
||||
println!("• Pricing is per GB consumed");
|
||||
println!("• OHLCV bars (cheapest): ~$0.50-$2.00 per GB");
|
||||
println!("• Trades: ~$5-$15 per GB");
|
||||
println!("• L2 Order Book (MBP): ~$10-$30 per GB");
|
||||
println!("• L3 Order Book (MBO): ~$30-$100 per GB");
|
||||
|
||||
// 4. Recommended minimal test download
|
||||
println!("\n🎯 Recommended Minimal Test Download:");
|
||||
println!("====================================");
|
||||
println!("Symbol: ES.FUT (E-mini S&P 500 futures)");
|
||||
println!("Dataset: GLBX.MDP3 (CME MDP 3.0)");
|
||||
println!("Schema: ohlcv-1m (1-minute bars)");
|
||||
println!("Date Range: 1 day (e.g., 2024-01-02)");
|
||||
println!("Est. Size: ~5-20 MB");
|
||||
println!("Est. Cost: ~$0.01-$0.05 (well under $125 limit)");
|
||||
|
||||
println!("\nAlternative (even cheaper):");
|
||||
println!("Symbol: SPY (S&P 500 ETF)");
|
||||
println!("Dataset: XNAS.ITCH (Nasdaq)");
|
||||
println!("Schema: ohlcv-1m");
|
||||
println!("Date Range: 1 day");
|
||||
println!("Est. Cost: ~$0.005-$0.02");
|
||||
|
||||
// 5. Cost estimation tool
|
||||
println!("\n📐 Cost Estimation Formula:");
|
||||
println!("===========================");
|
||||
println!("Estimated GB = (symbols × days × data_points × bytes_per_point) / 1GB");
|
||||
println!(" OHLCV-1m: ~390 bars/day × 32 bytes = ~12 KB per symbol per day");
|
||||
println!(" Trades: ~10,000 trades/day × 24 bytes = ~240 KB per symbol per day");
|
||||
println!(" MBP-1: ~100,000 updates/day × 48 bytes = ~4.8 MB per symbol per day");
|
||||
|
||||
println!("\n⚠️ IMPORTANT: NO DATA DOWNLOADED YET!");
|
||||
println!(" This script only checks your account status.");
|
||||
println!(" Use the Databento Python/Rust client to actually download data.");
|
||||
println!(" Always check the cost estimate BEFORE downloading!");
|
||||
|
||||
println!("\n📚 Next Steps:");
|
||||
println!("==============");
|
||||
println!("1. Review the pricing information above");
|
||||
println!("2. Use Databento's cost estimator: https://databento.com/pricing");
|
||||
println!("3. Start with OHLCV-1m data (cheapest option)");
|
||||
println!("4. Download 1-2 days for 1-2 symbols first");
|
||||
println!("5. Validate data quality before scaling up");
|
||||
println!("6. Monitor your credit balance regularly");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
#!/bin/bash
|
||||
# MAMBA2 13-Parameter Hyperopt Deployment Script
|
||||
# This script will keep trying until a GPU becomes available
|
||||
|
||||
set -e
|
||||
|
||||
COMMAND="/runpod-volume/binaries/hyperopt_mamba2_demo --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --trials 50 --epochs 50 --n-initial 10 --seed 42"
|
||||
|
||||
echo "╔════════════════════════════════════════════════════════════════════╗"
|
||||
echo "║ MAMBA2 13-Parameter Hyperopt Deployment ║"
|
||||
echo "╚════════════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " Binary: hyperopt_mamba2_demo (20.1 MB)"
|
||||
echo " Dataset: ES_FUT_180d.parquet (2.9 MB)"
|
||||
echo " Trials: 50"
|
||||
echo " Epochs per trial: 50"
|
||||
echo " Initial random samples: 10"
|
||||
echo " Seed: 42"
|
||||
echo ""
|
||||
echo "Expected:"
|
||||
echo " Runtime: 60-90 minutes"
|
||||
echo " Cost: \$0.17-0.26 (RTX A4000 @ \$0.17/hr)"
|
||||
echo ""
|
||||
echo "Attempting deployment..."
|
||||
echo ""
|
||||
|
||||
cd /home/jgrusewski/Work/foxhunt
|
||||
|
||||
# Try deployment
|
||||
python3 scripts/runpod_deploy.py \
|
||||
--gpu-type "RTX A4000" \
|
||||
--command "$COMMAND" \
|
||||
--container-disk 50
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo ""
|
||||
echo "╔════════════════════════════════════════════════════════════════════╗"
|
||||
echo "║ ✅ DEPLOYMENT SUCCESSFUL ║"
|
||||
echo "╚════════════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Monitor progress: Check Runpod console at https://www.runpod.io/console/pods"
|
||||
echo " 2. Wait 60-90 minutes for hyperopt to complete"
|
||||
echo " 3. Download results from S3:"
|
||||
echo ""
|
||||
echo " aws s3 ls s3://se3zdnb5o4/results/ \\"
|
||||
echo " --profile runpod \\"
|
||||
echo " --endpoint-url https://s3api-eur-is-1.runpod.io \\"
|
||||
echo " --recursive"
|
||||
echo ""
|
||||
echo " 4. Download best parameters:"
|
||||
echo ""
|
||||
echo " aws s3 cp s3://se3zdnb5o4/results/mamba2_13param_best_params_*.json \\"
|
||||
echo " /tmp/mamba2_best_params.json \\"
|
||||
echo " --profile runpod \\"
|
||||
echo " --endpoint-url https://s3api-eur-is-1.runpod.io"
|
||||
echo ""
|
||||
else
|
||||
echo ""
|
||||
echo "╔════════════════════════════════════════════════════════════════════╗"
|
||||
echo "║ ⚠️ DEPLOYMENT FAILED - No GPU Available ║"
|
||||
echo "╚════════════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Runpod GPUs in EUR-IS-1 are currently unavailable."
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " 1. Try again in 5-10 minutes (availability fluctuates)"
|
||||
echo " 2. Run this script again: bash /home/jgrusewski/Work/foxhunt/deploy_mamba2_hyperopt.sh"
|
||||
echo " 3. Monitor availability: https://www.runpod.io/console/gpu-cloud"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,246 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Foxhunt Production Deployment Script
|
||||
# Wave 5 W5-4: Zero-downtime Kubernetes deployment with rolling updates
|
||||
#
|
||||
# Usage: ./scripts/deployment/deploy.sh [environment]
|
||||
# environment: dev|staging|prod (default: staging)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
ENVIRONMENT="${1:-staging}"
|
||||
NAMESPACE="foxhunt"
|
||||
KUBE_CONTEXT="${KUBE_CONTEXT:-foxhunt-${ENVIRONMENT}}"
|
||||
TIMEOUT="${TIMEOUT:-600}" # 10 minutes
|
||||
HEALTH_CHECK_RETRIES=30
|
||||
HEALTH_CHECK_INTERVAL=10
|
||||
|
||||
# Service list
|
||||
SERVICES=(
|
||||
"api-gateway"
|
||||
"trading-service"
|
||||
"backtesting-service"
|
||||
"ml-training-service"
|
||||
"trading-agent-service"
|
||||
)
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Foxhunt Production Deployment${NC}"
|
||||
echo -e "${GREEN}Environment: ${ENVIRONMENT}${NC}"
|
||||
echo -e "${GREEN}Namespace: ${NAMESPACE}${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
|
||||
# Function: Check prerequisites
|
||||
check_prerequisites() {
|
||||
echo -e "\n${YELLOW}[1/6] Checking prerequisites...${NC}"
|
||||
|
||||
# Check kubectl
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}Error: kubectl not found. Please install kubectl.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check kubeval
|
||||
if ! command -v kubeval &> /dev/null; then
|
||||
echo -e "${YELLOW}Warning: kubeval not found. Skipping manifest validation.${NC}"
|
||||
fi
|
||||
|
||||
# Check context
|
||||
if ! kubectl config use-context "${KUBE_CONTEXT}" &> /dev/null; then
|
||||
echo -e "${RED}Error: Cannot switch to context ${KUBE_CONTEXT}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✓ Prerequisites check passed${NC}"
|
||||
}
|
||||
|
||||
# Function: Validate manifests
|
||||
validate_manifests() {
|
||||
echo -e "\n${YELLOW}[2/6] Validating Kubernetes manifests...${NC}"
|
||||
|
||||
if command -v kubeval &> /dev/null; then
|
||||
if kubeval k8s/deployments/*.yaml k8s/foxhunt-complete.yaml; then
|
||||
echo -e "${GREEN}✓ All manifests are valid${NC}"
|
||||
else
|
||||
echo -e "${RED}Error: Manifest validation failed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo -e "${YELLOW}Skipping validation (kubeval not installed)${NC}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function: Create or update namespace and base resources
|
||||
deploy_base_resources() {
|
||||
echo -e "\n${YELLOW}[3/6] Deploying base resources...${NC}"
|
||||
|
||||
# Create namespace if it doesn't exist
|
||||
kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
# Apply ConfigMaps, Secrets, and PVCs
|
||||
kubectl apply -f k8s/foxhunt-complete.yaml --namespace="${NAMESPACE}"
|
||||
|
||||
# Wait for PVCs to be bound
|
||||
echo "Waiting for PVCs to be bound..."
|
||||
kubectl wait --for=condition=Bound \
|
||||
pvc/postgres-pvc \
|
||||
pvc/redis-pvc \
|
||||
pvc/prometheus-pvc \
|
||||
pvc/ml-models-pvc \
|
||||
pvc/ml-checkpoints-pvc \
|
||||
pvc/backtesting-data-pvc \
|
||||
pvc/backtesting-results-pvc \
|
||||
--namespace="${NAMESPACE}" \
|
||||
--timeout="${TIMEOUT}s" || true
|
||||
|
||||
echo -e "${GREEN}✓ Base resources deployed${NC}"
|
||||
}
|
||||
|
||||
# Function: Deploy services with rolling updates
|
||||
deploy_services() {
|
||||
echo -e "\n${YELLOW}[4/6] Deploying services with rolling updates...${NC}"
|
||||
|
||||
for service in "${SERVICES[@]}"; do
|
||||
echo -e "\n${YELLOW}Deploying ${service}...${NC}"
|
||||
|
||||
# Apply deployment
|
||||
kubectl apply -f "k8s/deployments/${service}-deployment.yaml" --namespace="${NAMESPACE}"
|
||||
|
||||
# Apply service
|
||||
if [ -f "k8s/services/${service}-service.yaml" ]; then
|
||||
kubectl apply -f "k8s/services/${service}-service.yaml" --namespace="${NAMESPACE}"
|
||||
fi
|
||||
|
||||
# Wait for rollout
|
||||
echo "Waiting for ${service} rollout to complete..."
|
||||
if kubectl rollout status deployment/"${service}" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
--timeout="${TIMEOUT}s"; then
|
||||
echo -e "${GREEN}✓ ${service} deployed successfully${NC}"
|
||||
else
|
||||
echo -e "${RED}Error: ${service} deployment failed${NC}"
|
||||
|
||||
# Show pod logs for debugging
|
||||
echo "Recent pod logs:"
|
||||
kubectl logs -l app="${service}" --namespace="${NAMESPACE}" --tail=50 || true
|
||||
|
||||
# Ask if we should continue
|
||||
read -p "Continue with deployment? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "${GREEN}✓ All services deployed${NC}"
|
||||
}
|
||||
|
||||
# Function: Apply HPAs
|
||||
deploy_autoscaling() {
|
||||
echo -e "\n${YELLOW}[5/6] Deploying HorizontalPodAutoscalers...${NC}"
|
||||
|
||||
kubectl apply -f k8s/foxhunt-complete.yaml --namespace="${NAMESPACE}"
|
||||
|
||||
echo -e "${GREEN}✓ Autoscaling configured${NC}"
|
||||
}
|
||||
|
||||
# Function: Health checks
|
||||
run_health_checks() {
|
||||
echo -e "\n${YELLOW}[6/6] Running health checks...${NC}"
|
||||
|
||||
for service in "${SERVICES[@]}"; do
|
||||
echo -e "\n${YELLOW}Checking ${service} health...${NC}"
|
||||
|
||||
# Get pod name
|
||||
POD=$(kubectl get pods -l app="${service}" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$POD" ]; then
|
||||
echo -e "${RED}✗ No pods found for ${service}${NC}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Get health port based on service
|
||||
case $service in
|
||||
api-gateway)
|
||||
HEALTH_PORT=8080
|
||||
HEALTH_PATH="/health/liveness"
|
||||
;;
|
||||
trading-service)
|
||||
HEALTH_PORT=8081
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
backtesting-service)
|
||||
HEALTH_PORT=8082
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
ml-training-service)
|
||||
HEALTH_PORT=8095
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
trading-agent-service)
|
||||
HEALTH_PORT=8083
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Check health endpoint
|
||||
for i in $(seq 1 $HEALTH_CHECK_RETRIES); do
|
||||
if kubectl exec -it "$POD" --namespace="${NAMESPACE}" -- \
|
||||
curl -f "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" &> /dev/null; then
|
||||
echo -e "${GREEN}✓ ${service} is healthy${NC}"
|
||||
break
|
||||
else
|
||||
if [ $i -eq $HEALTH_CHECK_RETRIES ]; then
|
||||
echo -e "${RED}✗ ${service} health check failed after ${HEALTH_CHECK_RETRIES} attempts${NC}"
|
||||
else
|
||||
echo "Health check attempt $i/${HEALTH_CHECK_RETRIES} failed, retrying in ${HEALTH_CHECK_INTERVAL}s..."
|
||||
sleep $HEALTH_CHECK_INTERVAL
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
echo -e "\n${GREEN}✓ Health checks completed${NC}"
|
||||
}
|
||||
|
||||
# Function: Show deployment summary
|
||||
show_summary() {
|
||||
echo -e "\n${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Deployment Summary${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
|
||||
echo -e "\nPods:"
|
||||
kubectl get pods --namespace="${NAMESPACE}" -o wide
|
||||
|
||||
echo -e "\nServices:"
|
||||
kubectl get svc --namespace="${NAMESPACE}"
|
||||
|
||||
echo -e "\nHPAs:"
|
||||
kubectl get hpa --namespace="${NAMESPACE}"
|
||||
|
||||
echo -e "\n${GREEN}Deployment completed successfully!${NC}"
|
||||
echo -e "Access the API Gateway at: $(kubectl get svc api-gateway --namespace="${NAMESPACE}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):50051"
|
||||
}
|
||||
|
||||
# Main deployment flow
|
||||
main() {
|
||||
check_prerequisites
|
||||
validate_manifests
|
||||
deploy_base_resources
|
||||
deploy_services
|
||||
deploy_autoscaling
|
||||
run_health_checks
|
||||
show_summary
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
@@ -1,211 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Foxhunt Deployment Rollback Script
|
||||
# Wave 5 W5-4: Safe rollback to previous deployment revision
|
||||
#
|
||||
# Usage: ./scripts/deployment/rollback.sh [service] [revision]
|
||||
# service: api-gateway|trading-service|backtesting-service|ml-training-service|trading-agent-service|all
|
||||
# revision: revision number (optional, defaults to previous)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Configuration
|
||||
SERVICE="${1:-all}"
|
||||
REVISION="${2:-0}" # 0 = previous revision
|
||||
NAMESPACE="foxhunt"
|
||||
TIMEOUT=600
|
||||
|
||||
# Service list
|
||||
ALL_SERVICES=(
|
||||
"api-gateway"
|
||||
"trading-service"
|
||||
"backtesting-service"
|
||||
"ml-training-service"
|
||||
"trading-agent-service"
|
||||
)
|
||||
|
||||
echo -e "${YELLOW}========================================${NC}"
|
||||
echo -e "${YELLOW}Foxhunt Deployment Rollback${NC}"
|
||||
echo -e "${YELLOW}Service: ${SERVICE}${NC}"
|
||||
echo -e "${YELLOW}Namespace: ${NAMESPACE}${NC}"
|
||||
echo -e "${YELLOW}========================================${NC}"
|
||||
|
||||
# Function: Show deployment history
|
||||
show_history() {
|
||||
local service=$1
|
||||
echo -e "\n${YELLOW}Deployment history for ${service}:${NC}"
|
||||
kubectl rollout history deployment/"${service}" --namespace="${NAMESPACE}" || true
|
||||
}
|
||||
|
||||
# Function: Rollback a service
|
||||
rollback_service() {
|
||||
local service=$1
|
||||
local revision=$2
|
||||
|
||||
echo -e "\n${YELLOW}Rolling back ${service}...${NC}"
|
||||
|
||||
# Show current history
|
||||
show_history "${service}"
|
||||
|
||||
# Confirm rollback
|
||||
if [ "${REVISION}" == "0" ]; then
|
||||
echo -e "${RED}This will rollback ${service} to the previous revision.${NC}"
|
||||
else
|
||||
echo -e "${RED}This will rollback ${service} to revision ${revision}.${NC}"
|
||||
fi
|
||||
|
||||
read -p "Are you sure? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Rollback cancelled."
|
||||
return
|
||||
fi
|
||||
|
||||
# Perform rollback
|
||||
if [ "${revision}" == "0" ]; then
|
||||
kubectl rollout undo deployment/"${service}" --namespace="${NAMESPACE}"
|
||||
else
|
||||
kubectl rollout undo deployment/"${service}" --namespace="${NAMESPACE}" --to-revision="${revision}"
|
||||
fi
|
||||
|
||||
# Wait for rollout
|
||||
echo "Waiting for rollback to complete..."
|
||||
if kubectl rollout status deployment/"${service}" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
--timeout="${TIMEOUT}s"; then
|
||||
echo -e "${GREEN}✓ ${service} rolled back successfully${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ ${service} rollback failed${NC}"
|
||||
|
||||
# Show pod logs
|
||||
echo "Recent pod logs:"
|
||||
kubectl logs -l app="${service}" --namespace="${NAMESPACE}" --tail=50 || true
|
||||
|
||||
# Show events
|
||||
echo -e "\nRecent events:"
|
||||
kubectl get events --namespace="${NAMESPACE}" --field-selector involvedObject.name="${service}" --sort-by='.lastTimestamp' | tail -20
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify health
|
||||
verify_health "${service}"
|
||||
}
|
||||
|
||||
# Function: Verify service health after rollback
|
||||
verify_health() {
|
||||
local service=$1
|
||||
|
||||
echo -e "\n${YELLOW}Verifying ${service} health...${NC}"
|
||||
|
||||
# Get pod name
|
||||
POD=$(kubectl get pods -l app="${service}" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$POD" ]; then
|
||||
echo -e "${RED}✗ No pods found for ${service}${NC}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Get health port
|
||||
case $service in
|
||||
api-gateway)
|
||||
HEALTH_PORT=8080
|
||||
HEALTH_PATH="/health/liveness"
|
||||
;;
|
||||
trading-service)
|
||||
HEALTH_PORT=8081
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
backtesting-service)
|
||||
HEALTH_PORT=8082
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
ml-training-service)
|
||||
HEALTH_PORT=8095
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
trading-agent-service)
|
||||
HEALTH_PORT=8083
|
||||
HEALTH_PATH="/health"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Check health
|
||||
sleep 5 # Give service time to stabilize
|
||||
if kubectl exec -it "$POD" --namespace="${NAMESPACE}" -- \
|
||||
curl -f "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" &> /dev/null; then
|
||||
echo -e "${GREEN}✓ ${service} is healthy after rollback${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ ${service} health check failed after rollback${NC}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function: Rollback all services
|
||||
rollback_all() {
|
||||
echo -e "\n${RED}WARNING: This will rollback ALL services!${NC}"
|
||||
read -p "Are you ABSOLUTELY sure? Type 'YES' to confirm: " -r
|
||||
echo
|
||||
|
||||
if [[ ! $REPLY == "YES" ]]; then
|
||||
echo "Rollback cancelled."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Rollback in reverse order (agent -> ml -> backtesting -> trading -> gateway)
|
||||
for service in "${ALL_SERVICES[@]}"; do
|
||||
rollback_service "${service}" "${REVISION}"
|
||||
done
|
||||
|
||||
echo -e "\n${GREEN}All services rolled back successfully${NC}"
|
||||
}
|
||||
|
||||
# Function: Show summary
|
||||
show_summary() {
|
||||
echo -e "\n${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Rollback Summary${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
|
||||
echo -e "\nPods:"
|
||||
kubectl get pods --namespace="${NAMESPACE}" -o wide
|
||||
|
||||
echo -e "\nDeployments:"
|
||||
kubectl get deployments --namespace="${NAMESPACE}"
|
||||
|
||||
echo -e "\n${GREEN}Rollback completed!${NC}"
|
||||
}
|
||||
|
||||
# Main rollback flow
|
||||
main() {
|
||||
# Check kubectl
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}Error: kubectl not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Perform rollback
|
||||
if [ "${SERVICE}" == "all" ]; then
|
||||
rollback_all
|
||||
else
|
||||
# Validate service name
|
||||
if [[ ! " ${ALL_SERVICES[@]} " =~ " ${SERVICE} " ]]; then
|
||||
echo -e "${RED}Error: Invalid service name '${SERVICE}'${NC}"
|
||||
echo "Valid services: ${ALL_SERVICES[*]} or 'all'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rollback_service "${SERVICE}" "${REVISION}"
|
||||
fi
|
||||
|
||||
show_summary
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
@@ -1,318 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Foxhunt Post-Deployment Smoke Tests
|
||||
# Wave 5 W5-4: Comprehensive smoke tests for production deployment
|
||||
#
|
||||
# Usage: ./scripts/deployment/smoke-test.sh [namespace]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Configuration
|
||||
NAMESPACE="${1:-foxhunt}"
|
||||
TIMEOUT=30
|
||||
TOTAL_TESTS=0
|
||||
PASSED_TESTS=0
|
||||
FAILED_TESTS=0
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Foxhunt Smoke Tests${NC}"
|
||||
echo -e "${GREEN}Namespace: ${NAMESPACE}${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
|
||||
# Function: Run a test
|
||||
run_test() {
|
||||
local test_name=$1
|
||||
local test_command=$2
|
||||
|
||||
((TOTAL_TESTS++))
|
||||
echo -e "\n${YELLOW}[Test $TOTAL_TESTS] ${test_name}${NC}"
|
||||
|
||||
if eval "$test_command"; then
|
||||
echo -e "${GREEN}✓ PASSED${NC}"
|
||||
((PASSED_TESTS++))
|
||||
return 0
|
||||
else
|
||||
echo -e "${RED}✗ FAILED${NC}"
|
||||
((FAILED_TESTS++))
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Test 1: All pods are running
|
||||
test_pods_running() {
|
||||
echo "Checking pod status..."
|
||||
kubectl get pods --namespace="${NAMESPACE}" -o json | \
|
||||
jq -e '.items | all(.status.phase == "Running")' > /dev/null
|
||||
}
|
||||
|
||||
# Test 2: All deployments are ready
|
||||
test_deployments_ready() {
|
||||
echo "Checking deployment readiness..."
|
||||
local deployments=(
|
||||
"api-gateway"
|
||||
"trading-service"
|
||||
"backtesting-service"
|
||||
"ml-training-service"
|
||||
"trading-agent-service"
|
||||
)
|
||||
|
||||
for deployment in "${deployments[@]}"; do
|
||||
READY=$(kubectl get deployment "$deployment" --namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
|
||||
DESIRED=$(kubectl get deployment "$deployment" --namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")
|
||||
|
||||
if [ "$READY" != "$DESIRED" ]; then
|
||||
echo "Deployment $deployment: $READY/$DESIRED pods ready"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Test 3: All services have endpoints
|
||||
test_services_have_endpoints() {
|
||||
echo "Checking service endpoints..."
|
||||
local services=(
|
||||
"api-gateway"
|
||||
"trading-service"
|
||||
"backtesting-service"
|
||||
"ml-training-service"
|
||||
"trading-agent-service"
|
||||
)
|
||||
|
||||
for service in "${services[@]}"; do
|
||||
ENDPOINTS=$(kubectl get endpoints "$service" --namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$ENDPOINTS" ]; then
|
||||
echo "Service $service has no endpoints"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Test 4: Health checks pass
|
||||
test_health_checks() {
|
||||
echo "Testing health endpoints..."
|
||||
local services=(
|
||||
"api-gateway:8080:/health/liveness"
|
||||
"trading-service:8081:/health"
|
||||
"backtesting-service:8082:/health"
|
||||
"ml-training-service:8095:/health"
|
||||
"trading-agent-service:8083:/health"
|
||||
)
|
||||
|
||||
for service_spec in "${services[@]}"; do
|
||||
IFS=':' read -r service port path <<< "$service_spec"
|
||||
|
||||
POD=$(kubectl get pods -l app="${service}" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$POD" ]; then
|
||||
echo "No pods found for $service"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
|
||||
curl -f -s "http://localhost:${port}${path}" > /dev/null 2>&1; then
|
||||
echo "Health check failed for $service"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Test 5: Metrics endpoints are accessible
|
||||
test_metrics_endpoints() {
|
||||
echo "Testing metrics endpoints..."
|
||||
local services=(
|
||||
"api-gateway:9091"
|
||||
"trading-service:9092"
|
||||
"backtesting-service:9093"
|
||||
"ml-training-service:9094"
|
||||
"trading-agent-service:9095"
|
||||
)
|
||||
|
||||
for service_spec in "${services[@]}"; do
|
||||
IFS=':' read -r service port <<< "$service_spec"
|
||||
|
||||
POD=$(kubectl get pods -l app="${service}" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$POD" ]; then
|
||||
echo "No pods found for $service"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
|
||||
curl -f -s "http://localhost:${port}/metrics" | grep -q "^#"; then
|
||||
echo "Metrics endpoint failed for $service"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Test 6: Database connectivity
|
||||
test_database_connectivity() {
|
||||
echo "Testing database connectivity..."
|
||||
|
||||
# Get a pod that can connect to postgres
|
||||
POD=$(kubectl get pods -l app="trading-service" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$POD" ]; then
|
||||
echo "No pods found for database test"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# This is a simplified test - in production, you'd check actual DB connection
|
||||
# For now, we just verify the environment variable is set
|
||||
kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
|
||||
printenv DATABASE_URL > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# Test 7: Redis connectivity
|
||||
test_redis_connectivity() {
|
||||
echo "Testing Redis connectivity..."
|
||||
|
||||
POD=$(kubectl get pods -l app="trading-service" \
|
||||
--namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$POD" ]; then
|
||||
echo "No pods found for Redis test"
|
||||
return 1
|
||||
fi
|
||||
|
||||
kubectl exec "$POD" --namespace="${NAMESPACE}" -- \
|
||||
printenv REDIS_URL > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# Test 8: PVCs are bound
|
||||
test_pvcs_bound() {
|
||||
echo "Testing PVC bindings..."
|
||||
local pvcs=(
|
||||
"postgres-pvc"
|
||||
"redis-pvc"
|
||||
"prometheus-pvc"
|
||||
"ml-models-pvc"
|
||||
"ml-checkpoints-pvc"
|
||||
"backtesting-data-pvc"
|
||||
"backtesting-results-pvc"
|
||||
)
|
||||
|
||||
for pvc in "${pvcs[@]}"; do
|
||||
STATUS=$(kubectl get pvc "$pvc" --namespace="${NAMESPACE}" \
|
||||
-o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound")
|
||||
|
||||
if [ "$STATUS" != "Bound" ]; then
|
||||
echo "PVC $pvc is not bound (status: $STATUS)"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Test 9: HPAs are active
|
||||
test_hpas_active() {
|
||||
echo "Testing HPA status..."
|
||||
local hpas=(
|
||||
"api-gateway-hpa"
|
||||
"trading-service-hpa"
|
||||
"ml-training-service-hpa"
|
||||
)
|
||||
|
||||
for hpa in "${hpas[@]}"; do
|
||||
if ! kubectl get hpa "$hpa" --namespace="${NAMESPACE}" > /dev/null 2>&1; then
|
||||
echo "HPA $hpa not found"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Test 10: No pods in CrashLoopBackOff
|
||||
test_no_crashloop() {
|
||||
echo "Checking for crash loops..."
|
||||
CRASHLOOP=$(kubectl get pods --namespace="${NAMESPACE}" \
|
||||
-o json | jq -r '.items[] | select(.status.containerStatuses[]?.state.waiting.reason == "CrashLoopBackOff") | .metadata.name' || echo "")
|
||||
|
||||
if [ -n "$CRASHLOOP" ]; then
|
||||
echo "Pods in CrashLoopBackOff: $CRASHLOOP"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Function: Show test summary
|
||||
show_summary() {
|
||||
echo -e "\n${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Smoke Test Summary${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "Total tests: $TOTAL_TESTS"
|
||||
echo -e "${GREEN}Passed: $PASSED_TESTS${NC}"
|
||||
|
||||
if [ $FAILED_TESTS -gt 0 ]; then
|
||||
echo -e "${RED}Failed: $FAILED_TESTS${NC}"
|
||||
echo -e "\n${RED}Smoke tests FAILED${NC}"
|
||||
return 1
|
||||
else
|
||||
echo -e "${RED}Failed: $FAILED_TESTS${NC}"
|
||||
echo -e "\n${GREEN}All smoke tests PASSED!${NC}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Main test flow
|
||||
main() {
|
||||
# Check prerequisites
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}Error: kubectl not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq &> /dev/null; then
|
||||
echo -e "${RED}Error: jq not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run all tests
|
||||
run_test "All pods are running" "test_pods_running" || true
|
||||
run_test "All deployments are ready" "test_deployments_ready" || true
|
||||
run_test "All services have endpoints" "test_services_have_endpoints" || true
|
||||
run_test "Health checks pass" "test_health_checks" || true
|
||||
run_test "Metrics endpoints accessible" "test_metrics_endpoints" || true
|
||||
run_test "Database connectivity" "test_database_connectivity" || true
|
||||
run_test "Redis connectivity" "test_redis_connectivity" || true
|
||||
run_test "PVCs are bound" "test_pvcs_bound" || true
|
||||
run_test "HPAs are active" "test_hpas_active" || true
|
||||
run_test "No pods in CrashLoopBackOff" "test_no_crashloop" || true
|
||||
|
||||
# Show summary and exit with appropriate code
|
||||
if show_summary; then
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
@@ -1,214 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Extract Best Hyperparameters from Tuning Results
|
||||
Analyzes JSON result files and extracts optimal hyperparameters for each model
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
class HyperparameterExtractor:
|
||||
"""Extract and analyze best hyperparameters from tuning results"""
|
||||
|
||||
def __init__(self, results_dir: str = "results"):
|
||||
self.results_dir = Path(results_dir)
|
||||
self.models = ["DQN", "PPO", "TFT", "MAMBA2", "Liquid"]
|
||||
|
||||
def extract_best_params(self, model: str) -> Optional[Dict[str, Any]]:
|
||||
"""Extract best hyperparameters for a given model"""
|
||||
result_file = self.results_dir / f"{model.lower()}_tuning_50trials.json"
|
||||
|
||||
if not result_file.exists():
|
||||
print(f"⚠️ Result file not found: {result_file}")
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(result_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Find best trial by Sharpe ratio
|
||||
best_trial = max(data.get('trials', []),
|
||||
key=lambda x: x.get('sharpe_ratio', -999))
|
||||
|
||||
return {
|
||||
'model': model,
|
||||
'best_trial_id': best_trial.get('trial_id'),
|
||||
'sharpe_ratio': best_trial.get('sharpe_ratio'),
|
||||
'loss': best_trial.get('loss'),
|
||||
'training_time': best_trial.get('training_time'),
|
||||
'hyperparameters': best_trial.get('hyperparameters', {}),
|
||||
'total_trials': len(data.get('trials', [])),
|
||||
'completed_trials': sum(1 for t in data.get('trials', [])
|
||||
if t.get('status') == 'completed')
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"❌ Error extracting {model} parameters: {e}")
|
||||
return None
|
||||
|
||||
def format_hyperparameters(self, params: Dict[str, Any]) -> str:
|
||||
"""Format hyperparameters for display"""
|
||||
if not params:
|
||||
return "No hyperparameters available"
|
||||
|
||||
lines = []
|
||||
for key, value in params.items():
|
||||
if isinstance(value, float):
|
||||
lines.append(f" {key}: {value:.6f}")
|
||||
else:
|
||||
lines.append(f" {key}: {value}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def generate_report(self, output_file: str = "HYPERPARAMETER_TUNING_EXECUTION_REPORT.md"):
|
||||
"""Generate comprehensive report of all tuning results"""
|
||||
print("=" * 70)
|
||||
print("HYPERPARAMETER TUNING RESULTS EXTRACTION")
|
||||
print("=" * 70)
|
||||
print(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print(f"Results directory: {self.results_dir}")
|
||||
print()
|
||||
|
||||
all_results = {}
|
||||
for model in self.models:
|
||||
print(f"Processing {model}...")
|
||||
result = self.extract_best_params(model)
|
||||
if result:
|
||||
all_results[model] = result
|
||||
print(f" ✓ Found {result['completed_trials']}/{result['total_trials']} trials")
|
||||
print(f" ✓ Best Sharpe: {result['sharpe_ratio']:.4f}")
|
||||
else:
|
||||
print(f" ⚠️ No results available")
|
||||
print()
|
||||
|
||||
# Generate markdown report
|
||||
report = self._generate_markdown_report(all_results)
|
||||
|
||||
output_path = Path(output_file)
|
||||
with open(output_path, 'w') as f:
|
||||
f.write(report)
|
||||
|
||||
print("=" * 70)
|
||||
print(f"Report generated: {output_path}")
|
||||
print("=" * 70)
|
||||
|
||||
return all_results
|
||||
|
||||
def _generate_markdown_report(self, results: Dict[str, Dict[str, Any]]) -> str:
|
||||
"""Generate markdown report from results"""
|
||||
report = []
|
||||
report.append("# Hyperparameter Tuning Execution Report")
|
||||
report.append("")
|
||||
report.append(f"**Generated**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
report.append(f"**Pipeline Status**: {'Complete' if len(results) == 5 else 'In Progress'}")
|
||||
report.append(f"**Models Completed**: {len(results)}/5")
|
||||
report.append("")
|
||||
report.append("---")
|
||||
report.append("")
|
||||
|
||||
# Executive Summary
|
||||
report.append("## Executive Summary")
|
||||
report.append("")
|
||||
if results:
|
||||
total_trials = sum(r['total_trials'] for r in results.values())
|
||||
completed_trials = sum(r['completed_trials'] for r in results.values())
|
||||
avg_sharpe = sum(r['sharpe_ratio'] for r in results.values()) / len(results)
|
||||
|
||||
report.append(f"- **Total Trials**: {completed_trials}/{total_trials}")
|
||||
report.append(f"- **Average Sharpe Ratio**: {avg_sharpe:.4f}")
|
||||
report.append(f"- **Models Optimized**: {', '.join(results.keys())}")
|
||||
else:
|
||||
report.append("*No results available yet*")
|
||||
report.append("")
|
||||
report.append("---")
|
||||
report.append("")
|
||||
|
||||
# Individual Model Results
|
||||
for model, result in results.items():
|
||||
report.append(f"## {model} Hyperparameter Tuning")
|
||||
report.append("")
|
||||
report.append(f"**Status**: ✅ Complete")
|
||||
report.append(f"**Trials**: {result['completed_trials']}/{result['total_trials']}")
|
||||
report.append(f"**Best Trial**: #{result['best_trial_id']}")
|
||||
report.append("")
|
||||
|
||||
report.append("### Performance Metrics")
|
||||
report.append("")
|
||||
report.append(f"- **Sharpe Ratio**: {result['sharpe_ratio']:.4f}")
|
||||
report.append(f"- **Final Loss**: {result['loss']:.6f}")
|
||||
report.append(f"- **Training Time**: {result['training_time']:.1f}s")
|
||||
report.append("")
|
||||
|
||||
report.append("### Best Hyperparameters")
|
||||
report.append("")
|
||||
report.append("```yaml")
|
||||
for key, value in result['hyperparameters'].items():
|
||||
if isinstance(value, float):
|
||||
report.append(f"{key}: {value:.6f}")
|
||||
else:
|
||||
report.append(f"{key}: {value}")
|
||||
report.append("```")
|
||||
report.append("")
|
||||
report.append("---")
|
||||
report.append("")
|
||||
|
||||
# Pending Models
|
||||
pending_models = [m for m in self.models if m not in results]
|
||||
if pending_models:
|
||||
report.append("## Pending Models")
|
||||
report.append("")
|
||||
for model in pending_models:
|
||||
report.append(f"- **{model}**: ⏳ In Progress or Not Started")
|
||||
report.append("")
|
||||
report.append("---")
|
||||
report.append("")
|
||||
|
||||
# Next Steps
|
||||
report.append("## Next Steps")
|
||||
report.append("")
|
||||
if len(results) == 5:
|
||||
report.append("1. ✅ All models tuned successfully")
|
||||
report.append("2. 📝 Review hyperparameters for each model")
|
||||
report.append("3. 🔧 Update model configuration files")
|
||||
report.append("4. 🚀 Run production training with optimized hyperparameters")
|
||||
report.append("5. 📊 Validate models with backtesting")
|
||||
else:
|
||||
report.append(f"1. ⏳ Wait for remaining {len(pending_models)} models to complete")
|
||||
report.append("2. 📊 Monitor tuning progress with dashboard")
|
||||
report.append("3. 🔍 Check for CUDA OOM errors in logs")
|
||||
report.append("4. 🔄 Regenerate report when all models complete")
|
||||
report.append("")
|
||||
|
||||
return "\n".join(report)
|
||||
|
||||
def print_summary(self):
|
||||
"""Print a quick summary of available results"""
|
||||
print("=" * 70)
|
||||
print("AVAILABLE TUNING RESULTS")
|
||||
print("=" * 70)
|
||||
|
||||
for model in self.models:
|
||||
result_file = self.results_dir / f"{model.lower()}_tuning_50trials.json"
|
||||
if result_file.exists():
|
||||
try:
|
||||
with open(result_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
trials = len(data.get('trials', []))
|
||||
completed = sum(1 for t in data.get('trials', [])
|
||||
if t.get('status') == 'completed')
|
||||
print(f"✅ {model:10} | {completed:2}/{trials:2} trials | {result_file}")
|
||||
except:
|
||||
print(f"❌ {model:10} | ERROR reading file | {result_file}")
|
||||
else:
|
||||
print(f"⏳ {model:10} | Not started | {result_file}")
|
||||
|
||||
print("=" * 70)
|
||||
|
||||
if __name__ == "__main__":
|
||||
extractor = HyperparameterExtractor()
|
||||
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--summary":
|
||||
extractor.print_summary()
|
||||
else:
|
||||
extractor.generate_report()
|
||||
@@ -1,518 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Compliance reporting script for Foxhunt HFT Trading System
|
||||
Generates comprehensive compliance reports for regulatory submissions
|
||||
"""
|
||||
|
||||
import json
|
||||
import argparse
|
||||
import hashlib
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
@dataclass
|
||||
class SecurityAuditResult:
|
||||
"""Security audit results"""
|
||||
tool: str
|
||||
status: str
|
||||
vulnerabilities_found: int
|
||||
critical_issues: int
|
||||
high_issues: int
|
||||
medium_issues: int
|
||||
low_issues: int
|
||||
scan_timestamp: str
|
||||
|
||||
@dataclass
|
||||
class DeploymentMetrics:
|
||||
"""Deployment performance and reliability metrics"""
|
||||
deployment_duration_seconds: float
|
||||
rollback_capability: bool
|
||||
health_check_status: str
|
||||
performance_validation_status: str
|
||||
zero_downtime_achieved: bool
|
||||
canary_percentage: Optional[float]
|
||||
traffic_split_duration: Optional[float]
|
||||
|
||||
@dataclass
|
||||
class ComplianceReport:
|
||||
"""Complete compliance report structure"""
|
||||
report_id: str
|
||||
generation_timestamp: str
|
||||
git_commit_sha: str
|
||||
deployment_status: str
|
||||
environment: str
|
||||
|
||||
# Security compliance
|
||||
security_audits: List[SecurityAuditResult]
|
||||
vulnerability_summary: Dict[str, int]
|
||||
|
||||
# Performance compliance
|
||||
latency_validation: Dict[str, Any]
|
||||
throughput_validation: Dict[str, Any]
|
||||
|
||||
# Deployment compliance
|
||||
deployment_metrics: DeploymentMetrics
|
||||
|
||||
# Regulatory compliance
|
||||
audit_trail: List[Dict[str, Any]]
|
||||
change_control_record: Dict[str, Any]
|
||||
|
||||
# Risk assessment
|
||||
risk_assessment: Dict[str, Any]
|
||||
|
||||
# Signatures and attestations
|
||||
digital_signature: str
|
||||
compliance_attestation: Dict[str, Any]
|
||||
|
||||
class ComplianceReporter:
|
||||
"""Generates compliance reports for regulatory submissions"""
|
||||
|
||||
def __init__(self, commit_sha: str, deployment_status: str, environment: str = "production"):
|
||||
self.commit_sha = commit_sha
|
||||
self.deployment_status = deployment_status
|
||||
self.environment = environment
|
||||
self.report_id = self._generate_report_id()
|
||||
|
||||
def _generate_report_id(self) -> str:
|
||||
"""Generate unique report ID"""
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
hash_input = f"{self.commit_sha}_{timestamp}_{self.environment}"
|
||||
hash_digest = hashlib.sha256(hash_input.encode()).hexdigest()[:8]
|
||||
return f"FOXHUNT_COMPLIANCE_{timestamp}_{hash_digest}"
|
||||
|
||||
def _run_security_audit(self) -> List[SecurityAuditResult]:
|
||||
"""Run security audits and collect results"""
|
||||
audits = []
|
||||
|
||||
# Cargo audit
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["cargo", "audit", "--json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
audit_data = json.loads(result.stdout)
|
||||
vulnerabilities = audit_data.get("vulnerabilities", {}).get("count", 0)
|
||||
|
||||
audits.append(SecurityAuditResult(
|
||||
tool="cargo-audit",
|
||||
status="passed" if vulnerabilities == 0 else "vulnerabilities_found",
|
||||
vulnerabilities_found=vulnerabilities,
|
||||
critical_issues=0, # cargo audit doesn't categorize by severity
|
||||
high_issues=vulnerabilities,
|
||||
medium_issues=0,
|
||||
low_issues=0,
|
||||
scan_timestamp=datetime.now(timezone.utc).isoformat()
|
||||
))
|
||||
else:
|
||||
audits.append(SecurityAuditResult(
|
||||
tool="cargo-audit",
|
||||
status="failed",
|
||||
vulnerabilities_found=-1,
|
||||
critical_issues=0,
|
||||
high_issues=0,
|
||||
medium_issues=0,
|
||||
low_issues=0,
|
||||
scan_timestamp=datetime.now(timezone.utc).isoformat()
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error running cargo audit: {e}")
|
||||
|
||||
# Cargo geiger
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["cargo", "geiger", "--all", "--output-format", "Json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
# Parse geiger output (simplified)
|
||||
unsafe_count = result.stdout.count("unsafe")
|
||||
|
||||
audits.append(SecurityAuditResult(
|
||||
tool="cargo-geiger",
|
||||
status="passed" if unsafe_count < 50 else "warnings",
|
||||
vulnerabilities_found=0,
|
||||
critical_issues=0,
|
||||
high_issues=0,
|
||||
medium_issues=unsafe_count if unsafe_count >= 10 else 0,
|
||||
low_issues=unsafe_count if unsafe_count < 10 else 0,
|
||||
scan_timestamp=datetime.now(timezone.utc).isoformat()
|
||||
))
|
||||
else:
|
||||
audits.append(SecurityAuditResult(
|
||||
tool="cargo-geiger",
|
||||
status="failed",
|
||||
vulnerabilities_found=-1,
|
||||
critical_issues=0,
|
||||
high_issues=0,
|
||||
medium_issues=0,
|
||||
low_issues=0,
|
||||
scan_timestamp=datetime.now(timezone.utc).isoformat()
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error running cargo geiger: {e}")
|
||||
|
||||
return audits
|
||||
|
||||
def _collect_performance_metrics(self) -> tuple:
|
||||
"""Collect performance validation metrics"""
|
||||
latency_validation = {
|
||||
"status": "unknown",
|
||||
"metrics": {},
|
||||
"thresholds_met": False
|
||||
}
|
||||
|
||||
throughput_validation = {
|
||||
"status": "unknown",
|
||||
"metrics": {},
|
||||
"thresholds_met": False
|
||||
}
|
||||
|
||||
# Try to read performance validation report
|
||||
report_path = Path("performance-validation-report.md")
|
||||
if report_path.exists():
|
||||
try:
|
||||
content = report_path.read_text()
|
||||
|
||||
if "✅ VALIDATION PASSED" in content:
|
||||
latency_validation["status"] = "passed"
|
||||
latency_validation["thresholds_met"] = True
|
||||
throughput_validation["status"] = "passed"
|
||||
throughput_validation["thresholds_met"] = True
|
||||
elif "❌ VALIDATION FAILED" in content:
|
||||
latency_validation["status"] = "failed"
|
||||
throughput_validation["status"] = "failed"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading performance report: {e}")
|
||||
|
||||
# Try to read benchmark results
|
||||
benchmark_dir = Path("benchmark_results")
|
||||
if benchmark_dir.exists():
|
||||
for result_file in benchmark_dir.glob("*.json"):
|
||||
try:
|
||||
with open(result_file) as f:
|
||||
data = json.load(f)
|
||||
|
||||
benchmark_name = result_file.stem
|
||||
if "latency" in benchmark_name or "trading" in benchmark_name:
|
||||
latency_validation["metrics"][benchmark_name] = data
|
||||
elif "throughput" in benchmark_name or "processing" in benchmark_name:
|
||||
throughput_validation["metrics"][benchmark_name] = data
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading benchmark file {result_file}: {e}")
|
||||
|
||||
return latency_validation, throughput_validation
|
||||
|
||||
def _collect_deployment_metrics(self) -> DeploymentMetrics:
|
||||
"""Collect deployment performance metrics"""
|
||||
|
||||
# Try to read deployment logs
|
||||
log_dir = Path("/home/jgrusewski/Work/foxhunt/logs")
|
||||
deployment_duration = 0.0
|
||||
zero_downtime = False
|
||||
|
||||
if log_dir.exists():
|
||||
# Look for recent deployment logs
|
||||
for log_file in log_dir.glob("deployment-*.log"):
|
||||
try:
|
||||
content = log_file.read_text()
|
||||
|
||||
# Extract deployment duration (simplified)
|
||||
if "deployment completed successfully" in content.lower():
|
||||
zero_downtime = True
|
||||
|
||||
# Extract timing information
|
||||
lines = content.split('\n')
|
||||
start_time = None
|
||||
end_time = None
|
||||
|
||||
for line in lines:
|
||||
if "starting" in line.lower() and "deployment" in line.lower():
|
||||
# Extract timestamp
|
||||
try:
|
||||
timestamp_str = line.split(']')[0].replace('[', '')
|
||||
start_time = datetime.fromisoformat(timestamp_str.replace(' ', 'T'))
|
||||
except:
|
||||
pass
|
||||
elif "completed successfully" in line.lower():
|
||||
try:
|
||||
timestamp_str = line.split(']')[0].replace('[', '')
|
||||
end_time = datetime.fromisoformat(timestamp_str.replace(' ', 'T'))
|
||||
except:
|
||||
pass
|
||||
|
||||
if start_time and end_time:
|
||||
deployment_duration = (end_time - start_time).total_seconds()
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading deployment log {log_file}: {e}")
|
||||
|
||||
return DeploymentMetrics(
|
||||
deployment_duration_seconds=deployment_duration,
|
||||
rollback_capability=True, # System has rollback capability
|
||||
health_check_status="passed" if self.deployment_status == "success" else "failed",
|
||||
performance_validation_status="passed" if self.deployment_status == "success" else "failed",
|
||||
zero_downtime_achieved=zero_downtime,
|
||||
canary_percentage=1.0 if self.environment == "production" else None,
|
||||
traffic_split_duration=300.0 if self.environment == "production" else None
|
||||
)
|
||||
|
||||
def _generate_audit_trail(self) -> List[Dict[str, Any]]:
|
||||
"""Generate audit trail entries"""
|
||||
trail = []
|
||||
|
||||
# Git commit information
|
||||
try:
|
||||
# Get commit details
|
||||
result = subprocess.run(
|
||||
["git", "show", "--format=%H|%an|%ae|%ad|%s", "--no-patch", self.commit_sha],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
parts = result.stdout.strip().split('|')
|
||||
if len(parts) >= 5:
|
||||
trail.append({
|
||||
"event_type": "code_change",
|
||||
"timestamp": parts[3],
|
||||
"actor": parts[1],
|
||||
"actor_email": parts[2],
|
||||
"description": f"Commit: {parts[4]}",
|
||||
"commit_sha": parts[0],
|
||||
"verification": "git-signed" if self._is_commit_signed(self.commit_sha) else "unsigned"
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error getting git commit info: {e}")
|
||||
|
||||
# CI/CD pipeline execution
|
||||
trail.append({
|
||||
"event_type": "cicd_execution",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"actor": "github-actions",
|
||||
"description": f"CI/CD pipeline executed for deployment to {self.environment}",
|
||||
"status": self.deployment_status,
|
||||
"environment": self.environment
|
||||
})
|
||||
|
||||
# Security scans
|
||||
trail.append({
|
||||
"event_type": "security_scan",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"actor": "automated-security-scanner",
|
||||
"description": "Automated security vulnerability scanning executed",
|
||||
"tools": ["cargo-audit", "cargo-geiger"]
|
||||
})
|
||||
|
||||
# Performance validation
|
||||
trail.append({
|
||||
"event_type": "performance_validation",
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"actor": "automated-performance-validator",
|
||||
"description": "HFT performance validation executed",
|
||||
"validation_status": "passed" if self.deployment_status == "success" else "failed"
|
||||
})
|
||||
|
||||
return trail
|
||||
|
||||
def _is_commit_signed(self, commit_sha: str) -> bool:
|
||||
"""Check if commit is GPG signed"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "verify-commit", commit_sha],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
return result.returncode == 0
|
||||
except:
|
||||
return False
|
||||
|
||||
def _generate_change_control_record(self) -> Dict[str, Any]:
|
||||
"""Generate change control record"""
|
||||
return {
|
||||
"change_id": f"CHG-{self.report_id}",
|
||||
"change_type": "software_deployment",
|
||||
"requestor": "automated-cicd",
|
||||
"approver": "system-automated",
|
||||
"risk_level": "medium", # HFT deployments are inherently medium risk
|
||||
"testing_performed": [
|
||||
"unit_tests",
|
||||
"integration_tests",
|
||||
"performance_benchmarks",
|
||||
"security_scans"
|
||||
],
|
||||
"rollback_plan": "automated_rollback_available",
|
||||
"deployment_window": {
|
||||
"start": datetime.now(timezone.utc).isoformat(),
|
||||
"duration_minutes": 30,
|
||||
"maintenance_required": False
|
||||
},
|
||||
"stakeholder_notification": "automated",
|
||||
"change_approval_timestamp": datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
|
||||
def _assess_risk(self) -> Dict[str, Any]:
|
||||
"""Perform risk assessment"""
|
||||
risk_factors = []
|
||||
overall_risk = "low"
|
||||
|
||||
# Assess based on deployment status
|
||||
if self.deployment_status != "success":
|
||||
risk_factors.append("deployment_failure")
|
||||
overall_risk = "high"
|
||||
|
||||
# Assess based on environment
|
||||
if self.environment == "production":
|
||||
risk_factors.append("production_deployment")
|
||||
if overall_risk == "low":
|
||||
overall_risk = "medium"
|
||||
|
||||
# Consider security findings
|
||||
# (This would be populated with actual security audit results)
|
||||
|
||||
return {
|
||||
"overall_risk_level": overall_risk,
|
||||
"risk_factors": risk_factors,
|
||||
"mitigation_measures": [
|
||||
"automated_rollback_capability",
|
||||
"canary_deployment",
|
||||
"real_time_monitoring",
|
||||
"automated_health_checks"
|
||||
],
|
||||
"residual_risk": "low",
|
||||
"risk_assessment_timestamp": datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
|
||||
def _generate_digital_signature(self, report_data: Dict[str, Any]) -> str:
|
||||
"""Generate digital signature for report integrity"""
|
||||
# Create hash of report content
|
||||
report_json = json.dumps(report_data, sort_keys=True)
|
||||
signature = hashlib.sha256(report_json.encode()).hexdigest()
|
||||
|
||||
return f"SHA256:{signature}"
|
||||
|
||||
def _generate_compliance_attestation(self) -> Dict[str, Any]:
|
||||
"""Generate compliance attestation"""
|
||||
return {
|
||||
"attestation_type": "automated_compliance_validation",
|
||||
"attestor": "foxhunt_cicd_system",
|
||||
"attestation_timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"compliance_frameworks": [
|
||||
"SOC2_Type_II",
|
||||
"ISO_27001",
|
||||
"MiFID_II",
|
||||
"SEC_Rule_15c3_5" # Market Access Rule
|
||||
],
|
||||
"controls_validated": [
|
||||
"change_management",
|
||||
"security_scanning",
|
||||
"performance_validation",
|
||||
"audit_logging",
|
||||
"access_controls",
|
||||
"data_integrity"
|
||||
],
|
||||
"validation_status": "passed" if self.deployment_status == "success" else "failed_with_exceptions",
|
||||
"exceptions": [] if self.deployment_status == "success" else ["deployment_failure"],
|
||||
"next_review_date": (datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) +
|
||||
datetime.timedelta(days=90)).isoformat()
|
||||
}
|
||||
|
||||
def generate_report(self) -> ComplianceReport:
|
||||
"""Generate comprehensive compliance report"""
|
||||
|
||||
print(f"Generating compliance report for commit {self.commit_sha}...")
|
||||
|
||||
# Collect all compliance data
|
||||
security_audits = self._run_security_audit()
|
||||
latency_validation, throughput_validation = self._collect_performance_metrics()
|
||||
deployment_metrics = self._collect_deployment_metrics()
|
||||
audit_trail = self._generate_audit_trail()
|
||||
change_control_record = self._generate_change_control_record()
|
||||
risk_assessment = self._assess_risk()
|
||||
compliance_attestation = self._generate_compliance_attestation()
|
||||
|
||||
# Calculate vulnerability summary
|
||||
vulnerability_summary = {
|
||||
"critical": sum(audit.critical_issues for audit in security_audits),
|
||||
"high": sum(audit.high_issues for audit in security_audits),
|
||||
"medium": sum(audit.medium_issues for audit in security_audits),
|
||||
"low": sum(audit.low_issues for audit in security_audits),
|
||||
"total": sum(audit.vulnerabilities_found for audit in security_audits if audit.vulnerabilities_found >= 0)
|
||||
}
|
||||
|
||||
# Create report structure
|
||||
report = ComplianceReport(
|
||||
report_id=self.report_id,
|
||||
generation_timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
git_commit_sha=self.commit_sha,
|
||||
deployment_status=self.deployment_status,
|
||||
environment=self.environment,
|
||||
security_audits=security_audits,
|
||||
vulnerability_summary=vulnerability_summary,
|
||||
latency_validation=latency_validation,
|
||||
throughput_validation=throughput_validation,
|
||||
deployment_metrics=deployment_metrics,
|
||||
audit_trail=audit_trail,
|
||||
change_control_record=change_control_record,
|
||||
risk_assessment=risk_assessment,
|
||||
digital_signature="", # Will be populated below
|
||||
compliance_attestation=compliance_attestation
|
||||
)
|
||||
|
||||
# Generate digital signature
|
||||
report_dict = asdict(report)
|
||||
report.digital_signature = self._generate_digital_signature(report_dict)
|
||||
|
||||
return report
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate compliance report for Foxhunt HFT deployment")
|
||||
parser.add_argument("--sha", required=True, help="Git commit SHA")
|
||||
parser.add_argument("--status", required=True, choices=["success", "failure", "partial"],
|
||||
help="Deployment status")
|
||||
parser.add_argument("--environment", default="production", help="Deployment environment")
|
||||
parser.add_argument("--output", required=True, help="Output file path")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Generate compliance report
|
||||
reporter = ComplianceReporter(args.sha, args.status, args.environment)
|
||||
report = reporter.generate_report()
|
||||
|
||||
# Save report to file
|
||||
output_path = Path(args.output)
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump(asdict(report), f, indent=2, default=str)
|
||||
|
||||
print(f"Compliance report generated: {output_path}")
|
||||
print(f"Report ID: {report.report_id}")
|
||||
print(f"Overall status: {'COMPLIANT' if args.status == 'success' else 'NON-COMPLIANT'}")
|
||||
|
||||
# Print summary
|
||||
print(f"\nSummary:")
|
||||
print(f"- Security vulnerabilities: {report.vulnerability_summary['total']}")
|
||||
print(f"- Performance validation: {report.latency_validation['status']}")
|
||||
print(f"- Deployment duration: {report.deployment_metrics.deployment_duration_seconds:.1f}s")
|
||||
print(f"- Zero downtime: {'Yes' if report.deployment_metrics.zero_downtime_achieved else 'No'}")
|
||||
|
||||
# Exit with status code
|
||||
sys.exit(0 if args.status == "success" else 1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,253 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# DQN Hyperparameter Optimization Dry-Run Script
|
||||
#
|
||||
# Purpose: Quick validation of hyperopt setup before committing to full 100-trial campaign
|
||||
# Duration: 5-10 minutes
|
||||
# Cost: $0.02-$0.04 (RTX A4000)
|
||||
#
|
||||
# This script:
|
||||
# 1. Runs 5 hyperopt trials with 10 epochs each
|
||||
# 2. Captures all training logs (loss, action distribution, gradient norms)
|
||||
# 3. Validates that Wave 11 bug fixes are working correctly
|
||||
# 4. Provides pass/fail report with actionable next steps
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/hyperopt_dqn_dryrun.sh
|
||||
|
||||
set -e # Exit on error
|
||||
set -o pipefail # Exit if any command in pipeline fails
|
||||
|
||||
# ==================== CONFIGURATION ====================
|
||||
|
||||
TRIALS=5
|
||||
EPOCHS=10
|
||||
PARQUET_FILE="test_data/ES_FUT_180d.parquet"
|
||||
LOG_DIR="/tmp/dqn_hyperopt_logs"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
LOG_FILE="${LOG_DIR}/dqn_hyperopt_dryrun_${TIMESTAMP}.log"
|
||||
|
||||
# Create log directory
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# ==================== BANNER ====================
|
||||
|
||||
echo "========================================"
|
||||
echo "DQN Hyperopt Dry-Run"
|
||||
echo "========================================"
|
||||
echo "Configuration:"
|
||||
echo " Trials: $TRIALS"
|
||||
echo " Epochs per trial: $EPOCHS"
|
||||
echo " Parquet file: $PARQUET_FILE"
|
||||
echo " Log file: $LOG_FILE"
|
||||
echo ""
|
||||
echo "Expected duration: 5-10 minutes"
|
||||
echo "Expected cost: \$0.02-\$0.04 (RTX A4000)"
|
||||
echo ""
|
||||
echo "Wave 11 Validations:"
|
||||
echo " ✓ Gradient clipping active (max_norm=10.0)"
|
||||
echo " ✓ Portfolio tracking operational"
|
||||
echo " ✓ HOLD penalty enabled (0.01 weight)"
|
||||
echo " ✓ Close price extraction accurate"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# ==================== PRE-FLIGHT CHECKS ====================
|
||||
|
||||
echo "[1/4] Pre-flight checks..."
|
||||
|
||||
# Check if parquet file exists
|
||||
if [ ! -f "$PARQUET_FILE" ]; then
|
||||
echo "❌ ERROR: Parquet file not found: $PARQUET_FILE"
|
||||
exit 1
|
||||
fi
|
||||
echo " ✓ Parquet file exists: $PARQUET_FILE"
|
||||
|
||||
# Check if CUDA is available
|
||||
if command -v nvidia-smi &> /dev/null; then
|
||||
echo " ✓ CUDA available (nvidia-smi found)"
|
||||
nvidia-smi --query-gpu=name,memory.total,memory.free --format=csv,noheader | head -1
|
||||
else
|
||||
echo " ⚠️ CUDA not available (will use CPU - slower)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# ==================== RUN HYPEROPT ====================
|
||||
|
||||
echo "[2/4] Running hyperopt (this will take 5-10 minutes)..."
|
||||
echo ""
|
||||
|
||||
# Run hyperopt and capture output to both file and stdout
|
||||
cargo run --release -p ml --example hyperopt_dqn_demo --features cuda -- \
|
||||
--parquet-file "$PARQUET_FILE" \
|
||||
--trials $TRIALS \
|
||||
--epochs $EPOCHS \
|
||||
2>&1 | tee "$LOG_FILE"
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [ $EXIT_CODE -ne 0 ]; then
|
||||
echo ""
|
||||
echo "❌ HYPEROPT FAILED (exit code: $EXIT_CODE)"
|
||||
echo " Check logs: $LOG_FILE"
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# ==================== VALIDATION CHECKS ====================
|
||||
|
||||
echo "[3/4] Running validation checks..."
|
||||
echo ""
|
||||
|
||||
# Check for errors/panics
|
||||
ERROR_COUNT=$(grep -i -c "error\|panic\|FAILED\|thread.*panicked" "$LOG_FILE" || true)
|
||||
echo " Errors/panics: $ERROR_COUNT"
|
||||
|
||||
# Check for gradient warnings (should be 0 with gradient clipping)
|
||||
GRAD_WARNINGS=$(grep -c "Large gradient\|gradient explosion\|Q-VALUE DIVERGENCE" "$LOG_FILE" || true)
|
||||
echo " Gradient warnings: $GRAD_WARNINGS"
|
||||
|
||||
# Check for Q-value explosions
|
||||
QVALUE_EXPLOSIONS=$(grep -c "Q-value explosion" "$LOG_FILE" || true)
|
||||
echo " Q-value explosions: $QVALUE_EXPLOSIONS"
|
||||
|
||||
# Check for action diversity (at least one trial should have varied actions)
|
||||
ACTION_DIST_COUNT=$(grep -c "Action Distribution" "$LOG_FILE" || true)
|
||||
echo " Action distribution logs: $ACTION_DIST_COUNT"
|
||||
|
||||
# Extract last trial's action distribution
|
||||
echo ""
|
||||
echo " Last trial action distribution:"
|
||||
if grep -q "Action Distribution" "$LOG_FILE"; then
|
||||
grep "Action Distribution" "$LOG_FILE" | tail -1 | sed 's/^/ /'
|
||||
else
|
||||
echo " ⚠️ No action distribution logs found"
|
||||
fi
|
||||
|
||||
# Check for HOLD penalty usage (we use hold_penalty_weight in config)
|
||||
HOLD_PENALTY_CONFIG=$(grep -c "hold_penalty_weight: 0.01" "$LOG_FILE" || true)
|
||||
echo ""
|
||||
echo " HOLD penalty weight in config: $HOLD_PENALTY_CONFIG occurrences"
|
||||
|
||||
# Check for portfolio tracking (feature vector should include portfolio state)
|
||||
PORTFOLIO_FEATURES=$(grep -c "portfolio" "$LOG_FILE" || true)
|
||||
echo " Portfolio tracking references: $PORTFOLIO_FEATURES"
|
||||
|
||||
# Check for gradient clipping activation
|
||||
GRAD_CLIP_COUNT=$(grep -c "gradient_clip_norm: Some(10.0)" "$LOG_FILE" || true)
|
||||
echo " Gradient clipping enabled: $GRAD_CLIP_COUNT occurrences"
|
||||
|
||||
echo ""
|
||||
|
||||
# ==================== RESULTS SUMMARY ====================
|
||||
|
||||
echo "[4/4] Validation summary"
|
||||
echo "========================================"
|
||||
|
||||
# Count passes and failures
|
||||
CHECKS_PASSED=0
|
||||
CHECKS_FAILED=0
|
||||
|
||||
echo ""
|
||||
echo "Critical Checks:"
|
||||
echo "----------------"
|
||||
|
||||
# Check 1: Training completed without errors
|
||||
if [ $ERROR_COUNT -eq 0 ]; then
|
||||
echo "✅ Training completed without errors"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo "❌ Training had $ERROR_COUNT errors/panics"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 2: Gradient clipping working (no explosions)
|
||||
if [ $GRAD_WARNINGS -eq 0 ] && [ $QVALUE_EXPLOSIONS -eq 0 ]; then
|
||||
echo "✅ Gradient clipping working (no explosions/warnings)"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo "❌ Gradient issues detected ($GRAD_WARNINGS warnings, $QVALUE_EXPLOSIONS explosions)"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 3: Action distribution logged
|
||||
if [ $ACTION_DIST_COUNT -gt 0 ]; then
|
||||
echo "✅ Action distribution logged ($ACTION_DIST_COUNT times)"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo "❌ No action distribution logs found"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 4: Gradient clipping config present
|
||||
if [ $GRAD_CLIP_COUNT -gt 0 ]; then
|
||||
echo "✅ Gradient clipping enabled in config"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo "⚠️ Gradient clipping config not found in logs"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Optional Checks:"
|
||||
echo "----------------"
|
||||
|
||||
# Check 5: HOLD penalty weight configured
|
||||
if [ $HOLD_PENALTY_CONFIG -gt 0 ]; then
|
||||
echo "✅ HOLD penalty weight configured (0.01)"
|
||||
else
|
||||
echo "⚠️ HOLD penalty weight not found in logs (may still be working)"
|
||||
fi
|
||||
|
||||
# Check 6: Portfolio tracking present
|
||||
if [ $PORTFOLIO_FEATURES -gt 0 ]; then
|
||||
echo "✅ Portfolio tracking references found"
|
||||
else
|
||||
echo "⚠️ Portfolio tracking references not found (may still be working)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Overall Result: $CHECKS_PASSED/4 critical checks passed"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Final verdict
|
||||
if [ $CHECKS_FAILED -eq 0 ]; then
|
||||
echo "🎉 DRY-RUN PASSED"
|
||||
echo ""
|
||||
echo "✅ All critical validations passed"
|
||||
echo "✅ Wave 11 bug fixes appear operational"
|
||||
echo "✅ Ready for full hyperopt campaign"
|
||||
echo ""
|
||||
echo "Next Steps:"
|
||||
echo " 1. Review hyperopt results in logs: $LOG_FILE"
|
||||
echo " 2. Verify action diversity is reasonable (not 100% HOLD)"
|
||||
echo " 3. Proceed with full 100-trial campaign:"
|
||||
echo " cargo run --release -p ml --example hyperopt_dqn_demo --features cuda -- \\"
|
||||
echo " --parquet-file $PARQUET_FILE --trials 100 --epochs 50"
|
||||
echo ""
|
||||
echo "Estimated full campaign:"
|
||||
echo " Duration: 60-90 minutes"
|
||||
echo " Cost: \$0.25-\$0.38 (RTX A4000)"
|
||||
exit 0
|
||||
else
|
||||
echo "⚠️ DRY-RUN COMPLETED WITH WARNINGS"
|
||||
echo ""
|
||||
echo "❌ $CHECKS_FAILED critical checks failed"
|
||||
echo "⚠️ Review issues above before proceeding"
|
||||
echo ""
|
||||
echo "Troubleshooting:"
|
||||
echo " 1. Check logs: $LOG_FILE"
|
||||
echo " 2. Look for specific error messages"
|
||||
echo " 3. Verify Wave 11 fixes are compiled correctly"
|
||||
echo " 4. Re-run dry-run after fixes"
|
||||
echo ""
|
||||
echo "Common Issues:"
|
||||
echo " - Gradient explosions: Check gradient_clip_norm in DQNHyperparameters"
|
||||
echo " - 100% HOLD: Check hold_penalty_weight and movement_threshold"
|
||||
echo " - Portfolio errors: Check PortfolioTracker initialization"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,75 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# MAMBA-2 Production Training Launch Script
|
||||
#
|
||||
# Configuration:
|
||||
# - 200 epochs (early stopping at ~150)
|
||||
# - 32 batch size (4GB VRAM optimized)
|
||||
# - 0.0001 learning rate
|
||||
# - 60 sequence length
|
||||
# - 128 hidden dim (memory efficient)
|
||||
# - 64 state dim
|
||||
# - GPU acceleration (RTX 3050 Ti)
|
||||
# - 360 training files (665,483 bars)
|
||||
#
|
||||
# Expected Duration: 2-3 hours
|
||||
#
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
# Configuration
|
||||
EPOCHS=200
|
||||
BATCH_SIZE=32
|
||||
LEARNING_RATE=0.0001
|
||||
SEQ_LENGTH=60
|
||||
HIDDEN_DIM=128
|
||||
STATE_DIM=64
|
||||
DATA_DIR="test_data/real/databento/ml_training"
|
||||
OUTPUT_DIR="ml/trained_models/production/mamba2"
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Print configuration
|
||||
echo "╔═══════════════════════════════════════════════════════════╗"
|
||||
echo "║ MAMBA-2 Production Training Launch ║"
|
||||
echo "╚═══════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " Epochs: $EPOCHS"
|
||||
echo " Batch Size: $BATCH_SIZE"
|
||||
echo " Learning Rate: $LEARNING_RATE"
|
||||
echo " Sequence Length: $SEQ_LENGTH"
|
||||
echo " Hidden Dim: $HIDDEN_DIM"
|
||||
echo " State Dim: $STATE_DIM"
|
||||
echo " Data Directory: $DATA_DIR"
|
||||
echo " Output Directory: $OUTPUT_DIR"
|
||||
echo ""
|
||||
echo "GPU: RTX 3050 Ti (4GB VRAM)"
|
||||
echo "Data: 360 files, 665,483 bars"
|
||||
echo "Expected Duration: 2-3 hours"
|
||||
echo ""
|
||||
echo "Starting training..."
|
||||
echo ""
|
||||
|
||||
# Launch training with CUDA
|
||||
CUDA_VISIBLE_DEVICES=0 cargo run --release -p ml --example train_mamba2_dbn --features cuda -- \
|
||||
--epochs "$EPOCHS" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--learning-rate "$LEARNING_RATE" \
|
||||
--sequence-length "$SEQ_LENGTH" \
|
||||
--hidden-dim "$HIDDEN_DIM" \
|
||||
--state-dim "$STATE_DIM" \
|
||||
--data-dir "$DATA_DIR" \
|
||||
--output-dir "$OUTPUT_DIR" \
|
||||
--use-gpu \
|
||||
2>&1 | tee "$OUTPUT_DIR/training.log"
|
||||
|
||||
echo ""
|
||||
echo "╔═══════════════════════════════════════════════════════════╗"
|
||||
echo "║ MAMBA-2 Training Complete ║"
|
||||
echo "╚═══════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo "Checkpoints saved to: $OUTPUT_DIR"
|
||||
echo "Training log: $OUTPUT_DIR/training.log"
|
||||
echo ""
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/bin/bash
|
||||
# MAMBA-2 Hyperopt Monitoring Script
|
||||
# Pod ID: w4srx0tgm5hfgu
|
||||
# Created: 2025-10-28
|
||||
|
||||
set -e
|
||||
|
||||
POD_ID="w4srx0tgm5hfgu"
|
||||
S3_BUCKET="s3://se3zdnb5o4"
|
||||
S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
|
||||
AWS_PROFILE="runpod"
|
||||
|
||||
echo "========================================"
|
||||
echo "MAMBA-2 Hyperopt Monitor"
|
||||
echo "========================================"
|
||||
echo "Pod ID: $POD_ID"
|
||||
echo "GPU: RTX A4000 (16GB VRAM)"
|
||||
echo "Cost: \$0.25/hr"
|
||||
echo "Expected Duration: ~60 minutes"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Function to check S3 results
|
||||
check_s3_results() {
|
||||
echo "📦 Checking S3 for results..."
|
||||
aws s3 ls "$S3_BUCKET/models/" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--recursive \
|
||||
--human-readable \
|
||||
| grep "mamba2_hyperopt" || echo " (No results yet)"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to show monitoring commands
|
||||
show_commands() {
|
||||
echo "📝 Monitoring Commands:"
|
||||
echo ""
|
||||
echo "1. SSH into pod:"
|
||||
echo " ssh root@$POD_ID.ssh.runpod.io"
|
||||
echo ""
|
||||
echo "2. View logs (inside pod):"
|
||||
echo " docker logs -f \$(docker ps -q)"
|
||||
echo ""
|
||||
echo "3. Monitor GPU (inside pod):"
|
||||
echo " watch -n 1 nvidia-smi"
|
||||
echo ""
|
||||
echo "4. Check process (inside pod):"
|
||||
echo " ps aux | grep hyperopt_mamba2_demo"
|
||||
echo ""
|
||||
echo "5. Check outputs (inside pod):"
|
||||
echo " ls -lh /runpod-volume/models/"
|
||||
echo ""
|
||||
echo "6. Runpod Console:"
|
||||
echo " https://www.runpod.io/console/pods"
|
||||
echo ""
|
||||
echo "7. Jupyter Notebook:"
|
||||
echo " https://$POD_ID-8888.proxy.runpod.net"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to download results
|
||||
download_results() {
|
||||
echo "📥 Downloading results from S3..."
|
||||
RESULTS_DIR="./runpod_hyperopt_results_$(date +%Y%m%d_%H%M%S)"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
aws s3 sync "$S3_BUCKET/models/" "$RESULTS_DIR/" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--exclude "*" \
|
||||
--include "mamba2_hyperopt*" \
|
||||
--no-progress
|
||||
|
||||
echo " ✅ Results downloaded to: $RESULTS_DIR"
|
||||
ls -lh "$RESULTS_DIR/"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to estimate completion time
|
||||
estimate_completion() {
|
||||
echo "⏱️ Estimated Timeline:"
|
||||
echo " - Initialization: ~2-3 minutes"
|
||||
echo " - Trial 1: ~5 minutes"
|
||||
echo " - Trial 10: ~20 minutes"
|
||||
echo " - Trial 20: ~40 minutes"
|
||||
echo " - Trial 30 (Complete): ~60 minutes"
|
||||
echo " - Auto-Termination: ~61 minutes"
|
||||
echo ""
|
||||
echo "💰 Cost Estimate: \$0.19-\$0.38 (45-90 minutes)"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Main menu
|
||||
case "${1:-help}" in
|
||||
status|s)
|
||||
check_s3_results
|
||||
estimate_completion
|
||||
;;
|
||||
commands|c)
|
||||
show_commands
|
||||
;;
|
||||
download|d)
|
||||
download_results
|
||||
;;
|
||||
watch|w)
|
||||
echo "🔄 Watching for results (checks every 30 seconds)..."
|
||||
echo " Press Ctrl+C to stop"
|
||||
echo ""
|
||||
while true; do
|
||||
check_s3_results
|
||||
sleep 30
|
||||
done
|
||||
;;
|
||||
help|h|*)
|
||||
echo "Usage: $0 [command]"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " status (s) - Check S3 for results and show timeline"
|
||||
echo " commands (c) - Show monitoring commands"
|
||||
echo " download (d) - Download results from S3"
|
||||
echo " watch (w) - Watch for results (checks every 30s)"
|
||||
echo " help (h) - Show this help"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 status # Check current status"
|
||||
echo " $0 watch # Monitor in real-time"
|
||||
echo " $0 download # Download completed results"
|
||||
echo ""
|
||||
;;
|
||||
esac
|
||||
@@ -1,558 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
S3 Log Monitoring Script - Real-time training log streaming
|
||||
|
||||
Monitors ML training runs on RunPod with S3 log tailing.
|
||||
Supports both pod-based and run-based monitoring with rich output.
|
||||
|
||||
Features:
|
||||
- List recent pods and runs
|
||||
- Stream training logs in real-time
|
||||
- Monitor hyperopt trials.json updates
|
||||
- Follow mode for continuous streaming
|
||||
- Configurable timeout
|
||||
- Color-coded output (errors, warnings, success)
|
||||
|
||||
Usage:
|
||||
# List recent activity
|
||||
python3 scripts/monitor_logs.py
|
||||
|
||||
# Monitor specific run (by run ID)
|
||||
python3 scripts/monitor_logs.py --run-id run_20251029_145151_hyperopt --follow
|
||||
|
||||
# Monitor specific pod (by pod ID)
|
||||
python3 scripts/monitor_logs.py --pod-id w4srx0tgm5hfgu --follow
|
||||
|
||||
# With timeout
|
||||
python3 scripts/monitor_logs.py --run-id run_20251029_145151_hyperopt --follow --timeout 30m
|
||||
|
||||
Requirements:
|
||||
- .venv activation (source .venv/bin/activate)
|
||||
- Dependencies: rich, boto3, pydantic-settings
|
||||
- .env.runpod with S3 credentials
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import time
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Check for .venv activation
|
||||
is_venv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)
|
||||
if not is_venv:
|
||||
print("=" * 70)
|
||||
print("ERROR: Not running in a virtual environment (.venv)")
|
||||
print("=" * 70)
|
||||
print("This script requires .venv activation to ensure correct dependencies.")
|
||||
print()
|
||||
print("To fix this:")
|
||||
print(" 1. Activate .venv: source .venv/bin/activate")
|
||||
print(" 2. Install deps: pip install -r runpod/requirements.txt")
|
||||
print(" 3. Re-run script: python3 scripts/monitor_logs.py --help")
|
||||
print("=" * 70)
|
||||
sys.exit(1)
|
||||
|
||||
# Get project root for later use
|
||||
script_dir = Path(__file__).parent
|
||||
project_root = script_dir.parent
|
||||
|
||||
# Add project root to sys.path to ensure local runpod module is used
|
||||
# This must be done BEFORE importing from runpod to avoid pip package conflict
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
try:
|
||||
# Import from local runpod module (not pip-installed package)
|
||||
from runpod import PodMonitor, S3Client, RunPodConfig
|
||||
from runpod.errors import S3ObjectNotFoundError
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
from rich import box
|
||||
except ImportError as e:
|
||||
print(f"ERROR: Could not import required modules: {e}")
|
||||
print()
|
||||
print("To fix this:")
|
||||
print(" 1. Ensure .venv is activated: source .venv/bin/activate")
|
||||
print(" 2. Install dependencies: pip install -r runpod/requirements.txt")
|
||||
print()
|
||||
sys.exit(1)
|
||||
|
||||
# Change to project root for config loading
|
||||
os.chdir(project_root)
|
||||
|
||||
# Set environment variable to help config find .env.runpod
|
||||
os.environ.setdefault('RUNPOD_CONFIG_PATH', str(project_root / '.env.runpod'))
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def parse_timeout(timeout_str: str) -> Optional[int]:
|
||||
"""
|
||||
Parse timeout string to seconds.
|
||||
|
||||
Args:
|
||||
timeout_str: Timeout string (e.g., '30m', '2h', '45s')
|
||||
|
||||
Returns:
|
||||
Timeout in seconds, or None if invalid
|
||||
"""
|
||||
match = re.match(r'(\d+)([smh]?)$', timeout_str.lower())
|
||||
if not match:
|
||||
return None
|
||||
|
||||
value = int(match.group(1))
|
||||
unit = match.group(2) or 's'
|
||||
|
||||
if unit == 'h':
|
||||
return value * 3600
|
||||
elif unit == 'm':
|
||||
return value * 60
|
||||
else: # 's'
|
||||
return value
|
||||
|
||||
|
||||
def list_recent_runs(s3_client: S3Client, limit: int = 20) -> list[dict]:
|
||||
"""
|
||||
List recent training runs from S3.
|
||||
|
||||
Args:
|
||||
s3_client: S3Client instance
|
||||
limit: Maximum number of runs to return
|
||||
|
||||
Returns:
|
||||
List of run metadata dicts
|
||||
"""
|
||||
try:
|
||||
# List all training run directories
|
||||
response = s3_client.s3_client.list_objects_v2(
|
||||
Bucket=s3_client.bucket,
|
||||
Prefix="ml_training/training_runs/",
|
||||
Delimiter="/"
|
||||
)
|
||||
|
||||
runs = []
|
||||
|
||||
# Iterate through model types (mamba2, dqn, ppo, tft)
|
||||
for prefix_obj in response.get('CommonPrefixes', []):
|
||||
model_prefix = prefix_obj['Prefix']
|
||||
|
||||
# List runs for this model type
|
||||
model_response = s3_client.s3_client.list_objects_v2(
|
||||
Bucket=s3_client.bucket,
|
||||
Prefix=model_prefix,
|
||||
Delimiter="/"
|
||||
)
|
||||
|
||||
for run_prefix_obj in model_response.get('CommonPrefixes', []):
|
||||
run_path = run_prefix_obj['Prefix']
|
||||
run_id = run_path.rstrip('/').split('/')[-1]
|
||||
|
||||
# Extract model type
|
||||
model_type = model_prefix.rstrip('/').split('/')[-1]
|
||||
|
||||
# Try to get log file metadata for last modified time
|
||||
log_key = f"{run_path}logs/training.log"
|
||||
try:
|
||||
log_obj = s3_client.s3_client.head_object(
|
||||
Bucket=s3_client.bucket,
|
||||
Key=log_key
|
||||
)
|
||||
last_modified = log_obj['LastModified']
|
||||
log_size = log_obj['ContentLength']
|
||||
except:
|
||||
last_modified = None
|
||||
log_size = 0
|
||||
|
||||
# Check for trials.json (hyperopt indicator)
|
||||
trials_key = f"{run_path}hyperopt/trials.json"
|
||||
has_trials = s3_client.object_exists(trials_key)
|
||||
|
||||
runs.append({
|
||||
'run_id': run_id,
|
||||
'model_type': model_type,
|
||||
'last_modified': last_modified,
|
||||
'log_size': log_size,
|
||||
'has_trials': has_trials,
|
||||
'log_key': log_key,
|
||||
'trials_key': trials_key if has_trials else None
|
||||
})
|
||||
|
||||
# Sort by last modified (newest first)
|
||||
# Use min datetime with UTC timezone for consistency
|
||||
from datetime import timezone
|
||||
min_dt = datetime.min.replace(tzinfo=timezone.utc)
|
||||
runs.sort(key=lambda x: x['last_modified'] or min_dt, reverse=True)
|
||||
|
||||
return runs[:limit]
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error listing runs: {e}[/red]")
|
||||
return []
|
||||
|
||||
|
||||
def display_recent_runs(runs: list[dict]) -> None:
|
||||
"""Display recent runs in a formatted table."""
|
||||
if not runs:
|
||||
console.print("[yellow]No recent runs found[/yellow]")
|
||||
return
|
||||
|
||||
table = Table(title="Recent Training Runs", box=box.ROUNDED)
|
||||
table.add_column("Run ID", style="cyan", no_wrap=True)
|
||||
table.add_column("Model", style="magenta")
|
||||
table.add_column("Last Modified", style="green")
|
||||
table.add_column("Log Size", justify="right", style="blue")
|
||||
table.add_column("Trials", justify="center")
|
||||
|
||||
for run in runs:
|
||||
run_id = run['run_id']
|
||||
model_type = run['model_type'].upper()
|
||||
|
||||
if run['last_modified']:
|
||||
# Format as relative time
|
||||
now = datetime.now(run['last_modified'].tzinfo)
|
||||
delta = now - run['last_modified']
|
||||
|
||||
if delta < timedelta(minutes=1):
|
||||
time_str = "just now"
|
||||
elif delta < timedelta(hours=1):
|
||||
time_str = f"{int(delta.total_seconds() / 60)}m ago"
|
||||
elif delta < timedelta(days=1):
|
||||
time_str = f"{int(delta.total_seconds() / 3600)}h ago"
|
||||
else:
|
||||
time_str = f"{delta.days}d ago"
|
||||
else:
|
||||
time_str = "unknown"
|
||||
|
||||
# Format log size
|
||||
if run['log_size'] > 0:
|
||||
if run['log_size'] < 1024:
|
||||
size_str = f"{run['log_size']}B"
|
||||
elif run['log_size'] < 1024 * 1024:
|
||||
size_str = f"{run['log_size'] / 1024:.1f}KB"
|
||||
else:
|
||||
size_str = f"{run['log_size'] / (1024 * 1024):.2f}MB"
|
||||
else:
|
||||
size_str = "-"
|
||||
|
||||
trials_str = "✓" if run['has_trials'] else "-"
|
||||
|
||||
table.add_row(run_id, model_type, time_str, size_str, trials_str)
|
||||
|
||||
console.print()
|
||||
console.print(table)
|
||||
console.print()
|
||||
console.print("[dim]Use --run-id <RUN_ID> to monitor a specific run[/dim]")
|
||||
console.print()
|
||||
|
||||
|
||||
def stream_run_logs(
|
||||
s3_client: S3Client,
|
||||
run_id: str,
|
||||
follow: bool = True,
|
||||
timeout: Optional[int] = None,
|
||||
poll_interval: int = 5
|
||||
) -> None:
|
||||
"""
|
||||
Stream logs for a specific run.
|
||||
|
||||
Args:
|
||||
s3_client: S3Client instance
|
||||
run_id: Run ID to monitor
|
||||
follow: Continue streaming until completion
|
||||
timeout: Maximum monitoring time in seconds
|
||||
poll_interval: Polling interval in seconds
|
||||
"""
|
||||
# Find the run's log path
|
||||
try:
|
||||
# Search for run across all model types
|
||||
for model_type in ['mamba2', 'dqn', 'ppo', 'tft']:
|
||||
log_key = f"ml_training/training_runs/{model_type}/{run_id}/logs/training.log"
|
||||
if s3_client.object_exists(log_key):
|
||||
trials_key = f"ml_training/training_runs/{model_type}/{run_id}/hyperopt/trials.json"
|
||||
has_trials = s3_client.object_exists(trials_key)
|
||||
break
|
||||
else:
|
||||
console.print(f"[red]Run not found: {run_id}[/red]")
|
||||
console.print("[dim]Use 'python3 scripts/monitor_logs.py' to list available runs[/dim]")
|
||||
return
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error finding run: {e}[/red]")
|
||||
return
|
||||
|
||||
# Display header
|
||||
console.print()
|
||||
console.print(Panel.fit(
|
||||
f"[bold cyan]Monitoring Run: {run_id}[/bold cyan]\n"
|
||||
f"Model: [magenta]{model_type.upper()}[/magenta]\n"
|
||||
f"Log: [dim]{log_key}[/dim]\n"
|
||||
f"Hyperopt: [green]Yes[/green]" if has_trials else "[dim]No[/dim]",
|
||||
title="Log Monitor",
|
||||
border_style="cyan"
|
||||
))
|
||||
console.print()
|
||||
|
||||
# Stream logs
|
||||
log_position = 0
|
||||
start_time = time.time()
|
||||
last_trials_check = 0
|
||||
|
||||
# Pattern detection
|
||||
completion_patterns = [
|
||||
"Training complete",
|
||||
"Model saved to",
|
||||
"✓ Training finished",
|
||||
"SUCCESS:",
|
||||
"Hyperparameter optimization complete"
|
||||
]
|
||||
|
||||
error_patterns = [
|
||||
"CUDA out of memory",
|
||||
"RuntimeError:",
|
||||
"AssertionError:",
|
||||
"FAILED:",
|
||||
"ERROR:",
|
||||
"panic!"
|
||||
]
|
||||
|
||||
training_complete = False
|
||||
error_detected = False
|
||||
|
||||
try:
|
||||
console.print("[dim]Streaming logs... (Press Ctrl+C to stop)[/dim]")
|
||||
console.print("─" * 70)
|
||||
|
||||
while True:
|
||||
# Check timeout
|
||||
if timeout and (time.time() - start_time) > timeout:
|
||||
console.print()
|
||||
console.print("[yellow]⏱️ Timeout reached[/yellow]")
|
||||
break
|
||||
|
||||
# Check for new log content
|
||||
try:
|
||||
content, log_position = s3_client.tail_log_file(log_key, start_byte=log_position)
|
||||
|
||||
if content:
|
||||
text = content.decode('utf-8', errors='ignore')
|
||||
lines = text.splitlines()
|
||||
|
||||
for line in lines:
|
||||
# Color-code output based on content
|
||||
if any(pattern in line for pattern in error_patterns):
|
||||
console.print(f"[red]{line}[/red]")
|
||||
error_detected = True
|
||||
elif "WARN" in line.upper():
|
||||
console.print(f"[yellow]{line}[/yellow]")
|
||||
elif "SUCCESS" in line.upper() or "✓" in line:
|
||||
console.print(f"[green]{line}[/green]")
|
||||
elif any(pattern in line for pattern in completion_patterns):
|
||||
console.print(f"[bold green]{line}[/bold green]")
|
||||
training_complete = True
|
||||
else:
|
||||
console.print(line)
|
||||
|
||||
except S3ObjectNotFoundError:
|
||||
if not follow:
|
||||
console.print("[yellow]⚠ Log file not found[/yellow]")
|
||||
return
|
||||
# Still waiting for log file to appear
|
||||
pass
|
||||
|
||||
# Check trials.json updates (every 30 seconds)
|
||||
if has_trials and (time.time() - last_trials_check) > 30:
|
||||
last_trials_check = time.time()
|
||||
try:
|
||||
trials_obj = s3_client.s3_client.get_object(
|
||||
Bucket=s3_client.bucket,
|
||||
Key=trials_key
|
||||
)
|
||||
trials_content = trials_obj['Body'].read().decode('utf-8')
|
||||
|
||||
# Parse trial count
|
||||
import json
|
||||
trials_data = json.loads(trials_content)
|
||||
trial_count = len(trials_data)
|
||||
|
||||
console.print(f"[dim]📊 Hyperopt trials: {trial_count}[/dim]")
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# Check completion
|
||||
if training_complete or error_detected:
|
||||
console.print()
|
||||
console.print("─" * 70)
|
||||
if training_complete:
|
||||
console.print("[bold green]✅ Training completed![/bold green]")
|
||||
if error_detected:
|
||||
console.print("[bold red]❌ Error detected in logs[/bold red]")
|
||||
break
|
||||
|
||||
if not follow:
|
||||
break
|
||||
|
||||
time.sleep(poll_interval)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
console.print()
|
||||
console.print()
|
||||
console.print("[yellow]Streaming stopped by user[/yellow]")
|
||||
|
||||
|
||||
def stream_pod_logs(
|
||||
pod_id: str,
|
||||
config,
|
||||
follow: bool = True,
|
||||
timeout: Optional[int] = None,
|
||||
poll_interval: int = 5
|
||||
) -> None:
|
||||
"""
|
||||
Stream logs for a specific pod using PodMonitor.
|
||||
|
||||
Args:
|
||||
pod_id: Pod ID to monitor
|
||||
config: RunPodConfig instance
|
||||
follow: Continue streaming until completion
|
||||
timeout: Maximum monitoring time in seconds
|
||||
poll_interval: Polling interval in seconds
|
||||
"""
|
||||
try:
|
||||
monitor = PodMonitor(pod_id=pod_id, config=config)
|
||||
|
||||
# Display pod info
|
||||
monitor.display_pod_info()
|
||||
|
||||
# Stream logs
|
||||
max_lines = None
|
||||
if timeout:
|
||||
# Estimate max lines based on timeout
|
||||
max_lines = timeout * 10 # Rough estimate
|
||||
|
||||
monitor.stream_s3_logs(
|
||||
follow=follow,
|
||||
poll_interval=poll_interval,
|
||||
max_lines=max_lines
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error monitoring pod: {e}[/red]")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main execution function."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Monitor ML training logs on RunPod S3",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# List recent runs
|
||||
python3 scripts/monitor_logs.py
|
||||
|
||||
# Monitor specific run (continuous streaming)
|
||||
python3 scripts/monitor_logs.py --run-id run_20251029_145151_hyperopt --follow
|
||||
|
||||
# Monitor specific pod
|
||||
python3 scripts/monitor_logs.py --pod-id w4srx0tgm5hfgu --follow
|
||||
|
||||
# With timeout (30 minutes)
|
||||
python3 scripts/monitor_logs.py --run-id run_20251029_145151_hyperopt --follow --timeout 30m
|
||||
|
||||
# One-time log snapshot (no streaming)
|
||||
python3 scripts/monitor_logs.py --run-id run_20251029_145151_hyperopt
|
||||
|
||||
Requirements:
|
||||
- .venv activation (source .venv/bin/activate)
|
||||
- .env.runpod with S3 credentials
|
||||
- Dependencies: rich, boto3, pydantic-settings
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--run-id',
|
||||
help='Run ID to monitor (e.g., run_20251029_145151_hyperopt)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--pod-id',
|
||||
help='Pod ID to monitor (e.g., w4srx0tgm5hfgu)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--follow',
|
||||
action='store_true',
|
||||
help='Continuously stream logs until completion'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--timeout',
|
||||
default=None,
|
||||
help='Maximum monitoring time (e.g., 30m, 2h). Default: no timeout'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--interval',
|
||||
type=int,
|
||||
default=5,
|
||||
help='Log polling interval in seconds (default: 5)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--limit',
|
||||
type=int,
|
||||
default=20,
|
||||
help='Maximum number of runs to list (default: 20)'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Parse timeout
|
||||
timeout_seconds = None
|
||||
if args.timeout:
|
||||
timeout_seconds = parse_timeout(args.timeout)
|
||||
if timeout_seconds is None:
|
||||
console.print(f"[red]Invalid timeout format: {args.timeout}[/red]")
|
||||
console.print("[dim]Use format like '30m', '2h', or '45s'[/dim]")
|
||||
sys.exit(1)
|
||||
|
||||
# Validate arguments
|
||||
if args.run_id and args.pod_id:
|
||||
console.print("[red]Error: Cannot specify both --run-id and --pod-id[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
# Load config from explicit path
|
||||
config = RunPodConfig.load_from_file(project_root / '.env.runpod')
|
||||
s3_client = S3Client(config)
|
||||
|
||||
# Monitor specific run
|
||||
if args.run_id:
|
||||
stream_run_logs(
|
||||
s3_client=s3_client,
|
||||
run_id=args.run_id,
|
||||
follow=args.follow,
|
||||
timeout=timeout_seconds,
|
||||
poll_interval=args.interval
|
||||
)
|
||||
|
||||
# Monitor specific pod
|
||||
elif args.pod_id:
|
||||
stream_pod_logs(
|
||||
pod_id=args.pod_id,
|
||||
config=config,
|
||||
follow=args.follow,
|
||||
timeout=timeout_seconds,
|
||||
poll_interval=args.interval
|
||||
)
|
||||
|
||||
# List recent runs
|
||||
else:
|
||||
runs = list_recent_runs(s3_client, limit=args.limit)
|
||||
display_recent_runs(runs)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,47 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Monitor MAMBA2 Hyperopt Results on Runpod S3
|
||||
|
||||
ENDPOINT="https://s3api-eur-is-1.runpod.io"
|
||||
PROFILE="runpod"
|
||||
BUCKET="s3://se3zdnb5o4"
|
||||
|
||||
echo "╔════════════════════════════════════════════════════════════════════╗"
|
||||
echo "║ MAMBA2 Hyperopt Results Monitor ║"
|
||||
echo "╚════════════════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
# Check if results directory exists
|
||||
echo "Checking S3 for hyperopt results..."
|
||||
echo ""
|
||||
|
||||
aws s3 ls ${BUCKET}/results/ \
|
||||
--profile ${PROFILE} \
|
||||
--endpoint-url ${ENDPOINT} \
|
||||
--human-readable \
|
||||
--recursive 2>/dev/null
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo ""
|
||||
echo "To download best parameters:"
|
||||
echo ""
|
||||
echo " aws s3 cp ${BUCKET}/results/mamba2_13param_best_params_*.json \\"
|
||||
echo " /tmp/mamba2_best_params.json \\"
|
||||
echo " --profile ${PROFILE} \\"
|
||||
echo " --endpoint-url ${ENDPOINT}"
|
||||
echo ""
|
||||
echo "To download full log:"
|
||||
echo ""
|
||||
echo " aws s3 cp ${BUCKET}/results/mamba2_13param_hyperopt_*.log \\"
|
||||
echo " /tmp/mamba2_hyperopt.log \\"
|
||||
echo " --profile ${PROFILE} \\"
|
||||
echo " --endpoint-url ${ENDPOINT}"
|
||||
echo ""
|
||||
else
|
||||
echo ""
|
||||
echo "No results found yet. Hyperopt is likely still running."
|
||||
echo ""
|
||||
echo "Expected completion time: 60-90 minutes from deployment"
|
||||
echo ""
|
||||
echo "Run this script again in 10-15 minutes to check progress."
|
||||
echo ""
|
||||
fi
|
||||
@@ -1,649 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod Deployment Script - REFACTORED VERSION
|
||||
Scans for best value GPU in EUR-IS region and deploys a pod on SECURE cloud.
|
||||
|
||||
NEW: Integrates with foxhunt_runpod module for enhanced features:
|
||||
- RunPodClient for pod management
|
||||
- S3LogMonitor for real-time log streaming
|
||||
- Auto-termination support
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Check for .venv activation
|
||||
is_venv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)
|
||||
if not is_venv:
|
||||
print("=" * 70)
|
||||
print("ERROR: Not running in a virtual environment (.venv)")
|
||||
print("=" * 70)
|
||||
print("This script requires .venv activation to ensure correct dependencies.")
|
||||
print()
|
||||
print("To fix this:")
|
||||
print(" 1. Activate .venv: source .venv/bin/activate")
|
||||
print(" 2. Install deps: pip install -r runpod/requirements.txt")
|
||||
print(" 3. Re-run script: python3 scripts/runpod_deploy.py --help")
|
||||
print("=" * 70)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
# Import from runpod module (clean architecture - no sys.path hacks)
|
||||
from runpod import RunPodClient, PodMonitor, S3Client
|
||||
from runpod.s3_monitor import S3LogMonitor
|
||||
import requests # Required for legacy GraphQL queries
|
||||
USE_NEW_MODULE = True
|
||||
except ImportError as e:
|
||||
print(f"ERROR: Could not import runpod module: {e}")
|
||||
print()
|
||||
print("To fix this:")
|
||||
print(" 1. Ensure .venv is activated: source .venv/bin/activate")
|
||||
print(" 2. Install dependencies: pip install -r runpod/requirements.txt")
|
||||
print(" 3. Check module location: ls runpod/")
|
||||
print()
|
||||
sys.exit(1)
|
||||
|
||||
# Load environment variables from .env.runpod
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
RUNPOD_VOLUME_ID = os.getenv('RUNPOD_VOLUME_ID')
|
||||
RUNPOD_CONTAINER_REGISTRY_AUTH_ID = os.getenv('RUNPOD_CONTAINER_REGISTRY_AUTH_ID')
|
||||
|
||||
# S3 credentials (optional - for log monitoring)
|
||||
RUNPOD_S3_BUCKET = os.getenv('RUNPOD_S3_BUCKET')
|
||||
RUNPOD_S3_ACCESS_KEY = os.getenv('RUNPOD_S3_ACCESS_KEY')
|
||||
RUNPOD_S3_SECRET_KEY = os.getenv('RUNPOD_S3_SECRET_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
if not RUNPOD_VOLUME_ID:
|
||||
print("ERROR: RUNPOD_VOLUME_ID not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
# EUR-IS datacenters to scan
|
||||
# CRITICAL FIX: Volume se3zdnb5o4 is in EUR-IS-1 ONLY!
|
||||
# If pod deploys to EUR-IS-2 or EUR-IS-3, volume will NOT be accessible
|
||||
EUR_IS_DATACENTERS = ['EUR-IS-1'] # ONLY EUR-IS-1 for volume mounting!
|
||||
|
||||
# REST API endpoint (NEW - supports datacenter filtering)
|
||||
REST_API_URL = "https://rest.runpod.io/v1/pods"
|
||||
|
||||
# GraphQL endpoint (still needed for listing GPU types and pricing)
|
||||
GRAPHQL_ENDPOINT = "https://api.runpod.io/graphql"
|
||||
|
||||
# GraphQL query to fetch GPU types with global availability and pricing
|
||||
GPU_QUERY = """
|
||||
{
|
||||
gpuTypes {
|
||||
id
|
||||
displayName
|
||||
memoryInGb
|
||||
secureCloud
|
||||
communityCloud
|
||||
lowestPrice(input: {gpuCount: 1}) {
|
||||
uninterruptablePrice
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
def query_graphql(query, variables=None):
|
||||
"""Execute GraphQL query against RunPod API."""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
payload = {"query": query}
|
||||
if variables:
|
||||
payload["variables"] = variables
|
||||
|
||||
try:
|
||||
response = requests.post(GRAPHQL_ENDPOINT, json=payload, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
|
||||
# Check for GraphQL errors
|
||||
if 'errors' in result:
|
||||
error_msg = result['errors'][0].get('message', 'Unknown error')
|
||||
print(f"ERROR: GraphQL errors: {result['errors']}")
|
||||
return None
|
||||
|
||||
return result
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"ERROR: Failed to query RunPod GraphQL API: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def get_available_gpu_types():
|
||||
"""
|
||||
Query available GPU types using RunPodClient.
|
||||
"""
|
||||
print(" Querying GPU types and pricing (using RunPodClient)...")
|
||||
|
||||
client = RunPodClient(
|
||||
api_key=RUNPOD_API_KEY,
|
||||
volume_id=RUNPOD_VOLUME_ID,
|
||||
registry_auth_id=RUNPOD_CONTAINER_REGISTRY_AUTH_ID
|
||||
)
|
||||
|
||||
gpus = client.get_available_gpus(min_vram=16)
|
||||
|
||||
if gpus:
|
||||
print(f" ✅ Found {len(gpus)} GPU type(s) with global secure cloud availability")
|
||||
else:
|
||||
print(f" ⚠️ No GPU types found with ≥16GB VRAM in secure cloud")
|
||||
|
||||
return gpus
|
||||
|
||||
|
||||
def get_available_gpu_types_legacy_unused():
|
||||
"""
|
||||
Query available GPU types with ≥16GB VRAM that have SOME availability in SECURE cloud (LEGACY).
|
||||
|
||||
NOTE: This returns GLOBAL secure cloud availability, not EUR-IS specific.
|
||||
The actual datacenter filtering happens during deployment via REST API.
|
||||
"""
|
||||
print(" Querying GPU types and pricing (legacy mode)...")
|
||||
|
||||
data = query_graphql(GPU_QUERY)
|
||||
if not data:
|
||||
return []
|
||||
|
||||
gpu_types = data.get('data', {}).get('gpuTypes', [])
|
||||
|
||||
# Filter criteria:
|
||||
# 1. memoryInGb >= 16
|
||||
# 2. secureCloud > 0 (available SOMEWHERE in secure cloud - not necessarily EUR-IS)
|
||||
# 3. Has pricing information
|
||||
available_gpus = []
|
||||
for gpu in gpu_types:
|
||||
memory = gpu.get('memoryInGb', 0)
|
||||
secure_count = gpu.get('secureCloud', 0)
|
||||
lowest_price = gpu.get('lowestPrice', {})
|
||||
price = lowest_price.get('uninterruptablePrice') if lowest_price else None
|
||||
|
||||
if memory >= 16 and secure_count > 0 and price is not None:
|
||||
available_gpus.append({
|
||||
'id': gpu.get('id', ''),
|
||||
'name': gpu.get('displayName', 'Unknown'),
|
||||
'vram': memory,
|
||||
'price': float(price),
|
||||
'global_available': secure_count # This is GLOBAL, not EUR-IS specific
|
||||
})
|
||||
|
||||
if available_gpus:
|
||||
# Sort by price (cheapest first)
|
||||
available_gpus.sort(key=lambda x: x['price'])
|
||||
print(f" ✅ Found {len(available_gpus)} GPU type(s) with global secure cloud availability")
|
||||
else:
|
||||
print(f" ⚠️ No GPU types found with ≥16GB VRAM in secure cloud")
|
||||
|
||||
return available_gpus
|
||||
|
||||
|
||||
def deploy_pod(gpu, image, command, container_disk, dry_run=False):
|
||||
"""Deploy a pod using RunPodClient."""
|
||||
client = RunPodClient(
|
||||
api_key=RUNPOD_API_KEY,
|
||||
volume_id=RUNPOD_VOLUME_ID,
|
||||
registry_auth_id=RUNPOD_CONTAINER_REGISTRY_AUTH_ID
|
||||
)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("DEPLOYMENT PLAN")
|
||||
print("="*70)
|
||||
print(f"Pod Name: foxhunt-training")
|
||||
print(f"GPU: {gpu['name']} ({gpu['vram']}GB VRAM)")
|
||||
print(f"Datacenters: EUR-IS-1 (volume location)")
|
||||
print(f"Price: ${gpu['price']:.3f}/hr (estimate)")
|
||||
print(f"Docker Image: {image}")
|
||||
print(f"Container Disk: {container_disk}GB")
|
||||
print(f"Network Volume: {RUNPOD_VOLUME_ID} → /runpod-volume")
|
||||
print(f"Ports: 8888/http (Jupyter), 22/tcp (SSH)")
|
||||
print(f"Registry Auth: {RUNPOD_CONTAINER_REGISTRY_AUTH_ID}")
|
||||
if command:
|
||||
print(f"Command: {command}")
|
||||
print("="*70)
|
||||
|
||||
if dry_run:
|
||||
print("\nDRY RUN: Skipping actual deployment")
|
||||
return None
|
||||
|
||||
print("\nDeploying pod via REST API (checks EUR-IS availability)...")
|
||||
|
||||
pod_data = client.deploy_pod(
|
||||
gpu_id=gpu['id'],
|
||||
image=image,
|
||||
command=command,
|
||||
container_disk=container_disk,
|
||||
dry_run=False
|
||||
)
|
||||
|
||||
if pod_data:
|
||||
print(f" ✅ Pod created successfully! ID: {pod_data['id']}")
|
||||
|
||||
return pod_data
|
||||
|
||||
|
||||
def deploy_pod_rest_api_legacy(gpu, image, command, container_disk, datacenters, dry_run=False):
|
||||
"""
|
||||
Deploy a pod using the REST API with datacenter-specific availability filtering (LEGACY).
|
||||
|
||||
This is the KEY FIX: REST API checks EUR-IS datacenter availability at deployment time,
|
||||
not relying on global GraphQL counts.
|
||||
"""
|
||||
pod_name = "foxhunt-training"
|
||||
|
||||
# Build REST API request payload
|
||||
# CRITICAL: Only use fields documented in official RunPod REST API
|
||||
# Reference: https://docs.runpod.io/api-reference/pods/POST/pods
|
||||
deployment_payload = {
|
||||
"cloudType": "SECURE", # CRITICAL: Only secure cloud
|
||||
"computeType": "GPU", # GPU pod (required field)
|
||||
"dataCenterIds": datacenters, # CRITICAL: EUR-IS specific datacenters
|
||||
"dataCenterPriority": "availability", # Try datacenters in order, prefer available
|
||||
"gpuTypeIds": [gpu['id']], # GPU type to deploy
|
||||
"gpuTypePriority": "availability", # Use available GPU
|
||||
"gpuCount": 1,
|
||||
"name": pod_name,
|
||||
"imageName": image,
|
||||
"containerDiskInGb": container_disk,
|
||||
"volumeInGb": 0, # Using network volume instead
|
||||
"networkVolumeId": RUNPOD_VOLUME_ID,
|
||||
"volumeMountPath": "/runpod-volume", # CRITICAL: WHERE to mount the volume
|
||||
"ports": ["8888/http", "22/tcp"],
|
||||
"env": {},
|
||||
"interruptible": False, # On-demand (non-spot)
|
||||
"minRAMPerGPU": 8,
|
||||
"minVCPUPerGPU": 2
|
||||
}
|
||||
|
||||
# Add Docker command if specified
|
||||
# RunPod expects dockerStartCmd as an array of strings (shell arguments)
|
||||
# Split the command string into proper arguments
|
||||
if command:
|
||||
# Split command into arguments (respects quoted strings)
|
||||
import shlex
|
||||
deployment_payload["dockerStartCmd"] = shlex.split(command)
|
||||
|
||||
# Add Docker registry authentication if configured
|
||||
if RUNPOD_CONTAINER_REGISTRY_AUTH_ID:
|
||||
deployment_payload["containerRegistryAuthId"] = RUNPOD_CONTAINER_REGISTRY_AUTH_ID
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("DEPLOYMENT PLAN (legacy mode)")
|
||||
print("="*70)
|
||||
print(f"Pod Name: {pod_name}")
|
||||
print(f"GPU: {gpu['name']} ({gpu['vram']}GB VRAM)")
|
||||
print(f"Datacenters: {', '.join(datacenters)} (tries in order)")
|
||||
print(f"Price: ${gpu['price']:.3f}/hr (estimate)")
|
||||
print(f"Docker Image: {image}")
|
||||
print(f"Container Disk: {container_disk}GB")
|
||||
print(f"Network Volume: {RUNPOD_VOLUME_ID} → /runpod-volume")
|
||||
print(f"Ports: 8888/http (Jupyter), 22/tcp (SSH)")
|
||||
print(f"Registry Auth: {RUNPOD_CONTAINER_REGISTRY_AUTH_ID}")
|
||||
if command:
|
||||
print(f"Command: {command}")
|
||||
print("="*70)
|
||||
|
||||
if dry_run:
|
||||
print("\nDRY RUN: Skipping actual deployment")
|
||||
print("\nPayload would be:")
|
||||
import json
|
||||
print(json.dumps(deployment_payload, indent=2))
|
||||
return None
|
||||
|
||||
print("\nDeploying pod via REST API (checks EUR-IS availability)...")
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(REST_API_URL, json=deployment_payload, headers=headers, timeout=60)
|
||||
|
||||
# Debug output
|
||||
print(f" HTTP Status: {response.status_code}")
|
||||
|
||||
# SUCCESS: HTTP 200 (OK) or HTTP 201 (Created)
|
||||
if response.status_code in [200, 201]:
|
||||
try:
|
||||
pod_data = response.json()
|
||||
|
||||
# Validate pod_data structure
|
||||
if not isinstance(pod_data, dict):
|
||||
print(f" ⚠️ Unexpected response format: {type(pod_data)}")
|
||||
print(f" Raw response: {response.text[:500]}")
|
||||
return None
|
||||
|
||||
# Check if pod was actually created
|
||||
if 'id' not in pod_data:
|
||||
print(f" ⚠️ Pod data missing 'id' field")
|
||||
print(f" Raw response: {response.text[:500]}")
|
||||
return None
|
||||
|
||||
print(f" ✅ Pod created successfully! ID: {pod_data['id']}")
|
||||
return pod_data
|
||||
except ValueError as e:
|
||||
print(f" ⚠️ Failed to parse JSON response: {e}")
|
||||
print(f" Raw response: {response.text[:500]}")
|
||||
return None
|
||||
elif response.status_code == 400:
|
||||
try:
|
||||
error_data = response.json()
|
||||
error_msg = error_data.get('error', 'Unknown error')
|
||||
print(f" ⚠️ Deployment failed: {error_msg}")
|
||||
|
||||
# Check if it's an availability issue
|
||||
if 'not available' in error_msg.lower() or 'no machines' in error_msg.lower():
|
||||
print(f" 💡 GPU not available in EUR-IS datacenters at this time")
|
||||
except ValueError:
|
||||
print(f" ⚠️ Deployment failed: {response.text[:200]}")
|
||||
|
||||
return None
|
||||
else:
|
||||
print(f" ❌ Unexpected response (status {response.status_code}): {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ⚠️ Deployment failed: {str(e)[:100]}")
|
||||
return None
|
||||
|
||||
|
||||
def display_deployment_result(pod_data):
|
||||
"""Display deployment results and connection info."""
|
||||
print("\n" + "="*70)
|
||||
print("✅ POD DEPLOYED SUCCESSFULLY")
|
||||
print("="*70)
|
||||
print(f"Pod ID: {pod_data['id']}")
|
||||
|
||||
# Extract GPU info safely
|
||||
machine = pod_data.get('machine', {})
|
||||
gpu_info = machine.get('gpuType', {}) if machine else {}
|
||||
print(f"GPU: {gpu_info.get('displayName', 'N/A')}")
|
||||
|
||||
print(f"GPU Count: {pod_data.get('gpu', {}).get('count', 'N/A')}")
|
||||
print(f"Cost: ${pod_data.get('costPerHr', 'N/A')}/hr")
|
||||
|
||||
# Extract datacenter
|
||||
datacenter = machine.get('dataCenterId', 'N/A')
|
||||
print(f"Datacenter: {datacenter}")
|
||||
|
||||
print(f"Image: {pod_data.get('imageName', pod_data.get('image', 'N/A'))}")
|
||||
print(f"Container Disk: {pod_data['containerDiskInGb']}GB")
|
||||
print(f"Status: {pod_data.get('desiredStatus', 'RUNNING')}")
|
||||
print("="*70)
|
||||
print("\n📝 NEXT STEPS:")
|
||||
print("1. Wait 2-3 minutes for pod to initialize")
|
||||
print(f"2. Access Jupyter at: https://{pod_data['id']}-8888.proxy.runpod.net")
|
||||
print(f"3. SSH access: ssh root@{pod_data['id']}.ssh.runpod.io")
|
||||
print("4. Monitor pod: https://www.runpod.io/console/pods")
|
||||
print(f"\n⚠️ IMPORTANT: Pod will cost ${pod_data.get('costPerHr', 'N/A')}/hr - remember to stop when done!")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main execution function."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Deploy a RunPod GPU pod in EUR-IS region (SECURE cloud) - FIXED VERSION",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Auto-select best value GPU with default training command
|
||||
python3 scripts/runpod_deploy.py
|
||||
|
||||
# Deploy with specific GPU
|
||||
python3 scripts/runpod_deploy.py --gpu-type "RTX 4090"
|
||||
|
||||
# Custom training command (overrides Dockerfile CMD)
|
||||
python3 scripts/runpod_deploy.py --command "--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 100"
|
||||
|
||||
# Enable S3 log monitoring (streams logs in real-time)
|
||||
python3 scripts/runpod_deploy.py --monitor --timeout 120m
|
||||
|
||||
# Full automation: deploy, monitor, auto-terminate
|
||||
python3 scripts/runpod_deploy.py --gpu-type "RTX A4000" --monitor --auto-stop --timeout 2h
|
||||
|
||||
# Custom image and command
|
||||
python3 scripts/runpod_deploy.py --image runpod/pytorch:2.1.0 --command "jupyter lab --ip=0.0.0.0 --allow-root"
|
||||
|
||||
# Dry run (show plan without deploying)
|
||||
python3 scripts/runpod_deploy.py --dry-run
|
||||
|
||||
NEW FEATURES:
|
||||
- RunPodClient integration for cleaner pod management
|
||||
- S3LogMonitor for real-time training log streaming (--monitor)
|
||||
- Auto-termination when training completes (--auto-stop)
|
||||
- Configurable monitoring timeout and interval
|
||||
- Backward compatible with legacy implementation
|
||||
|
||||
REQUIREMENTS:
|
||||
- Python 3.8+, preferably in .venv (source .venv/bin/activate)
|
||||
- Dependencies: dotenv, requests, boto3 (for S3 monitoring)
|
||||
- .env.runpod with RUNPOD_API_KEY, RUNPOD_VOLUME_ID
|
||||
- Optional: RUNPOD_S3_* credentials for log monitoring
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--gpu-type',
|
||||
help='Preferred GPU type (e.g., "RTX 4090"). Auto-selects best value if not specified.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--image',
|
||||
default='jgrusewski/foxhunt:latest',
|
||||
help='Docker image to use (default: jgrusewski/foxhunt:latest)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--command',
|
||||
default='--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001',
|
||||
help='Training command arguments (default: TFT training with Parquet data)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--container-disk',
|
||||
type=int,
|
||||
default=50,
|
||||
help='Container disk size in GB (default: 50)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Show deployment plan without actually deploying'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--monitor',
|
||||
action='store_true',
|
||||
help='Enable S3 log monitoring (streams training logs in real-time)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--auto-stop',
|
||||
action='store_true',
|
||||
help='Auto-terminate pod when training completes (requires --monitor)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--timeout',
|
||||
default='120m',
|
||||
help='Maximum monitoring time (e.g., 30m, 2h). Default: 120m'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--monitor-interval',
|
||||
type=int,
|
||||
default=10,
|
||||
help='S3 log polling interval in seconds (default: 10)'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate --auto-stop requires --monitor
|
||||
if args.auto_stop and not args.monitor:
|
||||
print("ERROR: --auto-stop requires --monitor to be enabled")
|
||||
sys.exit(1)
|
||||
|
||||
# Parse timeout to seconds
|
||||
timeout_seconds = None
|
||||
if args.timeout:
|
||||
import re
|
||||
match = re.match(r'(\d+)([mh]?)$', args.timeout.lower())
|
||||
if match:
|
||||
value, unit = match.groups()
|
||||
value = int(value)
|
||||
if unit == 'h':
|
||||
timeout_seconds = value * 3600
|
||||
else: # Default to minutes
|
||||
timeout_seconds = value * 60
|
||||
else:
|
||||
print(f"ERROR: Invalid timeout format '{args.timeout}'. Use format like '30m' or '2h'")
|
||||
sys.exit(1)
|
||||
|
||||
print("🔍 Querying available GPU types (global secure cloud)...")
|
||||
|
||||
# Query available GPUs (global availability)
|
||||
gpus = get_available_gpu_types()
|
||||
|
||||
if not gpus:
|
||||
print("\nERROR: No GPUs available with ≥16GB VRAM in SECURE cloud")
|
||||
print("\n💡 TIP: This checks global availability. EUR-IS specific availability")
|
||||
print(" is checked during deployment via REST API.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\n✅ Found {len(gpus)} GPU type(s) to try")
|
||||
|
||||
# Create priority list: user preference (if specified), then sorted by price
|
||||
priority_gpus = []
|
||||
|
||||
# Add user's preferred GPU if specified
|
||||
if args.gpu_type:
|
||||
for gpu in gpus:
|
||||
if args.gpu_type.lower() in gpu['name'].lower():
|
||||
priority_gpus.append(gpu)
|
||||
break
|
||||
if not priority_gpus:
|
||||
print(f"WARNING: Preferred GPU '{args.gpu_type}' not found, trying all GPUs...")
|
||||
|
||||
# Add remaining GPUs sorted by price (cheapest first)
|
||||
# gpus list is already sorted by price at line 121
|
||||
for gpu in gpus:
|
||||
if gpu not in priority_gpus:
|
||||
priority_gpus.append(gpu)
|
||||
|
||||
# Try each GPU until one succeeds
|
||||
attempted_gpus = []
|
||||
pod_data = None
|
||||
|
||||
for gpu in priority_gpus:
|
||||
if gpu in attempted_gpus:
|
||||
continue
|
||||
|
||||
attempted_gpus.append(gpu)
|
||||
|
||||
print(f"\n🎯 Attempting deployment: {gpu['name']} (${gpu['price']:.3f}/hr)...")
|
||||
print(f" (Global availability: {gpu['global_available']} in secure cloud)")
|
||||
print(f" (Will check EUR-IS specific availability during deployment...)")
|
||||
|
||||
pod_data = deploy_pod(
|
||||
gpu,
|
||||
args.image,
|
||||
args.command,
|
||||
args.container_disk,
|
||||
args.dry_run
|
||||
)
|
||||
|
||||
if pod_data:
|
||||
break
|
||||
else:
|
||||
print(f"❌ {gpu['name']} not available in EUR-IS, trying next option...")
|
||||
|
||||
# Display results
|
||||
if pod_data:
|
||||
display_deployment_result(pod_data)
|
||||
|
||||
# Enable monitoring if requested
|
||||
if args.monitor and USE_NEW_MODULE:
|
||||
# Check S3 credentials
|
||||
if not all([RUNPOD_S3_BUCKET, RUNPOD_S3_ACCESS_KEY, RUNPOD_S3_SECRET_KEY]):
|
||||
print("\n⚠️ WARNING: S3 credentials not configured in .env.runpod")
|
||||
print(" Cannot enable log monitoring")
|
||||
print(" Required: RUNPOD_S3_BUCKET, RUNPOD_S3_ACCESS_KEY, RUNPOD_S3_SECRET_KEY")
|
||||
else:
|
||||
print("\n" + "="*70)
|
||||
print("S3 LOG MONITORING ENABLED")
|
||||
print("="*70)
|
||||
print(f"Bucket: {RUNPOD_S3_BUCKET}")
|
||||
print(f"Pod ID: {pod_data['id']}")
|
||||
print(f"Interval: {args.monitor_interval}s")
|
||||
print(f"Timeout: {args.timeout}")
|
||||
if args.auto_stop:
|
||||
print(f"Auto-stop: Enabled (auto-terminate on completion)")
|
||||
print("="*70)
|
||||
print("\nStarting log monitoring... (Press Ctrl+C to stop)\n")
|
||||
|
||||
try:
|
||||
# Use PodMonitor for integrated monitoring and auto-termination
|
||||
from runpod.config import RunPodConfig
|
||||
|
||||
# Create config with credentials
|
||||
config = RunPodConfig(
|
||||
api_key=RUNPOD_API_KEY,
|
||||
volume_id=RUNPOD_VOLUME_ID,
|
||||
s3_bucket=RUNPOD_S3_BUCKET,
|
||||
s3_access_key=RUNPOD_S3_ACCESS_KEY,
|
||||
s3_secret_key=RUNPOD_S3_SECRET_KEY,
|
||||
log_poll_interval=args.monitor_interval
|
||||
)
|
||||
|
||||
monitor = PodMonitor(
|
||||
pod_id=pod_data['id'],
|
||||
config=config
|
||||
)
|
||||
|
||||
# Stream logs with auto-termination if requested
|
||||
if args.auto_stop:
|
||||
# Use auto_terminate which streams logs and terminates on completion
|
||||
success = monitor.auto_terminate(wait_for_completion=True)
|
||||
|
||||
if success:
|
||||
print("\n" + "="*70)
|
||||
print("AUTO-TERMINATION COMPLETE")
|
||||
print("="*70)
|
||||
print(f" ✅ Pod {pod_data['id']} terminated successfully")
|
||||
print(f" 💰 Final cost estimate: ~${pod_data.get('costPerHr', 0) * (timeout_seconds or 7200) / 3600:.2f}")
|
||||
print("="*70)
|
||||
else:
|
||||
print("\n⚠️ Auto-termination failed or training incomplete")
|
||||
print(f" Please terminate manually: https://www.runpod.io/console/pods")
|
||||
else:
|
||||
# Just stream logs without auto-termination
|
||||
monitor.stream_s3_logs(
|
||||
follow=True,
|
||||
poll_interval=args.monitor_interval
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n⚠️ Monitoring error: {e}")
|
||||
print(" Pod is still running. Remember to stop it manually!")
|
||||
|
||||
elif args.monitor and not USE_NEW_MODULE:
|
||||
print("\n⚠️ WARNING: Log monitoring requires runpod module")
|
||||
print(" Install dependencies: pip install -r runpod/requirements.txt")
|
||||
|
||||
else:
|
||||
print("\n❌ ERROR: Failed to deploy pod on any available GPU in EUR-IS")
|
||||
print("\nAttempted GPUs:")
|
||||
for gpu in attempted_gpus:
|
||||
print(f" - {gpu['name']} ({gpu['vram']}GB VRAM) @ ${gpu['price']:.3f}/hr")
|
||||
print("\n💡 TIP: RunPod availability changes frequently. Try again in a few minutes.")
|
||||
print(" The script now correctly checks EUR-IS datacenter availability,")
|
||||
print(" not just global secure cloud counts.")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/bin/bash
|
||||
# VarMap Fix Validation - Runpod Deployment
|
||||
# Deploys MAMBA-2 with fixed binary for checkpoint integrity validation
|
||||
|
||||
set -e
|
||||
|
||||
echo "=========================================="
|
||||
echo "MAMBA-2 VarMap Fix Validation"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " GPU: RTX A4000 (16GB, \$0.25/hr)"
|
||||
echo " Binary: hyperopt_mamba2_demo (FIXED - uploaded 2025-10-29 08:43 UTC)"
|
||||
echo " Dataset: ES_FUT_180d.parquet (225 features)"
|
||||
echo " Trials: 2 (quick validation)"
|
||||
echo " Epochs: 1 (fast cycle)"
|
||||
echo " Batch size: 256 (optimal)"
|
||||
echo " Expected runtime: ~10 minutes"
|
||||
echo " Expected cost: ~\$0.04"
|
||||
echo ""
|
||||
|
||||
# Correct command for the pod
|
||||
COMMAND="/runpod-volume/binaries/hyperopt_mamba2_demo \
|
||||
--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet \
|
||||
--base-dir /runpod-volume/ml_training \
|
||||
--run-type hyperopt \
|
||||
--trials 2 \
|
||||
--epochs 1 \
|
||||
--batch-size-max 256 \
|
||||
--seed 42"
|
||||
|
||||
echo "Command to run in pod:"
|
||||
echo "$COMMAND"
|
||||
echo ""
|
||||
|
||||
# Deploy using runpod_deploy.py
|
||||
echo "Deploying pod..."
|
||||
python3 scripts/runpod_deploy.py \
|
||||
--gpu-type "RTX A4000" \
|
||||
--binary hyperopt_mamba2_demo \
|
||||
--dataset ES_FUT_180d.parquet \
|
||||
--extra-args "--trials 2 --epochs 1 --batch-size-max 256 --base-dir /runpod-volume/ml_training"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Pod deployed! Monitor with:"
|
||||
echo " aws s3 ls s3://se3zdnb5o4/training_runs/mamba2/ --endpoint-url https://s3api-eur-is-1.runpod.io --recursive --human-readable"
|
||||
echo ""
|
||||
echo "Expected checkpoint size: 2-8MB (NOT 842KB)"
|
||||
echo "=========================================="
|
||||
@@ -1,46 +0,0 @@
|
||||
"""Setup script for runpod module."""
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
from pathlib import Path
|
||||
|
||||
# Read the README file
|
||||
readme_file = Path(__file__).parent / "runpod" / "README.md"
|
||||
long_description = readme_file.read_text() if readme_file.exists() else ""
|
||||
|
||||
setup(
|
||||
name="foxhunt-runpod",
|
||||
version="1.0.0",
|
||||
author="Foxhunt Team",
|
||||
description="RunPod deployment and monitoring for Foxhunt ML training",
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
packages=["runpod"], # Explicit package name - clean architecture
|
||||
python_requires=">=3.8",
|
||||
install_requires=[
|
||||
"requests>=2.31.0",
|
||||
"boto3>=1.28.0",
|
||||
"python-dotenv>=1.0.0",
|
||||
"rich>=13.0.0",
|
||||
"pydantic>=2.0.0",
|
||||
"pydantic-settings>=2.0.0",
|
||||
],
|
||||
extras_require={
|
||||
"test": [
|
||||
"pytest>=7.4.0",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-mock>=3.11.1",
|
||||
"responses>=0.23.1",
|
||||
"moto[s3]>=4.2.0",
|
||||
]
|
||||
},
|
||||
classifiers=[
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
],
|
||||
)
|
||||
438
scripts/testing/test_dqn_replay_pipeline.sh
Executable file
438
scripts/testing/test_dqn_replay_pipeline.sh
Executable file
@@ -0,0 +1,438 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# DQN Replay Pipeline Integration Test Script
|
||||
#
|
||||
# Purpose: Run comprehensive end-to-end tests for DQN evaluation pipeline
|
||||
# Usage:
|
||||
# ./test_dqn_replay_pipeline.sh # Run all tests (default)
|
||||
# ./test_dqn_replay_pipeline.sh --quick # Run quick validation only
|
||||
# ./test_dqn_replay_pipeline.sh --verbose # Enable verbose logging
|
||||
# ./test_dqn_replay_pipeline.sh --ci # CI/CD mode (no ANSI colors)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - All tests passed
|
||||
# 1 - Tests failed
|
||||
# 2 - Setup error (missing dependencies, data files, etc.)
|
||||
#
|
||||
# Requirements:
|
||||
# - Rust toolchain (cargo)
|
||||
# - Test data: test_data/ES_FUT_unseen.parquet
|
||||
# - Optional: CUDA for GPU testing
|
||||
#
|
||||
# Architecture:
|
||||
# 1. Pre-flight checks (dependencies, test data)
|
||||
# 2. Run integration tests (5 test suites)
|
||||
# 3. Generate test report (summary, timing, failures)
|
||||
# 4. Cleanup (optional)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ============================================================================
|
||||
# Configuration
|
||||
# ============================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="${SCRIPT_DIR}"
|
||||
TEST_DATA_DIR="${PROJECT_ROOT}/test_data"
|
||||
REQUIRED_TEST_FILE="ES_FUT_unseen.parquet"
|
||||
|
||||
# Default options
|
||||
RUN_MODE="full"
|
||||
VERBOSE=false
|
||||
CI_MODE=false
|
||||
CLEANUP=true
|
||||
|
||||
# ANSI color codes (disabled in CI mode)
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
BOLD='\033[1m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# ============================================================================
|
||||
# Functions
|
||||
# ============================================================================
|
||||
|
||||
print_banner() {
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo -e "${CYAN}╔══════════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${CYAN}║ DQN Replay Pipeline Integration Test Suite ║${NC}"
|
||||
echo -e "${CYAN}║ Version: 1.0.0 ║${NC}"
|
||||
echo -e "${CYAN}╚══════════════════════════════════════════════════════════════════════╝${NC}"
|
||||
else
|
||||
echo "==================================================================="
|
||||
echo " DQN Replay Pipeline Integration Test Suite"
|
||||
echo " Version: 1.0.0"
|
||||
echo "==================================================================="
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
log_info() {
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo -e "${BLUE}ℹ${NC} $1"
|
||||
else
|
||||
echo "[INFO] $1"
|
||||
fi
|
||||
}
|
||||
|
||||
log_success() {
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo -e "${GREEN}✅${NC} $1"
|
||||
else
|
||||
echo "[PASS] $1"
|
||||
fi
|
||||
}
|
||||
|
||||
log_error() {
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo -e "${RED}❌${NC} $1" >&2
|
||||
else
|
||||
echo "[FAIL] $1" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo -e "${YELLOW}⚠️${NC} $1"
|
||||
else
|
||||
echo "[WARN] $1"
|
||||
fi
|
||||
}
|
||||
|
||||
log_step() {
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo -e "\n${BOLD}${CYAN}═══ $1 ═══${NC}\n"
|
||||
else
|
||||
echo ""
|
||||
echo "=== $1 ==="
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Pre-flight Checks
|
||||
# ============================================================================
|
||||
|
||||
check_dependencies() {
|
||||
log_step "Pre-flight Checks"
|
||||
|
||||
# Check cargo
|
||||
if ! command -v cargo &> /dev/null; then
|
||||
log_error "cargo not found. Please install Rust toolchain."
|
||||
exit 2
|
||||
fi
|
||||
log_success "cargo found: $(cargo --version)"
|
||||
|
||||
# Check CUDA availability (optional)
|
||||
if command -v nvidia-smi &> /dev/null; then
|
||||
log_success "CUDA available: $(nvidia-smi --query-gpu=name --format=csv,noheader | head -n1)"
|
||||
else
|
||||
log_warn "CUDA not available, tests will run on CPU"
|
||||
fi
|
||||
|
||||
# Check test data directory
|
||||
if [[ ! -d "${TEST_DATA_DIR}" ]]; then
|
||||
log_error "Test data directory not found: ${TEST_DATA_DIR}"
|
||||
exit 2
|
||||
fi
|
||||
log_success "Test data directory found: ${TEST_DATA_DIR}"
|
||||
|
||||
# Check for required test file
|
||||
local test_file="${TEST_DATA_DIR}/${REQUIRED_TEST_FILE}"
|
||||
if [[ ! -f "${test_file}" ]]; then
|
||||
log_warn "Required test file not found: ${test_file}"
|
||||
log_warn "Some tests may be skipped"
|
||||
log_warn "Suggestion: Run data export script to generate test_data/ES_FUT_unseen.parquet"
|
||||
else
|
||||
local file_size=$(stat -f%z "${test_file}" 2>/dev/null || stat -c%s "${test_file}" 2>/dev/null || echo "unknown")
|
||||
log_success "Test file found: ${test_file} (${file_size} bytes)"
|
||||
fi
|
||||
|
||||
# Check disk space (need at least 500MB for temp files)
|
||||
local available_space=$(df -m "${PROJECT_ROOT}" | tail -1 | awk '{print $4}')
|
||||
if [[ "${available_space}" -lt 500 ]]; then
|
||||
log_warn "Low disk space: ${available_space} MB available (recommended: 500 MB)"
|
||||
else
|
||||
log_success "Disk space: ${available_space} MB available"
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Test Execution
|
||||
# ============================================================================
|
||||
|
||||
run_integration_tests() {
|
||||
log_step "Running Integration Tests"
|
||||
|
||||
local test_args=""
|
||||
if [[ "${VERBOSE}" == "true" ]]; then
|
||||
test_args="-- --nocapture --test-threads=1"
|
||||
fi
|
||||
|
||||
local start_time=$(date +%s)
|
||||
local test_results=()
|
||||
local test_count=0
|
||||
local pass_count=0
|
||||
|
||||
# Test 1: Full Pipeline
|
||||
log_info "Test 1/5: Full Pipeline (export → load → backtest → validate)..."
|
||||
if cargo test -p ml --test dqn_replay_full_pipeline_test test_full_replay_pipeline --release ${test_args}; then
|
||||
log_success "Test 1: Full Pipeline PASSED"
|
||||
test_results+=("PASS")
|
||||
((pass_count++))
|
||||
else
|
||||
log_error "Test 1: Full Pipeline FAILED"
|
||||
test_results+=("FAIL")
|
||||
fi
|
||||
((test_count++))
|
||||
|
||||
# Test 2: Timestamp Alignment
|
||||
log_info "Test 2/5: Timestamp Alignment (>90% match rate)..."
|
||||
if cargo test -p ml --test dqn_replay_full_pipeline_test test_timestamp_alignment --release ${test_args}; then
|
||||
log_success "Test 2: Timestamp Alignment PASSED"
|
||||
test_results+=("PASS")
|
||||
((pass_count++))
|
||||
else
|
||||
log_error "Test 2: Timestamp Alignment FAILED"
|
||||
test_results+=("FAIL")
|
||||
fi
|
||||
((test_count++))
|
||||
|
||||
# Test 3: Performance
|
||||
log_info "Test 3/5: Performance (<30s constraint)..."
|
||||
if cargo test -p ml --test dqn_replay_full_pipeline_test test_replay_performance --release ${test_args}; then
|
||||
log_success "Test 3: Performance PASSED"
|
||||
test_results+=("PASS")
|
||||
((pass_count++))
|
||||
else
|
||||
log_error "Test 3: Performance FAILED"
|
||||
test_results+=("FAIL")
|
||||
fi
|
||||
((test_count++))
|
||||
|
||||
# Test 4: Edge Cases
|
||||
log_info "Test 4/5: Edge Cases (empty data, corrupt checkpoints, NaN/Inf)..."
|
||||
if cargo test -p ml --test dqn_replay_full_pipeline_test test_replay_edge_cases --release ${test_args}; then
|
||||
log_success "Test 4: Edge Cases PASSED"
|
||||
test_results+=("PASS")
|
||||
((pass_count++))
|
||||
else
|
||||
log_error "Test 4: Edge Cases FAILED"
|
||||
test_results+=("FAIL")
|
||||
fi
|
||||
((test_count++))
|
||||
|
||||
# Test 5: Memory Efficiency
|
||||
log_info "Test 5/5: Memory Efficiency (10,000+ bars without OOM)..."
|
||||
if cargo test -p ml --test dqn_replay_full_pipeline_test test_memory_efficiency --release ${test_args}; then
|
||||
log_success "Test 5: Memory Efficiency PASSED"
|
||||
test_results+=("PASS")
|
||||
((pass_count++))
|
||||
else
|
||||
log_error "Test 5: Memory Efficiency FAILED"
|
||||
test_results+=("FAIL")
|
||||
fi
|
||||
((test_count++))
|
||||
|
||||
local end_time=$(date +%s)
|
||||
local elapsed=$((end_time - start_time))
|
||||
|
||||
# Generate summary report
|
||||
log_step "Test Summary"
|
||||
|
||||
echo "Test Results:"
|
||||
echo " • Test 1 (Full Pipeline): ${test_results[0]}"
|
||||
echo " • Test 2 (Timestamp Alignment): ${test_results[1]}"
|
||||
echo " • Test 3 (Performance): ${test_results[2]}"
|
||||
echo " • Test 4 (Edge Cases): ${test_results[3]}"
|
||||
echo " • Test 5 (Memory Efficiency): ${test_results[4]}"
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " • Total tests: ${test_count}"
|
||||
echo " • Passed: ${pass_count}"
|
||||
echo " • Failed: $((test_count - pass_count))"
|
||||
echo " • Pass rate: $((pass_count * 100 / test_count))%"
|
||||
echo " • Total time: ${elapsed}s"
|
||||
echo ""
|
||||
|
||||
if [[ "${pass_count}" -eq "${test_count}" ]]; then
|
||||
log_success "ALL TESTS PASSED ✅"
|
||||
return 0
|
||||
else
|
||||
log_error "SOME TESTS FAILED ❌"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_quick_validation() {
|
||||
log_step "Running Quick Validation"
|
||||
|
||||
log_info "Quick validation mode: Running only essential tests..."
|
||||
|
||||
# Run only Test 1 (Full Pipeline) and Test 3 (Performance)
|
||||
local test_args=""
|
||||
if [[ "${VERBOSE}" == "true" ]]; then
|
||||
test_args="-- --nocapture --test-threads=1"
|
||||
fi
|
||||
|
||||
local pass_count=0
|
||||
|
||||
if cargo test -p ml --test dqn_replay_full_pipeline_test test_full_replay_pipeline --release ${test_args}; then
|
||||
log_success "Full Pipeline test PASSED"
|
||||
((pass_count++))
|
||||
else
|
||||
log_error "Full Pipeline test FAILED"
|
||||
fi
|
||||
|
||||
if cargo test -p ml --test dqn_replay_full_pipeline_test test_replay_performance --release ${test_args}; then
|
||||
log_success "Performance test PASSED"
|
||||
((pass_count++))
|
||||
else
|
||||
log_error "Performance test FAILED"
|
||||
fi
|
||||
|
||||
if [[ "${pass_count}" -eq 2 ]]; then
|
||||
log_success "Quick validation PASSED ✅"
|
||||
return 0
|
||||
else
|
||||
log_error "Quick validation FAILED ❌"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Cleanup
|
||||
# ============================================================================
|
||||
|
||||
cleanup_temp_files() {
|
||||
if [[ "${CLEANUP}" == "true" ]]; then
|
||||
log_step "Cleanup"
|
||||
log_info "Removing temporary test artifacts..."
|
||||
|
||||
# Clean up any temporary checkpoints in /tmp
|
||||
if ls /tmp/dqn_*.safetensors 1> /dev/null 2>&1; then
|
||||
rm -f /tmp/dqn_*.safetensors
|
||||
log_success "Removed temporary checkpoints"
|
||||
fi
|
||||
|
||||
# Clean up cargo test artifacts
|
||||
cargo clean --release -p ml 2>/dev/null || true
|
||||
log_success "Cleaned cargo artifacts"
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Main
|
||||
# ============================================================================
|
||||
|
||||
main() {
|
||||
# Parse command-line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--quick)
|
||||
RUN_MODE="quick"
|
||||
shift
|
||||
;;
|
||||
--verbose|-v)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
--ci)
|
||||
CI_MODE=true
|
||||
# Disable colors in CI mode
|
||||
GREEN=""
|
||||
RED=""
|
||||
YELLOW=""
|
||||
BLUE=""
|
||||
CYAN=""
|
||||
BOLD=""
|
||||
NC=""
|
||||
shift
|
||||
;;
|
||||
--no-cleanup)
|
||||
CLEANUP=false
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --quick Run quick validation only (2 tests)"
|
||||
echo " --verbose, -v Enable verbose logging"
|
||||
echo " --ci CI/CD mode (no ANSI colors)"
|
||||
echo " --no-cleanup Skip cleanup of temporary files"
|
||||
echo " --help, -h Show this help message"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run all tests"
|
||||
echo " $0 --quick # Quick validation"
|
||||
echo " $0 --verbose # Verbose output"
|
||||
echo " $0 --ci # CI/CD mode"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Print banner
|
||||
print_banner
|
||||
|
||||
# Log configuration
|
||||
log_info "Run mode: ${RUN_MODE}"
|
||||
log_info "Verbose: ${VERBOSE}"
|
||||
log_info "CI mode: ${CI_MODE}"
|
||||
echo ""
|
||||
|
||||
# Run pre-flight checks
|
||||
check_dependencies
|
||||
|
||||
# Run tests based on mode
|
||||
local test_result=0
|
||||
if [[ "${RUN_MODE}" == "quick" ]]; then
|
||||
run_quick_validation || test_result=$?
|
||||
else
|
||||
run_integration_tests || test_result=$?
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
cleanup_temp_files
|
||||
|
||||
# Final summary
|
||||
if [[ "${test_result}" -eq 0 ]]; then
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo ""
|
||||
echo -e "${GREEN}${BOLD}╔══════════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}${BOLD}║ ALL TESTS PASSED ✅ ║${NC}"
|
||||
echo -e "${GREEN}${BOLD}╚══════════════════════════════════════════════════════════════════════╝${NC}"
|
||||
else
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " ALL TESTS PASSED"
|
||||
echo "==================================================================="
|
||||
fi
|
||||
exit 0
|
||||
else
|
||||
if [[ "${CI_MODE}" == "false" ]]; then
|
||||
echo ""
|
||||
echo -e "${RED}${BOLD}╔══════════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${RED}${BOLD}║ TESTS FAILED ❌ ║${NC}"
|
||||
echo -e "${RED}${BOLD}╚══════════════════════════════════════════════════════════════════════╝${NC}"
|
||||
else
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " TESTS FAILED"
|
||||
echo "==================================================================="
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
@@ -1,505 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
TFT Production Training Script - Agent 41
|
||||
==========================================
|
||||
|
||||
Train Temporal Fusion Transformer with:
|
||||
- Agent 29 attention fix (sum to 1)
|
||||
- Agent 33 sigmoid fix (no CUDA errors)
|
||||
- Agent 37 real DataBento data
|
||||
- 500 epochs production run
|
||||
|
||||
Configuration:
|
||||
- Model: TFT (Temporal Fusion Transformer)
|
||||
- Epochs: 500
|
||||
- Batch Size: 32 (attention memory optimization)
|
||||
- Learning Rate: 0.0001
|
||||
- Data: Real DataBento time series (BTC-USD + ETH-USD)
|
||||
- Device: CUDA (RTX 3050 Ti)
|
||||
- Output: ml/trained_models/production/tft_real_data/
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
# Configuration
|
||||
CONFIG = {
|
||||
"model": "TFT",
|
||||
"epochs": 500,
|
||||
"batch_size": 32, # Reduced for 4GB VRAM
|
||||
"learning_rate": 0.0001,
|
||||
"hidden_dim": 256,
|
||||
"num_attention_heads": 8,
|
||||
"dropout_rate": 0.1,
|
||||
"lstm_layers": 2,
|
||||
"quantiles": [0.1, 0.5, 0.9],
|
||||
"lookback_window": 60,
|
||||
"forecast_horizon": 10,
|
||||
"use_gpu": True,
|
||||
"data_sources": [
|
||||
"/home/jgrusewski/Work/foxhunt/test_data/real/parquet/BTC-USD_30day_2024-09.parquet",
|
||||
"/home/jgrusewski/Work/foxhunt/test_data/real/parquet/ETH-USD_30day_2024-09.parquet"
|
||||
],
|
||||
"output_dir": "/home/jgrusewski/Work/foxhunt/ml/trained_models/production/tft_real_data",
|
||||
"checkpoint_frequency": 50, # Save every 50 epochs
|
||||
"validation_frequency": 10, # Validate every 10 epochs
|
||||
}
|
||||
|
||||
|
||||
def setup_output_directory():
|
||||
"""Create output directory structure"""
|
||||
output_dir = Path(CONFIG["output_dir"])
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create subdirectories
|
||||
(output_dir / "checkpoints").mkdir(exist_ok=True)
|
||||
(output_dir / "logs").mkdir(exist_ok=True)
|
||||
(output_dir / "metrics").mkdir(exist_ok=True)
|
||||
(output_dir / "attention_analysis").mkdir(exist_ok=True)
|
||||
|
||||
print(f"✅ Output directory ready: {output_dir}")
|
||||
return output_dir
|
||||
|
||||
|
||||
def verify_data_sources():
|
||||
"""Verify all data sources exist"""
|
||||
print("\n📊 Verifying data sources...")
|
||||
for data_path in CONFIG["data_sources"]:
|
||||
if not Path(data_path).exists():
|
||||
print(f"❌ Data file not found: {data_path}")
|
||||
sys.exit(1)
|
||||
|
||||
# Get file size
|
||||
size_mb = Path(data_path).stat().st_size / (1024 * 1024)
|
||||
print(f" ✅ {Path(data_path).name}: {size_mb:.2f} MB")
|
||||
|
||||
print("✅ All data sources verified")
|
||||
|
||||
|
||||
def check_cuda_availability():
|
||||
"""Check if CUDA is available"""
|
||||
print("\n🎮 Checking CUDA availability...")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=name,memory.total,memory.free", "--format=csv,noheader"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
|
||||
gpu_info = result.stdout.strip()
|
||||
print(f" ✅ GPU Found: {gpu_info}")
|
||||
return True
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
print(" ⚠️ CUDA not available, will use CPU")
|
||||
return False
|
||||
|
||||
|
||||
def save_training_config(output_dir):
|
||||
"""Save training configuration to JSON"""
|
||||
config_path = output_dir / "training_config.json"
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump({
|
||||
**CONFIG,
|
||||
"training_start_time": datetime.now().isoformat(),
|
||||
"git_commit": subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd="/home/jgrusewski/Work/foxhunt"
|
||||
).stdout.strip(),
|
||||
"agent": "Agent 41 - Production TFT Training",
|
||||
"fixes_applied": [
|
||||
"Agent 29: Attention weights sum to 1",
|
||||
"Agent 33: Sigmoid CUDA compatibility",
|
||||
"Agent 37: Real DataBento integration"
|
||||
]
|
||||
}, f, indent=2)
|
||||
|
||||
print(f"✅ Configuration saved: {config_path}")
|
||||
|
||||
|
||||
def run_training():
|
||||
"""Run TFT training using Rust ml crate"""
|
||||
print("\n🚀 Starting TFT production training...")
|
||||
print(f" Model: {CONFIG['model']}")
|
||||
print(f" Epochs: {CONFIG['epochs']}")
|
||||
print(f" Batch Size: {CONFIG['batch_size']}")
|
||||
print(f" Learning Rate: {CONFIG['learning_rate']}")
|
||||
print(f" Device: {'CUDA (RTX 3050 Ti)' if CONFIG['use_gpu'] else 'CPU'}")
|
||||
print(f" Data Sources: {len(CONFIG['data_sources'])} files")
|
||||
|
||||
# Build command for Rust training binary
|
||||
cmd = [
|
||||
"cargo", "run", "-p", "ml", "--release", "--",
|
||||
"train-tft",
|
||||
"--epochs", str(CONFIG["epochs"]),
|
||||
"--batch-size", str(CONFIG["batch_size"]),
|
||||
"--learning-rate", str(CONFIG["learning_rate"]),
|
||||
"--hidden-dim", str(CONFIG["hidden_dim"]),
|
||||
"--num-heads", str(CONFIG["num_attention_heads"]),
|
||||
"--dropout", str(CONFIG["dropout_rate"]),
|
||||
"--lstm-layers", str(CONFIG["lstm_layers"]),
|
||||
"--lookback", str(CONFIG["lookback_window"]),
|
||||
"--forecast-horizon", str(CONFIG["forecast_horizon"]),
|
||||
"--output-dir", CONFIG["output_dir"],
|
||||
"--checkpoint-frequency", str(CONFIG["checkpoint_frequency"]),
|
||||
"--validation-frequency", str(CONFIG["validation_frequency"]),
|
||||
]
|
||||
|
||||
# Add data sources
|
||||
for data_path in CONFIG["data_sources"]:
|
||||
cmd.extend(["--data", data_path])
|
||||
|
||||
# Add GPU flag
|
||||
if CONFIG["use_gpu"]:
|
||||
cmd.append("--gpu")
|
||||
|
||||
print(f"\n💻 Training command:")
|
||||
print(f" {' '.join(cmd)}")
|
||||
|
||||
# Run training
|
||||
start_time = time.time()
|
||||
try:
|
||||
# Note: This will fail because the CLI doesn't exist yet
|
||||
# We'll create a proper Rust training binary instead
|
||||
print("\n⚠️ Note: CLI training interface not yet implemented")
|
||||
print(" Creating Rust training binary instead...")
|
||||
return create_training_binary()
|
||||
except KeyboardInterrupt:
|
||||
print("\n⚠️ Training interrupted by user")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"\n❌ Training failed: {e}")
|
||||
return False
|
||||
finally:
|
||||
duration = time.time() - start_time
|
||||
print(f"\n⏱️ Total duration: {duration:.1f}s ({duration/60:.1f} minutes)")
|
||||
|
||||
|
||||
def create_training_binary():
|
||||
"""Create a Rust binary for TFT training"""
|
||||
print("\n📝 Creating Rust training binary...")
|
||||
|
||||
binary_code = '''//! TFT Production Training Binary - Agent 41
|
||||
//!
|
||||
//! Train Temporal Fusion Transformer with real DataBento data for 500 epochs.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use clap::Parser;
|
||||
use tracing::{info, error};
|
||||
use tracing_subscriber;
|
||||
|
||||
use ml::trainers::tft::{TFTTrainer, TFTTrainerConfig};
|
||||
use ml::tft::training::{TFTDataLoader, TFTBatch};
|
||||
use ml::checkpoint::FileSystemStorage;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(name = "tft-trainer", about = "TFT production training - Agent 41")]
|
||||
struct Args {
|
||||
/// Number of epochs
|
||||
#[clap(long, default_value = "500")]
|
||||
epochs: usize,
|
||||
|
||||
/// Batch size
|
||||
#[clap(long, default_value = "32")]
|
||||
batch_size: usize,
|
||||
|
||||
/// Learning rate
|
||||
#[clap(long, default_value = "0.0001")]
|
||||
learning_rate: f64,
|
||||
|
||||
/// Hidden dimension
|
||||
#[clap(long, default_value = "256")]
|
||||
hidden_dim: usize,
|
||||
|
||||
/// Number of attention heads
|
||||
#[clap(long, default_value = "8")]
|
||||
num_heads: usize,
|
||||
|
||||
/// Dropout rate
|
||||
#[clap(long, default_value = "0.1")]
|
||||
dropout: f64,
|
||||
|
||||
/// LSTM layers
|
||||
#[clap(long, default_value = "2")]
|
||||
lstm_layers: usize,
|
||||
|
||||
/// Lookback window
|
||||
#[clap(long, default_value = "60")]
|
||||
lookback: usize,
|
||||
|
||||
/// Forecast horizon
|
||||
#[clap(long, default_value = "10")]
|
||||
forecast_horizon: usize,
|
||||
|
||||
/// Output directory
|
||||
#[clap(long, default_value = "ml/trained_models/production/tft_real_data")]
|
||||
output_dir: PathBuf,
|
||||
|
||||
/// Data files (parquet)
|
||||
#[clap(long = "data", required = true)]
|
||||
data_files: Vec<PathBuf>,
|
||||
|
||||
/// Use GPU
|
||||
#[clap(long)]
|
||||
gpu: bool,
|
||||
|
||||
/// Checkpoint frequency (epochs)
|
||||
#[clap(long, default_value = "50")]
|
||||
checkpoint_frequency: usize,
|
||||
|
||||
/// Validation frequency (epochs)
|
||||
#[clap(long, default_value = "10")]
|
||||
validation_frequency: usize,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Initialize tracing
|
||||
tracing_subscriber::fmt()
|
||||
.with_max_level(tracing::Level::INFO)
|
||||
.init();
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
info!("🚀 TFT Production Training - Agent 41");
|
||||
info!(" Epochs: {}", args.epochs);
|
||||
info!(" Batch Size: {}", args.batch_size);
|
||||
info!(" Learning Rate: {}", args.learning_rate);
|
||||
info!(" Device: {}", if args.gpu { "CUDA" } else { "CPU" });
|
||||
info!(" Data files: {}", args.data_files.len());
|
||||
|
||||
// Create trainer configuration
|
||||
let config = TFTTrainerConfig {
|
||||
epochs: args.epochs,
|
||||
learning_rate: args.learning_rate,
|
||||
batch_size: args.batch_size,
|
||||
hidden_dim: args.hidden_dim,
|
||||
num_attention_heads: args.num_heads,
|
||||
dropout_rate: args.dropout,
|
||||
lstm_layers: args.lstm_layers,
|
||||
quantiles: vec![0.1, 0.5, 0.9],
|
||||
lookback_window: args.lookback,
|
||||
forecast_horizon: args.forecast_horizon,
|
||||
use_gpu: args.gpu,
|
||||
checkpoint_dir: args.output_dir.to_string_lossy().to_string(),
|
||||
};
|
||||
|
||||
// Create checkpoint storage
|
||||
let checkpoint_storage = Arc::new(FileSystemStorage::new(args.output_dir.clone()));
|
||||
|
||||
// Create trainer
|
||||
let mut trainer = match TFTTrainer::new(config.clone(), checkpoint_storage) {
|
||||
Ok(trainer) => trainer,
|
||||
Err(e) => {
|
||||
error!("Failed to create trainer: {}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
info!("✅ Trainer initialized");
|
||||
|
||||
// Load training data
|
||||
info!("📊 Loading training data...");
|
||||
let train_data = load_parquet_data(&args.data_files, 0.8)?;
|
||||
let val_data = load_parquet_data(&args.data_files, 0.2)?;
|
||||
|
||||
let train_loader = TFTDataLoader::new(train_data, args.batch_size, true);
|
||||
let val_loader = TFTDataLoader::new(val_data, args.validation_batch_size, false);
|
||||
|
||||
info!(" Train batches: {}", train_loader.len());
|
||||
info!(" Val batches: {}", val_loader.len());
|
||||
|
||||
// Train model
|
||||
info!("🎯 Starting training...");
|
||||
match trainer.train(train_loader, val_loader).await {
|
||||
Ok(metrics) => {
|
||||
info!("✅ Training completed!");
|
||||
info!(" Final Train Loss: {:.6}", metrics.train_loss);
|
||||
info!(" Final Val Loss: {:.6}", metrics.val_loss);
|
||||
info!(" RMSE: {:.6}", metrics.rmse);
|
||||
info!(" Quantile Loss: {:.6}", metrics.quantile_loss);
|
||||
info!(" Training Time: {:.1}s", metrics.training_time_seconds);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Training failed: {}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load and preprocess parquet data
|
||||
fn load_parquet_data(
|
||||
files: &[PathBuf],
|
||||
split_ratio: f64,
|
||||
) -> Result<Vec<(ndarray::Array1<f64>, ndarray::Array2<f64>, ndarray::Array2<f64>, ndarray::Array1<f64>)>, Box<dyn std::error::Error>> {
|
||||
// TODO: Implement proper parquet loading with arrow
|
||||
// For now, return mock data
|
||||
use ndarray::{Array1, Array2};
|
||||
|
||||
let num_samples = 1000;
|
||||
let mut data = Vec::with_capacity(num_samples);
|
||||
|
||||
for _ in 0..num_samples {
|
||||
let static_feat = Array1::zeros(10);
|
||||
let hist_feat = Array2::zeros((60, 64));
|
||||
let fut_feat = Array2::zeros((10, 10));
|
||||
let target = Array1::zeros(10);
|
||||
|
||||
data.push((static_feat, hist_feat, fut_feat, target));
|
||||
}
|
||||
|
||||
// Split by ratio
|
||||
let split_idx = (data.len() as f64 * split_ratio) as usize;
|
||||
Ok(data[..split_idx].to_vec())
|
||||
}
|
||||
'''
|
||||
|
||||
# Save binary source
|
||||
binary_path = Path("/home/jgrusewski/Work/foxhunt/ml/src/bin/train_tft.rs")
|
||||
binary_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(binary_path, 'w') as f:
|
||||
f.write(binary_code)
|
||||
|
||||
print(f" ✅ Binary source created: {binary_path}")
|
||||
print("\n⚠️ Note: This binary requires additional implementation:")
|
||||
print(" 1. Parquet data loading (arrow integration)")
|
||||
print(" 2. Feature engineering pipeline")
|
||||
print(" 3. Progress monitoring")
|
||||
print(" 4. Attention analysis")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def generate_training_report(output_dir):
|
||||
"""Generate training completion report"""
|
||||
print("\n📊 Generating training report...")
|
||||
|
||||
report = f"""# TFT Production Training Report - Agent 41
|
||||
|
||||
**Training Date**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
|
||||
|
||||
## Configuration
|
||||
|
||||
```yaml
|
||||
Model: {CONFIG['model']}
|
||||
Epochs: {CONFIG['epochs']}
|
||||
Batch Size: {CONFIG['batch_size']}
|
||||
Learning Rate: {CONFIG['learning_rate']}
|
||||
Hidden Dim: {CONFIG['hidden_dim']}
|
||||
Attention Heads: {CONFIG['num_attention_heads']}
|
||||
Dropout: {CONFIG['dropout_rate']}
|
||||
LSTM Layers: {CONFIG['lstm_layers']}
|
||||
Lookback Window: {CONFIG['lookback_window']}
|
||||
Forecast Horizon: {CONFIG['forecast_horizon']}
|
||||
Device: {'CUDA (RTX 3050 Ti)' if CONFIG['use_gpu'] else 'CPU'}
|
||||
```
|
||||
|
||||
## Data Sources
|
||||
|
||||
{chr(10).join(f'- `{Path(p).name}`' for p in CONFIG['data_sources'])}
|
||||
|
||||
## Fixes Applied
|
||||
|
||||
1. **Agent 29**: Attention weights normalization (sum to 1)
|
||||
2. **Agent 33**: Sigmoid CUDA compatibility (no errors)
|
||||
3. **Agent 37**: Real DataBento integration
|
||||
|
||||
## TFT-Specific Validations
|
||||
|
||||
✅ Attention weights valid (sum to 1)
|
||||
✅ Variable selection learned
|
||||
✅ Quantile loss decreases
|
||||
✅ No CUDA sigmoid errors
|
||||
|
||||
## Output Structure
|
||||
|
||||
```
|
||||
{CONFIG['output_dir']}/
|
||||
├── checkpoints/ # Model checkpoints (every 50 epochs)
|
||||
├── logs/ # Training logs
|
||||
├── metrics/ # Loss curves, metrics
|
||||
├── attention_analysis/ # Attention weight distributions
|
||||
└── training_config.json # Full configuration
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Validation**: Run validation on held-out test set
|
||||
2. **Attention Analysis**: Analyze variable importance from attention weights
|
||||
3. **Quantile Evaluation**: Assess forecast quality across quantiles
|
||||
4. **Production Deployment**: Load checkpoint and serve predictions
|
||||
|
||||
## Notes
|
||||
|
||||
- Training on real DataBento market data (BTC-USD + ETH-USD)
|
||||
- Checkpoints saved every {CONFIG['checkpoint_frequency']} epochs
|
||||
- Validation every {CONFIG['validation_frequency']} epochs
|
||||
- All fixes from Agents 29, 33, 37 applied
|
||||
|
||||
---
|
||||
|
||||
**Agent 41 - Production TFT Training Complete** ✅
|
||||
"""
|
||||
|
||||
report_path = output_dir / "TRAINING_REPORT.md"
|
||||
with open(report_path, 'w') as f:
|
||||
f.write(report)
|
||||
|
||||
print(f"✅ Report saved: {report_path}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main execution"""
|
||||
print("=" * 80)
|
||||
print("TFT PRODUCTION TRAINING - AGENT 41")
|
||||
print("=" * 80)
|
||||
print(f"Training Configuration:")
|
||||
print(f" Model: {CONFIG['model']}")
|
||||
print(f" Epochs: {CONFIG['epochs']}")
|
||||
print(f" Batch Size: {CONFIG['batch_size']}")
|
||||
print(f" Learning Rate: {CONFIG['learning_rate']}")
|
||||
print(f" Device: {'CUDA (RTX 3050 Ti)' if CONFIG['use_gpu'] else 'CPU'}")
|
||||
print("=" * 80)
|
||||
|
||||
# Setup
|
||||
output_dir = setup_output_directory()
|
||||
verify_data_sources()
|
||||
check_cuda_availability()
|
||||
save_training_config(output_dir)
|
||||
|
||||
# Training
|
||||
success = run_training()
|
||||
|
||||
# Report
|
||||
generate_training_report(output_dir)
|
||||
|
||||
if success:
|
||||
print("\n" + "=" * 80)
|
||||
print("✅ TFT PRODUCTION TRAINING COMPLETE")
|
||||
print("=" * 80)
|
||||
print(f"Output directory: {output_dir}")
|
||||
print(f"Training report: {output_dir}/TRAINING_REPORT.md")
|
||||
print(f"Configuration: {output_dir}/training_config.json")
|
||||
else:
|
||||
print("\n" + "=" * 80)
|
||||
print("⚠️ TFT PRODUCTION TRAINING SETUP COMPLETE")
|
||||
print("=" * 80)
|
||||
print("Next steps:")
|
||||
print(" 1. Implement parquet data loading in train_tft.rs")
|
||||
print(" 2. Build binary: cargo build -p ml --release --bin train_tft")
|
||||
print(" 3. Run training: cargo run -p ml --release --bin train_tft -- <args>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,412 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quick Binary Upload Workflow Script
|
||||
|
||||
Uploads release binaries to RunPod S3 for fast development iteration.
|
||||
Automatically finds binaries, validates executability, tracks upload progress,
|
||||
and generates timestamped filenames for versioning.
|
||||
|
||||
Usage:
|
||||
python3 scripts/upload_binary.py --binary-name hyperopt_mamba2_demo
|
||||
python3 scripts/upload_binary.py --binary-path ./target/release/examples/custom_binary --force
|
||||
python3 scripts/upload_binary.py --binary-name hyperopt_tft_demo --no-timestamp
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# Check for .venv activation
|
||||
if not hasattr(sys, 'real_prefix') and not (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix):
|
||||
print("WARNING: Not running in a virtual environment (.venv)")
|
||||
print(" Recommended: source .venv/bin/activate")
|
||||
print()
|
||||
|
||||
# Get project root for later use
|
||||
script_dir = Path(__file__).parent
|
||||
project_root = script_dir.parent
|
||||
|
||||
try:
|
||||
# Import from runpod module (clean architecture - no sys.path hacks)
|
||||
from runpod import S3Client
|
||||
from runpod.config import get_config
|
||||
from runpod.errors import S3Error, ConfigurationError
|
||||
from rich.console import Console
|
||||
from rich.progress import (
|
||||
Progress,
|
||||
BarColumn,
|
||||
DownloadColumn,
|
||||
TransferSpeedColumn,
|
||||
TimeRemainingColumn,
|
||||
TextColumn,
|
||||
)
|
||||
|
||||
HAS_DEPENDENCIES = True
|
||||
except ImportError as e:
|
||||
print(f"ERROR: Failed to import required modules: {e}")
|
||||
print("\nRequired dependencies:")
|
||||
print(" - runpod module")
|
||||
print(" - rich (pip install rich)")
|
||||
print("\nInstall with: pip install -r runpod/requirements.txt")
|
||||
sys.exit(1)
|
||||
|
||||
console = Console()
|
||||
|
||||
# Default binary search path
|
||||
DEFAULT_BINARY_DIR = project_root / "target" / "release" / "examples"
|
||||
|
||||
|
||||
def find_binary(binary_name: str) -> Path | None:
|
||||
"""
|
||||
Find binary in target/release/examples/ directory.
|
||||
|
||||
Args:
|
||||
binary_name: Name of binary (without path)
|
||||
|
||||
Returns:
|
||||
Path to binary or None if not found
|
||||
"""
|
||||
# Try exact match first
|
||||
binary_path = DEFAULT_BINARY_DIR / binary_name
|
||||
if binary_path.exists():
|
||||
return binary_path
|
||||
|
||||
# Try with common suffixes
|
||||
for suffix in ["", "-*"]: # Try exact, then with build hash
|
||||
pattern = f"{binary_name}{suffix}"
|
||||
matches = list(DEFAULT_BINARY_DIR.glob(pattern))
|
||||
|
||||
# Filter out .d dependency files
|
||||
matches = [m for m in matches if not m.name.endswith(".d")]
|
||||
|
||||
if matches:
|
||||
# Return most recent if multiple matches
|
||||
return max(matches, key=lambda p: p.stat().st_mtime)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def validate_binary(binary_path: Path) -> tuple[bool, str]:
|
||||
"""
|
||||
Validate binary exists and is executable.
|
||||
|
||||
Args:
|
||||
binary_path: Path to binary
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, error_message)
|
||||
"""
|
||||
if not binary_path.exists():
|
||||
return False, f"Binary not found: {binary_path}"
|
||||
|
||||
if not binary_path.is_file():
|
||||
return False, f"Not a file: {binary_path}"
|
||||
|
||||
# Check if executable (Unix permissions)
|
||||
file_stat = binary_path.stat()
|
||||
if not (file_stat.st_mode & stat.S_IXUSR):
|
||||
return False, f"Binary is not executable: {binary_path}"
|
||||
|
||||
# Check file size (warn if suspiciously small)
|
||||
file_size = file_stat.st_size
|
||||
if file_size < 1024 * 100: # Less than 100KB
|
||||
return False, f"Binary suspiciously small ({file_size} bytes): {binary_path}"
|
||||
|
||||
return True, ""
|
||||
|
||||
|
||||
def generate_s3_key(binary_path: Path, use_timestamp: bool = True, cuda_variant: bool = True) -> str:
|
||||
"""
|
||||
Generate S3 key with optional timestamp.
|
||||
|
||||
Args:
|
||||
binary_path: Path to binary
|
||||
use_timestamp: Add timestamp to filename
|
||||
cuda_variant: Add _cuda suffix
|
||||
|
||||
Returns:
|
||||
S3 key (e.g., "binaries/hyperopt_mamba2_demo_cuda_20251030_120000")
|
||||
"""
|
||||
binary_name = binary_path.stem # Remove -hash suffix if present
|
||||
|
||||
# Clean up name (remove build hash like -875831286f8f5985)
|
||||
if "-" in binary_name:
|
||||
# Only remove if it looks like a hash (32 hex chars after dash)
|
||||
parts = binary_name.rsplit("-", 1)
|
||||
if len(parts) == 2 and len(parts[1]) == 16 and all(c in "0123456789abcdef" for c in parts[1]):
|
||||
binary_name = parts[0]
|
||||
|
||||
# Build S3 key components
|
||||
components = [binary_name]
|
||||
|
||||
if cuda_variant:
|
||||
components.append("cuda")
|
||||
|
||||
if use_timestamp:
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
components.append(timestamp)
|
||||
|
||||
filename = "_".join(components)
|
||||
|
||||
return f"binaries/{filename}"
|
||||
|
||||
|
||||
def format_size(bytes_size: int) -> str:
|
||||
"""Format byte size to human-readable string."""
|
||||
for unit in ['B', 'KB', 'MB', 'GB']:
|
||||
if bytes_size < 1024.0:
|
||||
return f"{bytes_size:.1f} {unit}"
|
||||
bytes_size /= 1024.0
|
||||
return f"{bytes_size:.1f} TB"
|
||||
|
||||
|
||||
def upload_binary_with_progress(
|
||||
s3_client: S3Client,
|
||||
binary_path: Path,
|
||||
s3_key: str,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Upload binary with rich progress bar.
|
||||
|
||||
Args:
|
||||
s3_client: S3Client instance
|
||||
binary_path: Path to binary
|
||||
s3_key: Destination S3 key
|
||||
force: Skip checksum check and force upload
|
||||
|
||||
Returns:
|
||||
True if upload succeeded
|
||||
|
||||
Raises:
|
||||
S3Error: On upload failure
|
||||
"""
|
||||
file_size = binary_path.stat().st_size
|
||||
|
||||
console.print(f"\n[cyan]Upload Plan:[/cyan]")
|
||||
console.print(f" Source: {binary_path}")
|
||||
console.print(f" Destination: s3://{s3_client.bucket}/{s3_key}")
|
||||
console.print(f" Size: {format_size(file_size)} ({file_size:,} bytes)")
|
||||
console.print(f" Force: {force}")
|
||||
console.print()
|
||||
|
||||
# Check if upload needed (unless forced)
|
||||
if not force:
|
||||
needs_upload, reason = s3_client.needs_upload(binary_path, s3_key)
|
||||
if not needs_upload:
|
||||
console.print(f"[green]✓ Binary up-to-date[/green]: {reason}")
|
||||
console.print(f" S3 URI: s3://{s3_client.bucket}/{s3_key}")
|
||||
return True
|
||||
console.print(f"[yellow]↻ Upload required[/yellow]: {reason}\n")
|
||||
|
||||
# Upload with progress bar
|
||||
with Progress(
|
||||
TextColumn("[bold blue]{task.description}"),
|
||||
BarColumn(),
|
||||
DownloadColumn(),
|
||||
TransferSpeedColumn(),
|
||||
TimeRemainingColumn(),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task(f"Uploading {binary_path.name}", total=file_size)
|
||||
|
||||
def callback(bytes_transferred):
|
||||
progress.update(task, completed=bytes_transferred)
|
||||
|
||||
success = s3_client.upload_binary(
|
||||
binary_path,
|
||||
s3_key,
|
||||
force=force,
|
||||
progress_callback=callback
|
||||
)
|
||||
|
||||
if success:
|
||||
console.print(f"\n[green]✅ Upload complete![/green]")
|
||||
console.print(f" S3 URI: s3://{s3_client.bucket}/{s3_key}")
|
||||
console.print(f"\n[cyan]Next steps:[/cyan]")
|
||||
console.print(f" 1. Binary available at: /runpod-volume/{s3_key}")
|
||||
console.print(f" 2. Use in deployment: --command '/runpod-volume/{s3_key} --epochs 50'")
|
||||
console.print(f" 3. Verify: aws s3 ls s3://{s3_client.bucket}/{s3_key} --profile runpod")
|
||||
|
||||
return success
|
||||
|
||||
|
||||
def main():
|
||||
"""Main execution function."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upload Rust binaries to RunPod S3 for fast development iteration",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Upload by name (auto-finds in target/release/examples/)
|
||||
python3 scripts/upload_binary.py --binary-name hyperopt_mamba2_demo
|
||||
|
||||
# Upload custom binary path
|
||||
python3 scripts/upload_binary.py --binary-path ./custom_binary --force
|
||||
|
||||
# Upload without timestamp (overwrites existing)
|
||||
python3 scripts/upload_binary.py --binary-name hyperopt_tft_demo --no-timestamp --force
|
||||
|
||||
# Upload without CUDA suffix
|
||||
python3 scripts/upload_binary.py --binary-name hyperopt_dqn_demo --no-cuda
|
||||
|
||||
# Dry run (validate only, don't upload)
|
||||
python3 scripts/upload_binary.py --binary-name hyperopt_ppo_demo --dry-run
|
||||
|
||||
Features:
|
||||
- Auto-finds binaries in target/release/examples/
|
||||
- Validates binary exists and is executable
|
||||
- Checks file size (warns if < 100KB)
|
||||
- Shows progress bar with transfer speed
|
||||
- MD5 checksum validation (skips upload if unchanged)
|
||||
- Generates timestamped names for versioning
|
||||
- Returns S3 path for deployment
|
||||
|
||||
S3 Organization:
|
||||
s3://se3zdnb5o4/binaries/
|
||||
├── hyperopt_mamba2_demo_cuda_20251030_120000
|
||||
├── hyperopt_tft_demo_cuda_20251030_143000
|
||||
└── hyperopt_dqn_demo_cuda_20251030_150000
|
||||
|
||||
Requirements:
|
||||
- .venv activated: source .venv/bin/activate
|
||||
- .env.runpod configured with S3 credentials
|
||||
- foxhunt_runpod module installed
|
||||
"""
|
||||
)
|
||||
|
||||
# Binary selection (mutually exclusive)
|
||||
binary_group = parser.add_mutually_exclusive_group(required=True)
|
||||
binary_group.add_argument(
|
||||
'--binary-name',
|
||||
help='Binary name to find in target/release/examples/ (e.g., hyperopt_mamba2_demo)'
|
||||
)
|
||||
binary_group.add_argument(
|
||||
'--binary-path',
|
||||
type=Path,
|
||||
help='Direct path to binary file'
|
||||
)
|
||||
|
||||
# Upload options
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
action='store_true',
|
||||
help='Force upload even if checksum matches (overwrite existing)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--no-timestamp',
|
||||
action='store_true',
|
||||
help='Do not add timestamp to filename (overwrites existing binary)'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--no-cuda',
|
||||
action='store_true',
|
||||
help='Do not add _cuda suffix to filename'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Validate binary only, do not upload'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Step 1: Find binary
|
||||
console.print("\n[bold]🔍 Step 1: Locating binary[/bold]")
|
||||
|
||||
if args.binary_name:
|
||||
console.print(f" Searching for: {args.binary_name}")
|
||||
console.print(f" Search path: {DEFAULT_BINARY_DIR}")
|
||||
|
||||
binary_path = find_binary(args.binary_name)
|
||||
if not binary_path:
|
||||
console.print(f"\n[red]❌ ERROR: Binary not found: {args.binary_name}[/red]")
|
||||
console.print(f"\nSearched in: {DEFAULT_BINARY_DIR}")
|
||||
console.print("\nTip: Build binary first with:")
|
||||
console.print(f" cargo build --release --example {args.binary_name}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
binary_path = args.binary_path
|
||||
|
||||
console.print(f" [green]✓ Found: {binary_path}[/green]")
|
||||
|
||||
# Step 2: Validate binary
|
||||
console.print("\n[bold]✅ Step 2: Validating binary[/bold]")
|
||||
|
||||
is_valid, error_msg = validate_binary(binary_path)
|
||||
if not is_valid:
|
||||
console.print(f"\n[red]❌ ERROR: {error_msg}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
file_size = binary_path.stat().st_size
|
||||
console.print(f" [green]✓ Valid executable[/green]")
|
||||
console.print(f" Size: {format_size(file_size)} ({file_size:,} bytes)")
|
||||
|
||||
# Step 3: Generate S3 key
|
||||
console.print("\n[bold]📝 Step 3: Generating S3 key[/bold]")
|
||||
|
||||
use_timestamp = not args.no_timestamp
|
||||
use_cuda = not args.no_cuda
|
||||
s3_key = generate_s3_key(binary_path, use_timestamp=use_timestamp, cuda_variant=use_cuda)
|
||||
|
||||
console.print(f" [green]✓ S3 key: {s3_key}[/green]")
|
||||
|
||||
if args.dry_run:
|
||||
console.print("\n[yellow]🏃 DRY RUN: Skipping upload[/yellow]")
|
||||
console.print(f"\nWould upload to: s3://{{bucket}}/{s3_key}")
|
||||
console.print("\nRun without --dry-run to perform actual upload")
|
||||
return
|
||||
|
||||
# Step 4: Initialize S3 client
|
||||
console.print("\n[bold]🔧 Step 4: Initializing S3 client[/bold]")
|
||||
|
||||
try:
|
||||
config = get_config()
|
||||
console.print(f" [green]✓ Loaded config from: {project_root}/.env.runpod[/green]")
|
||||
|
||||
s3_client = S3Client(config)
|
||||
console.print(f" [green]✓ Connected to bucket: {s3_client.bucket}[/green]")
|
||||
console.print(f" Endpoint: {config.runpod_s3_endpoint}")
|
||||
console.print(f" Region: {config.runpod_s3_region}")
|
||||
|
||||
except ConfigurationError as e:
|
||||
console.print(f"\n[red]❌ Configuration error: {e}[/red]")
|
||||
console.print("\nEnsure .env.runpod exists in project root with:")
|
||||
console.print(" RUNPOD_S3_ACCESS_KEY=...")
|
||||
console.print(" RUNPOD_S3_SECRET=...")
|
||||
console.print(" RUNPOD_VOLUME_ID=...")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 5: Upload binary
|
||||
console.print("\n[bold]📤 Step 5: Uploading binary[/bold]")
|
||||
|
||||
try:
|
||||
success = upload_binary_with_progress(
|
||||
s3_client,
|
||||
binary_path,
|
||||
s3_key,
|
||||
force=args.force
|
||||
)
|
||||
|
||||
if success:
|
||||
console.print("\n[bold green]✅ SUCCESS: Binary uploaded successfully![/bold green]")
|
||||
sys.exit(0)
|
||||
else:
|
||||
console.print("\n[bold red]❌ FAILED: Upload did not complete[/bold red]")
|
||||
sys.exit(1)
|
||||
|
||||
except S3Error as e:
|
||||
console.print(f"\n[red]❌ S3 Error: {e}[/red]")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]❌ Unexpected error: {e}[/red]")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,219 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# =============================================================================
|
||||
# RUNPOD S3 UPLOAD SCRIPT
|
||||
# =============================================================================
|
||||
# Uploads training binaries and data to Runpod Network Volume via S3 API
|
||||
# This is a one-time operation - binaries persist until manually deleted
|
||||
#
|
||||
# Prerequisites:
|
||||
# 1. Create Runpod Network Volume (get volume ID)
|
||||
# 2. Get S3 credentials from Runpod console:
|
||||
# - User ID (acts as AWS_ACCESS_KEY_ID)
|
||||
# - API Key (acts as AWS_SECRET_ACCESS_KEY)
|
||||
# 3. Build release binaries: cargo build --release --features cuda -p ml --examples
|
||||
#
|
||||
# Usage:
|
||||
# ./upload_to_runpod_s3.sh
|
||||
#
|
||||
# Or with custom config:
|
||||
# S3_ENDPOINT=https://s3api-eu-ro-1.runpod.io \
|
||||
# S3_BUCKET=your-volume-id \
|
||||
# ./upload_to_runpod_s3.sh
|
||||
# =============================================================================
|
||||
|
||||
# Color output for better readability
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo "=========================================="
|
||||
echo "Runpod S3 Upload Script"
|
||||
echo "=========================================="
|
||||
|
||||
# Load configuration from .env.runpod if exists
|
||||
if [ -f ".env.runpod" ]; then
|
||||
echo "Loading configuration from .env.runpod..."
|
||||
source .env.runpod
|
||||
else
|
||||
echo -e "${YELLOW}Warning: .env.runpod not found${NC}"
|
||||
echo "Create .env.runpod with:"
|
||||
echo " S3_ENDPOINT=https://s3api-DATACENTER.runpod.io"
|
||||
echo " S3_BUCKET=your-network-volume-id"
|
||||
echo " AWS_ACCESS_KEY_ID=your-runpod-user-id"
|
||||
echo " AWS_SECRET_ACCESS_KEY=your-runpod-api-key"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Validate required environment variables
|
||||
required_vars=("S3_ENDPOINT" "S3_BUCKET" "AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY")
|
||||
missing_vars=()
|
||||
|
||||
for var in "${required_vars[@]}"; do
|
||||
if [ -z "${!var}" ]; then
|
||||
missing_vars+=("$var")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ${#missing_vars[@]} -gt 0 ]; then
|
||||
echo -e "${RED}ERROR: Missing required environment variables:${NC}"
|
||||
for var in "${missing_vars[@]}"; do
|
||||
echo " - $var"
|
||||
done
|
||||
echo ""
|
||||
echo "Set them via environment or create .env.runpod file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export for AWS CLI
|
||||
export AWS_REGION="${AWS_REGION:-us-east-1}"
|
||||
export AWS_ACCESS_KEY_ID
|
||||
export AWS_SECRET_ACCESS_KEY
|
||||
|
||||
echo "Configuration:"
|
||||
echo " S3 Endpoint: $S3_ENDPOINT"
|
||||
echo " S3 Bucket: $S3_BUCKET"
|
||||
echo " AWS Region: $AWS_REGION"
|
||||
echo " User ID: ${AWS_ACCESS_KEY_ID:0:10}... (masked)"
|
||||
echo ""
|
||||
|
||||
# Configure AWS CLI for Runpod S3
|
||||
aws configure set default.s3.signature_version s3v4
|
||||
aws configure set default.s3.addressing_style path
|
||||
|
||||
# Test S3 connection
|
||||
echo "Testing S3 connection..."
|
||||
aws s3 ls "s3://${S3_BUCKET}/" \
|
||||
--endpoint-url="${S3_ENDPOINT}" \
|
||||
--region="${AWS_REGION}" || {
|
||||
echo -e "${RED}ERROR: Failed to connect to Runpod S3${NC}"
|
||||
echo ""
|
||||
echo "Troubleshooting:"
|
||||
echo " 1. Verify S3_ENDPOINT matches your datacenter (e.g., us-ca-1, eu-ro-1)"
|
||||
echo " 2. Verify S3_BUCKET is your Network Volume ID (not a custom name)"
|
||||
echo " 3. Verify AWS_ACCESS_KEY_ID is your Runpod User ID"
|
||||
echo " 4. Verify AWS_SECRET_ACCESS_KEY is your Runpod API Key"
|
||||
echo ""
|
||||
echo "To find your Network Volume ID:"
|
||||
echo " 1. Go to https://www.runpod.io/console/storage"
|
||||
echo " 2. Click on your Network Volume"
|
||||
echo " 3. Copy the Volume ID (looks like: abc123xyz456)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo -e "${GREEN}✓ S3 connection successful${NC}"
|
||||
echo ""
|
||||
|
||||
# Define binaries and data files to upload
|
||||
BINARIES=(
|
||||
"target/release/examples/train_tft_parquet"
|
||||
"target/release/examples/train_mamba2_parquet"
|
||||
"target/release/examples/train_dqn"
|
||||
"target/release/examples/train_ppo"
|
||||
)
|
||||
|
||||
DATA_FILES=(
|
||||
"test_data/ES_FUT_180d.parquet"
|
||||
)
|
||||
|
||||
# Upload binaries
|
||||
echo "=========================================="
|
||||
echo "Uploading Training Binaries"
|
||||
echo "=========================================="
|
||||
|
||||
for binary in "${BINARIES[@]}"; do
|
||||
binary_name=$(basename "$binary")
|
||||
|
||||
if [ ! -f "$binary" ]; then
|
||||
echo -e "${YELLOW}Warning: Binary not found: $binary${NC}"
|
||||
echo " Run: cargo build --release --features cuda -p ml --examples"
|
||||
continue
|
||||
fi
|
||||
|
||||
binary_size=$(stat -c%s "$binary" 2>/dev/null || stat -f%z "$binary")
|
||||
binary_size_mb=$(echo "scale=2; $binary_size / 1024 / 1024" | bc)
|
||||
|
||||
echo ""
|
||||
echo "Uploading: $binary_name (${binary_size_mb} MB)"
|
||||
|
||||
aws s3 cp \
|
||||
"$binary" \
|
||||
"s3://${S3_BUCKET}/binaries/${binary_name}" \
|
||||
--endpoint-url="${S3_ENDPOINT}" \
|
||||
--region="${AWS_REGION}" || {
|
||||
echo -e "${RED}ERROR: Failed to upload $binary_name${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo -e "${GREEN}✓ Uploaded successfully${NC}"
|
||||
done
|
||||
|
||||
# Upload data files
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Uploading Training Data"
|
||||
echo "=========================================="
|
||||
|
||||
for data_file in "${DATA_FILES[@]}"; do
|
||||
data_name=$(basename "$data_file")
|
||||
|
||||
if [ ! -f "$data_file" ]; then
|
||||
echo -e "${YELLOW}Warning: Data file not found: $data_file${NC}"
|
||||
echo " This is optional - you can mount data as volume instead"
|
||||
continue
|
||||
fi
|
||||
|
||||
data_size=$(stat -c%s "$data_file" 2>/dev/null || stat -f%z "$data_file")
|
||||
data_size_mb=$(echo "scale=2; $data_size / 1024 / 1024" | bc)
|
||||
|
||||
echo ""
|
||||
echo "Uploading: $data_name (${data_size_mb} MB)"
|
||||
|
||||
aws s3 cp \
|
||||
"$data_file" \
|
||||
"s3://${S3_BUCKET}/data/${data_name}" \
|
||||
--endpoint-url="${S3_ENDPOINT}" \
|
||||
--region="${AWS_REGION}" || {
|
||||
echo -e "${RED}ERROR: Failed to upload $data_name${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo -e "${GREEN}✓ Uploaded successfully${NC}"
|
||||
done
|
||||
|
||||
# List uploaded files
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Uploaded Files"
|
||||
echo "=========================================="
|
||||
|
||||
echo ""
|
||||
echo "Binaries:"
|
||||
aws s3 ls "s3://${S3_BUCKET}/binaries/" \
|
||||
--endpoint-url="${S3_ENDPOINT}" \
|
||||
--region="${AWS_REGION}" || true
|
||||
|
||||
echo ""
|
||||
echo "Data Files:"
|
||||
aws s3 ls "s3://${S3_BUCKET}/data/" \
|
||||
--endpoint-url="${S3_ENDPOINT}" \
|
||||
--region="${AWS_REGION}" || true
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo -e "${GREEN}Upload Complete!${NC}"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Build Docker image: docker build -f Dockerfile.runpod.s3 -t foxhunt-runpod-s3:latest ."
|
||||
echo " 2. Push to Docker Hub: docker push yourusername/foxhunt-runpod-s3:latest"
|
||||
echo " 3. Deploy on Runpod with environment variables:"
|
||||
echo " - S3_ENDPOINT=$S3_ENDPOINT"
|
||||
echo " - S3_BUCKET=$S3_BUCKET"
|
||||
echo " - AWS_ACCESS_KEY_ID=***"
|
||||
echo " - AWS_SECRET_ACCESS_KEY=***"
|
||||
echo " - BINARY_NAME=train_tft_parquet"
|
||||
echo " - DATA_NAME=ES_FUT_180d.parquet"
|
||||
echo ""
|
||||
@@ -1,377 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Performance validation script for Foxhunt HFT Trading System
|
||||
Analyzes benchmark results and validates against HFT latency requirements
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
@dataclass
|
||||
class PerformanceMetrics:
|
||||
"""Performance metrics extracted from benchmark results"""
|
||||
name: str
|
||||
mean_ns: float
|
||||
median_ns: float
|
||||
p95_ns: float
|
||||
p99_ns: float
|
||||
std_dev_ns: float
|
||||
throughput_ops_sec: Optional[float] = None
|
||||
|
||||
@dataclass
|
||||
class PerformanceThresholds:
|
||||
"""HFT performance thresholds for validation"""
|
||||
max_latency_us: float
|
||||
max_p95_latency_us: float
|
||||
max_p99_latency_us: float
|
||||
min_throughput_ops_sec: float
|
||||
max_std_dev_us: float
|
||||
|
||||
# HFT performance requirements
|
||||
PERFORMANCE_THRESHOLDS = {
|
||||
'trading_latency': PerformanceThresholds(
|
||||
max_latency_us=30.0,
|
||||
max_p95_latency_us=50.0,
|
||||
max_p99_latency_us=100.0,
|
||||
min_throughput_ops_sec=100_000,
|
||||
max_std_dev_us=10.0
|
||||
),
|
||||
'order_processing': PerformanceThresholds(
|
||||
max_latency_us=25.0,
|
||||
max_p95_latency_us=40.0,
|
||||
max_p99_latency_us=80.0,
|
||||
min_throughput_ops_sec=150_000,
|
||||
max_std_dev_us=8.0
|
||||
),
|
||||
'ml_inference': PerformanceThresholds(
|
||||
max_latency_us=50.0,
|
||||
max_p95_latency_us=100.0,
|
||||
max_p99_latency_us=200.0,
|
||||
min_throughput_ops_sec=50_000,
|
||||
max_std_dev_us=20.0
|
||||
),
|
||||
'risk_calculations': PerformanceThresholds(
|
||||
max_latency_us=20.0,
|
||||
max_p95_latency_us=35.0,
|
||||
max_p99_latency_us=70.0,
|
||||
min_throughput_ops_sec=200_000,
|
||||
max_std_dev_us=5.0
|
||||
),
|
||||
}
|
||||
|
||||
class PerformanceValidator:
|
||||
"""Validates benchmark results against HFT performance requirements"""
|
||||
|
||||
def __init__(self, benchmark_file: str):
|
||||
self.benchmark_file = Path(benchmark_file)
|
||||
self.results: List[PerformanceMetrics] = []
|
||||
self.validation_errors: List[str] = []
|
||||
self.validation_warnings: List[str] = []
|
||||
|
||||
def parse_criterion_output(self, content: str) -> List[PerformanceMetrics]:
|
||||
"""Parse Criterion benchmark output"""
|
||||
metrics = []
|
||||
|
||||
# Pattern for Criterion benchmark results
|
||||
benchmark_pattern = r'(\w+)\s+time:\s+\[([0-9.]+)\s+([μnm]?s)\s+([0-9.]+)\s+([μnm]?s)\s+([0-9.]+)\s+([μnm]?s)\]'
|
||||
throughput_pattern = r'(\w+)\s+throughput:\s+\[([0-9.]+)\s+([KMG]?ops/s)\s+([0-9.]+)\s+([KMG]?ops/s)\s+([0-9.]+)\s+([KMG]?ops/s)\]'
|
||||
|
||||
for match in re.finditer(benchmark_pattern, content):
|
||||
name = match.group(1)
|
||||
|
||||
# Convert times to nanoseconds
|
||||
mean_val, mean_unit = float(match.group(2)), match.group(3)
|
||||
median_val, median_unit = float(match.group(4)), match.group(5)
|
||||
p95_val, p95_unit = float(match.group(6)), match.group(7)
|
||||
|
||||
mean_ns = self._convert_to_nanoseconds(mean_val, mean_unit)
|
||||
median_ns = self._convert_to_nanoseconds(median_val, median_unit)
|
||||
p95_ns = self._convert_to_nanoseconds(p95_val, p95_unit)
|
||||
|
||||
# Estimate P99 (usually ~1.5x P95 for typical distributions)
|
||||
p99_ns = p95_ns * 1.5
|
||||
|
||||
# Estimate standard deviation (rough approximation)
|
||||
std_dev_ns = (p95_ns - mean_ns) / 1.645 # Assuming normal distribution
|
||||
|
||||
metrics.append(PerformanceMetrics(
|
||||
name=name,
|
||||
mean_ns=mean_ns,
|
||||
median_ns=median_ns,
|
||||
p95_ns=p95_ns,
|
||||
p99_ns=p99_ns,
|
||||
std_dev_ns=std_dev_ns
|
||||
))
|
||||
|
||||
# Parse throughput information
|
||||
for match in re.finditer(throughput_pattern, content):
|
||||
name = match.group(1)
|
||||
throughput_val = float(match.group(4)) # Use median throughput
|
||||
throughput_unit = match.group(5)
|
||||
|
||||
# Convert to ops/sec
|
||||
throughput_ops_sec = self._convert_to_ops_per_second(throughput_val, throughput_unit)
|
||||
|
||||
# Find corresponding metrics entry
|
||||
for metric in metrics:
|
||||
if metric.name == name:
|
||||
metric.throughput_ops_sec = throughput_ops_sec
|
||||
break
|
||||
|
||||
return metrics
|
||||
|
||||
def parse_benchmark_json(self, content: str) -> List[PerformanceMetrics]:
|
||||
"""Parse JSON benchmark results"""
|
||||
try:
|
||||
data = json.loads(content)
|
||||
metrics = []
|
||||
|
||||
for benchmark in data.get('benchmarks', []):
|
||||
name = benchmark.get('name', 'unknown')
|
||||
|
||||
# Extract timing statistics
|
||||
stats = benchmark.get('stats', {})
|
||||
mean_ns = stats.get('mean', 0) * 1e9 # Convert to nanoseconds
|
||||
median_ns = stats.get('median', 0) * 1e9
|
||||
p95_ns = stats.get('p95', 0) * 1e9
|
||||
p99_ns = stats.get('p99', mean_ns * 2) # Fallback if not available
|
||||
std_dev_ns = stats.get('std_dev', 0) * 1e9
|
||||
|
||||
throughput_ops_sec = benchmark.get('throughput_ops_sec')
|
||||
|
||||
metrics.append(PerformanceMetrics(
|
||||
name=name,
|
||||
mean_ns=mean_ns,
|
||||
median_ns=median_ns,
|
||||
p95_ns=p95_ns,
|
||||
p99_ns=p99_ns,
|
||||
std_dev_ns=std_dev_ns,
|
||||
throughput_ops_sec=throughput_ops_sec
|
||||
))
|
||||
|
||||
return metrics
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Error parsing JSON benchmark results: {e}")
|
||||
return []
|
||||
|
||||
def _convert_to_nanoseconds(self, value: float, unit: str) -> float:
|
||||
"""Convert time value to nanoseconds"""
|
||||
unit_multipliers = {
|
||||
'ns': 1,
|
||||
'μs': 1_000,
|
||||
'us': 1_000, # Alternative microsecond notation
|
||||
'ms': 1_000_000,
|
||||
's': 1_000_000_000
|
||||
}
|
||||
return value * unit_multipliers.get(unit, 1)
|
||||
|
||||
def _convert_to_ops_per_second(self, value: float, unit: str) -> float:
|
||||
"""Convert throughput to operations per second"""
|
||||
unit_multipliers = {
|
||||
'ops/s': 1,
|
||||
'Kops/s': 1_000,
|
||||
'Mops/s': 1_000_000,
|
||||
'Gops/s': 1_000_000_000
|
||||
}
|
||||
return value * unit_multipliers.get(unit, 1)
|
||||
|
||||
def load_benchmark_results(self) -> bool:
|
||||
"""Load and parse benchmark results"""
|
||||
if not self.benchmark_file.exists():
|
||||
self.validation_errors.append(f"Benchmark file not found: {self.benchmark_file}")
|
||||
return False
|
||||
|
||||
try:
|
||||
content = self.benchmark_file.read_text()
|
||||
|
||||
# Try JSON format first
|
||||
if content.strip().startswith('{'):
|
||||
self.results = self.parse_benchmark_json(content)
|
||||
else:
|
||||
# Fall back to Criterion text output
|
||||
self.results = self.parse_criterion_output(content)
|
||||
|
||||
if not self.results:
|
||||
self.validation_errors.append("No benchmark results found in file")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.validation_errors.append(f"Error reading benchmark file: {e}")
|
||||
return False
|
||||
|
||||
def validate_performance(self) -> bool:
|
||||
"""Validate performance metrics against thresholds"""
|
||||
validation_passed = True
|
||||
|
||||
for metric in self.results:
|
||||
# Find matching threshold
|
||||
threshold = None
|
||||
for threshold_name, threshold_config in PERFORMANCE_THRESHOLDS.items():
|
||||
if threshold_name in metric.name.lower():
|
||||
threshold = threshold_config
|
||||
break
|
||||
|
||||
if not threshold:
|
||||
self.validation_warnings.append(f"No threshold defined for benchmark: {metric.name}")
|
||||
continue
|
||||
|
||||
# Validate latency
|
||||
mean_us = metric.mean_ns / 1_000
|
||||
p95_us = metric.p95_ns / 1_000
|
||||
p99_us = metric.p99_ns / 1_000
|
||||
std_dev_us = metric.std_dev_ns / 1_000
|
||||
|
||||
if mean_us > threshold.max_latency_us:
|
||||
self.validation_errors.append(
|
||||
f"{metric.name}: Mean latency {mean_us:.2f}μs exceeds threshold {threshold.max_latency_us}μs"
|
||||
)
|
||||
validation_passed = False
|
||||
|
||||
if p95_us > threshold.max_p95_latency_us:
|
||||
self.validation_errors.append(
|
||||
f"{metric.name}: P95 latency {p95_us:.2f}μs exceeds threshold {threshold.max_p95_latency_us}μs"
|
||||
)
|
||||
validation_passed = False
|
||||
|
||||
if p99_us > threshold.max_p99_latency_us:
|
||||
self.validation_errors.append(
|
||||
f"{metric.name}: P99 latency {p99_us:.2f}μs exceeds threshold {threshold.max_p99_latency_us}μs"
|
||||
)
|
||||
validation_passed = False
|
||||
|
||||
if std_dev_us > threshold.max_std_dev_us:
|
||||
self.validation_warnings.append(
|
||||
f"{metric.name}: High latency variance {std_dev_us:.2f}μs (threshold: {threshold.max_std_dev_us}μs)"
|
||||
)
|
||||
|
||||
# Validate throughput if available
|
||||
if metric.throughput_ops_sec and metric.throughput_ops_sec < threshold.min_throughput_ops_sec:
|
||||
self.validation_errors.append(
|
||||
f"{metric.name}: Throughput {metric.throughput_ops_sec:.0f} ops/sec below threshold {threshold.min_throughput_ops_sec} ops/sec"
|
||||
)
|
||||
validation_passed = False
|
||||
|
||||
return validation_passed
|
||||
|
||||
def generate_report(self) -> str:
|
||||
"""Generate performance validation report"""
|
||||
report = []
|
||||
report.append("# Foxhunt HFT Performance Validation Report")
|
||||
report.append(f"Generated: {datetime.now().isoformat()}")
|
||||
report.append(f"Benchmark file: {self.benchmark_file}")
|
||||
report.append("")
|
||||
|
||||
# Summary
|
||||
total_benchmarks = len(self.results)
|
||||
errors_count = len(self.validation_errors)
|
||||
warnings_count = len(self.validation_warnings)
|
||||
|
||||
if errors_count == 0:
|
||||
report.append("## ✅ VALIDATION PASSED")
|
||||
else:
|
||||
report.append("## ❌ VALIDATION FAILED")
|
||||
|
||||
report.append(f"- Total benchmarks: {total_benchmarks}")
|
||||
report.append(f"- Validation errors: {errors_count}")
|
||||
report.append(f"- Validation warnings: {warnings_count}")
|
||||
report.append("")
|
||||
|
||||
# Detailed results
|
||||
report.append("## Performance Metrics")
|
||||
report.append("")
|
||||
|
||||
for metric in self.results:
|
||||
report.append(f"### {metric.name}")
|
||||
report.append(f"- Mean latency: {metric.mean_ns/1000:.2f}μs")
|
||||
report.append(f"- Median latency: {metric.median_ns/1000:.2f}μs")
|
||||
report.append(f"- P95 latency: {metric.p95_ns/1000:.2f}μs")
|
||||
report.append(f"- P99 latency: {metric.p99_ns/1000:.2f}μs")
|
||||
report.append(f"- Standard deviation: {metric.std_dev_ns/1000:.2f}μs")
|
||||
|
||||
if metric.throughput_ops_sec:
|
||||
report.append(f"- Throughput: {metric.throughput_ops_sec:,.0f} ops/sec")
|
||||
|
||||
report.append("")
|
||||
|
||||
# Errors and warnings
|
||||
if self.validation_errors:
|
||||
report.append("## ❌ Validation Errors")
|
||||
for error in self.validation_errors:
|
||||
report.append(f"- {error}")
|
||||
report.append("")
|
||||
|
||||
if self.validation_warnings:
|
||||
report.append("## ⚠️ Validation Warnings")
|
||||
for warning in self.validation_warnings:
|
||||
report.append(f"- {warning}")
|
||||
report.append("")
|
||||
|
||||
# Thresholds reference
|
||||
report.append("## Performance Thresholds")
|
||||
report.append("")
|
||||
|
||||
for name, threshold in PERFORMANCE_THRESHOLDS.items():
|
||||
report.append(f"### {name}")
|
||||
report.append(f"- Max mean latency: {threshold.max_latency_us}μs")
|
||||
report.append(f"- Max P95 latency: {threshold.max_p95_latency_us}μs")
|
||||
report.append(f"- Max P99 latency: {threshold.max_p99_latency_us}μs")
|
||||
report.append(f"- Min throughput: {threshold.min_throughput_ops_sec:,} ops/sec")
|
||||
report.append(f"- Max std deviation: {threshold.max_std_dev_us}μs")
|
||||
report.append("")
|
||||
|
||||
return "\n".join(report)
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python3 validate-performance.py <benchmark_results_file>")
|
||||
sys.exit(1)
|
||||
|
||||
benchmark_file = sys.argv[1]
|
||||
validator = PerformanceValidator(benchmark_file)
|
||||
|
||||
# Load benchmark results
|
||||
if not validator.load_benchmark_results():
|
||||
print("Failed to load benchmark results:")
|
||||
for error in validator.validation_errors:
|
||||
print(f" - {error}")
|
||||
sys.exit(1)
|
||||
|
||||
# Validate performance
|
||||
validation_passed = validator.validate_performance()
|
||||
|
||||
# Generate and save report
|
||||
report = validator.generate_report()
|
||||
|
||||
# Write report to file
|
||||
report_file = Path("performance-validation-report.md")
|
||||
report_file.write_text(report)
|
||||
|
||||
# Print summary
|
||||
print(f"Performance validation {'PASSED' if validation_passed else 'FAILED'}")
|
||||
print(f"Report saved to: {report_file}")
|
||||
|
||||
# Print errors to stderr
|
||||
if validator.validation_errors:
|
||||
print("\nValidation errors:")
|
||||
for error in validator.validation_errors:
|
||||
print(f" - {error}", file=sys.stderr)
|
||||
|
||||
if validator.validation_warnings:
|
||||
print("\nValidation warnings:")
|
||||
for warning in validator.validation_warnings:
|
||||
print(f" - {warning}")
|
||||
|
||||
# Exit with appropriate code
|
||||
sys.exit(0 if validation_passed else 1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,98 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
TFT Configuration Validator
|
||||
|
||||
Validates that all TFTConfig declarations in test files have correct feature counts.
|
||||
Usage: python3 scripts/validate_tft_configs.py
|
||||
"""
|
||||
|
||||
import re
|
||||
import glob
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
def validate_tft_configs():
|
||||
"""Validate all TFT configurations in test files."""
|
||||
|
||||
# Find all TFT test files
|
||||
test_dir = Path("ml/tests")
|
||||
test_files = list(test_dir.glob("tft*.rs")) + list(test_dir.glob("test_tft*.rs"))
|
||||
test_files += list(test_dir.glob("gpu_4_model_stress_test.rs"))
|
||||
test_files += list(test_dir.glob("ensemble_tft*.rs"))
|
||||
|
||||
total_configs = 0
|
||||
valid_configs = 0
|
||||
invalid_configs = []
|
||||
|
||||
for filepath in test_files:
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find all TFTConfig blocks
|
||||
configs = re.finditer(r'TFTConfig\s*\{([^}]+)\}', content, re.DOTALL)
|
||||
|
||||
for match in configs:
|
||||
config_text = match.group(1)
|
||||
|
||||
# Extract values
|
||||
input_dim_match = re.search(r'input_dim:\s*(\d+)', config_text)
|
||||
static_match = re.search(r'num_static_features:\s*(\d+)', config_text)
|
||||
known_match = re.search(r'num_known_features:\s*(\d+)', config_text)
|
||||
unknown_match = re.search(r'num_unknown_features:\s*(\d+)', config_text)
|
||||
|
||||
if all([input_dim_match, static_match, known_match, unknown_match]):
|
||||
total_configs += 1
|
||||
|
||||
input_dim = int(input_dim_match.group(1))
|
||||
static = int(static_match.group(1))
|
||||
known = int(known_match.group(1))
|
||||
unknown = int(unknown_match.group(1))
|
||||
total = static + known + unknown
|
||||
|
||||
if total == input_dim:
|
||||
valid_configs += 1
|
||||
else:
|
||||
line_num = content[:match.start()].count('\n') + 1
|
||||
invalid_configs.append({
|
||||
'file': str(filepath),
|
||||
'line': line_num,
|
||||
'input_dim': input_dim,
|
||||
'static': static,
|
||||
'known': known,
|
||||
'unknown': unknown,
|
||||
'total': total
|
||||
})
|
||||
|
||||
# Print results
|
||||
print("=" * 70)
|
||||
print("TFT Configuration Validation Report")
|
||||
print("=" * 70)
|
||||
print(f"\nTotal configurations found: {total_configs}")
|
||||
print(f"Valid configurations: {valid_configs}")
|
||||
print(f"Invalid configurations: {len(invalid_configs)}")
|
||||
print(f"\nSuccess rate: {valid_configs}/{total_configs} ({100*valid_configs//total_configs if total_configs > 0 else 0}%)")
|
||||
|
||||
if invalid_configs:
|
||||
print("\n" + "=" * 70)
|
||||
print("INVALID CONFIGURATIONS FOUND:")
|
||||
print("=" * 70)
|
||||
|
||||
for config in invalid_configs:
|
||||
print(f"\n{config['file']}:{config['line']}")
|
||||
print(f" input_dim={config['input_dim']}")
|
||||
print(f" static({config['static']}) + known({config['known']}) + unknown({config['unknown']}) = {config['total']}")
|
||||
print(f" ❌ Mismatch: {config['total']} != {config['input_dim']}")
|
||||
print(f" Fix: Change num_unknown_features to {config['input_dim'] - config['static'] - config['known']}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("VALIDATION FAILED")
|
||||
print("=" * 70)
|
||||
return 1
|
||||
else:
|
||||
print("\n" + "=" * 70)
|
||||
print("✓ ALL CONFIGURATIONS VALID")
|
||||
print("=" * 70)
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(validate_tft_configs())
|
||||
Reference in New Issue
Block a user