feat(ml): Complete hyperopt infrastructure + documentation
Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jgrusewski
180
scripts/archive/backtest_runpod_225.sh
Executable file
180
scripts/archive/backtest_runpod_225.sh
Executable file
@@ -0,0 +1,180 @@
|
||||
#!/bin/bash
|
||||
# Runpod 225-Feature Backtesting Script
|
||||
# Tests trained TFT model against Wave D targets
|
||||
# Targets: Sharpe ≥2.0, Win Rate ≥60%, Drawdown ≤15%
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
MODEL_PATH="models/runpod_trained/tft_225_fp32.safetensors"
|
||||
DATA_PATH="test_data/ES_FUT_180d.parquet"
|
||||
INITIAL_CAPITAL=100000
|
||||
SYMBOLS="ES.FUT,NQ.FUT"
|
||||
STRATEGY="ml_adaptive_225"
|
||||
START_DATE="2024-01-01"
|
||||
END_DATE="2024-06-30"
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Runpod 225-Feature Backtesting${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Step 1: Verify model exists
|
||||
echo -e "${YELLOW}Step 1/4: Verifying trained model...${NC}"
|
||||
|
||||
if [ ! -f "${MODEL_PATH}" ]; then
|
||||
# Try to find any safetensors file
|
||||
FOUND_MODEL=$(find models/runpod_trained -name "*.safetensors" -type f | head -1)
|
||||
|
||||
if [ -z "$FOUND_MODEL" ]; then
|
||||
echo -e "${RED}❌ No trained model found in models/runpod_trained/${NC}"
|
||||
echo "Expected: ${MODEL_PATH}"
|
||||
echo ""
|
||||
echo "Available files:"
|
||||
ls -lh models/runpod_trained/ 2>/dev/null || echo "Directory does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}⚠️ Using found model: ${FOUND_MODEL}${NC}"
|
||||
MODEL_PATH="$FOUND_MODEL"
|
||||
fi
|
||||
|
||||
MODEL_SIZE=$(du -h "${MODEL_PATH}" | cut -f1)
|
||||
echo -e "${GREEN}✅ Model found: ${MODEL_SIZE}${NC}"
|
||||
|
||||
# Step 2: Verify data
|
||||
echo -e "\n${YELLOW}Step 2/4: Verifying training data...${NC}"
|
||||
|
||||
if [ ! -f "${DATA_PATH}" ]; then
|
||||
echo -e "${RED}❌ Training data not found: ${DATA_PATH}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DATA_SIZE=$(du -h "${DATA_PATH}" | cut -f1)
|
||||
ROW_COUNT=$(parquet-tools rowcount "${DATA_PATH}" 2>/dev/null || echo "unknown")
|
||||
echo -e "${GREEN}✅ Data found: ${DATA_SIZE}, Rows: ${ROW_COUNT}${NC}"
|
||||
|
||||
# Step 3: Run backtest
|
||||
echo -e "\n${YELLOW}Step 3/4: Running backtest...${NC}"
|
||||
echo "Configuration:"
|
||||
echo " • Model: ${MODEL_PATH}"
|
||||
echo " • Data: ${DATA_PATH}"
|
||||
echo " • Initial Capital: \$${INITIAL_CAPITAL}"
|
||||
echo " • Symbols: ${SYMBOLS}"
|
||||
echo " • Strategy: ${STRATEGY}"
|
||||
echo " • Period: ${START_DATE} to ${END_DATE}"
|
||||
echo ""
|
||||
|
||||
BACKTEST_START=$(date +%s)
|
||||
|
||||
# Run backtest (adjust command based on actual backtesting CLI)
|
||||
cargo run -p backtesting --release --features cuda --example feature_comparison_backtest -- \
|
||||
--model-path "${MODEL_PATH}" \
|
||||
--parquet-file "${DATA_PATH}" \
|
||||
--initial-capital ${INITIAL_CAPITAL} \
|
||||
--symbols "${SYMBOLS}" \
|
||||
--strategy "${STRATEGY}" \
|
||||
--start-date "${START_DATE}" \
|
||||
--end-date "${END_DATE}" \
|
||||
2>&1 | tee backtest_225.log
|
||||
|
||||
BACKTEST_END=$(date +%s)
|
||||
BACKTEST_DURATION=$((BACKTEST_END - BACKTEST_START))
|
||||
|
||||
echo -e "\n${GREEN}✅ Backtest completed in ${BACKTEST_DURATION}s${NC}"
|
||||
|
||||
# Step 4: Extract and validate metrics
|
||||
echo -e "\n${YELLOW}Step 4/4: Extracting Wave D metrics...${NC}"
|
||||
|
||||
# Extract key metrics from backtest output
|
||||
SHARPE=$(grep -oP "Sharpe Ratio[:\s]+\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
|
||||
WIN_RATE=$(grep -oP "Win Rate[:\s]+\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
|
||||
DRAWDOWN=$(grep -oP "Max Drawdown[:\s]+\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
|
||||
TOTAL_PNL=$(grep -oP "Total PnL[:\s]+\$?\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
|
||||
TOTAL_TRADES=$(grep -oP "Total Trades[:\s]+\K[0-9]+" backtest_225.log | tail -1 || echo "N/A")
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Wave D Backtest Results${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Sharpe Ratio validation
|
||||
echo -e "${BLUE}Sharpe Ratio:${NC} ${SHARPE}"
|
||||
if [ "$SHARPE" != "N/A" ] && [ "$(echo "${SHARPE} >= 2.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
|
||||
echo -e " ${GREEN}✅ Target: ≥2.0 (PASSED)${NC}"
|
||||
else
|
||||
echo -e " ${RED}❌ Target: ≥2.0 (FAILED)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Win Rate validation
|
||||
echo -e "${BLUE}Win Rate:${NC} ${WIN_RATE}%"
|
||||
if [ "$WIN_RATE" != "N/A" ] && [ "$(echo "${WIN_RATE} >= 60.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
|
||||
echo -e " ${GREEN}✅ Target: ≥60% (PASSED)${NC}"
|
||||
else
|
||||
echo -e " ${RED}❌ Target: ≥60% (FAILED)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Drawdown validation
|
||||
echo -e "${BLUE}Max Drawdown:${NC} ${DRAWDOWN}%"
|
||||
if [ "$DRAWDOWN" != "N/A" ] && [ "$(echo "${DRAWDOWN} <= 15.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
|
||||
echo -e " ${GREEN}✅ Target: ≤15% (PASSED)${NC}"
|
||||
else
|
||||
echo -e " ${RED}❌ Target: ≤15% (FAILED)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Additional metrics
|
||||
echo -e "${BLUE}Additional Metrics:${NC}"
|
||||
echo " • Total PnL: \$${TOTAL_PNL}"
|
||||
echo " • Total Trades: ${TOTAL_TRADES}"
|
||||
echo " • Backtest Duration: ${BACKTEST_DURATION}s"
|
||||
echo ""
|
||||
|
||||
# Overall assessment
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
PASSED_COUNT=0
|
||||
|
||||
if [ "$SHARPE" != "N/A" ] && [ "$(echo "${SHARPE} >= 2.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
|
||||
PASSED_COUNT=$((PASSED_COUNT + 1))
|
||||
fi
|
||||
|
||||
if [ "$WIN_RATE" != "N/A" ] && [ "$(echo "${WIN_RATE} >= 60.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
|
||||
PASSED_COUNT=$((PASSED_COUNT + 1))
|
||||
fi
|
||||
|
||||
if [ "$DRAWDOWN" != "N/A" ] && [ "$(echo "${DRAWDOWN} <= 15.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
|
||||
PASSED_COUNT=$((PASSED_COUNT + 1))
|
||||
fi
|
||||
|
||||
if [ $PASSED_COUNT -eq 3 ]; then
|
||||
echo -e "${GREEN}✅ All Wave D Targets PASSED (3/3)${NC}"
|
||||
echo ""
|
||||
echo "Model is ready for production deployment!"
|
||||
elif [ $PASSED_COUNT -ge 2 ]; then
|
||||
echo -e "${YELLOW}⚠️ Partial Success: ${PASSED_COUNT}/3 Targets Passed${NC}"
|
||||
echo ""
|
||||
echo "Model shows promise but may need fine-tuning."
|
||||
else
|
||||
echo -e "${RED}❌ Wave D Targets NOT MET (${PASSED_COUNT}/3 Passed)${NC}"
|
||||
echo ""
|
||||
echo "Model requires additional training or hyperparameter tuning."
|
||||
fi
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
echo "Next Steps:"
|
||||
echo " 1. Review detailed backtest log: backtest_225.log"
|
||||
echo " 2. Generate results report: see RUNPOD_225_FEATURE_TRAINING_RESULTS.md"
|
||||
echo " 3. Train additional models: DQN, PPO, MAMBA-2"
|
||||
echo " 4. Multi-asset validation: NQ.FUT, 6E.FUT, ZN.FUT"
|
||||
echo ""
|
||||
68
scripts/archive/check_pod_status.py
Executable file
68
scripts/archive/check_pod_status.py
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Check RunPod Pod Status via REST API
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
||||
# Load environment variables
|
||||
env_path = Path(__file__).parent.parent / '.env.runpod'
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
def get_pod_status(pod_id):
|
||||
"""Fetch pod status via REST API."""
|
||||
|
||||
url = f"https://rest.runpod.io/v1/pods/{pod_id}"
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"ERROR: Failed to query RunPod API: {e}")
|
||||
if hasattr(e.response, 'text'):
|
||||
print(f"Response: {e.response.text[:500]}")
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: ./check_pod_status.py <pod_id>")
|
||||
sys.exit(1)
|
||||
|
||||
pod_id = sys.argv[1]
|
||||
|
||||
print(f"Fetching pod status for: {pod_id}")
|
||||
print("=" * 70)
|
||||
|
||||
pod = get_pod_status(pod_id)
|
||||
|
||||
if not pod:
|
||||
print("Failed to fetch pod status")
|
||||
sys.exit(1)
|
||||
|
||||
# Display pod info (pretty print)
|
||||
import json
|
||||
print(json.dumps(pod, indent=2))
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("\nTo view logs:")
|
||||
print(f"1. Web UI: https://www.runpod.io/console/pods/{pod_id}")
|
||||
print(f"2. SSH: ssh root@{pod_id}.ssh.runpod.io")
|
||||
print(" Then run: docker ps # Get container ID")
|
||||
print(" docker logs <container_id>")
|
||||
300
scripts/archive/check_runpod_datacenter_field.py
Executable file
300
scripts/archive/check_runpod_datacenter_field.py
Executable file
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Runpod GraphQL Schema Introspection - Datacenter Field Check
|
||||
|
||||
This script introspects the Runpod GraphQL schema to:
|
||||
1. Find all available fields in PodFindAndDeployOnDemandInput
|
||||
2. Identify datacenter/region selection fields
|
||||
3. Document the exact field name and type
|
||||
4. Query available datacenters (if supported)
|
||||
|
||||
Prerequisites:
|
||||
- Set RUNPOD_API_KEY environment variable
|
||||
- Install requests: pip install requests
|
||||
|
||||
Usage:
|
||||
export RUNPOD_API_KEY='your-key-here'
|
||||
python3 scripts/check_runpod_datacenter_field.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import json
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
|
||||
API_URL = 'https://api.runpod.io/graphql'
|
||||
|
||||
|
||||
def get_api_key() -> Optional[str]:
|
||||
"""Get RunPod API key from environment"""
|
||||
api_key = os.environ.get('RUNPOD_API_KEY')
|
||||
if not api_key:
|
||||
print("❌ ERROR: RUNPOD_API_KEY environment variable not set")
|
||||
print("\nSet it with:")
|
||||
print(" export RUNPOD_API_KEY='your-key-here'")
|
||||
print("\nGet your API key from: https://www.runpod.io/console/user/settings")
|
||||
return None
|
||||
return api_key
|
||||
|
||||
|
||||
def make_graphql_request(api_key: str, query: str) -> Optional[Dict[str, Any]]:
|
||||
"""Make GraphQL request to RunPod API"""
|
||||
url = f'{API_URL}?api_key={api_key}'
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
payload = {'query': query}
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors:")
|
||||
for error in data['errors']:
|
||||
print(f" - {error.get('message', 'Unknown error')}")
|
||||
return None
|
||||
|
||||
return data
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def introspect_pod_input_type(api_key: str):
|
||||
"""Introspect PodFindAndDeployOnDemandInput type"""
|
||||
print("\n" + "="*70)
|
||||
print("🔍 INTROSPECTING: PodFindAndDeployOnDemandInput")
|
||||
print("="*70)
|
||||
|
||||
query = '''
|
||||
{
|
||||
__type(name: "PodFindAndDeployOnDemandInput") {
|
||||
name
|
||||
inputFields {
|
||||
name
|
||||
description
|
||||
type {
|
||||
name
|
||||
kind
|
||||
ofType {
|
||||
name
|
||||
kind
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
data = make_graphql_request(api_key, query)
|
||||
if not data or 'data' not in data:
|
||||
return None
|
||||
|
||||
type_info = data['data'].get('__type')
|
||||
if not type_info:
|
||||
print("❌ Type not found in schema")
|
||||
return None
|
||||
|
||||
fields = type_info['inputFields']
|
||||
print(f"\n✅ Found {len(fields)} fields\n")
|
||||
|
||||
# Print all fields
|
||||
print("📋 All Available Fields:")
|
||||
print("-" * 70)
|
||||
for field in sorted(fields, key=lambda f: f['name']):
|
||||
field_name = field['name']
|
||||
field_type = field['type']
|
||||
type_name = field_type.get('name') or field_type.get('ofType', {}).get('name', 'Unknown')
|
||||
required = '!' in str(field_type)
|
||||
req_marker = '(required)' if required else '(optional)'
|
||||
|
||||
print(f" {field_name:<30} {type_name:<20} {req_marker}")
|
||||
|
||||
if field.get('description'):
|
||||
print(f" → {field['description']}")
|
||||
|
||||
# Search for datacenter/region fields
|
||||
print("\n" + "="*70)
|
||||
print("🎯 SEARCHING FOR DATACENTER/REGION FIELDS")
|
||||
print("="*70)
|
||||
|
||||
keywords = ['datacenter', 'region', 'location', 'zone', 'area', 'center']
|
||||
datacenter_fields = [
|
||||
f for f in fields
|
||||
if any(keyword in f['name'].lower() for keyword in keywords)
|
||||
]
|
||||
|
||||
if datacenter_fields:
|
||||
print("\n✅ FOUND DATACENTER/REGION FIELDS:\n")
|
||||
for field in datacenter_fields:
|
||||
field_name = field['name']
|
||||
field_type = field['type']
|
||||
type_name = field_type.get('name') or field_type.get('ofType', {}).get('name', 'Unknown')
|
||||
|
||||
print(f" 🎯 {field_name}")
|
||||
print(f" Type: {type_name}")
|
||||
if field.get('description'):
|
||||
print(f" Description: {field['description']}")
|
||||
print()
|
||||
|
||||
return datacenter_fields
|
||||
else:
|
||||
print("\n❌ NO datacenter/region fields found")
|
||||
print("\nThis could mean:")
|
||||
print(" 1. Region is implicitly selected based on networkVolumeId")
|
||||
print(" 2. Region selection not supported in current API version")
|
||||
print(" 3. Different field naming convention")
|
||||
return None
|
||||
|
||||
|
||||
def query_datacenters(api_key: str):
|
||||
"""Try to query available datacenters"""
|
||||
print("\n" + "="*70)
|
||||
print("🌍 QUERYING AVAILABLE DATACENTERS")
|
||||
print("="*70)
|
||||
|
||||
# Try different possible query names
|
||||
queries = [
|
||||
('dataCenters', '{ dataCenters { id name location } }'),
|
||||
('datacenters', '{ datacenters { id name location } }'),
|
||||
('regions', '{ regions { id name location } }'),
|
||||
('locations', '{ locations { id name location } }'),
|
||||
]
|
||||
|
||||
for query_name, query in queries:
|
||||
print(f"\nTrying query: {query_name}...")
|
||||
data = make_graphql_request(api_key, query)
|
||||
|
||||
if data and 'data' in data:
|
||||
result = data['data'].get(query_name)
|
||||
if result is not None:
|
||||
print(f"\n✅ Found datacenters via '{query_name}' query!")
|
||||
print(json.dumps(result, indent=2))
|
||||
return result
|
||||
|
||||
print("\n⚠️ No datacenter query endpoint found")
|
||||
print("Datacenters may not be queryable via GraphQL")
|
||||
|
||||
|
||||
def generate_recommendations(datacenter_fields: Optional[List[Dict]]):
|
||||
"""Generate recommendations based on findings"""
|
||||
print("\n" + "="*70)
|
||||
print("📝 RECOMMENDATIONS")
|
||||
print("="*70)
|
||||
|
||||
if datacenter_fields:
|
||||
field = datacenter_fields[0]
|
||||
field_name = field['name']
|
||||
|
||||
print(f"\n✅ SOLUTION FOUND: Use '{field_name}' field")
|
||||
print("\nCode change for scripts/runpod_deploy_production.py:")
|
||||
print("-" * 70)
|
||||
print(f"""
|
||||
mutation = f\"\"\"
|
||||
mutation {{{{
|
||||
podFindAndDeployOnDemand(
|
||||
input: {{{{
|
||||
cloudType: {{cloud_type}}
|
||||
{field_name}: "eur-is-1" # <-- ADD THIS LINE
|
||||
gpuTypeId: "{{gpu_id}}"
|
||||
name: "{{name}}"
|
||||
imageName: "{{config['image_name']}}"
|
||||
networkVolumeId: "{{config['network_volume_id']}}"
|
||||
...
|
||||
}}}}
|
||||
) {{{{ ... }}}}
|
||||
}}}}
|
||||
\"\"\"
|
||||
""")
|
||||
print("-" * 70)
|
||||
|
||||
else:
|
||||
print("\n⚠️ NO DATACENTER FIELD FOUND")
|
||||
print("\nPossible solutions:")
|
||||
print(" 1. TEST CURRENT BEHAVIOR:")
|
||||
print(" - Deploy a test pod with current script")
|
||||
print(" - Check if volume mounts correctly")
|
||||
print(" - Runpod may auto-select region based on networkVolumeId")
|
||||
print()
|
||||
print(" 2. CONTACT RUNPOD SUPPORT:")
|
||||
print(" - Ask about region selection for network volumes")
|
||||
print(" - Request documentation for datacenter parameter")
|
||||
print()
|
||||
print(" 3. USE SPOT INSTANCES:")
|
||||
print(" - Try podRentInterruptable mutation")
|
||||
print(" - May have different region selection options")
|
||||
print()
|
||||
print(" 4. MIGRATE VOLUME:")
|
||||
print(" - Create new volume in high-availability region (us-ca-1)")
|
||||
print(" - Copy data to new volume")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main execution"""
|
||||
print("="*70)
|
||||
print("RUNPOD DATACENTER FIELD INTROSPECTION")
|
||||
print("="*70)
|
||||
|
||||
# Get API key
|
||||
api_key = get_api_key()
|
||||
if not api_key:
|
||||
return 1
|
||||
|
||||
print("\n✅ API key found")
|
||||
print(f"🌐 API endpoint: {API_URL}")
|
||||
|
||||
# Test authentication
|
||||
print("\n🔐 Testing authentication...")
|
||||
test_query = '{ gpuTypes { id displayName } }'
|
||||
test_data = make_graphql_request(api_key, test_query)
|
||||
|
||||
if not test_data:
|
||||
print("❌ Authentication failed")
|
||||
return 1
|
||||
|
||||
print("✅ Authentication successful")
|
||||
|
||||
# Introspect PodFindAndDeployOnDemandInput
|
||||
datacenter_fields = introspect_pod_input_type(api_key)
|
||||
|
||||
# Try to query datacenters
|
||||
query_datacenters(api_key)
|
||||
|
||||
# Generate recommendations
|
||||
generate_recommendations(datacenter_fields)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("✅ INTROSPECTION COMPLETE")
|
||||
print("="*70)
|
||||
|
||||
# Save results
|
||||
output_file = 'RUNPOD_DATACENTER_INTROSPECTION_RESULTS.json'
|
||||
results = {
|
||||
'timestamp': '2025-10-24',
|
||||
'datacenter_fields_found': bool(datacenter_fields),
|
||||
'datacenter_fields': datacenter_fields if datacenter_fields else [],
|
||||
}
|
||||
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
print(f"\n💾 Results saved to: {output_file}")
|
||||
print("\nNext steps:")
|
||||
print(" 1. Review recommendations above")
|
||||
print(" 2. Update deployment script if datacenter field found")
|
||||
print(" 3. Test with smoke test deployment")
|
||||
print(" 4. Update CLAUDE.md with findings")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
414
scripts/archive/deploy_dqn_staging.sh
Executable file
414
scripts/archive/deploy_dqn_staging.sh
Executable file
@@ -0,0 +1,414 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# DQN MODEL STAGING DEPLOYMENT SCRIPT
|
||||
# =============================================================================
|
||||
# Deploys production-ready DQN model to staging environment for paper trading
|
||||
#
|
||||
# Author: Agent F5
|
||||
# Date: 2025-10-18
|
||||
# Purpose: Deploy DQN v1 (100% production ready, 36.6μs inference) to staging
|
||||
#
|
||||
# Usage: ./scripts/deploy_dqn_staging.sh
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Color codes for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
DQN_CHECKPOINT="$PROJECT_ROOT/ml/trained_models/dqn_final_epoch100.safetensors"
|
||||
STAGING_MODEL_DIR="$PROJECT_ROOT/ml/trained_models/staging"
|
||||
STAGING_MODEL="$STAGING_MODEL_DIR/dqn_production_v1.safetensors"
|
||||
CONFIG_FILE="$PROJECT_ROOT/config/ml_models_staging.toml"
|
||||
LOG_DIR="$PROJECT_ROOT/logs/staging"
|
||||
DEPLOYMENT_LOG="$LOG_DIR/dqn_deployment_$(date +%Y%m%d_%H%M%S).log"
|
||||
|
||||
# Database credentials
|
||||
DB_HOST="localhost"
|
||||
DB_PORT="5432"
|
||||
DB_NAME="foxhunt_staging"
|
||||
DB_USER="foxhunt"
|
||||
DB_PASSWORD="foxhunt_dev_password"
|
||||
DB_URL="postgresql://$DB_USER:$DB_PASSWORD@$DB_HOST:$DB_PORT/$DB_NAME"
|
||||
|
||||
# Prometheus and Grafana ports
|
||||
PROMETHEUS_PORT=9090
|
||||
GRAFANA_PORT=3000
|
||||
|
||||
# Function to print colored output
|
||||
print_step() {
|
||||
echo -e "${BLUE}[STEP]${NC} $1"
|
||||
}
|
||||
|
||||
print_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
# Function to check prerequisites
|
||||
check_prerequisites() {
|
||||
print_step "Checking prerequisites..."
|
||||
|
||||
# Check if DQN checkpoint exists
|
||||
if [ ! -f "$DQN_CHECKPOINT" ]; then
|
||||
print_error "DQN checkpoint not found: $DQN_CHECKPOINT"
|
||||
exit 1
|
||||
fi
|
||||
print_success "DQN checkpoint found: $DQN_CHECKPOINT"
|
||||
|
||||
# Verify checkpoint size (should be 68 KB)
|
||||
CHECKPOINT_SIZE=$(stat -c%s "$DQN_CHECKPOINT")
|
||||
if [ "$CHECKPOINT_SIZE" -ne 69632 ]; then
|
||||
print_warning "Checkpoint size is $CHECKPOINT_SIZE bytes (expected 69632 bytes)"
|
||||
else
|
||||
print_success "Checkpoint size verified: $CHECKPOINT_SIZE bytes (68 KB)"
|
||||
fi
|
||||
|
||||
# Check if PostgreSQL is running
|
||||
if ! psql "$DB_URL" -c "SELECT 1" > /dev/null 2>&1; then
|
||||
print_error "Cannot connect to staging database: $DB_URL"
|
||||
exit 1
|
||||
fi
|
||||
print_success "Connected to staging database: $DB_NAME"
|
||||
|
||||
# Check if Docker services are running
|
||||
if ! docker ps | grep -q foxhunt; then
|
||||
print_warning "Docker services may not be running. Consider starting with 'docker-compose up -d'"
|
||||
else
|
||||
print_success "Docker services are running"
|
||||
fi
|
||||
|
||||
# Check CUDA availability
|
||||
if command -v nvidia-smi &> /dev/null; then
|
||||
GPU_INFO=$(nvidia-smi --query-gpu=name,memory.total --format=csv,noheader | head -1)
|
||||
print_success "GPU detected: $GPU_INFO"
|
||||
else
|
||||
print_warning "nvidia-smi not found. GPU may not be available."
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to create staging directory structure
|
||||
setup_directories() {
|
||||
print_step "Setting up staging directory structure..."
|
||||
|
||||
mkdir -p "$STAGING_MODEL_DIR"
|
||||
mkdir -p "$LOG_DIR"
|
||||
mkdir -p "$PROJECT_ROOT/data/staging"
|
||||
|
||||
print_success "Directories created"
|
||||
}
|
||||
|
||||
# Function to deploy DQN model
|
||||
deploy_model() {
|
||||
print_step "Deploying DQN model to staging..."
|
||||
|
||||
# Copy checkpoint to staging directory
|
||||
cp -v "$DQN_CHECKPOINT" "$STAGING_MODEL"
|
||||
|
||||
# Calculate SHA-256 checksum
|
||||
CHECKSUM=$(sha256sum "$STAGING_MODEL" | awk '{print $1}')
|
||||
print_success "Model deployed: $STAGING_MODEL"
|
||||
print_success "SHA-256: $CHECKSUM"
|
||||
|
||||
# Create model metadata file
|
||||
METADATA_FILE="$STAGING_MODEL_DIR/dqn_production_v1.json"
|
||||
cat > "$METADATA_FILE" <<EOF
|
||||
{
|
||||
"model_id": "DQN_v1",
|
||||
"model_type": "DQN",
|
||||
"version": "1.0.0",
|
||||
"checkpoint_epoch": 100,
|
||||
"deployment_date": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"deployment_environment": "staging",
|
||||
"checkpoint_path": "$STAGING_MODEL",
|
||||
"checksum_sha256": "$CHECKSUM",
|
||||
"file_size_bytes": $(stat -c%s "$STAGING_MODEL"),
|
||||
"training_metadata": {
|
||||
"training_samples": 665483,
|
||||
"training_duration_seconds": 192,
|
||||
"final_loss": 0.0234,
|
||||
"validation_accuracy": 0.891
|
||||
},
|
||||
"performance_metrics": {
|
||||
"inference_latency_us": 36.6,
|
||||
"target_latency_us": 100,
|
||||
"memory_usage_mb": 6,
|
||||
"gpu_memory_mb": 6
|
||||
},
|
||||
"model_architecture": {
|
||||
"input_features": 26,
|
||||
"hidden_layers": [128, 64, 32],
|
||||
"output_actions": 3,
|
||||
"activation": "relu",
|
||||
"optimizer": "adam",
|
||||
"learning_rate": 0.001
|
||||
}
|
||||
}
|
||||
EOF
|
||||
print_success "Model metadata created: $METADATA_FILE"
|
||||
}
|
||||
|
||||
# Function to register model in database
|
||||
register_model_in_db() {
|
||||
print_step "Registering DQN model in staging database..."
|
||||
|
||||
# Insert model registration record
|
||||
psql "$DB_URL" <<EOF
|
||||
INSERT INTO ml_models (
|
||||
model_id,
|
||||
model_type,
|
||||
version,
|
||||
checkpoint_path,
|
||||
deployment_date,
|
||||
status,
|
||||
metadata
|
||||
) VALUES (
|
||||
'DQN_v1',
|
||||
'DQN',
|
||||
'1.0.0',
|
||||
'$STAGING_MODEL',
|
||||
NOW(),
|
||||
'active',
|
||||
'{
|
||||
"checkpoint_epoch": 100,
|
||||
"training_samples": 665483,
|
||||
"inference_latency_us": 36.6,
|
||||
"memory_usage_mb": 6
|
||||
}'::jsonb
|
||||
)
|
||||
ON CONFLICT (model_id) DO UPDATE SET
|
||||
checkpoint_path = EXCLUDED.checkpoint_path,
|
||||
deployment_date = EXCLUDED.deployment_date,
|
||||
status = EXCLUDED.status,
|
||||
metadata = EXCLUDED.metadata;
|
||||
EOF
|
||||
|
||||
print_success "Model registered in database"
|
||||
}
|
||||
|
||||
# Function to configure paper trading
|
||||
configure_paper_trading() {
|
||||
print_step "Configuring paper trading with DQN predictions..."
|
||||
|
||||
# Update paper trading configuration
|
||||
psql "$DB_URL" <<EOF
|
||||
INSERT INTO paper_trading_config (
|
||||
config_name,
|
||||
enabled,
|
||||
initial_capital_usd,
|
||||
max_position_size_usd,
|
||||
max_positions,
|
||||
slippage_bps,
|
||||
commission_per_trade,
|
||||
symbols,
|
||||
ml_model_id,
|
||||
created_at,
|
||||
updated_at
|
||||
) VALUES (
|
||||
'dqn_staging',
|
||||
true,
|
||||
100000,
|
||||
10000,
|
||||
5,
|
||||
2,
|
||||
1.00,
|
||||
ARRAY['ES.FUT', 'NQ.FUT'],
|
||||
'DQN_v1',
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
ON CONFLICT (config_name) DO UPDATE SET
|
||||
enabled = EXCLUDED.enabled,
|
||||
ml_model_id = EXCLUDED.ml_model_id,
|
||||
updated_at = EXCLUDED.updated_at;
|
||||
EOF
|
||||
|
||||
print_success "Paper trading configured"
|
||||
}
|
||||
|
||||
# Function to validate deployment
|
||||
validate_deployment() {
|
||||
print_step "Validating deployment..."
|
||||
|
||||
# Check if model file exists
|
||||
if [ ! -f "$STAGING_MODEL" ]; then
|
||||
print_error "Model file not found: $STAGING_MODEL"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Verify model is registered in database
|
||||
MODEL_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM ml_models WHERE model_id = 'DQN_v1' AND status = 'active';" | xargs)
|
||||
if [ "$MODEL_COUNT" -eq "1" ]; then
|
||||
print_success "Model is registered and active in database"
|
||||
else
|
||||
print_error "Model is not properly registered in database (count: $MODEL_COUNT)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check paper trading configuration
|
||||
PAPER_TRADING_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM paper_trading_config WHERE config_name = 'dqn_staging' AND enabled = true;" | xargs)
|
||||
if [ "$PAPER_TRADING_COUNT" -eq "1" ]; then
|
||||
print_success "Paper trading is configured and enabled"
|
||||
else
|
||||
print_error "Paper trading is not properly configured (count: $PAPER_TRADING_COUNT)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if Prometheus is accessible
|
||||
if curl -s "http://localhost:$PROMETHEUS_PORT/-/healthy" > /dev/null 2>&1; then
|
||||
print_success "Prometheus is accessible on port $PROMETHEUS_PORT"
|
||||
else
|
||||
print_warning "Prometheus is not accessible on port $PROMETHEUS_PORT"
|
||||
fi
|
||||
|
||||
# Check if Grafana is accessible
|
||||
if curl -s "http://localhost:$GRAFANA_PORT/api/health" > /dev/null 2>&1; then
|
||||
print_success "Grafana is accessible on port $GRAFANA_PORT"
|
||||
else
|
||||
print_warning "Grafana is not accessible on port $GRAFANA_PORT"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to create deployment summary
|
||||
create_deployment_summary() {
|
||||
print_step "Creating deployment summary..."
|
||||
|
||||
SUMMARY_FILE="$LOG_DIR/dqn_deployment_summary_$(date +%Y%m%d_%H%M%S).txt"
|
||||
|
||||
cat > "$SUMMARY_FILE" <<EOF
|
||||
================================================================================
|
||||
DQN MODEL STAGING DEPLOYMENT SUMMARY
|
||||
================================================================================
|
||||
|
||||
Deployment Date: $(date -u +%Y-%m-%d\ %H:%M:%S\ UTC)
|
||||
Deployment Environment: Staging
|
||||
Model ID: DQN_v1
|
||||
Model Version: 1.0.0
|
||||
|
||||
MODEL DETAILS:
|
||||
--------------
|
||||
Checkpoint Path: $STAGING_MODEL
|
||||
Checkpoint Epoch: 100
|
||||
File Size: $(stat -c%s "$STAGING_MODEL") bytes (68 KB)
|
||||
SHA-256 Checksum: $(sha256sum "$STAGING_MODEL" | awk '{print $1}')
|
||||
|
||||
TRAINING METADATA:
|
||||
------------------
|
||||
Training Samples: 665,483
|
||||
Training Duration: 192 seconds (3.2 minutes)
|
||||
Final Loss: 0.0234
|
||||
Validation Accuracy: 89.1%
|
||||
|
||||
PERFORMANCE METRICS:
|
||||
--------------------
|
||||
Inference Latency: 36.6 μs
|
||||
Target Latency: < 100 μs
|
||||
Memory Usage: 6 MB
|
||||
GPU Memory: 6 MB
|
||||
|
||||
MODEL ARCHITECTURE:
|
||||
-------------------
|
||||
Input Features: 26 (Wave A features)
|
||||
Hidden Layers: [128, 64, 32]
|
||||
Output Actions: 3 (Buy, Sell, Hold)
|
||||
Activation: ReLU
|
||||
Optimizer: Adam
|
||||
Learning Rate: 0.001
|
||||
|
||||
PAPER TRADING CONFIGURATION:
|
||||
-----------------------------
|
||||
Initial Capital: \$100,000
|
||||
Max Position Size: \$10,000
|
||||
Max Positions: 5
|
||||
Slippage: 2 bps (0.02%)
|
||||
Commission: \$1.00 per trade
|
||||
Symbols: ES.FUT, NQ.FUT
|
||||
|
||||
DATABASE:
|
||||
---------
|
||||
Host: $DB_HOST:$DB_PORT
|
||||
Database: $DB_NAME
|
||||
Model Registration: Active
|
||||
Paper Trading: Enabled
|
||||
|
||||
MONITORING:
|
||||
-----------
|
||||
Prometheus: http://localhost:$PROMETHEUS_PORT
|
||||
Grafana: http://localhost:$GRAFANA_PORT
|
||||
|
||||
NEXT STEPS:
|
||||
-----------
|
||||
1. Start trading service: cargo run -p trading_service --release
|
||||
2. Monitor predictions: tail -f $LOG_DIR/ml_models.log
|
||||
3. View metrics: http://localhost:$PROMETHEUS_PORT/targets
|
||||
4. View dashboards: http://localhost:$GRAFANA_PORT (admin/foxhunt123)
|
||||
|
||||
VALIDATION COMMANDS:
|
||||
--------------------
|
||||
# Check model predictions in database
|
||||
psql $DB_URL -c "SELECT * FROM ensemble_predictions WHERE prediction_timestamp > NOW() - INTERVAL '1 hour' ORDER BY prediction_timestamp DESC LIMIT 10;"
|
||||
|
||||
# Check paper trading orders
|
||||
psql $DB_URL -c "SELECT * FROM agent_orders WHERE created_at > NOW() - INTERVAL '1 hour' ORDER BY created_at DESC LIMIT 10;"
|
||||
|
||||
# Monitor inference latency
|
||||
psql $DB_URL -c "SELECT AVG(inference_latency_us) as avg_latency_us, MAX(inference_latency_us) as max_latency_us FROM ensemble_predictions WHERE prediction_timestamp > NOW() - INTERVAL '1 hour';"
|
||||
|
||||
# Check paper trading PnL
|
||||
psql $DB_URL -c "SELECT symbol, SUM(pnl) as total_pnl FROM ensemble_predictions WHERE pnl IS NOT NULL GROUP BY symbol ORDER BY total_pnl DESC;"
|
||||
|
||||
================================================================================
|
||||
DEPLOYMENT SUCCESSFUL
|
||||
================================================================================
|
||||
EOF
|
||||
|
||||
print_success "Deployment summary created: $SUMMARY_FILE"
|
||||
echo ""
|
||||
cat "$SUMMARY_FILE"
|
||||
}
|
||||
|
||||
# Main deployment flow
|
||||
main() {
|
||||
echo "================================================================================"
|
||||
echo "DQN MODEL STAGING DEPLOYMENT"
|
||||
echo "================================================================================"
|
||||
echo ""
|
||||
|
||||
check_prerequisites
|
||||
setup_directories
|
||||
deploy_model
|
||||
register_model_in_db
|
||||
configure_paper_trading
|
||||
validate_deployment
|
||||
create_deployment_summary
|
||||
|
||||
echo ""
|
||||
print_success "✓ DQN deployment to staging completed successfully!"
|
||||
echo ""
|
||||
print_step "Deployment log saved to: $DEPLOYMENT_LOG"
|
||||
print_step "Model location: $STAGING_MODEL"
|
||||
print_step "Configuration: $CONFIG_FILE"
|
||||
echo ""
|
||||
print_step "To start paper trading:"
|
||||
echo " 1. cargo run -p trading_service --release"
|
||||
echo " 2. Monitor: tail -f $LOG_DIR/ml_models.log"
|
||||
echo " 3. Grafana: http://localhost:$GRAFANA_PORT"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Execute main function and log output
|
||||
main 2>&1 | tee "$DEPLOYMENT_LOG"
|
||||
245
scripts/archive/deploy_fp32_runpod.sh
Executable file
245
scripts/archive/deploy_fp32_runpod.sh
Executable file
@@ -0,0 +1,245 @@
|
||||
#!/bin/bash
|
||||
# FP32 Runpod Deployment Script
|
||||
# Generated: 2025-10-23
|
||||
# Purpose: One-command FP32 model training on Runpod
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TEST_DATA="${PROJECT_ROOT}/test_data/ES_FUT_180d.parquet"
|
||||
EPOCHS="${EPOCHS:-50}"
|
||||
MODEL_OUTPUT="${PROJECT_ROOT}/models"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Pre-deployment checks
|
||||
run_preflight_checks() {
|
||||
log_info "Running pre-deployment checks..."
|
||||
|
||||
# Check 1: Test data exists
|
||||
if [[ ! -f "${TEST_DATA}" ]]; then
|
||||
log_error "Test data not found: ${TEST_DATA}"
|
||||
exit 1
|
||||
fi
|
||||
log_success "Test data found: ${TEST_DATA} ($(du -h ${TEST_DATA} | cut -f1))"
|
||||
|
||||
# Check 2: Docker services running
|
||||
if ! docker ps | grep -q foxhunt-postgres; then
|
||||
log_error "Docker services not running. Start with: docker-compose up -d"
|
||||
exit 1
|
||||
fi
|
||||
log_success "Docker services running (postgres, redis, vault)"
|
||||
|
||||
# Check 3: GPU available
|
||||
if ! nvidia-smi &>/dev/null; then
|
||||
log_warn "nvidia-smi not found. GPU training may not work."
|
||||
log_warn "This is expected on CPU-only systems."
|
||||
else
|
||||
GPU_INFO=$(nvidia-smi --query-gpu=name,memory.free --format=csv,noheader)
|
||||
log_success "GPU available: ${GPU_INFO}"
|
||||
fi
|
||||
|
||||
# Check 4: Database migration 045 applied
|
||||
if ! psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt \
|
||||
-c "\dt regime_states" &>/dev/null; then
|
||||
log_error "Database migration 045 not applied. Run: cargo sqlx migrate run"
|
||||
exit 1
|
||||
fi
|
||||
log_success "Database migration 045 applied (regime tables exist)"
|
||||
|
||||
# Check 5: Cargo available
|
||||
if ! command -v cargo &>/dev/null; then
|
||||
log_error "Cargo not found. Install Rust toolchain first."
|
||||
exit 1
|
||||
fi
|
||||
log_success "Cargo available: $(cargo --version)"
|
||||
|
||||
log_success "All pre-flight checks passed ✅"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Build release binaries
|
||||
build_release() {
|
||||
log_info "Building release binaries (this may take 5-10 minutes)..."
|
||||
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
if cargo build --release --package ml --features cuda 2>&1 | tee /tmp/build.log; then
|
||||
log_success "Release build completed successfully"
|
||||
else
|
||||
log_error "Release build failed. Check /tmp/build.log for details."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Train FP32 model
|
||||
train_fp32_model() {
|
||||
log_info "Starting FP32 TFT training (${EPOCHS} epochs)..."
|
||||
log_info "Model: TFT-225 (FP32)"
|
||||
log_info "Dataset: ES.FUT 180 days"
|
||||
log_info "Expected Duration: ~3-5 minutes (RTX 4090) or ~10-15 minutes (RTX 3050 Ti)"
|
||||
echo ""
|
||||
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
# Create models directory if it doesn't exist
|
||||
mkdir -p "${MODEL_OUTPUT}"
|
||||
|
||||
# Record start time
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Run training (NO --use-qat flag for FP32)
|
||||
if cargo run -p ml --example train_tft_parquet --release --features cuda -- \
|
||||
--parquet-file "${TEST_DATA}" \
|
||||
--epochs "${EPOCHS}"; then
|
||||
|
||||
# Record end time
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - START_TIME))
|
||||
DURATION_MIN=$((DURATION / 60))
|
||||
DURATION_SEC=$((DURATION % 60))
|
||||
|
||||
log_success "Training completed in ${DURATION_MIN}m ${DURATION_SEC}s"
|
||||
echo ""
|
||||
|
||||
# Find latest model file
|
||||
LATEST_MODEL=$(ls -t "${MODEL_OUTPUT}"/tft_225_fp32_*.safetensors 2>/dev/null | head -1)
|
||||
if [[ -n "${LATEST_MODEL}" ]]; then
|
||||
MODEL_SIZE=$(du -h "${LATEST_MODEL}" | cut -f1)
|
||||
log_success "Model saved: ${LATEST_MODEL} (${MODEL_SIZE})"
|
||||
else
|
||||
log_warn "Model file not found in ${MODEL_OUTPUT}"
|
||||
fi
|
||||
else
|
||||
log_error "Training failed. Check logs above for details."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Monitor GPU during training
|
||||
monitor_gpu() {
|
||||
log_info "GPU monitoring enabled (press Ctrl+C to stop)"
|
||||
log_info "Watching GPU memory usage every 1 second..."
|
||||
echo ""
|
||||
|
||||
watch -n 1 nvidia-smi
|
||||
}
|
||||
|
||||
# Baseline metrics collection
|
||||
record_baseline_metrics() {
|
||||
log_info "Recording baseline metrics..."
|
||||
|
||||
METRICS_FILE="${PROJECT_ROOT}/FP32_BASELINE_METRICS.md"
|
||||
|
||||
cat > "${METRICS_FILE}" << EOF
|
||||
# FP32 Baseline Metrics (Runpod)
|
||||
|
||||
**Date**: $(date +"%Y-%m-%d %H:%M:%S")
|
||||
**GPU**: $(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo "N/A")
|
||||
**Model**: TFT-225 FP32
|
||||
**Dataset**: ES.FUT 180 days (test_data/ES_FUT_180d.parquet)
|
||||
|
||||
## Training Metrics
|
||||
- **Training Time**: ${DURATION_MIN}m ${DURATION_SEC}s
|
||||
- **Epochs Completed**: ${EPOCHS}
|
||||
- **GPU Memory Peak**: (monitor with nvidia-smi during training)
|
||||
|
||||
## Model Artifacts
|
||||
- **Model Path**: ${LATEST_MODEL}
|
||||
- **Model Size**: ${MODEL_SIZE}
|
||||
|
||||
## Next Steps
|
||||
1. ✅ FP32 model trained successfully
|
||||
2. 🔲 Run inference benchmark: \`cargo test -p ml --release test_tft_inference_latency\`
|
||||
3. 🔲 Compare with baseline RMSE/MAE metrics
|
||||
4. 🔲 Upload to S3/MinIO for production use
|
||||
5. 🔲 Begin QAT Phase 2 (after P0 fixes)
|
||||
|
||||
## Notes
|
||||
- FP32 deployment successful with zero blockers
|
||||
- QAT deferred to Phase 2 (1-2 weeks after P0 fixes)
|
||||
- Expected performance: Sharpe 2.00, Win Rate 60%, Drawdown 15%
|
||||
|
||||
---
|
||||
**Generated by**: scripts/deploy_fp32_runpod.sh
|
||||
EOF
|
||||
|
||||
log_success "Baseline metrics recorded: ${METRICS_FILE}"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo " FP32 Runpod Deployment Script"
|
||||
echo " Foxhunt HFT Trading System"
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
# Run pre-flight checks
|
||||
run_preflight_checks
|
||||
|
||||
# Ask user to confirm
|
||||
read -p "Proceed with FP32 model training? [y/N] " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
log_warn "Deployment cancelled by user"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Build release binaries
|
||||
build_release
|
||||
|
||||
# Train model
|
||||
train_fp32_model
|
||||
|
||||
# Record baseline metrics
|
||||
if [[ -n "${DURATION_MIN}" ]]; then
|
||||
record_baseline_metrics
|
||||
fi
|
||||
|
||||
# Success summary
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
log_success "FP32 DEPLOYMENT COMPLETE ✅"
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
log_info "Next Actions:"
|
||||
echo " 1. Review baseline metrics: cat FP32_BASELINE_METRICS.md"
|
||||
echo " 2. Run inference benchmark: cargo test -p ml --release test_tft_inference_latency"
|
||||
echo " 3. Upload model to S3/MinIO: cargo run -p storage --example upload_model"
|
||||
echo " 4. Begin QAT Phase 2 (after P0 fixes): See RUNPOD_DEPLOYMENT_CHECKLIST.md"
|
||||
echo ""
|
||||
log_info "For GPU monitoring during training, run in separate terminal:"
|
||||
echo " watch -n 1 nvidia-smi"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Execute main function
|
||||
main "$@"
|
||||
352
scripts/archive/deploy_fp32_runpod_test.sh
Executable file
352
scripts/archive/deploy_fp32_runpod_test.sh
Executable file
@@ -0,0 +1,352 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Runpod FP32 TFT-225 Training Test Deployment
|
||||
#
|
||||
# This script deploys a test FP32 TFT-225 model training job to Runpod using spot instances
|
||||
# for cost optimization. It's designed for a single 1-5 minute training run to validate
|
||||
# the deployment before scaling to production.
|
||||
#
|
||||
# COST TARGET: <$0.10 total (spot pricing with auto-termination)
|
||||
#
|
||||
# Requirements:
|
||||
# - runpodctl v1.14.6+ installed
|
||||
# - RUNPOD_API_KEY environment variable set
|
||||
# - test_data/ES_FUT_180d.parquet file exists (2.9MB)
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/deploy_fp32_runpod_test.sh [--dry-run]
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ==================== CONFIGURATION ====================
|
||||
|
||||
# Runpod Configuration
|
||||
GPU_TYPE="NVIDIA GeForce RTX 3090" # Cheapest GPU with 24GB VRAM (>4GB required)
|
||||
GPU_COUNT=1
|
||||
COST_CEILING=0.25 # Max $0.25/hr (spot RTX 3090 ~$0.14/hr)
|
||||
CONTAINER_IMAGE="runpod/pytorch:2.1.0-py3.10-cuda11.8.0-devel-ubuntu22.04"
|
||||
POD_NAME="foxhunt-fp32-tft-test-$(date +%s)"
|
||||
|
||||
# Training Configuration
|
||||
PARQUET_FILE="test_data/ES_FUT_180d.parquet"
|
||||
EPOCHS=10 # Reduced for quick test (vs 50 production)
|
||||
TRAINING_TIMEOUT=600 # 10 minutes max (safety timeout)
|
||||
|
||||
# Directories
|
||||
REMOTE_WORKSPACE="/workspace/foxhunt"
|
||||
REMOTE_DATA_DIR="${REMOTE_WORKSPACE}/test_data"
|
||||
REMOTE_OUTPUT_DIR="${REMOTE_WORKSPACE}/models"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# ==================== FUNCTIONS ====================
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $*"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $*"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $*"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $*"
|
||||
}
|
||||
|
||||
check_prerequisites() {
|
||||
log_info "Checking prerequisites..."
|
||||
|
||||
# Check runpodctl
|
||||
if ! command -v runpodctl &> /dev/null; then
|
||||
log_error "runpodctl not found. Install: brew install runpod/runpodctl/runpodctl"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local version=$(runpodctl --version | grep -oP 'v\K[0-9.]+' || echo "0.0.0")
|
||||
log_info "runpodctl version: v${version}"
|
||||
|
||||
# Check API key
|
||||
if [[ -z "${RUNPOD_API_KEY:-}" ]]; then
|
||||
log_error "RUNPOD_API_KEY environment variable not set"
|
||||
log_info "Get your API key from: https://www.runpod.io/console/user/settings"
|
||||
log_info "Then run: export RUNPOD_API_KEY='your-key-here'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Configure runpodctl
|
||||
log_info "Configuring runpodctl..."
|
||||
runpodctl config --apiKey="${RUNPOD_API_KEY}" 2>/dev/null || {
|
||||
log_error "Failed to configure runpodctl"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check training data
|
||||
if [[ ! -f "${PARQUET_FILE}" ]]; then
|
||||
log_error "Training data not found: ${PARQUET_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local file_size=$(du -h "${PARQUET_FILE}" | cut -f1)
|
||||
log_info "Training data: ${PARQUET_FILE} (${file_size})"
|
||||
|
||||
log_success "Prerequisites check passed"
|
||||
}
|
||||
|
||||
estimate_cost() {
|
||||
log_info "Cost Estimation:"
|
||||
echo " GPU Type: ${GPU_TYPE}"
|
||||
echo " Spot Price Ceiling: \$${COST_CEILING}/hr"
|
||||
echo " Expected Spot Rate: ~\$0.14/hr (typical RTX 3090 spot)"
|
||||
echo " Training Duration: ~1-5 minutes"
|
||||
echo " Storage (1GB): ~\$0.0002/hr"
|
||||
echo ""
|
||||
echo " ESTIMATED COST: \$0.02 - \$0.10 per run"
|
||||
echo " MAX COST (10 min): \$0.04 (with auto-termination)"
|
||||
echo ""
|
||||
log_warning "Spot instances can be interrupted. Save checkpoints frequently."
|
||||
}
|
||||
|
||||
create_pod() {
|
||||
log_info "Creating Runpod spot instance..."
|
||||
|
||||
# Note: runpodctl doesn't have a direct spot flag in the create command
|
||||
# Spot pricing is automatically used when --cost is specified and available
|
||||
local pod_output=$(runpodctl create pod \
|
||||
--name "${POD_NAME}" \
|
||||
--gpuType "${GPU_TYPE}" \
|
||||
--gpuCount ${GPU_COUNT} \
|
||||
--cost ${COST_CEILING} \
|
||||
--communityCloud \
|
||||
--imageName "${CONTAINER_IMAGE}" \
|
||||
--containerDiskSize 20 \
|
||||
--volumeSize 5 \
|
||||
--volumePath "/workspace" \
|
||||
--env "CUDA_VISIBLE_DEVICES=0" \
|
||||
--env "PYTHONUNBUFFERED=1" \
|
||||
--ports "8888/http" \
|
||||
--startSSH 2>&1) || {
|
||||
log_error "Failed to create pod"
|
||||
echo "${pod_output}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Extract pod ID from output
|
||||
POD_ID=$(echo "${pod_output}" | grep -oP '(?<=id: )[a-z0-9]+' || echo "")
|
||||
|
||||
if [[ -z "${POD_ID}" ]]; then
|
||||
log_error "Failed to extract pod ID from output:"
|
||||
echo "${pod_output}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_success "Pod created: ${POD_ID}"
|
||||
echo "${POD_ID}" > /tmp/foxhunt_runpod_test_id.txt
|
||||
|
||||
# Wait for pod to be ready
|
||||
log_info "Waiting for pod to be ready (max 120s)..."
|
||||
local wait_count=0
|
||||
while [[ $wait_count -lt 24 ]]; do
|
||||
local status=$(runpodctl get pod "${POD_ID}" 2>/dev/null | grep -i "status" || echo "UNKNOWN")
|
||||
if echo "${status}" | grep -qi "running"; then
|
||||
log_success "Pod is running"
|
||||
sleep 5 # Additional wait for SSH/filesystem
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
wait_count=$((wait_count + 1))
|
||||
done
|
||||
|
||||
log_error "Pod failed to become ready within 120s"
|
||||
cleanup_pod
|
||||
exit 1
|
||||
}
|
||||
|
||||
upload_data() {
|
||||
log_info "Uploading training data and code..."
|
||||
|
||||
# Create a temporary directory with all necessary files
|
||||
local tmp_dir=$(mktemp -d)
|
||||
mkdir -p "${tmp_dir}/foxhunt/test_data"
|
||||
|
||||
# Copy training data
|
||||
cp "${PARQUET_FILE}" "${tmp_dir}/foxhunt/test_data/"
|
||||
|
||||
# Create a simplified training script
|
||||
cat > "${tmp_dir}/foxhunt/train_remote.sh" << 'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
cd /workspace/foxhunt
|
||||
|
||||
echo "[INFO] Installing Rust and Cargo..."
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
source $HOME/.cargo/env
|
||||
|
||||
echo "[INFO] Cloning Foxhunt repository..."
|
||||
git clone https://github.com/yourusername/foxhunt.git repo || {
|
||||
echo "[ERROR] Failed to clone repository. Using local files."
|
||||
}
|
||||
|
||||
# If git clone failed, we'll use the uploaded data
|
||||
if [[ -d "repo" ]]; then
|
||||
cd repo
|
||||
else
|
||||
cd /workspace/foxhunt
|
||||
fi
|
||||
|
||||
echo "[INFO] Starting FP32 TFT-225 training..."
|
||||
echo "[INFO] Training data: test_data/ES_FUT_180d.parquet"
|
||||
echo "[INFO] Epochs: ${EPOCHS:-10}"
|
||||
echo "[INFO] GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader)"
|
||||
|
||||
# Run training
|
||||
cargo run -p ml --example train_tft_parquet --release --features cuda -- \
|
||||
--parquet-file test_data/ES_FUT_180d.parquet \
|
||||
--epochs ${EPOCHS:-10} \
|
||||
2>&1 | tee /workspace/foxhunt/training.log
|
||||
|
||||
echo "[INFO] Training complete!"
|
||||
echo "[INFO] Logs saved to: /workspace/foxhunt/training.log"
|
||||
EOF
|
||||
|
||||
chmod +x "${tmp_dir}/foxhunt/train_remote.sh"
|
||||
|
||||
# Upload via runpodctl send
|
||||
log_info "Initiating file transfer..."
|
||||
local send_output=$(runpodctl send "${tmp_dir}/foxhunt" 2>&1)
|
||||
local transfer_code=$(echo "${send_output}" | grep -oP '[0-9]{4}-[a-z]+-[a-z]+-[a-z]+' || echo "")
|
||||
|
||||
if [[ -z "${transfer_code}" ]]; then
|
||||
log_error "Failed to get transfer code"
|
||||
rm -rf "${tmp_dir}"
|
||||
cleanup_pod
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Transfer code: ${transfer_code}"
|
||||
log_info "Run this command on the pod to receive files:"
|
||||
echo " runpodctl receive ${transfer_code}"
|
||||
|
||||
# Attempt to SSH and receive files
|
||||
log_info "Attempting to receive files on pod..."
|
||||
# Note: This requires SSH access to the pod, which may not be immediate
|
||||
# In practice, you'd SSH into the pod manually and run the receive command
|
||||
|
||||
rm -rf "${tmp_dir}"
|
||||
|
||||
log_warning "Manual step required:"
|
||||
log_warning " 1. SSH into pod: runpodctl ssh ${POD_ID}"
|
||||
log_warning " 2. Run: runpodctl receive ${transfer_code}"
|
||||
log_warning " 3. Run: bash /workspace/foxhunt/train_remote.sh"
|
||||
}
|
||||
|
||||
run_training() {
|
||||
log_info "Training must be started manually via SSH"
|
||||
log_info "SSH command: runpodctl ssh ${POD_ID}"
|
||||
log_info "Then run: bash /workspace/foxhunt/train_remote.sh"
|
||||
log_info ""
|
||||
log_info "Training will:"
|
||||
log_info " - Install Rust/Cargo"
|
||||
log_info " - Clone Foxhunt repo (or use uploaded files)"
|
||||
log_info " - Run FP32 TFT-225 training for ${EPOCHS} epochs"
|
||||
log_info " - Save logs to /workspace/foxhunt/training.log"
|
||||
}
|
||||
|
||||
download_results() {
|
||||
log_info "To download results after training:"
|
||||
echo " 1. On pod: runpodctl send /workspace/foxhunt/training.log"
|
||||
echo " 2. Locally: runpodctl receive <code-from-step-1>"
|
||||
echo " 3. On pod: runpodctl send /workspace/foxhunt/models/"
|
||||
echo " 4. Locally: runpodctl receive <code-from-step-3>"
|
||||
}
|
||||
|
||||
cleanup_pod() {
|
||||
if [[ -n "${POD_ID:-}" ]]; then
|
||||
log_info "Terminating pod: ${POD_ID}"
|
||||
runpodctl remove pod "${POD_ID}" 2>/dev/null || {
|
||||
log_warning "Failed to terminate pod. Please terminate manually:"
|
||||
echo " runpodctl remove pod ${POD_ID}"
|
||||
}
|
||||
log_success "Pod terminated"
|
||||
rm -f /tmp/foxhunt_runpod_test_id.txt
|
||||
fi
|
||||
}
|
||||
|
||||
# ==================== MAIN ====================
|
||||
|
||||
main() {
|
||||
local dry_run=false
|
||||
|
||||
# Parse arguments
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--dry-run)
|
||||
dry_run=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [--dry-run]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --dry-run Show cost estimate and exit without creating pod"
|
||||
echo " --help Show this help message"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
log_info "Foxhunt FP32 TFT-225 Runpod Test Deployment"
|
||||
echo ""
|
||||
|
||||
check_prerequisites
|
||||
echo ""
|
||||
|
||||
estimate_cost
|
||||
|
||||
if [[ "$dry_run" == "true" ]]; then
|
||||
log_info "Dry run complete. No pod created."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
read -p "Proceed with deployment? (yes/no): " confirm
|
||||
if [[ "$confirm" != "yes" ]]; then
|
||||
log_info "Deployment cancelled"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
create_pod
|
||||
echo ""
|
||||
|
||||
upload_data
|
||||
echo ""
|
||||
|
||||
run_training
|
||||
echo ""
|
||||
|
||||
download_results
|
||||
echo ""
|
||||
|
||||
log_warning "IMPORTANT: Remember to terminate the pod when done to avoid charges!"
|
||||
echo " Terminate now: runpodctl remove pod ${POD_ID}"
|
||||
echo " Or save pod ID for later: ${POD_ID}"
|
||||
echo ""
|
||||
|
||||
log_info "Deployment complete!"
|
||||
}
|
||||
|
||||
# Trap to cleanup on exit
|
||||
trap cleanup_pod EXIT INT TERM
|
||||
|
||||
main "$@"
|
||||
230
scripts/archive/deploy_paper_trading.sh
Executable file
230
scripts/archive/deploy_paper_trading.sh
Executable file
@@ -0,0 +1,230 @@
|
||||
#!/bin/bash
|
||||
# ================================================================================================
|
||||
# Paper Trading Deployment Script
|
||||
# ================================================================================================
|
||||
# Purpose: Deploy 3-model ensemble (DQN + 2x PPO) to paper trading
|
||||
# Usage: bash scripts/deploy_paper_trading.sh
|
||||
# ================================================================================================
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# ================================================================================================
|
||||
# CONFIGURATION
|
||||
# ================================================================================================
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
CONFIG_FILE="$PROJECT_ROOT/config/paper_trading_config.yaml"
|
||||
CHECKPOINT_DIR="$PROJECT_ROOT/ml/trained_models/production"
|
||||
|
||||
# Service endpoints
|
||||
API_GATEWAY="localhost:50051"
|
||||
TRADING_SERVICE="localhost:50052"
|
||||
ML_TRAINING_SERVICE="localhost:50054"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# ================================================================================================
|
||||
# HELPER FUNCTIONS
|
||||
# ================================================================================================
|
||||
print_header() {
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}$1${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
}
|
||||
|
||||
print_success() {
|
||||
echo -e "${GREEN}✅ $1${NC}"
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo -e "${RED}❌ $1${NC}"
|
||||
}
|
||||
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}⚠️ $1${NC}"
|
||||
}
|
||||
|
||||
print_info() {
|
||||
echo -e "${BLUE}ℹ️ $1${NC}"
|
||||
}
|
||||
|
||||
check_command() {
|
||||
if ! command -v "$1" &> /dev/null; then
|
||||
print_error "Required command '$1' not found. Please install it first."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ================================================================================================
|
||||
# PRE-FLIGHT CHECKS
|
||||
# ================================================================================================
|
||||
print_header "Pre-Flight Checks"
|
||||
|
||||
# Check required commands
|
||||
print_info "Checking required commands..."
|
||||
check_command docker
|
||||
check_command docker-compose
|
||||
check_command psql
|
||||
check_command curl
|
||||
print_success "All required commands available"
|
||||
|
||||
# Check configuration file exists
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
print_error "Configuration file not found: $CONFIG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
print_success "Configuration file found: $CONFIG_FILE"
|
||||
|
||||
# Check checkpoint directory exists
|
||||
if [ ! -d "$CHECKPOINT_DIR" ]; then
|
||||
print_error "Checkpoint directory not found: $CHECKPOINT_DIR"
|
||||
exit 1
|
||||
fi
|
||||
print_success "Checkpoint directory found: $CHECKPOINT_DIR"
|
||||
|
||||
# ================================================================================================
|
||||
# CHECKPOINT VERIFICATION
|
||||
# ================================================================================================
|
||||
print_header "Checkpoint Verification"
|
||||
|
||||
print_info "Verifying model checkpoints..."
|
||||
|
||||
# DQN epoch 30
|
||||
DQN_CHECKPOINT="$CHECKPOINT_DIR/dqn/dqn_epoch_30.safetensors"
|
||||
if [ -f "$DQN_CHECKPOINT" ]; then
|
||||
SIZE=$(du -h "$DQN_CHECKPOINT" | cut -f1)
|
||||
print_success "DQN epoch 30: $SIZE"
|
||||
else
|
||||
print_error "DQN checkpoint not found: $DQN_CHECKPOINT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# PPO epoch 130 (actor + critic)
|
||||
PPO_130_ACTOR="$CHECKPOINT_DIR/ppo/ppo_actor_epoch_130.safetensors"
|
||||
PPO_130_CRITIC="$CHECKPOINT_DIR/ppo/ppo_critic_epoch_130.safetensors"
|
||||
if [ -f "$PPO_130_ACTOR" ] && [ -f "$PPO_130_CRITIC" ]; then
|
||||
SIZE_ACTOR=$(du -h "$PPO_130_ACTOR" | cut -f1)
|
||||
SIZE_CRITIC=$(du -h "$PPO_130_CRITIC" | cut -f1)
|
||||
print_success "PPO epoch 130: actor=$SIZE_ACTOR, critic=$SIZE_CRITIC"
|
||||
else
|
||||
print_error "PPO epoch 130 checkpoints not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# PPO epoch 420 (actor + critic)
|
||||
PPO_420_ACTOR="$CHECKPOINT_DIR/ppo/ppo_actor_epoch_420.safetensors"
|
||||
PPO_420_CRITIC="$CHECKPOINT_DIR/ppo/ppo_critic_epoch_420.safetensors"
|
||||
if [ -f "$PPO_420_ACTOR" ] && [ -f "$PPO_420_CRITIC" ]; then
|
||||
SIZE_ACTOR=$(du -h "$PPO_420_ACTOR" | cut -f1)
|
||||
SIZE_CRITIC=$(du -h "$PPO_420_CRITIC" | cut -f1)
|
||||
print_success "PPO epoch 420: actor=$SIZE_ACTOR, critic=$SIZE_CRITIC"
|
||||
else
|
||||
print_error "PPO epoch 420 checkpoints not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ================================================================================================
|
||||
# SERVICE HEALTH CHECKS
|
||||
# ================================================================================================
|
||||
print_header "Service Health Checks"
|
||||
|
||||
print_info "Checking Docker services..."
|
||||
if ! docker-compose ps | grep -q "Up"; then
|
||||
print_error "Docker services are not running. Start with: docker-compose up -d"
|
||||
exit 1
|
||||
fi
|
||||
print_success "Docker services running"
|
||||
|
||||
# Trading Service HTTP health check (gRPC health probe may not be installed)
|
||||
print_info "Checking Trading Service (port 8081 HTTP)..."
|
||||
if curl -s http://localhost:8081/health > /dev/null 2>&1; then
|
||||
print_success "Trading Service healthy (HTTP)"
|
||||
else
|
||||
print_warning "Trading Service HTTP health check failed (may be gRPC-only)"
|
||||
fi
|
||||
|
||||
# PostgreSQL health check
|
||||
print_info "Checking PostgreSQL..."
|
||||
if psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c "SELECT 1" > /dev/null 2>&1; then
|
||||
print_success "PostgreSQL healthy"
|
||||
else
|
||||
print_error "PostgreSQL unhealthy or not responding"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Redis health check
|
||||
print_info "Checking Redis..."
|
||||
if redis-cli -h localhost -p 6379 PING > /dev/null 2>&1; then
|
||||
print_success "Redis healthy"
|
||||
else
|
||||
print_warning "Redis not responding (non-critical)"
|
||||
fi
|
||||
|
||||
# Prometheus health check
|
||||
print_info "Checking Prometheus..."
|
||||
if curl -s http://localhost:9090/api/v1/targets > /dev/null 2>&1; then
|
||||
print_success "Prometheus healthy"
|
||||
else
|
||||
print_warning "Prometheus not responding (non-critical)"
|
||||
fi
|
||||
|
||||
# Grafana health check
|
||||
print_info "Checking Grafana..."
|
||||
if curl -s http://localhost:3000/api/health > /dev/null 2>&1; then
|
||||
print_success "Grafana healthy"
|
||||
else
|
||||
print_warning "Grafana not responding (non-critical)"
|
||||
fi
|
||||
|
||||
# ================================================================================================
|
||||
# DATABASE TABLE VERIFICATION
|
||||
# ================================================================================================
|
||||
print_header "Database Table Verification"
|
||||
|
||||
print_info "Checking ensemble prediction tables..."
|
||||
TABLE_COUNT=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c "SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public' AND tablename IN ('ensemble_predictions', 'model_performance_attribution', 'ab_test_experiments')" 2>/dev/null | tr -d ' ')
|
||||
|
||||
if [ "$TABLE_COUNT" = "3" ]; then
|
||||
print_success "All ensemble tables exist (3/3)"
|
||||
else
|
||||
print_warning "Only $TABLE_COUNT/3 ensemble tables found"
|
||||
print_info "Tables may need to be created manually from migration 022"
|
||||
fi
|
||||
|
||||
# ================================================================================================
|
||||
# DEPLOYMENT SUMMARY
|
||||
# ================================================================================================
|
||||
print_header "Deployment Summary"
|
||||
|
||||
echo ""
|
||||
print_success "Paper trading infrastructure verified!"
|
||||
echo ""
|
||||
print_info "Configuration Details:"
|
||||
echo " - Config file: $CONFIG_FILE"
|
||||
echo " - Checkpoints: $CHECKPOINT_DIR"
|
||||
echo " - Virtual capital: \$100,000"
|
||||
echo " - Symbols: ES.FUT, NQ.FUT"
|
||||
echo " - Models: DQN epoch 30, PPO epoch 130, PPO epoch 420"
|
||||
echo " - Risk limits: Max position \$10K, max daily loss \$2K"
|
||||
echo ""
|
||||
print_info "Next Steps:"
|
||||
echo " 1. Run smoke test: bash tests/paper_trading_smoke_test.sh"
|
||||
echo " 2. Monitor Grafana: http://localhost:3000/d/ensemble-ml-prod"
|
||||
echo " 3. Check predictions: psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt"
|
||||
echo " SQL: SELECT COUNT(*) FROM ensemble_predictions WHERE timestamp > NOW() - INTERVAL '1 hour';"
|
||||
echo " 4. Monitor logs: docker logs foxhunt-trading-service -f"
|
||||
echo ""
|
||||
print_info "Phase 1 Success Criteria (7 days):"
|
||||
echo " - Sharpe ratio > 1.5"
|
||||
echo " - Win rate > 52%"
|
||||
echo " - Max drawdown < 10%"
|
||||
echo " - Simulated P&L > \$10,000"
|
||||
echo " - Zero model errors"
|
||||
echo " - Latency P99 < 50μs"
|
||||
echo ""
|
||||
print_header "Deployment Ready"
|
||||
134
scripts/archive/deploy_runpod.sh
Executable file
134
scripts/archive/deploy_runpod.sh
Executable file
@@ -0,0 +1,134 @@
|
||||
#!/bin/bash
|
||||
# deploy_runpod.sh - Complete Runpod deployment workflow
|
||||
#
|
||||
# 100% RUNPOD INFRASTRUCTURE - Zero Cloud Dependencies
|
||||
# - All storage on Runpod infrastructure ($0.10/GB/month)
|
||||
# - Credentials are Runpod User ID + Runpod API Key
|
||||
# - Endpoint is Runpod's S3-compatible API (https://s3api-<datacenter>.runpod.io/)
|
||||
# - Tool: S3-compatible CLI (standard API interface for Runpod storage)
|
||||
# - GPU: Tesla V100-PCIE-16GB (16GB VRAM, $0.10/hr)
|
||||
# - Docker: PRIVATE repo (jgrusewski/foxhunt-runpod:latest)
|
||||
|
||||
set -e
|
||||
|
||||
echo "========================================="
|
||||
echo "Foxhunt Runpod Deployment Workflow"
|
||||
echo "100% RUNPOD INFRASTRUCTURE"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
echo "🔒 PRIVACY & SECURITY CHECK:"
|
||||
echo " Before proceeding, verify:"
|
||||
echo " 1. Docker Hub repo 'jgrusewski/foxhunt-runpod' is set to PRIVATE"
|
||||
echo " 2. Runpod Network Volume has access controls enabled"
|
||||
echo " 3. No credentials are baked into Docker image or scripts"
|
||||
echo " 4. Zero cloud dependencies (100% Runpod infrastructure)"
|
||||
echo " 5. All 9 data files ready for upload (ES.FUT, NQ.FUT, 6E.FUT, ZN.FUT)"
|
||||
echo ""
|
||||
read -p "Continue? (y/n) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Deployment cancelled. Ensure privacy checks pass first!"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Configuration
|
||||
DOCKER_IMAGE="jgrusewski/foxhunt-runpod:latest"
|
||||
RUNPOD_S3_ENDPOINT="https://s3api-us-ca-2.runpod.io/"
|
||||
RUNPOD_S3_REGION="US-CA-2"
|
||||
RUNPOD_NETWORK_VOLUME_ID="your-network-volume-id"
|
||||
|
||||
echo "Step 1: Build Docker image"
|
||||
echo "========================================="
|
||||
docker build -f Dockerfile.runpod -t "$DOCKER_IMAGE" .
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Docker build failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Step 2: Verify Docker Hub repo is PRIVATE"
|
||||
echo "========================================="
|
||||
echo "IMPORTANT: Before pushing, verify repo is PRIVATE:"
|
||||
echo " 1. Go to https://hub.docker.com/repository/docker/jgrusewski/foxhunt-runpod"
|
||||
echo " 2. Click 'Settings'"
|
||||
echo " 3. Ensure 'Visibility' is set to 'Private'"
|
||||
echo ""
|
||||
read -p "Is the repo PRIVATE? (y/n) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "ERROR: Make your Docker Hub repo PRIVATE before proceeding!"
|
||||
echo "Visit: https://hub.docker.com/repository/docker/jgrusewski/foxhunt-runpod/settings"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 3: Push Docker image to Docker Hub"
|
||||
echo "========================================="
|
||||
docker push "$DOCKER_IMAGE"
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Docker push failed"
|
||||
echo "Make sure you're logged in: docker login"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Step 4: Upload binaries & data to Runpod storage"
|
||||
echo "========================================="
|
||||
./scripts/upload_to_runpod_s3.sh
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Binary/data upload failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Verifying all data files uploaded:"
|
||||
echo " - ES.FUT_180d.parquet (2.9MB)"
|
||||
echo " - NQ.FUT_180d.parquet (4.4MB)"
|
||||
echo " - 6E.FUT_180d.parquet (2.8MB)"
|
||||
echo " - ZN.FUT_180d.parquet (2.8MB)"
|
||||
echo " + 5 additional parquet files"
|
||||
|
||||
echo "========================================="
|
||||
echo "Deployment Preparation Complete!"
|
||||
echo "100% RUNPOD INFRASTRUCTURE - ZERO CLOUD DEPENDENCIES"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
echo "Docker image: $DOCKER_IMAGE (PRIVATE repo)"
|
||||
echo "Runpod S3 endpoint: $RUNPOD_S3_ENDPOINT"
|
||||
echo "Runpod S3 region: $RUNPOD_S3_REGION"
|
||||
echo "Network Volume ID: $RUNPOD_NETWORK_VOLUME_ID"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo "1. Go to Runpod console: https://console.runpod.io"
|
||||
echo "2. Create a pod with the following configuration:"
|
||||
echo ""
|
||||
echo " GPU Type: Tesla V100-PCIE-16GB (16GB VRAM, $0.10/hr)"
|
||||
echo " Docker Image: $DOCKER_IMAGE (PRIVATE repo)"
|
||||
echo " Container Disk: 20GB minimum"
|
||||
echo " Volume Path: /runpod-volume (optional - we use S3 API)"
|
||||
echo ""
|
||||
echo " Environment Variables (100% Runpod infrastructure):"
|
||||
echo " - RUNPOD_S3_ENDPOINT=$RUNPOD_S3_ENDPOINT"
|
||||
echo " - RUNPOD_S3_REGION=$RUNPOD_S3_REGION"
|
||||
echo " - RUNPOD_S3_BUCKET=$RUNPOD_NETWORK_VOLUME_ID"
|
||||
echo " - RUNPOD_ACCESS_KEY_ID=<your-runpod-user-id>"
|
||||
echo " - RUNPOD_SECRET_ACCESS_KEY=<your-runpod-api-key>"
|
||||
echo " - BINARY_NAME=train_tft_parquet"
|
||||
echo " - DATA_NAME=ES_FUT_180d.parquet"
|
||||
echo ""
|
||||
echo " Container Arguments:"
|
||||
echo " --parquet-file /workspace/test_data/ES_FUT_180d.parquet"
|
||||
echo " --epochs 50"
|
||||
echo " --use-int8"
|
||||
echo ""
|
||||
echo "3. Start the pod and monitor logs"
|
||||
echo "4. Download trained models from Runpod storage after completion"
|
||||
echo ""
|
||||
echo "To download results (using S3-compatible CLI for Runpod storage):"
|
||||
echo " s3cmd sync --endpoint-url $RUNPOD_S3_ENDPOINT \\"
|
||||
echo " --region $RUNPOD_S3_REGION \\"
|
||||
echo " s3://${RUNPOD_NETWORK_VOLUME_ID}/foxhunt/models/ ./models/"
|
||||
echo ""
|
||||
echo "========================================="
|
||||
673
scripts/archive/deploy_runpod_graphql.py
Executable file
673
scripts/archive/deploy_runpod_graphql.py
Executable file
@@ -0,0 +1,673 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Runpod GraphQL API Deployment Script for Foxhunt FP32 Training
|
||||
|
||||
This script replaces runpodctl with direct GraphQL/REST API calls for more reliable deployments.
|
||||
It supports both smoke tests (1 epoch) and full training (50 epochs).
|
||||
|
||||
Requirements:
|
||||
pip install requests
|
||||
|
||||
Usage:
|
||||
# Smoke test (1 epoch, small dataset)
|
||||
./scripts/deploy_runpod_graphql.py --smoke-test
|
||||
|
||||
# Full training (50 epochs, 180 days)
|
||||
./scripts/deploy_runpod_graphql.py --full-training
|
||||
|
||||
# Custom configuration
|
||||
./scripts/deploy_runpod_graphql.py --epochs 10 --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet
|
||||
|
||||
# List available GPUs
|
||||
./scripts/deploy_runpod_graphql.py --list-gpus
|
||||
|
||||
# Get Docker credential ID
|
||||
./scripts/deploy_runpod_graphql.py --get-credentials
|
||||
|
||||
Environment Variables:
|
||||
RUNPOD_API_KEY: Your Runpod API key (required)
|
||||
|
||||
Author: Claude Code
|
||||
Date: 2025-10-24
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import argparse
|
||||
from typing import Dict, List, Optional, Any
|
||||
import requests
|
||||
|
||||
|
||||
class RunpodAPI:
|
||||
"""Runpod API client using GraphQL and REST endpoints."""
|
||||
|
||||
GRAPHQL_ENDPOINT = "https://api.runpod.io/graphql"
|
||||
REST_ENDPOINT = "https://api.runpod.io/v2"
|
||||
|
||||
def __init__(self, api_key: str):
|
||||
"""Initialize Runpod API client.
|
||||
|
||||
Args:
|
||||
api_key: Runpod API key
|
||||
"""
|
||||
if not api_key:
|
||||
raise ValueError("RUNPOD_API_KEY is required")
|
||||
|
||||
self.api_key = api_key
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
"Content-Type": "application/json"
|
||||
})
|
||||
|
||||
def _graphql_request(self, query: str, variables: Optional[Dict] = None) -> Dict:
|
||||
"""Execute a GraphQL query.
|
||||
|
||||
Args:
|
||||
query: GraphQL query string
|
||||
variables: Optional query variables
|
||||
|
||||
Returns:
|
||||
Response data dictionary
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the GraphQL request fails
|
||||
"""
|
||||
payload = {"query": query}
|
||||
if variables:
|
||||
payload["variables"] = variables
|
||||
|
||||
response = self.session.post(
|
||||
self.GRAPHQL_ENDPOINT,
|
||||
headers={"api-key": self.api_key},
|
||||
json=payload,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(f"GraphQL request failed: {response.status_code} - {response.text}")
|
||||
|
||||
data = response.json()
|
||||
|
||||
if "errors" in data:
|
||||
errors = data["errors"]
|
||||
error_messages = [err.get("message", str(err)) for err in errors]
|
||||
raise RuntimeError(f"GraphQL errors: {', '.join(error_messages)}")
|
||||
|
||||
return data.get("data", {})
|
||||
|
||||
def _rest_request(self, method: str, endpoint: str, data: Optional[Dict] = None) -> Dict:
|
||||
"""Execute a REST API request.
|
||||
|
||||
Args:
|
||||
method: HTTP method (GET, POST, DELETE, etc.)
|
||||
endpoint: API endpoint path
|
||||
data: Optional request body data
|
||||
|
||||
Returns:
|
||||
Response data dictionary
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the REST request fails
|
||||
"""
|
||||
url = f"{self.REST_ENDPOINT}/{endpoint}"
|
||||
|
||||
response = self.session.request(
|
||||
method,
|
||||
url,
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json=data,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code not in (200, 201):
|
||||
raise RuntimeError(f"REST request failed: {response.status_code} - {response.text}")
|
||||
|
||||
return response.json()
|
||||
|
||||
def get_docker_credential_id(self, name: str = "Docker") -> Optional[str]:
|
||||
"""Get Docker registry credential ID by name.
|
||||
|
||||
Args:
|
||||
name: Credential name (default: "Docker")
|
||||
|
||||
Returns:
|
||||
Credential ID or None if not found
|
||||
"""
|
||||
query = """
|
||||
query {
|
||||
myself {
|
||||
containerRegistryAuths {
|
||||
id
|
||||
name
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
try:
|
||||
data = self._graphql_request(query)
|
||||
auths = data.get("myself", {}).get("containerRegistryAuths", [])
|
||||
|
||||
for auth in auths:
|
||||
if auth.get("name") == name:
|
||||
return auth.get("id")
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to get Docker credentials: {e}")
|
||||
return None
|
||||
|
||||
def list_available_gpus(self) -> List[Dict]:
|
||||
"""List available GPU types.
|
||||
|
||||
Returns:
|
||||
List of GPU type dictionaries with id, name, and pricing
|
||||
"""
|
||||
query = """
|
||||
query {
|
||||
gpuTypes {
|
||||
id
|
||||
displayName
|
||||
memoryInGb
|
||||
secureCloud
|
||||
communityCloud
|
||||
lowestPrice(input: { gpuCount: 1 }) {
|
||||
minimumBidPrice
|
||||
uninterruptablePrice
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
data = self._graphql_request(query)
|
||||
return data.get("gpuTypes", [])
|
||||
|
||||
def create_pod_rest(
|
||||
self,
|
||||
name: str,
|
||||
image_name: str,
|
||||
gpu_type_id: str,
|
||||
docker_start_cmd: List[str],
|
||||
network_volume_id: Optional[str] = None,
|
||||
container_registry_auth_id: Optional[str] = None,
|
||||
env: Optional[Dict[str, str]] = None,
|
||||
cloud_type: str = "SECURE",
|
||||
gpu_count: int = 1,
|
||||
container_disk_in_gb: int = 50,
|
||||
volume_mount_path: str = "/workspace",
|
||||
ports: Optional[List[str]] = None
|
||||
) -> Dict:
|
||||
"""Create a pod using the REST API (better support for dockerStartCmd).
|
||||
|
||||
Args:
|
||||
name: Pod name
|
||||
image_name: Docker image name
|
||||
gpu_type_id: GPU type ID
|
||||
docker_start_cmd: Docker CMD override (list of strings)
|
||||
network_volume_id: Network volume ID (optional)
|
||||
container_registry_auth_id: Docker credential ID (optional)
|
||||
env: Environment variables (optional)
|
||||
cloud_type: "SECURE" or "COMMUNITY" (default: SECURE)
|
||||
gpu_count: Number of GPUs (default: 1)
|
||||
container_disk_in_gb: Container disk size (default: 50GB)
|
||||
volume_mount_path: Volume mount path (default: /workspace)
|
||||
ports: Exposed ports (optional)
|
||||
|
||||
Returns:
|
||||
Pod creation response with pod ID
|
||||
"""
|
||||
payload = {
|
||||
"name": name,
|
||||
"imageName": image_name,
|
||||
"gpuTypeIds": [gpu_type_id],
|
||||
"cloudType": cloud_type,
|
||||
"gpuCount": gpu_count,
|
||||
"containerDiskInGb": container_disk_in_gb,
|
||||
"volumeMountPath": volume_mount_path,
|
||||
"dockerStartCmd": docker_start_cmd,
|
||||
}
|
||||
|
||||
# Add optional parameters
|
||||
if network_volume_id:
|
||||
payload["networkVolumeId"] = network_volume_id
|
||||
|
||||
if container_registry_auth_id:
|
||||
payload["containerRegistryAuthId"] = container_registry_auth_id
|
||||
|
||||
if env:
|
||||
payload["env"] = env
|
||||
|
||||
if ports:
|
||||
payload["ports"] = ports
|
||||
|
||||
return self._rest_request("POST", "pods", payload)
|
||||
|
||||
def get_pod_status(self, pod_id: str) -> Dict:
|
||||
"""Get pod status.
|
||||
|
||||
Args:
|
||||
pod_id: Pod ID
|
||||
|
||||
Returns:
|
||||
Pod status dictionary
|
||||
"""
|
||||
query = """
|
||||
query GetPod($podId: String!) {
|
||||
pod(input: { podId: $podId }) {
|
||||
id
|
||||
name
|
||||
desiredStatus
|
||||
imageName
|
||||
costPerHr
|
||||
gpuCount
|
||||
machine {
|
||||
podHostId
|
||||
}
|
||||
runtime {
|
||||
uptimeInSeconds
|
||||
ports {
|
||||
ip
|
||||
isIpPublic
|
||||
privatePort
|
||||
publicPort
|
||||
type
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
data = self._graphql_request(query, {"podId": pod_id})
|
||||
return data.get("pod", {})
|
||||
|
||||
def terminate_pod(self, pod_id: str) -> Dict:
|
||||
"""Terminate a pod.
|
||||
|
||||
Args:
|
||||
pod_id: Pod ID
|
||||
|
||||
Returns:
|
||||
Termination response
|
||||
"""
|
||||
return self._rest_request("DELETE", f"pods/{pod_id}", None)
|
||||
|
||||
def list_pods(self) -> List[Dict]:
|
||||
"""List all user pods.
|
||||
|
||||
Returns:
|
||||
List of pod dictionaries
|
||||
"""
|
||||
query = """
|
||||
query {
|
||||
myself {
|
||||
pods {
|
||||
id
|
||||
name
|
||||
desiredStatus
|
||||
imageName
|
||||
costPerHr
|
||||
gpuCount
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
data = self._graphql_request(query)
|
||||
return data.get("myself", {}).get("pods", [])
|
||||
|
||||
|
||||
def format_docker_cmd(
|
||||
binary: str,
|
||||
parquet_file: str,
|
||||
epochs: int,
|
||||
use_int8: bool = False,
|
||||
use_qat: bool = False,
|
||||
additional_args: Optional[List[str]] = None
|
||||
) -> List[str]:
|
||||
"""Format Docker CMD for training.
|
||||
|
||||
Args:
|
||||
binary: Binary name (e.g., "train_tft_parquet")
|
||||
parquet_file: Path to parquet file
|
||||
epochs: Number of epochs
|
||||
use_int8: Enable INT8 quantization
|
||||
use_qat: Enable QAT (requires INT8)
|
||||
additional_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
List of command arguments
|
||||
"""
|
||||
cmd = [
|
||||
"--parquet-file", parquet_file,
|
||||
"--epochs", str(epochs)
|
||||
]
|
||||
|
||||
if use_int8:
|
||||
cmd.append("--use-int8")
|
||||
|
||||
if use_qat:
|
||||
cmd.append("--use-qat")
|
||||
|
||||
if additional_args:
|
||||
cmd.extend(additional_args)
|
||||
|
||||
return cmd
|
||||
|
||||
|
||||
def wait_for_pod_ready(api: RunpodAPI, pod_id: str, timeout: int = 300) -> bool:
|
||||
"""Wait for pod to be ready.
|
||||
|
||||
Args:
|
||||
api: Runpod API client
|
||||
pod_id: Pod ID
|
||||
timeout: Timeout in seconds (default: 300)
|
||||
|
||||
Returns:
|
||||
True if pod is ready, False if timeout
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
status = api.get_pod_status(pod_id)
|
||||
desired_status = status.get("desiredStatus", "")
|
||||
|
||||
if desired_status == "RUNNING":
|
||||
print(f"✅ Pod {pod_id} is RUNNING")
|
||||
return True
|
||||
|
||||
print(f"⏳ Pod status: {desired_status} (waiting...)")
|
||||
time.sleep(10)
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to get pod status: {e}")
|
||||
time.sleep(10)
|
||||
|
||||
print(f"❌ Timeout waiting for pod to be ready")
|
||||
return False
|
||||
|
||||
|
||||
def print_pod_info(pod_data: Dict):
|
||||
"""Print pod information.
|
||||
|
||||
Args:
|
||||
pod_data: Pod data dictionary
|
||||
"""
|
||||
print("\n" + "="*80)
|
||||
print("POD CREATED SUCCESSFULLY")
|
||||
print("="*80)
|
||||
|
||||
pod_id = pod_data.get("id") or pod_data.get("podId")
|
||||
print(f"\n📦 Pod ID: {pod_id}")
|
||||
print(f"📛 Name: {pod_data.get('name', 'N/A')}")
|
||||
print(f"🖼️ Image: {pod_data.get('imageName', 'N/A')}")
|
||||
print(f"💰 Cost: ${pod_data.get('costPerHr', 0):.4f}/hr")
|
||||
|
||||
# Print SSH/connection info if available
|
||||
runtime = pod_data.get("runtime", {})
|
||||
if runtime:
|
||||
ports = runtime.get("ports", [])
|
||||
for port in ports:
|
||||
if port.get("type") == "tcp" and port.get("privatePort") == 22:
|
||||
ip = port.get("ip")
|
||||
public_port = port.get("publicPort")
|
||||
print(f"\n🔌 SSH Connection:")
|
||||
print(f" ssh root@{ip} -p {public_port}")
|
||||
|
||||
print(f"\n⚠️ IMPORTANT: Terminate pod when done to avoid charges!")
|
||||
print(f" Terminate: curl -X DELETE https://api.runpod.io/v2/pods/{pod_id} \\")
|
||||
print(f" -H 'Authorization: Bearer $RUNPOD_API_KEY'")
|
||||
print("\n" + "="*80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Deploy Foxhunt ML training to Runpod using GraphQL API",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Smoke test (1 epoch)
|
||||
%(prog)s --smoke-test
|
||||
|
||||
# Full training (50 epochs)
|
||||
%(prog)s --full-training
|
||||
|
||||
# Custom training
|
||||
%(prog)s --epochs 10 --gpu-type "NVIDIA RTX A4000"
|
||||
|
||||
# List available GPUs
|
||||
%(prog)s --list-gpus
|
||||
|
||||
# Get Docker credential ID
|
||||
%(prog)s --get-credentials
|
||||
"""
|
||||
)
|
||||
|
||||
# Deployment options
|
||||
parser.add_argument("--smoke-test", action="store_true",
|
||||
help="Run smoke test (1 epoch, small dataset)")
|
||||
parser.add_argument("--full-training", action="store_true",
|
||||
help="Run full training (50 epochs, 180 days)")
|
||||
|
||||
# Training parameters
|
||||
parser.add_argument("--epochs", type=int, default=10,
|
||||
help="Number of training epochs (default: 10)")
|
||||
parser.add_argument("--binary", default="train_tft_parquet",
|
||||
help="Training binary name (default: train_tft_parquet)")
|
||||
parser.add_argument("--parquet-file",
|
||||
default="/runpod-volume/test_data/ES_FUT_180d.parquet",
|
||||
help="Path to parquet file on network volume")
|
||||
parser.add_argument("--use-int8", action="store_true",
|
||||
help="Enable INT8 post-training quantization")
|
||||
parser.add_argument("--use-qat", action="store_true",
|
||||
help="Enable QAT quantization (experimental)")
|
||||
|
||||
# Pod configuration
|
||||
parser.add_argument("--pod-name",
|
||||
help="Pod name (auto-generated if not specified)")
|
||||
parser.add_argument("--image", default="jgrusewski/foxhunt:latest",
|
||||
help="Docker image (default: jgrusewski/foxhunt:latest)")
|
||||
parser.add_argument("--gpu-type", default="NVIDIA RTX A4000",
|
||||
help="GPU type ID (default: NVIDIA RTX A4000)")
|
||||
parser.add_argument("--network-volume-id", default="se3zdnb5o4",
|
||||
help="Network volume ID (default: se3zdnb5o4)")
|
||||
parser.add_argument("--cloud-type", default="SECURE",
|
||||
choices=["SECURE", "COMMUNITY"],
|
||||
help="Cloud type (default: SECURE)")
|
||||
parser.add_argument("--container-disk", type=int, default=50,
|
||||
help="Container disk size in GB (default: 50)")
|
||||
|
||||
# Utility options
|
||||
parser.add_argument("--list-gpus", action="store_true",
|
||||
help="List available GPU types and exit")
|
||||
parser.add_argument("--get-credentials", action="store_true",
|
||||
help="Get Docker credential ID and exit")
|
||||
parser.add_argument("--list-pods", action="store_true",
|
||||
help="List all active pods and exit")
|
||||
parser.add_argument("--terminate", metavar="POD_ID",
|
||||
help="Terminate a specific pod and exit")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get API key
|
||||
api_key = os.getenv("RUNPOD_API_KEY")
|
||||
if not api_key:
|
||||
print("❌ Error: RUNPOD_API_KEY environment variable is not set")
|
||||
print("\nTo set your API key:")
|
||||
print(" export RUNPOD_API_KEY='your-api-key-here'")
|
||||
print("\nGet your API key from: https://www.runpod.io/console/user/settings")
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize API client
|
||||
try:
|
||||
api = RunpodAPI(api_key)
|
||||
except Exception as e:
|
||||
print(f"❌ Error initializing Runpod API: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Handle utility commands
|
||||
if args.list_gpus:
|
||||
print("\n📊 Available GPU Types:\n")
|
||||
gpus = api.list_available_gpus()
|
||||
for gpu in gpus:
|
||||
name = gpu.get("displayName", "Unknown")
|
||||
gpu_id = gpu.get("id", "Unknown")
|
||||
memory = gpu.get("memoryInGb", 0)
|
||||
lowest_price = gpu.get("lowestPrice", {})
|
||||
spot_price = lowest_price.get("minimumBidPrice", 0)
|
||||
on_demand = lowest_price.get("uninterruptablePrice", 0)
|
||||
|
||||
print(f" {name} ({gpu_id})")
|
||||
print(f" Memory: {memory}GB")
|
||||
print(f" Spot: ${spot_price:.4f}/hr | On-Demand: ${on_demand:.4f}/hr")
|
||||
print()
|
||||
sys.exit(0)
|
||||
|
||||
if args.get_credentials:
|
||||
print("\n🔑 Fetching Docker credentials...\n")
|
||||
cred_id = api.get_docker_credential_id("Docker")
|
||||
if cred_id:
|
||||
print(f"✅ Docker credential ID: {cred_id}")
|
||||
else:
|
||||
print("❌ No credential named 'Docker' found")
|
||||
print("\nCreate credentials at: https://www.runpod.io/console/user/settings")
|
||||
sys.exit(0)
|
||||
|
||||
if args.list_pods:
|
||||
print("\n📦 Active Pods:\n")
|
||||
pods = api.list_pods()
|
||||
if not pods:
|
||||
print(" No active pods")
|
||||
else:
|
||||
for pod in pods:
|
||||
print(f" {pod.get('id')} - {pod.get('name')}")
|
||||
print(f" Status: {pod.get('desiredStatus')}")
|
||||
print(f" Image: {pod.get('imageName')}")
|
||||
print(f" Cost: ${pod.get('costPerHr', 0):.4f}/hr")
|
||||
print()
|
||||
sys.exit(0)
|
||||
|
||||
if args.terminate:
|
||||
print(f"\n🛑 Terminating pod {args.terminate}...\n")
|
||||
try:
|
||||
api.terminate_pod(args.terminate)
|
||||
print(f"✅ Pod {args.terminate} terminated successfully")
|
||||
except Exception as e:
|
||||
print(f"❌ Error terminating pod: {e}")
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
# Handle deployment presets
|
||||
if args.smoke_test:
|
||||
args.epochs = 1
|
||||
args.parquet_file = "/runpod-volume/test_data/ES_FUT_small.parquet"
|
||||
print("\n🧪 SMOKE TEST MODE: 1 epoch, small dataset\n")
|
||||
|
||||
if args.full_training:
|
||||
args.epochs = 50
|
||||
args.parquet_file = "/runpod-volume/test_data/ES_FUT_180d.parquet"
|
||||
print("\n🚀 FULL TRAINING MODE: 50 epochs, 180 days\n")
|
||||
|
||||
# Generate pod name if not specified
|
||||
if not args.pod_name:
|
||||
timestamp = int(time.time())
|
||||
args.pod_name = f"foxhunt-{args.binary}-{args.epochs}ep-{timestamp}"
|
||||
|
||||
# Get Docker credential ID
|
||||
print("🔑 Fetching Docker credentials...")
|
||||
container_registry_auth_id = api.get_docker_credential_id("Docker")
|
||||
if container_registry_auth_id:
|
||||
print(f"✅ Using Docker credential: {container_registry_auth_id}")
|
||||
else:
|
||||
print("⚠️ Warning: No Docker credential found (using public images only)")
|
||||
|
||||
# Format Docker CMD
|
||||
docker_cmd = format_docker_cmd(
|
||||
args.binary,
|
||||
args.parquet_file,
|
||||
args.epochs,
|
||||
args.use_int8,
|
||||
args.use_qat
|
||||
)
|
||||
|
||||
# Environment variables
|
||||
env = {
|
||||
"BINARY_NAME": args.binary,
|
||||
"RUST_LOG": "info",
|
||||
"CUDA_VISIBLE_DEVICES": "0"
|
||||
}
|
||||
|
||||
# Print deployment summary
|
||||
print("\n" + "="*80)
|
||||
print("DEPLOYMENT SUMMARY")
|
||||
print("="*80)
|
||||
print(f"Pod Name: {args.pod_name}")
|
||||
print(f"Image: {args.image}")
|
||||
print(f"GPU Type: {args.gpu_type}")
|
||||
print(f"Cloud Type: {args.cloud_type}")
|
||||
print(f"Network Volume: {args.network_volume_id}")
|
||||
print(f"Binary: {args.binary}")
|
||||
print(f"Parquet File: {args.parquet_file}")
|
||||
print(f"Epochs: {args.epochs}")
|
||||
print(f"INT8: {args.use_int8}")
|
||||
print(f"QAT: {args.use_qat}")
|
||||
print(f"Docker CMD: {' '.join(docker_cmd)}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
# Confirm deployment
|
||||
confirm = input("Proceed with deployment? (yes/no): ")
|
||||
if confirm.lower() != "yes":
|
||||
print("❌ Deployment cancelled")
|
||||
sys.exit(0)
|
||||
|
||||
# Create pod
|
||||
print("\n🚀 Creating pod...")
|
||||
try:
|
||||
response = api.create_pod_rest(
|
||||
name=args.pod_name,
|
||||
image_name=args.image,
|
||||
gpu_type_id=args.gpu_type,
|
||||
docker_start_cmd=docker_cmd,
|
||||
network_volume_id=args.network_volume_id,
|
||||
container_registry_auth_id=container_registry_auth_id,
|
||||
env=env,
|
||||
cloud_type=args.cloud_type,
|
||||
container_disk_in_gb=args.container_disk,
|
||||
volume_mount_path="/runpod-volume",
|
||||
ports=["8888/http", "22/tcp"]
|
||||
)
|
||||
|
||||
pod_id = response.get("id") or response.get("podId")
|
||||
if not pod_id:
|
||||
raise RuntimeError(f"No pod ID in response: {response}")
|
||||
|
||||
print(f"✅ Pod created: {pod_id}")
|
||||
|
||||
# Wait for pod to be ready
|
||||
print("\n⏳ Waiting for pod to be ready...")
|
||||
if wait_for_pod_ready(api, pod_id, timeout=300):
|
||||
# Get full pod info
|
||||
pod_data = api.get_pod_status(pod_id)
|
||||
print_pod_info(pod_data)
|
||||
|
||||
print("\n📝 Next Steps:")
|
||||
print(" 1. Monitor pod logs in Runpod console")
|
||||
print(" 2. Training will start automatically")
|
||||
print(" 3. Checkpoints saved to /runpod-volume/models/")
|
||||
print(" 4. Terminate pod when done to avoid charges")
|
||||
print(f"\n Terminate command:")
|
||||
print(f" ./scripts/deploy_runpod_graphql.py --terminate {pod_id}")
|
||||
else:
|
||||
print("⚠️ Pod creation timed out, but pod may still be starting")
|
||||
print(f" Check status: ./scripts/deploy_runpod_graphql.py --list-pods")
|
||||
print(f" Terminate if needed: ./scripts/deploy_runpod_graphql.py --terminate {pod_id}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error creating pod: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
427
scripts/archive/deploy_runpod_training.py
Executable file
427
scripts/archive/deploy_runpod_training.py
Executable file
@@ -0,0 +1,427 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod Training Pod Deployment Script
|
||||
|
||||
Deploys a Foxhunt training pod to RunPod with proper configuration.
|
||||
Based on thorough documentation research and incremental testing.
|
||||
|
||||
Prerequisites:
|
||||
1. Run test_runpod_auth.py to verify API authentication
|
||||
2. Run test_runpod_pod_creation.py to test basic pod creation
|
||||
3. Set RUNPOD_API_KEY environment variable
|
||||
4. Ensure jgrusewski/foxhunt:latest image is pushed to Docker Hub
|
||||
|
||||
Usage:
|
||||
./scripts/deploy_runpod_training.py [--model MODEL] [--epochs EPOCHS] [--dry-run]
|
||||
|
||||
Examples:
|
||||
# Deploy TFT training with default settings
|
||||
./scripts/deploy_runpod_training.py
|
||||
|
||||
# Deploy MAMBA-2 training for 100 epochs
|
||||
./scripts/deploy_runpod_training.py --model mamba2 --epochs 100
|
||||
|
||||
# Test without actually creating pod
|
||||
./scripts/deploy_runpod_training.py --dry-run
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
from typing import Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
# Configuration defaults
|
||||
DEFAULT_CONFIG = {
|
||||
'gpu_type': 'NVIDIA RTX A4000',
|
||||
'gpu_count': 1,
|
||||
'cloud_type': 'SECURE',
|
||||
'container_disk_gb': 50,
|
||||
'network_volume_id': 'se3zdnb5o4',
|
||||
'volume_mount_path': '/workspace',
|
||||
'min_vcpu': 4,
|
||||
'min_memory_gb': 16,
|
||||
'image_name': 'jgrusewski/foxhunt:latest',
|
||||
}
|
||||
|
||||
# Model training commands
|
||||
TRAINING_COMMANDS = {
|
||||
'tft': 'cargo run -p ml --example train_tft_parquet --release --features cuda -- --parquet-file /workspace/test_data/ES_FUT_180d.parquet --epochs {epochs}',
|
||||
'tft-int8': 'cargo run -p ml --example train_tft_parquet --release --features cuda -- --parquet-file /workspace/test_data/ES_FUT_180d.parquet --epochs {epochs} --use-int8',
|
||||
'tft-qat': 'cargo run -p ml --example train_tft_parquet --release --features cuda -- --parquet-file /workspace/test_data/ES_FUT_180d.parquet --epochs {epochs} --use-qat',
|
||||
'mamba2': 'cargo run -p ml --example train_mamba2_dbn --release --features cuda -- --epochs {epochs}',
|
||||
'dqn': 'cargo run -p ml --example train_dqn --release --features cuda -- --epochs {epochs}',
|
||||
'ppo': 'cargo run -p ml --example train_ppo --release --features cuda -- --epochs {epochs}',
|
||||
}
|
||||
|
||||
|
||||
def get_api_key() -> Optional[str]:
|
||||
"""Get RunPod API key from environment"""
|
||||
api_key = os.environ.get('RUNPOD_API_KEY')
|
||||
if not api_key:
|
||||
print("❌ ERROR: RUNPOD_API_KEY environment variable not set")
|
||||
print("Set it with: export RUNPOD_API_KEY='your_api_key'")
|
||||
return None
|
||||
return api_key
|
||||
|
||||
|
||||
def build_pod_mutation(
|
||||
name: str,
|
||||
docker_cmd: str,
|
||||
config: Dict[str, Any]
|
||||
) -> str:
|
||||
"""
|
||||
Build GraphQL mutation for pod creation
|
||||
|
||||
Args:
|
||||
name: Pod name
|
||||
docker_cmd: Docker CMD override string
|
||||
config: Configuration dictionary
|
||||
|
||||
Returns:
|
||||
GraphQL mutation string
|
||||
"""
|
||||
# Escape quotes in docker command for GraphQL
|
||||
escaped_cmd = docker_cmd.replace('"', '\\"')
|
||||
|
||||
mutation = f"""
|
||||
mutation {{
|
||||
podFindAndDeployOnDemand(
|
||||
input: {{
|
||||
cloudType: {config['cloud_type']}
|
||||
gpuTypeId: "{config['gpu_type']}"
|
||||
gpuCount: {config['gpu_count']}
|
||||
name: "{name}"
|
||||
imageName: "{config['image_name']}"
|
||||
|
||||
containerDiskInGb: {config['container_disk_gb']}
|
||||
networkVolumeId: "{config['network_volume_id']}"
|
||||
volumeMountPath: "{config['volume_mount_path']}"
|
||||
|
||||
minVcpuCount: {config['min_vcpu']}
|
||||
minMemoryInGb: {config['min_memory_gb']}
|
||||
|
||||
dockerArgs: "{escaped_cmd}"
|
||||
|
||||
env: [
|
||||
{{ key: "RUST_LOG", value: "info" }},
|
||||
{{ key: "BINARY_NAME", value: "train_model" }}
|
||||
]
|
||||
|
||||
ports: "22/tcp"
|
||||
startSsh: true
|
||||
}}
|
||||
) {{
|
||||
id
|
||||
name
|
||||
imageName
|
||||
desiredStatus
|
||||
machineId
|
||||
machine {{
|
||||
podHostId
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
return mutation
|
||||
|
||||
|
||||
def deploy_pod(
|
||||
api_key: str,
|
||||
name: str,
|
||||
docker_cmd: str,
|
||||
config: Dict[str, Any],
|
||||
dry_run: bool = False
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Deploy a training pod to RunPod
|
||||
|
||||
Args:
|
||||
api_key: RunPod API key
|
||||
name: Pod name
|
||||
docker_cmd: Docker CMD override
|
||||
config: Configuration dictionary
|
||||
dry_run: If True, print mutation but don't execute
|
||||
|
||||
Returns:
|
||||
Pod information dict if successful, None otherwise
|
||||
"""
|
||||
print("\n" + "="*70)
|
||||
print("🚀 Deploying Training Pod to RunPod")
|
||||
print("="*70)
|
||||
|
||||
# Build mutation
|
||||
mutation = build_pod_mutation(name, docker_cmd, config)
|
||||
|
||||
# Print configuration
|
||||
print("\n📋 Configuration:")
|
||||
print(f" Pod Name: {name}")
|
||||
print(f" GPU: {config['gpu_type']} x{config['gpu_count']}")
|
||||
print(f" Cloud: {config['cloud_type']}")
|
||||
print(f" Image: {config['image_name']}")
|
||||
print(f" Resources: {config['min_vcpu']} vCPU, {config['min_memory_gb']}GB RAM")
|
||||
print(f" Storage: {config['container_disk_gb']}GB container + network volume")
|
||||
print(f" Network Volume: {config['network_volume_id']}")
|
||||
print(f" Mount Path: {config['volume_mount_path']}")
|
||||
print(f"\n🔧 Training Command:")
|
||||
print(f" {docker_cmd}")
|
||||
|
||||
if dry_run:
|
||||
print("\n🔍 DRY RUN MODE - GraphQL Mutation:")
|
||||
print("="*70)
|
||||
print(mutation)
|
||||
print("="*70)
|
||||
print("\n✅ Dry run complete. No pod was created.")
|
||||
return None
|
||||
|
||||
# Execute mutation
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
payload = {'query': mutation}
|
||||
|
||||
print("\n📡 Sending deployment request to RunPod...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=60)
|
||||
|
||||
print(f"📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Check for errors
|
||||
if 'errors' in data:
|
||||
print(f"\n❌ GraphQL Errors:")
|
||||
for error in data['errors']:
|
||||
print(f" - {error.get('message', str(error))}")
|
||||
return None
|
||||
|
||||
# Check for successful pod creation
|
||||
if 'data' in data and data['data'].get('podFindAndDeployOnDemand'):
|
||||
pod = data['data']['podFindAndDeployOnDemand']
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("✅ Pod Deployed Successfully!")
|
||||
print("="*70)
|
||||
print(f"\n🆔 Pod ID: {pod['id']}")
|
||||
print(f"📛 Name: {pod['name']}")
|
||||
print(f"🖼️ Image: {pod['imageName']}")
|
||||
print(f"📊 Status: {pod['desiredStatus']}")
|
||||
|
||||
if pod.get('machineId'):
|
||||
print(f"🖥️ Machine ID: {pod['machineId']}")
|
||||
|
||||
if pod.get('machine') and pod['machine'].get('podHostId'):
|
||||
print(f"🏠 Host ID: {pod['machine']['podHostId']}")
|
||||
|
||||
print(f"\n🌐 Access your pod:")
|
||||
print(f" https://www.runpod.io/console/pods/{pod['id']}")
|
||||
|
||||
print(f"\n💰 Cost Estimate:")
|
||||
print(f" ~$0.50/hour for {config['gpu_type']}")
|
||||
print(f" ~$0.05-0.10 per training run (3-5 minutes)")
|
||||
|
||||
print(f"\n⚠️ IMPORTANT:")
|
||||
print(f" - Training will start automatically")
|
||||
print(f" - Monitor progress in RunPod console")
|
||||
print(f" - STOP the pod when training completes to avoid charges")
|
||||
print(f" - Models saved to network volume will persist")
|
||||
|
||||
return pod
|
||||
else:
|
||||
print(f"\n❌ Pod creation returned null")
|
||||
print(f"This usually means no capacity available for {config['gpu_type']}")
|
||||
print(f"\nResponse: {json.dumps(data, indent=2)}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
print("\n❌ Request timed out after 60 seconds")
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"\n❌ Request failed: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"\n❌ Unexpected error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
|
||||
def stop_pod(api_key: str, pod_id: str) -> bool:
|
||||
"""Stop a running pod"""
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
mutation = f"""
|
||||
mutation {{
|
||||
podStop(input: {{podId: "{pod_id}"}}) {{
|
||||
id
|
||||
desiredStatus
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
payload = {'query': mutation}
|
||||
|
||||
print(f"\n🛑 Stopping pod {pod_id}...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ Failed to stop pod: HTTP {response.status_code}")
|
||||
return False
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ Errors stopping pod: {data['errors']}")
|
||||
return False
|
||||
|
||||
print(f"✅ Pod stopped successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping pod: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Parse command line arguments"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Deploy Foxhunt training pod to RunPod',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Deploy TFT training with default settings
|
||||
%(prog)s
|
||||
|
||||
# Deploy MAMBA-2 training for 100 epochs
|
||||
%(prog)s --model mamba2 --epochs 100
|
||||
|
||||
# Deploy TFT with INT8 quantization
|
||||
%(prog)s --model tft-int8 --epochs 50
|
||||
|
||||
# Test without creating pod
|
||||
%(prog)s --dry-run
|
||||
|
||||
Available models:
|
||||
tft - Temporal Fusion Transformer (FP32)
|
||||
tft-int8 - TFT with INT8 post-training quantization
|
||||
tft-qat - TFT with quantization-aware training (QAT)
|
||||
mamba2 - MAMBA-2 state space model
|
||||
dqn - Deep Q-Network
|
||||
ppo - Proximal Policy Optimization
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--model',
|
||||
type=str,
|
||||
default='tft',
|
||||
choices=list(TRAINING_COMMANDS.keys()),
|
||||
help='Model to train (default: tft)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--epochs',
|
||||
type=int,
|
||||
default=50,
|
||||
help='Number of training epochs (default: 50)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Print configuration without creating pod'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--gpu',
|
||||
type=str,
|
||||
default=None,
|
||||
help='GPU type (default: NVIDIA RTX A4000)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--name',
|
||||
type=str,
|
||||
default=None,
|
||||
help='Custom pod name (default: auto-generated)'
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main deployment function"""
|
||||
args = parse_args()
|
||||
|
||||
print("="*70)
|
||||
print("🦊 Foxhunt Training Pod Deployment")
|
||||
print("="*70)
|
||||
print(f"\nTimestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
|
||||
# Get API key
|
||||
api_key = get_api_key()
|
||||
if not api_key:
|
||||
return 1
|
||||
|
||||
# Build configuration
|
||||
config = DEFAULT_CONFIG.copy()
|
||||
if args.gpu:
|
||||
config['gpu_type'] = args.gpu
|
||||
|
||||
# Generate pod name
|
||||
if args.name:
|
||||
pod_name = args.name
|
||||
else:
|
||||
timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
|
||||
pod_name = f"foxhunt-{args.model}-{timestamp}"
|
||||
|
||||
# Build training command
|
||||
training_cmd = TRAINING_COMMANDS[args.model].format(epochs=args.epochs)
|
||||
|
||||
# Deploy pod
|
||||
pod = deploy_pod(
|
||||
api_key=api_key,
|
||||
name=pod_name,
|
||||
docker_cmd=training_cmd,
|
||||
config=config,
|
||||
dry_run=args.dry_run
|
||||
)
|
||||
|
||||
if pod:
|
||||
print("\n" + "="*70)
|
||||
print("✅ Deployment Complete!")
|
||||
print("="*70)
|
||||
print(f"\n📝 Save this Pod ID: {pod['id']}")
|
||||
print(f"\n🔧 To stop the pod later:")
|
||||
print(f" python -c \"import sys; sys.path.insert(0, 'scripts'); from deploy_runpod_training import stop_pod; stop_pod('{api_key[:10]}...', '{pod['id']}')\"")
|
||||
print(f"\n Or use RunPod console: https://www.runpod.io/console/pods")
|
||||
return 0
|
||||
elif not args.dry_run:
|
||||
print("\n❌ Deployment failed")
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
50
scripts/archive/fetch_pod_logs_via_web.py
Executable file
50
scripts/archive/fetch_pod_logs_via_web.py
Executable file
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple script to guide user to fetch pod logs from RunPod Web UI
|
||||
Since RunPod API doesn't expose container logs directly, this provides instructions.
|
||||
"""
|
||||
import sys
|
||||
import webbrowser
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: ./fetch_pod_logs_via_web.py <pod_id>")
|
||||
sys.exit(1)
|
||||
|
||||
pod_id = sys.argv[1]
|
||||
url = f"https://www.runpod.io/console/pods/{pod_id}"
|
||||
|
||||
print("=" * 70)
|
||||
print("RunPod Pod Logs - Access Instructions")
|
||||
print("=" * 70)
|
||||
print(f"\nPod ID: {pod_id}")
|
||||
print(f"\n1. Opening Web UI in browser...")
|
||||
print(f" URL: {url}")
|
||||
print("\n2. Once the page loads:")
|
||||
print(" a. Click on the pod name (foxhunt-training)")
|
||||
print(" b. Navigate to the 'Logs' tab")
|
||||
print(" c. Look for training output")
|
||||
print("\n3. What to verify:")
|
||||
print(" ✓ 'Using device: Cuda(...)' → GPU is active")
|
||||
print(" ✓ 'Epoch 1/5' → Training started")
|
||||
print(" ✓ GPU memory usage (should be ~12-13GB, NOT 15.9GB)")
|
||||
print(" ✗ NO 'CUDA_ERROR_OUT_OF_MEMORY' messages")
|
||||
print("\n4. Alternative: SSH access (once ready)")
|
||||
print(f" ssh root@{pod_id}.ssh.runpod.io")
|
||||
print(" docker logs $(docker ps -aq | head -1)")
|
||||
print("\n" + "=" * 70)
|
||||
print("\nOpening browser in 3 seconds...")
|
||||
|
||||
try:
|
||||
import time
|
||||
time.sleep(3)
|
||||
webbrowser.open(url)
|
||||
print("✓ Browser opened successfully")
|
||||
except Exception as e:
|
||||
print(f"✗ Failed to open browser: {e}")
|
||||
print(f"\nManually open: {url}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
255
scripts/archive/fix_runpod_deployment.py
Executable file
255
scripts/archive/fix_runpod_deployment.py
Executable file
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fix RunPod Deployment - Terminate failing pod and redeploy in EUR-IS-1
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
# API endpoints
|
||||
REST_API_URL = "https://rest.runpod.io/v1/pods"
|
||||
|
||||
def terminate_pod(pod_id):
|
||||
"""Terminate a RunPod pod."""
|
||||
print(f"\n🗑️ Terminating pod {pod_id}...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.delete(
|
||||
f"{REST_API_URL}/{pod_id}",
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code in [200, 204]:
|
||||
print(f" ✅ Pod {pod_id} terminated successfully")
|
||||
return True
|
||||
else:
|
||||
print(f" ⚠️ Termination failed (status {response.status_code}): {response.text[:200]}")
|
||||
return False
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to terminate pod: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def get_pod_status(pod_id):
|
||||
"""Get current status of a pod."""
|
||||
print(f"\n🔍 Checking pod {pod_id} status...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{REST_API_URL}/{pod_id}",
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
pod_data = response.json()
|
||||
print(f" Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
|
||||
print(f" Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}")
|
||||
|
||||
machine = pod_data.get('machine', {})
|
||||
datacenter = machine.get('dataCenterId', 'N/A')
|
||||
print(f" Datacenter: {datacenter}")
|
||||
|
||||
gpu_info = machine.get('gpuType', {})
|
||||
print(f" GPU: {gpu_info.get('displayName', 'N/A')}")
|
||||
|
||||
return pod_data
|
||||
elif response.status_code == 404:
|
||||
print(f" ⚠️ Pod not found (may already be terminated)")
|
||||
return None
|
||||
else:
|
||||
print(f" ⚠️ Failed to get pod status (status {response.status_code}): {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to check pod status: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def list_all_pods():
|
||||
"""List all active pods."""
|
||||
print("\n📋 Listing all active pods...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
REST_API_URL,
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
pods_data = response.json()
|
||||
|
||||
# Handle both array response and object with 'pods' key
|
||||
if isinstance(pods_data, list):
|
||||
pods = pods_data
|
||||
elif isinstance(pods_data, dict) and 'pods' in pods_data:
|
||||
pods = pods_data['pods']
|
||||
else:
|
||||
print(f" ⚠️ Unexpected response format: {type(pods_data)}")
|
||||
return []
|
||||
|
||||
print(f" Found {len(pods)} active pod(s)")
|
||||
|
||||
for pod in pods:
|
||||
pod_id = pod.get('id', 'N/A')
|
||||
pod_name = pod.get('name', 'N/A')
|
||||
status = pod.get('desiredStatus', 'UNKNOWN')
|
||||
machine = pod.get('machine', {})
|
||||
datacenter = machine.get('dataCenterId', 'N/A')
|
||||
gpu_info = machine.get('gpuType', {})
|
||||
gpu_name = gpu_info.get('displayName', 'N/A')
|
||||
|
||||
print(f"\n Pod: {pod_id}")
|
||||
print(f" Name: {pod_name}")
|
||||
print(f" Status: {status}")
|
||||
print(f" Datacenter: {datacenter}")
|
||||
print(f" GPU: {gpu_name}")
|
||||
|
||||
return pods
|
||||
else:
|
||||
print(f" ⚠️ Failed to list pods (status {response.status_code}): {response.text[:200]}")
|
||||
return []
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to list pods: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_pod_logs(pod_id):
|
||||
"""Get logs from a pod."""
|
||||
print(f"\n📝 Fetching logs for pod {pod_id}...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{REST_API_URL}/{pod_id}/logs",
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
logs_data = response.json()
|
||||
|
||||
# Display logs
|
||||
if isinstance(logs_data, list):
|
||||
print("\n" + "="*70)
|
||||
print("CONTAINER LOGS")
|
||||
print("="*70)
|
||||
for log_entry in logs_data[-50:]: # Last 50 lines
|
||||
timestamp = log_entry.get('timestamp', '')
|
||||
message = log_entry.get('message', '')
|
||||
print(f"{timestamp} {message}")
|
||||
print("="*70)
|
||||
elif isinstance(logs_data, dict) and 'logs' in logs_data:
|
||||
print(logs_data['logs'])
|
||||
else:
|
||||
print(f" Raw logs response: {logs_data}")
|
||||
|
||||
return logs_data
|
||||
else:
|
||||
print(f" ⚠️ Failed to get logs (status {response.status_code}): {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to fetch logs: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print("="*70)
|
||||
print("RUNPOD DEPLOYMENT FIX SCRIPT")
|
||||
print("="*70)
|
||||
print("\nThis script will:")
|
||||
print("1. List all active pods")
|
||||
print("2. Check status of pod 3518shhoirbof4 (if it exists)")
|
||||
print("3. Get logs from the pod (to debug volume mounting)")
|
||||
print("4. Terminate the failing pod")
|
||||
print("5. Instructions for redeploying with fixed script")
|
||||
print("="*70)
|
||||
|
||||
# List all active pods first
|
||||
pods = list_all_pods()
|
||||
|
||||
# Check specific failing pod
|
||||
failing_pod_id = "3518shhoirbof4"
|
||||
pod_data = get_pod_status(failing_pod_id)
|
||||
|
||||
if pod_data:
|
||||
# Get logs before terminating
|
||||
get_pod_logs(failing_pod_id)
|
||||
|
||||
# Ask user confirmation
|
||||
print(f"\n⚠️ Ready to terminate pod {failing_pod_id}")
|
||||
print(" This will stop the restart loop and prepare for clean redeployment")
|
||||
|
||||
confirm = input("\nContinue with termination? [y/N]: ").strip().lower()
|
||||
|
||||
if confirm == 'y':
|
||||
if terminate_pod(failing_pod_id):
|
||||
print("\n✅ Pod terminated successfully!")
|
||||
print("\n📋 NEXT STEPS:")
|
||||
print("="*70)
|
||||
print("1. Deployment script has been fixed (EUR-IS-1 only)")
|
||||
print("2. Wait 30 seconds for pod cleanup to complete")
|
||||
print("3. Deploy new pod with:")
|
||||
print()
|
||||
print(" cd /home/jgrusewski/Work/foxhunt")
|
||||
print(" ./scripts/runpod_deploy.py \\")
|
||||
print(" --image jgrusewski/foxhunt:debug \\")
|
||||
print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
|
||||
print()
|
||||
print("4. Verify new pod is in EUR-IS-1 datacenter")
|
||||
print("5. Check logs show volume mounted successfully")
|
||||
print("="*70)
|
||||
else:
|
||||
print("\n❌ Failed to terminate pod")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("\n❌ Termination cancelled by user")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"\n⚠️ Pod {failing_pod_id} not found or already terminated")
|
||||
print("\n✅ Ready to deploy new pod!")
|
||||
print("\n📋 DEPLOYMENT COMMAND:")
|
||||
print("="*70)
|
||||
print("cd /home/jgrusewski/Work/foxhunt")
|
||||
print("./scripts/runpod_deploy.py \\")
|
||||
print(" --image jgrusewski/foxhunt:debug \\")
|
||||
print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
|
||||
print("="*70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
200
scripts/archive/get_pod_info.py
Executable file
200
scripts/archive/get_pod_info.py
Executable file
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Get comprehensive pod information including datacenter and status
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
GRAPHQL_ENDPOINT = "https://api.runpod.io/graphql"
|
||||
|
||||
def get_pod_details_graphql(pod_id):
|
||||
"""Get pod details using GraphQL."""
|
||||
print(f"\n🔍 Fetching pod {pod_id} details via GraphQL...")
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
# Updated query with correct fields
|
||||
query = """
|
||||
query GetPodDetails($podId: String!) {
|
||||
pod(input: {podId: $podId}) {
|
||||
id
|
||||
name
|
||||
desiredStatus
|
||||
imageName
|
||||
costPerHr
|
||||
machine {
|
||||
podHostId
|
||||
dataCenterId
|
||||
gpuTypeId
|
||||
}
|
||||
runtime {
|
||||
uptimeInSeconds
|
||||
ports {
|
||||
ip
|
||||
isIpPublic
|
||||
privatePort
|
||||
publicPort
|
||||
type
|
||||
}
|
||||
gpus {
|
||||
id
|
||||
gpuUtilPercent
|
||||
memoryUtilPercent
|
||||
}
|
||||
container {
|
||||
cpuPercent
|
||||
memoryPercent
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
variables = {
|
||||
"podId": pod_id
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
GRAPHQL_ENDPOINT,
|
||||
json={"query": query, "variables": variables},
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
|
||||
if 'errors' in result:
|
||||
print(f" ⚠️ GraphQL errors: {result['errors']}")
|
||||
return None
|
||||
|
||||
pod_data = result.get('data', {}).get('pod', {})
|
||||
|
||||
if pod_data:
|
||||
print("\n" + "="*70)
|
||||
print("POD DETAILS")
|
||||
print("="*70)
|
||||
|
||||
print(f"Pod ID: {pod_data.get('id', 'N/A')}")
|
||||
print(f"Name: {pod_data.get('name', 'N/A')}")
|
||||
print(f"Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
|
||||
|
||||
machine = pod_data.get('machine', {})
|
||||
if machine:
|
||||
datacenter = machine.get('dataCenterId', 'N/A')
|
||||
print(f"Datacenter: {datacenter}")
|
||||
print(f"GPU Type ID: {machine.get('gpuTypeId', 'N/A')}")
|
||||
|
||||
# Check if datacenter is EUR-IS-1
|
||||
if datacenter == 'EUR-IS-1':
|
||||
print(f"\n✅ DATACENTER VERIFIED: Pod is in EUR-IS-1 (volume accessible)")
|
||||
elif datacenter != 'N/A':
|
||||
print(f"\n❌ DATACENTER MISMATCH: Pod is in {datacenter}, volume is in EUR-IS-1!")
|
||||
else:
|
||||
print(f"\n⚠️ Datacenter unknown - pod may still be initializing")
|
||||
|
||||
print(f"\nImage: {pod_data.get('imageName', 'N/A')}")
|
||||
print(f"Cost: ${pod_data.get('costPerHr', 'N/A')}/hr")
|
||||
|
||||
runtime = pod_data.get('runtime', {})
|
||||
if runtime:
|
||||
uptime = runtime.get('uptimeInSeconds', 0)
|
||||
print(f"\nRuntime Uptime: {uptime}s")
|
||||
|
||||
ports = runtime.get('ports', [])
|
||||
if ports:
|
||||
print("\nPorts:")
|
||||
for port in ports:
|
||||
print(f" {port.get('privatePort')} -> {port.get('publicPort')} ({port.get('type')})")
|
||||
|
||||
gpus = runtime.get('gpus', [])
|
||||
if gpus:
|
||||
print("\nGPU Status:")
|
||||
for gpu in gpus:
|
||||
gpu_util = gpu.get('gpuUtilPercent', 'N/A')
|
||||
mem_util = gpu.get('memoryUtilPercent', 'N/A')
|
||||
print(f" GPU {gpu.get('id')}: {gpu_util}% util, {mem_util}% memory")
|
||||
|
||||
container = runtime.get('container', {})
|
||||
if container:
|
||||
print(f"\nContainer:")
|
||||
print(f" CPU: {container.get('cpuPercent', 'N/A')}%")
|
||||
print(f" Memory: {container.get('memoryPercent', 'N/A')}%")
|
||||
|
||||
print("="*70)
|
||||
|
||||
return pod_data
|
||||
else:
|
||||
print(" ⚠️ No pod data returned")
|
||||
return None
|
||||
else:
|
||||
print(f" ⚠️ Failed to get pod details (status {response.status_code}): {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to fetch pod details: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print("="*70)
|
||||
print("GET RUNPOD POD INFO")
|
||||
print("="*70)
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("\nUsage: python3 get_pod_info.py <pod_id> [--watch]")
|
||||
print("\nExample:")
|
||||
print(" python3 get_pod_info.py b1m7v451nexg5r")
|
||||
print(" python3 get_pod_info.py b1m7v451nexg5r --watch # Check every 30s")
|
||||
sys.exit(1)
|
||||
|
||||
pod_id = sys.argv[1]
|
||||
watch_mode = '--watch' in sys.argv
|
||||
|
||||
if watch_mode:
|
||||
print("\n⏰ Watch mode enabled - will check every 30 seconds (Ctrl+C to stop)")
|
||||
print("="*70)
|
||||
|
||||
while True:
|
||||
pod_data = get_pod_details_graphql(pod_id)
|
||||
|
||||
if pod_data:
|
||||
machine = pod_data.get('machine', {})
|
||||
datacenter = machine.get('dataCenterId', 'N/A')
|
||||
runtime = pod_data.get('runtime', {})
|
||||
uptime = runtime.get('uptimeInSeconds', 0)
|
||||
|
||||
if datacenter == 'EUR-IS-1' and uptime > 0:
|
||||
print("\n✅ Pod is ready in EUR-IS-1!")
|
||||
break
|
||||
|
||||
print("\n⏳ Waiting 30 seconds before next check...")
|
||||
time.sleep(30)
|
||||
else:
|
||||
get_pod_details_graphql(pod_id)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("DONE")
|
||||
print("="*70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
128
scripts/archive/get_runpod_logs.py
Executable file
128
scripts/archive/get_runpod_logs.py
Executable file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch RunPod Pod Logs via API
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
||||
# Load environment variables
|
||||
env_path = Path(__file__).parent.parent / '.env.runpod'
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
def get_pod_logs(pod_id, lines=50):
|
||||
"""Fetch pod logs via GraphQL API."""
|
||||
|
||||
# GraphQL query to get pod info including logs
|
||||
query = """
|
||||
query GetPod($input: PodFindInput!) {
|
||||
pod(input: $input) {
|
||||
id
|
||||
name
|
||||
desiredStatus
|
||||
runtime {
|
||||
uptimeInSeconds
|
||||
ports {
|
||||
ip
|
||||
isIpPublic
|
||||
privatePort
|
||||
publicPort
|
||||
type
|
||||
}
|
||||
}
|
||||
machine {
|
||||
gpuDisplayName
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"query": query,
|
||||
"variables": {
|
||||
"input": {"podId": pod_id}
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
"https://api.runpod.io/graphql",
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
|
||||
if 'errors' in result:
|
||||
print(f"ERROR: GraphQL errors: {result['errors']}")
|
||||
return None
|
||||
|
||||
return result.get('data', {}).get('pod')
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"ERROR: Failed to query RunPod API: {e}")
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: ./get_runpod_logs.py <pod_id> [lines]")
|
||||
sys.exit(1)
|
||||
|
||||
pod_id = sys.argv[1]
|
||||
lines = int(sys.argv[2]) if len(sys.argv) > 2 else 50
|
||||
|
||||
print(f"Fetching pod info for: {pod_id}")
|
||||
print("=" * 70)
|
||||
|
||||
pod = get_pod_logs(pod_id, lines)
|
||||
|
||||
if not pod:
|
||||
print("Failed to fetch pod information")
|
||||
sys.exit(1)
|
||||
|
||||
# Display pod info
|
||||
print(f"\nPod ID: {pod.get('id')}")
|
||||
print(f"Name: {pod.get('name')}")
|
||||
print(f"Status: {pod.get('desiredStatus')}")
|
||||
|
||||
runtime = pod.get('runtime')
|
||||
if runtime:
|
||||
uptime = runtime.get('uptimeInSeconds', 0)
|
||||
print(f"Uptime: {uptime}s ({uptime // 60}m)")
|
||||
|
||||
# Show ports
|
||||
ports = runtime.get('ports', [])
|
||||
if ports:
|
||||
print("\nPorts:")
|
||||
for port in ports:
|
||||
print(f" {port.get('privatePort')} -> {port.get('publicPort')} ({port.get('type')})")
|
||||
|
||||
machine = pod.get('machine')
|
||||
if machine:
|
||||
print(f"\nGPU: {machine.get('gpuDisplayName', 'N/A')}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("\nNOTE: RunPod GraphQL API does not expose container logs directly.")
|
||||
print("To view logs, use one of these methods:")
|
||||
print(f"1. Web UI: https://www.runpod.io/console/pods/{pod_id}")
|
||||
print(f"2. SSH: ssh root@{pod_id}.ssh.runpod.io")
|
||||
print("3. RunPod CLI: runpodctl logs <pod_id>")
|
||||
print("\nOnce connected via SSH, check logs with:")
|
||||
print(" - docker logs <container_id>")
|
||||
print(" - tail -f /var/log/entrypoint.log (if logging to file)")
|
||||
print(" - ps aux | grep train_tft_parquet (check if process running)")
|
||||
128
scripts/archive/monitor_pod.py
Executable file
128
scripts/archive/monitor_pod.py
Executable file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod Pod Monitoring Script
|
||||
Continuously monitors pod metrics and displays GPU utilization, memory usage, etc.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load RunPod API key
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
||||
load_dotenv(env_path)
|
||||
|
||||
api_key = os.getenv('RUNPOD_API_KEY')
|
||||
if not api_key:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
# Default pod ID (can be overridden via command line)
|
||||
pod_id = 'k18xwnvja2mk1s'
|
||||
if len(sys.argv) > 1:
|
||||
pod_id = sys.argv[1]
|
||||
|
||||
query = """
|
||||
query GetPodMetrics($podId: String!) {
|
||||
pod(input: {podId: $podId}) {
|
||||
id
|
||||
name
|
||||
desiredStatus
|
||||
runtime {
|
||||
uptimeInSeconds
|
||||
container {
|
||||
cpuPercent
|
||||
memoryPercent
|
||||
}
|
||||
gpus {
|
||||
gpuUtilPercent
|
||||
memoryUtilPercent
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
print(f"📊 Monitoring Pod: {pod_id}")
|
||||
print(f"Press Ctrl+C to stop\n")
|
||||
|
||||
iteration = 0
|
||||
while True:
|
||||
try:
|
||||
response = requests.post(
|
||||
"https://api.runpod.io/graphql",
|
||||
json={"query": query, "variables": {"podId": pod_id}},
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"⚠️ API Error: HTTP {response.status_code}")
|
||||
time.sleep(60)
|
||||
continue
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"⚠️ GraphQL Error: {data['errors']}")
|
||||
time.sleep(60)
|
||||
continue
|
||||
|
||||
pod = data.get('data', {}).get('pod', {})
|
||||
if not pod:
|
||||
print(f"⚠️ Pod {pod_id} not found")
|
||||
time.sleep(60)
|
||||
continue
|
||||
|
||||
status = pod.get('desiredStatus', 'UNKNOWN')
|
||||
runtime = pod.get('runtime', {})
|
||||
|
||||
timestamp = datetime.now().strftime('%H:%M:%S')
|
||||
|
||||
if runtime:
|
||||
uptime = runtime.get('uptimeInSeconds', 0)
|
||||
container = runtime.get('container', {})
|
||||
gpus = runtime.get('gpus', [{}])
|
||||
|
||||
cpu_pct = container.get('cpuPercent', 0)
|
||||
mem_pct = container.get('memoryPercent', 0)
|
||||
gpu_util = gpus[0].get('gpuUtilPercent', 0) if gpus else 0
|
||||
gpu_mem = gpus[0].get('memoryUtilPercent', 0) if gpus else 0
|
||||
|
||||
# Format uptime
|
||||
hours = uptime // 3600
|
||||
minutes = (uptime % 3600) // 60
|
||||
seconds = uptime % 60
|
||||
uptime_str = f"{hours}h {minutes}m {seconds}s"
|
||||
|
||||
# Color coding for GPU utilization
|
||||
gpu_indicator = "🟢" if gpu_util > 80 else ("🟡" if gpu_util > 50 else "🔴")
|
||||
|
||||
print(f"[{timestamp}] {gpu_indicator} Uptime: {uptime_str:>12} | "
|
||||
f"CPU: {cpu_pct:5.1f}% | Mem: {mem_pct:5.1f}% | "
|
||||
f"GPU: {gpu_util:5.1f}% | VRAM: {gpu_mem:5.1f}% | "
|
||||
f"Status: {status}")
|
||||
|
||||
# Alert on low GPU utilization after 10 minutes
|
||||
if uptime > 600 and gpu_util < 50:
|
||||
print(f" ⚠️ WARNING: Low GPU utilization after {uptime}s - check if training started")
|
||||
|
||||
# Alert on high memory usage
|
||||
if mem_pct > 90:
|
||||
print(f" ⚠️ WARNING: High memory usage ({mem_pct:.1f}%) - possible OOM risk")
|
||||
|
||||
else:
|
||||
print(f"[{timestamp}] ⏳ Pod initializing... (Status: {status})")
|
||||
|
||||
iteration += 1
|
||||
time.sleep(60) # Check every 60 seconds
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n✅ Monitoring stopped")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error: {e}")
|
||||
time.sleep(60)
|
||||
152
scripts/archive/monitor_runpod.sh
Executable file
152
scripts/archive/monitor_runpod.sh
Executable file
@@ -0,0 +1,152 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Runpod Pod Monitoring Script
|
||||
# Usage: ./scripts/monitor_runpod.sh [pod_id]
|
||||
# Default pod: six9yaydb5f19b
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
POD_ID="${1:-six9yaydb5f19b}"
|
||||
CHECK_INTERVAL=30 # seconds between checks
|
||||
MAX_WAIT=900 # 15 minutes max wait time
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo "========================================="
|
||||
echo "Runpod Pod Monitoring Script"
|
||||
echo "========================================="
|
||||
echo "Pod ID: $POD_ID"
|
||||
echo "Check Interval: ${CHECK_INTERVAL}s"
|
||||
echo "Max Wait Time: ${MAX_WAIT}s ($(($MAX_WAIT / 60)) minutes)"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Function to check pod status
|
||||
check_pod_status() {
|
||||
runpodctl get pod "$POD_ID" 2>&1
|
||||
}
|
||||
|
||||
# Function to check SSH readiness
|
||||
check_ssh_ready() {
|
||||
OUTPUT=$(runpodctl ssh connect "$POD_ID" 2>&1)
|
||||
if echo "$OUTPUT" | grep -q "not yet ready"; then
|
||||
return 1 # Not ready
|
||||
else
|
||||
return 0 # Ready
|
||||
fi
|
||||
}
|
||||
|
||||
# Main monitoring loop
|
||||
ELAPSED=0
|
||||
SSH_READY=0
|
||||
|
||||
echo "Starting pod monitoring..."
|
||||
echo ""
|
||||
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Check pod status
|
||||
echo "[$TIMESTAMP] Checking pod status..."
|
||||
POD_STATUS=$(check_pod_status)
|
||||
|
||||
if echo "$POD_STATUS" | grep -q "RUNNING"; then
|
||||
echo -e "${GREEN}✓ Pod is RUNNING${NC}"
|
||||
|
||||
# Check SSH readiness
|
||||
if check_ssh_ready; then
|
||||
echo -e "${GREEN}✓ SSH is READY${NC}"
|
||||
SSH_READY=1
|
||||
|
||||
# Get SSH connection command
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "SSH CONNECTION AVAILABLE"
|
||||
echo "========================================="
|
||||
runpodctl ssh connect "$POD_ID"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo "1. Use the SSH command above to connect to the pod"
|
||||
echo "2. Run validation commands:"
|
||||
echo " - ls -lh /runpod-volume/"
|
||||
echo " - nvidia-smi"
|
||||
echo " - ps aux | grep train_tft_parquet"
|
||||
echo " - tail -f /var/log/entrypoint.log (if logged)"
|
||||
echo ""
|
||||
echo "3. Or use web UI for logs:"
|
||||
echo " https://www.runpod.io/console/pods"
|
||||
echo ""
|
||||
|
||||
break # Exit loop
|
||||
else
|
||||
echo -e "${YELLOW}⏳ SSH not yet ready (initializing...)${NC}"
|
||||
fi
|
||||
|
||||
elif echo "$POD_STATUS" | grep -q "STOPPED"; then
|
||||
echo -e "${RED}✗ Pod has STOPPED${NC}"
|
||||
echo ""
|
||||
echo "Pod may have completed or crashed."
|
||||
echo "Check logs at: https://www.runpod.io/console/pods"
|
||||
echo ""
|
||||
exit 1
|
||||
|
||||
else
|
||||
echo -e "${YELLOW}⏳ Pod status: UNKNOWN${NC}"
|
||||
echo "$POD_STATUS"
|
||||
fi
|
||||
|
||||
# Show elapsed time and wait
|
||||
ELAPSED=$((ELAPSED + CHECK_INTERVAL))
|
||||
REMAINING=$((MAX_WAIT - ELAPSED))
|
||||
echo "Elapsed: ${ELAPSED}s | Remaining: ${REMAINING}s"
|
||||
echo ""
|
||||
|
||||
if [ $ELAPSED -lt $MAX_WAIT ]; then
|
||||
echo "Waiting ${CHECK_INTERVAL}s before next check..."
|
||||
sleep $CHECK_INTERVAL
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
|
||||
# Final status
|
||||
echo "========================================="
|
||||
echo "MONITORING SUMMARY"
|
||||
echo "========================================="
|
||||
echo "Pod ID: $POD_ID"
|
||||
echo "Total Time: ${ELAPSED}s ($(($ELAPSED / 60)) minutes)"
|
||||
|
||||
if [ $SSH_READY -eq 1 ]; then
|
||||
echo -e "Status: ${GREEN}SUCCESS - SSH Ready${NC}"
|
||||
echo ""
|
||||
echo "Pod is ready for validation!"
|
||||
echo "Use the SSH connection command above or web UI logs."
|
||||
else
|
||||
echo -e "Status: ${YELLOW}TIMEOUT - SSH Not Ready${NC}"
|
||||
echo ""
|
||||
echo "SSH service did not become ready within $((MAX_WAIT / 60)) minutes."
|
||||
echo ""
|
||||
echo "Possible causes:"
|
||||
echo "1. Pod is still initializing (wait longer)"
|
||||
echo "2. SSH service disabled in Docker image"
|
||||
echo "3. Pod crashed during startup"
|
||||
echo ""
|
||||
echo "Recommended actions:"
|
||||
echo "1. Check web UI logs: https://www.runpod.io/console/pods"
|
||||
echo "2. Verify pod status: runpodctl get pod $POD_ID"
|
||||
echo "3. Check if training completed (may have run before SSH ready)"
|
||||
fi
|
||||
|
||||
echo "========================================="
|
||||
echo ""
|
||||
echo "Full pod details:"
|
||||
runpodctl get pod "$POD_ID"
|
||||
echo ""
|
||||
echo "To terminate pod when done:"
|
||||
echo " runpodctl remove pod $POD_ID"
|
||||
echo ""
|
||||
573
scripts/archive/runpod_deploy.sh
Executable file
573
scripts/archive/runpod_deploy.sh
Executable file
@@ -0,0 +1,573 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# FOXHUNT RUNPOD MASTER DEPLOYMENT SCRIPT
|
||||
# =============================================================================
|
||||
# Orchestrates complete Runpod deployment workflow for FP32 model training
|
||||
#
|
||||
# Prerequisites:
|
||||
# - cargo (Rust toolchain)
|
||||
# - docker (Docker daemon running)
|
||||
# - aws cli (S3 API client for Runpod storage)
|
||||
# - RUNPOD_S3_ENDPOINT environment variable
|
||||
# - AWS_PROFILE=runpod (AWS CLI profile configured)
|
||||
# - DOCKER_USERNAME=jgrusewski
|
||||
#
|
||||
# What this script does:
|
||||
# 1. Validates prerequisites
|
||||
# 2. Builds release binaries (5-6 min)
|
||||
# 3. Uploads binaries to Runpod S3 volume
|
||||
# 4. Uploads test data to Runpod S3 volume
|
||||
# 5. Prompts for .env upload (optional, secure confirmation)
|
||||
# 6. Builds Docker image
|
||||
# 7. Pushes to Docker Hub (jgrusewski/foxhunt:latest)
|
||||
# 8. Prints deployment instructions
|
||||
#
|
||||
# Safety:
|
||||
# - Idempotent (safe to re-run)
|
||||
# - Progress indicators for long operations
|
||||
# - Validation at each step
|
||||
# - Confirmation prompts for sensitive operations
|
||||
#
|
||||
# Usage:
|
||||
# export RUNPOD_S3_ENDPOINT=https://s3api-eur-is-1.runpod.io # Your datacenter
|
||||
# export AWS_PROFILE=runpod
|
||||
# export DOCKER_USERNAME=jgrusewski
|
||||
# ./scripts/runpod_deploy.sh
|
||||
# =============================================================================
|
||||
|
||||
set -e # Exit on error
|
||||
set -u # Exit on undefined variable
|
||||
|
||||
# Color codes for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
FOXHUNT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DOCKER_IMAGE="${DOCKER_USERNAME:-jgrusewski}/foxhunt:latest"
|
||||
S3_BUCKET="${S3_BUCKET:-se3zdnb5o4}" # Override with your Network Volume ID
|
||||
|
||||
# Progress indicator
|
||||
progress() {
|
||||
echo -e "${CYAN}▶${NC} $1"
|
||||
}
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
}
|
||||
|
||||
warning() {
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}✗${NC} $1"
|
||||
}
|
||||
|
||||
section() {
|
||||
echo ""
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${BLUE}$1${NC}"
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# STEP 0: BANNER
|
||||
# =============================================================================
|
||||
|
||||
echo -e "${CYAN}"
|
||||
cat << "EOF"
|
||||
╔═══════════════════════════════════════════════════════════════════════════╗
|
||||
║ ║
|
||||
║ FOXHUNT RUNPOD DEPLOYMENT ORCHESTRATOR ║
|
||||
║ ║
|
||||
║ FP32 Model Training - Production Ready ║
|
||||
║ Target GPU: Tesla V100 16GB / RTX 4090 / A4000+ ║
|
||||
║ Deployment Time: ~15-20 minutes ║
|
||||
║ ║
|
||||
╚═══════════════════════════════════════════════════════════════════════════╝
|
||||
EOF
|
||||
echo -e "${NC}"
|
||||
|
||||
# =============================================================================
|
||||
# STEP 1: VALIDATE PREREQUISITES
|
||||
# =============================================================================
|
||||
|
||||
section "Step 1/8: Validating Prerequisites"
|
||||
|
||||
PREREQUISITES_OK=true
|
||||
|
||||
# Check cargo
|
||||
progress "Checking Rust toolchain..."
|
||||
if command -v cargo &> /dev/null; then
|
||||
RUST_VERSION=$(cargo --version | awk '{print $2}')
|
||||
success "Cargo installed: $RUST_VERSION"
|
||||
else
|
||||
error "Cargo not found. Install Rust: https://rustup.rs/"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
|
||||
# Check docker
|
||||
progress "Checking Docker..."
|
||||
if command -v docker &> /dev/null; then
|
||||
if docker info &> /dev/null; then
|
||||
DOCKER_VERSION=$(docker --version | awk '{print $3}' | tr -d ',')
|
||||
success "Docker running: $DOCKER_VERSION"
|
||||
else
|
||||
error "Docker daemon not running. Start with: sudo systemctl start docker"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
else
|
||||
error "Docker not found. Install Docker: https://docs.docker.com/get-docker/"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
|
||||
# Check aws cli
|
||||
progress "Checking AWS CLI (for Runpod S3)..."
|
||||
if command -v aws &> /dev/null; then
|
||||
AWS_VERSION=$(aws --version 2>&1 | awk '{print $1}' | cut -d'/' -f2)
|
||||
success "AWS CLI installed: $AWS_VERSION"
|
||||
else
|
||||
error "AWS CLI not found. Install: pip install awscli"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
|
||||
# Check RUNPOD_S3_ENDPOINT
|
||||
progress "Checking RUNPOD_S3_ENDPOINT environment variable..."
|
||||
if [ -n "${RUNPOD_S3_ENDPOINT:-}" ]; then
|
||||
success "RUNPOD_S3_ENDPOINT: $RUNPOD_S3_ENDPOINT"
|
||||
else
|
||||
error "RUNPOD_S3_ENDPOINT not set. Example: export RUNPOD_S3_ENDPOINT=https://s3api-eur-is-1.runpod.io"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
|
||||
# Check AWS_PROFILE
|
||||
progress "Checking AWS_PROFILE=runpod..."
|
||||
if [ "${AWS_PROFILE:-}" = "runpod" ]; then
|
||||
success "AWS_PROFILE: runpod"
|
||||
|
||||
# Validate profile exists
|
||||
if aws configure list --profile runpod &> /dev/null; then
|
||||
success "AWS profile 'runpod' configured"
|
||||
else
|
||||
error "AWS profile 'runpod' not configured. Run: aws configure --profile runpod"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
else
|
||||
error "AWS_PROFILE not set to 'runpod'. Run: export AWS_PROFILE=runpod"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
|
||||
# Check DOCKER_USERNAME
|
||||
progress "Checking DOCKER_USERNAME..."
|
||||
if [ -n "${DOCKER_USERNAME:-}" ]; then
|
||||
success "DOCKER_USERNAME: $DOCKER_USERNAME"
|
||||
|
||||
# Check Docker Hub login
|
||||
if docker info 2>&1 | grep -q "Username: $DOCKER_USERNAME"; then
|
||||
success "Docker Hub authenticated as $DOCKER_USERNAME"
|
||||
else
|
||||
warning "Not logged into Docker Hub. Will prompt for login later."
|
||||
fi
|
||||
else
|
||||
error "DOCKER_USERNAME not set. Run: export DOCKER_USERNAME=jgrusewski"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
|
||||
# Check workspace directory
|
||||
progress "Checking workspace directory..."
|
||||
if [ -d "$FOXHUNT_ROOT" ]; then
|
||||
success "Workspace: $FOXHUNT_ROOT"
|
||||
else
|
||||
error "Workspace directory not found: $FOXHUNT_ROOT"
|
||||
PREREQUISITES_OK=false
|
||||
fi
|
||||
|
||||
# Final validation
|
||||
if [ "$PREREQUISITES_OK" = false ]; then
|
||||
error "Prerequisites check failed. Fix errors above and re-run."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "All prerequisites validated"
|
||||
|
||||
# =============================================================================
|
||||
# STEP 2: BUILD RELEASE BINARIES
|
||||
# =============================================================================
|
||||
|
||||
section "Step 2/8: Building Release Binaries"
|
||||
|
||||
cd "$FOXHUNT_ROOT"
|
||||
|
||||
progress "Building workspace with CUDA support..."
|
||||
progress "This will take 5-6 minutes (grab coffee ☕)..."
|
||||
|
||||
BUILD_START=$(date +%s)
|
||||
|
||||
# Build with release profile and CUDA features
|
||||
if cargo build --release --workspace --features cuda 2>&1 | tee /tmp/foxhunt_build.log | grep -E "(Compiling|Finished|error)"; then
|
||||
BUILD_END=$(date +%s)
|
||||
BUILD_DURATION=$((BUILD_END - BUILD_START))
|
||||
BUILD_MINUTES=$((BUILD_DURATION / 60))
|
||||
BUILD_SECONDS=$((BUILD_DURATION % 60))
|
||||
|
||||
success "Build complete in ${BUILD_MINUTES}m ${BUILD_SECONDS}s"
|
||||
else
|
||||
error "Build failed. See /tmp/foxhunt_build.log for details."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify critical binaries exist
|
||||
progress "Verifying training binaries..."
|
||||
|
||||
BINARIES=(
|
||||
"train_tft_parquet"
|
||||
"train_mamba2_parquet"
|
||||
"train_dqn"
|
||||
"train_ppo"
|
||||
)
|
||||
|
||||
TOTAL_BIN_SIZE=0
|
||||
for binary in "${BINARIES[@]}"; do
|
||||
BINARY_PATH="$FOXHUNT_ROOT/target/release/examples/$binary"
|
||||
if [ -f "$BINARY_PATH" ]; then
|
||||
SIZE=$(stat -c%s "$BINARY_PATH" 2>/dev/null || stat -f%z "$BINARY_PATH")
|
||||
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
|
||||
TOTAL_BIN_SIZE=$((TOTAL_BIN_SIZE + SIZE))
|
||||
success "$binary ($SIZE_MB MB)"
|
||||
else
|
||||
error "$binary not found at $BINARY_PATH"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
TOTAL_BIN_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BIN_SIZE/1024/1024}")
|
||||
success "Total binaries: $TOTAL_BIN_MB MB"
|
||||
|
||||
# =============================================================================
|
||||
# STEP 3: UPLOAD BINARIES TO RUNPOD S3
|
||||
# =============================================================================
|
||||
|
||||
section "Step 3/8: Uploading Binaries to Runpod S3"
|
||||
|
||||
progress "Uploading 4 training binaries to s3://${S3_BUCKET}/binaries/..."
|
||||
|
||||
UPLOAD_COUNT=0
|
||||
for binary in "${BINARIES[@]}"; do
|
||||
BINARY_PATH="$FOXHUNT_ROOT/target/release/examples/$binary"
|
||||
SIZE_MB=$(stat -c%s "$BINARY_PATH" 2>/dev/null | awk '{printf "%.2f", $1/1024/1024}')
|
||||
|
||||
progress "Uploading $binary ($SIZE_MB MB)..."
|
||||
|
||||
if aws s3 cp "$BINARY_PATH" "s3://${S3_BUCKET}/binaries/$binary" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT" \
|
||||
--profile runpod 2>&1 | grep -q "upload:"; then
|
||||
success "$binary uploaded"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + 1))
|
||||
else
|
||||
error "Failed to upload $binary"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
success "Uploaded $UPLOAD_COUNT binaries to Runpod S3"
|
||||
|
||||
# =============================================================================
|
||||
# STEP 4: UPLOAD TEST DATA TO RUNPOD S3
|
||||
# =============================================================================
|
||||
|
||||
section "Step 4/8: Uploading Test Data to Runpod S3"
|
||||
|
||||
TEST_DATA_DIR="$FOXHUNT_ROOT/test_data"
|
||||
|
||||
if [ ! -d "$TEST_DATA_DIR" ]; then
|
||||
error "Test data directory not found: $TEST_DATA_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Count parquet files
|
||||
PARQUET_COUNT=$(ls -1 "$TEST_DATA_DIR"/*.parquet 2>/dev/null | wc -l)
|
||||
|
||||
if [ "$PARQUET_COUNT" -eq 0 ]; then
|
||||
error "No parquet files found in $TEST_DATA_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
progress "Found $PARQUET_COUNT parquet files"
|
||||
|
||||
# Upload all parquet files
|
||||
TOTAL_DATA_SIZE=0
|
||||
DATA_UPLOAD_COUNT=0
|
||||
|
||||
for file in "$TEST_DATA_DIR"/*.parquet; do
|
||||
if [ -f "$file" ]; then
|
||||
filename=$(basename "$file")
|
||||
SIZE=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file")
|
||||
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
|
||||
TOTAL_DATA_SIZE=$((TOTAL_DATA_SIZE + SIZE))
|
||||
|
||||
progress "Uploading $filename ($SIZE_MB MB)..."
|
||||
|
||||
if aws s3 cp "$file" "s3://${S3_BUCKET}/test_data/$filename" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT" \
|
||||
--profile runpod 2>&1 | grep -q "upload:"; then
|
||||
success "$filename uploaded"
|
||||
DATA_UPLOAD_COUNT=$((DATA_UPLOAD_COUNT + 1))
|
||||
else
|
||||
error "Failed to upload $filename"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
TOTAL_DATA_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_DATA_SIZE/1024/1024}")
|
||||
success "Uploaded $DATA_UPLOAD_COUNT data files ($TOTAL_DATA_MB MB)"
|
||||
|
||||
# =============================================================================
|
||||
# STEP 5: UPLOAD .ENV FILE (OPTIONAL, WITH CONFIRMATION)
|
||||
# =============================================================================
|
||||
|
||||
section "Step 5/8: Upload .env File (Optional)"
|
||||
|
||||
warning "The .env file may contain sensitive credentials (Vault tokens, API keys, etc.)"
|
||||
warning "Only upload if you need these for training (usually not required for FP32)"
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}Do you want to upload .env to Runpod S3?${NC}"
|
||||
echo " - Yes: Upload .env for pod configuration"
|
||||
echo " - No: Skip (recommended for FP32 training)"
|
||||
echo ""
|
||||
|
||||
read -p "Upload .env? (y/N): " -n 1 -r
|
||||
echo
|
||||
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
ENV_FILE="$FOXHUNT_ROOT/.env"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
error ".env file not found at $ENV_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
warning "Uploading .env to s3://${S3_BUCKET}/config/.env"
|
||||
warning "This file will be accessible on Runpod pods"
|
||||
|
||||
echo ""
|
||||
read -p "Are you SURE? This may expose secrets. (y/N): " -n 1 -r
|
||||
echo
|
||||
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
if aws s3 cp "$ENV_FILE" "s3://${S3_BUCKET}/config/.env" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT" \
|
||||
--profile runpod 2>&1 | grep -q "upload:"; then
|
||||
success ".env uploaded to Runpod S3"
|
||||
else
|
||||
error "Failed to upload .env"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warning ".env upload cancelled"
|
||||
fi
|
||||
else
|
||||
success ".env upload skipped (recommended)"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# STEP 6: BUILD DOCKER IMAGE
|
||||
# =============================================================================
|
||||
|
||||
section "Step 6/8: Building Docker Image"
|
||||
|
||||
progress "Building Docker image: $DOCKER_IMAGE"
|
||||
progress "This will take 2-3 minutes (multistage build with caching)..."
|
||||
|
||||
BUILD_START=$(date +%s)
|
||||
|
||||
if docker build -f "$FOXHUNT_ROOT/Dockerfile.runpod" -t "$DOCKER_IMAGE" "$FOXHUNT_ROOT" 2>&1 | \
|
||||
tee /tmp/foxhunt_docker_build.log | grep -E "(Step|Successfully built|error)"; then
|
||||
BUILD_END=$(date +%s)
|
||||
BUILD_DURATION=$((BUILD_END - BUILD_START))
|
||||
BUILD_MINUTES=$((BUILD_DURATION / 60))
|
||||
BUILD_SECONDS=$((BUILD_DURATION % 60))
|
||||
|
||||
success "Docker image built in ${BUILD_MINUTES}m ${BUILD_SECONDS}s"
|
||||
else
|
||||
error "Docker build failed. See /tmp/foxhunt_docker_build.log for details."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check image size
|
||||
IMAGE_SIZE=$(docker images "$DOCKER_IMAGE" --format "{{.Size}}")
|
||||
success "Image size: $IMAGE_SIZE"
|
||||
|
||||
# =============================================================================
|
||||
# STEP 7: PUSH TO DOCKER HUB
|
||||
# =============================================================================
|
||||
|
||||
section "Step 7/8: Pushing to Docker Hub"
|
||||
|
||||
# Verify Docker Hub login
|
||||
progress "Verifying Docker Hub authentication..."
|
||||
|
||||
if ! docker info 2>&1 | grep -q "Username:"; then
|
||||
warning "Not logged into Docker Hub. Please login now."
|
||||
docker login
|
||||
fi
|
||||
|
||||
# Verify private repository
|
||||
echo ""
|
||||
warning "IMPORTANT: Verify your Docker Hub repository is PRIVATE"
|
||||
warning "URL: https://hub.docker.com/repository/docker/${DOCKER_USERNAME}/foxhunt/general"
|
||||
echo ""
|
||||
|
||||
read -p "Is your Docker Hub repo PRIVATE? (y/N): " -n 1 -r
|
||||
echo
|
||||
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
error "Make your Docker Hub repository PRIVATE before pushing!"
|
||||
error "Go to: https://hub.docker.com/repository/docker/${DOCKER_USERNAME}/foxhunt/settings"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Push image
|
||||
progress "Pushing $DOCKER_IMAGE to Docker Hub..."
|
||||
progress "This will take 3-5 minutes (uploading ~2GB image)..."
|
||||
|
||||
PUSH_START=$(date +%s)
|
||||
|
||||
if docker push "$DOCKER_IMAGE" 2>&1 | tee /tmp/foxhunt_docker_push.log | grep -E "(Pushed|Layer already exists|error)"; then
|
||||
PUSH_END=$(date +%s)
|
||||
PUSH_DURATION=$((PUSH_END - PUSH_START))
|
||||
PUSH_MINUTES=$((PUSH_DURATION / 60))
|
||||
PUSH_SECONDS=$((PUSH_DURATION % 60))
|
||||
|
||||
success "Image pushed in ${PUSH_MINUTES}m ${PUSH_SECONDS}s"
|
||||
else
|
||||
error "Docker push failed. See /tmp/foxhunt_docker_push.log for details."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# STEP 8: PRINT DEPLOYMENT INSTRUCTIONS
|
||||
# =============================================================================
|
||||
|
||||
section "Step 8/8: Deployment Instructions"
|
||||
|
||||
success "Deployment preparation complete! 🎉"
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${GREEN}READY FOR RUNPOD DEPLOYMENT${NC}"
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}📦 Uploaded Resources:${NC}"
|
||||
echo " - Binaries: s3://${S3_BUCKET}/binaries/ ($TOTAL_BIN_MB MB)"
|
||||
echo " - Test Data: s3://${S3_BUCKET}/test_data/ ($TOTAL_DATA_MB MB)"
|
||||
echo " - Docker Image: $DOCKER_IMAGE ($IMAGE_SIZE)"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}🚀 Deploy on Runpod Console:${NC}"
|
||||
echo " 1. Go to: https://www.runpod.io/console/pods"
|
||||
echo " 2. Click 'Deploy' and configure:"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN} GPU Configuration:${NC}"
|
||||
echo " GPU Type: Tesla V100 16GB (\$0.14-0.39/hr) or RTX 4090 24GB (\$0.60/hr)"
|
||||
echo " vCPU: 6-8 cores (recommended)"
|
||||
echo " RAM: 30GB+ (recommended)"
|
||||
echo " Container Disk: 20GB minimum"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN} Docker Configuration:${NC}"
|
||||
echo " Docker Image: $DOCKER_IMAGE"
|
||||
echo " Docker Hub Credentials: $DOCKER_USERNAME (required for private repo)"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN} Volume Configuration:${NC}"
|
||||
echo " Volume Path: /runpod-volume"
|
||||
echo " Network Volume: $S3_BUCKET (select from dropdown)"
|
||||
echo " Access Mode: Read/Write"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN} Environment Variables:${NC}"
|
||||
echo " BINARY_NAME=train_tft_parquet"
|
||||
echo " RUST_LOG=info"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN} Container Arguments (override CMD):${NC}"
|
||||
echo " --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet"
|
||||
echo " --epochs 50"
|
||||
echo " --batch-size 32"
|
||||
echo " --lookback-window 60"
|
||||
echo " --forecast-horizon 10"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}📊 Expected Training Performance:${NC}"
|
||||
echo " - DQN: ~15-20 seconds (6MB memory)"
|
||||
echo " - PPO: ~7-10 seconds (145MB memory)"
|
||||
echo " - MAMBA-2: ~2-3 minutes (164MB memory)"
|
||||
echo " - TFT-FP32: ~3-5 minutes (500MB memory)"
|
||||
echo " - Total: ~10-15 minutes (815MB peak memory)"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}💰 Cost Estimation:${NC}"
|
||||
echo " - Tesla V100 @ \$0.25/hr: ~\$0.06 per full training run"
|
||||
echo " - RTX 4090 @ \$0.60/hr: ~\$0.15 per full training run"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}📈 Wave D Backtest Results (Expected):${NC}"
|
||||
echo " - Sharpe Ratio: 2.00 (≥2.0 target ✅)"
|
||||
echo " - Win Rate: 60% (≥60% target ✅)"
|
||||
echo " - Max Drawdown: 15% (≤15% target ✅)"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}🔍 Monitor Training:${NC}"
|
||||
echo " 1. Click 'Logs' tab in Runpod console"
|
||||
echo " 2. Watch for training progress:"
|
||||
echo " - Epoch 1/50 [██████████] 100% | Loss: 0.0123"
|
||||
echo " - Validation RMSE: 0.0045"
|
||||
echo " - Training complete! Model saved."
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}📥 Download Trained Models:${NC}"
|
||||
echo " 1. SSH into pod (get SSH command from Runpod console)"
|
||||
echo " 2. Download models:"
|
||||
echo " scp root@<pod-ssh>:/workspace/models/*.pt ./models/"
|
||||
echo " Or:"
|
||||
echo " aws s3 sync s3://${S3_BUCKET}/models/ ./models/ \\"
|
||||
echo " --endpoint-url $RUNPOD_S3_ENDPOINT \\"
|
||||
echo " --profile runpod"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}🔧 Troubleshooting:${NC}"
|
||||
echo " - Volume not mounted: Check pod settings → Volumes → /runpod-volume"
|
||||
echo " - Binary not found: Verify binaries uploaded to s3://${S3_BUCKET}/binaries/"
|
||||
echo " - Out of memory: Use smaller batch size (--batch-size 16)"
|
||||
echo " - CUDA not found: Verify GPU type is V100/4090 and CUDA runtime available"
|
||||
echo ""
|
||||
|
||||
echo -e "${CYAN}📚 Documentation:${NC}"
|
||||
echo " - Full guide: $FOXHUNT_ROOT/RUNPOD_DEPLOYMENT_CHECKLIST.md"
|
||||
echo " - QAT blockers: $FOXHUNT_ROOT/QAT_BLOCKERS_ROOT_CAUSE_ANALYSIS.md"
|
||||
echo " - System status: $FOXHUNT_ROOT/CLAUDE.md"
|
||||
echo ""
|
||||
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${GREEN}✅ DEPLOYMENT READY - GO TRAIN YOUR MODELS!${NC}"
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo ""
|
||||
|
||||
# Log summary
|
||||
TOTAL_TIME=$(($(date +%s) - BUILD_START))
|
||||
TOTAL_MINUTES=$((TOTAL_TIME / 60))
|
||||
TOTAL_SECONDS=$((TOTAL_TIME % 60))
|
||||
|
||||
echo -e "${CYAN}⏱️ Total deployment time: ${TOTAL_MINUTES}m ${TOTAL_SECONDS}s${NC}"
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
216
scripts/archive/runpod_deploy_test.sh
Executable file
216
scripts/archive/runpod_deploy_test.sh
Executable file
@@ -0,0 +1,216 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# FOXHUNT RUNPOD DEPLOYMENT TEST SCRIPT
|
||||
# =============================================================================
|
||||
# Tests the deployment script without actually deploying
|
||||
# Validates prerequisites and dry-run checks without uploads
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/runpod_deploy_test.sh
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
# Color codes
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}✗${NC} $1"
|
||||
}
|
||||
|
||||
warning() {
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
}
|
||||
|
||||
section() {
|
||||
echo ""
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${BLUE}$1${NC}"
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
}
|
||||
|
||||
section "Runpod Deployment Test - Dry Run"
|
||||
|
||||
ISSUES_FOUND=0
|
||||
|
||||
# Test 1: Cargo
|
||||
echo -e "\n${CYAN}Test 1: Cargo${NC}"
|
||||
if command -v cargo &> /dev/null; then
|
||||
RUST_VERSION=$(cargo --version | awk '{print $2}')
|
||||
success "Cargo installed: $RUST_VERSION"
|
||||
else
|
||||
error "Cargo not found"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
# Test 2: Docker
|
||||
echo -e "\n${CYAN}Test 2: Docker${NC}"
|
||||
if command -v docker &> /dev/null; then
|
||||
if docker info &> /dev/null 2>&1; then
|
||||
DOCKER_VERSION=$(docker --version | awk '{print $3}' | tr -d ',')
|
||||
success "Docker running: $DOCKER_VERSION"
|
||||
else
|
||||
error "Docker daemon not running"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
else
|
||||
error "Docker not found"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
# Test 3: AWS CLI
|
||||
echo -e "\n${CYAN}Test 3: AWS CLI${NC}"
|
||||
if command -v aws &> /dev/null; then
|
||||
AWS_VERSION=$(aws --version 2>&1 | awk '{print $1}' | cut -d'/' -f2)
|
||||
success "AWS CLI installed: $AWS_VERSION"
|
||||
else
|
||||
error "AWS CLI not found"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
# Test 4: Environment variables
|
||||
echo -e "\n${CYAN}Test 4: Environment Variables${NC}"
|
||||
|
||||
if [ -n "${RUNPOD_S3_ENDPOINT:-}" ]; then
|
||||
success "RUNPOD_S3_ENDPOINT: $RUNPOD_S3_ENDPOINT"
|
||||
else
|
||||
error "RUNPOD_S3_ENDPOINT not set"
|
||||
warning "Set with: export RUNPOD_S3_ENDPOINT=https://s3api-us-ca-1.runpod.io"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
if [ "${AWS_PROFILE:-}" = "runpod" ]; then
|
||||
success "AWS_PROFILE: runpod"
|
||||
else
|
||||
error "AWS_PROFILE not set to 'runpod'"
|
||||
warning "Set with: export AWS_PROFILE=runpod"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
if [ -n "${DOCKER_USERNAME:-}" ]; then
|
||||
success "DOCKER_USERNAME: $DOCKER_USERNAME"
|
||||
else
|
||||
error "DOCKER_USERNAME not set"
|
||||
warning "Set with: export DOCKER_USERNAME=jgrusewski"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
# Test 5: AWS Profile
|
||||
echo -e "\n${CYAN}Test 5: AWS Profile Configuration${NC}"
|
||||
if [ "${AWS_PROFILE:-}" = "runpod" ]; then
|
||||
if aws configure list --profile runpod &> /dev/null; then
|
||||
success "AWS profile 'runpod' configured"
|
||||
|
||||
# Show credentials (redacted)
|
||||
ACCESS_KEY=$(aws configure get aws_access_key_id --profile runpod)
|
||||
if [ -n "$ACCESS_KEY" ]; then
|
||||
REDACTED_KEY="${ACCESS_KEY:0:8}***"
|
||||
success "Access Key: $REDACTED_KEY"
|
||||
fi
|
||||
else
|
||||
error "AWS profile 'runpod' not configured"
|
||||
warning "Configure with: aws configure --profile runpod"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
fi
|
||||
|
||||
# Test 6: Test data
|
||||
echo -e "\n${CYAN}Test 6: Test Data Files${NC}"
|
||||
TEST_DATA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/test_data"
|
||||
|
||||
if [ -d "$TEST_DATA_DIR" ]; then
|
||||
PARQUET_COUNT=$(ls -1 "$TEST_DATA_DIR"/*.parquet 2>/dev/null | wc -l)
|
||||
if [ "$PARQUET_COUNT" -gt 0 ]; then
|
||||
success "Found $PARQUET_COUNT parquet files in $TEST_DATA_DIR"
|
||||
|
||||
# Show first 3 files
|
||||
for file in $(ls "$TEST_DATA_DIR"/*.parquet 2>/dev/null | head -3); do
|
||||
filename=$(basename "$file")
|
||||
SIZE=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file")
|
||||
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
|
||||
echo " - $filename ($SIZE_MB MB)"
|
||||
done
|
||||
else
|
||||
error "No parquet files found in $TEST_DATA_DIR"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
else
|
||||
error "Test data directory not found: $TEST_DATA_DIR"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
# Test 7: Dockerfile
|
||||
echo -e "\n${CYAN}Test 7: Dockerfile${NC}"
|
||||
DOCKERFILE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/Dockerfile.runpod"
|
||||
|
||||
if [ -f "$DOCKERFILE" ]; then
|
||||
success "Dockerfile.runpod exists"
|
||||
else
|
||||
error "Dockerfile.runpod not found"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
# Test 8: Entrypoint script
|
||||
echo -e "\n${CYAN}Test 8: Entrypoint Script${NC}"
|
||||
ENTRYPOINT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/entrypoint.sh"
|
||||
|
||||
if [ -f "$ENTRYPOINT" ]; then
|
||||
if [ -x "$ENTRYPOINT" ]; then
|
||||
success "entrypoint.sh exists and is executable"
|
||||
else
|
||||
warning "entrypoint.sh exists but not executable"
|
||||
echo " Fix with: chmod +x $ENTRYPOINT"
|
||||
fi
|
||||
else
|
||||
error "entrypoint.sh not found"
|
||||
ISSUES_FOUND=$((ISSUES_FOUND + 1))
|
||||
fi
|
||||
|
||||
# Test 9: S3 connectivity
|
||||
echo -e "\n${CYAN}Test 9: S3 Connectivity (Optional)${NC}"
|
||||
if [ -n "${RUNPOD_S3_ENDPOINT:-}" ] && [ "${AWS_PROFILE:-}" = "runpod" ]; then
|
||||
echo "Testing S3 connectivity to $RUNPOD_S3_ENDPOINT..."
|
||||
|
||||
# Try to list buckets (may fail if no permissions, but tests connectivity)
|
||||
if aws s3 ls --endpoint-url "$RUNPOD_S3_ENDPOINT" --profile runpod &> /dev/null; then
|
||||
success "S3 connectivity works"
|
||||
else
|
||||
warning "S3 connectivity test failed (may need to create bucket first)"
|
||||
echo " This is OK if you haven't created a Network Volume yet"
|
||||
fi
|
||||
else
|
||||
warning "Skipping S3 connectivity test (missing env vars)"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
section "Test Summary"
|
||||
|
||||
if [ $ISSUES_FOUND -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ All tests passed! Ready for deployment.${NC}"
|
||||
echo ""
|
||||
echo "Run deployment with:"
|
||||
echo " ./scripts/runpod_deploy.sh"
|
||||
else
|
||||
echo -e "${RED}❌ Found $ISSUES_FOUND issue(s). Fix before deploying.${NC}"
|
||||
echo ""
|
||||
echo "Quick fixes:"
|
||||
echo " 1. Install missing tools (cargo, docker, aws cli)"
|
||||
echo " 2. Set environment variables:"
|
||||
echo " export RUNPOD_S3_ENDPOINT=https://s3api-us-ca-1.runpod.io"
|
||||
echo " export AWS_PROFILE=runpod"
|
||||
echo " export DOCKER_USERNAME=jgrusewski"
|
||||
echo " 3. Configure AWS profile:"
|
||||
echo " aws configure --profile runpod"
|
||||
echo " 4. Re-run this test script"
|
||||
fi
|
||||
|
||||
exit $ISSUES_FOUND
|
||||
411
scripts/archive/runpod_full_deploy.py
Executable file
411
scripts/archive/runpod_full_deploy.py
Executable file
@@ -0,0 +1,411 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod Full Deployment Orchestrator
|
||||
|
||||
All-in-one script that handles the complete deployment workflow:
|
||||
1. Check/upload binaries and test data to volume
|
||||
2. Deploy pod with training job
|
||||
3. Monitor training progress
|
||||
4. Download trained models from volume
|
||||
5. Cleanup pod (optional)
|
||||
|
||||
This script orchestrates `upload_to_runpod_volume.py` and
|
||||
`runpod_deploy_production.py` to provide a seamless deployment experience.
|
||||
|
||||
Prerequisites:
|
||||
1. Set RUNPOD_API_KEY environment variable
|
||||
2. Ensure .env.runpod exists with S3 credentials
|
||||
3. Build release binaries: cargo build --release --examples
|
||||
|
||||
Usage:
|
||||
# Smoke test (upload if needed, deploy, monitor, download)
|
||||
./scripts/runpod_full_deploy.py --smoke-test
|
||||
|
||||
# Full training with auto-cleanup after download
|
||||
./scripts/runpod_full_deploy.py --full-training --cleanup
|
||||
|
||||
# Skip upload (binaries already on volume)
|
||||
./scripts/runpod_full_deploy.py --smoke-test --skip-upload
|
||||
|
||||
# Force re-upload binaries even if unchanged
|
||||
./scripts/runpod_full_deploy.py --full-training --force-upload
|
||||
|
||||
# Custom configuration
|
||||
./scripts/runpod_full_deploy.py --model tft --epochs 25 --batch-size 16
|
||||
|
||||
# Dry run (show plan without executing)
|
||||
./scripts/runpod_full_deploy.py --smoke-test --dry-run
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def run_command(cmd: list, description: str, dry_run: bool = False) -> bool:
|
||||
"""
|
||||
Run a command and return success status.
|
||||
|
||||
Args:
|
||||
cmd: Command to run (list of args)
|
||||
description: Description for logging
|
||||
dry_run: If True, print command without executing
|
||||
|
||||
Returns:
|
||||
True if command succeeded, False otherwise
|
||||
"""
|
||||
if dry_run:
|
||||
console.print(f"\n[dim]Would run: {' '.join(cmd)}[/dim]")
|
||||
return True
|
||||
|
||||
console.print(f"\n[bold cyan]→ {description}[/bold cyan]")
|
||||
console.print(f"[dim] Command: {' '.join(cmd)}[/dim]\n")
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, check=True)
|
||||
console.print(f"[green]✓ {description} completed[/green]")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
console.print(f"[red]✗ {description} failed with exit code {e.returncode}[/red]")
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
console.print(f"[red]✗ Command not found: {cmd[0]}[/red]")
|
||||
return False
|
||||
|
||||
|
||||
def check_prerequisites() -> bool:
|
||||
"""Check if all prerequisites are met."""
|
||||
console.print("\n[bold cyan]Checking Prerequisites...[/bold cyan]")
|
||||
|
||||
checks = []
|
||||
|
||||
# Check RUNPOD_API_KEY
|
||||
if os.getenv("RUNPOD_API_KEY"):
|
||||
console.print("[green]✓ RUNPOD_API_KEY is set[/green]")
|
||||
checks.append(True)
|
||||
else:
|
||||
console.print("[red]✗ RUNPOD_API_KEY not set[/red]")
|
||||
console.print(" Set with: export RUNPOD_API_KEY='your-key-here'")
|
||||
checks.append(False)
|
||||
|
||||
# Check .env.runpod
|
||||
env_file = Path(__file__).parent.parent / ".env.runpod"
|
||||
if env_file.exists():
|
||||
console.print(f"[green]✓ .env.runpod exists[/green]")
|
||||
checks.append(True)
|
||||
else:
|
||||
console.print(f"[red]✗ .env.runpod not found[/red]")
|
||||
console.print(" Create it first (see RUNPOD_S3_DEPLOYMENT_GUIDE.md)")
|
||||
checks.append(False)
|
||||
|
||||
# Check upload script
|
||||
upload_script = Path(__file__).parent / "upload_to_runpod_volume.py"
|
||||
if upload_script.exists():
|
||||
console.print(f"[green]✓ upload_to_runpod_volume.py exists[/green]")
|
||||
checks.append(True)
|
||||
else:
|
||||
console.print(f"[red]✗ upload_to_runpod_volume.py not found[/red]")
|
||||
checks.append(False)
|
||||
|
||||
# Check deployment script
|
||||
deploy_script = Path(__file__).parent / "runpod_deploy_production.py"
|
||||
if deploy_script.exists():
|
||||
console.print(f"[green]✓ runpod_deploy_production.py exists[/green]")
|
||||
checks.append(True)
|
||||
else:
|
||||
console.print(f"[red]✗ runpod_deploy_production.py not found[/red]")
|
||||
checks.append(False)
|
||||
|
||||
# Check binaries
|
||||
binaries_dir = Path("target/release/examples")
|
||||
if binaries_dir.exists():
|
||||
binaries = list(binaries_dir.glob("train_*"))
|
||||
if binaries:
|
||||
console.print(f"[green]✓ Found {len(binaries)} training binaries[/green]")
|
||||
checks.append(True)
|
||||
else:
|
||||
console.print(f"[yellow]⚠ No training binaries found[/yellow]")
|
||||
console.print(" Run: cargo build --release --examples")
|
||||
checks.append(False)
|
||||
else:
|
||||
console.print(f"[yellow]⚠ Binaries directory not found[/yellow]")
|
||||
console.print(" Run: cargo build --release --examples")
|
||||
checks.append(False)
|
||||
|
||||
return all(checks)
|
||||
|
||||
|
||||
def upload_to_volume(force: bool = False, dry_run: bool = False) -> bool:
|
||||
"""Upload binaries and test data to volume."""
|
||||
script = Path(__file__).parent / "upload_to_runpod_volume.py"
|
||||
|
||||
cmd = [str(script), "--all"]
|
||||
if force:
|
||||
cmd.append("--force")
|
||||
if dry_run:
|
||||
cmd.append("--dry-run")
|
||||
|
||||
return run_command(cmd, "Upload binaries and test data to volume", dry_run)
|
||||
|
||||
|
||||
def deploy_pod(args: argparse.Namespace, dry_run: bool = False) -> Optional[str]:
|
||||
"""
|
||||
Deploy training pod and return pod ID if successful.
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
dry_run: If True, show plan without deploying
|
||||
|
||||
Returns:
|
||||
Pod ID if deployment succeeded, None otherwise
|
||||
"""
|
||||
script = Path(__file__).parent / "runpod_deploy_production.py"
|
||||
|
||||
cmd = [str(script)]
|
||||
|
||||
# Add training mode
|
||||
if args.smoke_test:
|
||||
cmd.append("--smoke-test")
|
||||
elif args.full_training:
|
||||
cmd.append("--full-training")
|
||||
|
||||
# Add model and training params
|
||||
if args.model:
|
||||
cmd.extend(["--model", args.model])
|
||||
if args.epochs:
|
||||
cmd.extend(["--epochs", str(args.epochs)])
|
||||
if args.batch_size:
|
||||
cmd.extend(["--batch-size", str(args.batch_size)])
|
||||
if args.dataset:
|
||||
cmd.extend(["--dataset", args.dataset])
|
||||
if args.use_int8:
|
||||
cmd.append("--use-int8")
|
||||
|
||||
# Add GPU preference
|
||||
if args.gpu_type:
|
||||
cmd.extend(["--gpu-type", args.gpu_type])
|
||||
|
||||
# Add execution options
|
||||
if dry_run:
|
||||
cmd.append("--dry-run")
|
||||
if args.monitor:
|
||||
cmd.append("--monitor")
|
||||
if args.yes:
|
||||
cmd.append("--yes")
|
||||
|
||||
success = run_command(cmd, "Deploy training pod", dry_run)
|
||||
|
||||
if not success:
|
||||
return None
|
||||
|
||||
# Parse pod ID from output (if not dry-run)
|
||||
# TODO: Modify runpod_deploy_production.py to output pod ID in machine-readable format
|
||||
# For now, return placeholder
|
||||
return "pod-id-placeholder" if not dry_run else "dry-run-pod-id"
|
||||
|
||||
|
||||
def download_models(model: str, dry_run: bool = False) -> bool:
|
||||
"""Download trained models from volume via S3."""
|
||||
if dry_run:
|
||||
console.print("\n[dim]Would download models from volume[/dim]")
|
||||
return True
|
||||
|
||||
console.print("\n[bold cyan]Downloading Models from Volume...[/bold cyan]")
|
||||
|
||||
try:
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load credentials
|
||||
env_path = Path(__file__).parent.parent / ".env.runpod"
|
||||
load_dotenv(env_path)
|
||||
|
||||
# Initialize S3 client
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
aws_access_key_id=os.getenv("RUNPOD_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key=os.getenv("RUNPOD_S3_SECRET"),
|
||||
region_name=os.getenv("RUNPOD_S3_REGION"),
|
||||
endpoint_url=os.getenv("RUNPOD_S3_ENDPOINT"),
|
||||
)
|
||||
|
||||
volume_id = os.getenv("RUNPOD_VOLUME_ID")
|
||||
|
||||
# List models in volume
|
||||
prefix = f"models/{model}/"
|
||||
response = s3_client.list_objects_v2(Bucket=volume_id, Prefix=prefix)
|
||||
|
||||
if "Contents" not in response:
|
||||
console.print(f"[yellow]⚠ No models found at {prefix}[/yellow]")
|
||||
return False
|
||||
|
||||
# Download each model file
|
||||
local_dir = Path("models/runpod_trained") / model
|
||||
local_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for obj in response["Contents"]:
|
||||
s3_key = obj["Key"]
|
||||
filename = Path(s3_key).name
|
||||
local_path = local_dir / filename
|
||||
|
||||
console.print(f" Downloading {filename}...")
|
||||
s3_client.download_file(volume_id, s3_key, str(local_path))
|
||||
console.print(f" [green]✓ Saved to {local_path}[/green]")
|
||||
|
||||
console.print(f"\n[green]✓ Downloaded {len(response['Contents'])} model files[/green]")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]✗ Failed to download models: {e}[/red]")
|
||||
return False
|
||||
|
||||
|
||||
def cleanup_pod(pod_id: str, dry_run: bool = False) -> bool:
|
||||
"""Terminate pod after training completes."""
|
||||
if dry_run:
|
||||
console.print(f"\n[dim]Would terminate pod: {pod_id}[/dim]")
|
||||
return True
|
||||
|
||||
console.print(f"\n[bold cyan]Terminating Pod: {pod_id}[/bold cyan]")
|
||||
|
||||
# TODO: Implement GraphQL mutation to terminate pod
|
||||
# For now, just print instructions
|
||||
console.print("[yellow]⚠ Auto-cleanup not yet implemented[/yellow]")
|
||||
console.print(f" Terminate manually: https://www.runpod.io/console/pods/{pod_id}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Full RunPod deployment orchestrator (upload → deploy → monitor → download)",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Complete smoke test workflow
|
||||
%(prog)s --smoke-test
|
||||
|
||||
# Full training with cleanup
|
||||
%(prog)s --full-training --cleanup
|
||||
|
||||
# Skip upload if binaries already on volume
|
||||
%(prog)s --smoke-test --skip-upload
|
||||
|
||||
# Force re-upload binaries
|
||||
%(prog)s --full-training --force-upload
|
||||
|
||||
# Custom configuration
|
||||
%(prog)s --model tft --epochs 25 --batch-size 16
|
||||
|
||||
# Dry run (show plan)
|
||||
%(prog)s --smoke-test --dry-run
|
||||
""",
|
||||
)
|
||||
|
||||
# Training mode
|
||||
parser.add_argument("--smoke-test", action="store_true", help="Quick smoke test")
|
||||
parser.add_argument("--full-training", action="store_true", help="Full training run")
|
||||
|
||||
# Upload options
|
||||
parser.add_argument(
|
||||
"--skip-upload", action="store_true", help="Skip upload (binaries already on volume)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force-upload",
|
||||
action="store_true",
|
||||
help="Force re-upload even if checksums match",
|
||||
)
|
||||
|
||||
# Training parameters
|
||||
parser.add_argument("--model", default="tft", help="Model to train (default: tft)")
|
||||
parser.add_argument("--epochs", type=int, help="Number of epochs")
|
||||
parser.add_argument("--batch-size", type=int, help="Batch size")
|
||||
parser.add_argument("--dataset", help="Dataset filename")
|
||||
parser.add_argument("--use-int8", action="store_true", help="Use INT8 quantization (TFT only)")
|
||||
|
||||
# GPU selection
|
||||
parser.add_argument("--gpu-type", help="Preferred GPU type")
|
||||
|
||||
# Execution options
|
||||
parser.add_argument("--monitor", action="store_true", help="Monitor pod in real-time")
|
||||
parser.add_argument("--cleanup", action="store_true", help="Terminate pod after completion")
|
||||
parser.add_argument("--yes", action="store_true", help="Skip confirmation prompts")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show plan without executing")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Display header
|
||||
console.print(Panel.fit(
|
||||
"[bold cyan]🦊 Foxhunt Full RunPod Deployment Orchestrator[/bold cyan]",
|
||||
border_style="cyan"
|
||||
))
|
||||
|
||||
# Check prerequisites
|
||||
if not check_prerequisites():
|
||||
console.print("\n[red]✗ Prerequisites check failed[/red]")
|
||||
console.print(" Fix the issues above and try again")
|
||||
return 1
|
||||
|
||||
# Step 1: Upload binaries and test data (optional)
|
||||
if not args.skip_upload:
|
||||
console.print("\n[bold]Step 1: Upload Binaries and Test Data[/bold]")
|
||||
if not upload_to_volume(force=args.force_upload, dry_run=args.dry_run):
|
||||
console.print("\n[red]✗ Upload failed[/red]")
|
||||
return 1
|
||||
else:
|
||||
console.print("\n[bold]Step 1: Upload[/bold] [dim](skipped)[/dim]")
|
||||
|
||||
# Step 2: Deploy pod
|
||||
console.print("\n[bold]Step 2: Deploy Training Pod[/bold]")
|
||||
pod_id = deploy_pod(args, dry_run=args.dry_run)
|
||||
if not pod_id:
|
||||
console.print("\n[red]✗ Deployment failed[/red]")
|
||||
return 1
|
||||
|
||||
# Step 3: Monitor (handled by runpod_deploy_production.py if --monitor flag passed)
|
||||
if args.monitor:
|
||||
console.print("\n[bold]Step 3: Monitor Training[/bold]")
|
||||
console.print("[dim](Monitoring handled by deployment script)[/dim]")
|
||||
|
||||
# Step 4: Download models
|
||||
if not args.dry_run:
|
||||
console.print("\n[bold]Step 4: Download Trained Models[/bold]")
|
||||
console.print("[yellow]⚠ Waiting for training to complete...[/yellow]")
|
||||
console.print(" Run this manually after training: ./scripts/download_models_from_volume.py")
|
||||
# TODO: Poll pod status and download automatically when complete
|
||||
|
||||
# Step 5: Cleanup (optional)
|
||||
if args.cleanup:
|
||||
console.print("\n[bold]Step 5: Cleanup Pod[/bold]")
|
||||
cleanup_pod(pod_id, dry_run=args.dry_run)
|
||||
|
||||
# Final summary
|
||||
console.print("\n" + "=" * 70)
|
||||
console.print("[bold green]✓ Deployment Orchestration Complete![/bold green]")
|
||||
console.print("=" * 70)
|
||||
|
||||
if not args.dry_run:
|
||||
console.print(f"\n📊 Pod ID: {pod_id}")
|
||||
console.print(f"🔗 Console: https://www.runpod.io/console/pods/{pod_id}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n\n[yellow]⚠ Interrupted by user[/yellow]")
|
||||
sys.exit(130)
|
||||
331
scripts/archive/runpod_upload.sh
Executable file
331
scripts/archive/runpod_upload.sh
Executable file
@@ -0,0 +1,331 @@
|
||||
#!/bin/bash
|
||||
################################################################################
|
||||
# runpod_upload.sh - Upload Foxhunt binaries and data to Runpod Network Volume
|
||||
#
|
||||
# Purpose:
|
||||
# Builds release binaries with CUDA support and uploads them to Runpod
|
||||
# Network Volume using AWS S3-compatible API. This script runs on LOCAL
|
||||
# CLIENT ONLY. Runpod pods will mount the volume and access files at
|
||||
# /runpod-volume/.
|
||||
#
|
||||
# Prerequisites:
|
||||
# 1. AWS CLI installed: apt-get install awscli
|
||||
# 2. Runpod credentials configured in ~/.aws/credentials:
|
||||
# [runpod]
|
||||
# aws_access_key_id = <your-runpod-user-id>
|
||||
# aws_secret_access_key = <your-runpod-api-key>
|
||||
# 3. Environment variable RUNPOD_S3_ENDPOINT set to your Runpod endpoint
|
||||
# Example: export RUNPOD_S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
|
||||
#
|
||||
# Environment Variables:
|
||||
# - RUNPOD_S3_ENDPOINT: Runpod S3-compatible endpoint (required)
|
||||
# - AWS_PROFILE: AWS profile to use (default: runpod)
|
||||
#
|
||||
# Usage:
|
||||
# export RUNPOD_S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
|
||||
# ./scripts/runpod_upload.sh
|
||||
#
|
||||
# What gets uploaded:
|
||||
# - Release binaries: train_tft_parquet, train_dqn, train_ppo, train_mamba2_dbn
|
||||
# - Test data: 9 Parquet files (ES.FUT, NQ.FUT, 6E.FUT, ZN.FUT variants)
|
||||
# - Directory structure: /runpod-volume/{bin,test_data,models}
|
||||
#
|
||||
# Post-upload access on Runpod pod:
|
||||
# - Binaries: /runpod-volume/bin/train_tft_parquet
|
||||
# - Data: /runpod-volume/test_data/ES_FUT_180d.parquet
|
||||
# - Models: /runpod-volume/models/ (for trained output)
|
||||
################################################################################
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# ANSI color codes for output formatting
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
BLUE='\033[0;34m'
|
||||
BOLD='\033[1m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
################################################################################
|
||||
# Configuration Validation
|
||||
################################################################################
|
||||
|
||||
echo -e "${BOLD}=========================================${NC}"
|
||||
echo -e "${BOLD}Foxhunt Runpod Upload Script${NC}"
|
||||
echo -e "${BOLD}=========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Validate RUNPOD_S3_ENDPOINT environment variable
|
||||
if [ -z "$RUNPOD_S3_ENDPOINT" ]; then
|
||||
echo -e "${RED}ERROR: RUNPOD_S3_ENDPOINT environment variable not set${NC}"
|
||||
echo ""
|
||||
echo "Please set your Runpod S3 endpoint:"
|
||||
echo " export RUNPOD_S3_ENDPOINT=\"https://s3api-<datacenter>.runpod.io\""
|
||||
echo ""
|
||||
echo "Find your endpoint in Runpod console:"
|
||||
echo " 1. Go to https://www.runpod.io/console/user/settings"
|
||||
echo " 2. Navigate to 'Network Volumes'"
|
||||
echo " 3. Copy the S3 endpoint URL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set AWS profile (default: runpod)
|
||||
AWS_PROFILE="${AWS_PROFILE:-runpod}"
|
||||
|
||||
# Validate AWS profile exists
|
||||
if ! aws configure list --profile "$AWS_PROFILE" &>/dev/null; then
|
||||
echo -e "${RED}ERROR: AWS profile '$AWS_PROFILE' not found${NC}"
|
||||
echo ""
|
||||
echo "Please configure Runpod credentials in ~/.aws/credentials:"
|
||||
echo " [runpod]"
|
||||
echo " aws_access_key_id = <your-runpod-user-id>"
|
||||
echo " aws_secret_access_key = <your-runpod-api-key>"
|
||||
echo ""
|
||||
echo "Find your credentials in Runpod console:"
|
||||
echo " 1. Go to https://www.runpod.io/console/user/settings"
|
||||
echo " 2. Navigate to 'API Keys'"
|
||||
echo " 3. Copy User ID and API Key"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Configuration summary
|
||||
echo -e "${BLUE}Configuration:${NC}"
|
||||
echo " Endpoint: $RUNPOD_S3_ENDPOINT"
|
||||
echo " AWS Profile: $AWS_PROFILE"
|
||||
echo " Upload Target: s3://runpod-volume/foxhunt/"
|
||||
echo ""
|
||||
|
||||
# Project root directory
|
||||
FOXHUNT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$FOXHUNT_ROOT"
|
||||
|
||||
################################################################################
|
||||
# Step 1: Build Release Binaries with CUDA
|
||||
################################################################################
|
||||
|
||||
echo -e "${BOLD}Step 1/5: Building release binaries with CUDA support...${NC}"
|
||||
echo "This may take 5-10 minutes depending on system."
|
||||
echo ""
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Build with release profile and CUDA features
|
||||
if cargo build --release --features cuda --workspace; then
|
||||
END_TIME=$(date +%s)
|
||||
BUILD_DURATION=$((END_TIME - START_TIME))
|
||||
BUILD_MIN=$(awk "BEGIN {printf \"%.2f\", $BUILD_DURATION/60}")
|
||||
echo -e "${GREEN}✅ Build complete in ${BUILD_MIN} minutes${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ ERROR: Cargo build failed${NC}"
|
||||
echo "Debug steps:"
|
||||
echo " 1. Check Rust toolchain: rustc --version"
|
||||
echo " 2. Verify CUDA installation: nvidia-smi"
|
||||
echo " 3. Check build logs above for specific errors"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
################################################################################
|
||||
# Step 2: Upload Training Binaries
|
||||
################################################################################
|
||||
|
||||
echo -e "${BOLD}Step 2/5: Uploading training binaries...${NC}"
|
||||
echo ""
|
||||
|
||||
# List of training binaries to upload
|
||||
BINARIES=(
|
||||
"train_tft_parquet"
|
||||
"train_dqn"
|
||||
"train_ppo"
|
||||
"train_mamba2_dbn"
|
||||
)
|
||||
|
||||
TOTAL_BIN_SIZE=0
|
||||
UPLOADED_COUNT=0
|
||||
|
||||
for binary in "${BINARIES[@]}"; do
|
||||
SRC="$FOXHUNT_ROOT/target/release/examples/$binary"
|
||||
|
||||
if [ -f "$SRC" ]; then
|
||||
# Get file size
|
||||
SIZE=$(stat -c%s "$SRC" 2>/dev/null || stat -f%z "$SRC" 2>/dev/null)
|
||||
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
|
||||
TOTAL_BIN_SIZE=$((TOTAL_BIN_SIZE + SIZE))
|
||||
|
||||
# Upload to Runpod S3
|
||||
DEST="s3://se3zdnb5o4/binaries/$binary"
|
||||
|
||||
if aws s3 cp "$SRC" "$DEST" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT"; then
|
||||
echo -e " ${GREEN}✓${NC} $binary (${SIZE_MB} MB)"
|
||||
UPLOADED_COUNT=$((UPLOADED_COUNT + 1))
|
||||
else
|
||||
echo -e " ${RED}✗${NC} $binary upload failed"
|
||||
fi
|
||||
else
|
||||
echo -e " ${YELLOW}⚠${NC} $binary not found (skipping)"
|
||||
fi
|
||||
done
|
||||
|
||||
TOTAL_BIN_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BIN_SIZE/1024/1024}")
|
||||
echo ""
|
||||
echo "Uploaded $UPLOADED_COUNT binaries (${TOTAL_BIN_MB} MB total)"
|
||||
echo ""
|
||||
|
||||
################################################################################
|
||||
# Step 3: Upload Test Data (Parquet Files)
|
||||
################################################################################
|
||||
|
||||
echo -e "${BOLD}Step 3/5: Uploading test data (Parquet files)...${NC}"
|
||||
echo ""
|
||||
|
||||
PARQUET_FILES=(
|
||||
"ES_FUT_180d.parquet"
|
||||
"NQ_FUT_180d.parquet"
|
||||
"6E_FUT_180d.parquet"
|
||||
"ZN_FUT_90d.parquet"
|
||||
"ZN_FUT_90d_clean.parquet"
|
||||
"ES_FUT_small.parquet"
|
||||
"NQ_FUT_small.parquet"
|
||||
"6E_FUT_small.parquet"
|
||||
"ZN_FUT_small.parquet"
|
||||
)
|
||||
|
||||
TOTAL_DATA_SIZE=0
|
||||
UPLOADED_DATA_COUNT=0
|
||||
|
||||
for file in "${PARQUET_FILES[@]}"; do
|
||||
SRC="$FOXHUNT_ROOT/test_data/$file"
|
||||
|
||||
if [ -f "$SRC" ]; then
|
||||
# Get file size
|
||||
SIZE=$(stat -c%s "$SRC" 2>/dev/null || stat -f%z "$SRC" 2>/dev/null)
|
||||
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
|
||||
TOTAL_DATA_SIZE=$((TOTAL_DATA_SIZE + SIZE))
|
||||
|
||||
# Upload to Runpod S3
|
||||
DEST="s3://se3zdnb5o4/test_data/$file"
|
||||
|
||||
if aws s3 cp "$SRC" "$DEST" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT"; then
|
||||
echo -e " ${GREEN}✓${NC} $file (${SIZE_MB} MB)"
|
||||
UPLOADED_DATA_COUNT=$((UPLOADED_DATA_COUNT + 1))
|
||||
else
|
||||
echo -e " ${RED}✗${NC} $file upload failed"
|
||||
fi
|
||||
else
|
||||
echo -e " ${YELLOW}⚠${NC} $file not found (skipping)"
|
||||
fi
|
||||
done
|
||||
|
||||
TOTAL_DATA_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_DATA_SIZE/1024/1024}")
|
||||
echo ""
|
||||
echo "Uploaded $UPLOADED_DATA_COUNT data files (${TOTAL_DATA_MB} MB total)"
|
||||
echo ""
|
||||
|
||||
################################################################################
|
||||
# Step 4: Create Models Directory
|
||||
################################################################################
|
||||
|
||||
echo -e "${BOLD}Step 4/5: Creating /runpod-volume/models/ directory...${NC}"
|
||||
echo ""
|
||||
|
||||
# Create empty marker file to ensure directory exists
|
||||
MARKER_FILE=$(mktemp)
|
||||
echo "Model storage directory created by runpod_upload.sh" > "$MARKER_FILE"
|
||||
echo "Date: $(date)" >> "$MARKER_FILE"
|
||||
|
||||
if aws s3 cp "$MARKER_FILE" "s3://se3zdnb5o4/models/.directory_created" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT"; then
|
||||
echo -e "${GREEN}✅ Models directory created${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Warning: Could not create models directory marker${NC}"
|
||||
fi
|
||||
|
||||
rm -f "$MARKER_FILE"
|
||||
echo ""
|
||||
|
||||
################################################################################
|
||||
# Step 5: Verify Uploads
|
||||
################################################################################
|
||||
|
||||
echo -e "${BOLD}Step 5/5: Verifying uploads...${NC}"
|
||||
echo ""
|
||||
|
||||
# List uploaded files to verify
|
||||
echo "Verifying binaries/ directory:"
|
||||
if aws s3 ls "s3://se3zdnb5o4/binaries/" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT" | head -10; then
|
||||
echo -e "${GREEN}✅ Binaries verified${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Could not verify binaries${NC}"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Verifying test_data/ directory:"
|
||||
if aws s3 ls "s3://se3zdnb5o4/test_data/" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT" | head -10; then
|
||||
echo -e "${GREEN}✅ Test data verified${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Could not verify test data${NC}"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Verifying models/ directory:"
|
||||
if aws s3 ls "s3://se3zdnb5o4/models/" \
|
||||
--profile "$AWS_PROFILE" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT" | head -5; then
|
||||
echo -e "${GREEN}✅ Models directory verified${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Warning: Models directory not found (non-critical)${NC}"
|
||||
fi
|
||||
|
||||
################################################################################
|
||||
# Upload Summary
|
||||
################################################################################
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}=========================================${NC}"
|
||||
echo -e "${BOLD}${GREEN}✅ Upload Complete!${NC}${BOLD}${NC}"
|
||||
echo -e "${BOLD}=========================================${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}Upload Summary:${NC}"
|
||||
echo " Binaries: $UPLOADED_COUNT files (${TOTAL_BIN_MB} MB)"
|
||||
echo " Test Data: $UPLOADED_DATA_COUNT files (${TOTAL_DATA_MB} MB)"
|
||||
TOTAL_SIZE_MB=$(awk "BEGIN {printf \"%.2f\", ($TOTAL_BIN_SIZE + $TOTAL_DATA_SIZE)/1024/1024}")
|
||||
echo " Total: ${TOTAL_SIZE_MB} MB"
|
||||
echo ""
|
||||
|
||||
echo -e "${BLUE}Files available at (on Runpod pod):${NC}"
|
||||
echo " Binaries: /runpod-volume/binaries/"
|
||||
echo " Test Data: /runpod-volume/test_data/"
|
||||
echo " Models (output): /runpod-volume/models/"
|
||||
echo ""
|
||||
|
||||
echo -e "${BLUE}Next Steps:${NC}"
|
||||
echo "1. Create Runpod pod with Tesla V100 16GB GPU"
|
||||
echo "2. Mount network volume (se3zdnb5o4) to /runpod-volume"
|
||||
echo "3. Run training on pod:"
|
||||
echo " cd /runpod-volume"
|
||||
echo " ./binaries/train_tft_parquet \\"
|
||||
echo " --parquet-file ./test_data/ES_FUT_180d.parquet \\"
|
||||
echo " --epochs 50 \\"
|
||||
echo " --output-dir ./models/tft_fp32"
|
||||
echo ""
|
||||
|
||||
echo -e "${BLUE}To download trained models:${NC}"
|
||||
echo " aws s3 sync \\"
|
||||
echo " --profile runpod \\"
|
||||
echo " --endpoint-url $RUNPOD_S3_ENDPOINT \\"
|
||||
echo " s3://se3zdnb5o4/models/ ./models/"
|
||||
echo ""
|
||||
|
||||
echo -e "${BOLD}=========================================${NC}"
|
||||
echo -e "${GREEN}Ready for Runpod deployment!${NC}"
|
||||
echo -e "${BOLD}=========================================${NC}"
|
||||
123
scripts/archive/scan_gpus.py
Executable file
123
scripts/archive/scan_gpus.py
Executable file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod GPU Scanner
|
||||
Queries RunPod API to show available SECURE cloud GPUs with ≥16GB VRAM.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env.runpod
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
GRAPHQL_ENDPOINT = "https://api.runpod.io/graphql"
|
||||
|
||||
# GraphQL query to fetch GPU types
|
||||
QUERY = """
|
||||
{
|
||||
gpuTypes {
|
||||
id
|
||||
displayName
|
||||
memoryInGb
|
||||
secureCloud
|
||||
communityCloud
|
||||
lowestPrice(input: {gpuCount: 1}) {
|
||||
uninterruptablePrice
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
def query_runpod_api():
|
||||
"""Query RunPod GraphQL API for GPU types."""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"query": QUERY
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(GRAPHQL_ENDPOINT, json=payload, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"ERROR: Failed to query RunPod API: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
def filter_and_sort_gpus(data):
|
||||
"""Filter for secure cloud GPUs with ≥16GB VRAM, sorted by price."""
|
||||
gpu_types = data.get('data', {}).get('gpuTypes', [])
|
||||
|
||||
# Filter criteria:
|
||||
# 1. memoryInGb >= 16
|
||||
# 2. secureCloud > 0 (available in secure cloud)
|
||||
# 3. Has pricing information
|
||||
filtered_gpus = []
|
||||
for gpu in gpu_types:
|
||||
memory = gpu.get('memoryInGb', 0)
|
||||
secure_count = gpu.get('secureCloud', 0)
|
||||
lowest_price = gpu.get('lowestPrice', {})
|
||||
price = lowest_price.get('uninterruptablePrice') if lowest_price else None
|
||||
|
||||
if memory >= 16 and secure_count > 0 and price is not None:
|
||||
filtered_gpus.append({
|
||||
'name': gpu.get('displayName', 'Unknown'),
|
||||
'vram': memory,
|
||||
'price': float(price),
|
||||
'available': secure_count
|
||||
})
|
||||
|
||||
# Sort by price (cheapest first)
|
||||
filtered_gpus.sort(key=lambda x: x['price'])
|
||||
|
||||
return filtered_gpus
|
||||
|
||||
def display_gpus(gpus):
|
||||
"""Display GPU information in clean format."""
|
||||
if not gpus:
|
||||
print("No secure cloud GPUs found with ≥16GB VRAM in EUR-IS region.")
|
||||
return
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("RUNPOD SECURE CLOUD GPUs (≥16GB VRAM) - EUR-IS REGION")
|
||||
print("="*70)
|
||||
print(f"{'GPU Name':<30} {'VRAM':<10} {'Price/hr':<12} {'Available':<10}")
|
||||
print("-"*70)
|
||||
|
||||
for gpu in gpus:
|
||||
name = gpu['name']
|
||||
vram = f"{gpu['vram']}GB"
|
||||
price = f"${gpu['price']:.3f}"
|
||||
available = f"{gpu['available']} pods"
|
||||
print(f"{name:<30} {vram:<10} {price:<12} {available:<10}")
|
||||
|
||||
print("="*70)
|
||||
print(f"Total GPUs found: {len(gpus)}")
|
||||
print()
|
||||
|
||||
def main():
|
||||
"""Main execution function."""
|
||||
print("Querying RunPod API...")
|
||||
|
||||
# Query API
|
||||
data = query_runpod_api()
|
||||
|
||||
# Filter and sort GPUs
|
||||
gpus = filter_and_sort_gpus(data)
|
||||
|
||||
# Display results
|
||||
display_gpus(gpus)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
127
scripts/archive/terminate_failing_pod.py
Executable file
127
scripts/archive/terminate_failing_pod.py
Executable file
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Terminate failing RunPod pod - non-interactive version
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
REST_API_URL = "https://rest.runpod.io/v1/pods"
|
||||
|
||||
def terminate_pod(pod_id):
|
||||
"""Terminate a RunPod pod."""
|
||||
print(f"\n🗑️ Terminating pod {pod_id}...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.delete(
|
||||
f"{REST_API_URL}/{pod_id}",
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code in [200, 204]:
|
||||
print(f" ✅ Pod {pod_id} terminated successfully")
|
||||
return True
|
||||
else:
|
||||
print(f" ⚠️ Termination failed (status {response.status_code}): {response.text[:200]}")
|
||||
return False
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to terminate pod: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def get_pod_status(pod_id):
|
||||
"""Get current status of a pod."""
|
||||
print(f"\n🔍 Checking pod {pod_id} status...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{REST_API_URL}/{pod_id}",
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
pod_data = response.json()
|
||||
print(f" Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
|
||||
print(f" Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}")
|
||||
|
||||
machine = pod_data.get('machine', {})
|
||||
datacenter = machine.get('dataCenterId', 'N/A')
|
||||
print(f" Datacenter: {datacenter}")
|
||||
|
||||
gpu_info = machine.get('gpuType', {})
|
||||
print(f" GPU: {gpu_info.get('displayName', 'N/A')}")
|
||||
|
||||
return pod_data
|
||||
elif response.status_code == 404:
|
||||
print(f" ⚠️ Pod not found (may already be terminated)")
|
||||
return None
|
||||
else:
|
||||
print(f" ⚠️ Failed to get pod status (status {response.status_code}): {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to check pod status: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print("="*70)
|
||||
print("TERMINATE FAILING RUNPOD POD")
|
||||
print("="*70)
|
||||
|
||||
failing_pod_id = "3518shhoirbof4"
|
||||
|
||||
# Check pod status first
|
||||
pod_data = get_pod_status(failing_pod_id)
|
||||
|
||||
if pod_data:
|
||||
# Terminate the pod
|
||||
if terminate_pod(failing_pod_id):
|
||||
print("\n✅ Pod terminated successfully!")
|
||||
print("\n📋 NEXT STEPS:")
|
||||
print("="*70)
|
||||
print("1. Deployment script has been fixed (EUR-IS-1 only)")
|
||||
print("2. Wait 30 seconds for pod cleanup to complete")
|
||||
print("3. Deploy new pod with:")
|
||||
print()
|
||||
print(" cd /home/jgrusewski/Work/foxhunt")
|
||||
print(" ./scripts/runpod_deploy.py \\")
|
||||
print(" --image jgrusewski/foxhunt:debug \\")
|
||||
print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
|
||||
print()
|
||||
print("4. Verify new pod is in EUR-IS-1 datacenter")
|
||||
print("5. Check logs show volume mounted successfully")
|
||||
print("="*70)
|
||||
else:
|
||||
print("\n❌ Failed to terminate pod")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"\n⚠️ Pod {failing_pod_id} not found or already terminated")
|
||||
print("\n✅ Ready to deploy new pod!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
241
scripts/archive/test_binary_sync.sh
Executable file
241
scripts/archive/test_binary_sync.sh
Executable file
@@ -0,0 +1,241 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Binary Sync Validation Test Script
|
||||
# =============================================================================
|
||||
# Purpose: Validate the binary volume sync fix works correctly
|
||||
# Tests:
|
||||
# 1. Local binary has correct timestamp metadata
|
||||
# 2. S3 upload includes timestamp in metadata
|
||||
# 3. S3 binary can be downloaded and verified
|
||||
# 4. Timestamps are preserved and comparable
|
||||
#
|
||||
# Usage: ./test_binary_sync.sh <binary_name>
|
||||
# Example: ./test_binary_sync.sh hyperopt_mamba2_demo
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
BINARY_NAME="$1"
|
||||
if [ -z "$BINARY_NAME" ]; then
|
||||
echo "Usage: $0 <binary_name>"
|
||||
echo "Example: $0 hyperopt_mamba2_demo"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Configuration
|
||||
LOCAL_PATH="target/release/examples/${BINARY_NAME}"
|
||||
S3_KEY_CURRENT="binaries/current/${BINARY_NAME}"
|
||||
S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
|
||||
S3_BUCKET="se3zdnb5o4"
|
||||
S3_PROFILE="runpod"
|
||||
|
||||
# Note: Upload creates two S3 objects:
|
||||
# 1. binaries/timestamped/${BINARY_NAME}_YYYYMMDD_HHMMSS (timestamped version)
|
||||
# 2. binaries/current/${BINARY_NAME} (symlink/copy to latest)
|
||||
# This test validates the 'current' version which is what deployments use
|
||||
|
||||
echo "========================================================================"
|
||||
echo "BINARY SYNC VALIDATION TEST"
|
||||
echo "========================================================================"
|
||||
echo "Binary: $BINARY_NAME"
|
||||
echo "Local: $LOCAL_PATH"
|
||||
echo "S3: s3://$S3_BUCKET/$S3_KEY_CURRENT"
|
||||
echo "========================================================================"
|
||||
|
||||
# =============================================================================
|
||||
# TEST 1: Verify local binary exists and has timestamp
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo "TEST 1: Local Binary Validation"
|
||||
echo "------------------------------------------------------------------------"
|
||||
|
||||
if [ ! -f "$LOCAL_PATH" ]; then
|
||||
echo "❌ FAIL: Local binary not found at $LOCAL_PATH"
|
||||
echo " Build it with: cargo build --release --example $BINARY_NAME"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Local binary exists"
|
||||
|
||||
# Get local file modification time
|
||||
LOCAL_MTIME=$(stat -c %Y "$LOCAL_PATH" 2>/dev/null || stat -f %m "$LOCAL_PATH" 2>/dev/null)
|
||||
LOCAL_SIZE=$(stat -c %s "$LOCAL_PATH" 2>/dev/null || stat -f %z "$LOCAL_PATH" 2>/dev/null)
|
||||
LOCAL_TIMESTAMP=$(date -Iseconds -d @"$LOCAL_MTIME" 2>/dev/null || date -Iseconds -r "$LOCAL_MTIME" 2>/dev/null)
|
||||
|
||||
echo "✅ Local metadata extracted:"
|
||||
echo " Size: $LOCAL_SIZE bytes ($(echo "scale=1; $LOCAL_SIZE/1024/1024" | bc) MB)"
|
||||
echo " Modified: $LOCAL_TIMESTAMP"
|
||||
|
||||
# Calculate local SHA256
|
||||
LOCAL_SHA256=$(sha256sum "$LOCAL_PATH" | awk '{print $1}')
|
||||
echo " SHA256: $LOCAL_SHA256"
|
||||
|
||||
# =============================================================================
|
||||
# TEST 2: Check S3 binary metadata (if exists)
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo "TEST 2: S3 Binary Metadata Validation"
|
||||
echo "------------------------------------------------------------------------"
|
||||
|
||||
# Check if S3 binary exists
|
||||
if ! aws s3api head-object \
|
||||
--bucket "$S3_BUCKET" \
|
||||
--key "$S3_KEY_CURRENT" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--profile "$S3_PROFILE" &>/dev/null; then
|
||||
echo "⚠️ S3 binary does not exist yet"
|
||||
echo " Run: python3 scripts/runpod_deploy.py --skip-upload=false --dry-run"
|
||||
echo " This will upload the binary with timestamp metadata"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "✅ S3 binary exists"
|
||||
|
||||
# Get S3 metadata
|
||||
S3_METADATA=$(aws s3api head-object \
|
||||
--bucket "$S3_BUCKET" \
|
||||
--key "$S3_KEY_CURRENT" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--profile "$S3_PROFILE")
|
||||
|
||||
S3_SIZE=$(echo "$S3_METADATA" | jq -r '.ContentLength')
|
||||
S3_MODIFIED=$(echo "$S3_METADATA" | jq -r '.LastModified')
|
||||
S3_SHA256=$(echo "$S3_METADATA" | jq -r '.Metadata.sha256 // "N/A"')
|
||||
S3_BUILD_TIMESTAMP=$(echo "$S3_METADATA" | jq -r '.Metadata.build_timestamp // "N/A"')
|
||||
S3_UPLOADED=$(echo "$S3_METADATA" | jq -r '.Metadata.uploaded // "N/A"')
|
||||
|
||||
echo "✅ S3 metadata extracted:"
|
||||
echo " Size: $S3_SIZE bytes ($(echo "scale=1; $S3_SIZE/1024/1024" | bc) MB)"
|
||||
echo " Last Modified: $S3_MODIFIED"
|
||||
echo " SHA256: $S3_SHA256"
|
||||
echo " Build Timestamp: $S3_BUILD_TIMESTAMP"
|
||||
echo " Uploaded: $S3_UPLOADED"
|
||||
|
||||
# =============================================================================
|
||||
# TEST 3: Validate timestamp metadata
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo "TEST 3: Timestamp Validation"
|
||||
echo "------------------------------------------------------------------------"
|
||||
|
||||
if [ "$S3_BUILD_TIMESTAMP" = "N/A" ]; then
|
||||
echo "⚠️ WARNING: S3 binary missing build_timestamp metadata"
|
||||
echo " This binary was uploaded with old version of runpod_deploy.py"
|
||||
echo " Re-upload to add timestamp: python3 scripts/runpod_deploy.py --force-upload --dry-run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "✅ S3 binary has timestamp metadata"
|
||||
|
||||
# Compare timestamps (allowing for timezone differences)
|
||||
if [ "$LOCAL_TIMESTAMP" = "$S3_BUILD_TIMESTAMP" ]; then
|
||||
echo "✅ PASS: Timestamps match exactly"
|
||||
echo " Local: $LOCAL_TIMESTAMP"
|
||||
echo " S3: $S3_BUILD_TIMESTAMP"
|
||||
else
|
||||
echo "⚠️ Timestamps differ (expected if binary was rebuilt):"
|
||||
echo " Local: $LOCAL_TIMESTAMP"
|
||||
echo " S3: $S3_BUILD_TIMESTAMP"
|
||||
|
||||
# Check if local is newer (requires re-upload)
|
||||
LOCAL_EPOCH=$(date -d "$LOCAL_TIMESTAMP" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "$LOCAL_TIMESTAMP" +%s 2>/dev/null)
|
||||
S3_EPOCH=$(date -d "$S3_BUILD_TIMESTAMP" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "$S3_BUILD_TIMESTAMP" +%s 2>/dev/null)
|
||||
|
||||
if [ "$LOCAL_EPOCH" -gt "$S3_EPOCH" ]; then
|
||||
echo " ⚠️ Local binary is NEWER - re-upload required!"
|
||||
echo " Run: python3 scripts/runpod_deploy.py --force-upload --dry-run"
|
||||
else
|
||||
echo " ℹ️ S3 binary is same or newer - deployment will use S3 version"
|
||||
fi
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# TEST 4: Validate SHA256 checksum
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo "TEST 4: Checksum Validation"
|
||||
echo "------------------------------------------------------------------------"
|
||||
|
||||
if [ "$S3_SHA256" = "N/A" ]; then
|
||||
echo "⚠️ WARNING: S3 binary missing SHA256 metadata"
|
||||
echo " Re-upload to add checksum: python3 scripts/runpod_deploy.py --force-upload --dry-run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "✅ S3 binary has SHA256 metadata"
|
||||
|
||||
# Compare checksums
|
||||
if [ "$LOCAL_SHA256" = "$S3_SHA256" ]; then
|
||||
echo "✅ PASS: Checksums match - binaries are identical"
|
||||
echo " SHA256: $LOCAL_SHA256"
|
||||
else
|
||||
echo "❌ FAIL: Checksums DO NOT MATCH"
|
||||
echo " Local: $LOCAL_SHA256"
|
||||
echo " S3: $S3_SHA256"
|
||||
echo ""
|
||||
echo " REQUIRED ACTION:"
|
||||
echo " 1. Force re-upload: python3 scripts/runpod_deploy.py --force-upload --dry-run"
|
||||
echo " 2. Or delete and re-upload:"
|
||||
echo " aws s3 rm s3://$S3_BUCKET/$S3_KEY_CURRENT --endpoint-url $S3_ENDPOINT --profile $S3_PROFILE"
|
||||
echo " python3 scripts/runpod_deploy.py --dry-run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# TEST 5: Validate binary arguments (for hyperopt binaries)
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo "TEST 5: Binary Arguments Validation"
|
||||
echo "------------------------------------------------------------------------"
|
||||
|
||||
if [[ "$BINARY_NAME" == hyperopt* ]]; then
|
||||
if ! "$LOCAL_PATH" --help 2>&1 | grep -q -- "--base-dir"; then
|
||||
echo "❌ FAIL: Local binary missing --base-dir argument"
|
||||
echo " This binary was built BEFORE VarMap fix!"
|
||||
echo " Rebuild: cargo clean && cargo build --release --example $BINARY_NAME"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ PASS: Local binary has --base-dir argument (VarMap fix applied)"
|
||||
|
||||
# Test S3 binary (download temporarily)
|
||||
TEMP_S3="/tmp/${BINARY_NAME}_s3_test_$$"
|
||||
echo " Downloading S3 binary for argument test..."
|
||||
aws s3 cp \
|
||||
"s3://${S3_BUCKET}/${S3_KEY_CURRENT}" \
|
||||
"$TEMP_S3" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--profile "$S3_PROFILE" \
|
||||
--quiet
|
||||
|
||||
chmod +x "$TEMP_S3"
|
||||
|
||||
if ! "$TEMP_S3" --help 2>&1 | grep -q -- "--base-dir"; then
|
||||
echo "❌ FAIL: S3 binary missing --base-dir argument"
|
||||
echo " This binary was uploaded BEFORE VarMap fix!"
|
||||
echo " Re-upload: python3 scripts/runpod_deploy.py --force-upload --dry-run"
|
||||
rm -f "$TEMP_S3"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ PASS: S3 binary has --base-dir argument (VarMap fix applied)"
|
||||
|
||||
rm -f "$TEMP_S3"
|
||||
else
|
||||
echo "ℹ️ SKIP: Not a hyperopt binary, no argument validation needed"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# SUMMARY
|
||||
# =============================================================================
|
||||
echo ""
|
||||
echo "========================================================================"
|
||||
echo "✅ ALL TESTS PASSED"
|
||||
echo "========================================================================"
|
||||
echo "Binary: $BINARY_NAME"
|
||||
echo "Local Size: $(echo "scale=1; $LOCAL_SIZE/1024/1024" | bc) MB"
|
||||
echo "Local Timestamp: $LOCAL_TIMESTAMP"
|
||||
echo "S3 Timestamp: $S3_BUILD_TIMESTAMP"
|
||||
echo "Checksum: $LOCAL_SHA256"
|
||||
echo ""
|
||||
echo "DEPLOYMENT STATUS: ✅ READY"
|
||||
echo "This binary is safe to deploy to Runpod."
|
||||
echo "The entrypoint will sync it from S3 to volume on pod startup."
|
||||
echo "========================================================================"
|
||||
96
scripts/archive/test_binary_validation.sh
Executable file
96
scripts/archive/test_binary_validation.sh
Executable file
@@ -0,0 +1,96 @@
|
||||
#!/bin/bash
|
||||
# Test binary validation system
|
||||
|
||||
set -e
|
||||
|
||||
echo "========================================"
|
||||
echo "Binary Validation System Test Suite"
|
||||
echo "========================================"
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# Test 1: Validate existing binary
|
||||
echo ""
|
||||
echo "Test 1: Validate hyperopt_mamba2_demo binary"
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [ ! -f "target/release/examples/hyperopt_mamba2_demo" ]; then
|
||||
echo "⚠️ Binary not found, building first..."
|
||||
cargo build -p ml --example hyperopt_mamba2_demo --release --features cuda
|
||||
fi
|
||||
|
||||
./scripts/validate_binary.sh hyperopt_mamba2_demo
|
||||
echo "✅ Test 1 passed: Validation script works"
|
||||
|
||||
# Test 2: Local binary smoke test
|
||||
echo ""
|
||||
echo "Test 2: Smoke test binary functionality"
|
||||
echo "----------------------------------------"
|
||||
|
||||
if target/release/examples/hyperopt_mamba2_demo --help | grep -q "base-dir"; then
|
||||
echo "✅ Test 2 passed: Binary has correct CLI arguments"
|
||||
else
|
||||
echo "❌ Test 2 failed: Binary missing --base-dir argument"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 3: Checksum comparison
|
||||
echo ""
|
||||
echo "Test 3: Checksum calculation"
|
||||
echo "----------------------------------------"
|
||||
|
||||
LOCAL_SHA=$(sha256sum target/release/examples/hyperopt_mamba2_demo | awk '{print $1}')
|
||||
echo "Local SHA256: $LOCAL_SHA"
|
||||
|
||||
if [ -n "$LOCAL_SHA" ] && [ ${#LOCAL_SHA} -eq 64 ]; then
|
||||
echo "✅ Test 3 passed: Checksum calculated correctly (64 chars)"
|
||||
else
|
||||
echo "❌ Test 3 failed: Invalid checksum format"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 4: Python validation integration
|
||||
echo ""
|
||||
echo "Test 4: Python script integration"
|
||||
echo "----------------------------------------"
|
||||
|
||||
if python3 -c "
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
project_root = os.getcwd()
|
||||
result = subprocess.run(
|
||||
[os.path.join(project_root, 'scripts/validate_binary.sh'), 'hyperopt_mamba2_demo'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=project_root
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
print('✅ Test 4 passed: Python can call validation script')
|
||||
exit(0)
|
||||
elif 'VALIDATION: LOCAL_ONLY' in result.stdout:
|
||||
print('✅ Test 4 passed: Validation works (local-only mode)')
|
||||
exit(0)
|
||||
else:
|
||||
print('❌ Test 4 failed: Python integration broken')
|
||||
print(result.stdout)
|
||||
print(result.stderr)
|
||||
exit(1)
|
||||
"; then
|
||||
:
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "✅ All validation tests passed"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Validation system is ready for production use."
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo "1. Test deployment with: python3 scripts/runpod_deploy.py --dry-run"
|
||||
echo "2. Review SAFE_DEPLOYMENT_CHECKLIST.md for full workflow"
|
||||
303
scripts/archive/test_runpod_auth.py
Executable file
303
scripts/archive/test_runpod_auth.py
Executable file
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod API Authentication Test
|
||||
Tests that API key authentication works correctly before attempting pod creation.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
def get_api_key() -> Optional[str]:
|
||||
"""Get RunPod API key from environment"""
|
||||
api_key = os.environ.get('RUNPOD_API_KEY')
|
||||
if not api_key:
|
||||
print("❌ ERROR: RUNPOD_API_KEY environment variable not set")
|
||||
print("Set it with: export RUNPOD_API_KEY='your_api_key'")
|
||||
return None
|
||||
return api_key
|
||||
|
||||
|
||||
def test_authentication(api_key: str) -> bool:
|
||||
"""Test API authentication with simple GPU types query"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST 1: Basic Authentication")
|
||||
print("="*60)
|
||||
|
||||
# Correct URL format per documentation
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
# Simple query that doesn't create resources
|
||||
query = """
|
||||
query {
|
||||
gpuTypes {
|
||||
id
|
||||
displayName
|
||||
memoryInGb
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
payload = {'query': query}
|
||||
|
||||
print(f"\n📡 Sending request to: {url[:50]}...")
|
||||
print(f"📝 Query: Get all GPU types")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||
|
||||
print(f"\n📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
print(f"Response: {response.text[:500]}")
|
||||
return False
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Check for GraphQL errors
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors: {json.dumps(data['errors'], indent=2)}")
|
||||
return False
|
||||
|
||||
# Check for successful data
|
||||
if 'data' in data and 'gpuTypes' in data['data']:
|
||||
gpu_types = data['data']['gpuTypes']
|
||||
print(f"\n✅ Authentication successful!")
|
||||
print(f"📊 Found {len(gpu_types)} GPU types")
|
||||
|
||||
# Show first 5 GPU types
|
||||
print("\n🎮 Sample GPU Types:")
|
||||
for gpu in gpu_types[:5]:
|
||||
print(f" - {gpu['displayName']} ({gpu['memoryInGb']}GB)")
|
||||
|
||||
if len(gpu_types) > 5:
|
||||
print(f" ... and {len(gpu_types) - 5} more")
|
||||
|
||||
return True
|
||||
else:
|
||||
print(f"❌ Unexpected response structure: {json.dumps(data, indent=2)}")
|
||||
return False
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
print("❌ Request timed out after 10 seconds")
|
||||
return False
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request failed: {e}")
|
||||
return False
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"❌ Failed to parse JSON response: {e}")
|
||||
print(f"Raw response: {response.text[:500]}")
|
||||
return False
|
||||
|
||||
|
||||
def test_user_info(api_key: str) -> bool:
|
||||
"""Test getting user information (requires authentication)"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST 2: User Information (Authenticated Query)")
|
||||
print("="*60)
|
||||
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
# Query that requires authentication
|
||||
query = """
|
||||
query {
|
||||
myself {
|
||||
id
|
||||
email
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
payload = {'query': query}
|
||||
|
||||
print(f"\n📡 Sending authenticated request...")
|
||||
print(f"📝 Query: Get user information")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||
|
||||
print(f"\n📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
print(f"Response: {response.text[:500]}")
|
||||
return False
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Check for GraphQL errors
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors: {json.dumps(data['errors'], indent=2)}")
|
||||
return False
|
||||
|
||||
# Check for successful data
|
||||
if 'data' in data and 'myself' in data['data']:
|
||||
user = data['data']['myself']
|
||||
print(f"\n✅ User information retrieved!")
|
||||
print(f"👤 User ID: {user['id']}")
|
||||
print(f"📧 Email: {user['email']}")
|
||||
return True
|
||||
else:
|
||||
print(f"❌ Unexpected response structure: {json.dumps(data, indent=2)}")
|
||||
return False
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
print("❌ Request timed out after 10 seconds")
|
||||
return False
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request failed: {e}")
|
||||
return False
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"❌ Failed to parse JSON response: {e}")
|
||||
print(f"Raw response: {response.text[:500]}")
|
||||
return False
|
||||
|
||||
|
||||
def test_gpu_availability(api_key: str, gpu_type: str = "NVIDIA RTX A4000") -> bool:
|
||||
"""Test checking GPU availability and pricing"""
|
||||
print("\n" + "="*60)
|
||||
print(f"TEST 3: GPU Availability Check ({gpu_type})")
|
||||
print("="*60)
|
||||
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
# Query to check specific GPU availability
|
||||
query = f"""
|
||||
query {{
|
||||
gpuTypes(input: {{ id: "{gpu_type}" }}) {{
|
||||
id
|
||||
displayName
|
||||
memoryInGb
|
||||
secureCloud
|
||||
communityCloud
|
||||
lowestPrice(input: {{
|
||||
gpuCount: 1
|
||||
secureCloud: true
|
||||
}}) {{
|
||||
minimumBidPrice
|
||||
uninterruptablePrice
|
||||
stockStatus
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
payload = {'query': query}
|
||||
|
||||
print(f"\n📡 Checking availability for {gpu_type}...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||
|
||||
print(f"\n📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
return False
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors: {json.dumps(data['errors'], indent=2)}")
|
||||
return False
|
||||
|
||||
if 'data' in data and 'gpuTypes' in data['data']:
|
||||
gpu_types = data['data']['gpuTypes']
|
||||
|
||||
if not gpu_types:
|
||||
print(f"❌ GPU type '{gpu_type}' not found")
|
||||
return False
|
||||
|
||||
gpu = gpu_types[0]
|
||||
print(f"\n✅ GPU information retrieved!")
|
||||
print(f"🎮 Display Name: {gpu['displayName']}")
|
||||
print(f"💾 Memory: {gpu['memoryInGb']}GB")
|
||||
print(f"🔒 Secure Cloud: {gpu['secureCloud']}")
|
||||
print(f"🌍 Community Cloud: {gpu['communityCloud']}")
|
||||
|
||||
if gpu.get('lowestPrice'):
|
||||
price = gpu['lowestPrice']
|
||||
print(f"\n💰 Pricing:")
|
||||
print(f" On-Demand: ${price.get('uninterruptablePrice', 'N/A')}/hour")
|
||||
print(f" Spot Min Bid: ${price.get('minimumBidPrice', 'N/A')}/hour")
|
||||
print(f" Stock Status: {price.get('stockStatus', 'Unknown')}")
|
||||
|
||||
return True
|
||||
else:
|
||||
print(f"❌ Unexpected response structure")
|
||||
return False
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
print("❌ Request timed out after 10 seconds")
|
||||
return False
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request failed: {e}")
|
||||
return False
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"❌ Failed to parse JSON response: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all authentication tests"""
|
||||
print("="*60)
|
||||
print("🧪 RunPod API Authentication Test Suite")
|
||||
print("="*60)
|
||||
print("\nThis script tests authentication with the RunPod API")
|
||||
print("before attempting pod creation.")
|
||||
|
||||
# Get API key
|
||||
api_key = get_api_key()
|
||||
if not api_key:
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\n🔑 API Key: {api_key[:10]}...{api_key[-10:]}")
|
||||
|
||||
# Run tests
|
||||
results = []
|
||||
|
||||
results.append(("Authentication Test", test_authentication(api_key)))
|
||||
results.append(("User Info Test", test_user_info(api_key)))
|
||||
results.append(("GPU Availability Test", test_gpu_availability(api_key)))
|
||||
|
||||
# Summary
|
||||
print("\n" + "="*60)
|
||||
print("📊 TEST SUMMARY")
|
||||
print("="*60)
|
||||
|
||||
for test_name, result in results:
|
||||
status = "✅ PASS" if result else "❌ FAIL"
|
||||
print(f"{status} - {test_name}")
|
||||
|
||||
passed = sum(1 for _, result in results if result)
|
||||
total = len(results)
|
||||
|
||||
print(f"\n🎯 Overall: {passed}/{total} tests passed")
|
||||
|
||||
if passed == total:
|
||||
print("\n✅ All tests passed! Authentication is working correctly.")
|
||||
print("✅ Ready to proceed with pod creation tests.")
|
||||
return 0
|
||||
else:
|
||||
print("\n❌ Some tests failed. Fix authentication before proceeding.")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
488
scripts/archive/test_runpod_pod_creation.py
Executable file
488
scripts/archive/test_runpod_pod_creation.py
Executable file
@@ -0,0 +1,488 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod Pod Creation Test
|
||||
Tests pod creation with incremental feature additions.
|
||||
|
||||
IMPORTANT: Run test_runpod_auth.py first to verify authentication.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
def get_api_key() -> Optional[str]:
|
||||
"""Get RunPod API key from environment"""
|
||||
api_key = os.environ.get('RUNPOD_API_KEY')
|
||||
if not api_key:
|
||||
print("❌ ERROR: RUNPOD_API_KEY environment variable not set")
|
||||
return None
|
||||
return api_key
|
||||
|
||||
|
||||
def create_minimal_pod(api_key: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Test 1: Create pod with absolute minimum required fields
|
||||
|
||||
This tests the basic pod creation without any optional features.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST 1: Minimal Pod Creation")
|
||||
print("="*60)
|
||||
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
# Absolute minimum fields based on documentation
|
||||
mutation = """
|
||||
mutation {
|
||||
podFindAndDeployOnDemand(
|
||||
input: {
|
||||
cloudType: SECURE
|
||||
gpuTypeId: "NVIDIA RTX A4000"
|
||||
gpuCount: 1
|
||||
name: "foxhunt-test-minimal"
|
||||
imageName: "runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel"
|
||||
}
|
||||
) {
|
||||
id
|
||||
name
|
||||
imageName
|
||||
desiredStatus
|
||||
machineId
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
payload = {'query': mutation}
|
||||
|
||||
print("\n📋 Configuration:")
|
||||
print(" Cloud Type: SECURE")
|
||||
print(" GPU: NVIDIA RTX A4000 (1x)")
|
||||
print(" Image: runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel")
|
||||
print(" Name: foxhunt-test-minimal")
|
||||
print("\n📡 Creating minimal pod...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=30)
|
||||
|
||||
print(f"📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors: {json.dumps(data['errors'], indent=2)}")
|
||||
return None
|
||||
|
||||
if 'data' in data and data['data'].get('podFindAndDeployOnDemand'):
|
||||
pod = data['data']['podFindAndDeployOnDemand']
|
||||
print(f"\n✅ Pod created successfully!")
|
||||
print(f"🆔 Pod ID: {pod['id']}")
|
||||
print(f"📛 Name: {pod['name']}")
|
||||
print(f"🖼️ Image: {pod['imageName']}")
|
||||
print(f"📊 Status: {pod['desiredStatus']}")
|
||||
print(f"🖥️ Machine ID: {pod.get('machineId', 'N/A')}")
|
||||
return pod
|
||||
else:
|
||||
print(f"❌ Pod creation returned null (likely no capacity)")
|
||||
print(f"Response: {json.dumps(data, indent=2)}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
print("❌ Request timed out after 30 seconds")
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request failed: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"❌ Unexpected error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def create_pod_with_storage(api_key: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Test 2: Add storage configuration
|
||||
|
||||
Adds network volume mounting to the minimal pod.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST 2: Pod Creation with Network Volume")
|
||||
print("="*60)
|
||||
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
mutation = """
|
||||
mutation {
|
||||
podFindAndDeployOnDemand(
|
||||
input: {
|
||||
cloudType: SECURE
|
||||
gpuTypeId: "NVIDIA RTX A4000"
|
||||
gpuCount: 1
|
||||
name: "foxhunt-test-storage"
|
||||
imageName: "runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel"
|
||||
|
||||
networkVolumeId: "se3zdnb5o4"
|
||||
volumeMountPath: "/workspace"
|
||||
containerDiskInGb: 50
|
||||
}
|
||||
) {
|
||||
id
|
||||
name
|
||||
imageName
|
||||
desiredStatus
|
||||
machineId
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
payload = {'query': mutation}
|
||||
|
||||
print("\n📋 Configuration:")
|
||||
print(" Previous + Network Volume: se3zdnb5o4")
|
||||
print(" Mount Path: /workspace")
|
||||
print(" Container Disk: 50GB")
|
||||
print("\n📡 Creating pod with storage...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=30)
|
||||
|
||||
print(f"📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors: {json.dumps(data['errors'], indent=2)}")
|
||||
return None
|
||||
|
||||
if 'data' in data and data['data'].get('podFindAndDeployOnDemand'):
|
||||
pod = data['data']['podFindAndDeployOnDemand']
|
||||
print(f"\n✅ Pod with storage created successfully!")
|
||||
print(f"🆔 Pod ID: {pod['id']}")
|
||||
print(f"📛 Name: {pod['name']}")
|
||||
return pod
|
||||
else:
|
||||
print(f"❌ Pod creation returned null")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def create_pod_with_env(api_key: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Test 3: Add environment variables
|
||||
|
||||
Adds environment variable configuration.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST 3: Pod Creation with Environment Variables")
|
||||
print("="*60)
|
||||
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
mutation = """
|
||||
mutation {
|
||||
podFindAndDeployOnDemand(
|
||||
input: {
|
||||
cloudType: SECURE
|
||||
gpuTypeId: "NVIDIA RTX A4000"
|
||||
gpuCount: 1
|
||||
name: "foxhunt-test-env"
|
||||
imageName: "runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel"
|
||||
|
||||
networkVolumeId: "se3zdnb5o4"
|
||||
volumeMountPath: "/workspace"
|
||||
containerDiskInGb: 50
|
||||
|
||||
env: [
|
||||
{ key: "TEST_VAR", value: "test_value" },
|
||||
{ key: "RUST_LOG", value: "info" }
|
||||
]
|
||||
}
|
||||
) {
|
||||
id
|
||||
name
|
||||
imageName
|
||||
env
|
||||
desiredStatus
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
payload = {'query': mutation}
|
||||
|
||||
print("\n📋 Configuration:")
|
||||
print(" Previous + Environment Variables:")
|
||||
print(" TEST_VAR=test_value")
|
||||
print(" RUST_LOG=info")
|
||||
print("\n📡 Creating pod with environment...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=30)
|
||||
|
||||
print(f"📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors: {json.dumps(data['errors'], indent=2)}")
|
||||
return None
|
||||
|
||||
if 'data' in data and data['data'].get('podFindAndDeployOnDemand'):
|
||||
pod = data['data']['podFindAndDeployOnDemand']
|
||||
print(f"\n✅ Pod with environment created successfully!")
|
||||
print(f"🆔 Pod ID: {pod['id']}")
|
||||
print(f"📛 Name: {pod['name']}")
|
||||
if pod.get('env'):
|
||||
print(f"🔧 Environment: {pod['env']}")
|
||||
return pod
|
||||
else:
|
||||
print(f"❌ Pod creation returned null")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def create_pod_with_custom_image(api_key: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Test 4: Use custom Docker image
|
||||
|
||||
Tests with the actual Foxhunt image.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST 4: Pod Creation with Custom Image")
|
||||
print("="*60)
|
||||
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
mutation = """
|
||||
mutation {
|
||||
podFindAndDeployOnDemand(
|
||||
input: {
|
||||
cloudType: SECURE
|
||||
gpuTypeId: "NVIDIA RTX A4000"
|
||||
gpuCount: 1
|
||||
name: "foxhunt-test-custom"
|
||||
imageName: "jgrusewski/foxhunt:latest"
|
||||
|
||||
networkVolumeId: "se3zdnb5o4"
|
||||
volumeMountPath: "/workspace"
|
||||
containerDiskInGb: 50
|
||||
|
||||
minVcpuCount: 4
|
||||
minMemoryInGb: 16
|
||||
|
||||
env: [
|
||||
{ key: "RUST_LOG", value: "info" }
|
||||
]
|
||||
}
|
||||
) {
|
||||
id
|
||||
name
|
||||
imageName
|
||||
desiredStatus
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
payload = {'query': mutation}
|
||||
|
||||
print("\n📋 Configuration:")
|
||||
print(" Custom Image: jgrusewski/foxhunt:latest")
|
||||
print(" Resources: 4 vCPU, 16GB RAM")
|
||||
print(" Storage: Network volume + 50GB container disk")
|
||||
print("\n📡 Creating pod with custom image...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=30)
|
||||
|
||||
print(f"📥 Response Status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ HTTP Error: {response.status_code}")
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL Errors: {json.dumps(data['errors'], indent=2)}")
|
||||
return None
|
||||
|
||||
if 'data' in data and data['data'].get('podFindAndDeployOnDemand'):
|
||||
pod = data['data']['podFindAndDeployOnDemand']
|
||||
print(f"\n✅ Pod with custom image created!")
|
||||
print(f"🆔 Pod ID: {pod['id']}")
|
||||
print(f"📛 Name: {pod['name']}")
|
||||
print(f"🖼️ Image: {pod['imageName']}")
|
||||
return pod
|
||||
else:
|
||||
print(f"❌ Pod creation returned null")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def stop_pod(api_key: str, pod_id: str) -> bool:
|
||||
"""Stop a running pod"""
|
||||
url = f'https://api.runpod.io/graphql?api_key={api_key}'
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
mutation = f"""
|
||||
mutation {{
|
||||
podStop(input: {{podId: "{pod_id}"}}) {{
|
||||
id
|
||||
desiredStatus
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
|
||||
payload = {'query': mutation}
|
||||
|
||||
print(f"\n🛑 Stopping pod {pod_id}...")
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=10)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ Failed to stop pod: HTTP {response.status_code}")
|
||||
return False
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'errors' in data:
|
||||
print(f"❌ Errors stopping pod: {data['errors']}")
|
||||
return False
|
||||
|
||||
print(f"✅ Pod stopped successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping pod: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Run pod creation tests incrementally"""
|
||||
print("="*60)
|
||||
print("🧪 RunPod Pod Creation Test Suite")
|
||||
print("="*60)
|
||||
print("\nThis script tests pod creation with incremental features.")
|
||||
print("Each test builds on the previous one.")
|
||||
print("\n⚠️ WARNING: These tests will CREATE REAL PODS that incur costs!")
|
||||
print("Make sure to stop/terminate pods after testing.")
|
||||
|
||||
# Get API key
|
||||
api_key = get_api_key()
|
||||
if not api_key:
|
||||
sys.exit(1)
|
||||
|
||||
# Ask for confirmation
|
||||
print("\n" + "="*60)
|
||||
response = input("Continue with pod creation tests? (yes/no): ")
|
||||
if response.lower() != 'yes':
|
||||
print("Tests cancelled.")
|
||||
return 0
|
||||
|
||||
# Track created pods for cleanup
|
||||
created_pods = []
|
||||
|
||||
# Run tests
|
||||
print("\n🚀 Starting pod creation tests...\n")
|
||||
|
||||
# Test 1: Minimal pod
|
||||
pod = create_minimal_pod(api_key)
|
||||
if pod:
|
||||
created_pods.append(pod['id'])
|
||||
time.sleep(2) # Brief pause between tests
|
||||
else:
|
||||
print("\n❌ Minimal pod creation failed. Stopping tests.")
|
||||
return 1
|
||||
|
||||
# Test 2: With storage
|
||||
pod = create_pod_with_storage(api_key)
|
||||
if pod:
|
||||
created_pods.append(pod['id'])
|
||||
time.sleep(2)
|
||||
else:
|
||||
print("\n⚠️ Storage test failed, but continuing...")
|
||||
|
||||
# Test 3: With environment
|
||||
pod = create_pod_with_env(api_key)
|
||||
if pod:
|
||||
created_pods.append(pod['id'])
|
||||
time.sleep(2)
|
||||
else:
|
||||
print("\n⚠️ Environment test failed, but continuing...")
|
||||
|
||||
# Test 4: With custom image
|
||||
pod = create_pod_with_custom_image(api_key)
|
||||
if pod:
|
||||
created_pods.append(pod['id'])
|
||||
else:
|
||||
print("\n⚠️ Custom image test failed")
|
||||
|
||||
# Summary
|
||||
print("\n" + "="*60)
|
||||
print("📊 TEST SUMMARY")
|
||||
print("="*60)
|
||||
print(f"\n✅ Created {len(created_pods)} pods:")
|
||||
for pod_id in created_pods:
|
||||
print(f" - {pod_id}")
|
||||
|
||||
# Cleanup option
|
||||
print("\n" + "="*60)
|
||||
cleanup = input("\nStop all created pods? (yes/no): ")
|
||||
if cleanup.lower() == 'yes':
|
||||
print("\n🧹 Cleaning up...")
|
||||
for pod_id in created_pods:
|
||||
stop_pod(api_key, pod_id)
|
||||
|
||||
print("\n✅ Tests complete!")
|
||||
print("\n💡 Next step: Run the full deployment script")
|
||||
print(" ./scripts/deploy_runpod_training.py")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
106
scripts/archive/test_ssh_connection.py
Normal file
106
scripts/archive/test_ssh_connection.py
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test SSH connection to RunPod pod pdx8g5suuvrwkb."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv('/home/jgrusewski/Work/foxhunt/.env.runpod')
|
||||
api_key = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not api_key:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
# Query pod details via GraphQL
|
||||
query = '''
|
||||
{
|
||||
pod(input: {podId: "pdx8g5suuvrwkb"}) {
|
||||
id
|
||||
name
|
||||
desiredStatus
|
||||
runtime {
|
||||
uptimeInSeconds
|
||||
ports {
|
||||
ip
|
||||
isIpPublic
|
||||
privatePort
|
||||
publicPort
|
||||
type
|
||||
}
|
||||
}
|
||||
machine {
|
||||
podHostId
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
print("🔍 Querying RunPod GraphQL API for pod details...")
|
||||
response = requests.post(
|
||||
'https://api.runpod.io/graphql',
|
||||
json={'query': query},
|
||||
headers={'Authorization': f'Bearer {api_key}'}
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ GraphQL API error: {response.status_code}")
|
||||
print(response.text)
|
||||
sys.exit(1)
|
||||
|
||||
data = response.json()
|
||||
if 'errors' in data:
|
||||
print(f"❌ GraphQL errors: {data['errors']}")
|
||||
sys.exit(1)
|
||||
|
||||
pod = data.get('data', {}).get('pod')
|
||||
if not pod:
|
||||
print("❌ Pod not found in GraphQL response")
|
||||
sys.exit(1)
|
||||
|
||||
print("\n📊 Pod Details:")
|
||||
print(f" ID: {pod['id']}")
|
||||
print(f" Name: {pod.get('name', 'N/A')}")
|
||||
print(f" Status: {pod.get('desiredStatus', 'N/A')}")
|
||||
|
||||
runtime = pod.get('runtime')
|
||||
if runtime:
|
||||
print(f" Uptime: {runtime.get('uptimeInSeconds', 0)} seconds")
|
||||
ports = runtime.get('ports', [])
|
||||
if ports:
|
||||
print(f"\n🔌 Port Mappings:")
|
||||
for port in ports:
|
||||
print(f" - {port['privatePort']}/{port['type']} -> {port.get('publicPort', 'N/A')} ({port.get('ip', 'N/A')})")
|
||||
print(f" Public IP: {port.get('isIpPublic', False)}")
|
||||
else:
|
||||
print("\n⚠️ No port mappings found (pod may still be initializing)")
|
||||
else:
|
||||
print("\n⚠️ No runtime information (pod may not be running yet)")
|
||||
|
||||
# Extract SSH connection details
|
||||
ssh_hostname = None
|
||||
ssh_port = None
|
||||
|
||||
if runtime and runtime.get('ports'):
|
||||
for port in runtime['ports']:
|
||||
if port.get('privatePort') == 22:
|
||||
if port.get('publicPort'):
|
||||
ssh_port = port['publicPort']
|
||||
ssh_hostname = port.get('ip')
|
||||
break
|
||||
|
||||
if ssh_hostname and ssh_port:
|
||||
print(f"\n✅ SSH Connection Details:")
|
||||
print(f" Host: {ssh_hostname}")
|
||||
print(f" Port: {ssh_port}")
|
||||
print(f"\n📝 SSH Connection Command:")
|
||||
print(f" ssh -o StrictHostKeyChecking=no -p {ssh_port} root@{ssh_hostname}")
|
||||
else:
|
||||
# Try RunPod proxy format
|
||||
pod_id = pod['id']
|
||||
print(f"\n⚠️ Direct SSH details not available. Trying RunPod proxy format:")
|
||||
print(f" ssh -o StrictHostKeyChecking=no root@{pod_id}-22.proxy.runpod.net")
|
||||
|
||||
print("\n" + "="*80)
|
||||
242
scripts/archive/train_runpod_225_features.sh
Executable file
242
scripts/archive/train_runpod_225_features.sh
Executable file
@@ -0,0 +1,242 @@
|
||||
#!/bin/bash
|
||||
# Runpod 225-Feature Training Deployment Script
|
||||
# GPU: 16GB VRAM (RTX 4000 Ada or A4000)
|
||||
# Features: 225 (201 Wave C + 24 Wave D)
|
||||
# Training Time: ~15-20 minutes
|
||||
# Cost: ~$0.05-$0.10
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
GPU_TYPE="RTX 4000 Ada"
|
||||
VRAM_MIN=16
|
||||
COST_CEILING="0.30"
|
||||
TRAINING_TIMEOUT=1200
|
||||
FEATURE_COUNT=225
|
||||
CONTAINER_IMAGE="runpod/pytorch:2.0.1-py3.10-cuda11.8.0-devel"
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Runpod 225-Feature Training Deployment${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Step 1: Verify prerequisites
|
||||
echo -e "${YELLOW}Step 1/8: Verifying prerequisites...${NC}"
|
||||
|
||||
# Check API key
|
||||
if ! runpodctl config 2>&1 | grep -q "apiKey"; then
|
||||
echo -e "${RED}❌ Runpod API key not configured!${NC}"
|
||||
echo -e "${YELLOW}Configure with: runpodctl config --apiKey YOUR_API_KEY${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}✅ API key configured${NC}"
|
||||
|
||||
# Check training data
|
||||
if [ ! -f "test_data/ES_FUT_180d.parquet" ]; then
|
||||
echo -e "${RED}❌ Training data not found: test_data/ES_FUT_180d.parquet${NC}"
|
||||
exit 1
|
||||
fi
|
||||
DATA_SIZE=$(du -h test_data/ES_FUT_180d.parquet | cut -f1)
|
||||
echo -e "${GREEN}✅ Training data found: ${DATA_SIZE}${NC}"
|
||||
|
||||
# Verify 225 features
|
||||
if ! grep -q "features: \[f64; 225\]" ml/src/features/unified.rs; then
|
||||
echo -e "${RED}❌ 225 features not configured in ml/src/features/unified.rs!${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}✅ 225 features configured${NC}"
|
||||
|
||||
# Step 2: Build release binary
|
||||
echo -e "\n${YELLOW}Step 2/8: Building release binary...${NC}"
|
||||
echo "This may take 5-6 minutes..."
|
||||
|
||||
cargo build --release -p ml --example train_tft_parquet --features cuda 2>&1 | tee /tmp/build.log | grep -E "Compiling|Finished" || true
|
||||
|
||||
if [ ! -f "target/release/examples/train_tft_parquet" ]; then
|
||||
echo -e "${RED}❌ Build failed! Check /tmp/build.log${NC}"
|
||||
exit 1
|
||||
fi
|
||||
BINARY_SIZE=$(du -h target/release/examples/train_tft_parquet | cut -f1)
|
||||
echo -e "${GREEN}✅ Binary built: ${BINARY_SIZE}${NC}"
|
||||
|
||||
# Step 3: List available GPUs
|
||||
echo -e "\n${YELLOW}Step 3/8: Checking available GPUs...${NC}"
|
||||
echo "Looking for 16GB+ GPUs under \$${COST_CEILING}/hr..."
|
||||
|
||||
runpodctl get gpus --filter "vram>=16" 2>&1 | grep -E "RTX 4000|A4000|RTX 4090|A5000" | head -5 || {
|
||||
echo -e "${YELLOW}⚠️ No pre-filtered results, listing all GPUs...${NC}"
|
||||
runpodctl get gpus 2>&1 | head -20
|
||||
}
|
||||
|
||||
# Step 4: Create pod
|
||||
echo -e "\n${YELLOW}Step 4/8: Creating Runpod pod...${NC}"
|
||||
echo "GPU: ${GPU_TYPE}, VRAM: ${VRAM_MIN}GB, Max Cost: \$${COST_CEILING}/hr"
|
||||
|
||||
POD_CREATE_OUTPUT=$(runpodctl create pod \
|
||||
--name "foxhunt-tft-225-training-$(date +%s)" \
|
||||
--gpuType "RTX 4000 Ada" \
|
||||
--minVram ${VRAM_MIN} \
|
||||
--maxCost ${COST_CEILING} \
|
||||
--containerDiskSize 50 \
|
||||
--volumeSize 50 \
|
||||
--imageName "${CONTAINER_IMAGE}" \
|
||||
--env "FEATURE_COUNT=${FEATURE_COUNT}" \
|
||||
--env "CUDA_VISIBLE_DEVICES=0" 2>&1)
|
||||
|
||||
POD_ID=$(echo "$POD_CREATE_OUTPUT" | jq -r '.id' 2>/dev/null || echo "$POD_CREATE_OUTPUT" | grep -oP 'pod-[a-z0-9]+' | head -1)
|
||||
|
||||
if [ -z "$POD_ID" ]; then
|
||||
echo -e "${RED}❌ Failed to create pod!${NC}"
|
||||
echo "$POD_CREATE_OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✅ Pod created: ${POD_ID}${NC}"
|
||||
echo "$POD_ID" > /tmp/runpod_pod_id.txt
|
||||
|
||||
# Step 5: Wait for pod ready
|
||||
echo -e "\n${YELLOW}Step 5/8: Waiting for pod initialization...${NC}"
|
||||
echo "This usually takes 30-60 seconds..."
|
||||
|
||||
WAIT_COUNT=0
|
||||
MAX_WAIT=120
|
||||
while [ $WAIT_COUNT -lt $MAX_WAIT ]; do
|
||||
POD_STATUS=$(runpodctl get pod "${POD_ID}" 2>&1 | grep -oP 'status: \K[A-Z]+' || echo "UNKNOWN")
|
||||
|
||||
if [ "$POD_STATUS" = "RUNNING" ]; then
|
||||
echo -e "${GREEN}✅ Pod is running!${NC}"
|
||||
break
|
||||
fi
|
||||
|
||||
echo -n "."
|
||||
sleep 5
|
||||
WAIT_COUNT=$((WAIT_COUNT + 5))
|
||||
done
|
||||
|
||||
if [ $WAIT_COUNT -ge $MAX_WAIT ]; then
|
||||
echo -e "\n${RED}❌ Pod failed to start within ${MAX_WAIT} seconds${NC}"
|
||||
runpodctl remove pod "${POD_ID}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 6: Upload training data and binary
|
||||
echo -e "\n${YELLOW}Step 6/8: Uploading training data and binary...${NC}"
|
||||
|
||||
# Create directories on pod
|
||||
runpodctl exec "${POD_ID}" -- mkdir -p /workspace/data /workspace/models
|
||||
|
||||
# Upload training data
|
||||
echo "Uploading training data (${DATA_SIZE})..."
|
||||
runpodctl send "${POD_ID}" test_data/ES_FUT_180d.parquet /workspace/data/ 2>&1 | grep -E "Success|Error" || echo "Upload in progress..."
|
||||
|
||||
# Upload binary
|
||||
echo "Uploading training binary (${BINARY_SIZE})..."
|
||||
runpodctl send "${POD_ID}" target/release/examples/train_tft_parquet /workspace/ 2>&1 | grep -E "Success|Error" || echo "Upload in progress..."
|
||||
|
||||
# Make binary executable
|
||||
runpodctl exec "${POD_ID}" -- chmod +x /workspace/train_tft_parquet
|
||||
|
||||
echo -e "${GREEN}✅ Files uploaded${NC}"
|
||||
|
||||
# Step 7: Execute training
|
||||
echo -e "\n${YELLOW}Step 7/8: Starting training...${NC}"
|
||||
echo "Training configuration:"
|
||||
echo " • Features: 225 (201 Wave C + 24 Wave D)"
|
||||
echo " • Epochs: 50"
|
||||
echo " • Batch size: 32"
|
||||
echo " • Data: ES.FUT 180 days"
|
||||
echo " • Estimated time: 15-20 minutes"
|
||||
echo ""
|
||||
|
||||
TRAINING_START=$(date +%s)
|
||||
|
||||
# Execute training with comprehensive logging
|
||||
runpodctl exec "${POD_ID}" -- /workspace/train_tft_parquet \
|
||||
--parquet-file /workspace/data/ES_FUT_180d.parquet \
|
||||
--epochs 50 \
|
||||
--batch-size 32 \
|
||||
--learning-rate 0.001 \
|
||||
--lookback-window 60 \
|
||||
--forecast-horizon 10 \
|
||||
--hidden-dim 256 \
|
||||
--num-attention-heads 8 \
|
||||
2>&1 | tee /tmp/training_output.log
|
||||
|
||||
TRAINING_END=$(date +%s)
|
||||
TRAINING_DURATION=$((TRAINING_END - TRAINING_START))
|
||||
TRAINING_MINUTES=$((TRAINING_DURATION / 60))
|
||||
TRAINING_SECONDS=$((TRAINING_DURATION % 60))
|
||||
|
||||
echo -e "${GREEN}✅ Training completed in ${TRAINING_MINUTES}m ${TRAINING_SECONDS}s${NC}"
|
||||
|
||||
# Step 8: Download trained model
|
||||
echo -e "\n${YELLOW}Step 8/8: Downloading trained model...${NC}"
|
||||
|
||||
mkdir -p models/runpod_trained
|
||||
|
||||
# List models on pod
|
||||
echo "Available models:"
|
||||
runpodctl exec "${POD_ID}" -- ls -lh /workspace/*.safetensors 2>/dev/null || {
|
||||
echo -e "${YELLOW}⚠️ No .safetensors found, checking models directory...${NC}"
|
||||
runpodctl exec "${POD_ID}" -- ls -lh /workspace/models/ 2>/dev/null || echo "No models found"
|
||||
}
|
||||
|
||||
# Download all model files
|
||||
runpodctl receive "${POD_ID}" "/workspace/*.safetensors" models/runpod_trained/ 2>&1 || {
|
||||
echo -e "${YELLOW}⚠️ Trying alternative model location...${NC}"
|
||||
runpodctl receive "${POD_ID}" "/workspace/models/*.safetensors" models/runpod_trained/ 2>&1
|
||||
}
|
||||
|
||||
# Also download training logs if available
|
||||
runpodctl receive "${POD_ID}" "/workspace/*.log" models/runpod_trained/ 2>&1 || echo "No logs to download"
|
||||
|
||||
if ls models/runpod_trained/*.safetensors 1>/dev/null 2>&1; then
|
||||
MODEL_SIZE=$(du -h models/runpod_trained/*.safetensors | head -1 | cut -f1)
|
||||
echo -e "${GREEN}✅ Model downloaded: ${MODEL_SIZE}${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ No model files downloaded!${NC}"
|
||||
fi
|
||||
|
||||
# Step 9: Calculate cost
|
||||
echo -e "\n${YELLOW}Calculating training cost...${NC}"
|
||||
|
||||
POD_INFO=$(runpodctl get pod "${POD_ID}" 2>&1)
|
||||
GPU_COST=$(echo "$POD_INFO" | grep -oP 'costPerHr: \K[0-9.]+' || echo "0.25")
|
||||
COST_HOURS=$(echo "scale=4; ${TRAINING_DURATION} / 3600" | bc)
|
||||
TOTAL_COST=$(echo "scale=4; ${GPU_COST} * ${COST_HOURS}" | bc)
|
||||
|
||||
echo -e "${BLUE}Cost Breakdown:${NC}"
|
||||
echo " • GPU Rate: \$${GPU_COST}/hr"
|
||||
echo " • Training Time: ${TRAINING_MINUTES}m ${TRAINING_SECONDS}s"
|
||||
echo " • Total Cost: \$${TOTAL_COST}"
|
||||
|
||||
# Step 10: Terminate pod
|
||||
echo -e "\n${YELLOW}Terminating pod...${NC}"
|
||||
runpodctl remove pod "${POD_ID}" 2>&1 | grep -E "Success|Removed" || echo "Pod termination in progress..."
|
||||
echo -e "${GREEN}✅ Pod terminated (no ongoing charges)${NC}"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${GREEN}Training Deployment Complete!${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo "Results:"
|
||||
echo " • Pod ID: ${POD_ID}"
|
||||
echo " • Training Time: ${TRAINING_MINUTES}m ${TRAINING_SECONDS}s"
|
||||
echo " • Total Cost: \$${TOTAL_COST}"
|
||||
echo " • Model Location: models/runpod_trained/"
|
||||
echo " • Training Log: /tmp/training_output.log"
|
||||
echo ""
|
||||
echo "Next Steps:"
|
||||
echo " 1. Run backtesting: ./scripts/backtest_runpod_225.sh"
|
||||
echo " 2. Validate Wave D targets (Sharpe ≥2.0, Win Rate ≥60%, Drawdown ≤15%)"
|
||||
echo " 3. Train remaining models (DQN, PPO, MAMBA-2)"
|
||||
echo ""
|
||||
173
scripts/archive/upload_env_to_runpod.sh
Executable file
173
scripts/archive/upload_env_to_runpod.sh
Executable file
@@ -0,0 +1,173 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# =============================================================================
|
||||
# RUNPOD .ENV UPLOAD SCRIPT
|
||||
# =============================================================================
|
||||
# Uploads .env file to Runpod Network Volume via S3 API
|
||||
#
|
||||
# USAGE:
|
||||
# ./scripts/upload_env_to_runpod.sh
|
||||
#
|
||||
# PREREQUISITES:
|
||||
# - AWS CLI installed (for S3 API access)
|
||||
# - ~/.aws/credentials configured with [runpod] profile
|
||||
# - RUNPOD_S3_ENDPOINT environment variable set
|
||||
# - .env file exists in current directory
|
||||
#
|
||||
# SECURITY:
|
||||
# - .env file MUST be gitignored (verify with: git status)
|
||||
# - File permissions set to 600 (owner read/write only) in pod
|
||||
# - Never commit .env to git repository
|
||||
# =============================================================================
|
||||
|
||||
echo "=========================================="
|
||||
echo "Runpod .env Upload Script"
|
||||
echo "=========================================="
|
||||
|
||||
# Check prerequisites
|
||||
echo ""
|
||||
echo "Checking prerequisites..."
|
||||
|
||||
# Check AWS CLI installed
|
||||
if ! command -v aws &> /dev/null; then
|
||||
echo "ERROR: AWS CLI not found"
|
||||
echo "Install: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ AWS CLI installed"
|
||||
|
||||
# Check .env file exists
|
||||
if [ ! -f .env ]; then
|
||||
echo "ERROR: .env file not found in current directory"
|
||||
echo "Create .env file with required credentials (see RUNPOD_VOLUME_DEPLOYMENT_GUIDE.md)"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ .env file exists"
|
||||
|
||||
# Check .env is gitignored
|
||||
if git check-ignore .env &> /dev/null; then
|
||||
echo "✓ .env file is gitignored"
|
||||
else
|
||||
echo "WARNING: .env file is NOT gitignored"
|
||||
echo "Add .env to .gitignore to prevent accidental commits"
|
||||
read -p "Continue anyway? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check RUNPOD_S3_ENDPOINT is set
|
||||
if [ -z "$RUNPOD_S3_ENDPOINT" ]; then
|
||||
echo "ERROR: RUNPOD_S3_ENDPOINT environment variable not set"
|
||||
echo ""
|
||||
echo "Get your S3 endpoint from Runpod console:"
|
||||
echo " 1. Go to https://www.runpod.io/console/pods"
|
||||
echo " 2. Click 'Storage' → Select your volume"
|
||||
echo " 3. Click 'Access Credentials'"
|
||||
echo " 4. Copy 'S3 Endpoint URL'"
|
||||
echo ""
|
||||
read -p "Enter RUNPOD_S3_ENDPOINT: " RUNPOD_S3_ENDPOINT
|
||||
if [ -z "$RUNPOD_S3_ENDPOINT" ]; then
|
||||
echo "ERROR: No endpoint provided"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo "✓ RUNPOD_S3_ENDPOINT set: $RUNPOD_S3_ENDPOINT"
|
||||
|
||||
# Check AWS profile configured
|
||||
if ! aws configure list --profile runpod &> /dev/null; then
|
||||
echo "ERROR: AWS profile 'runpod' not configured"
|
||||
echo ""
|
||||
echo "Configure AWS CLI profile:"
|
||||
echo " aws configure --profile runpod"
|
||||
echo ""
|
||||
echo "Enter Runpod S3 credentials:"
|
||||
echo " - AWS Access Key ID: (from Runpod console)"
|
||||
echo " - AWS Secret Access Key: (from Runpod console)"
|
||||
echo " - Default region: us-east-1"
|
||||
echo " - Default output format: json"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ AWS profile 'runpod' configured"
|
||||
|
||||
# Display .env file info (without printing contents)
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo ".env File Information"
|
||||
echo "=========================================="
|
||||
ENV_SIZE=$(stat -c%s .env 2>/dev/null || stat -f%z .env)
|
||||
ENV_LINES=$(wc -l < .env)
|
||||
echo "Size: $(numfmt --to=iec-i --suffix=B ${ENV_SIZE} 2>/dev/null || echo ${ENV_SIZE} bytes)"
|
||||
echo "Lines: ${ENV_LINES}"
|
||||
echo ""
|
||||
echo "Sample variables (values hidden):"
|
||||
grep -E "^[A-Z_]+" .env | sed 's/=.*/=***REDACTED***/' | head -5
|
||||
|
||||
# Confirm upload
|
||||
echo ""
|
||||
read -p "Upload .env to Runpod Network Volume? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Upload cancelled"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Upload .env file to Runpod Network Volume
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Uploading .env to Runpod Network Volume"
|
||||
echo "=========================================="
|
||||
|
||||
VOLUME_NAME="foxhunt-ml-volume"
|
||||
|
||||
echo "Uploading to s3://${VOLUME_NAME}/.env ..."
|
||||
aws s3 cp .env "s3://${VOLUME_NAME}/.env" \
|
||||
--endpoint-url "$RUNPOD_S3_ENDPOINT" \
|
||||
--profile runpod
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo ""
|
||||
echo "ERROR: Upload failed"
|
||||
echo ""
|
||||
echo "Troubleshooting:"
|
||||
echo " 1. Verify RUNPOD_S3_ENDPOINT is correct"
|
||||
echo " 2. Verify AWS credentials in ~/.aws/credentials [runpod] profile"
|
||||
echo " 3. Verify volume exists in Runpod console"
|
||||
echo " 4. Check network connectivity"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Upload successful"
|
||||
|
||||
# Verify upload
|
||||
echo ""
|
||||
echo "Verifying upload..."
|
||||
aws s3 ls "s3://${VOLUME_NAME}/" --endpoint-url "$RUNPOD_S3_ENDPOINT" --profile runpod | grep .env
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo ""
|
||||
echo "WARNING: Could not verify .env file in volume"
|
||||
echo "File may have uploaded but verification failed"
|
||||
else
|
||||
echo "✓ .env file verified in volume"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Upload Complete"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Deploy pod with volume mount (see RUNPOD_VOLUME_DEPLOYMENT_GUIDE.md)"
|
||||
echo " 2. SSH into pod and verify .env loaded:"
|
||||
echo " ls -lh /runpod-volume/.env"
|
||||
echo " env | grep -E 'DATABASE_URL|REDIS_URL|VAULT_ADDR' | sed 's/=.*/=***REDACTED***/'"
|
||||
echo " 3. Check pod logs for '✓ Loaded credentials from /runpod-volume/.env'"
|
||||
echo ""
|
||||
echo "Security reminder:"
|
||||
echo " - Never commit .env to git repository"
|
||||
echo " - Rotate credentials monthly"
|
||||
echo " - Use separate .env files per environment (dev/staging/prod)"
|
||||
echo ""
|
||||
532
scripts/archive/upload_to_runpod_s3.sh
Executable file
532
scripts/archive/upload_to_runpod_s3.sh
Executable file
@@ -0,0 +1,532 @@
|
||||
#!/bin/bash
|
||||
# upload_to_runpod_s3.sh - Package Foxhunt binaries for Runpod deployment
|
||||
#
|
||||
# NO AWS CLI REQUIRED - Uses simple tar packaging + manual/curl upload
|
||||
# Optimized for Tesla V100 16GB GPU ($0.14-0.39/hr on various providers)
|
||||
#
|
||||
# DEPLOYMENT OPTIONS:
|
||||
# 1. Manual Web Upload: Use Runpod web interface to upload tar.gz to Network Volume
|
||||
# 2. Direct Mount: Upload via SSH to mounted Network Volume on running pod
|
||||
# 3. Docker Image: Bundle everything in Docker image (recommended for reproducibility)
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
FOXHUNT_ROOT="/home/jgrusewski/Work/foxhunt"
|
||||
OUTPUT_DIR="$FOXHUNT_ROOT/runpod_package"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
PACKAGE_NAME="foxhunt_${TIMESTAMP}.tar.gz"
|
||||
|
||||
echo "========================================="
|
||||
echo "Foxhunt Runpod Package Builder"
|
||||
echo "========================================="
|
||||
echo "Target GPU: Tesla V100 16GB"
|
||||
echo "Estimated Pricing: \$0.14-0.39/hr (Runpod/DataCrunch)"
|
||||
echo "Package: $PACKAGE_NAME"
|
||||
echo "========================================="
|
||||
|
||||
# Clean and create output directory
|
||||
rm -rf "$OUTPUT_DIR"
|
||||
mkdir -p "$OUTPUT_DIR/bin"
|
||||
mkdir -p "$OUTPUT_DIR/test_data"
|
||||
mkdir -p "$OUTPUT_DIR/config"
|
||||
|
||||
# Build release binaries
|
||||
echo ""
|
||||
echo "📦 Step 1/4: Building release binaries..."
|
||||
cd "$FOXHUNT_ROOT"
|
||||
cargo build --release --features cuda --workspace
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "❌ ERROR: Cargo build failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Build complete"
|
||||
|
||||
# Copy training binaries
|
||||
echo ""
|
||||
echo "📦 Step 2/4: Packaging training binaries..."
|
||||
BINARIES=(
|
||||
"train_tft_parquet"
|
||||
"train_dqn"
|
||||
"train_ppo"
|
||||
"train_mamba2_dbn"
|
||||
)
|
||||
|
||||
TOTAL_BIN_SIZE=0
|
||||
for binary in "${BINARIES[@]}"; do
|
||||
SRC="$FOXHUNT_ROOT/target/release/examples/$binary"
|
||||
if [ -f "$SRC" ]; then
|
||||
cp "$SRC" "$OUTPUT_DIR/bin/"
|
||||
SIZE=$(stat -c%s "$SRC" 2>/dev/null || stat -f%z "$SRC" 2>/dev/null)
|
||||
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
|
||||
TOTAL_BIN_SIZE=$((TOTAL_BIN_SIZE + SIZE))
|
||||
echo " ✓ $binary ($SIZE_MB MB)"
|
||||
else
|
||||
echo " ⚠ $binary not found (skipping)"
|
||||
fi
|
||||
done
|
||||
|
||||
TOTAL_BIN_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BIN_SIZE/1024/1024}")
|
||||
echo "Total binaries: $TOTAL_BIN_MB MB"
|
||||
|
||||
# Copy test data (ALL Parquet files)
|
||||
echo ""
|
||||
echo "📦 Step 3/4: Packaging test data..."
|
||||
if [ -d "$FOXHUNT_ROOT/test_data" ]; then
|
||||
PARQUET_COUNT=$(ls $FOXHUNT_ROOT/test_data/*.parquet 2>/dev/null | wc -l)
|
||||
if [ "$PARQUET_COUNT" -gt 0 ]; then
|
||||
echo "Found $PARQUET_COUNT Parquet files:"
|
||||
TOTAL_DATA_SIZE=0
|
||||
for file in $FOXHUNT_ROOT/test_data/*.parquet; do
|
||||
if [ -f "$file" ]; then
|
||||
filename=$(basename "$file")
|
||||
SIZE=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null)
|
||||
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
|
||||
TOTAL_DATA_SIZE=$((TOTAL_DATA_SIZE + SIZE))
|
||||
echo " ✓ $filename ($SIZE_MB MB)"
|
||||
cp "$file" "$OUTPUT_DIR/test_data/"
|
||||
fi
|
||||
done
|
||||
TOTAL_DATA_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_DATA_SIZE/1024/1024}")
|
||||
echo "Total data: $TOTAL_DATA_MB MB"
|
||||
else
|
||||
echo "⚠ WARNING: No Parquet files found in test_data/"
|
||||
fi
|
||||
else
|
||||
echo "⚠ WARNING: test_data/ directory not found"
|
||||
fi
|
||||
|
||||
# Create deployment script
|
||||
echo ""
|
||||
echo "📦 Step 4/4: Creating deployment scripts..."
|
||||
cat > "$OUTPUT_DIR/runpod_setup.sh" << 'RUNPOD_SETUP_EOF'
|
||||
#!/bin/bash
|
||||
# runpod_setup.sh - Setup script to run inside Runpod pod
|
||||
#
|
||||
# Usage:
|
||||
# 1. Extract package: tar -xzf foxhunt_*.tar.gz
|
||||
# 2. Run setup: ./runpod_setup.sh
|
||||
# 3. Start training: ./train_fp32.sh
|
||||
|
||||
set -e
|
||||
|
||||
echo "========================================="
|
||||
echo "Foxhunt Runpod Environment Setup"
|
||||
echo "========================================="
|
||||
|
||||
# Verify CUDA
|
||||
echo ""
|
||||
echo "🔍 Checking CUDA availability..."
|
||||
if command -v nvidia-smi &> /dev/null; then
|
||||
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv
|
||||
echo "✅ CUDA available"
|
||||
else
|
||||
echo "❌ WARNING: nvidia-smi not found"
|
||||
fi
|
||||
|
||||
# Verify binaries
|
||||
echo ""
|
||||
echo "🔍 Checking binaries..."
|
||||
for binary in bin/*; do
|
||||
if [ -f "$binary" ] && [ -x "$binary" ]; then
|
||||
size=$(stat -c%s "$binary" 2>/dev/null || stat -f%z "$binary" 2>/dev/null)
|
||||
size_mb=$(awk "BEGIN {printf \"%.2f\", $size/1024/1024}")
|
||||
echo " ✓ $(basename $binary) ($size_mb MB)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Verify test data
|
||||
echo ""
|
||||
echo "🔍 Checking test data..."
|
||||
if [ -d "test_data" ]; then
|
||||
PARQUET_COUNT=$(ls test_data/*.parquet 2>/dev/null | wc -l)
|
||||
echo "Found $PARQUET_COUNT Parquet files:"
|
||||
for file in test_data/*.parquet; do
|
||||
if [ -f "$file" ]; then
|
||||
size=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null)
|
||||
size_mb=$(awk "BEGIN {printf \"%.2f\", $size/1024/1024}")
|
||||
echo " ✓ $(basename $file) ($size_mb MB)"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "⚠ WARNING: test_data/ directory not found"
|
||||
fi
|
||||
|
||||
# Make binaries executable
|
||||
chmod +x bin/*
|
||||
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "✅ Setup complete!"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
echo "Ready to train. Run one of:"
|
||||
echo " ./train_fp32.sh # Full FP32 training (recommended)"
|
||||
echo " ./train_tft_only.sh # TFT model only"
|
||||
echo " ./bin/train_tft_parquet --help # Manual training"
|
||||
RUNPOD_SETUP_EOF
|
||||
|
||||
chmod +x "$OUTPUT_DIR/runpod_setup.sh"
|
||||
echo " ✓ runpod_setup.sh"
|
||||
|
||||
# Create FP32 training script
|
||||
cat > "$OUTPUT_DIR/train_fp32.sh" << 'TRAIN_FP32_EOF'
|
||||
#!/bin/bash
|
||||
# train_fp32.sh - Train all FP32 models (PRODUCTION READY)
|
||||
#
|
||||
# Models: DQN, PPO, MAMBA-2, TFT-FP32
|
||||
# GPU Memory: ~815MB total (fits Tesla V100 16GB easily)
|
||||
# Training Time: ~10-15 minutes total on V100
|
||||
|
||||
set -e
|
||||
|
||||
echo "========================================="
|
||||
echo "Foxhunt FP32 Training Pipeline"
|
||||
echo "========================================="
|
||||
echo "GPU: Tesla V100 16GB"
|
||||
echo "Models: DQN, PPO, MAMBA-2, TFT-FP32"
|
||||
echo "Expected Memory: 815MB"
|
||||
echo "Expected Time: 10-15 minutes"
|
||||
echo "========================================="
|
||||
|
||||
# Set CUDA environment
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
|
||||
# Create output directory
|
||||
OUTPUT_DIR="models_$(date +%Y%m%d_%H%M%S)"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Training function
|
||||
train_model() {
|
||||
MODEL=$1
|
||||
BINARY=$2
|
||||
DATA_FILE=$3
|
||||
EPOCHS=$4
|
||||
EXPECTED_TIME=$5
|
||||
|
||||
echo ""
|
||||
echo "🚀 Training $MODEL..."
|
||||
echo " Binary: $BINARY"
|
||||
echo " Data: $DATA_FILE"
|
||||
echo " Epochs: $EPOCHS"
|
||||
echo " Expected: $EXPECTED_TIME"
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
./bin/$BINARY \
|
||||
--parquet-file "test_data/$DATA_FILE" \
|
||||
--epochs "$EPOCHS" \
|
||||
--output-dir "$OUTPUT_DIR/$MODEL" \
|
||||
2>&1 | tee "$OUTPUT_DIR/${MODEL}_training.log"
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - START_TIME))
|
||||
DURATION_MIN=$(awk "BEGIN {printf \"%.2f\", $DURATION/60}")
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ $MODEL complete in ${DURATION_MIN} minutes"
|
||||
else
|
||||
echo "❌ $MODEL training failed (see $OUTPUT_DIR/${MODEL}_training.log)"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Train all models
|
||||
train_model "DQN" "train_dqn" "ES_FUT_180d.parquet" 20 "15-20 seconds"
|
||||
train_model "PPO" "train_ppo" "ES_FUT_180d.parquet" 20 "7-10 seconds"
|
||||
train_model "MAMBA2" "train_mamba2_dbn" "ES_FUT_180d.parquet" 50 "2-3 minutes"
|
||||
train_model "TFT_FP32" "train_tft_parquet" "ES_FUT_180d.parquet" 50 "3-5 minutes"
|
||||
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "✅ All models trained successfully!"
|
||||
echo "========================================="
|
||||
echo "Output directory: $OUTPUT_DIR"
|
||||
echo ""
|
||||
echo "Models ready for deployment:"
|
||||
ls -lh "$OUTPUT_DIR"
|
||||
TRAIN_FP32_EOF
|
||||
|
||||
chmod +x "$OUTPUT_DIR/train_fp32.sh"
|
||||
echo " ✓ train_fp32.sh"
|
||||
|
||||
# Create TFT-only training script
|
||||
cat > "$OUTPUT_DIR/train_tft_only.sh" << 'TRAIN_TFT_EOF'
|
||||
#!/bin/bash
|
||||
# train_tft_only.sh - Train TFT model only (faster iteration)
|
||||
#
|
||||
# Model: TFT-FP32 (225 features)
|
||||
# GPU Memory: ~500MB
|
||||
# Training Time: ~3-5 minutes on V100
|
||||
|
||||
set -e
|
||||
|
||||
echo "========================================="
|
||||
echo "TFT-FP32 Training (225 Features)"
|
||||
echo "========================================="
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
OUTPUT_DIR="models_tft_$(date +%Y%m%d_%H%M%S)"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
./bin/train_tft_parquet \
|
||||
--parquet-file test_data/ES_FUT_180d.parquet \
|
||||
--epochs 50 \
|
||||
--output-dir "$OUTPUT_DIR" \
|
||||
2>&1 | tee "$OUTPUT_DIR/training.log"
|
||||
|
||||
echo ""
|
||||
echo "✅ TFT training complete!"
|
||||
echo "Output: $OUTPUT_DIR"
|
||||
TRAIN_TFT_EOF
|
||||
|
||||
chmod +x "$OUTPUT_DIR/train_tft_only.sh"
|
||||
echo " ✓ train_tft_only.sh"
|
||||
|
||||
# Create README
|
||||
cat > "$OUTPUT_DIR/README.md" << 'README_EOF'
|
||||
# Foxhunt Runpod Deployment Package
|
||||
|
||||
## 🎯 Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Extract package
|
||||
tar -xzf foxhunt_*.tar.gz
|
||||
cd foxhunt_*/
|
||||
|
||||
# 2. Run setup
|
||||
./runpod_setup.sh
|
||||
|
||||
# 3. Start training (choose one)
|
||||
./train_fp32.sh # All models (10-15 min)
|
||||
./train_tft_only.sh # TFT only (3-5 min)
|
||||
```
|
||||
|
||||
## 📊 GPU Requirements
|
||||
|
||||
**Recommended: Tesla V100 16GB**
|
||||
- Pricing: $0.14-0.39/hour (Runpod/DataCrunch)
|
||||
- Memory: 16GB (815MB FP32 models = 95% headroom)
|
||||
- Training time: 10-15 minutes total
|
||||
- Cost per training run: ~$0.10
|
||||
|
||||
**Minimum: RTX 3060 12GB**
|
||||
- Works but slower (2-3x training time)
|
||||
- Budget option if V100 unavailable
|
||||
|
||||
## 📦 Package Contents
|
||||
|
||||
```
|
||||
foxhunt_YYYYMMDD_HHMMSS/
|
||||
├── bin/ # Training binaries
|
||||
│ ├── train_tft_parquet # TFT model (225 features)
|
||||
│ ├── train_dqn # Deep Q-Network
|
||||
│ ├── train_ppo # Proximal Policy Optimization
|
||||
│ └── train_mamba2_dbn # MAMBA-2 model
|
||||
├── test_data/ # Parquet training data
|
||||
│ ├── ES_FUT_180d.parquet # ES futures (180 days)
|
||||
│ ├── NQ_FUT_180d.parquet # NQ futures (180 days)
|
||||
│ ├── 6E_FUT_180d.parquet # 6E futures (180 days)
|
||||
│ └── *.parquet # Other datasets
|
||||
├── runpod_setup.sh # Environment setup
|
||||
├── train_fp32.sh # Full training pipeline
|
||||
├── train_tft_only.sh # TFT only (faster)
|
||||
└── README.md # This file
|
||||
```
|
||||
|
||||
## 🚀 Training Pipeline
|
||||
|
||||
### Full FP32 Training (Recommended)
|
||||
```bash
|
||||
./train_fp32.sh
|
||||
```
|
||||
|
||||
Trains all 4 models:
|
||||
1. **DQN** (~15-20 sec, 6MB memory)
|
||||
2. **PPO** (~7-10 sec, 145MB memory)
|
||||
3. **MAMBA-2** (~2-3 min, 164MB memory)
|
||||
4. **TFT-FP32** (~3-5 min, 500MB memory)
|
||||
|
||||
**Total**: 10-15 minutes, 815MB peak memory
|
||||
|
||||
### TFT Only (Faster Iteration)
|
||||
```bash
|
||||
./train_tft_only.sh
|
||||
```
|
||||
|
||||
Trains only TFT model:
|
||||
- **Time**: 3-5 minutes
|
||||
- **Memory**: 500MB
|
||||
- **Use case**: Quick experiments, hyperparameter tuning
|
||||
|
||||
### Manual Training
|
||||
```bash
|
||||
./bin/train_tft_parquet \
|
||||
--parquet-file test_data/ES_FUT_180d.parquet \
|
||||
--epochs 50 \
|
||||
--output-dir ./models_custom
|
||||
```
|
||||
|
||||
## 📈 Expected Performance
|
||||
|
||||
Based on Wave D backtest validation:
|
||||
- **Sharpe Ratio**: 2.00 (≥2.0 target ✅)
|
||||
- **Win Rate**: 60% (≥60% target ✅)
|
||||
- **Max Drawdown**: 15% (≤15% target ✅)
|
||||
|
||||
## 🔧 Troubleshooting
|
||||
|
||||
### CUDA Not Found
|
||||
```bash
|
||||
# Check GPU
|
||||
nvidia-smi
|
||||
|
||||
# If missing, verify pod has GPU attached
|
||||
# Runpod: Settings → GPU Type → Tesla V100
|
||||
```
|
||||
|
||||
### Out of Memory
|
||||
```bash
|
||||
# Use smaller dataset
|
||||
./bin/train_tft_parquet \
|
||||
--parquet-file test_data/ES_FUT_small.parquet \
|
||||
--epochs 50
|
||||
```
|
||||
|
||||
### Binary Not Executable
|
||||
```bash
|
||||
chmod +x bin/*
|
||||
```
|
||||
|
||||
## 📊 Cost Estimation
|
||||
|
||||
**Tesla V100 16GB @ $0.25/hour**
|
||||
- Full training (15 min): $0.06
|
||||
- TFT only (5 min): $0.02
|
||||
- 100 training runs: $6.00
|
||||
|
||||
**Best for**:
|
||||
- Initial model training
|
||||
- Hyperparameter tuning (100+ runs)
|
||||
- Production model retraining
|
||||
|
||||
## 🔐 Security Notes
|
||||
|
||||
- All files are private in your Runpod account
|
||||
- No AWS credentials needed
|
||||
- No external network access required
|
||||
- Training runs entirely on local GPU
|
||||
|
||||
## 📝 Next Steps
|
||||
|
||||
After training:
|
||||
1. Download models from pod
|
||||
2. Deploy to production infrastructure
|
||||
3. Monitor with Grafana dashboards
|
||||
4. Begin paper trading validation
|
||||
|
||||
See RUNPOD_DEPLOYMENT_CHECKLIST.md in main repo for full deployment guide.
|
||||
README_EOF
|
||||
|
||||
echo " ✓ README.md"
|
||||
|
||||
# Create tarball
|
||||
echo ""
|
||||
echo "📦 Creating package tarball..."
|
||||
cd "$FOXHUNT_ROOT"
|
||||
tar -czf "$OUTPUT_DIR/$PACKAGE_NAME" -C "$OUTPUT_DIR" .
|
||||
|
||||
PACKAGE_SIZE=$(stat -c%s "$OUTPUT_DIR/$PACKAGE_NAME" 2>/dev/null || stat -f%z "$OUTPUT_DIR/$PACKAGE_NAME" 2>/dev/null)
|
||||
PACKAGE_SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $PACKAGE_SIZE/1024/1024}")
|
||||
|
||||
echo "✅ Package created: $PACKAGE_NAME ($PACKAGE_SIZE_MB MB)"
|
||||
|
||||
# Generate deployment instructions
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "✅ Package ready for deployment!"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
echo "📦 Package: $OUTPUT_DIR/$PACKAGE_NAME"
|
||||
echo "📊 Size: $PACKAGE_SIZE_MB MB"
|
||||
echo ""
|
||||
echo "🚀 DEPLOYMENT OPTIONS:"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "Option 1: Runpod Web Interface (Easiest)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "1. Go to https://www.runpod.io/console/pods"
|
||||
echo "2. Deploy pod:"
|
||||
echo " - GPU: Tesla V100 16GB (\$0.14-0.39/hr)"
|
||||
echo " - Template: PyTorch or CUDA base"
|
||||
echo " - Network Volume: Create new (10GB minimum)"
|
||||
echo "3. Upload via web interface:"
|
||||
echo " - Navigate to 'Files' tab in pod"
|
||||
echo " - Upload: $PACKAGE_NAME"
|
||||
echo "4. SSH into pod and run:"
|
||||
echo " cd /workspace"
|
||||
echo " tar -xzf $PACKAGE_NAME"
|
||||
echo " ./runpod_setup.sh"
|
||||
echo " ./train_fp32.sh"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "Option 2: SSH Direct Upload"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "1. Deploy pod with SSH enabled"
|
||||
echo "2. Get SSH connection string from Runpod console"
|
||||
echo "3. Upload via SCP:"
|
||||
echo " scp $OUTPUT_DIR/$PACKAGE_NAME root@<pod-ip>:/workspace/"
|
||||
echo "4. SSH and extract:"
|
||||
echo " ssh root@<pod-ip>"
|
||||
echo " cd /workspace"
|
||||
echo " tar -xzf $PACKAGE_NAME"
|
||||
echo " ./runpod_setup.sh"
|
||||
echo " ./train_fp32.sh"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "Option 3: HTTP Server Upload (Fast)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "1. On local machine, start HTTP server:"
|
||||
echo " cd $OUTPUT_DIR"
|
||||
echo " python3 -m http.server 8000"
|
||||
echo "2. Get your public IP:"
|
||||
echo " curl ifconfig.me"
|
||||
echo "3. In Runpod pod, download:"
|
||||
echo " cd /workspace"
|
||||
echo " wget http://<your-ip>:8000/$PACKAGE_NAME"
|
||||
echo " tar -xzf $PACKAGE_NAME"
|
||||
echo " ./runpod_setup.sh"
|
||||
echo " ./train_fp32.sh"
|
||||
echo " ⚠️ WARNING: Only use on trusted networks!"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "💰 PRICING REFERENCE (Tesla V100 16GB)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "Runpod Community Cloud: \$0.14-0.39/hour"
|
||||
echo "DataCrunch.io: \$0.39/hour"
|
||||
echo "Google Cloud: \$2.48/hour"
|
||||
echo "Azure NCv3: \$3.06/hour"
|
||||
echo ""
|
||||
echo "💡 TIP: Use Runpod Community Cloud for best pricing"
|
||||
echo " (~\$0.10 per full training run)"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "📊 EXPECTED TRAINING PERFORMANCE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "Full FP32 Pipeline: 10-15 minutes"
|
||||
echo "TFT Only: 3-5 minutes"
|
||||
echo "Peak GPU Memory: 815MB (95% headroom on V100)"
|
||||
echo "Cost per run: ~\$0.06-0.10"
|
||||
echo ""
|
||||
echo "Wave D Backtest Results:"
|
||||
echo " Sharpe Ratio: 2.00 (≥2.0 target ✅)"
|
||||
echo " Win Rate: 60% (≥60% target ✅)"
|
||||
echo " Max Drawdown: 15% (≤15% target ✅)"
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "🎯 READY FOR DEPLOYMENT"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
echo "Next: Choose deployment option above and train models"
|
||||
echo "Full docs: $OUTPUT_DIR/README.md"
|
||||
477
scripts/archive/upload_to_runpod_volume.py
Executable file
477
scripts/archive/upload_to_runpod_volume.py
Executable file
@@ -0,0 +1,477 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RunPod Network Volume S3 Upload Script
|
||||
|
||||
Uploads binaries and test data to RunPod network volume via S3 API with smart
|
||||
checksum comparison to avoid unnecessary uploads.
|
||||
|
||||
Usage:
|
||||
./scripts/upload_to_runpod_volume.py --binaries # Upload binaries only
|
||||
./scripts/upload_to_runpod_volume.py --test-data # Upload test data only
|
||||
./scripts/upload_to_runpod_volume.py --all # Upload everything
|
||||
./scripts/upload_to_runpod_volume.py --all --force # Force re-upload (ignore checksums)
|
||||
./scripts/upload_to_runpod_volume.py --all --dry-run # Show what would be uploaded
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from dotenv import load_dotenv
|
||||
from rich.console import Console
|
||||
from rich.progress import (
|
||||
BarColumn,
|
||||
DownloadColumn,
|
||||
Progress,
|
||||
TaskID,
|
||||
TextColumn,
|
||||
TimeRemainingColumn,
|
||||
TransferSpeedColumn,
|
||||
)
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
class RunPodVolumeUploader:
|
||||
"""Uploads files to RunPod network volume via S3 API."""
|
||||
|
||||
def __init__(self, env_file: str = ".env.runpod"):
|
||||
"""Initialize uploader with credentials from env file."""
|
||||
# Load credentials
|
||||
env_path = Path(__file__).parent.parent / env_file
|
||||
if not env_path.exists():
|
||||
console.print(f"[red]✗ Error: {env_file} not found at {env_path}[/red]")
|
||||
console.print(f"[yellow]Run this script from the project root directory[/yellow]")
|
||||
sys.exit(1)
|
||||
|
||||
load_dotenv(env_path)
|
||||
|
||||
# Validate required env vars
|
||||
required_vars = [
|
||||
"RUNPOD_S3_ACCESS_KEY",
|
||||
"RUNPOD_S3_SECRET",
|
||||
"RUNPOD_S3_ENDPOINT",
|
||||
"RUNPOD_S3_REGION",
|
||||
"RUNPOD_VOLUME_ID",
|
||||
]
|
||||
missing = [var for var in required_vars if not os.getenv(var)]
|
||||
if missing:
|
||||
console.print(f"[red]✗ Missing required env vars: {', '.join(missing)}[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize S3 client
|
||||
self.bucket = os.getenv("RUNPOD_VOLUME_ID")
|
||||
self.s3_client = boto3.client(
|
||||
"s3",
|
||||
aws_access_key_id=os.getenv("RUNPOD_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key=os.getenv("RUNPOD_S3_SECRET"),
|
||||
region_name=os.getenv("RUNPOD_S3_REGION"),
|
||||
endpoint_url=os.getenv("RUNPOD_S3_ENDPOINT"),
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
|
||||
console.print(f"[green]✓ Initialized S3 client[/green]")
|
||||
console.print(f" Endpoint: {os.getenv('RUNPOD_S3_ENDPOINT')}")
|
||||
console.print(f" Volume ID: {self.bucket}")
|
||||
|
||||
def compute_md5(self, file_path: Path) -> str:
|
||||
"""Compute MD5 checksum of local file."""
|
||||
md5 = hashlib.md5()
|
||||
with open(file_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
md5.update(chunk)
|
||||
return md5.hexdigest()
|
||||
|
||||
def needs_upload(self, local_file: Path, s3_key: str) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Check if file needs upload based on checksum.
|
||||
|
||||
Returns:
|
||||
(needs_upload, reason)
|
||||
"""
|
||||
local_md5 = self.compute_md5(local_file)
|
||||
|
||||
try:
|
||||
response = self.s3_client.head_object(Bucket=self.bucket, Key=s3_key)
|
||||
remote_etag = response["ETag"].strip('"')
|
||||
|
||||
if local_md5 == remote_etag:
|
||||
return False, "up-to-date (checksum match)"
|
||||
else:
|
||||
return True, "changed (checksum mismatch)"
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "404":
|
||||
return True, "new file (doesn't exist on S3)"
|
||||
else:
|
||||
return True, f"error checking: {e}"
|
||||
|
||||
def upload_file_with_progress(
|
||||
self, local_file: Path, s3_key: str, progress: Progress, task: TaskID
|
||||
) -> bool:
|
||||
"""Upload file with progress bar."""
|
||||
file_size = local_file.stat().st_size
|
||||
|
||||
try:
|
||||
# Upload with progress callback
|
||||
def callback(bytes_transferred):
|
||||
progress.update(task, completed=bytes_transferred)
|
||||
|
||||
self.s3_client.upload_file(
|
||||
str(local_file),
|
||||
self.bucket,
|
||||
s3_key,
|
||||
Callback=callback,
|
||||
ExtraArgs={"ContentType": "application/octet-stream"},
|
||||
)
|
||||
|
||||
# Verify upload
|
||||
response = self.s3_client.head_object(Bucket=self.bucket, Key=s3_key)
|
||||
remote_size = response["ContentLength"]
|
||||
|
||||
if remote_size == file_size:
|
||||
return True
|
||||
else:
|
||||
console.print(
|
||||
f"[red]✗ Size mismatch: local={file_size}, remote={remote_size}[/red]"
|
||||
)
|
||||
self.s3_client.delete_object(Bucket=self.bucket, Key=s3_key)
|
||||
return False
|
||||
|
||||
except ClientError as e:
|
||||
console.print(f"[red]✗ Upload failed: {e}[/red]")
|
||||
return False
|
||||
|
||||
def list_volume_contents(self, prefix: str = "") -> List[Dict]:
|
||||
"""List contents of volume directory."""
|
||||
try:
|
||||
response = self.s3_client.list_objects_v2(Bucket=self.bucket, Prefix=prefix)
|
||||
return response.get("Contents", [])
|
||||
except ClientError as e:
|
||||
console.print(f"[red]✗ Failed to list volume: {e}[/red]")
|
||||
return []
|
||||
|
||||
def create_directory(self, s3_key: str, dry_run: bool = False) -> bool:
|
||||
"""Create directory on volume (by uploading empty marker object)."""
|
||||
if dry_run:
|
||||
console.print(f"[dim]Would create directory: {s3_key}[/dim]")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Check if directory already exists
|
||||
objects = self.list_volume_contents(prefix=s3_key)
|
||||
if objects:
|
||||
console.print(f"[dim] Directory exists: {s3_key}[/dim]")
|
||||
return True
|
||||
|
||||
# Create empty directory marker (trailing slash)
|
||||
self.s3_client.put_object(Bucket=self.bucket, Key=s3_key)
|
||||
console.print(f"[green]✓ Created directory: {s3_key}[/green]")
|
||||
return True
|
||||
except ClientError as e:
|
||||
console.print(f"[red]✗ Failed to create directory {s3_key}: {e}[/red]")
|
||||
return False
|
||||
|
||||
def upload_binaries(self, force: bool = False, dry_run: bool = False) -> bool:
|
||||
"""Upload training binaries to volume."""
|
||||
console.print("\n[bold cyan]Step 1: Upload Binaries[/bold cyan]")
|
||||
|
||||
binaries = [
|
||||
"train_tft_parquet",
|
||||
"train_mamba2_parquet",
|
||||
"train_dqn",
|
||||
"train_ppo",
|
||||
]
|
||||
|
||||
binaries_dir = Path("target/release/examples")
|
||||
if not binaries_dir.exists():
|
||||
console.print(f"[red]✗ Binaries directory not found: {binaries_dir}[/red]")
|
||||
console.print("[yellow]Run: cargo build --release --examples[/yellow]")
|
||||
return False
|
||||
|
||||
# Check which binaries exist locally
|
||||
existing_binaries = []
|
||||
for binary in binaries:
|
||||
binary_path = binaries_dir / binary
|
||||
if binary_path.exists():
|
||||
existing_binaries.append(binary)
|
||||
else:
|
||||
console.print(f"[yellow]⚠ Binary not found: {binary}[/yellow]")
|
||||
|
||||
if not existing_binaries:
|
||||
console.print("[red]✗ No binaries found to upload[/red]")
|
||||
return False
|
||||
|
||||
console.print(f"Found {len(existing_binaries)}/{len(binaries)} binaries")
|
||||
|
||||
# Upload each binary
|
||||
stats = {"uploaded": 0, "skipped": 0, "failed": 0, "total_bytes": 0}
|
||||
|
||||
with Progress(
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
DownloadColumn(),
|
||||
TransferSpeedColumn(),
|
||||
TimeRemainingColumn(),
|
||||
console=console,
|
||||
) as progress:
|
||||
for binary in existing_binaries:
|
||||
local_file = binaries_dir / binary
|
||||
s3_key = f"binaries/{binary}"
|
||||
file_size = local_file.stat().st_size
|
||||
|
||||
# Check if upload needed
|
||||
if not force:
|
||||
needs_upload, reason = self.needs_upload(local_file, s3_key)
|
||||
if not needs_upload:
|
||||
console.print(f"[green]✓ {binary}[/green]: {reason}")
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
else:
|
||||
console.print(f"[cyan]↻ {binary}[/cyan]: {reason}")
|
||||
|
||||
if dry_run:
|
||||
console.print(f"[dim]Would upload: {binary} ({file_size:,} bytes)[/dim]")
|
||||
stats["uploaded"] += 1
|
||||
stats["total_bytes"] += file_size
|
||||
continue
|
||||
|
||||
# Upload with progress bar
|
||||
task = progress.add_task(f"Uploading {binary}", total=file_size)
|
||||
success = self.upload_file_with_progress(local_file, s3_key, progress, task)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]✓ Uploaded: {binary} ({file_size:,} bytes)[/green]")
|
||||
stats["uploaded"] += 1
|
||||
stats["total_bytes"] += file_size
|
||||
else:
|
||||
console.print(f"[red]✗ Failed: {binary}[/red]")
|
||||
stats["failed"] += 1
|
||||
|
||||
# Summary
|
||||
console.print(f"\n[bold]Summary:[/bold]")
|
||||
console.print(f" Uploaded: {stats['uploaded']}")
|
||||
console.print(f" Skipped: {stats['skipped']}")
|
||||
console.print(f" Failed: {stats['failed']}")
|
||||
console.print(f" Total bytes: {stats['total_bytes']:,}")
|
||||
|
||||
return stats["failed"] == 0
|
||||
|
||||
def upload_test_data(self, force: bool = False, dry_run: bool = False) -> bool:
|
||||
"""Upload test data files to volume."""
|
||||
console.print("\n[bold cyan]Step 2: Upload Test Data[/bold cyan]")
|
||||
|
||||
test_data_dir = Path("test_data")
|
||||
if not test_data_dir.exists():
|
||||
console.print(f"[red]✗ Test data directory not found: {test_data_dir}[/red]")
|
||||
return False
|
||||
|
||||
# Find all parquet files
|
||||
parquet_files = list(test_data_dir.rglob("*.parquet"))
|
||||
if not parquet_files:
|
||||
console.print("[yellow]⚠ No parquet files found[/yellow]")
|
||||
return True
|
||||
|
||||
console.print(f"Found {len(parquet_files)} parquet files")
|
||||
|
||||
# Upload each file
|
||||
stats = {"uploaded": 0, "skipped": 0, "failed": 0, "total_bytes": 0}
|
||||
|
||||
with Progress(
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
DownloadColumn(),
|
||||
TransferSpeedColumn(),
|
||||
TimeRemainingColumn(),
|
||||
console=console,
|
||||
) as progress:
|
||||
for local_file in parquet_files:
|
||||
# Preserve directory structure
|
||||
rel_path = local_file.relative_to(test_data_dir)
|
||||
s3_key = f"test_data/{rel_path}"
|
||||
file_size = local_file.stat().st_size
|
||||
|
||||
# Check if upload needed
|
||||
if not force:
|
||||
needs_upload, reason = self.needs_upload(local_file, s3_key)
|
||||
if not needs_upload:
|
||||
console.print(f"[green]✓ {rel_path}[/green]: {reason}")
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
else:
|
||||
console.print(f"[cyan]↻ {rel_path}[/cyan]: {reason}")
|
||||
|
||||
if dry_run:
|
||||
console.print(f"[dim]Would upload: {rel_path} ({file_size:,} bytes)[/dim]")
|
||||
stats["uploaded"] += 1
|
||||
stats["total_bytes"] += file_size
|
||||
continue
|
||||
|
||||
# Upload with progress bar
|
||||
task = progress.add_task(f"Uploading {rel_path}", total=file_size)
|
||||
success = self.upload_file_with_progress(local_file, s3_key, progress, task)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]✓ Uploaded: {rel_path} ({file_size:,} bytes)[/green]")
|
||||
stats["uploaded"] += 1
|
||||
stats["total_bytes"] += file_size
|
||||
else:
|
||||
console.print(f"[red]✗ Failed: {rel_path}[/red]")
|
||||
stats["failed"] += 1
|
||||
|
||||
# Summary
|
||||
console.print(f"\n[bold]Summary:[/bold]")
|
||||
console.print(f" Uploaded: {stats['uploaded']}")
|
||||
console.print(f" Skipped: {stats['skipped']}")
|
||||
console.print(f" Failed: {stats['failed']}")
|
||||
console.print(f" Total bytes: {stats['total_bytes']:,}")
|
||||
|
||||
return stats["failed"] == 0
|
||||
|
||||
def create_output_directories(self, dry_run: bool = False) -> bool:
|
||||
"""Create output directories on volume."""
|
||||
console.print("\n[bold cyan]Step 3: Create Output Directories[/bold cyan]")
|
||||
|
||||
directories = [
|
||||
"models/",
|
||||
"models/tft/",
|
||||
"models/mamba2/",
|
||||
"models/dqn/",
|
||||
"models/ppo/",
|
||||
"logs/",
|
||||
"checkpoints/",
|
||||
]
|
||||
|
||||
# Directory creation is non-critical - models will create subdirs as needed
|
||||
for directory in directories:
|
||||
self.create_directory(directory, dry_run)
|
||||
|
||||
# Always return success - directory creation is optional
|
||||
return True
|
||||
|
||||
def verify_upload(self) -> None:
|
||||
"""Verify all uploads by listing volume contents."""
|
||||
console.print("\n[bold cyan]Step 4: Verify Upload[/bold cyan]")
|
||||
|
||||
# List binaries
|
||||
binaries = self.list_volume_contents(prefix="binaries/")
|
||||
console.print(f"\n[bold]Binaries:[/bold] ({len(binaries)} files)")
|
||||
for obj in binaries[:10]: # Show first 10
|
||||
size_mb = obj["Size"] / (1024 * 1024)
|
||||
console.print(f" {obj['Key']} ({size_mb:.1f} MB)")
|
||||
|
||||
# List test data
|
||||
test_data = self.list_volume_contents(prefix="test_data/")
|
||||
console.print(f"\n[bold]Test Data:[/bold] ({len(test_data)} files)")
|
||||
for obj in test_data[:10]: # Show first 10
|
||||
size_mb = obj["Size"] / (1024 * 1024)
|
||||
console.print(f" {obj['Key']} ({size_mb:.1f} MB)")
|
||||
|
||||
# List models directory
|
||||
models = self.list_volume_contents(prefix="models/")
|
||||
console.print(f"\n[bold]Models:[/bold] ({len(models)} files)")
|
||||
if models:
|
||||
for obj in models[:5]:
|
||||
size_mb = obj["Size"] / (1024 * 1024)
|
||||
console.print(f" {obj['Key']} ({size_mb:.1f} MB)")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upload binaries and test data to RunPod network volume via S3 API",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Upload binaries only
|
||||
./scripts/upload_to_runpod_volume.py --binaries
|
||||
|
||||
# Upload test data only
|
||||
./scripts/upload_to_runpod_volume.py --test-data
|
||||
|
||||
# Upload everything
|
||||
./scripts/upload_to_runpod_volume.py --all
|
||||
|
||||
# Force re-upload (ignore checksums)
|
||||
./scripts/upload_to_runpod_volume.py --all --force
|
||||
|
||||
# Dry run (show what would be uploaded)
|
||||
./scripts/upload_to_runpod_volume.py --all --dry-run
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--binaries", action="store_true", help="Upload training binaries"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test-data", action="store_true", help="Upload test data files"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all", action="store_true", help="Upload binaries and test data"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Force re-upload (ignore checksums)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would be uploaded without uploading",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--env-file",
|
||||
default=".env.runpod",
|
||||
help="Path to env file (default: .env.runpod)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate arguments
|
||||
if not (args.binaries or args.test_data or args.all):
|
||||
parser.print_help()
|
||||
console.print("\n[red]Error: Must specify --binaries, --test-data, or --all[/red]")
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize uploader
|
||||
uploader = RunPodVolumeUploader(env_file=args.env_file)
|
||||
|
||||
# Execute uploads
|
||||
success = True
|
||||
|
||||
if args.all or args.binaries:
|
||||
if not uploader.upload_binaries(force=args.force, dry_run=args.dry_run):
|
||||
success = False
|
||||
|
||||
if args.all or args.test_data:
|
||||
if not uploader.upload_test_data(force=args.force, dry_run=args.dry_run):
|
||||
success = False
|
||||
|
||||
if args.all:
|
||||
if not uploader.create_output_directories(dry_run=args.dry_run):
|
||||
success = False
|
||||
|
||||
# Verify upload
|
||||
if not args.dry_run:
|
||||
uploader.verify_upload()
|
||||
|
||||
# Exit with appropriate status
|
||||
if success:
|
||||
console.print("\n[bold green]✓ Upload complete![/bold green]")
|
||||
sys.exit(0)
|
||||
else:
|
||||
console.print("\n[bold red]✗ Upload failed![/bold red]")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
191
scripts/archive/verify_pod_deployment.py
Executable file
191
scripts/archive/verify_pod_deployment.py
Executable file
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Verify RunPod pod deployment - check datacenter and logs
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
||||
load_dotenv(env_path)
|
||||
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
sys.exit(1)
|
||||
|
||||
REST_API_URL = "https://rest.runpod.io/v1/pods"
|
||||
GRAPHQL_ENDPOINT = "https://api.runpod.io/graphql"
|
||||
|
||||
def get_pod_status(pod_id):
|
||||
"""Get current status of a pod."""
|
||||
print(f"\n🔍 Checking pod {pod_id} status...")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{REST_API_URL}/{pod_id}",
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
pod_data = response.json()
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("POD STATUS")
|
||||
print("="*70)
|
||||
|
||||
print(f"Pod ID: {pod_data.get('id', 'N/A')}")
|
||||
print(f"Name: {pod_data.get('name', 'N/A')}")
|
||||
print(f"Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
|
||||
|
||||
runtime = pod_data.get('runtime', {})
|
||||
print(f"Runtime Status: {runtime.get('status', 'UNKNOWN')}")
|
||||
|
||||
machine = pod_data.get('machine', {})
|
||||
datacenter = machine.get('dataCenterId', 'N/A')
|
||||
print(f"Datacenter: {datacenter}")
|
||||
|
||||
gpu_info = machine.get('gpuType', {})
|
||||
print(f"GPU: {gpu_info.get('displayName', 'N/A')}")
|
||||
print(f"GPU Count: {machine.get('gpuCount', 'N/A')}")
|
||||
|
||||
print(f"Image: {pod_data.get('imageName', 'N/A')}")
|
||||
print(f"Cost: ${pod_data.get('costPerHr', 'N/A')}/hr")
|
||||
|
||||
print("="*70)
|
||||
|
||||
# Check if datacenter is EUR-IS-1
|
||||
if datacenter == 'EUR-IS-1':
|
||||
print(f"\n✅ DATACENTER VERIFIED: Pod is in EUR-IS-1 (volume accessible)")
|
||||
else:
|
||||
print(f"\n❌ DATACENTER MISMATCH: Pod is in {datacenter}, volume is in EUR-IS-1!")
|
||||
|
||||
return pod_data
|
||||
elif response.status_code == 404:
|
||||
print(f" ⚠️ Pod not found")
|
||||
return None
|
||||
else:
|
||||
print(f" ⚠️ Failed to get pod status (status {response.status_code}): {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to check pod status: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def get_pod_logs_graphql(pod_id):
|
||||
"""Get logs from a pod using GraphQL (REST API logs endpoint doesn't work)."""
|
||||
print(f"\n📝 Fetching logs for pod {pod_id} via GraphQL...")
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
query = """
|
||||
query GetPodLogs($podId: String!) {
|
||||
pod(input: {podId: $podId}) {
|
||||
logs
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
variables = {
|
||||
"podId": pod_id
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
GRAPHQL_ENDPOINT,
|
||||
json={"query": query, "variables": variables},
|
||||
headers=headers,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
|
||||
if 'errors' in result:
|
||||
print(f" ⚠️ GraphQL errors: {result['errors']}")
|
||||
return None
|
||||
|
||||
logs = result.get('data', {}).get('pod', {}).get('logs', '')
|
||||
|
||||
if logs:
|
||||
print("\n" + "="*70)
|
||||
print("CONTAINER LOGS")
|
||||
print("="*70)
|
||||
# Print last 50 lines
|
||||
log_lines = logs.split('\n')
|
||||
for line in log_lines[-50:]:
|
||||
print(line)
|
||||
print("="*70)
|
||||
|
||||
# Check for volume mounting indicators
|
||||
if '/runpod-volume' in logs:
|
||||
print("\n✅ Volume path /runpod-volume found in logs")
|
||||
if 'No such file or directory' in logs and '/runpod-volume' in logs:
|
||||
print("\n❌ Volume NOT mounted - file not found errors detected")
|
||||
if 'test_data/ES_FUT_180d.parquet' in logs:
|
||||
print("\n✅ Parquet file path found in logs")
|
||||
|
||||
return logs
|
||||
else:
|
||||
print(" ⚠️ No logs available yet (pod may still be initializing)")
|
||||
return None
|
||||
else:
|
||||
print(f" ⚠️ Failed to get logs (status {response.status_code}): {response.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Failed to fetch logs: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print("="*70)
|
||||
print("VERIFY RUNPOD DEPLOYMENT")
|
||||
print("="*70)
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("\nUsage: python3 verify_pod_deployment.py <pod_id>")
|
||||
print("\nExample:")
|
||||
print(" python3 verify_pod_deployment.py b1m7v451nexg5r")
|
||||
sys.exit(1)
|
||||
|
||||
pod_id = sys.argv[1]
|
||||
|
||||
# Check pod status
|
||||
pod_data = get_pod_status(pod_id)
|
||||
|
||||
if not pod_data:
|
||||
print(f"\n❌ Failed to get pod status")
|
||||
sys.exit(1)
|
||||
|
||||
# Wait a bit for container to start
|
||||
runtime_status = pod_data.get('runtime', {}).get('status', 'UNKNOWN')
|
||||
if runtime_status != 'running':
|
||||
print(f"\n⏳ Container not running yet (status: {runtime_status})")
|
||||
print(" Waiting 30 seconds for container to start...")
|
||||
time.sleep(30)
|
||||
|
||||
# Get logs
|
||||
get_pod_logs_graphql(pod_id)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("VERIFICATION COMPLETE")
|
||||
print("="*70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
161
scripts/archive/verify_runpod_config.sh
Executable file
161
scripts/archive/verify_runpod_config.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
# Runpod Configuration Verification Script
|
||||
# Checks all prerequisites before executing training
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Runpod Configuration Verification${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
CHECKS_PASSED=0
|
||||
CHECKS_FAILED=0
|
||||
|
||||
# Check 1: runpodctl installation
|
||||
echo -n "Checking runpodctl installation... "
|
||||
if command -v runpodctl &>/dev/null; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}❌${NC}"
|
||||
echo " Error: runpodctl not found"
|
||||
echo " Install: curl -fsSL https://raw.githubusercontent.com/runpod/runpodctl/main/install.sh | bash"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 2: API key configuration
|
||||
echo -n "Checking Runpod API key... "
|
||||
if runpodctl config 2>&1 | grep -q "apiKey"; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}❌${NC}"
|
||||
echo " Error: API key not configured"
|
||||
echo " Get key: https://www.runpod.io/console/user/settings"
|
||||
echo " Configure: runpodctl config --apiKey YOUR_API_KEY"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 3: Training data
|
||||
echo -n "Checking training data (ES_FUT_180d.parquet)... "
|
||||
if [ -f "test_data/ES_FUT_180d.parquet" ]; then
|
||||
SIZE=$(du -h test_data/ES_FUT_180d.parquet | cut -f1)
|
||||
echo -e "${GREEN}✅ (${SIZE})${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}❌${NC}"
|
||||
echo " Error: test_data/ES_FUT_180d.parquet not found"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 4: 225 features configured
|
||||
echo -n "Checking 225-feature configuration... "
|
||||
if grep -q "features: \[f64; 225\]" ml/src/features/unified.rs; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}❌${NC}"
|
||||
echo " Error: 225 features not configured in ml/src/features/unified.rs"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 5: Training example exists
|
||||
echo -n "Checking TFT training example... "
|
||||
if [ -f "ml/examples/train_tft_parquet.rs" ]; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}❌${NC}"
|
||||
echo " Error: ml/examples/train_tft_parquet.rs not found"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 6: Cargo workspace valid
|
||||
echo -n "Checking Cargo workspace... "
|
||||
if cargo check --workspace --quiet 2>/dev/null; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ (Has warnings)${NC}"
|
||||
echo " Warning: Cargo check has warnings (non-blocking)"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
fi
|
||||
|
||||
# Check 7: Training script exists
|
||||
echo -n "Checking training script... "
|
||||
if [ -f "scripts/train_runpod_225_features.sh" ] && [ -x "scripts/train_runpod_225_features.sh" ]; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}❌${NC}"
|
||||
echo " Error: scripts/train_runpod_225_features.sh not found or not executable"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 8: Backtesting script exists
|
||||
echo -n "Checking backtesting script... "
|
||||
if [ -f "scripts/backtest_runpod_225.sh" ] && [ -x "scripts/backtest_runpod_225.sh" ]; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}❌${NC}"
|
||||
echo " Error: scripts/backtest_runpod_225.sh not found or not executable"
|
||||
CHECKS_FAILED=$((CHECKS_FAILED + 1))
|
||||
fi
|
||||
|
||||
# Check 9: CUDA availability (optional)
|
||||
echo -n "Checking CUDA availability (optional)... "
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
GPU_INFO=$(nvidia-smi --query-gpu=name,memory.total --format=csv,noheader | head -1)
|
||||
echo -e "${GREEN}✅ (${GPU_INFO})${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ (Local GPU not required for Runpod)${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
fi
|
||||
|
||||
# Check 10: Docker services (optional)
|
||||
echo -n "Checking Docker services (optional)... "
|
||||
if docker-compose ps 2>/dev/null | grep -q "postgres"; then
|
||||
echo -e "${GREEN}✅${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ (Not required for Runpod training)${NC}"
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
fi
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Verification Summary${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "Checks Passed: ${GREEN}${CHECKS_PASSED}${NC}"
|
||||
echo -e "Checks Failed: ${RED}${CHECKS_FAILED}${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $CHECKS_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ All checks passed! Ready for Runpod training.${NC}"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Execute training: ./scripts/train_runpod_225_features.sh"
|
||||
echo " 2. Monitor progress: tail -f runpod_training_225.log"
|
||||
echo " 3. Run backtesting: ./scripts/backtest_runpod_225.sh"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}❌ ${CHECKS_FAILED} check(s) failed. Please fix errors before training.${NC}"
|
||||
echo ""
|
||||
echo "Common fixes:"
|
||||
echo " • API key: runpodctl config --apiKey YOUR_API_KEY"
|
||||
echo " • Training data: Download ES_FUT_180d.parquet to test_data/"
|
||||
echo " • Scripts: chmod +x scripts/*.sh"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user