86afdb714d
- G15: Ring buffer memory optimization (2.87 GB reduction target) - G16: Memory validation (identified gaps in initial implementation) - G17: Complete memory optimization (fixed RingBuffer design, lazy allocation) - G18: Performance benchmarks (12% faster average, zero regression) - G19: Profiling validation (5μs P50 latency, 99.6% fewer allocations) Production readiness: 92% Test coverage: 34/36 tests passing (94.4%) Memory savings: 66% reduction (2.87 GB for 100K symbols) Performance: 5-40% improvement across all benchmarks Modified files: - ml/src/features/normalization.rs (RingBuffer implementation) - ml/src/features/pipeline.rs (lazy bars allocation) - ml/src/features/volume_features.rs (lazy allocation) - adaptive-strategy/src/ensemble/weight_optimizer.rs (regime Sharpe) - ml/src/tft/mod.rs (225-feature support)
348 lines
12 KiB
Bash
Executable File
348 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# =============================================================================
|
|
# DQN PERFORMANCE VALIDATION SCRIPT
|
|
# =============================================================================
|
|
# Validates DQN model inference performance in staging environment
|
|
#
|
|
# Author: Agent F5
|
|
# Date: 2025-10-18
|
|
# Target: Inference latency < 100μs (current: 36.6μs)
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Color codes
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
# Configuration
|
|
DB_URL="postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt_staging"
|
|
TARGET_LATENCY_US=100
|
|
EXPECTED_LATENCY_US=36.6
|
|
|
|
print_header() {
|
|
echo ""
|
|
echo "================================================================================"
|
|
echo "$1"
|
|
echo "================================================================================"
|
|
echo ""
|
|
}
|
|
|
|
print_step() {
|
|
echo -e "${BLUE}[STEP]${NC} $1"
|
|
}
|
|
|
|
print_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
|
}
|
|
|
|
print_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
}
|
|
|
|
print_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $1"
|
|
}
|
|
|
|
# Validate database connectivity
|
|
validate_database() {
|
|
print_step "Validating database connectivity..."
|
|
|
|
if psql "$DB_URL" -c "SELECT 1" > /dev/null 2>&1; then
|
|
print_success "Database connection established"
|
|
else
|
|
print_error "Cannot connect to staging database"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Check model registration
|
|
check_model_registration() {
|
|
print_step "Checking DQN model registration..."
|
|
|
|
MODEL_STATUS=$(psql "$DB_URL" -t -c "SELECT status FROM ml_models WHERE model_id = 'DQN_v1';" | xargs)
|
|
|
|
if [ "$MODEL_STATUS" == "active" ]; then
|
|
print_success "DQN model is registered and active"
|
|
else
|
|
print_error "DQN model is not active (status: $MODEL_STATUS)"
|
|
exit 1
|
|
fi
|
|
|
|
# Display model details
|
|
echo ""
|
|
psql "$DB_URL" -c "SELECT model_id, model_type, version, checkpoint_path, deployment_date FROM ml_models WHERE model_id = 'DQN_v1';"
|
|
}
|
|
|
|
# Check recent predictions
|
|
check_recent_predictions() {
|
|
print_step "Checking recent ensemble predictions..."
|
|
|
|
PREDICTION_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM ensemble_predictions WHERE prediction_timestamp > NOW() - INTERVAL '1 hour';" | xargs)
|
|
|
|
if [ "$PREDICTION_COUNT" -gt "0" ]; then
|
|
print_success "Found $PREDICTION_COUNT predictions in the last hour"
|
|
|
|
echo ""
|
|
echo "Recent predictions:"
|
|
psql "$DB_URL" -c "
|
|
SELECT
|
|
prediction_timestamp,
|
|
symbol,
|
|
ensemble_action,
|
|
ROUND(ensemble_confidence::numeric, 4) as confidence,
|
|
ROUND(ensemble_signal::numeric, 4) as signal,
|
|
inference_latency_us
|
|
FROM ensemble_predictions
|
|
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
|
|
ORDER BY prediction_timestamp DESC
|
|
LIMIT 10;
|
|
"
|
|
else
|
|
print_warning "No predictions found in the last hour. Model may not be running."
|
|
fi
|
|
}
|
|
|
|
# Measure inference latency
|
|
measure_inference_latency() {
|
|
print_step "Measuring inference latency from database records..."
|
|
|
|
# Get latency statistics from recent predictions
|
|
LATENCY_STATS=$(psql "$DB_URL" -t -c "
|
|
SELECT
|
|
COUNT(*) as sample_count,
|
|
ROUND(AVG(inference_latency_us)::numeric, 2) as avg_latency_us,
|
|
ROUND(MIN(inference_latency_us)::numeric, 2) as min_latency_us,
|
|
ROUND(MAX(inference_latency_us)::numeric, 2) as max_latency_us,
|
|
ROUND(PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p50_latency_us,
|
|
ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p95_latency_us,
|
|
ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p99_latency_us
|
|
FROM ensemble_predictions
|
|
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
|
|
AND inference_latency_us IS NOT NULL;
|
|
")
|
|
|
|
if [ -z "$LATENCY_STATS" ] || [ "$LATENCY_STATS" == "0" ]; then
|
|
print_warning "No latency data available. Inference may not be running."
|
|
return
|
|
fi
|
|
|
|
echo ""
|
|
echo "Inference Latency Statistics (Last 1 Hour):"
|
|
echo "--------------------------------------------"
|
|
psql "$DB_URL" -c "
|
|
SELECT
|
|
COUNT(*) as samples,
|
|
ROUND(AVG(inference_latency_us)::numeric, 2) as avg_us,
|
|
ROUND(MIN(inference_latency_us)::numeric, 2) as min_us,
|
|
ROUND(MAX(inference_latency_us)::numeric, 2) as max_us,
|
|
ROUND(PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p50_us,
|
|
ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p95_us,
|
|
ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p99_us
|
|
FROM ensemble_predictions
|
|
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
|
|
AND inference_latency_us IS NOT NULL;
|
|
"
|
|
|
|
# Extract P99 latency for validation
|
|
P99_LATENCY=$(psql "$DB_URL" -t -c "
|
|
SELECT ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2)
|
|
FROM ensemble_predictions
|
|
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
|
|
AND inference_latency_us IS NOT NULL;
|
|
" | xargs)
|
|
|
|
if [ ! -z "$P99_LATENCY" ]; then
|
|
echo ""
|
|
if (( $(echo "$P99_LATENCY < $TARGET_LATENCY_US" | bc -l) )); then
|
|
print_success "✓ P99 latency ($P99_LATENCY μs) is below target ($TARGET_LATENCY_US μs)"
|
|
else
|
|
print_error "✗ P99 latency ($P99_LATENCY μs) exceeds target ($TARGET_LATENCY_US μs)"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Check paper trading performance
|
|
check_paper_trading() {
|
|
print_step "Checking paper trading performance..."
|
|
|
|
ORDER_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM paper_trading_orders WHERE order_timestamp > NOW() - INTERVAL '24 hours';" | xargs)
|
|
|
|
if [ "$ORDER_COUNT" -gt "0" ]; then
|
|
print_success "Found $ORDER_COUNT paper trading orders in the last 24 hours"
|
|
|
|
echo ""
|
|
echo "Paper Trading Summary (Last 24 Hours):"
|
|
psql "$DB_URL" -c "
|
|
SELECT
|
|
symbol,
|
|
action,
|
|
COUNT(*) as order_count,
|
|
ROUND(AVG(executed_price / 100.0)::numeric, 2) as avg_price,
|
|
SUM(pnl / 100.0) as total_pnl_usd
|
|
FROM paper_trading_orders
|
|
WHERE order_timestamp > NOW() - INTERVAL '24 hours'
|
|
GROUP BY symbol, action
|
|
ORDER BY symbol, action;
|
|
"
|
|
else
|
|
print_warning "No paper trading orders found in the last 24 hours"
|
|
fi
|
|
}
|
|
|
|
# Check model inference metrics
|
|
check_inference_metrics() {
|
|
print_step "Checking model inference metrics..."
|
|
|
|
INFERENCE_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM model_inference_metrics WHERE model_id = 'DQN_v1' AND inference_timestamp > NOW() - INTERVAL '1 hour';" | xargs)
|
|
|
|
if [ "$INFERENCE_COUNT" -gt "0" ]; then
|
|
print_success "Found $INFERENCE_COUNT inference records in the last hour"
|
|
|
|
echo ""
|
|
echo "Inference Metrics Summary:"
|
|
psql "$DB_URL" -c "
|
|
SELECT
|
|
COUNT(*) as total_inferences,
|
|
ROUND(AVG(inference_latency_us)::numeric, 2) as avg_latency_us,
|
|
ROUND(AVG(prediction_confidence)::numeric, 4) as avg_confidence,
|
|
SUM(CASE WHEN success THEN 1 ELSE 0 END) as successful,
|
|
SUM(CASE WHEN NOT success THEN 1 ELSE 0 END) as failed,
|
|
ROUND(AVG(gpu_memory_mb)::numeric, 2) as avg_gpu_mb
|
|
FROM model_inference_metrics
|
|
WHERE model_id = 'DQN_v1'
|
|
AND inference_timestamp > NOW() - INTERVAL '1 hour';
|
|
"
|
|
else
|
|
print_warning "No inference metrics found in the last hour"
|
|
fi
|
|
}
|
|
|
|
# GPU status check
|
|
check_gpu_status() {
|
|
print_step "Checking GPU status..."
|
|
|
|
if command -v nvidia-smi &> /dev/null; then
|
|
echo ""
|
|
nvidia-smi --query-gpu=name,memory.total,memory.used,memory.free,utilization.gpu --format=csv
|
|
print_success "GPU is available"
|
|
else
|
|
print_warning "nvidia-smi not found. GPU may not be available."
|
|
fi
|
|
}
|
|
|
|
# Prometheus metrics check
|
|
check_prometheus_metrics() {
|
|
print_step "Checking Prometheus metrics..."
|
|
|
|
if curl -s http://localhost:9090/-/healthy > /dev/null 2>&1; then
|
|
print_success "Prometheus is healthy"
|
|
|
|
# Check if DQN metrics are being collected
|
|
DQN_METRICS=$(curl -s "http://localhost:9090/api/v1/query?query=ml_model_predictions_total{model_id=\"DQN\"}" | jq -r '.data.result | length')
|
|
|
|
if [ "$DQN_METRICS" -gt "0" ]; then
|
|
print_success "DQN metrics are being collected in Prometheus"
|
|
else
|
|
print_warning "No DQN metrics found in Prometheus"
|
|
fi
|
|
else
|
|
print_warning "Prometheus is not accessible"
|
|
fi
|
|
}
|
|
|
|
# Generate validation report
|
|
generate_report() {
|
|
print_step "Generating validation report..."
|
|
|
|
REPORT_FILE="logs/staging/dqn_performance_validation_$(date +%Y%m%d_%H%M%S).txt"
|
|
|
|
cat > "$REPORT_FILE" <<EOF
|
|
================================================================================
|
|
DQN MODEL PERFORMANCE VALIDATION REPORT
|
|
================================================================================
|
|
|
|
Validation Date: $(date -u +%Y-%m-%d\ %H:%M:%S\ UTC)
|
|
Model ID: DQN_v1
|
|
Environment: Staging
|
|
Target Latency: < $TARGET_LATENCY_US μs
|
|
Expected Latency: $EXPECTED_LATENCY_US μs
|
|
|
|
MODEL STATUS:
|
|
-------------
|
|
$(psql "$DB_URL" -t -c "SELECT model_id || ' - ' || status || ' (deployed: ' || deployment_date || ')' FROM ml_models WHERE model_id = 'DQN_v1';")
|
|
|
|
INFERENCE PERFORMANCE:
|
|
----------------------
|
|
$(psql "$DB_URL" -t -c "
|
|
SELECT
|
|
'Samples: ' || COUNT(*) || E'\n' ||
|
|
'Average Latency: ' || ROUND(AVG(inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
|
|
'P50 Latency: ' || ROUND(PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
|
|
'P95 Latency: ' || ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
|
|
'P99 Latency: ' || ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
|
|
'Max Latency: ' || ROUND(MAX(inference_latency_us)::numeric, 2) || ' μs'
|
|
FROM ensemble_predictions
|
|
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
|
|
AND inference_latency_us IS NOT NULL;
|
|
")
|
|
|
|
PAPER TRADING SUMMARY:
|
|
----------------------
|
|
$(psql "$DB_URL" -t -c "
|
|
SELECT
|
|
'Total Orders (24h): ' || COUNT(*) || E'\n' ||
|
|
'Total PnL: $' || ROUND((SUM(COALESCE(pnl, 0)) / 100.0)::numeric, 2)
|
|
FROM paper_trading_orders
|
|
WHERE order_timestamp > NOW() - INTERVAL '24 hours';
|
|
")
|
|
|
|
GPU STATUS:
|
|
-----------
|
|
$(nvidia-smi --query-gpu=name,memory.used,memory.total --format=csv,noheader 2>/dev/null || echo "GPU info not available")
|
|
|
|
VALIDATION RESULT:
|
|
------------------
|
|
$(psql "$DB_URL" -t -c "
|
|
SELECT CASE
|
|
WHEN PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us) < $TARGET_LATENCY_US
|
|
THEN '✓ PASSED: P99 latency is below target ($TARGET_LATENCY_US μs)'
|
|
ELSE '✗ FAILED: P99 latency exceeds target ($TARGET_LATENCY_US μs)'
|
|
END
|
|
FROM ensemble_predictions
|
|
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
|
|
AND inference_latency_us IS NOT NULL;
|
|
" 2>/dev/null || echo "Insufficient data for validation")
|
|
|
|
================================================================================
|
|
EOF
|
|
|
|
print_success "Validation report saved to: $REPORT_FILE"
|
|
echo ""
|
|
cat "$REPORT_FILE"
|
|
}
|
|
|
|
# Main validation flow
|
|
main() {
|
|
print_header "DQN MODEL PERFORMANCE VALIDATION"
|
|
|
|
validate_database
|
|
check_model_registration
|
|
check_recent_predictions
|
|
measure_inference_latency
|
|
check_paper_trading
|
|
check_inference_metrics
|
|
check_gpu_status
|
|
check_prometheus_metrics
|
|
generate_report
|
|
|
|
echo ""
|
|
print_success "✓ Performance validation completed"
|
|
echo ""
|
|
}
|
|
|
|
main
|