433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
108 lines
3.6 KiB
Bash
Executable File
108 lines
3.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# Profile-Guided Optimization (PGO) Implementation Script
|
|
# Expected Impact: 5-15% performance improvement
|
|
# See: AGENT_15_RUST_COMPILER_OPTIMIZATION_ANALYSIS.md
|
|
|
|
set -e
|
|
|
|
echo "=== Profile-Guided Optimization (PGO) Implementation ==="
|
|
echo ""
|
|
|
|
# Check if cargo-pgo is installed
|
|
if ! command -v cargo-pgo &> /dev/null; then
|
|
echo "Installing cargo-pgo..."
|
|
cargo install cargo-pgo
|
|
fi
|
|
|
|
# Step 1: Build instrumented binary
|
|
echo "Step 1: Building instrumented binary for profiling..."
|
|
RUSTFLAGS="-C target-cpu=native" cargo pgo build --release
|
|
|
|
# Step 2: Generate profile data with representative workloads
|
|
echo ""
|
|
echo "Step 2: Generating profile data with representative workloads..."
|
|
|
|
# Workload 1: Backtesting with real market data (ES.FUT 180 days)
|
|
echo " - Running backtest with ES.FUT (180 days)..."
|
|
if [ -f "test_data/ES_FUT_180d.parquet" ]; then
|
|
timeout 60s cargo run --release -p backtesting_service -- \
|
|
--symbol ES.FUT --duration 180d --profile-output /tmp/pgo-profile || true
|
|
fi
|
|
|
|
# Workload 2: TFT training (ML hot path)
|
|
echo " - Running TFT training (10 epochs for profiling)..."
|
|
if [ -f "test_data/ES_FUT_180d.parquet" ]; then
|
|
timeout 120s cargo run --release -p ml --example train_tft_parquet --features cuda -- \
|
|
--parquet-file test_data/ES_FUT_180d.parquet --epochs 10 \
|
|
--profile-output /tmp/pgo-profile || true
|
|
fi
|
|
|
|
# Workload 3: Order matching benchmarks
|
|
echo " - Running order matching benchmarks..."
|
|
timeout 60s cargo bench --bench performance_regression -- \
|
|
--profile-output /tmp/pgo-profile || true
|
|
|
|
# Workload 4: API Gateway load test
|
|
echo " - Running API Gateway proxy benchmarks..."
|
|
timeout 30s cargo bench --bench trading_latency -- \
|
|
--profile-output /tmp/pgo-profile || true
|
|
|
|
# Step 3: Build optimized binary with profile data
|
|
echo ""
|
|
echo "Step 3: Building PGO-optimized binary..."
|
|
RUSTFLAGS="-C target-cpu=native" cargo pgo optimize --release
|
|
|
|
# Step 4: Run validation benchmarks
|
|
echo ""
|
|
echo "Step 4: Running validation benchmarks..."
|
|
echo " - Baseline (before PGO)..."
|
|
git stash
|
|
cargo build --release
|
|
cargo bench --bench performance_regression -- --save-baseline pgo-before
|
|
git stash pop
|
|
|
|
echo " - Optimized (after PGO)..."
|
|
cargo build --release --profile release-pgo
|
|
cargo bench --bench performance_regression -- --baseline pgo-before
|
|
|
|
# Step 5: Generate performance report
|
|
echo ""
|
|
echo "Step 5: Generating performance improvement report..."
|
|
cat << 'EOF' > /tmp/pgo_report.md
|
|
# PGO Performance Improvement Report
|
|
|
|
**Date**: $(date -u +%Y-%m-%d)
|
|
**Profile Data**: Representative HFT workload (backtest + ML training + benchmarks)
|
|
|
|
## Expected Improvements
|
|
|
|
| Benchmark | Before PGO | After PGO | Improvement |
|
|
|-----------|------------|-----------|-------------|
|
|
| Order Matching | TBD | TBD | TBD |
|
|
| Authentication | TBD | TBD | TBD |
|
|
| Order Submission | TBD | TBD | TBD |
|
|
| API Gateway Proxy | TBD | TBD | TBD |
|
|
| DBN Data Loading | TBD | TBD | TBD |
|
|
| TFT Inference | TBD | TBD | TBD |
|
|
|
|
**Target**: 5-15% improvement in CPU-bound hot paths
|
|
|
|
## Next Steps
|
|
|
|
1. Review benchmark results in `target/criterion/`
|
|
2. If improvements meet expectations (5-15%), deploy PGO builds to production
|
|
3. Update CI/CD pipeline to use PGO for release builds
|
|
4. Consider BOLT post-link optimization for additional 2-8% gains
|
|
|
|
See: AGENT_15_RUST_COMPILER_OPTIMIZATION_ANALYSIS.md for full analysis
|
|
EOF
|
|
|
|
echo ""
|
|
echo "✅ PGO implementation complete!"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo "1. Review benchmark results in target/criterion/"
|
|
echo "2. Check PGO report in /tmp/pgo_report.md"
|
|
echo "3. If successful, integrate into CI/CD pipeline"
|
|
echo "4. Proceed to Priority 2: Static linking (musl target)"
|