From 2da5bafc0e48d292b89b3b34d6ee63fd4a4607e6 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 24 Feb 2026 10:31:47 +0100 Subject: [PATCH] =?UTF-8?q?refactor:=20rename=20tli=E2=86=92fxt,=20delete?= =?UTF-8?q?=20legacy=20scripts/RunPod/deploy=20artifacts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename tli/ directory to fxt/, update package + binary name to "fxt" - Replace all `use tli::` → `use fxt::` across 52 Rust files - Update build.rs proto paths (tli/proto → fxt/proto) in 6 services - Update Dockerfiles, CI workflows, deploy.sh for new paths - Delete ~170 legacy shell scripts (kept 15 essential ones) - Delete RunPod Python client (runpod/), tests (tests/runpod/) - Delete foxhunt-deploy crate (RunPod-only deployment tool) - Delete terraform/runpod/ (moved to Scaleway) - Delete ML Python hyperopt scripts (replaced by Rust Argmin PSO) - Delete .gitlab-ci.yml (using GitHub + Gitea) - Remove foxhunt-deploy from workspace members 504 files changed, -74,355 lines of legacy code removed. Workspace compiles clean (0 errors, 0 warnings). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/build.yml | 22 +- .github/workflows/ci-cd-pipeline.yml | 10 +- .github/workflows/comprehensive-testing.yml | 28 +- .gitlab-ci.yml | 454 ----------- Cargo.lock | 140 ++-- Cargo.toml | 7 +- config/mutants.toml | 2 +- deploy.sh | 6 +- foxhunt-deploy/Cargo.toml | 63 -- foxhunt-deploy/DEPLOYMENT_QUICK_START.md | 289 ------- foxhunt-deploy/DOCKER_BUILD_MILESTONE2.md | 378 --------- foxhunt-deploy/MILESTONE_1_COMPLETE.md | 221 ----- foxhunt-deploy/MILESTONE_3_SUMMARY.md | 427 ---------- foxhunt-deploy/MILESTONE_4_SUMMARY.md | 480 ----------- foxhunt-deploy/MONITOR_QUICK_REF.md | 99 --- foxhunt-deploy/QUICK_START_BUILD.md | 302 ------- foxhunt-deploy/README.md | 420 ---------- .../RUNPOD_DEPLOYMENT_IMPLEMENTATION.md | 359 --------- foxhunt-deploy/S3_MONITOR_IMPLEMENTATION.md | 398 --------- foxhunt-deploy/examples/monitor_demo.sh | 123 --- foxhunt-deploy/examples/sample_config.toml | 50 -- foxhunt-deploy/src/cli/build.rs | 67 -- foxhunt-deploy/src/cli/deploy.rs | 249 ------ foxhunt-deploy/src/cli/mod.rs | 48 -- foxhunt-deploy/src/cli/monitor.rs | 56 -- foxhunt-deploy/src/cli/run.rs | 34 - foxhunt-deploy/src/config/mod.rs | 207 ----- foxhunt-deploy/src/config/types.rs | 167 ---- foxhunt-deploy/src/docker/build.rs | 226 ------ foxhunt-deploy/src/docker/mod.rs | 65 -- foxhunt-deploy/src/docker/push.rs | 130 --- foxhunt-deploy/src/error.rs | 45 -- foxhunt-deploy/src/main.rs | 102 --- foxhunt-deploy/src/runpod/client.rs | 120 --- foxhunt-deploy/src/runpod/deployment.rs | 304 ------- foxhunt-deploy/src/runpod/mod.rs | 8 - foxhunt-deploy/src/runpod/types.rs | 168 ---- foxhunt-deploy/src/s3/mod.rs | 165 ---- foxhunt-deploy/src/s3/monitor.rs | 253 ------ foxhunt-deploy/src/s3/parser.rs | 214 ----- foxhunt-deploy/src/utils/mod.rs | 4 - foxhunt-deploy/src/utils/terminal.rs | 50 -- .../tests/docker_integration_tests.rs | 169 ---- {tli => fxt}/CONFIG_FILE_SUPPORT.md | 0 {tli => fxt}/Cargo.toml | 8 +- {tli => fxt}/TUNE_COMMAND_README.md | 0 {tli => fxt}/benches/client_performance.rs | 14 +- .../benches/configuration_benchmarks.rs | 6 +- .../benches/encryption_performance.rs | 4 +- .../benches/serialization_benchmarks.rs | 6 +- {tli => fxt}/build.rs | 0 {tli => fxt}/config.toml.example | 0 {tli => fxt}/docs/AUTHENTICATION.md | 0 {tli => fxt}/docs/ML_TRADING_COMMANDS.md | 0 {tli => fxt}/docs/USAGE.md | 0 .../examples/complete_client_example.rs | 2 +- {tli => fxt}/examples/config_demo.rs | 0 .../examples/config_management_placeholder.rs | 0 {tli => fxt}/examples/security_example.rs | 0 {tli => fxt}/proptest-regressions/tests.txt | 0 {tli => fxt}/proto/config.proto | 0 {tli => fxt}/proto/health.proto | 0 {tli => fxt}/proto/ml.proto | 0 {tli => fxt}/proto/ml_training.proto | 0 {tli => fxt}/proto/trading.proto | 0 {tli => fxt}/proto/trading_agent.proto | 0 {tli => fxt}/src/auth/encryption.rs | 12 +- {tli => fxt}/src/auth/interceptor.rs | 0 {tli => fxt}/src/auth/jwt_generator.rs | 0 {tli => fxt}/src/auth/key_manager.rs | 0 {tli => fxt}/src/auth/login.rs | 0 {tli => fxt}/src/auth/mod.rs | 0 {tli => fxt}/src/auth/token_manager.rs | 0 {tli => fxt}/src/client/backtesting_client.rs | 0 {tli => fxt}/src/client/connection_manager.rs | 0 {tli => fxt}/src/client/ml_training_client.rs | 0 {tli => fxt}/src/client/mod.rs | 0 {tli => fxt}/src/client/trading_client.rs | 0 {tli => fxt}/src/commands/agent.rs | 0 {tli => fxt}/src/commands/auth.rs | 0 {tli => fxt}/src/commands/backtest_ml.rs | 0 {tli => fxt}/src/commands/mod.rs | 0 {tli => fxt}/src/commands/trade.rs | 4 +- {tli => fxt}/src/commands/trade_ml.rs | 0 {tli => fxt}/src/commands/train/list.rs | 0 {tli => fxt}/src/commands/train/mod.rs | 0 .../src/commands/train/progress_tracker.rs | 0 {tli => fxt}/src/commands/train/status.rs | 0 {tli => fxt}/src/commands/tune.rs | 0 {tli => fxt}/src/commands/tune_impl.rs | 0 {tli => fxt}/src/commands/tune_stream.rs | 0 {tli => fxt}/src/config.rs | 4 +- {tli => fxt}/src/error.rs | 0 {tli => fxt}/src/lib.rs | 2 +- {tli => fxt}/src/main.rs | 62 +- {tli => fxt}/src/prelude.rs | 0 {tli => fxt}/src/tests.rs | 0 {tli => fxt}/src/types.rs | 0 {tli => fxt}/tests/INTEGRATION_TEST_GUIDE.md | 0 {tli => fxt}/tests/TEST_EXECUTION_README.md | 0 {tli => fxt}/tests/agent_commands_test.rs | 2 +- {tli => fxt}/tests/auth_login_tests.rs | 2 +- {tli => fxt}/tests/cli_integration_test.rs | 0 {tli => fxt}/tests/client_builder_tests.rs | 2 +- .../tests/client_connection_manager_tests.rs | 2 +- .../tests/client_trading_client_tests.rs | 2 +- {tli => fxt}/tests/commands/mod.rs | 0 .../tests/commands/train_list_test.rs | 4 +- {tli => fxt}/tests/debug_file_storage.rs | 2 +- .../tests/e2e/full_user_workflow_test.rs | 2 +- .../tests/e2e/mock_ml_training_service.rs | 2 +- {tli => fxt}/tests/e2e/mod.rs | 0 {tli => fxt}/tests/e2e/test_fixtures.rs | 2 +- {tli => fxt}/tests/e2e/train_list_e2e_test.rs | 2 +- .../tests/e2e/train_start_e2e_test.rs | 2 +- .../tests/e2e/train_status_e2e_test.rs | 2 +- {tli => fxt}/tests/e2e/train_stop_e2e_test.rs | 2 +- .../tests/e2e/train_watch_e2e_test.rs | 2 +- .../tests/encryption_security_audit.rs | 2 +- {tli => fxt}/tests/error_tests.rs | 2 +- {tli => fxt}/tests/file_storage_encryption.rs | 2 +- .../tests/fxt_auth_integration_test.rs | 6 +- .../tests/integration/client_integration.rs | 4 +- .../tests/integration/end_to_end_tests.rs | 6 +- .../tests/integration/error_handling_tests.rs | 6 +- {tli => fxt}/tests/integration/mod.rs | 0 .../tests/integration/performance_tests.rs | 4 +- .../integration/service_integration_tests.rs | 4 +- {tli => fxt}/tests/integration_tests.rs | 0 .../tests/keyring_persistence_tests.rs | 2 +- {tli => fxt}/tests/lib.rs | 0 {tli => fxt}/tests/minimal_keyring_test.rs | 0 .../tests/ml_trading_commands_test.rs | 8 +- {tli => fxt}/tests/mocks/grpc_server.rs | 0 {tli => fxt}/tests/mocks/mod.rs | 0 {tli => fxt}/tests/mod.rs | 0 {tli => fxt}/tests/performance_tests.rs | 0 {tli => fxt}/tests/property_tests.rs | 0 {tli => fxt}/tests/regime_command_tests.rs | 2 +- {tli => fxt}/tests/test_helpers/mod.rs | 0 {tli => fxt}/tests/test_monitoring.rs | 0 {tli => fxt}/tests/tune_integration_test.rs | 0 {tli => fxt}/tests/types_tests.rs | 2 +- {tli => fxt}/tests/unit_tests.rs | 2 +- migrations/tests/run_all_tests.sh | 170 ---- ml/scripts/compare_checkpoint_sizes.sh | 100 --- ml/scripts/profile_memory.sh | 56 -- runpod/QUICK_START.md | 76 -- runpod/README.md | 367 --------- runpod/__init__.py | 53 -- runpod/client.py | 286 ------- runpod/config.py | 265 ------ runpod/errors.py | 90 --- runpod/example_usage.py | 219 ----- runpod/monitor.py | 302 ------- runpod/requirements.txt | 10 - runpod/s3_client.py | 419 ---------- runpod/s3_monitor.py | 178 ----- scripts/EPSILON_FIX3_VALIDATION_README.md | 199 ----- scripts/LOCAL_CI_QUICK_REF.md | 132 --- scripts/README.md | 222 ----- scripts/README_UPLOAD_BINARY.md | 266 ------ scripts/README_scan_gpus.md | 75 -- scripts/README_test_tli_tuning.md | 599 -------------- scripts/README_validate_training.md | 359 --------- scripts/RUNPOD_DEPLOYMENT_SCRIPTS.md | 613 -------------- scripts/SCRIPTS_INVENTORY.md | 197 ----- scripts/VALIDATION_QUICKSTART.md | 217 ----- scripts/analyze_clippy.sh | 53 -- scripts/analyze_dead_code.sh | 50 -- scripts/apply_batch_size_fix.sh | 71 -- scripts/apply_ml_test_fixes.sh | 97 --- scripts/archive/backtest_runpod_225.sh | 180 ----- scripts/archive/check_pod_status.py | 68 -- .../archive/check_runpod_datacenter_field.py | 300 ------- .../cleanup_2025_10_30/activate_venv.sh | 17 - .../cleanup_2025_10_30/auto_fix_safe.sh | 123 --- .../cleanup_2025_10_30/auto_launch_ppo.sh | 69 -- .../auto_monitor_and_launch.sh | 133 --- .../cleanup_2025_10_30/check-warnings.sh | 71 -- .../cleanup_2025_10_30/check_dependencies.sh | 160 ---- .../cleanup_2025_10_30/compare_checkpoints.sh | 71 -- .../cleanup_2025_10_30/dashboard_monitor.sh | 136 ---- .../databento_minimal_download.sh | 183 ----- .../cleanup_2025_10_30/deploy_tuning.sh | 663 --------------- .../cleanup_2025_10_30/download_mbp10.sh | 134 ---- .../e2e_latency_benchmark.sh | 217 ----- .../cleanup_2025_10_30/fix_all_audit_tests.py | 170 ---- .../fix_async_audit_queue_tests.py | 237 ------ .../fix_async_audit_queue_tests_v2.py | 165 ---- .../fix_audit_compliance.py | 67 -- .../cleanup_2025_10_30/fix_mamba2_lr.sh | 176 ---- .../cleanup_2025_10_30/fix_services_unwrap.sh | 65 -- .../fix_services_unwrap2.sh | 59 -- .../fix_services_unwrap3.sh | 61 -- .../fix_services_unwrap4.sh | 55 -- .../fix_services_unwrap5.sh | 28 - .../generate_flame_graphs.sh | 292 ------- .../cleanup_2025_10_30/grpc_load_test.sh | 310 ------- .../grpc_load_test_wave78.sh | 333 -------- .../cleanup_2025_10_30/implement_pgo.sh | 107 --- .../cleanup_2025_10_30/load_test_wave79.sh | 179 ----- .../monitor_all_training.sh | 583 -------------- .../monitor_paper_trading.sh | 227 ------ .../cleanup_2025_10_30/monitor_tuning.sh | 111 --- .../cleanup_2025_10_30/ppo_tuning_prep.sh | 349 -------- .../profile_trading_cycle.sh | 81 -- .../record_baseline_metrics.sh | 294 ------- .../cleanup_2025_10_30/run-coverage-llvm.sh | 37 - .../cleanup_2025_10_30/run_coverage.sh | 58 -- .../sequential_tuning_launcher.sh | 108 --- .../test_alert_resolution.sh | 146 ---- .../cleanup_2025_10_30/test_alerting.sh | 232 ------ .../test_coverage_edge_cases.sh | 422 ---------- .../test_coverage_enforcement.sh | 288 ------- .../cleanup_2025_10_30/test_cuda_fix.sh | 102 --- .../test_direct_training.sh | 84 -- .../cleanup_2025_10_30/test_dqn_training.sh | 73 -- .../test_ensemble_alerts.sh | 283 ------- .../test_full_3month_training.sh | 171 ---- .../test_grafana_dashboard.sh | 88 -- .../test_optimized_dockerfile.sh | 338 -------- .../test_regime_endpoints.sh | 195 ----- .../test_service_integration.sh | 265 ------ .../test_service_startup.sh | 237 ------ .../cleanup_2025_10_30/test_tli_commands.sh | 184 ----- .../cleanup_2025_10_30/test_tli_tuning.sh | 562 ------------- .../cleanup_2025_10_30/test_tuning_small.sh | 81 -- .../test_vault_integration.sh | 276 ------- .../cleanup_2025_10_30/test_wave_d_alerts.sh | 193 ----- .../train_all_models_full.sh | 191 ----- .../cleanup_2025_10_30/upload_checkpoints.sh | 146 ---- scripts/archive/deploy_dqn_staging.sh | 414 ---------- scripts/archive/deploy_fp32_runpod.sh | 245 ------ scripts/archive/deploy_fp32_runpod_test.sh | 352 -------- scripts/archive/deploy_paper_trading.sh | 230 ------ scripts/archive/deploy_runpod.sh | 134 ---- scripts/archive/deploy_runpod_graphql.py | 673 ---------------- scripts/archive/deploy_runpod_training.py | 427 ---------- scripts/archive/fetch_pod_logs_via_web.py | 50 -- scripts/archive/fix_runpod_deployment.py | 255 ------ scripts/archive/get_pod_info.py | 200 ----- scripts/archive/get_runpod_logs.py | 128 --- scripts/archive/monitor_pod.py | 128 --- scripts/archive/monitor_runpod.sh | 152 ---- scripts/archive/runpod_deploy.sh | 573 ------------- scripts/archive/runpod_deploy_test.sh | 216 ----- scripts/archive/runpod_full_deploy.py | 411 ---------- scripts/archive/runpod_upload.sh | 331 -------- scripts/archive/scan_gpus.py | 123 --- scripts/archive/terminate_failing_pod.py | 127 --- scripts/archive/test_binary_sync.sh | 241 ------ scripts/archive/test_binary_validation.sh | 96 --- scripts/archive/test_runpod_auth.py | 303 ------- scripts/archive/test_runpod_pod_creation.py | 488 ----------- scripts/archive/test_ssh_connection.py | 106 --- scripts/archive/train_runpod_225_features.sh | 242 ------ scripts/archive/upload_env_to_runpod.sh | 173 ---- scripts/archive/upload_to_runpod_s3.sh | 532 ------------ scripts/archive/upload_to_runpod_volume.py | 477 ----------- scripts/archive/verify_pod_deployment.py | 191 ----- scripts/archive/verify_runpod_config.sh | 161 ---- .../wave_scripts/WAVE112_QUICKSTART.sh | 114 --- .../WAVE113_AGENT34_VERIFICATION.sh | 63 -- .../WAVE113_CLIPPY_ACTION_PLAN.sh | 259 ------ scripts/backtest_dqn_trials.sh | 154 ---- scripts/backtest_dqn_trials_enhanced.sh | 280 ------- scripts/benchmark_ensemble_db.sh | 390 --------- scripts/benchmark_ensemble_db_quick.sh | 173 ---- scripts/benchmark_mimalloc.sh | 120 --- scripts/benchmark_nextest.sh | 79 -- scripts/build.sh | 9 - scripts/bypass_cache.sh | 29 - scripts/check-status.sh | 42 - scripts/check.sh | 9 - scripts/check_backtesting_health.sh | 38 - scripts/check_service_binaries.sh | 43 - scripts/cleanup_deprecated_scripts.sh | 148 ---- scripts/cleanup_docker_tags.sh | 104 --- scripts/comprehensive_health_check.sh | 157 ---- scripts/concurrent_connection_test.sh | 225 ------ scripts/cross_service_integration_test.sh | 345 -------- scripts/database_validation.sh | 363 --------- scripts/db_load_test.sh | 266 ------ scripts/db_load_test_pgbench.sh | 218 ----- scripts/db_load_test_simple.sh | 140 ---- scripts/db_load_test_v2.sh | 266 ------ scripts/deploy_broker_gateway.sh | 457 ----------- scripts/deploy_debug_image.sh | 99 --- scripts/deploy_fp32_production.sh | 259 ------ scripts/deployment_log.txt | 55 -- scripts/e2e_integration_test.sh | 291 ------- scripts/entrypoint-debug.sh | 80 -- scripts/entrypoint_debug.sh | 277 ------- scripts/export_vault_passwords.sh | 42 - scripts/fix_api_gateway_mfa.sh | 115 --- scripts/fix_audit_compliance_part2.sh | 32 - scripts/fix_mfa_compilation.sh | 55 -- scripts/fix_ml_tests.sh | 40 - scripts/fix_oom_retry_compilation.sh | 29 - scripts/fix_unsafe_blocks.sh | 16 - scripts/fix_wave112_compilation.sh | 78 -- scripts/grpc_integration_test.sh | 307 ------- scripts/grpc_load_test_quickstart.md | 89 --- scripts/health-check.sh | 4 +- scripts/health_check.sh | 473 ----------- scripts/install_cron.sh | 139 ---- scripts/local_ci_pipeline.sh | 612 -------------- scripts/mark_tests_ignored.sh | 59 -- scripts/measure_vram.sh | 191 ----- scripts/monitor_quick_reference.txt | 90 --- scripts/offline_service_validation.sh | 212 ----- scripts/optimize_batch_sizes.sh | 101 --- scripts/plan_databento_download.sh | 411 ---------- scripts/prefix_unused_vars.sh | 52 -- scripts/production-security-hardening.sh | 449 ----------- .../analysis/analyze_backtest_results.py | 334 -------- .../analysis/analyze_clippy_warnings.py | 145 ---- scripts/python/analysis/analyze_failures.py | 32 - scripts/python/analysis/analyze_gc_data.py | 95 --- .../python/analysis/analyze_price_action.py | 237 ------ .../analysis/generate_backtest_summary.py | 193 ----- scripts/python/analysis/parse_test_results.py | 35 - .../python/analysis/parse_wave_141_results.py | 128 --- scripts/python/analyze_features.py | 222 ----- .../benchmarks/benchmark_training_time.py | 378 --------- .../benchmarks/extract_dqn_hyperparameters.py | 755 ------------------ .../python/benchmarks/extract_dqn_results.py | 177 ---- scripts/python/data/concat_zn_existing.py | 91 --- scripts/python/data/convert_nq_to_parquet.py | 88 -- scripts/python/data/download_6e_fut.py | 155 ---- scripts/python/data/download_6e_fut_180d.py | 192 ----- .../data/download_es_90d_multi_contract.py | 262 ------ scripts/python/data/download_es_90d_unseen.py | 180 ----- scripts/python/data/download_es_databento.py | 166 ---- .../python/data/download_es_databento_v2.py | 216 ----- scripts/python/data/download_gc_timeseries.py | 136 ---- .../python/data/download_ml_training_data.py | 366 --------- scripts/python/data/download_nq_fut_180d.py | 167 ---- scripts/python/data/download_zn_fut_180d.py | 164 ---- .../python/data/download_zn_fut_180d_v2.py | 237 ------ scripts/python/data/final_6e_download.py | 136 ---- scripts/python/data/find_6e_contracts.py | 88 -- scripts/python/data/inspect_nq_parquet.py | 69 -- scripts/python/data/list_datasets.py | 64 -- scripts/python/data/list_glbx_instruments.py | 98 --- scripts/python/data/search_euro_symbols.py | 87 -- scripts/python/data/validate_es_multiday.py | 219 ----- scripts/python/data/verify_6e_data.py | 172 ---- .../testing/concurrent_connection_test.py | 228 ------ scripts/python/testing/db_load_test.py | 342 -------- scripts/python/testing/load_test.py | 451 ----------- scripts/python/testing/sustained_load_test.py | 483 ----------- .../python/utils/check_registry_creds_v2.py | 91 --- .../python/utils/check_registry_creds_v3.py | 87 -- scripts/python/utils/check_subscription.py | 92 --- scripts/quarterly_retrain.sh | 379 --------- scripts/quick_health_check.sh | 134 ---- scripts/quick_status.sh | 86 -- scripts/requirements.txt | 88 -- scripts/run-coverage.sh | 109 --- scripts/run_comprehensive_tests.sh | 130 --- scripts/run_cross_validation.sh | 111 --- scripts/run_final_benchmarks.sh | 39 - scripts/run_ghz_load_test.sh | 152 ---- scripts/run_liquid_nn_tuning.sh | 671 ---------------- scripts/run_load_tests.sh | 235 ------ scripts/run_performance_benchmarks.sh | 73 -- scripts/run_ppo_comprehensive_tuning.sh | 466 ----------- scripts/run_smoke_tests.sh | 218 ----- scripts/run_tests.sh | 37 - scripts/run_tft_training.sh | 69 -- scripts/run_training.sh | 156 ---- scripts/run_wave26_p1_12_tests.sh | 35 - scripts/security-hardening.sh | 421 ---------- scripts/setup_lld.sh | 133 --- scripts/setup_production_passwords.sh | 399 --------- scripts/simple_concurrent_test.sh | 128 --- scripts/smoke_test.sh | 323 -------- scripts/staging_e2e_tests.sh | 271 ------- scripts/{start-tli.sh => start-fxt.sh} | 4 +- scripts/start.sh | 8 +- scripts/start_all_services.sh | 93 --- scripts/start_backtesting.sh | 19 - scripts/start_foxhunt.sh | 133 --- scripts/start_services.sh | 205 ----- scripts/stop_foxhunt.sh | 85 -- scripts/stop_services.sh | 42 - scripts/stub_ignored_tests.sh | 54 -- scripts/sustained_load_grpc_test.sh | 352 -------- scripts/system_resource_monitor.sh | 554 ------------- scripts/test_alerts.sh | 92 --- scripts/test_circuit_breakers.sh | 135 ---- scripts/test_crash_logging.sh | 100 --- scripts/test_dbn_loader_fix.sh | 19 - scripts/test_dqn_checkpoints_quick.sh | 133 --- scripts/test_dropout_scheduler.sh | 117 --- scripts/test_graceful_degradation.sh | 360 --------- scripts/test_grpc_proxies.sh | 313 -------- scripts/test_liquid_nn_readiness.sh | 87 -- scripts/test_market_data.sh | 16 - scripts/test_ppo_checkpoint.sh | 22 - scripts/testing/test_dqn_replay_pipeline.sh | 438 ---------- scripts/train_all_models_fixed.sh | 211 ----- scripts/train_dqn_production.sh | 230 ------ scripts/train_launcher.sh | 226 ------ scripts/validate-docker-config.sh | 224 ------ scripts/validate-monitoring-performance.sh | 476 ----------- scripts/validate_ab_testing_tdd.sh | 61 -- scripts/validate_agent_9_13.sh | 113 --- scripts/validate_auth_enabled.sh | 138 ---- scripts/validate_binary.sh | 121 --- scripts/validate_clippy.sh | 240 ------ scripts/validate_dqn_performance.sh | 347 -------- scripts/validate_epsilon_fix3.sh | 209 ----- scripts/validate_gitlab_cicd.sh | 387 --------- scripts/validate_grpc_endpoints.sh | 97 --- scripts/validate_h5_alerting.sh | 217 ----- scripts/validate_jwt_config.sh | 182 ----- scripts/validate_ml_monitoring_metrics.sh | 207 ----- scripts/validate_ppo_adapter.sh | 100 --- scripts/validate_ppo_fix.sh | 137 ---- scripts/validate_tls_setup.sh | 338 -------- scripts/validate_train_script.sh | 108 --- scripts/validate_training.sh | 268 ------- scripts/verify_ci_setup.sh | 165 ---- scripts/verify_dataset_coverage.sh | 120 --- scripts/verify_db_optimization.sh | 72 -- scripts/verify_dbn_fix.sh | 92 --- scripts/verify_dbn_loader_fix.sh | 134 ---- scripts/verify_dockerfile_updates.sh | 124 --- scripts/verify_documentation_structure.sh | 62 -- scripts/verify_ml_dependencies.sh | 64 -- scripts/verify_tft_checkpoint_fix.sh | 65 -- scripts/verify_tft_cuda_fix.sh | 117 --- scripts/verify_tft_cuda_setup.sh | 76 -- scripts/verify_vault_setup.sh | 26 - scripts/watch_gpu_availability.sh | 60 -- services/api_gateway/Cargo.toml | 2 +- services/api_gateway/Dockerfile | 2 +- services/api_gateway/build.rs | 6 +- .../api_gateway/tests/grpc_error_handling.rs | 2 +- .../tests/real_backend_integration_test.rs | 14 +- services/backtesting_service/Cargo.toml | 2 +- services/backtesting_service/Dockerfile | 2 +- services/backtesting_service/build.rs | 2 +- .../tests/grpc_error_handling.rs | 2 +- .../Dockerfile.production | 2 +- services/integration_tests/build.rs | 18 +- services/ml_training_service/Dockerfile | 2 +- services/ml_training_service/build.rs | 6 +- .../hyperparameter_tuner.py | 664 --------------- .../hyperparameter_tuner_ppo_enhanced.py | 603 -------------- .../run_hyperparameter_optimization.py | 520 ------------ .../scripts/example_tuning_job.sh | 92 --- .../scripts/generate_python_proto.sh | 37 - .../tests/docker/test_setup.sh | 52 -- .../tests/docker/test_teardown.sh | 18 - .../tests/test_hyperparameter_tuner.py | 393 --------- .../ml_training_service/validate_test_data.py | 115 --- .../validate_test_data_simple.sh | 59 -- services/trading_agent_service/Dockerfile | 2 +- .../scripts/verify_tls.sh | 192 ----- services/trading_service/Dockerfile | 2 +- terraform/runpod/.gitignore | 43 - .../runpod/CREDENTIAL_MANAGEMENT_QUICK_REF.md | 183 ----- terraform/runpod/DEPLOYMENT_SUMMARY.md | 542 ------------- terraform/runpod/QUICK_START.md | 124 --- terraform/runpod/README.md | 631 --------------- terraform/runpod/deploy.sh | 207 ----- terraform/runpod/generate_credentials.sh | 137 ---- terraform/runpod/main.tf | 134 ---- terraform/runpod/outputs.tf | 171 ---- terraform/runpod/terraform.tfvars.example | 241 ------ terraform/runpod/validate.sh | 239 ------ terraform/runpod/validate_credentials.sh | 187 ----- terraform/runpod/variables.tf | 206 ----- test_data/tuning_config.yaml | 2 +- tests/Cargo.toml | 2 +- tests/e2e/build.rs | 2 +- tests/e2e/integration/auth_flow_test.sh | 273 ------- tests/e2e/integration/e2e_test_suite.sh | 225 ------ tests/e2e/integration/hot_reload_test.sh | 304 ------- tests/e2e/integration/trading_flow_test.sh | 344 -------- tests/e2e/vault_e2e_test.sh | 255 ------ tests/e2e/vault_e2e_test_curl.sh | 268 ------- .../fixtures/vault-setup/setup-vault.sh | 185 ----- .../infrastructure_health_check.sh | 211 ----- tests/fixtures/mod.rs | 2 +- tests/load_tests/ghz_authenticated.sh | 266 ------ tests/load_tests/ghz_authenticated_fixed.sh | 266 ------ tests/load_tests/ghz_quick_auth_test.sh | 81 -- tests/load_tests/ghz_quick_test.sh | 55 -- tests/mocks/mod.rs | 2 +- tests/paper_trading_smoke_test.sh | 263 ------ tests/runpod/__init__.py | 1 - tests/runpod/conftest.py | 281 ------- tests/runpod/test_client.py | 389 --------- tests/runpod/test_config.py | 198 ----- tests/runpod/test_monitor.py | 344 -------- tests/runpod/test_s3_client.py | 439 ---------- tli/ci_cd.sh | 489 ------------ trading_engine/tests/stub_tests.sh | 28 - web-gateway/build.rs | 26 +- 504 files changed, 257 insertions(+), 74355 deletions(-) delete mode 100644 .gitlab-ci.yml delete mode 100644 foxhunt-deploy/Cargo.toml delete mode 100644 foxhunt-deploy/DEPLOYMENT_QUICK_START.md delete mode 100644 foxhunt-deploy/DOCKER_BUILD_MILESTONE2.md delete mode 100644 foxhunt-deploy/MILESTONE_1_COMPLETE.md delete mode 100644 foxhunt-deploy/MILESTONE_3_SUMMARY.md delete mode 100644 foxhunt-deploy/MILESTONE_4_SUMMARY.md delete mode 100644 foxhunt-deploy/MONITOR_QUICK_REF.md delete mode 100644 foxhunt-deploy/QUICK_START_BUILD.md delete mode 100644 foxhunt-deploy/README.md delete mode 100644 foxhunt-deploy/RUNPOD_DEPLOYMENT_IMPLEMENTATION.md delete mode 100644 foxhunt-deploy/S3_MONITOR_IMPLEMENTATION.md delete mode 100755 foxhunt-deploy/examples/monitor_demo.sh delete mode 100644 foxhunt-deploy/examples/sample_config.toml delete mode 100644 foxhunt-deploy/src/cli/build.rs delete mode 100644 foxhunt-deploy/src/cli/deploy.rs delete mode 100644 foxhunt-deploy/src/cli/mod.rs delete mode 100644 foxhunt-deploy/src/cli/monitor.rs delete mode 100644 foxhunt-deploy/src/cli/run.rs delete mode 100644 foxhunt-deploy/src/config/mod.rs delete mode 100644 foxhunt-deploy/src/config/types.rs delete mode 100644 foxhunt-deploy/src/docker/build.rs delete mode 100644 foxhunt-deploy/src/docker/mod.rs delete mode 100644 foxhunt-deploy/src/docker/push.rs delete mode 100644 foxhunt-deploy/src/error.rs delete mode 100644 foxhunt-deploy/src/main.rs delete mode 100644 foxhunt-deploy/src/runpod/client.rs delete mode 100644 foxhunt-deploy/src/runpod/deployment.rs delete mode 100644 foxhunt-deploy/src/runpod/mod.rs delete mode 100644 foxhunt-deploy/src/runpod/types.rs delete mode 100644 foxhunt-deploy/src/s3/mod.rs delete mode 100644 foxhunt-deploy/src/s3/monitor.rs delete mode 100644 foxhunt-deploy/src/s3/parser.rs delete mode 100644 foxhunt-deploy/src/utils/mod.rs delete mode 100644 foxhunt-deploy/src/utils/terminal.rs delete mode 100644 foxhunt-deploy/tests/docker_integration_tests.rs rename {tli => fxt}/CONFIG_FILE_SUPPORT.md (100%) rename {tli => fxt}/Cargo.toml (97%) rename {tli => fxt}/TUNE_COMMAND_README.md (100%) rename {tli => fxt}/benches/client_performance.rs (98%) rename {tli => fxt}/benches/configuration_benchmarks.rs (99%) rename {tli => fxt}/benches/encryption_performance.rs (96%) rename {tli => fxt}/benches/serialization_benchmarks.rs (99%) rename {tli => fxt}/build.rs (100%) rename {tli => fxt}/config.toml.example (100%) rename {tli => fxt}/docs/AUTHENTICATION.md (100%) rename {tli => fxt}/docs/ML_TRADING_COMMANDS.md (100%) rename {tli => fxt}/docs/USAGE.md (100%) rename {tli => fxt}/examples/complete_client_example.rs (99%) rename {tli => fxt}/examples/config_demo.rs (100%) rename {tli => fxt}/examples/config_management_placeholder.rs (100%) rename {tli => fxt}/examples/security_example.rs (100%) rename {tli => fxt}/proptest-regressions/tests.txt (100%) rename {tli => fxt}/proto/config.proto (100%) rename {tli => fxt}/proto/health.proto (100%) rename {tli => fxt}/proto/ml.proto (100%) rename {tli => fxt}/proto/ml_training.proto (100%) rename {tli => fxt}/proto/trading.proto (100%) rename {tli => fxt}/proto/trading_agent.proto (100%) rename {tli => fxt}/src/auth/encryption.rs (99%) rename {tli => fxt}/src/auth/interceptor.rs (100%) rename {tli => fxt}/src/auth/jwt_generator.rs (100%) rename {tli => fxt}/src/auth/key_manager.rs (100%) rename {tli => fxt}/src/auth/login.rs (100%) rename {tli => fxt}/src/auth/mod.rs (100%) rename {tli => fxt}/src/auth/token_manager.rs (100%) rename {tli => fxt}/src/client/backtesting_client.rs (100%) rename {tli => fxt}/src/client/connection_manager.rs (100%) rename {tli => fxt}/src/client/ml_training_client.rs (100%) rename {tli => fxt}/src/client/mod.rs (100%) rename {tli => fxt}/src/client/trading_client.rs (100%) rename {tli => fxt}/src/commands/agent.rs (100%) rename {tli => fxt}/src/commands/auth.rs (100%) rename {tli => fxt}/src/commands/backtest_ml.rs (100%) rename {tli => fxt}/src/commands/mod.rs (100%) rename {tli => fxt}/src/commands/trade.rs (97%) rename {tli => fxt}/src/commands/trade_ml.rs (100%) rename {tli => fxt}/src/commands/train/list.rs (100%) rename {tli => fxt}/src/commands/train/mod.rs (100%) rename {tli => fxt}/src/commands/train/progress_tracker.rs (100%) rename {tli => fxt}/src/commands/train/status.rs (100%) rename {tli => fxt}/src/commands/tune.rs (100%) rename {tli => fxt}/src/commands/tune_impl.rs (100%) rename {tli => fxt}/src/commands/tune_stream.rs (100%) rename {tli => fxt}/src/config.rs (98%) rename {tli => fxt}/src/error.rs (100%) rename {tli => fxt}/src/lib.rs (99%) rename {tli => fxt}/src/main.rs (91%) rename {tli => fxt}/src/prelude.rs (100%) rename {tli => fxt}/src/tests.rs (100%) rename {tli => fxt}/src/types.rs (100%) rename {tli => fxt}/tests/INTEGRATION_TEST_GUIDE.md (100%) rename {tli => fxt}/tests/TEST_EXECUTION_README.md (100%) rename {tli => fxt}/tests/agent_commands_test.rs (99%) rename {tli => fxt}/tests/auth_login_tests.rs (99%) rename {tli => fxt}/tests/cli_integration_test.rs (100%) rename {tli => fxt}/tests/client_builder_tests.rs (99%) rename {tli => fxt}/tests/client_connection_manager_tests.rs (99%) rename {tli => fxt}/tests/client_trading_client_tests.rs (99%) rename {tli => fxt}/tests/commands/mod.rs (100%) rename {tli => fxt}/tests/commands/train_list_test.rs (99%) rename {tli => fxt}/tests/debug_file_storage.rs (97%) rename {tli => fxt}/tests/e2e/full_user_workflow_test.rs (99%) rename {tli => fxt}/tests/e2e/mock_ml_training_service.rs (99%) rename {tli => fxt}/tests/e2e/mod.rs (100%) rename {tli => fxt}/tests/e2e/test_fixtures.rs (98%) rename {tli => fxt}/tests/e2e/train_list_e2e_test.rs (99%) rename {tli => fxt}/tests/e2e/train_start_e2e_test.rs (99%) rename {tli => fxt}/tests/e2e/train_status_e2e_test.rs (99%) rename {tli => fxt}/tests/e2e/train_stop_e2e_test.rs (99%) rename {tli => fxt}/tests/e2e/train_watch_e2e_test.rs (99%) rename {tli => fxt}/tests/encryption_security_audit.rs (97%) rename {tli => fxt}/tests/error_tests.rs (99%) rename {tli => fxt}/tests/file_storage_encryption.rs (99%) rename tli/tests/tli_auth_integration_test.rs => fxt/tests/fxt_auth_integration_test.rs (99%) rename {tli => fxt}/tests/integration/client_integration.rs (99%) rename {tli => fxt}/tests/integration/end_to_end_tests.rs (99%) rename {tli => fxt}/tests/integration/error_handling_tests.rs (99%) rename {tli => fxt}/tests/integration/mod.rs (100%) rename {tli => fxt}/tests/integration/performance_tests.rs (99%) rename {tli => fxt}/tests/integration/service_integration_tests.rs (99%) rename {tli => fxt}/tests/integration_tests.rs (100%) rename {tli => fxt}/tests/keyring_persistence_tests.rs (99%) rename {tli => fxt}/tests/lib.rs (100%) rename {tli => fxt}/tests/minimal_keyring_test.rs (100%) rename {tli => fxt}/tests/ml_trading_commands_test.rs (98%) rename {tli => fxt}/tests/mocks/grpc_server.rs (100%) rename {tli => fxt}/tests/mocks/mod.rs (100%) rename {tli => fxt}/tests/mod.rs (100%) rename {tli => fxt}/tests/performance_tests.rs (100%) rename {tli => fxt}/tests/property_tests.rs (100%) rename {tli => fxt}/tests/regime_command_tests.rs (99%) rename {tli => fxt}/tests/test_helpers/mod.rs (100%) rename {tli => fxt}/tests/test_monitoring.rs (100%) rename {tli => fxt}/tests/tune_integration_test.rs (100%) rename {tli => fxt}/tests/types_tests.rs (99%) rename {tli => fxt}/tests/unit_tests.rs (92%) delete mode 100755 migrations/tests/run_all_tests.sh delete mode 100755 ml/scripts/compare_checkpoint_sizes.sh delete mode 100755 ml/scripts/profile_memory.sh delete mode 100644 runpod/QUICK_START.md delete mode 100644 runpod/README.md delete mode 100644 runpod/__init__.py delete mode 100644 runpod/client.py delete mode 100644 runpod/config.py delete mode 100644 runpod/errors.py delete mode 100755 runpod/example_usage.py delete mode 100644 runpod/monitor.py delete mode 100644 runpod/requirements.txt delete mode 100644 runpod/s3_client.py delete mode 100644 runpod/s3_monitor.py delete mode 100644 scripts/EPSILON_FIX3_VALIDATION_README.md delete mode 100644 scripts/LOCAL_CI_QUICK_REF.md delete mode 100644 scripts/README.md delete mode 100644 scripts/README_UPLOAD_BINARY.md delete mode 100644 scripts/README_scan_gpus.md delete mode 100644 scripts/README_test_tli_tuning.md delete mode 100644 scripts/README_validate_training.md delete mode 100644 scripts/RUNPOD_DEPLOYMENT_SCRIPTS.md delete mode 100644 scripts/SCRIPTS_INVENTORY.md delete mode 100644 scripts/VALIDATION_QUICKSTART.md delete mode 100755 scripts/analyze_clippy.sh delete mode 100755 scripts/analyze_dead_code.sh delete mode 100755 scripts/apply_batch_size_fix.sh delete mode 100755 scripts/apply_ml_test_fixes.sh delete mode 100755 scripts/archive/backtest_runpod_225.sh delete mode 100755 scripts/archive/check_pod_status.py delete mode 100755 scripts/archive/check_runpod_datacenter_field.py delete mode 100755 scripts/archive/cleanup_2025_10_30/activate_venv.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/auto_fix_safe.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/auto_launch_ppo.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/auto_monitor_and_launch.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/check-warnings.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/check_dependencies.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/compare_checkpoints.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/dashboard_monitor.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/databento_minimal_download.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/deploy_tuning.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/download_mbp10.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/e2e_latency_benchmark.sh delete mode 100644 scripts/archive/cleanup_2025_10_30/fix_all_audit_tests.py delete mode 100755 scripts/archive/cleanup_2025_10_30/fix_async_audit_queue_tests.py delete mode 100755 scripts/archive/cleanup_2025_10_30/fix_async_audit_queue_tests_v2.py delete mode 100644 scripts/archive/cleanup_2025_10_30/fix_audit_compliance.py delete mode 100644 scripts/archive/cleanup_2025_10_30/fix_mamba2_lr.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/fix_services_unwrap.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/fix_services_unwrap2.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/fix_services_unwrap3.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/fix_services_unwrap4.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/fix_services_unwrap5.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/generate_flame_graphs.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/grpc_load_test.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/grpc_load_test_wave78.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/implement_pgo.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/load_test_wave79.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/monitor_all_training.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/monitor_paper_trading.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/monitor_tuning.sh delete mode 100644 scripts/archive/cleanup_2025_10_30/ppo_tuning_prep.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/profile_trading_cycle.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/record_baseline_metrics.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/run-coverage-llvm.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/run_coverage.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/sequential_tuning_launcher.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_alert_resolution.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_alerting.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_coverage_edge_cases.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_coverage_enforcement.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_cuda_fix.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_direct_training.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_dqn_training.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_ensemble_alerts.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_full_3month_training.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_grafana_dashboard.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_optimized_dockerfile.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_regime_endpoints.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_service_integration.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_service_startup.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_tli_commands.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_tli_tuning.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_tuning_small.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_vault_integration.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/test_wave_d_alerts.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/train_all_models_full.sh delete mode 100755 scripts/archive/cleanup_2025_10_30/upload_checkpoints.sh delete mode 100755 scripts/archive/deploy_dqn_staging.sh delete mode 100755 scripts/archive/deploy_fp32_runpod.sh delete mode 100755 scripts/archive/deploy_fp32_runpod_test.sh delete mode 100755 scripts/archive/deploy_paper_trading.sh delete mode 100755 scripts/archive/deploy_runpod.sh delete mode 100755 scripts/archive/deploy_runpod_graphql.py delete mode 100755 scripts/archive/deploy_runpod_training.py delete mode 100755 scripts/archive/fetch_pod_logs_via_web.py delete mode 100755 scripts/archive/fix_runpod_deployment.py delete mode 100755 scripts/archive/get_pod_info.py delete mode 100755 scripts/archive/get_runpod_logs.py delete mode 100755 scripts/archive/monitor_pod.py delete mode 100755 scripts/archive/monitor_runpod.sh delete mode 100755 scripts/archive/runpod_deploy.sh delete mode 100755 scripts/archive/runpod_deploy_test.sh delete mode 100755 scripts/archive/runpod_full_deploy.py delete mode 100755 scripts/archive/runpod_upload.sh delete mode 100755 scripts/archive/scan_gpus.py delete mode 100755 scripts/archive/terminate_failing_pod.py delete mode 100755 scripts/archive/test_binary_sync.sh delete mode 100755 scripts/archive/test_binary_validation.sh delete mode 100755 scripts/archive/test_runpod_auth.py delete mode 100755 scripts/archive/test_runpod_pod_creation.py delete mode 100644 scripts/archive/test_ssh_connection.py delete mode 100755 scripts/archive/train_runpod_225_features.sh delete mode 100755 scripts/archive/upload_env_to_runpod.sh delete mode 100755 scripts/archive/upload_to_runpod_s3.sh delete mode 100755 scripts/archive/upload_to_runpod_volume.py delete mode 100755 scripts/archive/verify_pod_deployment.py delete mode 100755 scripts/archive/verify_runpod_config.sh delete mode 100755 scripts/archive/wave_scripts/WAVE112_QUICKSTART.sh delete mode 100755 scripts/archive/wave_scripts/WAVE113_AGENT34_VERIFICATION.sh delete mode 100755 scripts/archive/wave_scripts/WAVE113_CLIPPY_ACTION_PLAN.sh delete mode 100755 scripts/backtest_dqn_trials.sh delete mode 100755 scripts/backtest_dqn_trials_enhanced.sh delete mode 100755 scripts/benchmark_ensemble_db.sh delete mode 100755 scripts/benchmark_ensemble_db_quick.sh delete mode 100755 scripts/benchmark_mimalloc.sh delete mode 100755 scripts/benchmark_nextest.sh delete mode 100755 scripts/build.sh delete mode 100755 scripts/bypass_cache.sh delete mode 100755 scripts/check-status.sh delete mode 100755 scripts/check.sh delete mode 100755 scripts/check_backtesting_health.sh delete mode 100755 scripts/check_service_binaries.sh delete mode 100755 scripts/cleanup_deprecated_scripts.sh delete mode 100755 scripts/cleanup_docker_tags.sh delete mode 100755 scripts/comprehensive_health_check.sh delete mode 100755 scripts/concurrent_connection_test.sh delete mode 100755 scripts/cross_service_integration_test.sh delete mode 100755 scripts/database_validation.sh delete mode 100755 scripts/db_load_test.sh delete mode 100755 scripts/db_load_test_pgbench.sh delete mode 100755 scripts/db_load_test_simple.sh delete mode 100755 scripts/db_load_test_v2.sh delete mode 100755 scripts/deploy_broker_gateway.sh delete mode 100755 scripts/deploy_debug_image.sh delete mode 100755 scripts/deploy_fp32_production.sh delete mode 100644 scripts/deployment_log.txt delete mode 100755 scripts/e2e_integration_test.sh delete mode 100755 scripts/entrypoint-debug.sh delete mode 100755 scripts/entrypoint_debug.sh delete mode 100755 scripts/export_vault_passwords.sh delete mode 100644 scripts/fix_api_gateway_mfa.sh delete mode 100644 scripts/fix_audit_compliance_part2.sh delete mode 100755 scripts/fix_mfa_compilation.sh delete mode 100644 scripts/fix_ml_tests.sh delete mode 100755 scripts/fix_oom_retry_compilation.sh delete mode 100644 scripts/fix_unsafe_blocks.sh delete mode 100755 scripts/fix_wave112_compilation.sh delete mode 100755 scripts/grpc_integration_test.sh delete mode 100644 scripts/grpc_load_test_quickstart.md delete mode 100755 scripts/health_check.sh delete mode 100755 scripts/install_cron.sh delete mode 100755 scripts/local_ci_pipeline.sh delete mode 100644 scripts/mark_tests_ignored.sh delete mode 100755 scripts/measure_vram.sh delete mode 100644 scripts/monitor_quick_reference.txt delete mode 100755 scripts/offline_service_validation.sh delete mode 100755 scripts/optimize_batch_sizes.sh delete mode 100755 scripts/plan_databento_download.sh delete mode 100755 scripts/prefix_unused_vars.sh delete mode 100755 scripts/production-security-hardening.sh delete mode 100644 scripts/python/analysis/analyze_backtest_results.py delete mode 100644 scripts/python/analysis/analyze_clippy_warnings.py delete mode 100644 scripts/python/analysis/analyze_failures.py delete mode 100644 scripts/python/analysis/analyze_gc_data.py delete mode 100644 scripts/python/analysis/analyze_price_action.py delete mode 100644 scripts/python/analysis/generate_backtest_summary.py delete mode 100644 scripts/python/analysis/parse_test_results.py delete mode 100755 scripts/python/analysis/parse_wave_141_results.py delete mode 100755 scripts/python/analyze_features.py delete mode 100755 scripts/python/benchmarks/benchmark_training_time.py delete mode 100644 scripts/python/benchmarks/extract_dqn_hyperparameters.py delete mode 100644 scripts/python/benchmarks/extract_dqn_results.py delete mode 100755 scripts/python/data/concat_zn_existing.py delete mode 100755 scripts/python/data/convert_nq_to_parquet.py delete mode 100644 scripts/python/data/download_6e_fut.py delete mode 100755 scripts/python/data/download_6e_fut_180d.py delete mode 100644 scripts/python/data/download_es_90d_multi_contract.py delete mode 100644 scripts/python/data/download_es_90d_unseen.py delete mode 100644 scripts/python/data/download_es_databento.py delete mode 100644 scripts/python/data/download_es_databento_v2.py delete mode 100644 scripts/python/data/download_gc_timeseries.py delete mode 100755 scripts/python/data/download_ml_training_data.py delete mode 100644 scripts/python/data/download_nq_fut_180d.py delete mode 100755 scripts/python/data/download_zn_fut_180d.py delete mode 100755 scripts/python/data/download_zn_fut_180d_v2.py delete mode 100644 scripts/python/data/final_6e_download.py delete mode 100644 scripts/python/data/find_6e_contracts.py delete mode 100644 scripts/python/data/inspect_nq_parquet.py delete mode 100644 scripts/python/data/list_datasets.py delete mode 100644 scripts/python/data/list_glbx_instruments.py delete mode 100644 scripts/python/data/search_euro_symbols.py delete mode 100644 scripts/python/data/validate_es_multiday.py delete mode 100644 scripts/python/data/verify_6e_data.py delete mode 100755 scripts/python/testing/concurrent_connection_test.py delete mode 100755 scripts/python/testing/db_load_test.py delete mode 100755 scripts/python/testing/load_test.py delete mode 100755 scripts/python/testing/sustained_load_test.py delete mode 100755 scripts/python/utils/check_registry_creds_v2.py delete mode 100755 scripts/python/utils/check_registry_creds_v3.py delete mode 100644 scripts/python/utils/check_subscription.py delete mode 100755 scripts/quarterly_retrain.sh delete mode 100755 scripts/quick_health_check.sh delete mode 100755 scripts/quick_status.sh delete mode 100644 scripts/requirements.txt delete mode 100755 scripts/run-coverage.sh delete mode 100755 scripts/run_comprehensive_tests.sh delete mode 100644 scripts/run_cross_validation.sh delete mode 100755 scripts/run_final_benchmarks.sh delete mode 100755 scripts/run_ghz_load_test.sh delete mode 100755 scripts/run_liquid_nn_tuning.sh delete mode 100755 scripts/run_load_tests.sh delete mode 100755 scripts/run_performance_benchmarks.sh delete mode 100755 scripts/run_ppo_comprehensive_tuning.sh delete mode 100755 scripts/run_smoke_tests.sh delete mode 100755 scripts/run_tests.sh delete mode 100755 scripts/run_tft_training.sh delete mode 100755 scripts/run_training.sh delete mode 100755 scripts/run_wave26_p1_12_tests.sh delete mode 100755 scripts/security-hardening.sh delete mode 100755 scripts/setup_lld.sh delete mode 100755 scripts/setup_production_passwords.sh delete mode 100755 scripts/simple_concurrent_test.sh delete mode 100755 scripts/smoke_test.sh delete mode 100755 scripts/staging_e2e_tests.sh rename scripts/{start-tli.sh => start-fxt.sh} (97%) delete mode 100755 scripts/start_all_services.sh delete mode 100755 scripts/start_backtesting.sh delete mode 100755 scripts/start_foxhunt.sh delete mode 100755 scripts/start_services.sh delete mode 100755 scripts/stop_foxhunt.sh delete mode 100755 scripts/stop_services.sh delete mode 100644 scripts/stub_ignored_tests.sh delete mode 100644 scripts/sustained_load_grpc_test.sh delete mode 100755 scripts/system_resource_monitor.sh delete mode 100755 scripts/test_alerts.sh delete mode 100755 scripts/test_circuit_breakers.sh delete mode 100755 scripts/test_crash_logging.sh delete mode 100755 scripts/test_dbn_loader_fix.sh delete mode 100755 scripts/test_dqn_checkpoints_quick.sh delete mode 100755 scripts/test_dropout_scheduler.sh delete mode 100755 scripts/test_graceful_degradation.sh delete mode 100755 scripts/test_grpc_proxies.sh delete mode 100755 scripts/test_liquid_nn_readiness.sh delete mode 100755 scripts/test_market_data.sh delete mode 100755 scripts/test_ppo_checkpoint.sh delete mode 100755 scripts/testing/test_dqn_replay_pipeline.sh delete mode 100755 scripts/train_all_models_fixed.sh delete mode 100755 scripts/train_dqn_production.sh delete mode 100755 scripts/train_launcher.sh delete mode 100755 scripts/validate-docker-config.sh delete mode 100755 scripts/validate-monitoring-performance.sh delete mode 100755 scripts/validate_ab_testing_tdd.sh delete mode 100755 scripts/validate_agent_9_13.sh delete mode 100755 scripts/validate_auth_enabled.sh delete mode 100755 scripts/validate_binary.sh delete mode 100755 scripts/validate_clippy.sh delete mode 100755 scripts/validate_dqn_performance.sh delete mode 100755 scripts/validate_epsilon_fix3.sh delete mode 100755 scripts/validate_gitlab_cicd.sh delete mode 100755 scripts/validate_grpc_endpoints.sh delete mode 100755 scripts/validate_h5_alerting.sh delete mode 100755 scripts/validate_jwt_config.sh delete mode 100755 scripts/validate_ml_monitoring_metrics.sh delete mode 100755 scripts/validate_ppo_adapter.sh delete mode 100755 scripts/validate_ppo_fix.sh delete mode 100755 scripts/validate_tls_setup.sh delete mode 100755 scripts/validate_train_script.sh delete mode 100755 scripts/validate_training.sh delete mode 100755 scripts/verify_ci_setup.sh delete mode 100755 scripts/verify_dataset_coverage.sh delete mode 100755 scripts/verify_db_optimization.sh delete mode 100755 scripts/verify_dbn_fix.sh delete mode 100755 scripts/verify_dbn_loader_fix.sh delete mode 100755 scripts/verify_dockerfile_updates.sh delete mode 100755 scripts/verify_documentation_structure.sh delete mode 100755 scripts/verify_ml_dependencies.sh delete mode 100755 scripts/verify_tft_checkpoint_fix.sh delete mode 100755 scripts/verify_tft_cuda_fix.sh delete mode 100755 scripts/verify_tft_cuda_setup.sh delete mode 100755 scripts/verify_vault_setup.sh delete mode 100755 scripts/watch_gpu_availability.sh delete mode 100644 services/ml_training_service/hyperparameter_tuner.py delete mode 100755 services/ml_training_service/hyperparameter_tuner_ppo_enhanced.py delete mode 100755 services/ml_training_service/run_hyperparameter_optimization.py delete mode 100755 services/ml_training_service/scripts/example_tuning_job.sh delete mode 100755 services/ml_training_service/scripts/generate_python_proto.sh delete mode 100755 services/ml_training_service/tests/docker/test_setup.sh delete mode 100755 services/ml_training_service/tests/docker/test_teardown.sh delete mode 100644 services/ml_training_service/tests/test_hyperparameter_tuner.py delete mode 100755 services/ml_training_service/validate_test_data.py delete mode 100755 services/ml_training_service/validate_test_data_simple.sh delete mode 100755 services/trading_agent_service/scripts/verify_tls.sh delete mode 100644 terraform/runpod/.gitignore delete mode 100644 terraform/runpod/CREDENTIAL_MANAGEMENT_QUICK_REF.md delete mode 100644 terraform/runpod/DEPLOYMENT_SUMMARY.md delete mode 100644 terraform/runpod/QUICK_START.md delete mode 100644 terraform/runpod/README.md delete mode 100755 terraform/runpod/deploy.sh delete mode 100755 terraform/runpod/generate_credentials.sh delete mode 100644 terraform/runpod/main.tf delete mode 100644 terraform/runpod/outputs.tf delete mode 100644 terraform/runpod/terraform.tfvars.example delete mode 100755 terraform/runpod/validate.sh delete mode 100755 terraform/runpod/validate_credentials.sh delete mode 100644 terraform/runpod/variables.tf delete mode 100755 tests/e2e/integration/auth_flow_test.sh delete mode 100755 tests/e2e/integration/e2e_test_suite.sh delete mode 100755 tests/e2e/integration/hot_reload_test.sh delete mode 100755 tests/e2e/integration/trading_flow_test.sh delete mode 100755 tests/e2e/vault_e2e_test.sh delete mode 100755 tests/e2e/vault_e2e_test_curl.sh delete mode 100755 tests/e2e/vault_integration/fixtures/vault-setup/setup-vault.sh delete mode 100755 tests/e2e_helpers/infrastructure_health_check.sh delete mode 100755 tests/load_tests/ghz_authenticated.sh delete mode 100755 tests/load_tests/ghz_authenticated_fixed.sh delete mode 100755 tests/load_tests/ghz_quick_auth_test.sh delete mode 100755 tests/load_tests/ghz_quick_test.sh delete mode 100755 tests/paper_trading_smoke_test.sh delete mode 100644 tests/runpod/__init__.py delete mode 100644 tests/runpod/conftest.py delete mode 100644 tests/runpod/test_client.py delete mode 100644 tests/runpod/test_config.py delete mode 100644 tests/runpod/test_monitor.py delete mode 100644 tests/runpod/test_s3_client.py delete mode 100644 tli/ci_cd.sh delete mode 100755 trading_engine/tests/stub_tests.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac1a724e3..eb2cfe41b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,7 +108,7 @@ jobs: exit 1 fi - build-tli: + build-fxt: name: Build TLI Client runs-on: ${{ matrix.os }} timeout-minutes: 20 @@ -119,16 +119,16 @@ jobs: include: - os: ubuntu-latest target: x86_64-unknown-linux-gnu - artifact_name: tli - asset_name: tli-linux-amd64 + artifact_name: fxt + asset_name: fxt-linux-amd64 - os: macos-latest target: x86_64-apple-darwin - artifact_name: tli - asset_name: tli-macos-amd64 + artifact_name: fxt + asset_name: fxt-macos-amd64 - os: windows-latest target: x86_64-pc-windows-msvc - artifact_name: tli.exe - asset_name: tli-windows-amd64.exe + artifact_name: fxt.exe + asset_name: fxt-windows-amd64.exe steps: - name: Checkout Code @@ -151,7 +151,7 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Build TLI - run: cargo build --release -p tli --target ${{ matrix.target }} + run: cargo build --release -p fxt --target ${{ matrix.target }} - name: Strip Binary (Linux/macOS) if: matrix.os != 'windows-latest' @@ -167,7 +167,7 @@ jobs: create-release: name: Create GitHub Release runs-on: ubuntu-latest - needs: [docker-build, build-tli] + needs: [docker-build, build-fxt] if: startsWith(github.ref, 'refs/tags/v') timeout-minutes: 10 @@ -252,7 +252,7 @@ jobs: summary: name: Build Summary runs-on: ubuntu-latest - needs: [docker-build, build-tli] + needs: [docker-build, build-fxt] if: always() steps: @@ -263,7 +263,7 @@ jobs: echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY echo "| Docker Build | ${{ needs.docker-build.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| TLI Build | ${{ needs.build-tli.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| FXT Build | ${{ needs.build-fxt.result }} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY if [ "${{ github.ref }}" = "refs/heads/main" ]; then diff --git a/.github/workflows/ci-cd-pipeline.yml b/.github/workflows/ci-cd-pipeline.yml index 833dd5adc..e13370f15 100644 --- a/.github/workflows/ci-cd-pipeline.yml +++ b/.github/workflows/ci-cd-pipeline.yml @@ -79,7 +79,7 @@ jobs: path: | target/release/trading_service target/release/backtesting_service - target/release/tli + target/release/fxt retention-days: 30 build-and-test: @@ -229,7 +229,7 @@ jobs: with: images: | ghcr.io/${{ github.repository }}/foxhunt-trading-engine - ghcr.io/${{ github.repository }}/foxhunt-tli + ghcr.io/${{ github.repository }}/foxhunt-fxt ghcr.io/${{ github.repository }}/foxhunt-ml ghcr.io/${{ github.repository }}/foxhunt-risk ghcr.io/${{ github.repository }}/foxhunt-data @@ -256,10 +256,10 @@ jobs: - name: Build and push TLI service uses: docker/build-push-action@v5 with: - context: ./tli - file: ./tli/Dockerfile.production + context: ./fxt + file: ./fxt/Dockerfile.production push: true - tags: ghcr.io/${{ github.repository }}/foxhunt-tli:${{ github.sha }} + tags: ghcr.io/${{ github.repository }}/foxhunt-fxt:${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/.github/workflows/comprehensive-testing.yml b/.github/workflows/comprehensive-testing.yml index 6a8089e3c..528c4c32b 100644 --- a/.github/workflows/comprehensive-testing.yml +++ b/.github/workflows/comprehensive-testing.yml @@ -145,7 +145,7 @@ jobs: EOF - name: Build test harness - run: cargo build --bin tli --bin ml-training-service --bin trading-service + run: cargo build --bin fxt --bin ml-training-service --bin trading-service - name: Run Layer 1 Foundation Tests run: | @@ -241,7 +241,7 @@ jobs: - name: Start services for integration testing run: | # Start services in background - cargo run --bin tli -- --config tests/config/tli-test.toml & + cargo run --bin fxt -- --config tests/config/fxt-test.toml & sleep 5 cargo run --bin ml-training-service -- --config tests/config/ml-test.toml & sleep 5 @@ -368,8 +368,8 @@ jobs: - name: Start complete service stack run: | # Start all services with proper config - cargo run --bin tli -- --config tests/config/tli-workflow.toml & - TLI_PID=$! + cargo run --bin fxt -- --config tests/config/fxt-workflow.toml & + FXT_PID=$! sleep 5 cargo run --bin ml-training-service -- --config tests/config/ml-workflow.toml & @@ -381,7 +381,7 @@ jobs: sleep 10 # Store PIDs for cleanup - echo $TLI_PID > tli.pid + echo $FXT_PID > fxt.pid echo $ML_PID > ml.pid echo $TRADING_PID > trading.pid @@ -393,7 +393,7 @@ jobs: - name: Cleanup services if: always() run: | - if [ -f tli.pid ]; then kill $(cat tli.pid) || true; fi + if [ -f fxt.pid ]; then kill $(cat fxt.pid) || true; fi if [ -f ml.pid ]; then kill $(cat ml.pid) || true; fi if [ -f trading.pid ]; then kill $(cat trading.pid) || true; fi @@ -521,8 +521,8 @@ jobs: - name: Start optimized service stack for performance testing run: | # Start services with performance-optimized configs - RUST_LOG=warn cargo run --release --bin tli -- --config tests/config/tli-performance.toml & - TLI_PID=$! + RUST_LOG=warn cargo run --release --bin fxt -- --config tests/config/fxt-performance.toml & + FXT_PID=$! sleep 5 RUST_LOG=warn cargo run --release --bin ml-training-service -- --config tests/config/ml-performance.toml & @@ -533,7 +533,7 @@ jobs: TRADING_PID=$! sleep 10 - echo $TLI_PID > tli.pid + echo $FXT_PID > fxt.pid echo $ML_PID > ml.pid echo $TRADING_PID > trading.pid @@ -567,7 +567,7 @@ jobs: - name: Cleanup performance test services if: always() run: | - if [ -f tli.pid ]; then kill $(cat tli.pid) || true; fi + if [ -f fxt.pid ]; then kill $(cat fxt.pid) || true; fi if [ -f ml.pid ]; then kill $(cat ml.pid) || true; fi if [ -f trading.pid ]; then kill $(cat trading.pid) || true; fi @@ -688,8 +688,8 @@ jobs: - name: Start resilient service stack for chaos testing run: | # Start services with resilience-focused configs - cargo run --release --bin tli -- --config tests/config/tli-chaos.toml & - TLI_PID=$! + cargo run --release --bin fxt -- --config tests/config/fxt-chaos.toml & + FXT_PID=$! sleep 5 cargo run --release --bin ml-training-service -- --config tests/config/ml-chaos.toml & @@ -700,7 +700,7 @@ jobs: TRADING_PID=$! sleep 10 - echo $TLI_PID > tli.pid + echo $FXT_PID > fxt.pid echo $ML_PID > ml.pid echo $TRADING_PID > trading.pid @@ -732,7 +732,7 @@ jobs: - name: Cleanup chaos test services if: always() run: | - if [ -f tli.pid ]; then kill $(cat tli.pid) || true; fi + if [ -f fxt.pid ]; then kill $(cat fxt.pid) || true; fi if [ -f ml.pid ]; then kill $(cat ml.pid) || true; fi if [ -f trading.pid ]; then kill $(cat trading.pid) || true; fi diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 3ebd405cd..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,454 +0,0 @@ -# ============================================================================= -# GitLab CI/CD Pipeline - Foxhunt Runpod Docker Deployment -# ============================================================================= -# Purpose: Automated Docker builds for Runpod GPU deployment -# Registry: Docker Hub (jgrusewski/foxhunt) - PRIVATE repository -# Base Image: Dockerfile.foxhunt-build (CUDA 12.4.1 + cuDNN 9, Ubuntu 22.04) -# Build Strategy: Layer caching with BuildKit -# Versioning: Git commit SHA + timestamp -# Deployment: Manual trigger for production -# -# CRITICAL: Requires Docker Hub authentication via GitLab CI/CD Variables -# ============================================================================= - -# Pipeline stages -stages: - - build - - test - - deploy - -# Global variables -variables: - # Docker configuration - DOCKER_DRIVER: overlay2 - DOCKER_BUILDKIT: 1 - DOCKER_TLS_CERTDIR: "/certs" - # Fix for GitLab Runner v18.5.0: Override invalid "if-not-available" with correct value - # Valid values: "always", "if-not-present", "never" - # Using "if-not-present" for faster builds with local cache fallback - DOCKER_PULL_POLICY: "if-not-present" - - # Image configuration - DOCKER_HUB_REGISTRY: docker.io - DOCKER_HUB_REPO: jgrusewski/foxhunt - IMAGE_TAG: ${DOCKER_HUB_REPO}:${CI_COMMIT_SHORT_SHA} - IMAGE_TAG_LATEST: ${DOCKER_HUB_REPO}:latest - - # Build arguments - GIT_COMMIT: ${CI_COMMIT_SHORT_SHA} - BUILD_DATE: ${CI_COMMIT_TIMESTAMP} - - # Rust configuration - RUST_BACKTRACE: "1" - RUST_LOG: "info" - -# ============================================================================= -# REQUIRED GitLab CI/CD Variables (Settings > CI/CD > Variables) -# ============================================================================= -# DOCKER_HUB_USERNAME - Docker Hub username (e.g., jgrusewski) -# DOCKER_HUB_PASSWORD - Docker Hub access token (NOT password, use token!) -# -# To create Docker Hub access token: -# 1. Login to hub.docker.com -# 2. Account Settings > Security > New Access Token -# 3. Name: "GitLab CI/CD", Permissions: Read, Write, Delete -# 4. Copy token to GitLab CI/CD variable DOCKER_HUB_PASSWORD -# 5. Mark variable as "Masked" and "Protected" in GitLab -# -# IMPORTANT: Docker Hub repository must be set to PRIVATE -# ============================================================================= - -# Docker-in-Docker service -services: - - name: docker:24.0.7-dind - # Fix for GitLab Runner v18.5.0: Use valid pull_policy value - # "if-not-present" allows local cache usage while falling back to pull if needed - pull_policy: ["if-not-present"] - -# Global before_script: Docker Hub authentication -before_script: - - echo "Authenticating with Docker Hub..." - - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY - - docker info - -# ============================================================================= -# BUILD STAGE - Docker Image Build with Layer Caching -# ============================================================================= - -build:docker: - stage: build - image: - name: docker:24.0.7 - pull_policy: ["if-not-present"] - tags: - - docker - before_script: - # Docker Hub authentication - - echo "Authenticating with Docker Hub..." - - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY - # Set build metadata - - export GIT_COMMIT=$CI_COMMIT_SHORT_SHA - - export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - - echo "Building Docker image..." - - 'echo " Commit: $GIT_COMMIT"' - - 'echo " Date: $BUILD_DATE"' - - 'echo " Tags: $IMAGE_TAG, $IMAGE_TAG_LATEST"' - script: - # Pull latest image for layer caching - - echo "Pulling latest image for layer caching..." - - docker pull $IMAGE_TAG_LATEST || true - - # Build Docker image with BuildKit and layer caching - - | - DOCKER_BUILDKIT=1 docker build \ - -f Dockerfile.foxhunt-build \ - -t $IMAGE_TAG \ - -t $IMAGE_TAG_LATEST \ - --build-arg GIT_COMMIT=$GIT_COMMIT \ - --build-arg BUILD_DATE=$BUILD_DATE \ - --cache-from $IMAGE_TAG_LATEST \ - --label "org.opencontainers.image.created=$BUILD_DATE" \ - --label "org.opencontainers.image.revision=$GIT_COMMIT" \ - --label "org.opencontainers.image.source=$CI_PROJECT_URL" \ - --label "org.opencontainers.image.url=$CI_PROJECT_URL" \ - --label "org.opencontainers.image.title=Foxhunt HFT Trading System" \ - --label "org.opencontainers.image.description=CUDA 12.4.1 + cuDNN 9 runtime for Runpod GPU deployment" \ - . - - # Push both tags to Docker Hub - - echo "Pushing Docker images to Docker Hub..." - - docker push $IMAGE_TAG - - docker push $IMAGE_TAG_LATEST - - # Display image info - - "echo \"Docker image build complete:\"" - - docker images | grep jgrusewski/foxhunt - - docker history --no-trunc $IMAGE_TAG | head -10 - - # Build artifacts (image metadata) - after_script: - - docker inspect $IMAGE_TAG > image-metadata.json || true - - artifacts: - name: "docker-image-${CI_COMMIT_SHORT_SHA}" - paths: - - image-metadata.json - expire_in: 30 days - reports: - dotenv: image-metadata.env - - # Build only on main branch - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - when: always - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - timeout: 30m - - # Retry on infrastructure failures - retry: - max: 2 - when: - - runner_system_failure - - stuck_or_timeout_failure - -# ============================================================================= -# TEST STAGE - GLIBC and CUDA Validation -# ============================================================================= - -test:glibc-validation: - stage: test - image: - name: docker:24.0.7 - pull_policy: ["if-not-present"] - tags: - - docker - before_script: - - echo "Authenticating with Docker Hub..." - - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY - - docker pull $IMAGE_TAG - script: - - "echo \"Test 1: Verifying GLIBC version...\"" - - docker run --rm $IMAGE_TAG ldd --version - - "echo \"Test 2: Verifying GLIBC symbols for hyperopt binaries...\"" - - | - for binary in hyperopt_mamba2_demo hyperopt_dqn_demo hyperopt_ppo_demo hyperopt_tft_demo; do - echo "Checking GLIBC symbols for $binary..." - docker run --rm -v /runpod-volume:/runpod-volume $IMAGE_TAG \ - sh -c "if [ -f /runpod-volume/binaries/$binary ]; then ldd /runpod-volume/binaries/$binary | grep 'GLIBC_2'; else echo 'Binary not found on volume (expected in CI)'; fi" - done - - "echo \"Test 3: Verifying base system libraries...\"" - - docker run --rm $IMAGE_TAG ldconfig -p | grep -E "(libstdc|libgcc_s|libm\.so|libc\.so)" - - "echo \"Test 4: Verifying ca-certificates...\"" - - docker run --rm $IMAGE_TAG ls -la /etc/ssl/certs/ | head -5 - - # Run tests only after successful build - needs: - - build:docker - - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - when: always - - timeout: 10m - -test:cuda-validation: - stage: test - image: - name: docker:24.0.7 - pull_policy: ["if-not-present"] - tags: - - docker - before_script: - - echo "Authenticating with Docker Hub..." - - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY - - docker pull $IMAGE_TAG - script: - - "echo \"Test 1: Verifying CUDA installation...\"" - - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/bin/ | grep nvcc || echo "nvcc not found (expected for runtime-only)" - - "echo \"Test 2: Verifying CUDA libraries...\"" - - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/lib64/ | grep -E "(libcuda|libcurand|libcublas|libcublasLt)" - - "echo \"Test 3: Verifying cuDNN 9 installation...\"" - - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/lib64/ | grep libcudnn || docker run --rm $IMAGE_TAG find /usr -name "libcudnn*" 2>/dev/null - - "echo \"Test 4: Verifying CUDA environment variables...\"" - - docker run --rm $IMAGE_TAG env | grep -E "(CUDA_HOME|LD_LIBRARY_PATH|NVIDIA_VISIBLE_DEVICES|CUDA_VISIBLE_DEVICES)" - - "echo \"Test 5: Verifying CUDA compat libraries...\"" - - docker run --rm $IMAGE_TAG ls -la /usr/local/cuda/compat/ | grep libcuda || echo "CUDA compat libraries available" - - # Run tests only after successful build - needs: - - build:docker - - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - when: always - - timeout: 10m - -test:entrypoint-validation: - stage: test - image: - name: docker:24.0.7 - pull_policy: ["if-not-present"] - tags: - - docker - before_script: - - echo "Authenticating with Docker Hub..." - - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY - - docker pull $IMAGE_TAG - script: - - "echo \"Test 1: Verifying entrypoint scripts...\"" - - docker run --rm $IMAGE_TAG ls -la /entrypoint.sh /entrypoint-generic.sh - - "echo \"Test 2: Verifying entrypoint scripts are executable...\"" - - docker run --rm $IMAGE_TAG sh -c "test -x /entrypoint.sh && test -x /entrypoint-generic.sh && echo 'Entrypoints are executable'" - - "echo \"Test 3: Running entrypoint help...\"" - - docker run --rm $IMAGE_TAG --help || echo "Entrypoint help executed (expected failure without volume)" - - # Run tests only after successful build - needs: - - build:docker - - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - when: always - - timeout: 5m - -# ============================================================================= -# DEPLOY STAGE - Manual Production Deployment -# ============================================================================= - -deploy:runpod: - stage: deploy - image: - name: docker:24.0.7 - pull_policy: ["if-not-present"] - tags: - - docker - before_script: - - echo "Preparing for Runpod deployment..." - script: - - echo "======================================================================" - - "echo \"Docker image ready for Runpod deployment:\"" - - "echo \" Image: $IMAGE_TAG\"" - - "echo \" Latest: $IMAGE_TAG_LATEST\"" - - "echo \" Commit: $GIT_COMMIT\"" - - "echo \" Built: $BUILD_DATE\"" - - echo "======================================================================" - - echo "" - - "echo \"Deployment Instructions:\"" - - "echo \"1. Login to Runpod.io\"" - - "echo \"2. Deploy GPU Pod:\"" - - "echo \" - Region: EUR-IS-1 (required for volume mount)\"" - - "echo \" - GPU: RTX A4000 (16GB, \\$0.25/hr) or Tesla V100 (16GB, \\$0.10/hr)\"" - - "echo \" - Docker Image: $IMAGE_TAG\"" - - "echo \" - Container Registry Auth: Select Docker Hub credentials\"" - - "echo \" - Volume Mount: /runpod-volume (select existing network volume)\"" - - "echo \"3. Pod will auto-start training from /runpod-volume/binaries/\"" - - "echo \"4. Models saved to /runpod-volume/models/ (auto-synced to S3)\"" - - echo "" - - "echo \"Alternative: Use automated deployment script:\"" - - "echo \" python3 scripts/runpod_deploy.py --gpu-type 'RTX A4000' --image $IMAGE_TAG\"" - - echo "======================================================================" - - # Manual deployment approval required - when: manual - - # Deploy only on main branch - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - when: manual - - # Require all tests to pass before deployment - needs: - - build:docker - - test:glibc-validation - - test:cuda-validation - - test:entrypoint-validation - - # Deployment environment - environment: - name: production/runpod - url: https://www.runpod.io/console/pods - deployment_tier: production - action: start - - timeout: 5m - -deploy:runpod-staging: - stage: deploy - image: - name: docker:24.0.7 - pull_policy: ["if-not-present"] - tags: - - docker - before_script: - - echo "Preparing for Runpod staging deployment..." - script: - - echo "======================================================================" - - "echo \"Docker image ready for Runpod STAGING deployment:\"" - - "echo \" Image: $IMAGE_TAG\"" - - "echo \" Commit: $GIT_COMMIT\"" - - echo "======================================================================" - - echo "" - - "echo \"Staging Deployment (Auto-triggered for testing):\"" - - "echo \"1. Deploy to Runpod staging environment\"" - - "echo \"2. Run basic smoke tests (1 epoch training)\"" - - "echo \"3. Verify model output and metrics\"" - - "echo \"4. Validate S3 sync functionality\"" - - echo "" - - "echo \"Use automated deployment script with --test flag:\"" - - "echo \" python3 scripts/runpod_deploy.py --gpu-type 'Tesla V100' --image $IMAGE_TAG --test\"" - - echo "======================================================================" - - # Auto-deploy to staging (for testing) - when: on_success - - # Deploy to staging on main branch - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - when: on_success - - # Require all tests to pass - needs: - - build:docker - - test:glibc-validation - - test:cuda-validation - - test:entrypoint-validation - - # Staging environment - environment: - name: staging/runpod - url: https://www.runpod.io/console/pods - deployment_tier: staging - action: start - auto_stop_in: 1 hour - - timeout: 5m - -# ============================================================================= -# CLEANUP - Docker Hub Old Images (Optional) -# ============================================================================= - -cleanup:docker-hub: - stage: deploy - image: - name: docker:24.0.7 - pull_policy: ["if-not-present"] - tags: - - docker - before_script: - - echo "Authenticating with Docker Hub..." - - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin $DOCKER_HUB_REGISTRY - script: - - echo "======================================================================" - - "echo \"Docker Hub Cleanup (Manual)\"" - - echo "======================================================================" - - "echo \"To clean up old images on Docker Hub:\"" - - "echo \"1. Login to hub.docker.com\"" - - "echo \"2. Navigate to jgrusewski/foxhunt repository\"" - - "echo \"3. Delete old tags (keep last 10-20 commits)\"" - - "echo \"4. Keep 'latest' tag for production\"" - - echo "" - - "echo \"Alternative: Use Docker Hub API to auto-cleanup\"" - - "echo \" (Requires additional scripting - out of scope for this pipeline)\"" - - echo "======================================================================" - - # Manual cleanup trigger - when: manual - - rules: - - if: '$CI_COMMIT_BRANCH == "main"' - when: manual - - timeout: 2m - -# ============================================================================= -# PIPELINE NOTIFICATIONS (Optional) -# ============================================================================= - -# Uncomment to enable Slack/Discord notifications -# notify:success: -# stage: .post -# script: -# - 'curl -X POST -H "Content-Type: application/json" --data "{\"text\":\"✅ Foxhunt Docker build SUCCESS: $IMAGE_TAG\"}" $SLACK_WEBHOOK_URL' -# when: on_success -# rules: -# - if: '$CI_COMMIT_BRANCH == "main"' -# -# notify:failure: -# stage: .post -# script: -# - 'curl -X POST -H "Content-Type: application/json" --data "{\"text\":\"❌ Foxhunt Docker build FAILED: $CI_PIPELINE_URL\"}" $SLACK_WEBHOOK_URL' -# when: on_failure -# rules: -# - if: '$CI_COMMIT_BRANCH == "main"' - -# ============================================================================= -# PIPELINE SUMMARY -# ============================================================================= -# Stages: -# 1. BUILD (build:docker) - Build Docker image with layer caching -# 2. TEST (test:*) - Validate GLIBC, CUDA, and entrypoint scripts -# 3. DEPLOY (deploy:*) - Manual production deployment, auto staging deployment -# -# Triggers: -# - Push to main branch: Auto-build + auto-test + manual deploy -# - Merge request: Pipeline disabled (no builds on MRs) -# -# Build time: ~2-3 minutes (cached), ~5-8 minutes (clean build) -# Image size: ~4.8GB (CUDA 12.4.1 + cuDNN 9 + Ubuntu 22.04) -# Registry: Docker Hub (jgrusewski/foxhunt) - PRIVATE -# -# Security: -# - Docker Hub credentials stored in GitLab CI/CD Variables (masked) -# - PRIVATE Docker Hub repository (authentication required) -# - Manual deployment approval for production -# - Staging environment with auto-stop (1 hour) -# -# Cost: -# - GitLab CI/CD: Free tier (400 CI/CD minutes/month) -# - Docker Hub: Free tier (1 private repo, unlimited pulls) -# - Runpod GPU: Pay-per-use ($0.10-0.29/hr depending on GPU) -# ============================================================================= diff --git a/Cargo.lock b/Cargo.lock index 0605b5dbb..48e30d600 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -217,6 +217,7 @@ dependencies = [ "criterion", "dashmap 6.1.0", "futures", + "fxt", "governor", "hdrhistogram", "hex", @@ -248,7 +249,6 @@ dependencies = [ "sha2", "sqlx", "thiserror 1.0.69", - "tli", "tokio", "tokio-stream", "tokio-test", @@ -1553,6 +1553,7 @@ dependencies = [ "dbn 0.42.0", "dotenvy", "futures", + "fxt", "influxdb2", "jsonwebtoken", "ml", @@ -1579,7 +1580,6 @@ dependencies = [ "storage", "tempfile", "thiserror 1.0.69", - "tli", "tokio", "tokio-stream", "tokio-test", @@ -3757,6 +3757,7 @@ dependencies = [ "fastrand", "flate2", "futures", + "fxt", "http 1.3.1", "lazy_static", "prometheus", @@ -3773,7 +3774,6 @@ dependencies = [ "sqlx", "tempfile", "thiserror 1.0.69", - "tli", "tokio", "tokio-stream", "tonic 0.14.2", @@ -3783,32 +3783,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "foxhunt-deploy" -version = "1.0.0" -dependencies = [ - "assert_cmd", - "aws-config", - "aws-sdk-s3", - "chrono", - "clap", - "colored", - "dotenvy", - "home", - "indicatif", - "predicates", - "regex", - "reqwest 0.12.23", - "serde", - "serde_json", - "tempfile", - "thiserror 1.0.69", - "tokio", - "toml", - "tracing", - "tracing-subscriber", -] - [[package]] name = "foxhunt_e2e" version = "0.1.0" @@ -4007,6 +3981,59 @@ dependencies = [ "slab", ] +[[package]] +name = "fxt" +version = "1.0.0" +dependencies = [ + "aes-gcm", + "anyhow", + "argon2", + "assert_cmd", + "async-trait", + "base64 0.22.1", + "chrono", + "clap", + "colored", + "comfy-table", + "common", + "console", + "criterion", + "dirs 5.0.1", + "futures", + "futures-util", + "getrandom 0.2.16", + "hex", + "indicatif", + "jsonwebtoken", + "keyring", + "once_cell", + "owo-colors", + "predicates", + "proptest", + "prost 0.14.1", + "rand 0.8.5", + "rpassword", + "rust_decimal", + "serde", + "serde_json", + "serial_test", + "sha2", + "tabled", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tokio-test", + "toml", + "tonic 0.14.2", + "tonic-prost", + "tonic-prost-build", + "tracing", + "tracing-subscriber", + "uuid", + "zeroize", +] + [[package]] name = "gemm" version = "0.18.2" @@ -9629,6 +9656,7 @@ dependencies = [ "dhat", "dotenvy", "futures", + "fxt", "hdrhistogram", "influxdb2", "jemalloc_pprof", @@ -9654,7 +9682,6 @@ dependencies = [ "sqlx", "tempfile", "thiserror 1.0.69", - "tli", "tokio", "tokio-stream", "tokio-test", @@ -9844,59 +9871,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tli" -version = "1.0.0" -dependencies = [ - "aes-gcm", - "anyhow", - "argon2", - "assert_cmd", - "async-trait", - "base64 0.22.1", - "chrono", - "clap", - "colored", - "comfy-table", - "common", - "console", - "criterion", - "dirs 5.0.1", - "futures", - "futures-util", - "getrandom 0.2.16", - "hex", - "indicatif", - "jsonwebtoken", - "keyring", - "once_cell", - "owo-colors", - "predicates", - "proptest", - "prost 0.14.1", - "rand 0.8.5", - "rpassword", - "rust_decimal", - "serde", - "serde_json", - "serial_test", - "sha2", - "tabled", - "tempfile", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tokio-test", - "toml", - "tonic 0.14.2", - "tonic-prost", - "tonic-prost-build", - "tracing", - "tracing-subscriber", - "uuid", - "zeroize", -] - [[package]] name = "tokio" version = "1.47.1" diff --git a/Cargo.toml b/Cargo.toml index d60e01aa0..67408aca3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,7 +109,7 @@ members = [ "risk", "risk-data", "trading-data", - "tli", + "fxt", "ml", "ml-data", "data", @@ -135,7 +135,6 @@ members = [ "tests", "tests/e2e", "tests/load_tests", - "foxhunt-deploy", "web-gateway", "ctrader-openapi", ] @@ -382,7 +381,7 @@ arc-swap = "1.6" # foxhunt-common-types = { path = "foxhunt-common-types" } # DELETED - types migrated to common/src/types.rs trading_engine = { path = "trading_engine" } data = { path = "data" } -tli = { path = "tli" } +fxt = { path = "fxt" } risk = { path = "risk" } risk-data = { path = "risk-data" } backtesting = { path = "backtesting" } @@ -438,7 +437,7 @@ rand_distr.workspace = true serde_json.workspace = true sqlx.workspace = true tempfile = "3.13" -tli.workspace = true # Required by tests/fixtures/mod.rs +fxt.workspace = true # Required by tests/fixtures/mod.rs tokio.workspace = true trading_engine.workspace = true uuid.workspace = true diff --git a/config/mutants.toml b/config/mutants.toml index 099d17fe9..7066c3767 100644 --- a/config/mutants.toml +++ b/config/mutants.toml @@ -10,7 +10,7 @@ minimum_test_score = 80 # Packages to test (focus on critical components) # We prioritize ML, trading engine, and risk management exclude_packages = [ - "tli", # Pure client - less critical + "fxt", # Pure client - less critical "data_acquisition_service", # Lower priority "monitoring_service", # Lower priority ] diff --git a/deploy.sh b/deploy.sh index 5a180fd80..9fba1c996 100755 --- a/deploy.sh +++ b/deploy.sh @@ -209,7 +209,7 @@ build_images() { docker build -t foxhunt/backtesting-service:latest -f services/backtesting_service/Dockerfile . # Build TLI - docker build -t foxhunt/tli:latest -f tli/Dockerfile . + docker build -t foxhunt/fxt:latest -f fxt/Dockerfile . log_success "Docker images built successfully" } @@ -222,7 +222,7 @@ deploy_services() { trading-service \ ml-training-service \ backtesting-service \ - tli + fxt log_info "Waiting for application services to start..." sleep 30 @@ -246,7 +246,7 @@ deploy_full() { health_check() { log_info "Performing health check..." - local services=("trading-service" "ml-training-service" "backtesting-service" "tli") + local services=("trading-service" "ml-training-service" "backtesting-service" "fxt") local failed_services=() for service in "${services[@]}"; do diff --git a/foxhunt-deploy/Cargo.toml b/foxhunt-deploy/Cargo.toml deleted file mode 100644 index 6f46d27fa..000000000 --- a/foxhunt-deploy/Cargo.toml +++ /dev/null @@ -1,63 +0,0 @@ -[package] -name = "foxhunt-deploy" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -documentation.workspace = true -publish.workspace = true -keywords.workspace = true -categories.workspace = true - -[dependencies] -# CLI Framework -clap = { version = "4.5", features = ["derive", "env", "color"] } - -# Async Runtime -tokio = { workspace = true } - -# HTTP Client -reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false } - -# AWS SDK (for S3 log monitoring) -aws-config = "1.5" -aws-sdk-s3 = "1.50" - -# Serialization -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -toml = "0.8" - -# Error Handling -thiserror = "1.0" - -# Terminal UI -indicatif = "0.17" -colored = "2.1" - -# Environment Variables -dotenvy = "0.15" - -# Logging -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } - -# Utilities -chrono = { version = "0.4", features = ["serde"] } -home = "0.5" -regex = "1.10" - -[dev-dependencies] -assert_cmd = "2.0" -predicates = "3.1" -tempfile = "3.13" - -[[bin]] -name = "foxhunt-deploy" -path = "src/main.rs" - -[lints] -workspace = true diff --git a/foxhunt-deploy/DEPLOYMENT_QUICK_START.md b/foxhunt-deploy/DEPLOYMENT_QUICK_START.md deleted file mode 100644 index 5375932b2..000000000 --- a/foxhunt-deploy/DEPLOYMENT_QUICK_START.md +++ /dev/null @@ -1,289 +0,0 @@ -# RunPod Deployment Quick Start Guide - -## Prerequisites - -1. **Initialize Configuration**: -```bash -foxhunt-deploy init -``` - -2. **Edit Configuration**: -Edit `~/.runpod/config.toml` and add your RunPod API key: -```toml -[runpod] -api_key = "YOUR_RUNPOD_API_KEY_HERE" -``` - -## Common Deployment Scenarios - -### 1. PPO Training (Production) - -Deploy PPO training with dual learning rates: - -```bash -foxhunt-deploy deploy \ - --command "train_ppo_parquet --policy-lr 1e-6 --value-lr 0.001 --epochs 200" \ - --gpu-type "RTX A4000" \ - --name "ppo-production" -``` - -**Expected Cost**: $0.25/hr (~$0.50 for 2 hours) - -### 2. DQN Hyperparameter Optimization - -Run hyperopt for DQN with 50 trials: - -```bash -foxhunt-deploy deploy \ - --command "hyperopt_dqn_demo --n-trials 50" \ - --gpu-type "RTX A4000" \ - --name "dqn-hyperopt" -``` - -**Expected Cost**: $0.25/hr (~$0.13 for 30 minutes) - -### 3. MAMBA-2 Training (High Memory) - -Train MAMBA-2 with RTX 4090 for larger batch sizes: - -```bash -foxhunt-deploy deploy \ - --command "train_mamba2_dbn --batch-size 256" \ - --gpu-type "RTX 4090" \ - --name "mamba2-batch256" -``` - -**Expected Cost**: $0.59/hr (~$0.30 for 30 minutes) - -### 4. TFT Training (Fast Cache) - -Train TFT with cache optimization: - -```bash -foxhunt-deploy deploy \ - --command "train_tft_parquet --parquet-file /runpod-volume/data/ES_FUT_180d.parquet --epochs 50" \ - --gpu-type "RTX A4000" \ - --name "tft-fp32" -``` - -**Expected Cost**: $0.25/hr (~$0.02 for 5 minutes) - -### 5. Multi-Model Hyperopt (Long Running) - -Run hyperopt for multiple models (TFT + PPO): - -```bash -# TFT Hyperopt -foxhunt-deploy deploy \ - --command "hyperopt_tft_demo --n-trials 50" \ - --gpu-type "RTX A4000" \ - --name "tft-hyperopt" - -# PPO Hyperopt -foxhunt-deploy deploy \ - --command "hyperopt_ppo_demo --n-trials 50" \ - --gpu-type "RTX A4000" \ - --name "ppo-hyperopt" -``` - -**Expected Cost**: $0.25/hr each (~$0.25 total for parallel execution) - -### 6. Custom Environment Variables - -Deploy with custom logging and debugging: - -```bash -foxhunt-deploy deploy \ - --command "train_ppo_parquet --epochs 100" \ - --env "RUST_LOG=debug" \ - --env "RUST_BACKTRACE=1" \ - --env "CUDA_VISIBLE_DEVICES=0" -``` - -### 7. Dry Run (Validation) - -Test deployment configuration without actually deploying: - -```bash -foxhunt-deploy deploy \ - --command "train_ppo --epochs 100" \ - --gpu-type "RTX 4090" \ - --dry-run -``` - -**Output**: -``` -✓ Selected GPU: RTX 4090 (24 GB VRAM, $0.59/hr) - -ℹ [DRY RUN] Deployment Summary -──────────────────────────────────────────────────────────── - Pod Name: foxhunt-20251102-091802 - GPU: RTX 4090 - VRAM: 24 GB - Cost: $0.59/hr - Datacenter: EUR-IS-1 - Image: jgrusewski/foxhunt:latest - Command: train_ppo --epochs 100 - Container Disk: 50 GB -──────────────────────────────────────────────────────────── - -✓ Dry run completed successfully. Request is valid. -``` - -## GPU Selection Guide - -### RTX A4000 (Default) -- **VRAM**: 16 GB -- **Cost**: $0.25/hr -- **Best For**: Most training jobs (PPO, DQN, TFT) -- **Selection**: Auto-selected or `--gpu-type "RTX A4000"` - -### RTX 4090 -- **VRAM**: 24 GB -- **Cost**: $0.59/hr -- **Best For**: Large batch sizes (MAMBA-2, TFT with large cache) -- **Selection**: `--gpu-type "RTX 4090"` - -### A40 -- **VRAM**: 48 GB -- **Cost**: $0.45/hr -- **Best For**: Very large models or ensembles -- **Selection**: `--gpu-type "A40"` - -## Datacenter Selection - -### EUR-IS-1 (Default) -- **Location**: Europe (Iceland) -- **Latency**: ~50ms from EU -- **Availability**: High -- **Selection**: Auto-selected or `--datacenter "EUR-IS-1"` - -### US-TX-3 -- **Location**: US (Texas) -- **Latency**: ~30ms from US East -- **Availability**: Medium -- **Selection**: `--datacenter "US-TX-3"` - -## Monitoring and Management - -### Monitor Pod Logs - -After deployment, monitor training progress: - -```bash -# Replace with actual pod ID from deployment output -foxhunt-deploy monitor -``` - -### Terminate Pod - -When training completes: - -```bash -# Replace with actual pod ID -foxhunt-deploy run terminate --pod-id -``` - -### Check Last Pod ID - -The CLI saves the last deployed pod ID to `.last_pod_id`: - -```bash -cat .last_pod_id -``` - -## Cost Estimates - -| Training Job | GPU | Duration | Cost | -|---|---|---|---| -| PPO Production | RTX A4000 | 2 hours | $0.50 | -| DQN Hyperopt | RTX A4000 | 30 min | $0.13 | -| MAMBA-2 Training | RTX 4090 | 30 min | $0.30 | -| TFT Training | RTX A4000 | 5 min | $0.02 | -| PPO Hyperopt | RTX A4000 | 15 min | $0.06 | - -**Total for full ML suite**: ~$1.00 (with hyperopt) - -## Troubleshooting - -### API Key Not Set - -**Error**: `RunPod API key not set` - -**Solution**: -```bash -# Edit config file -nano ~/.runpod/config.toml - -# Add API key -[runpod] -api_key = "YOUR_RUNPOD_API_KEY_HERE" -``` - -### GPU Not Available - -**Error**: `GPU type 'RTX 4090' not found` - -**Solution**: -1. Try auto-selection (remove `--gpu-type` flag) -2. Use RTX A4000 instead: `--gpu-type "RTX A4000"` -3. Check RunPod dashboard for availability - -### Command Failed - -**Error**: `Docker start command cannot be empty` - -**Solution**: -Ensure `--command` flag is provided: -```bash -foxhunt-deploy deploy --command "train_ppo --epochs 100" -``` - -### Invalid Environment Variable - -**Error**: `Invalid environment variable format: KEY` - -**Solution**: -Use `KEY=VALUE` format: -```bash -foxhunt-deploy deploy --command "train_ppo" --env "RUST_LOG=debug" -``` - -## Best Practices - -1. **Always use dry-run first** for new deployments: - ```bash - foxhunt-deploy deploy --command "..." --dry-run - ``` - -2. **Use descriptive pod names** for easy identification: - ```bash - --name "ppo-production-v2" instead of auto-generated - ``` - -3. **Monitor costs** using dry-run cost estimates before deploying - -4. **Terminate pods immediately** when training completes to avoid charges - -5. **Use RTX A4000** for most jobs (best value, sufficient for 99% of workloads) - -6. **Set environment variables** for debugging during development: - ```bash - --env "RUST_LOG=debug" --env "RUST_BACKTRACE=1" - ``` - -## Next Steps - -After successful deployment: - -1. **Monitor Logs**: `foxhunt-deploy monitor ` -2. **Check Training Progress**: Watch for epoch updates, loss metrics -3. **Download Results**: Results saved to `/runpod-volume/ml_training/` -4. **Terminate Pod**: `foxhunt-deploy run terminate --pod-id ` -5. **Retrieve Models**: Access via RunPod S3 or volume download - -## Support - -- **RunPod Dashboard**: https://www.runpod.io/console/pods -- **Foxhunt Docs**: See `RUNPOD_DEPLOYMENT_IMPLEMENTATION.md` -- **CLI Help**: `foxhunt-deploy deploy --help` diff --git a/foxhunt-deploy/DOCKER_BUILD_MILESTONE2.md b/foxhunt-deploy/DOCKER_BUILD_MILESTONE2.md deleted file mode 100644 index 782a99780..000000000 --- a/foxhunt-deploy/DOCKER_BUILD_MILESTONE2.md +++ /dev/null @@ -1,378 +0,0 @@ -# Milestone 2: Docker Build Implementation - -**Status**: ✅ COMPLETE -**Date**: 2025-11-02 -**Time**: ~45 minutes - -## Summary - -Implemented Docker build and push functionality for the foxhunt-deploy CLI. The implementation uses `std::process::Command` to execute Docker commands with real-time output streaming and visual progress indicators. - -## Implementation Details - -### Module Structure - -``` -src/docker/ -├── mod.rs # Docker verification utilities -├── build.rs # Docker build implementation -└── push.rs # Docker push implementation -``` - -### Key Components - -#### 1. Docker Module (`src/docker/mod.rs`) - -**Functions**: -- `verify_docker_available()` - Validates Docker installation and daemon status -- `image_exists(tag)` - Checks if a Docker image exists locally - -**Features**: -- Uses `which docker` to verify Docker is installed -- Checks Docker daemon status via `docker version` -- Provides clear error messages for common issues - -#### 2. Build Module (`src/docker/build.rs`) - -**Main Types**: -- `DockerBuildOptions` - Builder pattern for build configuration -- `build_image()` - Main build function with streaming output - -**Features**: -- Builder pattern API for flexible configuration -- Real-time output streaming with `BufReader` -- Progress spinner using `indicatif` crate -- Validates Dockerfile and build context existence -- Captures and returns image ID after successful build -- Detailed error messages with stderr output - -**Options**: -- `tag` - Docker image tag (e.g., "jgrusewski/foxhunt:latest") -- `dockerfile` - Path to Dockerfile (default: "Dockerfile.foxhunt-build") -- `context` - Build context directory (default: ".") -- `no_cache` - Disable Docker build cache -- `build_args` - Additional build arguments (reserved for future use) - -#### 3. Push Module (`src/docker/push.rs`) - -**Main Functions**: -- `push_image(tag)` - Push image to registry -- `check_registry_auth(registry)` - Verify registry authentication - -**Features**: -- Validates image exists before pushing -- Real-time progress streaming -- Progress spinner with upload status -- Special handling for authentication errors -- Clear error messages suggesting `docker login` - -### Integration - -Updated `src/cli/build.rs` to use the new Docker module: - -```rust -pub async fn execute(config: &FoxhuntConfig, args: &BuildArgs) -> Result<()> { - // Step 1: Verify Docker - docker::verify_docker_available()?; - - // Step 2: Build image - let full_tag = format!("{}/{}:{}", - config.docker.registry, - config.docker.image_name, - args.tag - ); - - let build_options = DockerBuildOptions::new(full_tag.clone()) - .dockerfile(args.dockerfile.clone()) - .context(args.context.clone()) - .no_cache(args.no_cache); - - let _image_id = docker::build::build_image(&build_options)?; - - // Step 3: Push (unless --no-push) - if !args.no_push { - docker::push::push_image(&full_tag)?; - } - - Ok(()) -} -``` - -## Testing - -### Unit Tests (5 tests) - -Located in: -- `src/docker/mod.rs` (2 tests) -- `src/docker/build.rs` (2 tests) -- `src/docker/push.rs` (1 test) - -**Coverage**: -- ✅ Builder pattern API -- ✅ Default values -- ✅ Docker availability check -- ✅ Image existence check -- ✅ Registry auth check - -### Integration Tests (14 tests) - -Located in `tests/docker_integration_tests.rs` - -**Coverage**: -- ✅ CLI help output -- ✅ Config validation -- ✅ Command-line arguments -- ✅ Error message format validation -- ✅ Flag recognition (--no-push, --no-cache, etc.) - -**Test Results**: -``` -running 14 tests -test test_build_args_defaults ... ok -test test_build_command_help ... ok -test test_build_error_handling_daemon_not_running ... ok -test test_build_error_handling_no_docker ... ok -test test_build_options_builder_pattern ... ok -test test_build_requires_config ... ok -test test_build_validates_dockerfile_existence ... ok -test test_build_with_custom_context ... ok -test test_build_with_custom_dockerfile ... ok -test test_build_with_custom_tag ... ok -test test_build_with_no_cache_flag ... ok -test test_build_with_no_push_flag ... ok -test test_push_error_handling_image_not_found ... ok -test test_push_error_handling_no_auth ... ok - -test result: ok. 14 passed; 0 failed; 0 ignored -``` - -## Usage Examples - -### Basic Build (with push) - -```bash -# Builds and pushes jgrusewski/foxhunt:latest -foxhunt-deploy build -``` - -### Build with Custom Tag (no push) - -```bash -# Builds jgrusewski/foxhunt:v1.0.0 without pushing -foxhunt-deploy build --tag v1.0.0 --no-push -``` - -### Build with Custom Dockerfile - -```bash -# Use a different Dockerfile -foxhunt-deploy build --dockerfile Dockerfile.custom --no-cache -``` - -### Build with Custom Context - -```bash -# Use a different build context -foxhunt-deploy build --context ../parent-dir --tag test -``` - -## Error Handling - -The implementation provides clear, actionable error messages: - -### Docker Not Installed -``` -Error: Docker is not installed. Please install Docker Desktop or Docker Engine. -``` - -### Docker Daemon Not Running -``` -Error: Docker daemon is not running. Please start Docker Desktop or Docker service. -``` - -### Dockerfile Not Found -``` -Error: Dockerfile not found: Dockerfile.foxhunt-build -``` - -### Build Context Not Found -``` -Error: Build context directory not found: . -``` - -### Authentication Failed -``` -Error: Docker registry authentication failed. Please run 'docker login' first. -``` - -### Image Doesn't Exist (for push) -``` -Error: Image 'jgrusewski/foxhunt:latest' does not exist locally. Build it first. -``` - -## Visual Feedback - -The implementation uses `indicatif` for progress indication: - -``` -[1/3] Verifying Docker installation... -ℹ Building Docker image: jgrusewski/foxhunt:latest -ℹ Dockerfile: Dockerfile.foxhunt-build -ℹ Context: . -⠋ Starting Docker build... -[2/3] Building Docker image... -⠙ Step 5/12: RUN cargo build --release -✓ Built image: jgrusewski/foxhunt:latest (sha256:abc123...) -[3/3] Pushing to Docker registry... -⠸ Pushing to registry... -✓ Successfully pushed: jgrusewski/foxhunt:latest -✓ Docker build completed successfully! -ℹ Image: jgrusewski/foxhunt:latest -``` - -## Design Decisions - -### 1. std::process::Command vs bollard - -**Choice**: `std::process::Command` -**Rationale**: -- Simpler implementation (no additional dependencies) -- Direct access to Docker CLI features -- Easier debugging (can run same commands manually) -- Better compatibility across Docker versions -- Minimal overhead - -### 2. Real-time Output Streaming - -**Choice**: `BufReader` with line-by-line streaming -**Rationale**: -- User sees build progress immediately -- Can debug build failures in real-time -- Better UX for long builds -- Standard pattern for CLI tools - -### 3. Builder Pattern for Options - -**Choice**: `DockerBuildOptions` with builder methods -**Rationale**: -- Flexible API for future extensions -- Self-documenting code -- Type-safe configuration -- Optional parameters with defaults - -### 4. Progress Indicators - -**Choice**: `indicatif::ProgressBar` with spinner -**Rationale**: -- Visual feedback that process is running -- Professional CLI appearance -- Updates with current build step -- Clears on completion (no clutter) - -### 5. Error Handling Strategy - -**Choice**: Validate early, fail fast, provide context -**Rationale**: -- Check Docker installation before attempting build -- Validate file paths before running commands -- Capture stderr for detailed error messages -- Suggest fixes in error messages - -## Configuration Integration - -The build command integrates with the config system: - -```toml -[docker] -registry = "jgrusewski" # Docker Hub username or registry -image_name = "foxhunt" # Image name -tag = "latest" # Default tag (overridden by --tag) -``` - -**Tag Resolution**: -``` -Full Tag = {config.docker.registry}/{config.docker.image_name}:{args.tag} -Example: jgrusewski/foxhunt:latest -``` - -## Future Enhancements - -While not implemented in Milestone 2, the design supports: - -1. **Build Args**: The `build_arg()` method exists but isn't exposed via CLI yet -2. **Multi-platform Builds**: Could add `--platform linux/amd64,linux/arm64` -3. **BuildKit Features**: Could enable BuildKit-specific features -4. **Parallel Builds**: Could build multiple tags simultaneously -5. **Registry Selection**: Could support multiple registries -6. **Image Inspection**: Could show image size, layers, etc. - -## Files Changed - -### New Files (3) -- `src/docker/mod.rs` - Docker utilities (57 lines) -- `src/docker/build.rs` - Build implementation (194 lines) -- `src/docker/push.rs` - Push implementation (119 lines) -- `tests/docker_integration_tests.rs` - Integration tests (177 lines) - -### Modified Files (2) -- `src/main.rs` - Added docker module import -- `src/cli/build.rs` - Replaced stub with real implementation (38 lines) - -### Fixed Files (1) -- `src/runpod/deployment.rs` - Fixed ownership issue on line 43 - -**Total**: ~585 lines of production code + tests - -## Compilation Status - -```bash -$ cargo build - Compiling foxhunt-deploy v1.0.0 - Finished `dev` profile [unoptimized + debuginfo] target(s) in 15.17s - -$ cargo test --package foxhunt-deploy docker:: - Finished `test` profile [unoptimized] target(s) in 2.24s - Running unittests src/main.rs - -running 5 tests -test docker::build::tests::test_build_options_defaults ... ok -test docker::build::tests::test_build_options_builder ... ok -test docker::tests::test_verify_docker_available ... ok -test docker::push::tests::test_check_registry_auth ... ok -test docker::tests::test_image_exists ... ok - -test result: ok. 5 passed; 0 failed; 0 ignored -``` - -## Milestone Completion Checklist - -- ✅ Create Docker module structure (mod.rs, build.rs, push.rs) -- ✅ Implement `DockerBuildOptions` with builder pattern -- ✅ Implement `build_image()` with real-time output streaming -- ✅ Implement `push_image()` with progress indication -- ✅ Add Docker daemon verification -- ✅ Update build subcommand to use Docker module -- ✅ Add unit tests (5 tests) -- ✅ Add integration tests (14 tests) -- ✅ Verify all tests pass -- ✅ Handle errors gracefully with actionable messages -- ✅ Use `indicatif` for progress bars -- ✅ Stream Docker output in real-time -- ✅ Support all CLI flags (--tag, --no-push, --no-cache, --dockerfile, --context) -- ✅ Validate Docker installation and image existence - -## Next Steps (Milestone 3) - -The next milestone should implement RunPod deployment functionality: -1. Complete `src/runpod/deployment.rs` implementation -2. Integrate with `src/cli/deploy.rs` -3. Add RunPod API client functionality -4. Test end-to-end deployment workflow - -## Notes - -- The implementation prioritizes reliability and error handling -- All error messages are actionable and suggest fixes -- The builder pattern makes it easy to extend functionality -- Real-time output streaming provides immediate feedback -- Tests validate both success and error paths diff --git a/foxhunt-deploy/MILESTONE_1_COMPLETE.md b/foxhunt-deploy/MILESTONE_1_COMPLETE.md deleted file mode 100644 index e63e35a10..000000000 --- a/foxhunt-deploy/MILESTONE_1_COMPLETE.md +++ /dev/null @@ -1,221 +0,0 @@ -# Milestone 1: Core Infrastructure - COMPLETE ✅ - -**Completion Date**: 2025-11-02 -**Duration**: ~25 minutes -**Status**: All tests passing (4/4), binary size: 4.2MB - -## Implementation Summary - -Successfully implemented all core modules for the foxhunt-deploy Rust CLI: - -### 1. Configuration System ✅ -**Files Created**: -- `src/config/types.rs` (179 lines) - Configuration structs with serde support -- `src/config/mod.rs` (171 lines) - ConfigManager with TOML loading and env overrides -- `examples/sample_config.toml` - Sample configuration file - -**Features**: -- `FoxhuntConfig` main struct with nested configs -- `RunPodConfig` (API key, GPU type, datacenter) -- `DockerConfig` (registry, image, tag) -- `S3Config` (endpoint, bucket, region, poll interval) -- `DeploymentDefaults` (disk size, volume settings) -- Default value functions for all fields -- Environment variable overrides from `.env.runpod` -- Config validation (API key, S3 bucket required) -- Sample config creation at `~/.runpod/config.toml` - -### 2. Terminal Utilities ✅ -**Files Created**: -- `src/utils/terminal.rs` (43 lines) - Colored terminal output -- `src/utils/mod.rs` (4 lines) - Module exports - -**Features**: -- `success(msg)` - Green checkmark ✓ -- `error(msg)` - Red X ✗ -- `info(msg)` - Blue info icon ℹ -- `warning(msg)` - Yellow warning ⚠ -- `step(step, total, msg)` - Numbered progress [1/5] - -### 3. CLI Framework ✅ -**Files Created**: -- `src/cli/mod.rs` (36 lines) - Main CLI struct with clap -- `src/cli/build.rs` (33 lines) - Build subcommand -- `src/cli/deploy.rs` (42 lines) - Deploy subcommand -- `src/cli/monitor.rs` (28 lines) - Monitor subcommand -- `src/cli/run.rs` (33 lines) - Unified run subcommand - -**Features**: -- Global flags: `--verbose`, `--config` -- `init` command - Creates sample config -- `build` command - Docker image building (stub) -- `deploy` command - RunPod deployment (stub) -- `monitor` command - Log monitoring (stub) -- `run` command - Unified workflow (stub) - -### 4. Main Entry Point ✅ -**File Modified**: -- `src/main.rs` (94 lines) - Complete async runtime with routing - -**Features**: -- Tokio async runtime -- Clap CLI parsing -- Tracing-subscriber logging (verbose/info levels) -- Config loading with path override -- Config validation -- Subcommand routing -- Graceful error handling - -### 5. Error Handling ✅ -**File** (existing): -- `src/error.rs` - Custom error types with thiserror - -**Error Types**: -- `ConfigNotFound` - Missing config file -- `MissingConfigField` - Required field validation -- `Config` - General config errors -- Plus Docker, RunPod API, S3, IO, HTTP errors - -## Test Results - -```bash -$ cargo test --package foxhunt-deploy -running 4 tests -test config::tests::test_default_config ... ok -test config::tests::test_env_override ... ok -test config::tests::test_validation ... ok -test utils::terminal::tests::test_terminal_functions ... ok - -test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out -``` - -## Verified Behavior - -### 1. CLI Help -```bash -$ foxhunt-deploy --help -Deploy and manage Foxhunt ML training workloads on RunPod GPU infrastructure. -Supports Docker image building, pod deployment, log monitoring, and unified workflows. - -Usage: foxhunt-deploy [OPTIONS] - -Commands: - init Initialize configuration file - build Build Docker image - deploy Deploy pod to RunPod - monitor Monitor pod logs - run Build and deploy in one command - help Print this message or the help of the given subcommand(s) -``` - -### 2. Init Command -```bash -$ foxhunt-deploy init -ℹ Initializing foxhunt-deploy configuration... -✓ Created sample configuration at: /home/user/.runpod/config.toml -ℹ Please edit the config file and add your RunPod API key. -ℹ You can also override settings with a .env.runpod file or environment variables. -``` - -### 3. Config Validation -```bash -$ rm ~/.runpod/config.toml -$ foxhunt-deploy build -✗ Error: Config file not found at "/home/user/.runpod/config.toml". Run 'foxhunt-deploy init' to create one. -``` - -### 4. Subcommand Stubs -All subcommands (build, deploy, monitor, run) execute successfully with placeholder messages indicating they're ready for Milestone 2+ implementation. - -### 5. Environment Override -```bash -$ RUNPOD_API_KEY="override" foxhunt-deploy build -# Uses overridden API key -``` - -## Code Statistics - -- **Total Files Created**: 11 -- **Total Lines of Code**: 770 -- **Test Coverage**: 4 unit tests -- **Binary Size**: 4.2MB (release, stripped) -- **Build Time**: 64 seconds (release) -- **Dependencies**: All from Cargo.toml (clap, tokio, serde, toml, etc.) - -## File Structure - -``` -foxhunt-deploy/ -├── src/ -│ ├── main.rs (94 lines) - Entry point -│ ├── error.rs (66 lines) - Error types -│ ├── cli/ -│ │ ├── mod.rs (36 lines) - CLI framework -│ │ ├── build.rs (33 lines) - Build command -│ │ ├── deploy.rs (42 lines) - Deploy command -│ │ ├── monitor.rs (28 lines) - Monitor command -│ │ └── run.rs (33 lines) - Run command -│ ├── config/ -│ │ ├── mod.rs (171 lines) - Config manager -│ │ └── types.rs (179 lines) - Config structs -│ └── utils/ -│ ├── mod.rs (4 lines) - Utils exports -│ └── terminal.rs (43 lines) - Terminal output -└── examples/ - └── sample_config.toml (52 lines) - Sample config -``` - -## Next Steps (Milestone 2) - -1. **Docker Builder Module** - Implement `src/docker/builder.rs` - - Multi-stage build support - - BuildKit caching - - Registry push - - Progress reporting - -2. **Build Command Implementation** - Wire up `cli::build::execute()` - - Call Docker builder - - Handle --no-push, --no-cache flags - - Show build progress - -3. **Integration Tests** - Add `tests/integration_tests.rs` - - Test config loading - - Test CLI parsing - - Test error handling - -## Dependencies Used - -- `clap` 4.5 - CLI framework with derive macros -- `tokio` 1.40 - Async runtime -- `serde` 1.0 - Serialization -- `toml` 0.8 - TOML parsing -- `colored` 2.1 - Terminal colors -- `tracing` 0.1 - Structured logging -- `tracing-subscriber` 0.3 - Log formatting -- `dotenvy` 0.15 - .env file loading -- `home` 0.5 - Home directory detection -- `thiserror` 1.0 - Error derive macros - -## Critical Design Decisions - -1. **Async-First**: All command handlers are async (prepare for HTTP/S3 operations) -2. **Config Hierarchy**: File → Environment → CLI args (standard precedence) -3. **Error Context**: Rich error messages with actionable suggestions -4. **Stub Pattern**: All subcommands return Ok(()) with placeholder messages -5. **Test Coverage**: Unit tests for config, validation, and terminal utils -6. **Binary Optimization**: Release profile with LTO, strip, single codegen unit - -## Known Issues - -- None. All functionality working as designed. - -## Warnings - -- 54 compiler warnings (all "unreachable pub item" - expected for now) -- Can be fixed with `cargo fix` later or ignored (not blocking) - ---- - -**Milestone 1 Status**: ✅ **COMPLETE AND CERTIFIED** - -Ready for Milestone 2: Docker Builder Implementation diff --git a/foxhunt-deploy/MILESTONE_3_SUMMARY.md b/foxhunt-deploy/MILESTONE_3_SUMMARY.md deleted file mode 100644 index 9fd86df34..000000000 --- a/foxhunt-deploy/MILESTONE_3_SUMMARY.md +++ /dev/null @@ -1,427 +0,0 @@ -# Milestone 3 Implementation Summary - -**Project**: foxhunt-deploy RunPod Deployment Functionality -**Status**: ✅ **COMPLETE** -**Date**: 2025-11-02 -**Time**: 55 minutes -**Code Quality**: Production-Ready - ---- - -## Deliverables - -### 1. RunPod Module Structure ✅ - -Created complete module hierarchy: - -``` -src/runpod/ -├── mod.rs # Module exports and public API -├── types.rs # 167 lines - API request/response types -├── client.rs # 165 lines - RunPodClient implementation -└── deployment.rs # 238 lines - Deployment logic and helpers -``` - -**Total**: 570 lines of production code - -### 2. RunPod API Types ✅ - -Implemented comprehensive serde types (`src/runpod/types.rs`): - -- ✅ `PodDeploymentRequest` - Complete deployment payload with 16 fields -- ✅ `PodDeploymentResponse` - Deployment result with machine/runtime info -- ✅ `GpuType` - GPU information with pricing and VRAM -- ✅ `MachineInfo` - Machine configuration details -- ✅ `RuntimeInfo` - Datacenter and runtime metadata -- ✅ `ApiError` - Error response parsing -- ✅ `PodTerminationResponse` - Termination result - -**Features**: -- Full serde serialization/deserialization -- CamelCase API compatibility -- Optional field support -- Type-safe API contracts - -### 3. RunPod Client ✅ - -Implemented async HTTP client (`src/runpod/client.rs`): - -```rust -impl RunPodClient { - pub fn new(api_key: String) -> Result - pub async fn list_gpus(&self) -> Result> - pub async fn deploy_pod(&self, request: PodDeploymentRequest) -> Result - pub async fn terminate_pod(&self, pod_id: &str) -> Result<()> - pub async fn get_pod(&self, pod_id: &str) -> Result -} -``` - -**Features**: -- ✅ API key validation -- ✅ 30-second HTTP timeout -- ✅ Bearer token authentication -- ✅ JSON request/response handling -- ✅ Comprehensive error handling -- ✅ Structured logging (info, debug) - -**API Integration**: -- Base URL: `https://rest.runpod.io/v1` -- Endpoints: `/pods`, `/pods/{id}` -- Methods: POST, GET, DELETE -- Content-Type: `application/json` - -### 4. Deployment Logic ✅ - -Implemented helper functions (`src/runpod/deployment.rs`): - -```rust -pub fn build_deployment_request() -> Result -pub fn select_best_gpu() -> Result -pub fn validate_deployment() -> Result<()> -``` - -**GPU Selection Algorithm**: -1. User-specified GPU (exact or fuzzy match) -2. Auto-select RTX A4000 (best value: $0.25/hr) -3. Fallback to cheapest available - -**Request Builder**: -- ✅ Command parsing (whitespace split) -- ✅ Environment variable map construction -- ✅ Pod name generation with timestamp -- ✅ Configuration defaults application -- ✅ Docker image name formatting - -**Validation**: -- ✅ GPU count validation (>= 1) -- ✅ Container disk validation (>= 10GB) -- ✅ Command validation (non-empty) - -### 5. Deploy Subcommand ✅ - -Updated `src/cli/deploy.rs` with full implementation: - -**CLI Arguments**: -```bash ---command # Required: Docker start command ---gpu-type # Optional: GPU selection ---datacenter # Optional: Datacenter preference ---tag # Optional: Docker image tag (default: latest) ---env # Optional: Environment variables (repeatable) ---name # Optional: Pod name ---volume-size # Optional: Volume size in GB ---dry-run # Optional: Validate without deploying -``` - -**Features**: -- ✅ Interactive deployment confirmation -- ✅ Cost estimates (hourly + daily) -- ✅ Deployment summary display -- ✅ Pod ID persistence (`.last_pod_id`) -- ✅ Next-step guidance (monitor, terminate) -- ✅ Dry-run mode for validation - -**User Experience**: -``` -ℹ Starting RunPod deployment... -✓ Selected GPU: RTX A4000 (16 GB VRAM, $0.25/hr) - -ℹ Deployment Summary -──────────────────────────────────────────────────────────── - Pod Name: foxhunt-20251102-091802 - GPU: RTX A4000 - VRAM: 16 GB - Cost: $0.25/hr - Datacenter: EUR-IS-1 - Image: jgrusewski/foxhunt:latest - Command: train_ppo --epochs 100 - Container Disk: 50 GB -──────────────────────────────────────────────────────────── - -Estimated cost: $0.25/hr ($6.00/day) -Deploy pod? [y/N]: y - -✓ Pod deployed successfully! -``` - -### 6. Error Handling ✅ - -Comprehensive error handling: - -**Configuration Errors**: -- ❌ Missing API key → User-friendly message -- ❌ Invalid GPU type → Available options listed -- ❌ Invalid environment variables → Format guidance - -**API Errors**: -- ❌ HTTP errors (network, timeout) → Parsed and displayed -- ❌ RunPod API errors (quota, GPU unavailable) → User-friendly -- ❌ JSON parsing errors → Debug info provided - -**Validation Errors**: -- ❌ Empty command → Clear error message -- ❌ Invalid GPU count → Minimum value enforced -- ❌ Invalid disk size → Minimum value enforced - -### 7. Testing ✅ - -**Unit Tests** (6 tests in `deployment.rs`): -``` -✓ test_parse_command # Command parsing -✓ test_parse_command_empty # Error handling -✓ test_select_best_gpu_preferred # User preference -✓ test_select_best_gpu_auto # Auto-selection -✓ test_validate_deployment # Valid request -✓ test_validate_deployment_invalid_gpu_count # Invalid request -``` - -**Integration Tests**: -- ✅ Dry-run validation -- ✅ Auto GPU selection -- ✅ Custom GPU selection -- ✅ Environment variables -- ✅ Cost estimation - -**Test Results**: -``` -Running unittests: 19 passed; 0 failed -Running integration tests: 14 passed; 0 failed -Total: 33 tests passed ✅ -``` - -### 8. Documentation ✅ - -Created comprehensive documentation: - -1. **RUNPOD_DEPLOYMENT_IMPLEMENTATION.md** (525 lines): - - Architecture overview - - API integration details - - Configuration guide - - Usage examples - - Testing documentation - - Future enhancements - -2. **DEPLOYMENT_QUICK_START.md** (350 lines): - - Common deployment scenarios - - GPU selection guide - - Cost estimates - - Troubleshooting - - Best practices - -3. **Inline Documentation**: - - Detailed rustdoc comments - - Function-level documentation - - Example usage in comments - ---- - -## Technical Specifications - -### Performance - -- **API Response Time**: <500ms for deployment requests -- **Validation**: <10ms for local validation -- **Binary Size**: 21MB (release build, optimized with LTO) -- **Memory**: <20MB for typical operations -- **Build Time**: 76 seconds (clean build) - -### Code Metrics - -| Component | Lines | Tests | Coverage | -|---|---|---|---| -| types.rs | 167 | - | N/A (data types) | -| client.rs | 165 | - | Manual validation | -| deployment.rs | 238 | 6 | 100% | -| deploy.rs | 242 | - | E2E tested | -| **Total** | **812** | **6** | **100%** | - -### Dependencies - -**Added**: -- `reqwest` (already in Cargo.toml) - HTTP client -- `serde` (already in Cargo.toml) - Serialization -- `chrono` (already in Cargo.toml) - Timestamps - -**No new dependencies required** ✅ - ---- - -## Usage Examples - -### Example 1: Basic Deployment - -```bash -foxhunt-deploy deploy --command "train_ppo --epochs 100" -``` - -**Output**: -- Auto-selects RTX A4000 -- Generates pod name with timestamp -- Displays deployment summary -- Saves pod ID to `.last_pod_id` - -### Example 2: Custom Configuration - -```bash -foxhunt-deploy deploy \ - --command "hyperopt_dqn_demo --n-trials 50" \ - --gpu-type "RTX 4090" \ - --datacenter "EUR-IS-1" \ - --env "RUST_LOG=debug" \ - --name "dqn-hyperopt" -``` - -### Example 3: Dry Run - -```bash -foxhunt-deploy deploy \ - --command "train_ppo --epochs 100" \ - --dry-run -``` - -**Output**: Validates request without deploying - ---- - -## API Integration - -### Request Example - -```json -POST https://rest.runpod.io/v1/pods -Authorization: Bearer - -{ - "cloudType": "SECURE", - "computeType": "GPU", - "dataCenterIds": ["EUR-IS-1"], - "gpuTypeIds": ["NVIDIA RTX A4000"], - "gpuCount": 1, - "name": "foxhunt-20251102-091802", - "imageName": "jgrusewski/foxhunt:latest", - "containerDiskInGb": 50, - "networkVolumeId": "se3zdnb5o4", - "volumeMountPath": "/runpod-volume", - "dockerStartCmd": ["train_ppo", "--epochs", "100"], - "containerRegistryAuthId": "cmh3ya1710001jo02vwqtisbf", - "ports": ["8888/http", "22/tcp"], - "interruptible": false -} -``` - -### Response Example - -```json -{ - "id": "pod_abc123xyz", - "machine": { - "gpuTypeId": "NVIDIA RTX A4000", - "gpuCount": 1, - "vramGb": 16 - }, - "runtime": { - "datacenterId": "EUR-IS-1" - }, - "costPerHr": 0.25 -} -``` - ---- - -## Completion Checklist - -- ✅ RunPod module structure created -- ✅ API types implemented with serde -- ✅ RunPod client with async HTTP -- ✅ Deployment logic with GPU selection -- ✅ Deploy command with interactive UX -- ✅ Dry-run mode for validation -- ✅ Error handling (config, API, validation) -- ✅ Unit tests (6/6 passing) -- ✅ Integration tests (manual validation) -- ✅ Documentation (2 guides + inline) -- ✅ Code compiles without errors -- ✅ Binary tested and working - ---- - -## Time Breakdown - -| Task | Time | Notes | -|---|---|---| -| Module setup | 5 min | Created runpod/ directory structure | -| API types | 10 min | Comprehensive serde types | -| RunPod client | 15 min | Async HTTP with error handling | -| Deployment logic | 12 min | GPU selection + validation | -| Deploy command | 10 min | CLI integration + UX | -| Testing | 8 min | Unit tests + manual validation | -| Documentation | 5 min | Implementation guide + quick start | -| **Total** | **55 min** | **Under 60-minute target** ✅ | - ---- - -## Quality Metrics - -### Code Quality -- ✅ Production-ready code -- ✅ Comprehensive error handling -- ✅ Type-safe API integration -- ✅ No unsafe code -- ✅ No unwrap() calls in production paths -- ✅ Structured logging throughout - -### User Experience -- ✅ Clear, informative output -- ✅ Cost estimates before deployment -- ✅ Interactive confirmation -- ✅ Next-step guidance -- ✅ Dry-run validation mode -- ✅ Helpful error messages - -### Maintainability -- ✅ Modular architecture -- ✅ Comprehensive documentation -- ✅ Test coverage -- ✅ Clear separation of concerns -- ✅ No hardcoded values -- ✅ Configuration-driven - ---- - -## Next Steps (Future Enhancements) - -1. **GraphQL Integration**: - - Replace hardcoded GPU list with real-time availability - - Support dynamic GPU type discovery - -2. **Pod Monitoring**: - - Implement `monitor` subcommand - - Real-time log streaming - - Training metrics parsing - -3. **Advanced Features**: - - Multi-GPU deployment support - - Pod templates and presets - - Cost optimization (spot instances) - - SSH key injection - -4. **Testing**: - - Mock RunPod API for integration tests - - End-to-end deployment tests - - Load testing for API client - ---- - -## Conclusion - -**Milestone 3 is 100% complete** with production-ready code, comprehensive testing, and extensive documentation. The implementation provides a robust, user-friendly CLI for deploying ML training workloads to RunPod GPU infrastructure. - -**Key Achievements**: -- ✅ Full REST API integration -- ✅ Smart GPU selection algorithm -- ✅ Interactive user experience -- ✅ Comprehensive error handling -- ✅ 100% test pass rate -- ✅ Complete documentation - -**Ready for**: Production deployment and integration with Foxhunt ML training workflows. diff --git a/foxhunt-deploy/MILESTONE_4_SUMMARY.md b/foxhunt-deploy/MILESTONE_4_SUMMARY.md deleted file mode 100644 index 016fba624..000000000 --- a/foxhunt-deploy/MILESTONE_4_SUMMARY.md +++ /dev/null @@ -1,480 +0,0 @@ -# Milestone 4: S3 Log Monitoring - Implementation Summary - -**Status**: ✅ COMPLETE -**Date**: 2025-11-02 -**Duration**: ~45 minutes -**Developer**: AI Assistant - -## Overview - -Successfully implemented S3 log monitoring functionality for the foxhunt-deploy CLI tool. This allows users to stream real-time training logs from RunPod S3 buckets with colorized output, filtering, and automatic completion detection. - -## Deliverables - -### 1. Core Implementation - -#### Files Created - -1. **`src/s3/mod.rs`** (186 lines) - - `S3LogClient` struct with AWS SDK wrapper - - Custom RunPod endpoint configuration - - Methods: list_log_files, download_log, download_log_range, get_log_size, log_exists - -2. **`src/s3/monitor.rs`** (246 lines) - - `LogMonitor` struct for real-time log streaming - - Poll-based streaming with configurable interval - - Methods: tail_logs, show_recent_logs, list_logs - - Features: auto-detection, retries, completion detection - -3. **`src/s3/parser.rs`** (192 lines) - - Log level detection and colorization - - Training metrics parsing (epoch, loss, LR) - - Completion detection - - Filtering and tail utilities - -#### Files Modified - -1. **`src/cli/monitor.rs`** (57 lines) - - Replaced stub implementation with real S3 client integration - - Added `--list` flag for listing log files - - Connected to LogMonitor for streaming - -2. **`src/main.rs`** (96 lines) - - Added `mod s3` declaration - -3. **`Cargo.toml`** (71 lines) - - Added `regex = "1.10"` dependency - -### 2. Documentation - -1. **`README.md`** (500+ lines) - - Complete CLI documentation - - Usage examples for all commands - - Configuration guide - - Troubleshooting section - -2. **`S3_MONITOR_IMPLEMENTATION.md`** (400+ lines) - - Detailed technical implementation - - Architecture overview - - API reference - - Performance characteristics - - Known limitations - -3. **`MILESTONE_4_SUMMARY.md`** (this file) - - Implementation summary - - Test results - - Next steps - -### 3. Examples - -1. **`examples/monitor_demo.sh`** (150+ lines) - - Interactive demo script - - 7 usage examples - - Prerequisite checks - - Common patterns reference - -## Features Implemented - -### ✅ Core Features - -1. **Real-Time Log Streaming** - - Poll-based S3 monitoring (5s interval, configurable) - - Byte-range downloads for efficiency - - Tracks last read position - - Only downloads new content - -2. **Colorized Output** - - Green: INFO - - Yellow: WARN - - Red: ERROR - - Cyan: DEBUG - - Dimmed: TRACE - -3. **Log Discovery** - - Auto-detects primary log file - - Priority: training.log > stdout > stderr - - Lists all available logs with `--list` flag - -4. **Tail Mode** - - Show last N lines: `--tail ` - - Works in both follow and snapshot modes - - Combines with follow: `--follow --tail 10` - -5. **Filtering** - - Regex pattern matching: `--filter "epoch|loss"` - - Filters before colorization - - Show only errors: `--filter "ERROR|WARN"` - -6. **Completion Detection** - - Detects training completion keywords - - Exits gracefully when complete - - Patterns: "training complete", "saved final model", etc. - -7. **Error Handling** - - Graceful retries (max 3 attempts) - - Network error recovery - - Missing log file handling - - Clear error messages - -### ✅ User Experience - -1. **Intuitive CLI** - - Consistent with other commands - - Helpful error messages - - Progress indicators - -2. **Configuration** - - TOML-based config file - - Environment variable overrides - - Sensible defaults - -3. **Documentation** - - Comprehensive README - - Usage examples - - Interactive demo script - - Troubleshooting guide - -## Technical Details - -### Architecture - -``` -S3LogClient (mod.rs) - ↓ (manages connection) -AWS SDK S3 Client - ↓ (authenticated requests) -RunPod S3 Endpoint (https://s3api-eur-is-1.runpod.io) - ↓ (downloads logs) -LogMonitor (monitor.rs) - ↓ (streams content) -LogParser (parser.rs) - ↓ (colorizes) -Terminal Output -``` - -### Dependencies - -```toml -aws-config = "1.5" -aws-sdk-s3 = "1.50" -regex = "1.10" -colored = "2.1" # Already in Cargo.toml -tokio = "1.40" # Already in Cargo.toml -``` - -### Configuration - -**S3 Config** (`~/.foxhunt/config.toml`): -```toml -[s3] -endpoint = "https://s3api-eur-is-1.runpod.io" -bucket = "se3zdnb5o4" -region = "us-east-1" -poll_interval_secs = 5 -``` - -**Environment Variables**: -```bash -export AWS_ACCESS_KEY_ID="your_key" -export AWS_SECRET_ACCESS_KEY="your_secret" -``` - -### S3 Path Structure - -``` -s3://se3zdnb5o4/ml_training/ -└── {pod_id}/ - ├── training_runs/ - │ └── {model}/ - │ └── run_{timestamp}/ - │ ├── logs/ - │ │ └── training.log (priority 1) - │ └── checkpoints/ - ├── stdout (priority 2) - └── stderr (priority 3) -``` - -## Testing - -### Build Tests - -```bash -# Debug build -cargo build -# Result: ✅ SUCCESS (11 warnings, 0 errors) - -# Release build -cargo build --release -# Result: ✅ SUCCESS (compiled in 36.23s) -``` - -### Unit Tests - -```bash -cargo test --package foxhunt-deploy s3 -``` - -**Tests Implemented**: -1. `test_detect_log_level` - ✅ PASS -2. `test_parse_training_metrics` - ✅ PASS -3. `test_detect_completion` - ✅ PASS -4. `test_get_last_n_lines` - ✅ PASS - -### Manual Testing - -**Commands Verified**: -```bash -# Help output -foxhunt-deploy monitor --help # ✅ PASS - -# List logs -foxhunt-deploy monitor --list # ✅ Ready (requires S3 creds) - -# Show tail -foxhunt-deploy monitor --tail 20 # ✅ Ready - -# Follow logs -foxhunt-deploy monitor --follow # ✅ Ready - -# Filter -foxhunt-deploy monitor --follow --filter "epoch" # ✅ Ready -``` - -## Performance - -### Metrics - -| Metric | Value | Notes | -|--------|-------|-------| -| Poll Interval | 5s | Configurable in config | -| Latency | 5-10s | Poll interval + S3 latency | -| Network | Byte-range only | Only downloads new content | -| Memory | Streaming | Doesn't load entire file | -| S3 Costs | ~$0.01/month | Minimal GET/HEAD requests | - -### Efficiency - -- ✅ **Byte-Range Requests**: Only downloads new content since last poll -- ✅ **HEAD Requests**: Checks file size before downloading -- ✅ **Streaming**: Processes chunks, doesn't buffer entire file -- ✅ **Retry Logic**: 3 retries with exponential backoff - -## Usage Examples - -### 1. List Available Logs - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --list -``` - -**Output**: -``` -Searching for logs for pod: dy2bn5ninzaxma -──────────────────────────────────────────────────────────────────────────────── -3 log file(s): - • ml_training/dy2bn5ninzaxma/training_runs/ppo/run_20251102/logs/training.log - • ml_training/dy2bn5ninzaxma/stdout - • ml_training/dy2bn5ninzaxma/stderr -``` - -### 2. Show Last 20 Lines - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --tail 20 -``` - -### 3. Follow Logs - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --follow -``` - -**Output**: -``` -Monitoring pod: dy2bn5ninzaxma -Poll interval: 5s -──────────────────────────────────────────────────────────────────────────────── -Found log file: ml_training/dy2bn5ninzaxma/training.log -──────────────────────────────────────────────────────────────────────────────── -INFO: Starting training... [green] -INFO: Epoch: 1, Loss: 0.345 [green] -WARN: Learning rate decayed [yellow] -... -``` - -### 4. Filter for Metrics - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "epoch|loss" -``` - -### 5. Show Errors Only - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "ERROR|WARN" -``` - -## Known Limitations - -1. **Poll Delay**: 5-10s lag due to polling (not true streaming) - - **Reason**: S3 doesn't support WebSockets - - **Mitigation**: Configurable poll interval - -2. **S3 Costs**: Frequent HEAD/GET requests - - **Impact**: Minimal (~$0.01/month) - - **Efficiency**: Byte-range requests reduce bandwidth - -3. **Regex Errors**: Invalid regex shows all lines - - **Reason**: Fail-open for user convenience - - **Alternative**: Could show error message - -4. **Single Log File**: Doesn't merge multiple logs - - **Reason**: Complexity vs. value - - **Workaround**: Use `--list` to see all logs - -## Future Enhancements - -### High Priority - -1. **WebSocket Support**: True real-time streaming (if RunPod adds support) -2. **Log Download**: Save logs to local file -3. **Multi-Pod Monitoring**: Monitor multiple pods simultaneously - -### Medium Priority - -4. **Metrics Dashboard**: Live plot of loss/epoch curves -5. **Log Aggregation**: Merge stdout + stderr + training.log -6. **Compression**: Support .gz log files - -### Low Priority - -7. **Pagination**: For very large log files -8. **Log Search**: Full-text search within logs -9. **Export**: Export logs to JSON/CSV - -## Integration with Existing Workflow - -### Before (Manual Process) - -1. Deploy pod via Python script -2. SSH into pod or use RunPod dashboard -3. Tail logs manually: `tail -f /runpod-volume/ml_training/*/logs/training.log` -4. Copy-paste to local terminal - -### After (Automated) - -1. Deploy pod: `foxhunt-deploy deploy --command "train_ppo --epochs 100"` -2. Monitor logs: `foxhunt-deploy monitor --follow` -3. Filter metrics: `--filter "epoch|loss"` -4. Automatic completion detection - -**Time Savings**: ~5 minutes per deployment - -## Code Quality - -### Metrics - -- **Lines of Code**: ~700 (3 new files, 2 modified) -- **Test Coverage**: 4 unit tests -- **Documentation**: 1,000+ lines (README + implementation doc) -- **Examples**: 1 interactive demo script - -### Best Practices - -- ✅ Error handling with custom error types -- ✅ Async/await throughout -- ✅ Configuration management -- ✅ Modular architecture -- ✅ Comprehensive documentation -- ✅ Type safety (strong typing) -- ✅ Resource cleanup (no leaks) - -## Deployment Checklist - -### Prerequisites - -- [x] AWS credentials set (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`) -- [x] Configuration file created (`~/.foxhunt/config.toml`) -- [x] Binary built (`cargo build --release`) -- [x] Binary in PATH (`/usr/local/bin/foxhunt-deploy`) - -### Verification - -```bash -# 1. Check binary -foxhunt-deploy --version -# Expected: foxhunt-deploy 1.0.0 - -# 2. Check config -cat ~/.foxhunt/config.toml -# Expected: Valid TOML with [s3] section - -# 3. Check credentials -echo $AWS_ACCESS_KEY_ID -# Expected: Non-empty string - -# 4. Test monitor help -foxhunt-deploy monitor --help -# Expected: Help text with options - -# 5. Test list (if pod exists) -foxhunt-deploy monitor --list -# Expected: List of log files or "No log files found" -``` - -## Success Criteria - -### ✅ All Met - -- [x] Real-time log streaming working -- [x] Colorized output implemented -- [x] Filtering functional -- [x] Tail mode working -- [x] Completion detection active -- [x] Error handling robust -- [x] Documentation complete -- [x] Examples provided -- [x] Build succeeds -- [x] Tests pass - -## Lessons Learned - -### Technical - -1. **AWS SDK**: Custom endpoint requires careful configuration -2. **Byte Ranges**: Essential for efficient streaming -3. **Error Handling**: Fail-open for better UX (e.g., invalid regex) -4. **Polling**: Trade-off between latency and S3 costs - -### Process - -1. **Modular Design**: Separated concerns (client, monitor, parser) -2. **Documentation First**: README written concurrently with code -3. **Examples**: Interactive demo adds significant value -4. **Testing**: Unit tests caught regex edge cases early - -## Conclusion - -Successfully delivered S3 log monitoring functionality for foxhunt-deploy CLI in ~45 minutes. The implementation is production-ready with: - -- ✅ Clean, modular architecture -- ✅ Comprehensive error handling -- ✅ Extensive documentation -- ✅ Interactive examples -- ✅ Unit tests -- ✅ Performance optimizations - -The feature integrates seamlessly with existing deploy/run workflows and provides significant time savings for ML training monitoring. - -**Next Steps**: -1. Deploy to production -2. Gather user feedback -3. Implement priority enhancements (WebSocket, multi-pod, download) - ---- - -**Implementation Time**: ~45 minutes -**Code Quality**: Production-ready -**User Experience**: Excellent -**Documentation**: Comprehensive -**Status**: ✅ READY FOR PRODUCTION diff --git a/foxhunt-deploy/MONITOR_QUICK_REF.md b/foxhunt-deploy/MONITOR_QUICK_REF.md deleted file mode 100644 index 5c6c8f97f..000000000 --- a/foxhunt-deploy/MONITOR_QUICK_REF.md +++ /dev/null @@ -1,99 +0,0 @@ -# S3 Monitor Quick Reference - -## Prerequisites - -```bash -export AWS_ACCESS_KEY_ID="your_runpod_access_key" -export AWS_SECRET_ACCESS_KEY="your_runpod_secret_key" -``` - -## Common Commands - -### List Logs -```bash -foxhunt-deploy monitor --list -``` - -### Quick Check (last 20 lines) -```bash -foxhunt-deploy monitor --tail 20 -``` - -### Follow in Real-Time -```bash -foxhunt-deploy monitor --follow -``` - -### Follow with Context -```bash -foxhunt-deploy monitor --follow --tail 10 -``` - -### Filter for Metrics -```bash -foxhunt-deploy monitor --follow --filter "epoch|loss|accuracy" -``` - -### Show Errors Only -```bash -foxhunt-deploy monitor --follow --filter "ERROR|WARN|CRITICAL" -``` - -## Config - -Location: `~/.foxhunt/config.toml` - -```toml -[s3] -endpoint = "https://s3api-eur-is-1.runpod.io" -bucket = "se3zdnb5o4" -region = "us-east-1" -poll_interval_secs = 5 # Decrease for faster updates -``` - -## Colorization - -- 🟢 INFO (green) -- 🟡 WARN (yellow) -- 🔴 ERROR (red) -- 🔵 DEBUG (cyan) -- ⚫ TRACE (dimmed) - -## Troubleshooting - -### No logs found -```bash -# Check credentials -echo $AWS_ACCESS_KEY_ID - -# List logs -foxhunt-deploy monitor --list - -# Verify pod ID in RunPod dashboard -``` - -### Permission denied -```bash -# Check S3 config -cat ~/.foxhunt/config.toml | grep -A 4 "\[s3\]" - -# Verify credentials match RunPod account -``` - -### Slow updates -```bash -# Edit config: reduce poll_interval_secs -nano ~/.foxhunt/config.toml -# Change: poll_interval_secs = 2 - -# Or check network -ping s3api-eur-is-1.runpod.io -``` - -## Tips - -1. **Ctrl+C** to exit follow mode -2. **Completion auto-detected**: Exits when training finishes -3. **Regex patterns**: Use standard regex syntax -4. **Multiple flags**: Combine `--follow --tail 10 --filter "epoch"` -5. **Log priority**: training.log > stdout > stderr (auto-selected) diff --git a/foxhunt-deploy/QUICK_START_BUILD.md b/foxhunt-deploy/QUICK_START_BUILD.md deleted file mode 100644 index b06ade679..000000000 --- a/foxhunt-deploy/QUICK_START_BUILD.md +++ /dev/null @@ -1,302 +0,0 @@ -# Quick Start: Docker Build Command - -## Prerequisites - -1. **Docker installed and running** - ```bash - docker --version - docker info # Verify daemon is running - ``` - -2. **foxhunt-deploy configured** - ```bash - foxhunt-deploy init - # Edit ~/.runpod/config.toml - ``` - -3. **Docker login** (for pushing) - ```bash - docker login - ``` - -## Basic Usage - -### Build and Push (Default) -```bash -foxhunt-deploy build -``` -- Builds: `jgrusewski/foxhunt:latest` (from config) -- Pushes to Docker Hub -- Uses: `Dockerfile.foxhunt-build` - -### Build Only (No Push) -```bash -foxhunt-deploy build --no-push -``` -- Builds locally -- Skips registry push -- Useful for testing - -### Custom Tag -```bash -foxhunt-deploy build --tag v1.0.0 -``` -- Builds: `jgrusewski/foxhunt:v1.0.0` -- Pushes to Docker Hub - -### No Cache Build -```bash -foxhunt-deploy build --no-cache -``` -- Rebuilds all layers -- Ignores Docker cache -- Useful after dependency changes - -### Custom Dockerfile -```bash -foxhunt-deploy build --dockerfile Dockerfile.custom -``` -- Uses different Dockerfile -- Keeps same build context - -### Custom Build Context -```bash -foxhunt-deploy build --context ../parent-dir -``` -- Uses different directory as context -- Useful for monorepos - -## Common Scenarios - -### Development Build -```bash -# Quick local test without pushing -foxhunt-deploy build --tag dev --no-push -``` - -### Production Build -```bash -# Clean build with version tag -foxhunt-deploy build --tag v1.2.3 --no-cache -``` - -### Test Build -```bash -# Build from different Dockerfile without pushing -foxhunt-deploy build --dockerfile Dockerfile.test --tag test --no-push -``` - -### Multi-Platform Build (Future) -```bash -# Not yet implemented, but architecture supports it -# foxhunt-deploy build --platform linux/amd64,linux/arm64 -``` - -## Configuration - -Edit `~/.runpod/config.toml`: - -```toml -[docker] -registry = "jgrusewski" # Your Docker Hub username -image_name = "foxhunt" # Image name -tag = "latest" # Default tag -``` - -**Result**: Full tag = `jgrusewski/foxhunt:latest` - -Override tag with: `--tag your-tag` - -## Troubleshooting - -### Docker Not Found -```bash -✗ Error: Docker is not installed. Please install Docker Desktop or Docker Engine. -``` -**Fix**: Install Docker from https://docker.com - -### Docker Daemon Not Running -```bash -✗ Error: Docker daemon is not running. Please start Docker Desktop or Docker service. -``` -**Fix**: Start Docker Desktop or `sudo systemctl start docker` - -### Dockerfile Not Found -```bash -✗ Error: Dockerfile not found: Dockerfile.foxhunt-build -``` -**Fix**: Create Dockerfile or use `--dockerfile` with correct path - -### Authentication Failed -```bash -✗ Error: Docker registry authentication failed. Please run 'docker login' first. -``` -**Fix**: Run `docker login` and enter credentials - -### Permission Denied -```bash -✗ Error: permission denied while trying to connect to Docker daemon -``` -**Fix**: Add user to docker group: `sudo usermod -aG docker $USER` - -## Environment Variables - -Override config with environment variables: - -```bash -export DOCKER_REGISTRY=myregistry.io -export DOCKER_IMAGE_NAME=my-app -foxhunt-deploy build --tag v1.0.0 -``` - -## Output Examples - -### Successful Build -``` -[1/3] Verifying Docker installation... -ℹ Building Docker image: jgrusewski/foxhunt:latest -ℹ Dockerfile: Dockerfile.foxhunt-build -ℹ Context: . -[2/3] Building Docker image... -⠋ Step 5/12: RUN cargo build --release -✓ Built image: jgrusewski/foxhunt:latest (sha256:abc123...) -[3/3] Pushing to Docker registry... -⠸ Pushing to registry... -✓ Successfully pushed: jgrusewski/foxhunt:latest -✓ Docker build completed successfully! -ℹ Image: jgrusewski/foxhunt:latest -``` - -### Build Failed -``` -[1/3] Verifying Docker installation... -[2/3] Building Docker image... -✗ Error: Docker build failed with exit code 1: -ERROR [5/12] RUN cargo build --release -error: package `serde v1.0.228` cannot be built because it requires... -``` - -### No Push -``` -[1/3] Verifying Docker installation... -[2/3] Building Docker image... -✓ Built image: jgrusewski/foxhunt:dev (sha256:xyz789...) -ℹ Skipping push (--no-push flag set) -✓ Docker build completed successfully! -ℹ Image: jgrusewski/foxhunt:dev -``` - -## Advanced Usage - -### Verbose Logging -```bash -foxhunt-deploy build --verbose -``` -- Shows debug logs -- Useful for troubleshooting - -### Custom Config File -```bash -foxhunt-deploy build --config ./my-config.toml -``` -- Uses different config file -- Useful for multiple environments - -### Combined Options -```bash -foxhunt-deploy build \ - --tag v1.0.0-rc1 \ - --dockerfile Dockerfile.prod \ - --no-cache \ - --verbose -``` - -## Integration with CI/CD - -### GitHub Actions -```yaml -- name: Build Docker Image - run: | - foxhunt-deploy build --tag ${{ github.sha }} -``` - -### GitLab CI -```yaml -build: - script: - - foxhunt-deploy build --tag ${CI_COMMIT_SHA} -``` - -### Jenkins -```groovy -stage('Build') { - steps { - sh 'foxhunt-deploy build --tag ${GIT_COMMIT}' - } -} -``` - -## Performance Tips - -1. **Use BuildKit** (Docker 18.09+) - ```bash - export DOCKER_BUILDKIT=1 - foxhunt-deploy build - ``` - -2. **Leverage Layer Caching** - - Don't use `--no-cache` unless necessary - - Order Dockerfile for optimal caching - - Copy dependency files before source code - -3. **Multi-Stage Builds** - - Use Dockerfile.foxhunt-build pattern - - Separate build and runtime stages - - Keep final image small - -4. **Parallel Builds** - - Build multiple tags in parallel (manual) - ```bash - foxhunt-deploy build --tag v1.0.0 & - foxhunt-deploy build --tag latest & - wait - ``` - -## Next Steps - -After building, you can: - -1. **Deploy to RunPod** - ```bash - foxhunt-deploy deploy --gpu "RTX A4000" - ``` - -2. **Run locally** - ```bash - docker run -it jgrusewski/foxhunt:latest bash - ``` - -3. **Inspect image** - ```bash - docker images jgrusewski/foxhunt - docker history jgrusewski/foxhunt:latest - ``` - -4. **Test image** - ```bash - docker run --rm jgrusewski/foxhunt:latest --version - ``` - -## Help - -```bash -foxhunt-deploy build --help -foxhunt-deploy --help -``` - -## Support - -- 📖 Documentation: `DOCKER_BUILD_MILESTONE2.md` -- 📋 Summary: `IMPLEMENTATION_SUMMARY.md` -- 🐛 Issues: Check stderr output with `--verbose` -- 💬 Questions: Review help text and config file diff --git a/foxhunt-deploy/README.md b/foxhunt-deploy/README.md deleted file mode 100644 index 2f6d7d6f2..000000000 --- a/foxhunt-deploy/README.md +++ /dev/null @@ -1,420 +0,0 @@ -# foxhunt-deploy - -Rust CLI tool for managing Foxhunt ML training deployments on RunPod GPU infrastructure. - -## Features - -- 🐳 **Docker Management**: Build and push Docker images to registry -- ☁️ **RunPod Deployment**: Deploy training jobs to GPU pods with one command -- 📊 **S3 Log Monitoring**: Stream real-time logs from RunPod S3 buckets -- 🎨 **Colorized Output**: Easy-to-read log levels (INFO, WARN, ERROR) -- 🔍 **Filtering**: Regex-based log filtering -- ⚙️ **Configuration**: Flexible TOML-based configuration - -## Installation - -```bash -cd foxhunt-deploy -cargo build --release - -# Copy to PATH -sudo cp target/release/foxhunt-deploy /usr/local/bin/ -``` - -## Quick Start - -### 1. Initialize Configuration - -```bash -foxhunt-deploy init -``` - -This creates `~/.foxhunt/config.toml` with default settings. - -### 2. Configure Credentials - -Edit `~/.foxhunt/config.toml`: - -```toml -[runpod] -api_key = "YOUR_RUNPOD_API_KEY" -default_gpu_type = "RTX A4000" -default_datacenter = "EUR-IS-1" - -[docker] -registry = "jgrusewski" -image_name = "foxhunt" -tag = "latest" - -[s3] -endpoint = "https://s3api-eur-is-1.runpod.io" -bucket = "se3zdnb5o4" -region = "us-east-1" -poll_interval_secs = 5 -``` - -Set S3 credentials: - -```bash -export AWS_ACCESS_KEY_ID="your_runpod_access_key" -export AWS_SECRET_ACCESS_KEY="your_runpod_secret_key" -``` - -### 3. Deploy a Training Job - -```bash -foxhunt-deploy deploy \ - --command "train_ppo --epochs 100" \ - --gpu "RTX A4000" \ - --datacenter "EUR-IS-1" -``` - -### 4. Monitor Logs - -```bash -# List available log files -foxhunt-deploy monitor --list - -# Show last 50 lines -foxhunt-deploy monitor --tail 50 - -# Follow logs in real-time -foxhunt-deploy monitor --follow - -# Filter for specific patterns -foxhunt-deploy monitor --follow --filter "epoch|loss" -``` - -## Commands - -### `init` - -Initialize configuration file. - -```bash -foxhunt-deploy init -``` - -Creates `~/.foxhunt/config.toml` with default settings. - -### `build` - -Build Docker image. - -```bash -foxhunt-deploy build [OPTIONS] - -Options: - --tag Docker image tag [default: latest] - --push Push to registry after building - --dockerfile Path to Dockerfile [default: Dockerfile.foxhunt-build] -``` - -**Example**: - -```bash -# Build and push -foxhunt-deploy build --tag v1.0.0 --push -``` - -### `deploy` - -Deploy training job to RunPod GPU. - -```bash -foxhunt-deploy deploy [OPTIONS] --command - -Required: - --command Training command to run - -Options: - --name Pod name (auto-generated if not provided) - --gpu GPU type [default: RTX A4000] - --datacenter Datacenter [default: EUR-IS-1] - --tag Docker image tag [default: latest] - --env Environment variables (can be repeated) - --volume-size Network volume size in GB [default: 100] -``` - -**Examples**: - -```bash -# Deploy PPO training -foxhunt-deploy deploy \ - --command "train_ppo --epochs 100" \ - --gpu "RTX A4000" - -# Deploy with custom env vars -foxhunt-deploy deploy \ - --command "train_dqn --data /data/ES_FUT.parquet" \ - --env "RUST_LOG=debug" \ - --env "CUDA_VISIBLE_DEVICES=0" - -# Deploy to specific datacenter -foxhunt-deploy deploy \ - --command "train_tft --epochs 50" \ - --datacenter "US-TX-1" \ - --gpu "RTX 4090" -``` - -### `monitor` - -Monitor pod logs from S3. - -```bash -foxhunt-deploy monitor [OPTIONS] - -Arguments: - Pod ID to monitor - -Options: - -f, --follow Follow logs in real-time - -t, --tail Number of recent lines to show - -l, --list List available log files - --filter Filter logs by regex pattern -``` - -**Examples**: - -```bash -# List available log files -foxhunt-deploy monitor dy2bn5ninzaxma --list - -# Show last 20 lines -foxhunt-deploy monitor dy2bn5ninzaxma --tail 20 - -# Follow logs in real-time -foxhunt-deploy monitor dy2bn5ninzaxma --follow - -# Follow with initial tail (show last 10 lines, then follow) -foxhunt-deploy monitor dy2bn5ninzaxma --follow --tail 10 - -# Filter for training metrics -foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "epoch.*loss" - -# Filter for errors only -foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "ERROR|WARN" -``` - -### `run` - -Execute commands in a running pod. - -```bash -foxhunt-deploy run - -Arguments: - Pod ID - Command to execute -``` - -**Example**: - -```bash -foxhunt-deploy run dy2bn5ninzaxma "nvidia-smi" -``` - -## Configuration - -### Config File - -Location: `~/.foxhunt/config.toml` - -```toml -[runpod] -api_key = "YOUR_API_KEY" -default_gpu_type = "RTX A4000" -default_datacenter = "EUR-IS-1" - -[docker] -registry = "jgrusewski" -image_name = "foxhunt" -tag = "latest" - -[s3] -endpoint = "https://s3api-eur-is-1.runpod.io" -bucket = "se3zdnb5o4" -region = "us-east-1" -poll_interval_secs = 5 - -[defaults] -container_disk_gb = 50 -volume_path = "/runpod-volume" -volume_size_gb = 100 -support_public_ip = false -``` - -### Environment Variables - -Override config values with environment variables: - -```bash -# RunPod -export RUNPOD_API_KEY="your_key" - -# S3 (required for monitor command) -export AWS_ACCESS_KEY_ID="your_access_key" -export AWS_SECRET_ACCESS_KEY="your_secret_key" - -# Optional overrides -export S3_ENDPOINT="https://s3api-eur-is-1.runpod.io" -export S3_BUCKET="se3zdnb5o4" -export S3_POLL_INTERVAL="10" -``` - -### GPU Types - -Common GPU options: - -- `RTX A4000` - 16GB VRAM, $0.25/hr (recommended) -- `RTX 4090` - 24GB VRAM, $0.59/hr -- `A40` - 48GB VRAM, $0.79/hr -- `A100 80GB` - 80GB VRAM, $1.89/hr - -### Datacenters - -Available datacenters: - -- `EUR-IS-1` - Europe (Iceland) -- `US-TX-1` - US (Texas) -- `US-OR-1` - US (Oregon) - -## S3 Log Monitoring - -### How It Works - -1. **Log Discovery**: Automatically finds log files in S3 bucket - - Priority: `training.log` > `stdout` > `stderr` - - Path: `s3://{bucket}/ml_training/{pod_id}/...` - -2. **Real-Time Streaming**: Polls S3 every 5 seconds (configurable) - - Tracks last read position - - Only downloads new content (byte-range requests) - -3. **Colorization**: Applies colors based on log level - - 🟢 INFO (green) - - 🟡 WARN (yellow) - - 🔴 ERROR (red) - - 🔵 DEBUG (cyan) - - ⚫ TRACE (dimmed) - -4. **Completion Detection**: Exits when training completes - - Detects keywords: "training complete", "saved final model", etc. - -### S3 Path Structure - -``` -s3://se3zdnb5o4/ml_training/ -└── {pod_id}/ - ├── training_runs/ - │ └── {model}/ - │ └── run_{timestamp}/ - │ ├── logs/ - │ │ └── training.log (priority 1) - │ └── checkpoints/ - ├── stdout (priority 2) - └── stderr (priority 3) -``` - -### Performance - -- **Poll Interval**: 5 seconds (configurable) -- **Latency**: ~5-10s lag -- **Network**: Byte-range downloads (efficient) -- **Memory**: Streams chunks, doesn't load entire file - -## Examples - -See `examples/monitor_demo.sh` for interactive demo: - -```bash -./examples/monitor_demo.sh -``` - -## Development - -### Build - -```bash -cargo build --release -``` - -### Run Tests - -```bash -cargo test -``` - -### Run with Verbose Logging - -```bash -foxhunt-deploy --verbose monitor --follow -``` - -## Troubleshooting - -### Monitor: No logs appearing - -**Cause**: Missing AWS credentials or incorrect pod ID - -**Solution**: -1. Check credentials: `echo $AWS_ACCESS_KEY_ID` -2. List log files: `foxhunt-deploy monitor --list` -3. Verify pod ID in RunPod dashboard - -### Monitor: Permission denied - -**Cause**: Invalid AWS credentials - -**Solution**: -1. Verify RunPod API key has S3 access -2. Check bucket name in config matches RunPod -3. Ensure credentials match RunPod account - -### Monitor: Slow updates - -**Cause**: High poll interval or network latency - -**Solution**: -1. Reduce poll interval in config: `poll_interval_secs = 2` -2. Check network: `ping s3api-eur-is-1.runpod.io` - -### Deploy: GPU not available - -**Cause**: Requested GPU type unavailable in datacenter - -**Solution**: -1. Try different datacenter: `--datacenter "US-TX-1"` -2. Try different GPU: `--gpu "RTX 4090"` -3. Check RunPod dashboard for availability - -## Architecture - -``` -foxhunt-deploy -├── src/ -│ ├── cli/ # Command-line interface -│ │ ├── build.rs # Docker build command -│ │ ├── deploy.rs # RunPod deployment command -│ │ ├── monitor.rs # S3 log monitoring command -│ │ └── run.rs # Pod execution command -│ ├── config/ # Configuration management -│ ├── docker/ # Docker operations -│ ├── runpod/ # RunPod API client -│ ├── s3/ # S3 log monitoring -│ │ ├── mod.rs # S3LogClient -│ │ ├── monitor.rs # LogMonitor -│ │ └── parser.rs # Log parsing/colorization -│ ├── utils/ # Utilities (terminal output) -│ └── error.rs # Error handling -└── examples/ - └── monitor_demo.sh # Interactive demo -``` - -## License - -Apache-2.0 - -## Credits - -Built for the Foxhunt HFT trading system. diff --git a/foxhunt-deploy/RUNPOD_DEPLOYMENT_IMPLEMENTATION.md b/foxhunt-deploy/RUNPOD_DEPLOYMENT_IMPLEMENTATION.md deleted file mode 100644 index 2d06a2900..000000000 --- a/foxhunt-deploy/RUNPOD_DEPLOYMENT_IMPLEMENTATION.md +++ /dev/null @@ -1,359 +0,0 @@ -# RunPod Deployment Implementation - Milestone 3 - -## Overview - -This document describes the implementation of the RunPod deployment functionality for the `foxhunt-deploy` CLI tool. The implementation enables automated deployment of ML training workloads to RunPod GPU infrastructure via REST API integration. - -## Architecture - -### Module Structure - -``` -src/runpod/ -├── mod.rs # Module exports and public API -├── types.rs # Serde types for RunPod REST API -├── client.rs # RunPodClient with HTTP operations -└── deployment.rs # Deployment logic and helpers -``` - -### Key Components - -#### 1. RunPod API Types (`types.rs`) - -Defines serde-compatible types for RunPod REST API communication: - -- **`PodDeploymentRequest`**: Complete deployment payload - - Cloud type, compute type, datacenter preferences - - GPU configuration (type, count, VRAM) - - Docker image and startup command - - Volume mounts and network configuration - - Environment variables - -- **`PodDeploymentResponse`**: Deployment result - - Pod ID (for monitoring and termination) - - Machine info (GPU type, VRAM) - - Runtime info (datacenter, cost) - -- **`GpuType`**: Available GPU information - - Display name, VRAM capacity - - Pricing (secure cloud vs community cloud) - -#### 2. RunPod Client (`client.rs`) - -HTTP client for RunPod REST API operations: - -```rust -impl RunPodClient { - pub fn new(api_key: String) -> Result - pub async fn list_gpus(&self) -> Result> - pub async fn deploy_pod(&self, request: PodDeploymentRequest) -> Result - pub async fn terminate_pod(&self, pod_id: &str) -> Result<()> - pub async fn get_pod(&self, pod_id: &str) -> Result -} -``` - -**API Endpoints**: -- Base URL: `https://rest.runpod.io/v1` -- Authentication: Bearer token (`Authorization: Bearer `) -- Content-Type: `application/json` - -**Error Handling**: -- HTTP status validation -- API error response parsing -- User-friendly error messages - -#### 3. Deployment Logic (`deployment.rs`) - -Helper functions for deployment operations: - -```rust -pub fn build_deployment_request( - args: &DeployArgs, - config: &FoxhuntConfig, - gpu: &GpuType, -) -> Result - -pub fn select_best_gpu( - gpus: &[GpuType], - preferred_type: Option<&str> -) -> Result - -pub fn validate_deployment( - request: &PodDeploymentRequest -) -> Result<()> -``` - -**GPU Selection Logic**: -1. User-specified GPU type (exact or fuzzy match) -2. Auto-select RTX A4000 (best value: $0.25/hr) -3. Fallback to cheapest available GPU - -**Request Builder**: -- Parses command string into Docker start command array -- Builds environment variable map from CLI args -- Generates pod name with timestamp if not provided -- Applies configuration defaults for volumes, disks, etc. - -#### 4. Deploy Command (`cli/deploy.rs`) - -User-facing CLI command implementation: - -```bash -foxhunt-deploy deploy \ - --command "train_ppo --epochs 100" \ - --gpu-type "RTX A4000" \ - --datacenter "EUR-IS-1" \ - --env "CUDA_VISIBLE_DEVICES=0" \ - --dry-run -``` - -**Features**: -- Interactive deployment confirmation with cost estimates -- Dry-run mode for validation without deployment -- Deployment summary display (GPU, cost, datacenter, image, command) -- Pod ID persistence (saved to `.last_pod_id` for reference) -- Next-step guidance (monitoring and termination commands) - -## Implementation Details - -### API Request Format - -Example deployment request to RunPod REST API: - -```json -POST https://rest.runpod.io/v1/pods -Authorization: Bearer -Content-Type: application/json - -{ - "cloudType": "SECURE", - "computeType": "GPU", - "dataCenterIds": ["EUR-IS-1"], - "gpuTypeIds": ["NVIDIA RTX A4000"], - "gpuCount": 1, - "name": "foxhunt-20251102-091802", - "imageName": "jgrusewski/foxhunt:latest", - "containerDiskInGb": 50, - "networkVolumeId": "se3zdnb5o4", - "volumeMountPath": "/runpod-volume", - "dockerStartCmd": ["train_ppo", "--epochs", "100"], - "containerRegistryAuthId": "cmh3ya1710001jo02vwqtisbf", - "ports": ["8888/http", "22/tcp"], - "interruptible": false, - "env": { - "CUDA_VISIBLE_DEVICES": "0", - "RUST_LOG": "debug" - }, - "supportPublicIp": false -} -``` - -### Configuration - -The deployment uses settings from `foxhunt.toml`: - -```toml -[runpod] -api_key = "YOUR_RUNPOD_API_KEY" -default_gpu_type = "RTX A4000" -default_datacenter = "EUR-IS-1" - -[docker] -registry = "jgrusewski" -image_name = "foxhunt" -tag = "latest" - -[defaults] -container_disk_gb = 50 -volume_path = "/runpod-volume" -volume_size_gb = 100 -support_public_ip = false -``` - -### Error Handling - -Comprehensive error handling for: - -1. **Configuration Errors**: - - Missing or invalid API key - - Invalid GPU type selection - -2. **API Errors**: - - HTTP errors (network, timeout) - - RunPod API errors (GPU not available, quota exceeded) - - JSON parsing errors - -3. **Validation Errors**: - - Invalid command format - - Invalid environment variable format - - Invalid GPU count or disk size - -### Security - -- API key validation before making requests -- API key stored in config file (gitignored) -- Support for environment variable overrides -- No hardcoded credentials - -## Usage Examples - -### Basic Deployment - -Auto-select GPU and deploy with default settings: - -```bash -foxhunt-deploy deploy --command "train_ppo --epochs 100" -``` - -### Custom GPU Selection - -Specify GPU type explicitly: - -```bash -foxhunt-deploy deploy \ - --command "hyperopt_dqn_demo --n-trials 50" \ - --gpu-type "RTX 4090" -``` - -### Full Configuration - -All available options: - -```bash -foxhunt-deploy deploy \ - --command "train_ppo --epochs 100" \ - --gpu-type "RTX A4000" \ - --datacenter "EUR-IS-1" \ - --tag "v1.2.3" \ - --name "ppo-production-run" \ - --env "CUDA_VISIBLE_DEVICES=0" \ - --env "RUST_LOG=debug" \ - --volume-size 150 -``` - -### Dry Run Mode - -Validate deployment request without actually deploying: - -```bash -foxhunt-deploy deploy \ - --command "train_ppo --epochs 100" \ - --dry-run -``` - -Output: -``` -ℹ Starting RunPod deployment... -✓ Selected GPU: RTX A4000 (16 GB VRAM, $0.25/hr) - -ℹ [DRY RUN] Deployment Summary -──────────────────────────────────────────────────────────── - Pod Name: foxhunt-20251102-091802 - GPU: RTX A4000 - VRAM: 16 GB - Cost: $0.25/hr - Datacenter: EUR-IS-1 - Image: jgrusewski/foxhunt:latest - Command: train_ppo --epochs 100 - Container Disk: 50 GB -──────────────────────────────────────────────────────────── - -✓ Dry run completed successfully. Request is valid. -``` - -## Testing - -### Unit Tests - -Comprehensive test coverage in `deployment.rs`: - -```rust -#[test] -fn test_parse_command() // Command parsing -fn test_parse_command_empty() // Error handling -fn test_select_best_gpu_preferred() // User preference -fn test_select_best_gpu_auto() // Auto-selection -fn test_validate_deployment() // Valid request -fn test_validate_deployment_invalid_gpu_count() // Invalid request -``` - -**Test Results**: -``` -running 19 tests -test runpod::deployment::tests::test_parse_command ... ok -test runpod::deployment::tests::test_parse_command_empty ... ok -test runpod::deployment::tests::test_select_best_gpu_auto ... ok -test runpod::deployment::tests::test_select_best_gpu_preferred ... ok -test runpod::deployment::tests::test_validate_deployment ... ok -test runpod::deployment::tests::test_validate_deployment_invalid_gpu_count ... ok - -test result: ok. 19 passed; 0 failed; 0 ignored -``` - -### Integration Testing - -Test with dry-run mode: - -```bash -# Test auto GPU selection -foxhunt-deploy deploy --command "train_ppo" --dry-run - -# Test custom GPU -foxhunt-deploy deploy --command "train_ppo" --gpu-type "RTX 4090" --dry-run - -# Test environment variables -foxhunt-deploy deploy --command "train_ppo" --env "KEY=VALUE" --dry-run -``` - -## Performance - -- **API Response Time**: <500ms for deployment requests -- **Validation**: <10ms for local validation -- **Binary Size**: ~14MB (release build, optimized) -- **Memory**: <20MB for typical operations - -## Future Enhancements - -1. **GraphQL API Integration**: - - Replace hardcoded GPU list with real-time availability query - - Support more GPU types dynamically - -2. **Pod Status Monitoring**: - - Real-time pod status updates - - Automatic retry on transient failures - -3. **Cost Optimization**: - - Interruptible instance support - - Multi-datacenter price comparison - - Automatic spot instance selection - -4. **Advanced Features**: - - Multi-GPU deployment support - - Custom network volumes - - SSH key injection - - Pod templates and presets - -## References - -- **RunPod REST API**: https://docs.runpod.io/api/rest -- **RunPod GraphQL API**: https://docs.runpod.io/api/graphql -- **Foxhunt Docker Image**: jgrusewski/foxhunt:latest -- **Private Registry Auth**: cmh3ya1710001jo02vwqtisbf - -## Completion Status - -- ✅ RunPod module structure created -- ✅ API types implemented with serde -- ✅ RunPod client with async HTTP operations -- ✅ Deployment logic with GPU selection -- ✅ Deploy command with interactive confirmation -- ✅ Dry-run mode for validation -- ✅ Unit tests (6/6 passing) -- ✅ Integration tests (manual validation) -- ✅ Documentation and examples - -**Milestone 3: COMPLETE** ✅ - -**Time Spent**: ~55 minutes -**Code Quality**: Production-ready with comprehensive error handling -**Test Coverage**: 100% for deployment logic diff --git a/foxhunt-deploy/S3_MONITOR_IMPLEMENTATION.md b/foxhunt-deploy/S3_MONITOR_IMPLEMENTATION.md deleted file mode 100644 index c6496c1db..000000000 --- a/foxhunt-deploy/S3_MONITOR_IMPLEMENTATION.md +++ /dev/null @@ -1,398 +0,0 @@ -# S3 Log Monitoring Implementation - -**Status**: ✅ COMPLETE -**Date**: 2025-11-02 -**Milestone**: 4 - Monitor Subcommand - -## Overview - -Implemented real-time S3 log monitoring functionality for the foxhunt-deploy CLI. This allows users to stream training logs from RunPod S3 buckets in real-time with colorized output, filtering, and automatic completion detection. - -## Architecture - -### Module Structure - -``` -src/s3/ -├── mod.rs # S3LogClient - AWS SDK wrapper -├── monitor.rs # LogMonitor - Log streaming logic -└── parser.rs # Log parsing and colorization utilities -``` - -### Components - -#### 1. S3LogClient (`src/s3/mod.rs`) - -AWS SDK wrapper with custom RunPod endpoint support: - -```rust -pub struct S3LogClient { - client: aws_sdk_s3::Client, - bucket: String, - endpoint: String, -} -``` - -**Methods**: -- `new(config: &S3Config)` - Create client with RunPod credentials -- `list_log_files(pod_id: &str)` - List all log files for a pod -- `download_log(path: &str)` - Download complete log file -- `download_log_range(path, start, end)` - Download byte range (for tailing) -- `get_log_size(path: &str)` - Get current file size -- `log_exists(path: &str)` - Check if log file exists - -**Configuration**: -- Endpoint: `https://s3api-eur-is-1.runpod.io` -- Bucket: `se3zdnb5o4` -- Region: `us-east-1` -- Credentials: From `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` env vars - -#### 2. LogMonitor (`src/s3/monitor.rs`) - -Real-time log streaming with polling: - -```rust -pub struct LogMonitor { - client: S3LogClient, - pod_id: String, - poll_interval: Duration, -} -``` - -**Methods**: -- `new(client, pod_id, poll_interval_secs)` - Create monitor -- `tail_logs(tail, filter)` - Stream logs in real-time (follow mode) -- `show_recent_logs(tail)` - Show recent logs (snapshot mode) -- `list_logs()` - List all available log files - -**Features**: -- ✅ Automatic log file detection (priority: training.log > stdout > stderr) -- ✅ Real-time streaming with configurable poll interval (default: 5s) -- ✅ Tail mode: show last N lines before following -- ✅ Regex filtering -- ✅ Automatic completion detection -- ✅ Graceful error handling with retries (max 3) -- ✅ Tracks last read position to avoid re-reading - -#### 3. LogParser (`src/s3/parser.rs`) - -Log parsing and colorization utilities: - -```rust -pub enum LogLevel { - Info, // Green - Warn, // Yellow - Error, // Red - Debug, // Cyan - Trace, // Dimmed - Unknown, // Default -} -``` - -**Functions**: -- `colorize_log_line(line)` - Apply colors based on log level -- `parse_training_metrics(line)` - Extract epoch, loss, LR -- `detect_completion(line)` - Identify training completion -- `filter_line(line, pattern)` - Regex filtering -- `get_last_n_lines(content, n)` - Get tail lines - -## Usage - -### Prerequisites - -1. **AWS Credentials**: Set environment variables - ```bash - export AWS_ACCESS_KEY_ID="your_runpod_access_key" - export AWS_SECRET_ACCESS_KEY="your_runpod_secret_key" - ``` - -2. **Configuration**: Create or update `~/.foxhunt/config.toml` - ```toml - [s3] - endpoint = "https://s3api-eur-is-1.runpod.io" - bucket = "se3zdnb5o4" - region = "us-east-1" - poll_interval_secs = 5 - ``` - -### Examples - -#### 1. Follow Logs in Real-Time - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --follow -``` - -**Output**: -``` -Monitoring pod: dy2bn5ninzaxma -Poll interval: 5s -──────────────────────────────────────────────────────────────────────────────── -Found log file: ml_training/dy2bn5ninzaxma/training_runs/ppo/run_20251102/logs/training.log -──────────────────────────────────────────────────────────────────────────────── -INFO: Starting training... [green] -INFO: Epoch: 1, Loss: 0.345 [green] -WARN: Learning rate decayed [yellow] -INFO: Epoch: 2, Loss: 0.298 [green] -ERROR: CUDA OOM detected [red] -... -──────────────────────────────────────────────────────────────────────────────── -✓ Training completed! [green] -``` - -#### 2. Show Last 50 Lines - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --tail 50 -``` - -#### 3. Follow with Initial Tail - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --follow --tail 20 -``` - -#### 4. Filter by Pattern - -```bash -# Show only lines containing "epoch" or "loss" -foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "epoch.*loss" - -# Show only errors -foxhunt-deploy monitor dy2bn5ninzaxma --follow --filter "ERROR|WARN" -``` - -#### 5. List Available Log Files - -```bash -foxhunt-deploy monitor dy2bn5ninzaxma --list -``` - -**Output**: -``` -Searching for logs for pod: dy2bn5ninzaxma -──────────────────────────────────────────────────────────────────────────────── -3 log file(s): - • ml_training/dy2bn5ninzaxma/training_runs/ppo/run_20251102/logs/training.log - • ml_training/dy2bn5ninzaxma/stdout - • ml_training/dy2bn5ninzaxma/stderr -``` - -## S3 Path Structure - -``` -s3://se3zdnb5o4/ml_training/ -├── {pod_id}/ -│ ├── training_runs/ -│ │ ├── {model}/ -│ │ │ ├── run_{timestamp}/ -│ │ │ │ ├── logs/ -│ │ │ │ │ ├── training.log (priority 1) -│ │ │ │ ├── checkpoints/ -│ ├── stdout (priority 2) -│ └── stderr (priority 3) -``` - -## Features - -### ✅ Implemented - -1. **Real-Time Streaming** - - Polls S3 every 5 seconds (configurable) - - Tracks last read position - - Only downloads new content (byte-range requests) - -2. **Colorized Output** - - Green: INFO - - Yellow: WARN - - Red: ERROR - - Cyan: DEBUG - - Dimmed: TRACE - -3. **Filtering** - - Regex pattern matching - - Filter applied before colorization - -4. **Tail Mode** - - Show last N lines before following - - Works in both follow and snapshot modes - -5. **Completion Detection** - - Detects training completion keywords - - Exits gracefully when training completes - -6. **Error Handling** - - Graceful retries (max 3) - - Network error recovery - - Missing log file handling - -7. **Multiple Log Files** - - Auto-selects primary log file - - Prioritizes training.log > stdout > stderr - - List mode shows all available logs - -### Performance - -- **Poll Interval**: 5 seconds (configurable) -- **Network Efficiency**: Byte-range downloads (only new content) -- **Memory**: Streams chunks, doesn't load entire file -- **Latency**: ~5-10s lag (poll interval + S3 latency) - -## Configuration - -### Default Settings - -```toml -[s3] -endpoint = "https://s3api-eur-is-1.runpod.io" -bucket = "se3zdnb5o4" -region = "us-east-1" -poll_interval_secs = 5 -``` - -### Environment Variables - -```bash -# Required -export AWS_ACCESS_KEY_ID="your_access_key" -export AWS_SECRET_ACCESS_KEY="your_secret_key" - -# Optional (override config) -export S3_ENDPOINT="https://s3api-eur-is-1.runpod.io" -export S3_BUCKET="se3zdnb5o4" -export S3_POLL_INTERVAL="10" # seconds -``` - -## Dependencies Added - -```toml -[dependencies] -aws-config = "1.5" -aws-sdk-s3 = "1.50" -regex = "1.10" -colored = "2.1" -``` - -## Error Handling - -### Network Errors - -```rust -Error: S3 error: Failed to get object: network timeout -(retrying 1/3...) -``` - -- Retries up to 3 times -- 5-second delay between retries -- Fails gracefully if all retries exhausted - -### Missing Log Files - -``` -No log files found yet (retrying in 5s...) -``` - -- Polls until log file appears -- Useful for newly created pods -- Ctrl+C to exit - -### Invalid Credentials - -``` -Error: Configuration error: AWS_ACCESS_KEY_ID not set -``` - -- Clear error messages -- Points to missing env vars - -## Testing - -### Unit Tests - -```bash -cargo test --package foxhunt-deploy s3 -``` - -**Tests**: -- `test_detect_log_level` - Log level detection -- `test_parse_training_metrics` - Metric extraction -- `test_detect_completion` - Completion detection -- `test_get_last_n_lines` - Tail functionality - -### Integration Test (Manual) - -1. Deploy a pod with training job -2. Monitor logs: - ```bash - foxhunt-deploy monitor --follow - ``` -3. Verify: - - ✅ Logs appear in real-time - - ✅ Colors applied correctly - - ✅ Completion detected - - ✅ Graceful exit - -## Known Limitations - -1. **Poll Delay**: 5-10s lag due to polling (not true streaming) -2. **S3 Costs**: Frequent HEAD/GET requests (minimal, ~$0.01/month) -3. **Regex Filtering**: Invalid regex shows all lines (no error) -4. **Byte Range**: S3 must support byte-range requests (RunPod does) - -## Future Enhancements - -### Potential Improvements - -1. **WebSocket Support**: True real-time streaming (if RunPod adds support) -2. **Log Aggregation**: Merge stdout + stderr + training.log -3. **Metrics Dashboard**: Live plot of loss/epoch curves -4. **Download Mode**: Save logs to local file -5. **Multi-Pod Monitoring**: Monitor multiple pods simultaneously -6. **Compression**: Support .gz log files -7. **Pagination**: For very large log files - -### Not Implemented (Out of Scope) - -- ❌ SSH log streaming (requires SSH keys) -- ❌ RunPod CLI integration (use their API) -- ❌ Log search/indexing (use grep/less locally) - -## Troubleshooting - -### Problem: No logs appearing - -**Solution**: -1. Check AWS credentials: `echo $AWS_ACCESS_KEY_ID` -2. Verify pod ID: `foxhunt-deploy monitor --list` -3. Check S3 bucket: `aws s3 ls s3://se3zdnb5o4/ --profile runpod` - -### Problem: Permission denied - -**Solution**: -1. Verify RunPod API key has S3 access -2. Check bucket name in config -3. Ensure credentials match RunPod account - -### Problem: Slow updates - -**Solution**: -1. Reduce poll interval: `poll_interval_secs = 2` (in config) -2. Check network latency: `ping s3api-eur-is-1.runpod.io` - -## Summary - -Successfully implemented S3 log monitoring with: -- ✅ Real-time streaming (5s poll interval) -- ✅ Colorized output (5 log levels) -- ✅ Regex filtering -- ✅ Tail mode -- ✅ Completion detection -- ✅ Graceful error handling -- ✅ Production-ready code (tested, documented) - -**Time**: ~45 minutes -**Lines of Code**: ~700 (3 files) -**Dependencies**: 3 added (aws-config, aws-sdk-s3, regex) -**Tests**: 4 unit tests - -The implementation provides a solid foundation for monitoring RunPod training jobs via S3 logs, with room for future enhancements. diff --git a/foxhunt-deploy/examples/monitor_demo.sh b/foxhunt-deploy/examples/monitor_demo.sh deleted file mode 100755 index b8017360a..000000000 --- a/foxhunt-deploy/examples/monitor_demo.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# S3 Log Monitor Demo - Usage examples for foxhunt-deploy monitor command - -set -e - -echo "=== S3 Log Monitor Demo ===" -echo "" - -# Check prerequisites -if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then - echo "ERROR: AWS_ACCESS_KEY_ID not set" - echo "Please set RunPod S3 credentials:" - echo " export AWS_ACCESS_KEY_ID='your_key'" - echo " export AWS_SECRET_ACCESS_KEY='your_secret'" - exit 1 -fi - -# Example pod ID (replace with your actual pod ID) -POD_ID="${1:-dy2bn5ninzaxma}" - -echo "Pod ID: $POD_ID" -echo "" - -# Function to run example -run_example() { - local description="$1" - local command="$2" - - echo "────────────────────────────────────────────────────────────────" - echo "Example: $description" - echo "Command: $command" - echo "────────────────────────────────────────────────────────────────" - echo "" - echo "Press Enter to run (or Ctrl+C to skip)..." - read -r - - eval "$command" - echo "" -} - -# Example 1: List available log files -run_example \ - "List all log files for the pod" \ - "foxhunt-deploy monitor $POD_ID --list" - -# Example 2: Show last 20 lines -run_example \ - "Show last 20 lines of logs" \ - "foxhunt-deploy monitor $POD_ID --tail 20" - -# Example 3: Show last 50 lines -run_example \ - "Show last 50 lines of logs" \ - "foxhunt-deploy monitor $POD_ID --tail 50" - -# Example 4: Follow logs in real-time -echo "────────────────────────────────────────────────────────────────" -echo "Example: Follow logs in real-time (Ctrl+C to exit)" -echo "Command: foxhunt-deploy monitor $POD_ID --follow" -echo "────────────────────────────────────────────────────────────────" -echo "" -echo "This will stream logs continuously. Press Ctrl+C to stop." -echo "Press Enter to start..." -read -r - -foxhunt-deploy monitor "$POD_ID" --follow - -# Example 5: Follow with initial tail -echo "" -echo "────────────────────────────────────────────────────────────────" -echo "Example: Follow logs with initial 10 lines" -echo "Command: foxhunt-deploy monitor $POD_ID --follow --tail 10" -echo "────────────────────────────────────────────────────────────────" -echo "" -echo "Press Enter to start (Ctrl+C to stop)..." -read -r - -foxhunt-deploy monitor "$POD_ID" --follow --tail 10 - -# Example 6: Filter by pattern -echo "" -echo "────────────────────────────────────────────────────────────────" -echo "Example: Filter logs for 'epoch' or 'loss'" -echo "Command: foxhunt-deploy monitor $POD_ID --follow --filter 'epoch|loss'" -echo "────────────────────────────────────────────────────────────────" -echo "" -echo "Press Enter to start (Ctrl+C to stop)..." -read -r - -foxhunt-deploy monitor "$POD_ID" --follow --filter "epoch|loss" - -# Example 7: Filter for errors only -echo "" -echo "────────────────────────────────────────────────────────────────" -echo "Example: Show only errors and warnings" -echo "Command: foxhunt-deploy monitor $POD_ID --follow --filter 'ERROR|WARN'" -echo "────────────────────────────────────────────────────────────────" -echo "" -echo "Press Enter to start (Ctrl+C to stop)..." -read -r - -foxhunt-deploy monitor "$POD_ID" --follow --filter "ERROR|WARN" - -echo "" -echo "=== Demo Complete ===" -echo "" -echo "Common usage patterns:" -echo "" -echo "1. Quick check: Show last 20 lines" -echo " foxhunt-deploy monitor $POD_ID --tail 20" -echo "" -echo "2. Monitor training: Follow in real-time" -echo " foxhunt-deploy monitor $POD_ID --follow" -echo "" -echo "3. Debug errors: Filter for problems" -echo " foxhunt-deploy monitor $POD_ID --follow --filter 'ERROR|WARN|CRITICAL'" -echo "" -echo "4. Track metrics: Filter for training stats" -echo " foxhunt-deploy monitor $POD_ID --follow --filter 'epoch.*loss|accuracy'" -echo "" -echo "5. List logs: See all available log files" -echo " foxhunt-deploy monitor $POD_ID --list" -echo "" diff --git a/foxhunt-deploy/examples/sample_config.toml b/foxhunt-deploy/examples/sample_config.toml deleted file mode 100644 index dc27edf82..000000000 --- a/foxhunt-deploy/examples/sample_config.toml +++ /dev/null @@ -1,50 +0,0 @@ -# Foxhunt RunPod Deployment Configuration -# Edit this file and save to ~/.runpod/config.toml -# You can override any value with environment variables (see .env.runpod example) - -[runpod] -# Your RunPod API key (REQUIRED) -# Get it from: https://www.runpod.io/console/user/settings -api_key = "YOUR_RUNPOD_API_KEY_HERE" - -# Default GPU type for deployments -default_gpu_type = "RTX A4000" - -# Default datacenter region -default_datacenter = "EUR-IS-1" - -[docker] -# Docker registry username -registry = "jgrusewski" - -# Docker image name -image_name = "foxhunt" - -# Default image tag -tag = "latest" - -[s3] -# RunPod S3 endpoint (region-specific) -endpoint = "https://s3api-eur-is-1.runpod.io" - -# S3 bucket for training outputs -bucket = "se3zdnb5o4" - -# AWS region for S3 -region = "us-east-1" - -# Log polling interval in seconds -poll_interval_secs = 5 - -[defaults] -# Container disk size in GB -container_disk_gb = 50 - -# Volume mount path inside container -volume_path = "/runpod-volume" - -# Volume size in GB -volume_size_gb = 100 - -# Whether to enable public IP -support_public_ip = false diff --git a/foxhunt-deploy/src/cli/build.rs b/foxhunt-deploy/src/cli/build.rs deleted file mode 100644 index 220f11dc5..000000000 --- a/foxhunt-deploy/src/cli/build.rs +++ /dev/null @@ -1,67 +0,0 @@ -use crate::config::types::FoxhuntConfig; -use crate::docker; -use crate::error::Result; -use crate::utils; -use clap::Args; - -/// Build Docker image with embedded binaries -#[derive(Args, Debug)] -pub(crate) struct BuildArgs { - /// Docker image tag (e.g., "latest", "v1.0.0") - #[arg(short, long, default_value = "latest")] - pub tag: String, - - /// Skip pushing to registry after build - #[arg(long)] - pub no_push: bool, - - /// Docker build context path - #[arg(long, default_value = ".")] - pub context: String, - - /// Dockerfile path - #[arg(long, default_value = "Dockerfile.foxhunt-build")] - pub dockerfile: String, - - /// Disable Docker build cache - #[arg(long)] - pub no_cache: bool, -} - -/// Execute build command -pub(crate) async fn execute(config: &FoxhuntConfig, args: &BuildArgs) -> Result<()> { - utils::step(1, 3, "Verifying Docker installation..."); - docker::verify_docker_available()?; - - // Build full image tag from config and args - let full_tag = format!( - "{}/{}:{}", - config.docker.registry, - config.docker.image_name, - args.tag - ); - - utils::step(2, 3, "Building Docker image..."); - - // Create build options - let build_options = docker::build::DockerBuildOptions::new(full_tag.clone()) - .dockerfile(args.dockerfile.clone()) - .context(args.context.clone()) - .no_cache(args.no_cache); - - // Build the image - let _image_id = docker::build::build_image(&build_options)?; - - // Push unless --no-push is specified - if !args.no_push { - utils::step(3, 3, "Pushing to Docker registry..."); - docker::push::push_image(&full_tag)?; - } else { - utils::info("Skipping push (--no-push flag set)"); - } - - utils::success("Docker build completed successfully!"); - utils::info(&format!("Image: {}", full_tag)); - - Ok(()) -} diff --git a/foxhunt-deploy/src/cli/deploy.rs b/foxhunt-deploy/src/cli/deploy.rs deleted file mode 100644 index ac6d2979a..000000000 --- a/foxhunt-deploy/src/cli/deploy.rs +++ /dev/null @@ -1,249 +0,0 @@ -use crate::config::types::FoxhuntConfig; -use crate::error::Result; -use crate::runpod::{ - build_deployment_request, select_best_gpu, validate_deployment, RunPodClient, -}; -use crate::utils; -use clap::Args; -use colored::Colorize; -use tracing::info; - -/// Deploy pod to RunPod -#[derive(Args, Debug)] -pub(crate) struct DeployArgs { - /// GPU type (e.g., "RTX A4000", "RTX 4090") - #[arg(short, long)] - pub gpu_type: Option, - - /// Datacenter region (e.g., "EUR-IS-1", "US-TX-3") - #[arg(short, long)] - pub datacenter: Option, - - /// Docker image tag to deploy - #[arg(short, long, default_value = "latest")] - pub tag: String, - - /// Training command to run - #[arg(short, long, required = true)] - pub command: String, - - /// Environment variables (format: KEY=VALUE) - #[arg(short, long, value_name = "KEY=VALUE")] - pub env: Vec, - - /// Pod name - #[arg(short, long)] - pub name: Option, - - /// Volume size in GB - #[arg(long)] - pub volume_size: Option, - - /// Dry run mode (validate request without deploying) - #[arg(long)] - pub dry_run: bool, - - /// Skip confirmation prompt (auto-approve deployment) - #[arg(short = 'y', long)] - pub yes: bool, -} - -/// Execute deploy command -pub(crate) async fn execute(config: &FoxhuntConfig, args: &DeployArgs) -> Result<()> { - utils::info("Starting RunPod deployment..."); - - // Create RunPod client - let client = RunPodClient::new(config.runpod.api_key.clone())?; - - // List available GPUs - let gpus = client.list_gpus().await?; - info!("Found {} available GPU types", gpus.len()); - - // Select GPU - let gpu = select_best_gpu(&gpus, args.gpu_type.as_deref())?; - - utils::success(&format!( - "Selected GPU: {} ({} GB VRAM, ${:.2}/hr)", - gpu.display_name.bold(), - gpu.memory_in_gb.unwrap_or(0), - gpu.secure_price.unwrap_or(0.0) - )); - - // Build deployment request - let request = build_deployment_request(args, config, &gpu)?; - - // Validate deployment request - validate_deployment(&request)?; - - // Display deployment summary - display_deployment_summary(&request, &gpu, args.dry_run); - - // Dry run mode - just validate and exit - if args.dry_run { - utils::success("Dry run completed successfully. Request is valid."); - return Ok(()); - } - - // Confirm deployment (skip if --yes flag is set) - if !args.yes && !confirm_deployment(&request, &gpu)? { - utils::info("Deployment cancelled by user."); - return Ok(()); - } - - // Deploy pod - utils::info("Deploying pod to RunPod..."); - let response = client.deploy_pod(request).await?; - - // Display success message - display_deployment_result(&response, &gpu); - - // Save pod ID for later reference - save_pod_id(&response.id)?; - - Ok(()) -} - -/// Display deployment summary -#[allow(clippy::non_ascii_literal)] -fn display_deployment_summary( - request: &crate::runpod::PodDeploymentRequest, - gpu: &crate::runpod::GpuType, - dry_run: bool, -) { - let na_fallback = "N/A".to_owned(); - println!(); - utils::info(&format!( - "{} Deployment Summary", - if dry_run { "[DRY RUN]" } else { "" } - )); - println!("{}", "─".repeat(60)); - println!(" Pod Name: {}", request.name.bold()); - println!(" GPU: {}", gpu.display_name.bold()); - println!( - " VRAM: {} GB", - gpu.memory_in_gb.unwrap_or(0).to_string().bold() - ); - println!( - " Cost: ${:.2}/hr", - gpu.secure_price.unwrap_or(0.0) - ); - println!( - " Datacenter: {}", - request - .data_center_ids - .as_ref() - .and_then(|d| d.first()) - .unwrap_or(&na_fallback) - .bold() - ); - println!(" Image: {}", request.image_name.bold()); - println!( - " Command: {}", - request - .docker_start_cmd - .as_ref() - .map(|cmd| cmd.join(" ")) - .unwrap_or_default() - .bold() - ); - println!( - " Container Disk: {} GB", - request.container_disk_in_gb.to_string().bold() - ); - if let Some(env) = &request.env { - if !env.is_empty() { - println!(" Environment:"); - for (key, value) in env { - println!(" {}={}", key, value); - } - } - } - println!("{}", "─".repeat(60)); - println!(); -} - -/// Confirm deployment with user -fn confirm_deployment( - _request: &crate::runpod::PodDeploymentRequest, - gpu: &crate::runpod::GpuType, -) -> Result { - let cost_per_hr = gpu.secure_price.unwrap_or(0.0); - let estimated_daily = cost_per_hr * 24.0; - - println!( - "{}", - format!( - "Estimated cost: ${:.2}/hr (${:.2}/day)", - cost_per_hr, estimated_daily - ) - .yellow() - ); - - print!("Deploy pod? [y/N]: "); - std::io::Write::flush(&mut std::io::stdout())?; - - let mut input = String::new(); - std::io::stdin().read_line(&mut input)?; - - Ok(input.trim().eq_ignore_ascii_case("y")) -} - -/// Display deployment result -#[allow(clippy::non_ascii_literal)] -fn display_deployment_result( - response: &crate::runpod::PodDeploymentResponse, - gpu: &crate::runpod::GpuType, -) { - println!(); - utils::success("Pod deployed successfully!"); - println!(); - println!("{}", "─".repeat(60)); - println!(" Pod ID: {}", response.id.green().bold()); - - if let Some(machine) = &response.machine { - if let Some(gpu_type) = &machine.gpu_type_id { - println!(" GPU: {}", gpu_type.bold()); - } - if let Some(vram) = machine.vram_gb { - println!(" VRAM: {} GB", vram.to_string().bold()); - } - } - - if let Some(runtime) = &response.runtime { - if let Some(datacenter) = &runtime.datacenter_id { - println!(" Datacenter: {}", datacenter.bold()); - } - } - - println!( - " Cost: ${:.2}/hr", - response.cost_per_hr.unwrap_or(gpu.secure_price.unwrap_or(0.0)) - ); - println!("{}", "─".repeat(60)); - println!(); - - utils::info("Monitor your pod:"); - println!( - " {}", - format!("foxhunt-deploy monitor {}", response.id).cyan() - ); - println!(); - utils::info("Terminate when done:"); - println!( - " {}", - format!("foxhunt-deploy run terminate --pod-id {}", response.id).cyan() - ); - println!(); -} - -/// Save pod ID to a file for later reference -fn save_pod_id(pod_id: &str) -> Result<()> { - use std::io::Write; - - let pod_file = std::path::PathBuf::from(".last_pod_id"); - let mut file = std::fs::File::create(&pod_file)?; - writeln!(file, "{}", pod_id)?; - - info!("Saved pod ID to {}", pod_file.display()); - Ok(()) -} diff --git a/foxhunt-deploy/src/cli/mod.rs b/foxhunt-deploy/src/cli/mod.rs deleted file mode 100644 index 073fa1bc3..000000000 --- a/foxhunt-deploy/src/cli/mod.rs +++ /dev/null @@ -1,48 +0,0 @@ -pub(crate) mod build; -pub(crate) mod deploy; -pub(crate) mod monitor; -pub(crate) mod run; - -use clap::{Parser, Subcommand}; -use std::path::PathBuf; - -/// Foxhunt RunPod deployment CLI -#[derive(Parser, Debug)] -#[command( - name = "foxhunt-deploy", - version, - about = "Foxhunt HFT Trading System - RunPod GPU Deployment CLI", - long_about = "Deploy and manage Foxhunt ML training workloads on RunPod GPU infrastructure.\n\ - Supports Docker image building, pod deployment, log monitoring, and unified workflows." -)] -pub(crate) struct Cli { - /// Enable verbose logging - #[arg(short, long, global = true)] - pub verbose: bool, - - /// Path to config file (default: ~/.runpod/config.toml) - #[arg(short, long, global = true, value_name = "FILE")] - pub config: Option, - - #[command(subcommand)] - pub command: Commands, -} - -/// Available CLI commands -#[derive(Subcommand, Debug)] -pub(crate) enum Commands { - /// Initialize configuration file - Init, - - /// Build Docker image - Build(build::BuildArgs), - - /// Deploy pod to RunPod - Deploy(deploy::DeployArgs), - - /// Monitor pod logs - Monitor(monitor::MonitorArgs), - - /// Build and deploy in one command - Run(run::RunArgs), -} diff --git a/foxhunt-deploy/src/cli/monitor.rs b/foxhunt-deploy/src/cli/monitor.rs deleted file mode 100644 index 559238c8c..000000000 --- a/foxhunt-deploy/src/cli/monitor.rs +++ /dev/null @@ -1,56 +0,0 @@ -use crate::config::types::FoxhuntConfig; -use crate::error::Result; -use crate::s3::monitor::LogMonitor; -use crate::s3::S3LogClient; -use clap::Args; - -/// Monitor pod logs from S3 -#[derive(Args, Debug)] -pub(crate) struct MonitorArgs { - /// Pod ID to monitor - #[arg(required = true)] - pub pod_id: String, - - /// Follow logs in real-time - #[arg(short, long)] - pub follow: bool, - - /// Number of recent lines to show - #[arg(short, long)] - pub tail: Option, - - /// Filter logs by pattern (regex) - #[arg(long)] - pub filter: Option, - - /// List available log files without displaying content - #[arg(short, long)] - pub list: bool, -} - -/// Execute monitor command -pub(crate) async fn execute(config: &FoxhuntConfig, args: &MonitorArgs) -> Result<()> { - // Create S3 client - let s3_client = S3LogClient::new(&config.s3).await?; - - // Create monitor - let monitor = LogMonitor::new( - s3_client, - args.pod_id.clone(), - config.s3.poll_interval_secs, - ); - - // Handle list mode - if args.list { - return monitor.list_logs().await; - } - - // Stream logs - if args.follow { - monitor.tail_logs(args.tail, args.filter.clone()).await?; - } else { - monitor.show_recent_logs(args.tail).await?; - } - - Ok(()) -} diff --git a/foxhunt-deploy/src/cli/run.rs b/foxhunt-deploy/src/cli/run.rs deleted file mode 100644 index 2c8f88b5d..000000000 --- a/foxhunt-deploy/src/cli/run.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::config::types::FoxhuntConfig; -use crate::error::Result; -use clap::Args; - -/// Build, deploy, and monitor in one command -#[derive(Args, Debug)] -pub(crate) struct RunArgs { - /// Training command to run - #[arg(short, long, required = true)] - pub command: String, - - /// GPU type (e.g., "RTX A4000", "RTX 4090") - #[arg(short, long)] - pub gpu_type: Option, - - /// Docker image tag - #[arg(short, long, default_value = "latest")] - pub tag: String, - - /// Skip Docker build step - #[arg(long)] - pub skip_build: bool, - - /// Don't monitor logs after deployment - #[arg(long)] - pub no_monitor: bool, -} - -/// Execute run command -pub(crate) async fn execute(_config: &FoxhuntConfig, _args: &RunArgs) -> Result<()> { - println!("Run command (Milestone 5)"); - println!("This will build, deploy, and monitor in one unified workflow"); - Ok(()) -} diff --git a/foxhunt-deploy/src/config/mod.rs b/foxhunt-deploy/src/config/mod.rs deleted file mode 100644 index 3d994ca2d..000000000 --- a/foxhunt-deploy/src/config/mod.rs +++ /dev/null @@ -1,207 +0,0 @@ -pub(crate) mod types; - -use crate::error::{FoxhuntError, Result}; -use std::fs; -use std::path::PathBuf; -use types::FoxhuntConfig; - -pub(crate) struct ConfigManager { - config_path: PathBuf, -} - -impl ConfigManager { - /// Create a new ConfigManager with default config path - pub(crate) fn new() -> Result { - let config_path = Self::default_config_path()?; - Ok(Self { config_path }) - } - - /// Create a ConfigManager with custom config path - pub(crate) fn with_path(path: PathBuf) -> Self { - Self { config_path: path } - } - - /// Get default config path: ~/.runpod/config.toml - pub(crate) fn default_config_path() -> Result { - let home = home::home_dir().ok_or_else(|| { - FoxhuntError::Config("Could not determine home directory".to_owned()) - })?; - Ok(home.join(".runpod").join("config.toml")) - } - - /// Load configuration from file and override with environment variables - pub(crate) fn load(&self) -> Result { - // Load from TOML file - let config = self.load_from_file()?; - - // Override with environment variables from .env.runpod - self.override_with_env(config) - } - - /// Load configuration from TOML file - fn load_from_file(&self) -> Result { - if !self.config_path.exists() { - return Err(FoxhuntError::ConfigNotFound { - path: self.config_path.clone(), - }); - } - - let contents = fs::read_to_string(&self.config_path)?; - let config: FoxhuntConfig = toml::from_str(&contents)?; - - Ok(config) - } - - /// Override configuration with environment variables - fn override_with_env(&self, mut config: FoxhuntConfig) -> Result { - // Try to load .env.runpod if it exists - drop(dotenvy::from_filename(".env.runpod")); - - // Override RunPod config - if let Ok(api_key) = std::env::var("RUNPOD_API_KEY") { - config.runpod.api_key = api_key; - } - if let Ok(gpu_type) = std::env::var("RUNPOD_GPU_TYPE") { - config.runpod.default_gpu_type = gpu_type; - } - if let Ok(datacenter) = std::env::var("RUNPOD_DATACENTER") { - config.runpod.default_datacenter = datacenter; - } - - // Override Docker config - if let Ok(registry) = std::env::var("DOCKER_REGISTRY") { - config.docker.registry = registry; - } - if let Ok(image_name) = std::env::var("DOCKER_IMAGE_NAME") { - config.docker.image_name = image_name; - } - if let Ok(tag) = std::env::var("DOCKER_TAG") { - config.docker.tag = tag; - } - - // Override S3 config - if let Ok(endpoint) = std::env::var("S3_ENDPOINT") { - config.s3.endpoint = endpoint; - } - if let Ok(bucket) = std::env::var("S3_BUCKET") { - config.s3.bucket = bucket; - } - if let Ok(region) = std::env::var("S3_REGION") { - config.s3.region = region; - } - if let Ok(poll_interval) = std::env::var("S3_POLL_INTERVAL_SECS") { - if let Ok(interval) = poll_interval.parse::() { - config.s3.poll_interval_secs = interval; - } - } - - // Override deployment defaults - if let Ok(disk) = std::env::var("CONTAINER_DISK_GB") { - if let Ok(disk_gb) = disk.parse::() { - config.defaults.container_disk_gb = disk_gb; - } - } - if let Ok(volume_path) = std::env::var("VOLUME_PATH") { - config.defaults.volume_path = volume_path; - } - if let Ok(volume_size) = std::env::var("VOLUME_SIZE_GB") { - if let Ok(size_gb) = volume_size.parse::() { - config.defaults.volume_size_gb = size_gb; - } - } - if let Ok(public_ip) = std::env::var("SUPPORT_PUBLIC_IP") { - if let Ok(support) = public_ip.parse::() { - config.defaults.support_public_ip = support; - } - } - - Ok(config) - } - - /// Validate required configuration fields - pub(crate) fn validate(config: &FoxhuntConfig) -> Result<()> { - // Validate API key - if config.runpod.api_key.is_empty() - || config.runpod.api_key == "YOUR_RUNPOD_API_KEY_HERE" - { - return Err(FoxhuntError::MissingConfigField { - field: "runpod.api_key".to_owned(), - }); - } - - // Validate S3 bucket - if config.s3.bucket.is_empty() { - return Err(FoxhuntError::MissingConfigField { - field: "s3.bucket".to_owned(), - }); - } - - Ok(()) - } - - /// Create sample config file - pub(crate) fn create_sample_config() -> Result { - let config_path = Self::default_config_path()?; - let config_dir = config_path - .parent() - .ok_or_else(|| FoxhuntError::Config("Invalid config path".to_owned()))?; - - // Create config directory if it doesn't exist - fs::create_dir_all(config_dir)?; - - // Create default config - let config = FoxhuntConfig::default(); - let toml_str = toml::to_string_pretty(&config).map_err(|e| { - FoxhuntError::Config(format!("Failed to serialize config: {}", e)) - })?; - - // Write to file - fs::write(&config_path, toml_str)?; - - Ok(config_path) - } -} - -impl Default for ConfigManager { - fn default() -> Self { - Self::new().unwrap_or_else(|_| { - // Fallback to a reasonable default path if home dir detection fails - Self::with_path(PathBuf::from(".runpod/config.toml")) - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::env; - - #[test] - fn test_default_config() { - let config = FoxhuntConfig::default(); - assert_eq!(config.runpod.default_gpu_type, "RTX A4000"); - assert_eq!(config.docker.registry, "jgrusewski"); - assert_eq!(config.s3.bucket, "se3zdnb5o4"); - } - - #[test] - fn test_env_override() { - env::set_var("RUNPOD_API_KEY", "test_key"); - let manager = ConfigManager::with_path(PathBuf::from("test_config.toml")); - let config = FoxhuntConfig::default(); - let overridden = manager.override_with_env(config).unwrap(); - assert_eq!(overridden.runpod.api_key, "test_key"); - env::remove_var("RUNPOD_API_KEY"); - } - - #[test] - fn test_validation() { - let mut config = FoxhuntConfig::default(); - // Should fail with default API key - assert!(ConfigManager::validate(&config).is_err()); - - // Should succeed with real API key - config.runpod.api_key = "real_api_key".to_owned(); - assert!(ConfigManager::validate(&config).is_ok()); - } -} diff --git a/foxhunt-deploy/src/config/types.rs b/foxhunt-deploy/src/config/types.rs deleted file mode 100644 index 4a4312f3a..000000000 --- a/foxhunt-deploy/src/config/types.rs +++ /dev/null @@ -1,167 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Main configuration struct -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct FoxhuntConfig { - pub runpod: RunPodConfig, - pub docker: DockerConfig, - pub s3: S3Config, - pub defaults: DeploymentDefaults, -} - -impl Default for FoxhuntConfig { - fn default() -> Self { - Self { - runpod: RunPodConfig::default(), - docker: DockerConfig::default(), - s3: S3Config::default(), - defaults: DeploymentDefaults::default(), - } - } -} - -/// RunPod API configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct RunPodConfig { - #[serde(default = "default_api_key")] - pub api_key: String, - #[serde(default = "default_gpu_type")] - pub default_gpu_type: String, - #[serde(default = "default_datacenter")] - pub default_datacenter: String, -} - -impl Default for RunPodConfig { - fn default() -> Self { - Self { - api_key: default_api_key(), - default_gpu_type: default_gpu_type(), - default_datacenter: default_datacenter(), - } - } -} - -fn default_api_key() -> String { - "YOUR_RUNPOD_API_KEY_HERE".to_owned() -} - -fn default_gpu_type() -> String { - "RTX A4000".to_owned() -} - -fn default_datacenter() -> String { - "EUR-IS-1".to_owned() -} - -/// Docker configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct DockerConfig { - #[serde(default = "default_registry")] - pub registry: String, - #[serde(default = "default_image_name")] - pub image_name: String, - #[serde(default = "default_tag")] - pub tag: String, -} - -impl Default for DockerConfig { - fn default() -> Self { - Self { - registry: default_registry(), - image_name: default_image_name(), - tag: default_tag(), - } - } -} - -fn default_registry() -> String { - "jgrusewski".to_owned() -} - -fn default_image_name() -> String { - "foxhunt".to_owned() -} - -fn default_tag() -> String { - "latest".to_owned() -} - -/// S3 configuration for log monitoring -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct S3Config { - #[serde(default = "default_endpoint")] - pub endpoint: String, - #[serde(default = "default_bucket")] - pub bucket: String, - #[serde(default = "default_region")] - pub region: String, - #[serde(default = "default_poll_interval")] - pub poll_interval_secs: u64, -} - -impl Default for S3Config { - fn default() -> Self { - Self { - endpoint: default_endpoint(), - bucket: default_bucket(), - region: default_region(), - poll_interval_secs: default_poll_interval(), - } - } -} - -fn default_endpoint() -> String { - "https://s3api-eur-is-1.runpod.io".to_owned() -} - -fn default_bucket() -> String { - "se3zdnb5o4".to_owned() -} - -fn default_region() -> String { - "us-east-1".to_owned() -} - -fn default_poll_interval() -> u64 { - 5 -} - -/// Deployment defaults -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct DeploymentDefaults { - #[serde(default = "default_container_disk_gb")] - pub container_disk_gb: u32, - #[serde(default = "default_volume_path")] - pub volume_path: String, - #[serde(default = "default_volume_size_gb")] - pub volume_size_gb: u32, - #[serde(default = "default_support_public_ip")] - pub support_public_ip: bool, -} - -impl Default for DeploymentDefaults { - fn default() -> Self { - Self { - container_disk_gb: default_container_disk_gb(), - volume_path: default_volume_path(), - volume_size_gb: default_volume_size_gb(), - support_public_ip: default_support_public_ip(), - } - } -} - -fn default_container_disk_gb() -> u32 { - 50 -} - -fn default_volume_path() -> String { - "/runpod-volume".to_owned() -} - -fn default_volume_size_gb() -> u32 { - 100 -} - -fn default_support_public_ip() -> bool { - false -} diff --git a/foxhunt-deploy/src/docker/build.rs b/foxhunt-deploy/src/docker/build.rs deleted file mode 100644 index 76d4c07a3..000000000 --- a/foxhunt-deploy/src/docker/build.rs +++ /dev/null @@ -1,226 +0,0 @@ -use crate::error::{FoxhuntError, Result}; -use crate::utils; -use indicatif::{ProgressBar, ProgressStyle}; -use std::io::{BufRead, BufReader}; -use std::process::{Command, Stdio}; -use std::time::Duration; - -/// Options for building a Docker image -#[derive(Debug, Clone)] -pub(crate) struct DockerBuildOptions { - /// Docker image tag (e.g., "jgrusewski/foxhunt:latest") - pub tag: String, - /// Path to Dockerfile - pub dockerfile: String, - /// Build context directory - pub context: String, - /// Disable build cache - pub no_cache: bool, - /// Additional build arguments - pub build_args: Vec<(String, String)>, -} - -impl DockerBuildOptions { - pub(crate) fn new(tag: impl Into) -> Self { - Self { - tag: tag.into(), - dockerfile: "Dockerfile.foxhunt-build".to_owned(), - context: ".".to_owned(), - no_cache: false, - build_args: Vec::new(), - } - } - - pub(crate) fn dockerfile(mut self, path: impl Into) -> Self { - self.dockerfile = path.into(); - self - } - - pub(crate) fn context(mut self, path: impl Into) -> Self { - self.context = path.into(); - self - } - - pub(crate) fn no_cache(mut self, no_cache: bool) -> Self { - self.no_cache = no_cache; - self - } - - #[allow(dead_code)] - pub(crate) fn build_arg(mut self, key: impl Into, value: impl Into) -> Self { - self.build_args.push((key.into(), value.into())); - self - } -} - -/// Build a Docker image and return the image ID -#[allow(clippy::non_ascii_literal)] -pub(crate) fn build_image(options: &DockerBuildOptions) -> Result { - // Verify Dockerfile exists - let dockerfile_path = std::path::Path::new(&options.dockerfile); - if !dockerfile_path.exists() { - return Err(FoxhuntError::Docker(format!( - "Dockerfile not found: {}", - options.dockerfile - ))); - } - - // Verify build context exists - let context_path = std::path::Path::new(&options.context); - if !context_path.exists() { - return Err(FoxhuntError::Docker(format!( - "Build context directory not found: {}", - options.context - ))); - } - - utils::info(&format!("Building Docker image: {}", options.tag)); - utils::info(&format!("Dockerfile: {}", options.dockerfile)); - utils::info(&format!("Context: {}", options.context)); - - // Create progress bar - let pb = ProgressBar::new_spinner(); - pb.set_style( - ProgressStyle::default_spinner() - .template("{spinner:.blue} {msg}") - .unwrap() - .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]), - ); - pb.enable_steady_tick(Duration::from_millis(100)); - pb.set_message("Starting Docker build..."); - - // Build docker command - let mut cmd = Command::new("docker"); - cmd.arg("build"); - cmd.arg("--tag").arg(&options.tag); - cmd.arg("--file").arg(&options.dockerfile); - - if options.no_cache { - cmd.arg("--no-cache"); - } - - // Add build args - for (key, value) in &options.build_args { - cmd.arg("--build-arg").arg(format!("{}={}", key, value)); - } - - // Add context last - cmd.arg(&options.context); - - // Configure to capture output - cmd.stdout(Stdio::piped()); - cmd.stderr(Stdio::piped()); - - // Spawn the process - let mut child = cmd - .spawn() - .map_err(|e| FoxhuntError::Docker(format!("Failed to spawn docker build: {}", e)))?; - - // Stream stdout - if let Some(stdout) = child.stdout.take() { - let reader = BufReader::new(stdout); - for raw_line in reader.lines() { - if let Ok(output_line) = raw_line { - // Update progress bar with build step - if output_line.contains("Step ") { - pb.set_message(output_line.clone()); - } - tracing::debug!("docker build: {}", output_line); - } - } - } - - // Wait for completion - let status = child - .wait() - .map_err(|e| FoxhuntError::Docker(format!("Failed to wait for docker build: {}", e)))?; - - pb.finish_and_clear(); - - if !status.success() { - // Try to get stderr for error details - let stderr = if let Some(stderr) = child.stderr { - let reader = BufReader::new(stderr); - let mut error_msg = String::new(); - for err_line in reader.lines().flatten() { - error_msg.push_str(&err_line); - error_msg.push('\n'); - } - error_msg - } else { - "Unknown error".to_owned() - }; - - return Err(FoxhuntError::Docker(format!( - "Docker build failed with exit code {:?}:\n{}", - status.code(), - stderr - ))); - } - - // Get the image ID - let image_id = get_image_id(&options.tag)?; - - utils::success(&format!("Built image: {} ({})", options.tag, image_id)); - - Ok(image_id) -} - -/// Get the image ID for a given tag -fn get_image_id(tag: &str) -> Result { - let output = Command::new("docker") - .args(["images", "-q", tag]) - .output() - .map_err(|e| FoxhuntError::Docker(format!("Failed to get image ID: {}", e)))?; - - if !output.status.success() { - return Err(FoxhuntError::Docker( - "Failed to retrieve image ID after build".to_owned(), - )); - } - - let image_id = String::from_utf8_lossy(&output.stdout) - .trim() - .to_owned(); - - if image_id.is_empty() { - return Err(FoxhuntError::Docker( - "Image ID is empty after build".to_owned(), - )); - } - - Ok(image_id) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_build_options_builder() { - let options = DockerBuildOptions::new("test:latest") - .dockerfile("Dockerfile.test") - .context("/tmp/test") - .no_cache(true) - .build_arg("VERSION", "1.0.0"); - - assert_eq!(options.tag, "test:latest"); - assert_eq!(options.dockerfile, "Dockerfile.test"); - assert_eq!(options.context, "/tmp/test"); - assert!(options.no_cache); - assert_eq!(options.build_args.len(), 1); - assert_eq!(options.build_args[0].0, "VERSION"); - assert_eq!(options.build_args[0].1, "1.0.0"); - } - - #[test] - fn test_build_options_defaults() { - let options = DockerBuildOptions::new("test:latest"); - - assert_eq!(options.tag, "test:latest"); - assert_eq!(options.dockerfile, "Dockerfile.foxhunt-build"); - assert_eq!(options.context, "."); - assert!(!options.no_cache); - assert!(options.build_args.is_empty()); - } -} diff --git a/foxhunt-deploy/src/docker/mod.rs b/foxhunt-deploy/src/docker/mod.rs deleted file mode 100644 index 814b75597..000000000 --- a/foxhunt-deploy/src/docker/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -pub(crate) mod build; -pub(crate) mod push; - -use crate::error::{FoxhuntError, Result}; -use std::process::Command; - -/// Verify Docker is installed and daemon is running -pub(crate) fn verify_docker_available() -> Result<()> { - // Check if docker command exists - let which_result = Command::new("which") - .arg("docker") - .output() - .map_err(|e| FoxhuntError::Docker(format!("Failed to check for Docker installation: {}", e)))?; - - if !which_result.status.success() { - return Err(FoxhuntError::Docker( - "Docker is not installed. Please install Docker Desktop or Docker Engine.".to_owned(), - )); - } - - // Check if Docker daemon is running - let version_result = Command::new("docker") - .arg("version") - .arg("--format") - .arg("{{.Server.Version}}") - .output() - .map_err(|e| FoxhuntError::Docker(format!("Failed to check Docker daemon: {}", e)))?; - - if !version_result.status.success() { - return Err(FoxhuntError::Docker( - "Docker daemon is not running. Please start Docker Desktop or Docker service.".to_owned(), - )); - } - - Ok(()) -} - -/// Check if a Docker image exists locally -pub(crate) fn image_exists(tag: &str) -> Result { - let output = Command::new("docker") - .args(["images", "-q", tag]) - .output() - .map_err(|e| FoxhuntError::Docker(format!("Failed to check if image exists: {}", e)))?; - - Ok(!output.stdout.is_empty()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_verify_docker_available() { - // This will succeed or fail depending on Docker installation - // We don't assert here since it's environment-dependent - let _ = verify_docker_available(); - } - - #[test] - fn test_image_exists() { - // Test with a non-existent image - let result = image_exists("nonexistent-image-12345:latest"); - assert!(result.is_ok()); - } -} diff --git a/foxhunt-deploy/src/docker/push.rs b/foxhunt-deploy/src/docker/push.rs deleted file mode 100644 index 378df9c23..000000000 --- a/foxhunt-deploy/src/docker/push.rs +++ /dev/null @@ -1,130 +0,0 @@ -use crate::error::{FoxhuntError, Result}; -use crate::utils; -use indicatif::{ProgressBar, ProgressStyle}; -use std::io::{BufRead, BufReader}; -use std::process::{Command, Stdio}; -use std::time::Duration; - -/// Push a Docker image to a registry -#[allow(clippy::non_ascii_literal)] -pub(crate) fn push_image(tag: &str) -> Result<()> { - // Verify image exists locally - if !super::image_exists(tag)? { - return Err(FoxhuntError::Docker(format!( - "Image '{}' does not exist locally. Build it first.", - tag - ))); - } - - utils::info(&format!("Pushing Docker image: {}", tag)); - - // Create progress bar - let pb = ProgressBar::new_spinner(); - pb.set_style( - ProgressStyle::default_spinner() - .template("{spinner:.blue} {msg}") - .unwrap() - .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]), - ); - pb.enable_steady_tick(Duration::from_millis(100)); - pb.set_message("Pushing to registry..."); - - // Build docker push command - let mut cmd = Command::new("docker"); - cmd.arg("push"); - cmd.arg(tag); - - // Configure to capture output - cmd.stdout(Stdio::piped()); - cmd.stderr(Stdio::piped()); - - // Spawn the process - let mut child = cmd - .spawn() - .map_err(|e| FoxhuntError::Docker(format!("Failed to spawn docker push: {}", e)))?; - - // Stream stdout to show progress - if let Some(stdout) = child.stdout.take() { - let reader = BufReader::new(stdout); - for raw_line in reader.lines() { - if let Ok(output_line) = raw_line { - // Update progress bar with push progress - if output_line.contains("Pushing") || output_line.contains("Pushed") || output_line.contains("Waiting") { - pb.set_message(output_line.clone()); - } - tracing::debug!("docker push: {}", output_line); - } - } - } - - // Wait for completion - let status = child - .wait() - .map_err(|e| FoxhuntError::Docker(format!("Failed to wait for docker push: {}", e)))?; - - pb.finish_and_clear(); - - if !status.success() { - // Try to get stderr for error details - let stderr = if let Some(stderr) = child.stderr { - let reader = BufReader::new(stderr); - let mut error_msg = String::new(); - for err_line in reader.lines().flatten() { - error_msg.push_str(&err_line); - error_msg.push('\n'); - } - error_msg - } else { - "Unknown error".to_owned() - }; - - // Check for authentication errors - if stderr.contains("authentication") || stderr.contains("unauthorized") { - return Err(FoxhuntError::Docker(format!( - "Docker registry authentication failed. Please run 'docker login' first.\n{}", - stderr - ))); - } - - return Err(FoxhuntError::Docker(format!( - "Docker push failed with exit code {:?}:\n{}", - status.code(), - stderr - ))); - } - - utils::success(&format!("Successfully pushed: {}", tag)); - - Ok(()) -} - -/// Check if user is logged in to Docker registry -#[allow(dead_code)] -pub(crate) fn check_registry_auth(registry: &str) -> Result { - // Try to get credentials from Docker config - let output = Command::new("docker") - .args(["login", "--help"]) - .output() - .map_err(|e| FoxhuntError::Docker(format!("Failed to check Docker login status: {}", e)))?; - - if !output.status.success() { - return Ok(false); - } - - // For simplicity, we assume if docker login command exists, auth is possible - // A more robust check would parse ~/.docker/config.json - tracing::debug!("Registry auth check for: {}", registry); - Ok(true) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_check_registry_auth() { - // This will succeed or fail depending on Docker installation - let result = check_registry_auth("docker.io"); - assert!(result.is_ok()); - } -} diff --git a/foxhunt-deploy/src/error.rs b/foxhunt-deploy/src/error.rs deleted file mode 100644 index 6d6af2100..000000000 --- a/foxhunt-deploy/src/error.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::path::PathBuf; -use thiserror::Error; - -/// Custom Result type for foxhunt-deploy -pub(crate) type Result = std::result::Result; - -/// Unified error type for all operations -#[derive(Error, Debug)] -pub(crate) enum FoxhuntError { - #[error("Configuration error: {0}")] - Config(String), - - #[error("Config file not found at {path:?}. Run 'foxhunt-deploy init' to create one.")] - ConfigNotFound { path: PathBuf }, - - #[error("Missing required config field: {field}")] - MissingConfigField { field: String }, - - #[error("Docker command failed: {0}")] - Docker(String), - - #[error("RunPod API error: {status} - {message}")] - RunPodApi { status: u16, message: String }, - - #[error("S3 error: {0}")] - S3(String), - - #[error("IO error: {0}")] - Io(#[from] std::io::Error), - - #[error("HTTP request failed: {0}")] - Http(#[from] reqwest::Error), - - #[error("Serialization error: {0}")] - Serde(#[from] serde_json::Error), - - #[error("TOML parsing error: {0}")] - Toml(#[from] toml::de::Error), - - #[error("Invalid argument: {0}")] - InvalidArgument(String), - - #[error("Unknown error: {0}")] - Unknown(String), -} diff --git a/foxhunt-deploy/src/main.rs b/foxhunt-deploy/src/main.rs deleted file mode 100644 index 519e98df7..000000000 --- a/foxhunt-deploy/src/main.rs +++ /dev/null @@ -1,102 +0,0 @@ -mod cli; -mod config; -mod docker; -mod error; -mod runpod; -mod s3; -mod utils; - -use clap::Parser; -use cli::{Cli, Commands}; -use config::ConfigManager; -use error::Result; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; - -#[tokio::main] -async fn main() { - if let Err(e) = run().await { - utils::error(format!("Error: {}", e)); - std::process::exit(1); - } -} - -async fn run() -> Result<()> { - // Parse CLI arguments - let cli = Cli::parse(); - - // Initialize logging - init_logging(cli.verbose); - - // Handle init command separately (doesn't need config) - if matches!(cli.command, Commands::Init) { - return handle_init(); - } - - // Load configuration - let config_manager = if let Some(config_path) = cli.config { - ConfigManager::with_path(config_path) - } else { - ConfigManager::new()? - }; - - let config = config_manager.load()?; - - // Validate configuration - ConfigManager::validate(&config)?; - - // Route to appropriate subcommand - match cli.command { - Commands::Init => { - // Already handled above; return Ok to satisfy the match arm - Ok(()) - } - Commands::Build(args) => { - cli::build::execute(&config, &args).await?; - Ok(()) - } - Commands::Deploy(args) => { - cli::deploy::execute(&config, &args).await?; - Ok(()) - } - Commands::Monitor(args) => { - cli::monitor::execute(&config, &args).await?; - Ok(()) - } - Commands::Run(args) => { - cli::run::execute(&config, &args).await?; - Ok(()) - } - } -} - -/// Initialize logging based on verbosity -fn init_logging(verbose: bool) { - let filter = if verbose { - EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("foxhunt_deploy=debug,info")) - } else { - EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("foxhunt_deploy=info")) - }; - - tracing_subscriber::registry() - .with(filter) - .with(tracing_subscriber::fmt::layer().with_target(false)) - .init(); -} - -/// Handle init command -fn handle_init() -> Result<()> { - utils::info("Initializing foxhunt-deploy configuration..."); - - let config_path = ConfigManager::create_sample_config()?; - - utils::success(format!( - "Created sample configuration at: {}", - config_path.display() - )); - utils::info("Please edit the config file and add your RunPod API key."); - utils::info("You can also override settings with a .env.runpod file or environment variables."); - - Ok(()) -} diff --git a/foxhunt-deploy/src/runpod/client.rs b/foxhunt-deploy/src/runpod/client.rs deleted file mode 100644 index 251170d39..000000000 --- a/foxhunt-deploy/src/runpod/client.rs +++ /dev/null @@ -1,120 +0,0 @@ -use super::types::{ - ApiError, GpuType, PodDeploymentRequest, PodDeploymentResponse, -}; -use crate::error::{FoxhuntError, Result}; -use reqwest::{header, Client}; -use tracing::{debug, info}; - -const RUNPOD_REST_BASE: &str = "https://rest.runpod.io/v1"; - -/// RunPod API client -pub(crate) struct RunPodClient { - api_key: String, - http_client: Client, -} - -impl RunPodClient { - /// Create a new RunPod client - pub(crate) fn new(api_key: String) -> Result { - if api_key.is_empty() || api_key == "YOUR_RUNPOD_API_KEY_HERE" { - return Err(FoxhuntError::Config( - "RunPod API key not set. Please configure it in foxhunt.toml or .env.runpod" - .to_owned(), - )); - } - - let http_client = Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build()?; - - Ok(Self { - api_key, - http_client, - }) - } - - /// List available GPU types (simplified implementation) - /// In production, this would use the GraphQL API to query gpuTypes - pub(crate) async fn list_gpus(&self) -> Result> { - info!("Fetching available GPU types from RunPod..."); - - // For now, return hardcoded list of common GPUs - // TODO: Implement GraphQL query to fetch real-time GPU availability - Ok(vec![ - GpuType { - id: "NVIDIA RTX A4000".to_owned(), - display_name: "RTX A4000".to_owned(), - memory_in_gb: Some(16), - secure_price: Some(0.25), - community_price: Some(0.18), - lowest_price: None, - }, - GpuType { - id: "NVIDIA RTX 4090".to_owned(), - display_name: "RTX 4090".to_owned(), - memory_in_gb: Some(24), - secure_price: Some(0.59), - community_price: Some(0.44), - lowest_price: None, - }, - GpuType { - id: "NVIDIA A40".to_owned(), - display_name: "A40".to_owned(), - memory_in_gb: Some(48), - secure_price: Some(0.45), - community_price: Some(0.35), - lowest_price: None, - }, - ]) - } - - /// Deploy a new pod - pub(crate) async fn deploy_pod(&self, request: PodDeploymentRequest) -> Result { - info!("Deploying pod: {}", request.name); - debug!("Deployment request: {:?}", request); - - let url = format!("{}/pods", RUNPOD_REST_BASE); - - let response = self - .http_client - .post(&url) - .header(header::AUTHORIZATION, format!("Bearer {}", self.api_key)) - .header(header::CONTENT_TYPE, "application/json") - .json(&request) - .send() - .await?; - - let status = response.status(); - let body = response.text().await?; - - debug!("API response (status {}): {}", status, body); - - if status.is_success() { - let pod_response: PodDeploymentResponse = serde_json::from_str(&body) - .map_err(|e| { - FoxhuntError::Unknown(format!( - "Failed to parse deployment response: {}. Body: {}", - e, body - )) - })?; - - info!("Pod deployed successfully: {}", pod_response.id); - Ok(pod_response) - } else { - // Try to parse error response - let error_msg = if let Ok(api_error) = serde_json::from_str::(&body) { - api_error - .error - .or(api_error.message) - .unwrap_or_else(|| body.clone()) - } else { - body - }; - - Err(FoxhuntError::RunPodApi { - status: status.as_u16(), - message: error_msg, - }) - } - } -} diff --git a/foxhunt-deploy/src/runpod/deployment.rs b/foxhunt-deploy/src/runpod/deployment.rs deleted file mode 100644 index 9ec44916f..000000000 --- a/foxhunt-deploy/src/runpod/deployment.rs +++ /dev/null @@ -1,304 +0,0 @@ -use super::types::{GpuType, PodDeploymentRequest}; -use crate::cli::deploy::DeployArgs; -use crate::config::types::FoxhuntConfig; -use crate::error::{FoxhuntError, Result}; -use std::collections::HashMap; -use tracing::{debug, info}; - -/// Build a deployment request from CLI args and config -pub(crate) fn build_deployment_request( - args: &DeployArgs, - config: &FoxhuntConfig, - gpu: &GpuType, -) -> Result { - // Parse command into docker start command - let docker_start_cmd = parse_command(&args.command)?; - - // Build environment variables - let mut env = HashMap::new(); - for env_var in &args.env { - let parts: Vec<&str> = env_var.splitn(2, '=').collect(); - if parts.len() != 2 { - return Err(FoxhuntError::InvalidArgument(format!( - "Invalid environment variable format: {}. Expected KEY=VALUE", - env_var - ))); - } - env.insert(parts[0].to_owned(), parts[1].to_owned()); - } - - // Determine datacenter - let datacenter = args - .datacenter - .clone() - .unwrap_or_else(|| config.runpod.default_datacenter.clone()); - - // Build image name - let image_name = format!( - "{}/{}:{}", - config.docker.registry, config.docker.image_name, args.tag - ); - - // Generate pod name - let pod_name = args.name.as_ref().cloned().unwrap_or_else(|| { - format!( - "foxhunt-{}", - chrono::Utc::now().format("%Y%m%d-%H%M%S") - ) - }); - - // Get volume configuration (unused for now, but reserved for future use) - let _volume_size = args - .volume_size - .unwrap_or(config.defaults.volume_size_gb); - - info!("Building deployment request:"); - info!(" Name: {}", pod_name); - info!(" GPU: {}", gpu.display_name); - info!(" Datacenter: {}", datacenter); - info!(" Image: {}", image_name); - info!(" Command: {:?}", docker_start_cmd); - - Ok(PodDeploymentRequest { - cloud_type: "SECURE".to_owned(), - compute_type: "GPU".to_owned(), - data_center_ids: Some(vec![datacenter]), - gpu_type_ids: vec![gpu.id.clone()], - gpu_count: 1, - name: pod_name, - image_name, - container_disk_in_gb: config.defaults.container_disk_gb, - network_volume_id: Some(config.s3.bucket.clone()), - volume_mount_path: Some(config.defaults.volume_path.clone()), - docker_start_cmd: Some(docker_start_cmd), - container_registry_auth_id: Some("cmh3ya1710001jo02vwqtisbf".to_owned()), - ports: Some(vec!["8888/http".to_owned(), "22/tcp".to_owned()]), - interruptible: Some(false), - env: if env.is_empty() { None } else { Some(env) }, - support_public_ip: Some(config.defaults.support_public_ip), - }) -} - -/// Parse command string into docker start command array -fn parse_command(command: &str) -> Result> { - // Simple split by whitespace (could be enhanced for quoted arguments) - let parts: Vec = command - .split_whitespace() - .map(|s| s.to_owned()) - .collect(); - - if parts.is_empty() { - return Err(FoxhuntError::InvalidArgument( - "Command cannot be empty".to_owned(), - )); - } - - debug!("Parsed command: {:?}", parts); - Ok(parts) -} - -/// Select the best GPU based on availability and price -pub(crate) fn select_best_gpu(gpus: &[GpuType], preferred_type: Option<&str>) -> Result { - if gpus.is_empty() { - return Err(FoxhuntError::Unknown( - "No GPUs available from RunPod".to_owned(), - )); - } - - // If user specified a GPU type, find it - if let Some(preferred) = preferred_type { - if let Some(gpu) = gpus.iter().find(|g| { - g.id.to_lowercase().contains(&preferred.to_lowercase()) - || g.display_name - .to_lowercase() - .contains(&preferred.to_lowercase()) - }) { - info!("Selected user-preferred GPU: {}", gpu.display_name); - return Ok(gpu.clone()); - } else { - return Err(FoxhuntError::InvalidArgument(format!( - "GPU type '{}' not found. Available: {}", - preferred, - gpus.iter() - .map(|g| g.display_name.as_str()) - .collect::>() - .join(", ") - ))); - } - } - - // Auto-select: prefer RTX A4000 (good balance of price and performance) - if let Some(a4000) = gpus - .iter() - .find(|g| g.id.contains("A4000") || g.display_name.contains("A4000")) - { - info!( - "Auto-selected RTX A4000 (best value): ${:.2}/hr", - a4000.secure_price.unwrap_or(0.0) - ); - return Ok(a4000.clone()); - } - - // Fallback to cheapest GPU - let cheapest = gpus - .iter() - .min_by(|a, b| { - let price_a = a.secure_price.unwrap_or(f64::MAX); - let price_b = b.secure_price.unwrap_or(f64::MAX); - price_a - .partial_cmp(&price_b) - .unwrap_or(std::cmp::Ordering::Equal) - }) - .unwrap(); // Safe because we checked gpus.is_empty() above - - info!( - "Auto-selected cheapest GPU: {} (${:.2}/hr)", - cheapest.display_name, - cheapest.secure_price.unwrap_or(0.0) - ); - Ok(cheapest.clone()) -} - -/// Validate deployment request before submitting -pub(crate) fn validate_deployment(request: &PodDeploymentRequest) -> Result<()> { - // Validate GPU count - if request.gpu_count == 0 { - return Err(FoxhuntError::InvalidArgument( - "GPU count must be at least 1".to_owned(), - )); - } - - // Validate container disk size - if request.container_disk_in_gb < 10 { - return Err(FoxhuntError::InvalidArgument( - "Container disk size must be at least 10GB".to_owned(), - )); - } - - // Validate docker command - if let Some(cmd) = &request.docker_start_cmd { - if cmd.is_empty() { - return Err(FoxhuntError::InvalidArgument( - "Docker start command cannot be empty".to_owned(), - )); - } - } - - debug!("Deployment validation passed"); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_command() { - let result = parse_command("train_ppo --epochs 100").unwrap(); - assert_eq!(result, vec!["train_ppo", "--epochs", "100"]); - } - - #[test] - fn test_parse_command_empty() { - let result = parse_command(""); - assert!(result.is_err()); - } - - #[test] - fn test_select_best_gpu_preferred() { - let gpus = vec![ - GpuType { - id: "NVIDIA RTX A4000".to_owned(), - display_name: "RTX A4000".to_owned(), - memory_in_gb: Some(16), - secure_price: Some(0.25), - community_price: None, - lowest_price: None, - }, - GpuType { - id: "NVIDIA RTX 4090".to_owned(), - display_name: "RTX 4090".to_owned(), - memory_in_gb: Some(24), - secure_price: Some(0.59), - community_price: None, - lowest_price: None, - }, - ]; - - let result = select_best_gpu(&gpus, Some("4090")).unwrap(); - assert_eq!(result.display_name, "RTX 4090"); - } - - #[test] - fn test_select_best_gpu_auto() { - let gpus = vec![ - GpuType { - id: "NVIDIA RTX A4000".to_owned(), - display_name: "RTX A4000".to_owned(), - memory_in_gb: Some(16), - secure_price: Some(0.25), - community_price: None, - lowest_price: None, - }, - GpuType { - id: "NVIDIA RTX 4090".to_owned(), - display_name: "RTX 4090".to_owned(), - memory_in_gb: Some(24), - secure_price: Some(0.59), - community_price: None, - lowest_price: None, - }, - ]; - - let result = select_best_gpu(&gpus, None).unwrap(); - assert_eq!(result.display_name, "RTX A4000"); // Should prefer A4000 - } - - #[test] - fn test_validate_deployment() { - let request = PodDeploymentRequest { - cloud_type: "SECURE".to_owned(), - compute_type: "GPU".to_owned(), - data_center_ids: Some(vec!["EUR-IS-1".to_owned()]), - gpu_type_ids: vec!["NVIDIA RTX A4000".to_owned()], - gpu_count: 1, - name: "test-pod".to_owned(), - image_name: "jgrusewski/foxhunt:latest".to_owned(), - container_disk_in_gb: 50, - network_volume_id: None, - volume_mount_path: None, - docker_start_cmd: Some(vec!["train_ppo".to_owned()]), - container_registry_auth_id: None, - ports: None, - interruptible: None, - env: None, - support_public_ip: None, - }; - - assert!(validate_deployment(&request).is_ok()); - } - - #[test] - fn test_validate_deployment_invalid_gpu_count() { - let request = PodDeploymentRequest { - cloud_type: "SECURE".to_owned(), - compute_type: "GPU".to_owned(), - data_center_ids: None, - gpu_type_ids: vec!["NVIDIA RTX A4000".to_owned()], - gpu_count: 0, // Invalid - name: "test-pod".to_owned(), - image_name: "jgrusewski/foxhunt:latest".to_owned(), - container_disk_in_gb: 50, - network_volume_id: None, - volume_mount_path: None, - docker_start_cmd: Some(vec!["train_ppo".to_owned()]), - container_registry_auth_id: None, - ports: None, - interruptible: None, - env: None, - support_public_ip: None, - }; - - assert!(validate_deployment(&request).is_err()); - } -} diff --git a/foxhunt-deploy/src/runpod/mod.rs b/foxhunt-deploy/src/runpod/mod.rs deleted file mode 100644 index 21bbd6793..000000000 --- a/foxhunt-deploy/src/runpod/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) mod client; -pub(crate) mod deployment; -pub(crate) mod types; - -// Re-export main types for convenience -pub(crate) use client::RunPodClient; -pub(crate) use deployment::{build_deployment_request, select_best_gpu, validate_deployment}; -pub(crate) use types::{GpuType, PodDeploymentRequest, PodDeploymentResponse}; diff --git a/foxhunt-deploy/src/runpod/types.rs b/foxhunt-deploy/src/runpod/types.rs deleted file mode 100644 index a0fe22922..000000000 --- a/foxhunt-deploy/src/runpod/types.rs +++ /dev/null @@ -1,168 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// Request payload for deploying a pod to RunPod -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PodDeploymentRequest { - /// Cloud type (SECURE, COMMUNITY, ALL) - pub cloud_type: String, - - /// Compute type (GPU, CPU) - pub compute_type: String, - - /// Datacenter IDs to prefer - #[serde(skip_serializing_if = "Option::is_none")] - pub data_center_ids: Option>, - - /// GPU type IDs - pub gpu_type_ids: Vec, - - /// Number of GPUs - pub gpu_count: u32, - - /// Pod name - pub name: String, - - /// Docker image name - pub image_name: String, - - /// Container disk size in GB - pub container_disk_in_gb: u32, - - /// Network volume ID - #[serde(skip_serializing_if = "Option::is_none")] - pub network_volume_id: Option, - - /// Volume mount path - #[serde(skip_serializing_if = "Option::is_none")] - pub volume_mount_path: Option, - - /// Docker start command - #[serde(skip_serializing_if = "Option::is_none")] - pub docker_start_cmd: Option>, - - /// Container registry auth ID - #[serde(skip_serializing_if = "Option::is_none")] - pub container_registry_auth_id: Option, - - /// Exposed ports - #[serde(skip_serializing_if = "Option::is_none")] - pub ports: Option>, - - /// Allow interruptible instances - #[serde(skip_serializing_if = "Option::is_none")] - pub interruptible: Option, - - /// Environment variables - #[serde(skip_serializing_if = "Option::is_none")] - pub env: Option>, - - /// Support public IP - #[serde(skip_serializing_if = "Option::is_none")] - pub support_public_ip: Option, -} - -/// Response from RunPod pod deployment -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PodDeploymentResponse { - /// Pod ID - pub id: String, - - /// Machine configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub machine: Option, - - /// Runtime information - #[serde(skip_serializing_if = "Option::is_none")] - pub runtime: Option, - - /// Cost per hour - #[serde(skip_serializing_if = "Option::is_none")] - pub cost_per_hr: Option, -} - -/// Machine information -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct MachineInfo { - /// GPU type - #[serde(skip_serializing_if = "Option::is_none")] - pub gpu_type_id: Option, - - /// GPU count - #[serde(skip_serializing_if = "Option::is_none")] - pub gpu_count: Option, - - /// VRAM per GPU in GB - #[serde(skip_serializing_if = "Option::is_none")] - pub vram_gb: Option, -} - -/// Runtime information -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct RuntimeInfo { - /// Datacenter ID - #[serde(skip_serializing_if = "Option::is_none")] - pub datacenter_id: Option, -} - -/// GPU type information -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct GpuType { - /// GPU type ID - pub id: String, - - /// Display name - pub display_name: String, - - /// VRAM in GB - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_in_gb: Option, - - /// Secure cloud price per hour - #[serde(skip_serializing_if = "Option::is_none")] - pub secure_price: Option, - - /// Community cloud price per hour - #[serde(skip_serializing_if = "Option::is_none")] - pub community_price: Option, - - /// Lowest price (used for auto-selection) - #[serde(skip_serializing_if = "Option::is_none")] - pub lowest_price: Option, -} - -/// Lowest price information -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct LowestPrice { - /// Minimum price per hour - pub min_price_per_hr: f64, - - /// Whether it's interruptible - #[serde(skip_serializing_if = "Option::is_none")] - pub interruptible: Option, -} - -/// RunPod API error response -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct ApiError { - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub message: Option, -} - -/// Pod termination response -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct PodTerminationResponse { - pub id: String, - - #[serde(skip_serializing_if = "Option::is_none")] - pub status: Option, -} diff --git a/foxhunt-deploy/src/s3/mod.rs b/foxhunt-deploy/src/s3/mod.rs deleted file mode 100644 index c682425c0..000000000 --- a/foxhunt-deploy/src/s3/mod.rs +++ /dev/null @@ -1,165 +0,0 @@ -use crate::config::types::S3Config; -use crate::error::{FoxhuntError, Result}; -use aws_config::meta::region::RegionProviderChain; -use aws_config::BehaviorVersion; -use aws_sdk_s3::config::Credentials; -use aws_sdk_s3::Client; -use std::env; - -pub(crate) mod monitor; -pub(crate) mod parser; - -/// S3 client wrapper for log operations -#[derive(Clone)] -pub(crate) struct S3LogClient { - client: Client, - bucket: String, -} - -impl S3LogClient { - /// Create a new S3 client with RunPod configuration - pub(crate) async fn new(config: &S3Config) -> Result { - // Get AWS credentials from environment - let access_key = env::var("AWS_ACCESS_KEY_ID") - .map_err(|_| FoxhuntError::Config("AWS_ACCESS_KEY_ID not set".to_owned()))?; - let secret_key = env::var("AWS_SECRET_ACCESS_KEY") - .map_err(|_| FoxhuntError::Config("AWS_SECRET_ACCESS_KEY not set".to_owned()))?; - - // Create credentials - let credentials = Credentials::new(access_key, secret_key, None, None, "runpod-s3"); - - // Configure region - let region_provider = RegionProviderChain::first_try( - aws_sdk_s3::config::Region::new(config.region.clone()), - ); - - // Build AWS config with custom endpoint - let sdk_config = aws_config::defaults(BehaviorVersion::latest()) - .region(region_provider) - .credentials_provider(credentials) - .load() - .await; - - // Build S3 client with custom endpoint - let s3_config = aws_sdk_s3::config::Builder::from(&sdk_config) - .endpoint_url(&config.endpoint) - .force_path_style(true) - .build(); - - let client = Client::from_conf(s3_config); - - Ok(Self { - client, - bucket: config.bucket.clone(), - }) - } - - /// List all log files for a specific pod - pub(crate) async fn list_log_files(&self, pod_id: &str) -> Result> { - let prefix = format!("ml_training/{}/", pod_id); - - let response = self - .client - .list_objects_v2() - .bucket(&self.bucket) - .prefix(&prefix) - .send() - .await - .map_err(|e| FoxhuntError::S3(format!("Failed to list objects: {}", e)))?; - - let mut log_files = Vec::new(); - let contents = response.contents(); - for object in contents { - if let Some(key) = object.key() { - // Filter for .log files or stdout/stderr - if key.ends_with(".log") - || key.ends_with("stdout") - || key.ends_with("stderr") - { - log_files.push(key.to_owned()); - } - } - } - - // Sort by name (typically includes timestamp) - log_files.sort(); - - Ok(log_files) - } - - /// Download a specific log file - pub(crate) async fn download_log(&self, path: &str) -> Result { - let response = self - .client - .get_object() - .bucket(&self.bucket) - .key(path) - .send() - .await - .map_err(|e| FoxhuntError::S3(format!("Failed to get object {}: {}", path, e)))?; - - let bytes = response - .body - .collect() - .await - .map_err(|e| FoxhuntError::S3(format!("Failed to read object body: {}", e)))?; - - let content = String::from_utf8(bytes.to_vec()) - .map_err(|e| FoxhuntError::S3(format!("Invalid UTF-8 in log file: {}", e)))?; - - Ok(content) - } - - /// Get the size of a log file in bytes - pub(crate) async fn get_log_size(&self, path: &str) -> Result> { - match self - .client - .head_object() - .bucket(&self.bucket) - .key(path) - .send() - .await - { - Ok(response) => Ok(response.content_length()), - Err(e) => { - let err_str = format!("{:?}", e); - if err_str.contains("NotFound") || err_str.contains("404") { - Ok(None) - } else { - Err(FoxhuntError::S3(format!( - "Failed to get object metadata: {}", - e - ))) - } - } - } - } - - /// Download a range of bytes from a log file - pub(crate) async fn download_log_range(&self, path: &str, start: i64, end: i64) -> Result { - let range = format!("bytes={}-{}", start, end); - - let response = self - .client - .get_object() - .bucket(&self.bucket) - .key(path) - .range(range) - .send() - .await - .map_err(|e| { - FoxhuntError::S3(format!("Failed to get object range {}: {}", path, e)) - })?; - - let bytes = response - .body - .collect() - .await - .map_err(|e| FoxhuntError::S3(format!("Failed to read object body: {}", e)))?; - - let content = String::from_utf8(bytes.to_vec()) - .map_err(|e| FoxhuntError::S3(format!("Invalid UTF-8 in log file: {}", e)))?; - - Ok(content) - } -} diff --git a/foxhunt-deploy/src/s3/monitor.rs b/foxhunt-deploy/src/s3/monitor.rs deleted file mode 100644 index 90a9a54a6..000000000 --- a/foxhunt-deploy/src/s3/monitor.rs +++ /dev/null @@ -1,253 +0,0 @@ -use super::parser::{colorize_log_line, detect_completion, filter_line, get_last_n_lines}; -use super::S3LogClient; -use crate::error::{FoxhuntError, Result}; -use colored::Colorize; -use std::time::Duration; -use tokio::time::sleep; - -/// Log monitor for streaming S3 logs -pub(crate) struct LogMonitor { - client: S3LogClient, - pod_id: String, - poll_interval: Duration, -} - -impl LogMonitor { - /// Create a new log monitor - pub(crate) fn new(client: S3LogClient, pod_id: String, poll_interval_secs: u64) -> Self { - Self { - client, - pod_id, - poll_interval: Duration::from_secs(poll_interval_secs), - } - } - - /// Find the primary log file for a pod - async fn find_log_file(&self) -> Result> { - let log_files = self.client.list_log_files(&self.pod_id).await?; - - if log_files.is_empty() { - return Ok(None); - } - - // Priority: training.log > stdout > stderr > other .log files - let training_log = log_files.iter().find(|f| f.ends_with("training.log")); - if let Some(log) = training_log { - return Ok(Some(log.clone())); - } - - let stdout = log_files.iter().find(|f| f.ends_with("stdout")); - if let Some(log) = stdout { - return Ok(Some(log.clone())); - } - - let stderr = log_files.iter().find(|f| f.ends_with("stderr")); - if let Some(log) = stderr { - return Ok(Some(log.clone())); - } - - // Return first .log file - Ok(log_files.first().cloned()) - } - - /// Show recent logs (non-following mode) - pub(crate) async fn show_recent_logs(&self, tail: Option) -> Result<()> { - let log_file = self - .find_log_file() - .await? - .ok_or_else(|| FoxhuntError::S3(format!("No log files found for pod {}", self.pod_id)))?; - - println!( - "{} {}", - "Found log file:".cyan().bold(), - log_file.dimmed() - ); - - let content = self.client.download_log(&log_file).await?; - - let lines = if let Some(n) = tail { - get_last_n_lines(&content, n) - } else { - content.lines().map(|s| s.to_owned()).collect() - }; - - for line in lines { - println!("{}", colorize_log_line(&line)); - } - - Ok(()) - } - - /// Tail logs in real-time (following mode) - #[allow(clippy::non_ascii_literal)] - pub(crate) async fn tail_logs(&self, tail: Option, filter: Option) -> Result<()> { - println!( - "{} {}", - "Monitoring pod:".cyan().bold(), - self.pod_id.yellow() - ); - println!( - "{} {}", - "Poll interval:".cyan().bold(), - format!("{}s", self.poll_interval.as_secs()).yellow() - ); - if let Some(ref pattern) = filter { - println!( - "{} {}", - "Filter pattern:".cyan().bold(), - pattern.yellow() - ); - } - println!("{}", "─".repeat(80).dimmed()); - - let log_file = loop { - match self.find_log_file().await { - Ok(Some(file)) => { - println!( - "{} {}", - "Found log file:".green().bold(), - file.dimmed() - ); - break file; - } - Ok(None) => { - println!( - "{} (retrying in {}s...)", - "No log files found yet".yellow(), - self.poll_interval.as_secs() - ); - sleep(self.poll_interval).await; - } - Err(e) => { - println!("{} {} (retrying...)", "Error:".red().bold(), e); - sleep(self.poll_interval).await; - } - } - }; - - println!("{}", "─".repeat(80).dimmed()); - - // Track last read position - let mut last_position: i64 = 0; - - // Show initial tail lines if requested - if let Some(n) = tail { - match self.client.download_log(&log_file).await { - Ok(content) => { - let lines = get_last_n_lines(&content, n); - for line in &lines { - if filter_line(line, filter.as_deref()) { - println!("{}", colorize_log_line(line)); - } - } - // Update position to end of file - last_position = content.len() as i64; - } - Err(e) => { - println!("{} {}", "Error reading initial logs:".yellow(), e); - } - } - } - - // Main monitoring loop - let mut retry_count = 0; - const MAX_RETRIES: u32 = 3; - - loop { - // Check current file size - match self.client.get_log_size(&log_file).await { - Ok(Some(current_size)) => { - retry_count = 0; // Reset on success - - if current_size > last_position { - // New content available - match self - .client - .download_log_range(&log_file, last_position, current_size - 1) - .await - { - Ok(new_content) => { - let lines: Vec<&str> = new_content.lines().collect(); - for line in lines { - if filter_line(line, filter.as_deref()) { - println!("{}", colorize_log_line(line)); - - // Check for completion - if detect_completion(line) { - println!("{}", "─".repeat(80).dimmed()); - println!( - "{} {}", - "✓".green().bold(), - "Training completed!".green().bold() - ); - return Ok(()); - } - } - } - last_position = current_size; - } - Err(e) => { - println!( - "{} {}", - "Error reading new content:".yellow(), - e - ); - } - } - } - } - Ok(None) => { - println!( - "{} (file may have been deleted)", - "Log file not found".yellow() - ); - } - Err(e) => { - retry_count += 1; - println!( - "{} {} (retry {}/{})", - "Error checking file size:".yellow(), - e, - retry_count, - MAX_RETRIES - ); - - if retry_count >= MAX_RETRIES { - return Err(FoxhuntError::S3(format!( - "Failed to monitor logs after {} retries", - MAX_RETRIES - ))); - } - } - } - - // Wait before next poll - sleep(self.poll_interval).await; - } - } - - /// List all available log files for the pod - #[allow(clippy::non_ascii_literal)] - pub(crate) async fn list_logs(&self) -> Result<()> { - println!( - "{} {}", - "Searching for logs for pod:".cyan().bold(), - self.pod_id.yellow() - ); - - let log_files = self.client.list_log_files(&self.pod_id).await?; - - if log_files.is_empty() { - println!("{}", "No log files found".yellow()); - return Ok(()); - } - - println!("{}", "─".repeat(80).dimmed()); - println!("{} log file(s):", log_files.len()); - for file in log_files { - println!(" {} {}", "•".cyan(), file); - } - - Ok(()) - } -} diff --git a/foxhunt-deploy/src/s3/parser.rs b/foxhunt-deploy/src/s3/parser.rs deleted file mode 100644 index 7409e999f..000000000 --- a/foxhunt-deploy/src/s3/parser.rs +++ /dev/null @@ -1,214 +0,0 @@ -use colored::Colorize; -use regex::Regex; - -/// Log level detected in log lines -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum LogLevel { - Info, - Warn, - Error, - Debug, - Trace, - Unknown, -} - -impl LogLevel { - /// Detect log level from a line - pub(crate) fn detect(line: &str) -> Self { - let upper = line.to_uppercase(); - if upper.contains("ERROR") || upper.contains("FATAL") || upper.contains("CRITICAL") { - LogLevel::Error - } else if upper.contains("WARN") || upper.contains("WARNING") { - LogLevel::Warn - } else if upper.contains("INFO") { - LogLevel::Info - } else if upper.contains("DEBUG") { - LogLevel::Debug - } else if upper.contains("TRACE") { - LogLevel::Trace - } else { - LogLevel::Unknown - } - } -} - -/// Colorize a log line based on its level -pub(crate) fn colorize_log_line(line: &str) -> String { - let level = LogLevel::detect(line); - - match level { - LogLevel::Error => line.red().to_string(), - LogLevel::Warn => line.yellow().to_string(), - LogLevel::Info => line.green().to_string(), - LogLevel::Debug => line.cyan().to_string(), - LogLevel::Trace => line.dimmed().to_string(), - LogLevel::Unknown => line.to_owned(), - } -} - -/// Training metrics extracted from log lines -#[derive(Debug, Clone, Default)] -pub(crate) struct TrainingMetrics { - pub epoch: Option, - pub loss: Option, - pub accuracy: Option, - pub learning_rate: Option, -} - -/// Parse training metrics from a log line -#[allow(dead_code)] -pub(crate) fn parse_training_metrics(line: &str) -> Option { - let mut metrics = TrainingMetrics::default(); - let mut found_any = false; - - // Epoch patterns - let epoch_patterns = [ - Regex::new(r"epoch[:\s]+(\d+)").ok()?, - Regex::new(r"Epoch[:\s]+(\d+)").ok()?, - Regex::new(r"EPOCH[:\s]+(\d+)").ok()?, - ]; - - for pattern in &epoch_patterns { - if let Some(caps) = pattern.captures(line) { - if let Some(epoch_str) = caps.get(1) { - if let Ok(epoch) = epoch_str.as_str().parse::() { - metrics.epoch = Some(epoch); - found_any = true; - break; - } - } - } - } - - // Loss patterns - let loss_patterns = [ - Regex::new(r"loss[:\s]+([0-9.]+)").ok()?, - Regex::new(r"Loss[:\s]+([0-9.]+)").ok()?, - Regex::new(r"LOSS[:\s]+([0-9.]+)").ok()?, - ]; - - for pattern in &loss_patterns { - if let Some(caps) = pattern.captures(line) { - if let Some(loss_str) = caps.get(1) { - if let Ok(loss) = loss_str.as_str().parse::() { - metrics.loss = Some(loss); - found_any = true; - break; - } - } - } - } - - // Learning rate patterns - let lr_patterns = [ - Regex::new(r"lr[:\s]+([0-9.e-]+)").ok()?, - Regex::new(r"learning_rate[:\s]+([0-9.e-]+)").ok()?, - ]; - - for pattern in &lr_patterns { - if let Some(caps) = pattern.captures(line) { - if let Some(lr_str) = caps.get(1) { - if let Ok(lr) = lr_str.as_str().parse::() { - metrics.learning_rate = Some(lr); - found_any = true; - break; - } - } - } - } - - found_any.then_some(metrics) -} - -/// Detect if a log line indicates training completion -pub(crate) fn detect_completion(line: &str) -> bool { - let lower = line.to_lowercase(); - lower.contains("training complete") - || lower.contains("training finished") - || lower.contains("training done") - || lower.contains("saved final model") - || lower.contains("checkpoint saved") - || (lower.contains("epoch") && lower.contains("/") && lower.contains("100%")) -} - -/// Filter a log line by regex pattern -pub(crate) fn filter_line(line: &str, pattern_str: Option<&str>) -> bool { - match pattern_str { - None => true, - Some(pat) => { - if let Ok(regex) = Regex::new(pat) { - regex.is_match(line) - } else { - // Invalid regex, show all lines - true - } - } - } -} - -/// Get last N lines from log content -pub(crate) fn get_last_n_lines(content: &str, n: usize) -> Vec { - let lines: Vec = content.lines().map(|s| s.to_owned()).collect(); - let start_index = lines.len().saturating_sub(n); - lines[start_index..].to_vec() -} - -/// Format a summary of training metrics -#[allow(dead_code)] -pub(crate) fn format_metrics_summary(metrics: &TrainingMetrics) -> String { - let mut parts = Vec::new(); - - if let Some(epoch) = metrics.epoch { - parts.push(format!("Epoch: {}", epoch)); - } - if let Some(loss) = metrics.loss { - parts.push(format!("Loss: {:.6}", loss)); - } - if let Some(acc) = metrics.accuracy { - parts.push(format!("Acc: {:.4}", acc)); - } - if let Some(lr) = metrics.learning_rate { - parts.push(format!("LR: {:.6}", lr)); - } - - if parts.is_empty() { - "No metrics".to_owned() - } else { - parts.join(" | ") - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_detect_log_level() { - assert_eq!(LogLevel::detect("ERROR: Something failed"), LogLevel::Error); - assert_eq!(LogLevel::detect("WARN: Low memory"), LogLevel::Warn); - assert_eq!(LogLevel::detect("INFO: Starting training"), LogLevel::Info); - } - - #[test] - fn test_parse_training_metrics() { - let line = "INFO: Epoch: 10, Loss: 0.345, lr: 0.001"; - let metrics = parse_training_metrics(line).unwrap(); - assert_eq!(metrics.epoch, Some(10)); - assert_eq!(metrics.loss, Some(0.345)); - assert_eq!(metrics.learning_rate, Some(0.001)); - } - - #[test] - fn test_detect_completion() { - assert!(detect_completion("Training complete! Model saved.")); - assert!(detect_completion("Checkpoint saved at epoch 100")); - assert!(!detect_completion("Starting training...")); - } - - #[test] - fn test_get_last_n_lines() { - let content = "line1\nline2\nline3\nline4\nline5"; - let last_3 = get_last_n_lines(content, 3); - assert_eq!(last_3, vec!["line3", "line4", "line5"]); - } -} diff --git a/foxhunt-deploy/src/utils/mod.rs b/foxhunt-deploy/src/utils/mod.rs deleted file mode 100644 index 227039877..000000000 --- a/foxhunt-deploy/src/utils/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) mod terminal; - -// Re-export terminal functions for convenience -pub(crate) use terminal::{error, info, step, success}; diff --git a/foxhunt-deploy/src/utils/terminal.rs b/foxhunt-deploy/src/utils/terminal.rs deleted file mode 100644 index 9f0503afc..000000000 --- a/foxhunt-deploy/src/utils/terminal.rs +++ /dev/null @@ -1,50 +0,0 @@ -use colored::Colorize; - -/// Print success message with green checkmark -#[allow(clippy::non_ascii_literal)] -pub(crate) fn success(msg: impl AsRef) { - println!("{} {}", "✓".green().bold(), msg.as_ref()); -} - -/// Print error message with red X -#[allow(clippy::non_ascii_literal)] -pub(crate) fn error(msg: impl AsRef) { - eprintln!("{} {}", "✗".red().bold(), msg.as_ref()); -} - -/// Print info message with blue icon -#[allow(clippy::non_ascii_literal)] -pub(crate) fn info(msg: impl AsRef) { - println!("{} {}", "ℹ".blue().bold(), msg.as_ref()); -} - -/// Print warning message with yellow icon -#[allow(dead_code)] -#[allow(clippy::non_ascii_literal)] -pub(crate) fn warning(msg: impl AsRef) { - println!("{} {}", "⚠".yellow().bold(), msg.as_ref()); -} - -/// Print numbered step progress -pub(crate) fn step(step: usize, total: usize, msg: impl AsRef) { - println!( - "{} {}", - format!("[{}/{}]", step, total).cyan().bold(), - msg.as_ref() - ); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_terminal_functions() { - // Just verify these don't panic - success("Test success"); - error("Test error"); - info("Test info"); - warning("Test warning"); - step(1, 5, "Test step"); - } -} diff --git a/foxhunt-deploy/tests/docker_integration_tests.rs b/foxhunt-deploy/tests/docker_integration_tests.rs deleted file mode 100644 index f8e66b2d5..000000000 --- a/foxhunt-deploy/tests/docker_integration_tests.rs +++ /dev/null @@ -1,169 +0,0 @@ -use assert_cmd::Command; -use predicates::prelude::*; - -#[test] -fn test_build_command_help() { - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("build").arg("--help"); - - cmd.assert() - .success() - .stdout(predicate::str::contains("Build Docker image")); -} - -#[test] -fn test_build_requires_config() { - // Test that build command requires valid config - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("--config") - .arg("/tmp/nonexistent-config.toml") - .arg("build"); - - cmd.assert() - .failure() - .stderr(predicate::str::contains("Config file not found")); -} - -#[test] -fn test_build_validates_dockerfile_existence() { - // This test verifies that the build command would check for Dockerfile existence - // Note: We can't actually run docker build in CI without Docker installed - // This is a documentation test showing expected behavior - - let result = std::panic::catch_unwind(|| { - // If Docker is not available, the verify_docker_available() will fail - // This is the expected behavior - }); - - assert!(result.is_ok()); -} - -#[test] -fn test_build_options_builder_pattern() { - // Test the DockerBuildOptions builder pattern (unit test style) - // This verifies the API design is correct - - // Note: We import via the binary's exposed API - // In a real scenario, these would be re-exported from lib.rs - // For now, this documents the expected API - - let expected_tag = "test:latest"; - let expected_dockerfile = "Dockerfile.test"; - let expected_context = "/tmp/test"; - - // Verify our design expectations - assert_eq!(expected_tag, "test:latest"); - assert_eq!(expected_dockerfile, "Dockerfile.test"); - assert_eq!(expected_context, "/tmp/test"); -} - -#[test] -fn test_build_error_handling_no_docker() { - // This test documents expected behavior when Docker is not installed - // The actual error would be: "Docker is not installed" - - let error_message = "Docker is not installed. Please install Docker Desktop or Docker Engine."; - assert!(error_message.contains("Docker")); - assert!(error_message.contains("install")); -} - -#[test] -fn test_build_error_handling_daemon_not_running() { - // This test documents expected behavior when Docker daemon is not running - // The actual error would be: "Docker daemon is not running" - - let error_message = "Docker daemon is not running. Please start Docker Desktop or Docker service."; - assert!(error_message.contains("daemon")); - assert!(error_message.contains("not running")); -} - -#[test] -fn test_push_error_handling_no_auth() { - // This test documents expected behavior when registry auth fails - // The actual error would contain "authentication" or "unauthorized" - - let error_message = "Docker registry authentication failed. Please run 'docker login' first."; - assert!(error_message.contains("authentication")); - assert!(error_message.contains("docker login")); -} - -#[test] -fn test_push_error_handling_image_not_found() { - // This test documents expected behavior when image doesn't exist - - let error_message = "Image 'test:latest' does not exist locally. Build it first."; - assert!(error_message.contains("does not exist")); - assert!(error_message.contains("Build it first")); -} - -#[test] -fn test_build_args_defaults() { - // Test CLI argument defaults - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("build").arg("--help"); - - cmd.assert() - .success() - .stdout(predicate::str::contains("default: latest")) - .stdout(predicate::str::contains("Dockerfile.foxhunt-build")); -} - -#[test] -fn test_build_with_no_push_flag() { - // Test that --no-push flag is recognized - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("build").arg("--help"); - - cmd.assert() - .success() - .stdout(predicate::str::contains("no-push")) - .stdout(predicate::str::contains("Skip pushing")); -} - -#[test] -fn test_build_with_no_cache_flag() { - // Test that --no-cache flag is recognized - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("build").arg("--help"); - - cmd.assert() - .success() - .stdout(predicate::str::contains("no-cache")) - .stdout(predicate::str::contains("Disable Docker build cache")); -} - -#[test] -fn test_build_with_custom_tag() { - // Test that custom tag argument works - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("build").arg("--help"); - - cmd.assert() - .success() - .stdout(predicate::str::contains("--tag")) - .stdout(predicate::str::contains("Docker image tag")); -} - -#[test] -fn test_build_with_custom_dockerfile() { - // Test that custom Dockerfile argument works - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("build").arg("--help"); - - cmd.assert() - .success() - .stdout(predicate::str::contains("--dockerfile")) - .stdout(predicate::str::contains("Dockerfile path")); -} - -#[test] -fn test_build_with_custom_context() { - // Test that custom context argument works - let mut cmd = Command::cargo_bin("foxhunt-deploy").unwrap(); - cmd.arg("build").arg("--help"); - - cmd.assert() - .success() - .stdout(predicate::str::contains("--context")) - .stdout(predicate::str::contains("Docker build context path")); -} diff --git a/tli/CONFIG_FILE_SUPPORT.md b/fxt/CONFIG_FILE_SUPPORT.md similarity index 100% rename from tli/CONFIG_FILE_SUPPORT.md rename to fxt/CONFIG_FILE_SUPPORT.md diff --git a/tli/Cargo.toml b/fxt/Cargo.toml similarity index 97% rename from tli/Cargo.toml rename to fxt/Cargo.toml index c5934b677..3a6d05478 100644 --- a/tli/Cargo.toml +++ b/fxt/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "tli" +name = "fxt" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -11,11 +11,11 @@ documentation.workspace = true publish.workspace = true keywords.workspace = true categories.workspace = true -description = "Terminal Line Interface for Foxhunt HFT Trading System" +description = "Foxhunt CLI — command-line interface for Foxhunt HFT Trading System" -# TLI binary - client-only terminal interface +# CLI binary [[bin]] -name = "tli" +name = "fxt" path = "src/main.rs" [dependencies] diff --git a/tli/TUNE_COMMAND_README.md b/fxt/TUNE_COMMAND_README.md similarity index 100% rename from tli/TUNE_COMMAND_README.md rename to fxt/TUNE_COMMAND_README.md diff --git a/tli/benches/client_performance.rs b/fxt/benches/client_performance.rs similarity index 98% rename from tli/benches/client_performance.rs rename to fxt/benches/client_performance.rs index 732e456c3..7f7003798 100644 --- a/tli/benches/client_performance.rs +++ b/fxt/benches/client_performance.rs @@ -23,8 +23,8 @@ // DISABLED: use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; // DISABLED: use std::time::Duration; -// DISABLED: use tli::prelude::*; -// DISABLED: use tli::{ServiceEndpoints, TliClient}; +// DISABLED: use fxt::prelude::*; +// DISABLED: use fxt::{ServiceEndpoints, TliClient}; // DISABLED: use tokio::runtime::Runtime; /* @@ -100,7 +100,7 @@ fn bench_endpoint_parsing(c: &mut Criterion) { fn bench_request_serialization(c: &mut Criterion) { let mut group = c.benchmark_group("request_serialization"); - use tli::proto::trading::*; + use fxt::proto::trading::*; // Order submission request group.bench_function("submit_order_request", |b| { @@ -166,7 +166,7 @@ fn bench_request_serialization(c: &mut Criterion) { fn bench_response_deserialization(c: &mut Criterion) { let mut group = c.benchmark_group("response_deserialization"); - use tli::proto::trading::*; + use fxt::proto::trading::*; // Order response let order_response_json = r#"{ @@ -350,7 +350,7 @@ fn bench_memory_patterns(c: &mut Criterion) { // Large response handling group.bench_function("large_response_handling", |b| { - use tli::proto::trading::*; + use fxt::proto::trading::*; b.iter(|| { let mut orders = Vec::new(); @@ -379,7 +379,7 @@ fn bench_memory_patterns(c: &mut Criterion) { // Streaming data structures group.bench_function("streaming_data_structures", |b| { - use tli::proto::trading::*; + use fxt::proto::trading::*; b.iter(|| { let mut updates = Vec::new(); @@ -457,7 +457,7 @@ fn bench_config_operations(c: &mut Criterion) { fn bench_health_check_operations(c: &mut Criterion) { let mut group = c.benchmark_group("health_check_operations"); - use tli::client::ServiceHealth; + use fxt::client::ServiceHealth; // Health status creation group.bench_function("health_status_creation", |b| { diff --git a/tli/benches/configuration_benchmarks.rs b/fxt/benches/configuration_benchmarks.rs similarity index 99% rename from tli/benches/configuration_benchmarks.rs rename to fxt/benches/configuration_benchmarks.rs index bbafeb0d5..d023a963c 100644 --- a/tli/benches/configuration_benchmarks.rs +++ b/fxt/benches/configuration_benchmarks.rs @@ -20,8 +20,8 @@ // DISABLED: use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; // DISABLED: use std::collections::HashMap; // DISABLED: use std::time::Duration; -// DISABLED: use tli::prelude::*; -// DISABLED: use tli::types::*; +// DISABLED: use fxt::prelude::*; +// DISABLED: use fxt::types::*; // DISABLED: use tokio::runtime::Runtime; /* @@ -140,7 +140,7 @@ fn bench_type_conversions(c: &mut Criterion) { let mut group = c.benchmark_group("type_conversions"); use common::OrderSide; - use tli::proto::trading::{OrderStatus, OrderType}; + use fxt::proto::trading::{OrderStatus, OrderType}; // Order side conversions let sides = vec![OrderSide::Buy, OrderSide::Sell]; diff --git a/tli/benches/encryption_performance.rs b/fxt/benches/encryption_performance.rs similarity index 96% rename from tli/benches/encryption_performance.rs rename to fxt/benches/encryption_performance.rs index 9c9ecaeb6..7be34f13c 100644 --- a/tli/benches/encryption_performance.rs +++ b/fxt/benches/encryption_performance.rs @@ -1,5 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use tli::auth::token_manager::{FileTokenStorage, TokenStorage}; +use fxt::auth::token_manager::{FileTokenStorage, TokenStorage}; fn create_test_storage() -> FileTokenStorage { // Use default directory - benchmark will measure real-world performance @@ -56,7 +56,7 @@ fn bench_roundtrip_encrypted(c: &mut Criterion) { } fn bench_key_derivation(c: &mut Criterion) { - use tli::auth::KeyManager; + use fxt::auth::KeyManager; c.bench_function("key_derivation_first_call", |b| { b.iter(|| { diff --git a/tli/benches/serialization_benchmarks.rs b/fxt/benches/serialization_benchmarks.rs similarity index 99% rename from tli/benches/serialization_benchmarks.rs rename to fxt/benches/serialization_benchmarks.rs index aeb605ee7..699f05281 100644 --- a/tli/benches/serialization_benchmarks.rs +++ b/fxt/benches/serialization_benchmarks.rs @@ -14,12 +14,12 @@ //! - `MetricValue` - proto has Metric instead //! //! TODO: Fix this benchmark by either: -//! 1. Adding the missing proto message definitions to tli/proto/trading.proto +//! 1. Adding the missing proto message definitions to fxt/proto/trading.proto //! 2. Rewriting benchmarks to use existing proto types (GetOrderStatusResponse, OrderUpdateEvent, Metric) //! 3. Creating separate benchmark-specific proto messages //! //! Related files: -//! - tli/proto/trading.proto - contains actual protobuf definitions +//! - fxt/proto/trading.proto - contains actual protobuf definitions //! - tli/build.rs - compiles proto files to Rust code //! //! See Wave 36 - Agent 1 for context @@ -30,7 +30,7 @@ // DISABLED: use prost::Message; // DISABLED: use std::collections::HashMap; // DISABLED: use std::time::Duration; -// DISABLED: use tli::proto::trading::*; +// DISABLED: use fxt::proto::trading::*; /* /// Benchmark protobuf serialization diff --git a/tli/build.rs b/fxt/build.rs similarity index 100% rename from tli/build.rs rename to fxt/build.rs diff --git a/tli/config.toml.example b/fxt/config.toml.example similarity index 100% rename from tli/config.toml.example rename to fxt/config.toml.example diff --git a/tli/docs/AUTHENTICATION.md b/fxt/docs/AUTHENTICATION.md similarity index 100% rename from tli/docs/AUTHENTICATION.md rename to fxt/docs/AUTHENTICATION.md diff --git a/tli/docs/ML_TRADING_COMMANDS.md b/fxt/docs/ML_TRADING_COMMANDS.md similarity index 100% rename from tli/docs/ML_TRADING_COMMANDS.md rename to fxt/docs/ML_TRADING_COMMANDS.md diff --git a/tli/docs/USAGE.md b/fxt/docs/USAGE.md similarity index 100% rename from tli/docs/USAGE.md rename to fxt/docs/USAGE.md diff --git a/tli/examples/complete_client_example.rs b/fxt/examples/complete_client_example.rs similarity index 99% rename from tli/examples/complete_client_example.rs rename to fxt/examples/complete_client_example.rs index c9f857b05..eed24682c 100644 --- a/tli/examples/complete_client_example.rs +++ b/fxt/examples/complete_client_example.rs @@ -4,7 +4,7 @@ //! to connect to both Trading Service and Backtesting Service. #![allow(unused_crate_dependencies)] -use tli::prelude::*; +use fxt::prelude::*; use tokio::time::{sleep, Duration}; use tracing::{error, info, warn}; diff --git a/tli/examples/config_demo.rs b/fxt/examples/config_demo.rs similarity index 100% rename from tli/examples/config_demo.rs rename to fxt/examples/config_demo.rs diff --git a/tli/examples/config_management_placeholder.rs b/fxt/examples/config_management_placeholder.rs similarity index 100% rename from tli/examples/config_management_placeholder.rs rename to fxt/examples/config_management_placeholder.rs diff --git a/tli/examples/security_example.rs b/fxt/examples/security_example.rs similarity index 100% rename from tli/examples/security_example.rs rename to fxt/examples/security_example.rs diff --git a/tli/proptest-regressions/tests.txt b/fxt/proptest-regressions/tests.txt similarity index 100% rename from tli/proptest-regressions/tests.txt rename to fxt/proptest-regressions/tests.txt diff --git a/tli/proto/config.proto b/fxt/proto/config.proto similarity index 100% rename from tli/proto/config.proto rename to fxt/proto/config.proto diff --git a/tli/proto/health.proto b/fxt/proto/health.proto similarity index 100% rename from tli/proto/health.proto rename to fxt/proto/health.proto diff --git a/tli/proto/ml.proto b/fxt/proto/ml.proto similarity index 100% rename from tli/proto/ml.proto rename to fxt/proto/ml.proto diff --git a/tli/proto/ml_training.proto b/fxt/proto/ml_training.proto similarity index 100% rename from tli/proto/ml_training.proto rename to fxt/proto/ml_training.proto diff --git a/tli/proto/trading.proto b/fxt/proto/trading.proto similarity index 100% rename from tli/proto/trading.proto rename to fxt/proto/trading.proto diff --git a/tli/proto/trading_agent.proto b/fxt/proto/trading_agent.proto similarity index 100% rename from tli/proto/trading_agent.proto rename to fxt/proto/trading_agent.proto diff --git a/tli/src/auth/encryption.rs b/fxt/src/auth/encryption.rs similarity index 99% rename from tli/src/auth/encryption.rs rename to fxt/src/auth/encryption.rs index 1a1daa4ec..f4238e172 100644 --- a/tli/src/auth/encryption.rs +++ b/fxt/src/auth/encryption.rs @@ -42,7 +42,7 @@ impl EncryptionFormat { /// /// # Examples /// ``` - /// use tli::auth::encryption::EncryptionFormat; + /// use fxt::auth::encryption::EncryptionFormat; /// /// let hex_data = "4a5b6c7d8e9f0a1b"; /// assert_eq!(EncryptionFormat::detect(hex_data), EncryptionFormat::HexEncoded); @@ -68,7 +68,7 @@ impl EncryptionFormat { /// /// # Examples /// ``` - /// use tli::auth::encryption::EncryptionFormat; + /// use fxt::auth::encryption::EncryptionFormat; /// /// assert!(!EncryptionFormat::is_encrypted("4a5b6c7d")); /// assert!(EncryptionFormat::is_encrypted("ENC:data")); @@ -101,7 +101,7 @@ impl EncryptionFormat { /// /// # Examples /// ```no_run -/// use tli::auth::encryption::encrypt_token; +/// use fxt::auth::encryption::encrypt_token; /// /// let key = [0u8; 32]; // 32-byte key (in production, use proper key derivation) /// let token = "my_secret_token"; @@ -180,7 +180,7 @@ pub fn encrypt_token(token: &str, key: &[u8]) -> Result { /// /// # Examples /// ```no_run -/// use tli::auth::encryption::{encrypt_token, decrypt_token}; +/// use fxt::auth::encryption::{encrypt_token, decrypt_token}; /// /// let key = [0u8; 32]; /// let token = "my_secret_token"; @@ -279,7 +279,7 @@ pub fn decrypt_token(encrypted: &str, key: &[u8]) -> Result /// /// # Examples /// ```no_run -/// use tli::auth::encryption::read_token_auto; +/// use fxt::auth::encryption::read_token_auto; /// /// let key = [0u8; 32]; /// @@ -338,7 +338,7 @@ pub fn read_token_auto(encrypted_data: &str, key: &[u8]) -> Result anyhow::Result<()> { /// let args = TradeArgs { diff --git a/tli/src/commands/trade_ml.rs b/fxt/src/commands/trade_ml.rs similarity index 100% rename from tli/src/commands/trade_ml.rs rename to fxt/src/commands/trade_ml.rs diff --git a/tli/src/commands/train/list.rs b/fxt/src/commands/train/list.rs similarity index 100% rename from tli/src/commands/train/list.rs rename to fxt/src/commands/train/list.rs diff --git a/tli/src/commands/train/mod.rs b/fxt/src/commands/train/mod.rs similarity index 100% rename from tli/src/commands/train/mod.rs rename to fxt/src/commands/train/mod.rs diff --git a/tli/src/commands/train/progress_tracker.rs b/fxt/src/commands/train/progress_tracker.rs similarity index 100% rename from tli/src/commands/train/progress_tracker.rs rename to fxt/src/commands/train/progress_tracker.rs diff --git a/tli/src/commands/train/status.rs b/fxt/src/commands/train/status.rs similarity index 100% rename from tli/src/commands/train/status.rs rename to fxt/src/commands/train/status.rs diff --git a/tli/src/commands/tune.rs b/fxt/src/commands/tune.rs similarity index 100% rename from tli/src/commands/tune.rs rename to fxt/src/commands/tune.rs diff --git a/tli/src/commands/tune_impl.rs b/fxt/src/commands/tune_impl.rs similarity index 100% rename from tli/src/commands/tune_impl.rs rename to fxt/src/commands/tune_impl.rs diff --git a/tli/src/commands/tune_stream.rs b/fxt/src/commands/tune_stream.rs similarity index 100% rename from tli/src/commands/tune_stream.rs rename to fxt/src/commands/tune_stream.rs diff --git a/tli/src/config.rs b/fxt/src/config.rs similarity index 98% rename from tli/src/config.rs rename to fxt/src/config.rs index ea782b4f4..2a562d8ce 100644 --- a/tli/src/config.rs +++ b/fxt/src/config.rs @@ -69,7 +69,7 @@ impl TliConfig { /// /// # Examples /// ```no_run - /// use tli::config::TliConfig; + /// use fxt::config::TliConfig; /// /// let config = TliConfig::load().unwrap(); /// println!("API Gateway: {}", config.api_gateway_url); @@ -96,7 +96,7 @@ impl TliConfig { /// /// # Examples /// ```no_run - /// use tli::config::TliConfig; + /// use fxt::config::TliConfig; /// /// let config = TliConfig { /// api_gateway_url: "http://localhost:50051".to_string(), diff --git a/tli/src/error.rs b/fxt/src/error.rs similarity index 100% rename from tli/src/error.rs rename to fxt/src/error.rs diff --git a/tli/src/lib.rs b/fxt/src/lib.rs similarity index 99% rename from tli/src/lib.rs rename to fxt/src/lib.rs index a00964812..8e8180a16 100644 --- a/tli/src/lib.rs +++ b/fxt/src/lib.rs @@ -30,7 +30,7 @@ //! //! ## Example Usage //! ```rust,no_run -//! use tli::prelude::*; +//! use fxt::prelude::*; //! //! #[tokio::main] //! async fn main() -> TliResult<()> { diff --git a/tli/src/main.rs b/fxt/src/main.rs similarity index 91% rename from tli/src/main.rs rename to fxt/src/main.rs index 3f86ed513..4e8826498 100644 --- a/tli/src/main.rs +++ b/fxt/src/main.rs @@ -11,8 +11,8 @@ use colored::Colorize; use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation}; use serde::{Deserialize, Serialize}; use std::time::{SystemTime, UNIX_EPOCH}; -use tli::auth::token_manager::FileTokenStorage; -use tli::{ +use fxt::auth::token_manager::FileTokenStorage; +use fxt::{ commands::{ agent::{execute_agent_command, AgentArgs}, auth::{execute_auth_command, AuthCommand}, @@ -68,7 +68,7 @@ use zeroize as _; /// TLI Command Line Interface #[derive(Parser)] #[clap( - name = "tli", + name = "fxt", version, about = "Foxhunt Trading System Terminal Interface", long_about = "Foxhunt Trading System Terminal Interface\n\n\ @@ -120,10 +120,10 @@ enum Commands { Supported models: DQN, PPO, MAMBA_2, TFT, TLOB, LIQUID\n\ Uses Optuna for Bayesian optimization.\n\n\ Examples:\n\ - tli tune start --model DQN --trials 100\n\ - tli tune status --job-id \n\ - tli tune best --job-id \n\ - tli tune stop --job-id " + fxt tune start --model DQN --trials 100\n\ + fxt tune status --job-id \n\ + fxt tune best --job-id \n\ + fxt tune stop --job-id " )] Tune { #[clap(subcommand)] @@ -139,11 +139,11 @@ enum Commands { - Watch real-time training progress\n\ - Stop jobs gracefully or forcefully\n\n\ Examples:\n\ - tli train list\n\ - tli train list --status RUNNING --model TFT\n\ - tli train status train_dqn_es_20251022_1430\n\ - tli train watch train_tft_nq_20251022_1500\n\ - tli train stop train_ppo_es_20251022_1600" + fxt train list\n\ + fxt train list --status RUNNING --model TFT\n\ + fxt train status train_dqn_es_20251022_1430\n\ + fxt train watch train_tft_nq_20251022_1500\n\ + fxt train stop train_ppo_es_20251022_1600" )] Train { #[clap(subcommand)] @@ -155,10 +155,10 @@ enum Commands { JWT tokens are stored securely in OS keyring.\n\ Tokens auto-refresh when expiring (within 60 seconds).\n\n\ Examples:\n\ - tli auth login --username trader1\n\ - tli auth status\n\ - tli auth refresh\n\ - tli auth logout")] + fxt auth login --username trader1\n\ + fxt auth status\n\ + fxt auth refresh\n\ + fxt auth logout")] Auth { #[clap(subcommand)] auth_cmd: AuthCommand, @@ -170,8 +170,8 @@ enum Commands { Subcommands:\n\ allocate-portfolio - Allocate capital across selected assets\n\n\ Examples:\n\ - tli agent allocate-portfolio --selection-id abc-123 --total-capital 100000\n\ - tli agent allocate-portfolio --selection-id abc-123 --total-capital 100000 --strategy risk-parity" + fxt agent allocate-portfolio --selection-id abc-123 --total-capital 100000\n\ + fxt agent allocate-portfolio --selection-id abc-123 --total-capital 100000 --strategy risk-parity" )] Agent { #[command(flatten)] @@ -223,8 +223,8 @@ struct Claims { /// - `Ok(String)` - Valid access token (possibly refreshed) /// - `Err(anyhow::Error)` - Token not found, refresh failed, or invalid format async fn load_jwt_token(api_gateway_url: &str) -> Result { - use tli::auth::login::LoginClient; - use tli::auth::token_manager::{AuthTokenManager, TokenStorage}; + use fxt::auth::login::LoginClient; + use fxt::auth::token_manager::{AuthTokenManager, TokenStorage}; use tonic::transport::Channel; let storage = FileTokenStorage::new().context("Failed to initialize file token storage")?; @@ -242,7 +242,7 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result { if storage.get_refresh_token().await?.is_none() { anyhow::bail!( "No refresh token available. Please login: {}", - "tli auth login".bright_cyan() + "fxt auth login".bright_cyan() ); } @@ -283,14 +283,14 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result { if stored_refresh.is_empty() { anyhow::bail!( "Token refresh succeeded but refresh token is empty in keyring. Please login again: {}", - "tli auth login".bright_cyan() + "fxt auth login".bright_cyan() ) } }, None => { anyhow::bail!( "Token refresh succeeded but refresh token not found in keyring. Please login again: {}", - "tli auth login".bright_cyan() + "fxt auth login".bright_cyan() ) }, } @@ -301,7 +301,7 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result { None => { anyhow::bail!( "Token refresh succeeded but new token not found in keyring. Please login again: {}", - "tli auth login".bright_cyan() + "fxt auth login".bright_cyan() ) }, } @@ -313,7 +313,7 @@ async fn load_jwt_token(api_gateway_url: &str) -> Result { None => { anyhow::bail!( "Not authenticated. Please run: {} first", - "tli auth login".bright_cyan() + "fxt auth login".bright_cyan() ) }, } @@ -426,7 +426,7 @@ mod tests { fn test_cli_parsing_tune_command() { // Test that Cli struct parses tune command correctly let cli = Cli::parse_from(&[ - "tli", + "fxt", "--api-gateway-url", "http://test.com", "tune", @@ -445,7 +445,7 @@ mod tests { #[test] fn test_cli_parsing_auth_command() { - let cli = Cli::parse_from(&["tli", "auth", "status"]); + let cli = Cli::parse_from(&["fxt", "auth", "status"]); match cli.command { Commands::Auth { .. } => {}, @@ -456,7 +456,7 @@ mod tests { #[test] fn test_cli_default_values() { // Test default values when no flags provided - let cli = Cli::parse_from(&["tli", "auth", "status"]); + let cli = Cli::parse_from(&["fxt", "auth", "status"]); assert_eq!(cli.api_gateway_url, "http://localhost:50051"); assert_eq!(cli.log_level, "info"); @@ -467,7 +467,7 @@ mod tests { fn test_cli_custom_values() { // Test that custom values override defaults let cli = Cli::parse_from(&[ - "tli", + "fxt", "--api-gateway-url", "http://custom.com:8080", "--log-level", @@ -486,7 +486,7 @@ mod tests { #[test] fn test_tune_command_with_all_args() { let cli = Cli::parse_from(&[ - "tli", + "fxt", "tune", "start", "--model", @@ -506,7 +506,7 @@ mod tests { #[test] fn test_auth_login_command_parsing() { - let cli = Cli::parse_from(&["tli", "auth", "login", "--username", "testuser"]); + let cli = Cli::parse_from(&["fxt", "auth", "login", "--username", "testuser"]); match cli.command { Commands::Auth { .. } => {}, diff --git a/tli/src/prelude.rs b/fxt/src/prelude.rs similarity index 100% rename from tli/src/prelude.rs rename to fxt/src/prelude.rs diff --git a/tli/src/tests.rs b/fxt/src/tests.rs similarity index 100% rename from tli/src/tests.rs rename to fxt/src/tests.rs diff --git a/tli/src/types.rs b/fxt/src/types.rs similarity index 100% rename from tli/src/types.rs rename to fxt/src/types.rs diff --git a/tli/tests/INTEGRATION_TEST_GUIDE.md b/fxt/tests/INTEGRATION_TEST_GUIDE.md similarity index 100% rename from tli/tests/INTEGRATION_TEST_GUIDE.md rename to fxt/tests/INTEGRATION_TEST_GUIDE.md diff --git a/tli/tests/TEST_EXECUTION_README.md b/fxt/tests/TEST_EXECUTION_README.md similarity index 100% rename from tli/tests/TEST_EXECUTION_README.md rename to fxt/tests/TEST_EXECUTION_README.md diff --git a/tli/tests/agent_commands_test.rs b/fxt/tests/agent_commands_test.rs similarity index 99% rename from tli/tests/agent_commands_test.rs rename to fxt/tests/agent_commands_test.rs index 7ee8c9e6b..b450ffbc9 100644 --- a/tli/tests/agent_commands_test.rs +++ b/fxt/tests/agent_commands_test.rs @@ -8,7 +8,7 @@ // This test may not use all deps, but they are required by other integration tests #![allow(unused_crate_dependencies)] -use tli::commands::agent::{handle_allocate_portfolio, AllocatePortfolioArgs}; +use fxt::commands::agent::{handle_allocate_portfolio, AllocatePortfolioArgs}; #[tokio::test] async fn test_allocate_portfolio_valid_args() { diff --git a/tli/tests/auth_login_tests.rs b/fxt/tests/auth_login_tests.rs similarity index 99% rename from tli/tests/auth_login_tests.rs rename to fxt/tests/auth_login_tests.rs index b563288c2..746519cb9 100644 --- a/tli/tests/auth_login_tests.rs +++ b/fxt/tests/auth_login_tests.rs @@ -7,7 +7,7 @@ // This test may not use all deps, but they are required by other integration tests #![allow(unused_crate_dependencies)] -use tli::auth::login::{LoginRequest, LoginResponse, MfaRequest, RefreshRequest, RefreshResponse}; +use fxt::auth::login::{LoginRequest, LoginResponse, MfaRequest, RefreshRequest, RefreshResponse}; /// Test LoginRequest creation #[test] diff --git a/tli/tests/cli_integration_test.rs b/fxt/tests/cli_integration_test.rs similarity index 100% rename from tli/tests/cli_integration_test.rs rename to fxt/tests/cli_integration_test.rs diff --git a/tli/tests/client_builder_tests.rs b/fxt/tests/client_builder_tests.rs similarity index 99% rename from tli/tests/client_builder_tests.rs rename to fxt/tests/client_builder_tests.rs index e14802c2d..cb312fc67 100644 --- a/tli/tests/client_builder_tests.rs +++ b/fxt/tests/client_builder_tests.rs @@ -7,7 +7,7 @@ // This test may not use all deps, but they are required by other integration tests #![allow(unused_crate_dependencies)] -use tli::client::{ +use fxt::client::{ backtesting_client::BacktestingClientConfig, connection_manager::ConnectionConfig, ml_training_client::MLTrainingClientConfig, trading_client::TradingClientConfig, ClientFactory, ServiceEndpoints, TliClientBuilder, diff --git a/tli/tests/client_connection_manager_tests.rs b/fxt/tests/client_connection_manager_tests.rs similarity index 99% rename from tli/tests/client_connection_manager_tests.rs rename to fxt/tests/client_connection_manager_tests.rs index 6b1359c20..bc52f0ac8 100644 --- a/tli/tests/client_connection_manager_tests.rs +++ b/fxt/tests/client_connection_manager_tests.rs @@ -7,7 +7,7 @@ // This test may not use all deps, but they are required by other integration tests #![allow(unused_crate_dependencies)] -use tli::client::connection_manager::{ConnectionConfig, ConnectionManager, ConnectionStats}; +use fxt::client::connection_manager::{ConnectionConfig, ConnectionManager, ConnectionStats}; /// Test ConnectionConfig default values #[test] diff --git a/tli/tests/client_trading_client_tests.rs b/fxt/tests/client_trading_client_tests.rs similarity index 99% rename from tli/tests/client_trading_client_tests.rs rename to fxt/tests/client_trading_client_tests.rs index 5c323d516..650d9c1bd 100644 --- a/tli/tests/client_trading_client_tests.rs +++ b/fxt/tests/client_trading_client_tests.rs @@ -7,7 +7,7 @@ // This test may not use all deps, but they are required by other integration tests #![allow(unused_crate_dependencies)] -use tli::client::trading_client::{TradingClient, TradingClientConfig}; +use fxt::client::trading_client::{TradingClient, TradingClientConfig}; /// Test TradingClientConfig default values #[test] diff --git a/tli/tests/commands/mod.rs b/fxt/tests/commands/mod.rs similarity index 100% rename from tli/tests/commands/mod.rs rename to fxt/tests/commands/mod.rs diff --git a/tli/tests/commands/train_list_test.rs b/fxt/tests/commands/train_list_test.rs similarity index 99% rename from tli/tests/commands/train_list_test.rs rename to fxt/tests/commands/train_list_test.rs index 10c40901d..994b5b48b 100644 --- a/tli/tests/commands/train_list_test.rs +++ b/fxt/tests/commands/train_list_test.rs @@ -7,8 +7,8 @@ #[cfg(test)] mod train_list_tests { - use tli::commands::train::list::ListCommand; - use tli::proto::ml_training::{ + use fxt::commands::train::list::ListCommand; + use fxt::proto::ml_training::{ ml_training_service_client::MlTrainingServiceClient, ListTrainingJobsRequest, ListTrainingJobsResponse, TrainingJobSummary, TrainingStatus, }; diff --git a/tli/tests/debug_file_storage.rs b/fxt/tests/debug_file_storage.rs similarity index 97% rename from tli/tests/debug_file_storage.rs rename to fxt/tests/debug_file_storage.rs index 9179c5cc1..0225eab23 100644 --- a/tli/tests/debug_file_storage.rs +++ b/fxt/tests/debug_file_storage.rs @@ -6,7 +6,7 @@ #![allow(unused_crate_dependencies)] use anyhow::Result; -use tli::auth::token_manager::{FileTokenStorage, TokenStorage}; +use fxt::auth::token_manager::{FileTokenStorage, TokenStorage}; #[tokio::test] async fn debug_file_storage() -> Result<()> { diff --git a/tli/tests/e2e/full_user_workflow_test.rs b/fxt/tests/e2e/full_user_workflow_test.rs similarity index 99% rename from tli/tests/e2e/full_user_workflow_test.rs rename to fxt/tests/e2e/full_user_workflow_test.rs index 88b82c5ae..e543898d8 100644 --- a/tli/tests/e2e/full_user_workflow_test.rs +++ b/fxt/tests/e2e/full_user_workflow_test.rs @@ -2,7 +2,7 @@ //! //! Tests complete user scenarios from start to finish -use tli::proto::ml_training::{ +use fxt::proto::ml_training::{ ml_training_service_server::MlTrainingServiceServer, TrainingStatus, ml_training_service_client::MlTrainingServiceClient, SubscribeToTrainingStatusRequest, StopTrainingRequest, GetTrainingJobDetailsRequest, diff --git a/tli/tests/e2e/mock_ml_training_service.rs b/fxt/tests/e2e/mock_ml_training_service.rs similarity index 99% rename from tli/tests/e2e/mock_ml_training_service.rs rename to fxt/tests/e2e/mock_ml_training_service.rs index a0d31c2d7..039cf8f93 100644 --- a/tli/tests/e2e/mock_ml_training_service.rs +++ b/fxt/tests/e2e/mock_ml_training_service.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; -use tli::proto::ml_training::{ +use fxt::proto::ml_training::{ ml_training_service_server::MlTrainingService, HealthCheckRequest, HealthCheckResponse, ListAvailableModelsRequest, ListAvailableModelsResponse, ListTrainingJobsRequest, ListTrainingJobsResponse, GetTrainingJobDetailsRequest, GetTrainingJobDetailsResponse, diff --git a/tli/tests/e2e/mod.rs b/fxt/tests/e2e/mod.rs similarity index 100% rename from tli/tests/e2e/mod.rs rename to fxt/tests/e2e/mod.rs diff --git a/tli/tests/e2e/test_fixtures.rs b/fxt/tests/e2e/test_fixtures.rs similarity index 98% rename from tli/tests/e2e/test_fixtures.rs rename to fxt/tests/e2e/test_fixtures.rs index 911a08d4a..a6424f42c 100644 --- a/tli/tests/e2e/test_fixtures.rs +++ b/fxt/tests/e2e/test_fixtures.rs @@ -3,7 +3,7 @@ //! Provides reusable test data, helper functions, and utilities for E2E testing. use std::collections::HashMap; -use tli::proto::ml_training::{TrainingJobSummary, TrainingStatus}; +use fxt::proto::ml_training::{TrainingJobSummary, TrainingStatus}; /// Create a test training job with default values pub fn create_test_job( diff --git a/tli/tests/e2e/train_list_e2e_test.rs b/fxt/tests/e2e/train_list_e2e_test.rs similarity index 99% rename from tli/tests/e2e/train_list_e2e_test.rs rename to fxt/tests/e2e/train_list_e2e_test.rs index 89e3602f4..94bee05f3 100644 --- a/tli/tests/e2e/train_list_e2e_test.rs +++ b/fxt/tests/e2e/train_list_e2e_test.rs @@ -2,7 +2,7 @@ //! //! Tests the complete flow: TLI CLI → API Gateway → ML Training Service -use tli::proto::ml_training::{ +use fxt::proto::ml_training::{ ml_training_service_server::MlTrainingServiceServer, TrainingStatus, }; use tonic::transport::Server; diff --git a/tli/tests/e2e/train_start_e2e_test.rs b/fxt/tests/e2e/train_start_e2e_test.rs similarity index 99% rename from tli/tests/e2e/train_start_e2e_test.rs rename to fxt/tests/e2e/train_start_e2e_test.rs index 292a029d4..427082586 100644 --- a/tli/tests/e2e/train_start_e2e_test.rs +++ b/fxt/tests/e2e/train_start_e2e_test.rs @@ -2,7 +2,7 @@ //! //! Tests training job creation and validation -use tli::proto::ml_training::{ +use fxt::proto::ml_training::{ ml_training_service_server::MlTrainingServiceServer, TrainingStatus, }; use tonic::transport::Server; diff --git a/tli/tests/e2e/train_status_e2e_test.rs b/fxt/tests/e2e/train_status_e2e_test.rs similarity index 99% rename from tli/tests/e2e/train_status_e2e_test.rs rename to fxt/tests/e2e/train_status_e2e_test.rs index bed04082f..f12940eeb 100644 --- a/tli/tests/e2e/train_status_e2e_test.rs +++ b/fxt/tests/e2e/train_status_e2e_test.rs @@ -2,7 +2,7 @@ //! //! Tests training job status queries -use tli::proto::ml_training::{ +use fxt::proto::ml_training::{ ml_training_service_server::MlTrainingServiceServer, TrainingStatus, ml_training_service_client::MlTrainingServiceClient, GetTrainingJobDetailsRequest, }; diff --git a/tli/tests/e2e/train_stop_e2e_test.rs b/fxt/tests/e2e/train_stop_e2e_test.rs similarity index 99% rename from tli/tests/e2e/train_stop_e2e_test.rs rename to fxt/tests/e2e/train_stop_e2e_test.rs index 40b5a0421..fb8558f90 100644 --- a/tli/tests/e2e/train_stop_e2e_test.rs +++ b/fxt/tests/e2e/train_stop_e2e_test.rs @@ -2,7 +2,7 @@ //! //! Tests stopping training jobs and cleanup -use tli::proto::ml_training::{ +use fxt::proto::ml_training::{ ml_training_service_server::MlTrainingServiceServer, TrainingStatus, ml_training_service_client::MlTrainingServiceClient, StopTrainingRequest, }; diff --git a/tli/tests/e2e/train_watch_e2e_test.rs b/fxt/tests/e2e/train_watch_e2e_test.rs similarity index 99% rename from tli/tests/e2e/train_watch_e2e_test.rs rename to fxt/tests/e2e/train_watch_e2e_test.rs index c28a4921c..939ddaf33 100644 --- a/tli/tests/e2e/train_watch_e2e_test.rs +++ b/fxt/tests/e2e/train_watch_e2e_test.rs @@ -2,7 +2,7 @@ //! //! Tests real-time training progress monitoring -use tli::proto::ml_training::{ +use fxt::proto::ml_training::{ ml_training_service_server::MlTrainingServiceServer, TrainingStatus, ml_training_service_client::MlTrainingServiceClient, SubscribeToTrainingStatusRequest, }; diff --git a/tli/tests/encryption_security_audit.rs b/fxt/tests/encryption_security_audit.rs similarity index 97% rename from tli/tests/encryption_security_audit.rs rename to fxt/tests/encryption_security_audit.rs index b3fbe1e13..04b3ce5a7 100644 --- a/tli/tests/encryption_security_audit.rs +++ b/fxt/tests/encryption_security_audit.rs @@ -10,7 +10,7 @@ #[ignore = "Ignored by default since it leaves files for inspection"] async fn security_audit_create_persistent_tokens() { use std::path::PathBuf; - use tli::auth::token_manager::{FileTokenStorage, TokenStorage}; + use fxt::auth::token_manager::{FileTokenStorage, TokenStorage}; // Create persistent test directory let test_dir = PathBuf::from("/tmp/foxhunt_security_audit_wave155"); diff --git a/tli/tests/error_tests.rs b/fxt/tests/error_tests.rs similarity index 99% rename from tli/tests/error_tests.rs rename to fxt/tests/error_tests.rs index 29764f846..490dba3d2 100644 --- a/tli/tests/error_tests.rs +++ b/fxt/tests/error_tests.rs @@ -8,7 +8,7 @@ #![allow(unused_crate_dependencies)] use std::io; -use tli::error::{TliError, TliResult}; +use fxt::error::{TliError, TliResult}; /// Test TliError::Connection variant #[test] diff --git a/tli/tests/file_storage_encryption.rs b/fxt/tests/file_storage_encryption.rs similarity index 99% rename from tli/tests/file_storage_encryption.rs rename to fxt/tests/file_storage_encryption.rs index 0a6039b5c..50b75eb5e 100644 --- a/tli/tests/file_storage_encryption.rs +++ b/fxt/tests/file_storage_encryption.rs @@ -19,7 +19,7 @@ use anyhow::Result; use std::fs; use std::path::PathBuf; use tempfile::TempDir; -use tli::auth::token_manager::{FileTokenStorage, TokenStorage}; +use fxt::auth::token_manager::{FileTokenStorage, TokenStorage}; /// Helper: Create FileTokenStorage with temporary directory fn create_test_storage() -> Result<(FileTokenStorage, TempDir)> { diff --git a/tli/tests/tli_auth_integration_test.rs b/fxt/tests/fxt_auth_integration_test.rs similarity index 99% rename from tli/tests/tli_auth_integration_test.rs rename to fxt/tests/fxt_auth_integration_test.rs index 05411c261..2a630a911 100644 --- a/tli/tests/tli_auth_integration_test.rs +++ b/fxt/tests/fxt_auth_integration_test.rs @@ -13,7 +13,7 @@ use anyhow::{Context, Result}; use std::time::{SystemTime, UNIX_EPOCH}; -use tli::auth::{ +use fxt::auth::{ token_manager::{InMemoryTokenStorage, KeyringTokenStorage, TokenInfo}, AuthInterceptor, AuthTokenManager, LoginClient, TokenStorage, }; @@ -358,7 +358,7 @@ async fn test_login_client_token_refresh() -> Result<()> { async fn test_connection_manager() -> Result<()> { println!("\n=== Test: Connection Manager Configuration ==="); - use tli::client::connection_manager::{ConnectionConfig, ConnectionManager}; + use fxt::client::connection_manager::{ConnectionConfig, ConnectionManager}; let config = ConnectionConfig { server_url: "https://localhost:50050".to_string(), // API Gateway @@ -404,7 +404,7 @@ async fn test_connection_manager() -> Result<()> { async fn test_tli_client_builder() -> Result<()> { println!("\n=== Test: TLI Client Builder ==="); - use tli::client::{ + use fxt::client::{ backtesting_client::BacktestingClientConfig, connection_manager::ConnectionConfig, ml_training_client::MLTrainingClientConfig, trading_client::TradingClientConfig, TliClientBuilder, diff --git a/tli/tests/integration/client_integration.rs b/fxt/tests/integration/client_integration.rs similarity index 99% rename from tli/tests/integration/client_integration.rs rename to fxt/tests/integration/client_integration.rs index 6cb58b158..44475bcab 100644 --- a/tli/tests/integration/client_integration.rs +++ b/fxt/tests/integration/client_integration.rs @@ -2,8 +2,8 @@ use std::time::Duration; use tokio::time::{sleep, timeout}; -use tli::prelude::*; -use tli::{TliClient, ServiceEndpoints}; +use fxt::prelude::*; +use fxt::{TliClient, ServiceEndpoints}; use crate::integration::{TestConfig, TestUtilities, integration_test}; use crate::mocks::grpc_server::MockGrpcServer; diff --git a/tli/tests/integration/end_to_end_tests.rs b/fxt/tests/integration/end_to_end_tests.rs similarity index 99% rename from tli/tests/integration/end_to_end_tests.rs rename to fxt/tests/integration/end_to_end_tests.rs index f58a2f3d3..ebb742ff2 100644 --- a/tli/tests/integration/end_to_end_tests.rs +++ b/fxt/tests/integration/end_to_end_tests.rs @@ -13,10 +13,10 @@ use uuid::Uuid; use crate::integration::{TestConfig, TestUtilities}; use crate::mocks::grpc_server::{MockBacktestingServer, MockRiskServer, MockTradingServer}; // Auth types removed - TLI is pure client -// use tli::auth::{AuthenticationManager, RbacManager, SessionManager}; -use tli::client::{TliClientBuilder, TliClientSuite}; +// use fxt::auth::{AuthenticationManager, RbacManager, SessionManager}; +use fxt::client::{TliClientBuilder, TliClientSuite}; // Database imports removed - TLI is pure client -use tli::prelude::*; +use fxt::prelude::*; /// End-to-end test environment pub struct EndToEndTestEnvironment { diff --git a/tli/tests/integration/error_handling_tests.rs b/fxt/tests/integration/error_handling_tests.rs similarity index 99% rename from tli/tests/integration/error_handling_tests.rs rename to fxt/tests/integration/error_handling_tests.rs index c80a1928b..b1d5918f4 100644 --- a/tli/tests/integration/error_handling_tests.rs +++ b/fxt/tests/integration/error_handling_tests.rs @@ -14,9 +14,9 @@ use wiremock::{ use crate::integration::{TestConfig, TestUtilities}; use crate::mocks::grpc_server::{FailureMode, MockBacktestingServer, MockTradingServer}; -use tli::client::{BacktestingClient, TliClientBuilder, TradingClient}; -use tli::error::{ErrorCode, ErrorSeverity, TliError}; -use tli::prelude::*; +use fxt::client::{BacktestingClient, TliClientBuilder, TradingClient}; +use fxt::error::{ErrorCode, ErrorSeverity, TliError}; +use fxt::prelude::*; /// Error handling test suite pub struct ErrorHandlingTests { diff --git a/tli/tests/integration/mod.rs b/fxt/tests/integration/mod.rs similarity index 100% rename from tli/tests/integration/mod.rs rename to fxt/tests/integration/mod.rs diff --git a/tli/tests/integration/performance_tests.rs b/fxt/tests/integration/performance_tests.rs similarity index 99% rename from tli/tests/integration/performance_tests.rs rename to fxt/tests/integration/performance_tests.rs index ef0aa3c5e..7c71807b2 100644 --- a/tli/tests/integration/performance_tests.rs +++ b/fxt/tests/integration/performance_tests.rs @@ -15,8 +15,8 @@ use uuid::Uuid; use crate::integration::{TestConfig, TestUtilities}; use crate::mocks::grpc_server::{MockBacktestingServer, MockTradingServer}; -use tli::client::{BacktestingClient, TliClientBuilder, TradingClient}; -use tli::prelude::*; +use fxt::client::{BacktestingClient, TliClientBuilder, TradingClient}; +use fxt::prelude::*; /// Performance test configuration #[derive(Debug, Clone)] diff --git a/tli/tests/integration/service_integration_tests.rs b/fxt/tests/integration/service_integration_tests.rs similarity index 99% rename from tli/tests/integration/service_integration_tests.rs rename to fxt/tests/integration/service_integration_tests.rs index 8b18deaa0..ffc2db442 100644 --- a/tli/tests/integration/service_integration_tests.rs +++ b/fxt/tests/integration/service_integration_tests.rs @@ -14,12 +14,12 @@ use uuid::Uuid; use crate::integration::{TestConfig, TestUtilities}; use crate::mocks::grpc_server::{MockBacktestingServer, MockRiskServer, MockTradingServer}; -use tli::client::{ +use fxt::client::{ BacktestingClient, BacktestingClientConfig, TliClientBuilder, TradingClient, TradingClientConfig, }; // Database imports removed - TLI is pure client -use tli::prelude::*; +use fxt::prelude::*; /// Service integration test suite pub struct ServiceIntegrationTests { diff --git a/tli/tests/integration_tests.rs b/fxt/tests/integration_tests.rs similarity index 100% rename from tli/tests/integration_tests.rs rename to fxt/tests/integration_tests.rs diff --git a/tli/tests/keyring_persistence_tests.rs b/fxt/tests/keyring_persistence_tests.rs similarity index 99% rename from tli/tests/keyring_persistence_tests.rs rename to fxt/tests/keyring_persistence_tests.rs index 15c96c044..be17ee7b0 100644 --- a/tli/tests/keyring_persistence_tests.rs +++ b/fxt/tests/keyring_persistence_tests.rs @@ -22,7 +22,7 @@ use anyhow::Result; use serial_test::serial; use std::time::{SystemTime, UNIX_EPOCH}; -use tli::auth::token_manager::{FileTokenStorage, TokenStorage}; +use fxt::auth::token_manager::{FileTokenStorage, TokenStorage}; /// Helper function to generate test JWT token fn generate_test_token(username: &str, expires_in_seconds: u64) -> String { diff --git a/tli/tests/lib.rs b/fxt/tests/lib.rs similarity index 100% rename from tli/tests/lib.rs rename to fxt/tests/lib.rs diff --git a/tli/tests/minimal_keyring_test.rs b/fxt/tests/minimal_keyring_test.rs similarity index 100% rename from tli/tests/minimal_keyring_test.rs rename to fxt/tests/minimal_keyring_test.rs diff --git a/tli/tests/ml_trading_commands_test.rs b/fxt/tests/ml_trading_commands_test.rs similarity index 98% rename from tli/tests/ml_trading_commands_test.rs rename to fxt/tests/ml_trading_commands_test.rs index b06a12aeb..a4f0885a1 100644 --- a/tli/tests/ml_trading_commands_test.rs +++ b/fxt/tests/ml_trading_commands_test.rs @@ -47,8 +47,8 @@ mod test_auth { /// * `Ok(PathBuf)` - Path to temporary token directory (for cleanup) /// * `Err(anyhow::Error)` - Failed to setup authentication pub fn setup_test_auth() -> Result { - use tli::auth::jwt_generator; - use tli::auth::token_manager::{FileTokenStorage, TokenStorage}; + use fxt::auth::jwt_generator; + use fxt::auth::token_manager::{FileTokenStorage, TokenStorage}; // Create isolated temp directory for this test run let temp_dir = @@ -155,8 +155,8 @@ mod test_auth { std::env::set_var("XDG_CONFIG_HOME", &config_home); // Generate and store tokens - use tli::auth::jwt_generator; - use tli::auth::token_manager::{FileTokenStorage, TokenStorage}; + use fxt::auth::jwt_generator; + use fxt::auth::token_manager::{FileTokenStorage, TokenStorage}; let storage = FileTokenStorage::new() .context("Failed to create FileTokenStorage (should use temp XDG_CONFIG_HOME)")?; diff --git a/tli/tests/mocks/grpc_server.rs b/fxt/tests/mocks/grpc_server.rs similarity index 100% rename from tli/tests/mocks/grpc_server.rs rename to fxt/tests/mocks/grpc_server.rs diff --git a/tli/tests/mocks/mod.rs b/fxt/tests/mocks/mod.rs similarity index 100% rename from tli/tests/mocks/mod.rs rename to fxt/tests/mocks/mod.rs diff --git a/tli/tests/mod.rs b/fxt/tests/mod.rs similarity index 100% rename from tli/tests/mod.rs rename to fxt/tests/mod.rs diff --git a/tli/tests/performance_tests.rs b/fxt/tests/performance_tests.rs similarity index 100% rename from tli/tests/performance_tests.rs rename to fxt/tests/performance_tests.rs diff --git a/tli/tests/property_tests.rs b/fxt/tests/property_tests.rs similarity index 100% rename from tli/tests/property_tests.rs rename to fxt/tests/property_tests.rs diff --git a/tli/tests/regime_command_tests.rs b/fxt/tests/regime_command_tests.rs similarity index 99% rename from tli/tests/regime_command_tests.rs rename to fxt/tests/regime_command_tests.rs index 451bae8b7..03b53ea44 100644 --- a/tli/tests/regime_command_tests.rs +++ b/fxt/tests/regime_command_tests.rs @@ -8,7 +8,7 @@ // This test may not use all deps, but they are required by other integration tests #![allow(unused_crate_dependencies)] -use tli::commands::trade_ml::{TradeMlArgs, TradeMlCommand}; +use fxt::commands::trade_ml::{TradeMlArgs, TradeMlCommand}; #[tokio::test] async fn test_regime_command_parses() { diff --git a/tli/tests/test_helpers/mod.rs b/fxt/tests/test_helpers/mod.rs similarity index 100% rename from tli/tests/test_helpers/mod.rs rename to fxt/tests/test_helpers/mod.rs diff --git a/tli/tests/test_monitoring.rs b/fxt/tests/test_monitoring.rs similarity index 100% rename from tli/tests/test_monitoring.rs rename to fxt/tests/test_monitoring.rs diff --git a/tli/tests/tune_integration_test.rs b/fxt/tests/tune_integration_test.rs similarity index 100% rename from tli/tests/tune_integration_test.rs rename to fxt/tests/tune_integration_test.rs diff --git a/tli/tests/types_tests.rs b/fxt/tests/types_tests.rs similarity index 99% rename from tli/tests/types_tests.rs rename to fxt/tests/types_tests.rs index 5bced0d9e..01d0ac422 100644 --- a/tli/tests/types_tests.rs +++ b/fxt/tests/types_tests.rs @@ -8,7 +8,7 @@ #![allow(unused_crate_dependencies)] use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use tli::types::*; +use fxt::types::*; /// Test unix_nanos_to_system_time conversion #[test] diff --git a/tli/tests/unit_tests.rs b/fxt/tests/unit_tests.rs similarity index 92% rename from tli/tests/unit_tests.rs rename to fxt/tests/unit_tests.rs index b0e22a6f7..48b9516c3 100644 --- a/tli/tests/unit_tests.rs +++ b/fxt/tests/unit_tests.rs @@ -4,7 +4,7 @@ //! when TLI was refactored to be a pure client without database dependencies. //! //! To re-enable: -//! 1. Update imports to use tli::client::trading_client::{TradingClient, TradingClientConfig} +//! 1. Update imports to use fxt::client::trading_client::{TradingClient, TradingClientConfig} //! 2. Remove references to non-existent config fields (order_validation, risk_management, etc.) //! 3. Use actual TradingClientConfig fields (endpoint, timeout_ms) //! 4. Remove database-related tests (TLI is pure client) diff --git a/migrations/tests/run_all_tests.sh b/migrations/tests/run_all_tests.sh deleted file mode 100755 index 76072764c..000000000 --- a/migrations/tests/run_all_tests.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/bash -# ================================================================================================ -# Migration Test Suite Runner -# PostgreSQL 16.10 + TimescaleDB 2.22.1 -# Runs all migration tests in sequence and reports results -# ================================================================================================ - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Database connection (override with environment variables) -DB_HOST="${DB_HOST:-localhost}" -DB_PORT="${DB_PORT:-5432}" -DB_NAME="${DB_NAME:-foxhunt_trading}" -DB_USER="${DB_USER:-foxhunt_admin}" - -PSQL_CMD="psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME" - -# Test result tracking -TOTAL_TESTS=0 -PASSED_TESTS=0 -FAILED_TESTS=0 -WARNINGS=0 - -echo -e "${BLUE}================================================================================================${NC}" -echo -e "${BLUE}Migration Test Suite - PostgreSQL 16.10 + TimescaleDB 2.22.1${NC}" -echo -e "${BLUE}================================================================================================${NC}" -echo "" -echo -e "Database: ${YELLOW}$DB_NAME${NC} on ${YELLOW}$DB_HOST:$DB_PORT${NC}" -echo -e "User: ${YELLOW}$DB_USER${NC}" -echo "" - -# Function to run a test file and parse results -run_test() { - local test_file=$1 - local test_name=$(basename "$test_file" .sql) - - echo -e "${BLUE}Running: ${YELLOW}$test_name${NC}" - echo "-----------------------------------" - - # Run test and capture output - OUTPUT=$($PSQL_CMD -f "$test_file" 2>&1 || true) - - # Count PASS/FAIL/WARNING messages - local pass_count=$(echo "$OUTPUT" | grep -c "PASS:" || true) - local fail_count=$(echo "$OUTPUT" | grep -c "FAIL:" || true) - local warn_count=$(echo "$OUTPUT" | grep -c "WARNING:" || true) - - TOTAL_TESTS=$((TOTAL_TESTS + pass_count + fail_count)) - PASSED_TESTS=$((PASSED_TESTS + pass_count)) - FAILED_TESTS=$((FAILED_TESTS + fail_count)) - WARNINGS=$((WARNINGS + warn_count)) - - # Display results - if [ $fail_count -gt 0 ]; then - echo -e "${RED}✗ FAILED${NC} - $fail_count test(s) failed" - echo "$OUTPUT" | grep "FAIL:" | sed "s/^/${RED} /${NC}" - else - echo -e "${GREEN}✓ PASSED${NC} - $pass_count test(s) passed" - fi - - if [ $warn_count -gt 0 ]; then - echo -e "${YELLOW}⚠ WARNINGS${NC} - $warn_count warning(s)" - echo "$OUTPUT" | grep "WARNING:" | sed "s/^/${YELLOW} /${NC}" - fi - - # Show NOTICE messages for additional info - echo "$OUTPUT" | grep "NOTICE:" | sed "s/NOTICE:/ ℹ /" || true - - echo "" -} - -# Check database connectivity -echo -e "${BLUE}Checking database connectivity...${NC}" -if $PSQL_CMD -c "SELECT version();" > /dev/null 2>&1; then - echo -e "${GREEN}✓ Connected successfully${NC}" - - # Check PostgreSQL version - PG_VERSION=$($PSQL_CMD -t -c "SELECT version();" | head -1) - echo -e " PostgreSQL: ${YELLOW}$PG_VERSION${NC}" - - # Check TimescaleDB version - TSDB_VERSION=$($PSQL_CMD -t -c "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';" 2>/dev/null || echo "Not installed") - if [ "$TSDB_VERSION" != "Not installed" ]; then - echo -e " TimescaleDB: ${YELLOW}$TSDB_VERSION${NC}" - else - echo -e "${RED} ⚠ TimescaleDB not detected${NC}" - fi -else - echo -e "${RED}✗ Failed to connect to database${NC}" - echo "Please check connection settings and try again." - exit 1 -fi -echo "" - -# Run test suites in order -echo -e "${BLUE}================================================================================================${NC}" -echo -e "${BLUE}Running Test Suites${NC}" -echo -e "${BLUE}================================================================================================${NC}" -echo "" - -# Test 1: Trading Events (Migration 001) -if [ -f "migrations/tests/test_trading_events.sql" ]; then - run_test "migrations/tests/test_trading_events.sql" -fi - -# Test 2: Risk Events (Migrations 002-003) -if [ -f "migrations/tests/test_risk_events.sql" ]; then - run_test "migrations/tests/test_risk_events.sql" -fi - -# Test 3: Compliance Views (Migration 004) -if [ -f "migrations/tests/test_compliance_views.sql" ]; then - run_test "migrations/tests/test_compliance_views.sql" -fi - -# Test 4: Configuration Schema (Migration 007) -if [ -f "migrations/tests/test_configuration_schema.sql" ]; then - run_test "migrations/tests/test_configuration_schema.sql" -fi - -# Test 5: Auth Schema (Migration 015) -if [ -f "migrations/tests/test_auth_schema.sql" ]; then - run_test "migrations/tests/test_auth_schema.sql" -fi - -# Test 6: TimescaleDB Features -if [ -f "migrations/tests/test_timescaledb_features.sql" ]; then - run_test "migrations/tests/test_timescaledb_features.sql" -fi - -# Test 7: Schema Validation -if [ -f "migrations/tests/test_schema_validation.sql" ]; then - run_test "migrations/tests/test_schema_validation.sql" -fi - -# Summary -echo -e "${BLUE}================================================================================================${NC}" -echo -e "${BLUE}Test Summary${NC}" -echo -e "${BLUE}================================================================================================${NC}" -echo "" -echo -e "Total Tests: ${YELLOW}$TOTAL_TESTS${NC}" -echo -e "Passed: ${GREEN}$PASSED_TESTS${NC}" -echo -e "Failed: ${RED}$FAILED_TESTS${NC}" -echo -e "Warnings: ${YELLOW}$WARNINGS${NC}" -echo "" - -# Calculate success rate -if [ $TOTAL_TESTS -gt 0 ]; then - SUCCESS_RATE=$(awk "BEGIN {printf \"%.1f\", ($PASSED_TESTS / $TOTAL_TESTS) * 100}") - echo -e "Success Rate: ${YELLOW}$SUCCESS_RATE%${NC}" - echo "" - - if [ $FAILED_TESTS -eq 0 ]; then - echo -e "${GREEN}✓ All tests passed!${NC}" - exit 0 - else - echo -e "${RED}✗ Some tests failed. Review output above for details.${NC}" - exit 1 - fi -else - echo -e "${YELLOW}⚠ No tests were run${NC}" - exit 1 -fi diff --git a/ml/scripts/compare_checkpoint_sizes.sh b/ml/scripts/compare_checkpoint_sizes.sh deleted file mode 100755 index 2f930eda8..000000000 --- a/ml/scripts/compare_checkpoint_sizes.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -# File Size Comparison Script for Quantized Checkpoints -# -# Compares FP32 vs INT8 checkpoint sizes across all models. -# -# Usage: -# ./ml/scripts/compare_checkpoint_sizes.sh - -set -e - -CHECKPOINT_DIR="ml/trained_models" - -echo "=== Checkpoint Size Comparison (FP32 vs INT8) ===" -echo "" - -# Function to get file size in MB -get_size_mb() { - local file="$1" - if [ -f "$file" ]; then - du -h "$file" | awk '{print $1}' - else - echo "N/A" - fi -} - -# Function to calculate percentage reduction -calc_reduction() { - local fp32_size="$1" - local int8_size="$2" - - if [ "$fp32_size" != "N/A" ] && [ "$int8_size" != "N/A" ]; then - # Strip units and calculate - fp32_bytes=$(du -b "$fp32_size" 2>/dev/null | awk '{print $1}') - int8_bytes=$(du -b "$int8_size" 2>/dev/null | awk '{print $1}') - - if [ "$fp32_bytes" -gt 0 ]; then - reduction=$(echo "scale=1; 100 * (1 - $int8_bytes / $fp32_bytes)" | bc) - echo "${reduction}%" - else - echo "N/A" - fi - else - echo "N/A" - fi -} - -echo "Model | FP32 Size | INT8 Size | Reduction" -echo "---------------|-----------|-----------|----------" - -# DQN checkpoints -DQN_FP32="${CHECKPOINT_DIR}/dqn_final_epoch3.safetensors" -DQN_INT8="${CHECKPOINT_DIR}/dqn_quantized_demo.safetensors" - -if [ -f "$DQN_FP32" ] || [ -f "$DQN_INT8" ]; then - fp32_size=$(get_size_mb "$DQN_FP32") - int8_size=$(get_size_mb "$DQN_INT8") - echo "DQN | $fp32_size | $int8_size | $(calc_reduction "$DQN_FP32" "$DQN_INT8")" -fi - -# PPO checkpoints -PPO_FP32="${CHECKPOINT_DIR}/ppo_checkpoint_epoch_3.safetensors" -PPO_INT8="${CHECKPOINT_DIR}/ppo_quantized_epoch_3.safetensors" - -if [ -f "$PPO_FP32" ] || [ -f "$PPO_INT8" ]; then - fp32_size=$(get_size_mb "$PPO_FP32") - int8_size=$(get_size_mb "$PPO_INT8") - echo "PPO | $fp32_size | $int8_size | $(calc_reduction "$PPO_FP32" "$PPO_INT8")" -fi - -# MAMBA-2 checkpoints -MAMBA_FP32="${CHECKPOINT_DIR}/mamba2_real_data/mamba2_epoch_24.safetensors" -MAMBA_INT8="${CHECKPOINT_DIR}/mamba2_real_data/mamba2_quantized_epoch_24.safetensors" - -if [ -f "$MAMBA_FP32" ] || [ -f "$MAMBA_INT8" ]; then - fp32_size=$(get_size_mb "$MAMBA_FP32") - int8_size=$(get_size_mb "$MAMBA_INT8") - echo "MAMBA-2 | $fp32_size | $int8_size | $(calc_reduction "$MAMBA_FP32" "$MAMBA_INT8")" -fi - -# TFT checkpoints -TFT_FP32="${CHECKPOINT_DIR}/tft_225_epoch_0.safetensors" -TFT_INT8="${CHECKPOINT_DIR}/tft_quantized_225_epoch_0.safetensors" - -if [ -f "$TFT_FP32" ] || [ -f "$TFT_INT8" ]; then - fp32_size=$(get_size_mb "$TFT_FP32") - int8_size=$(get_size_mb "$TFT_INT8") - echo "TFT | $fp32_size | $int8_size | $(calc_reduction "$TFT_FP32" "$TFT_INT8")" -fi - -echo "" -echo "Legend:" -echo " FP32 Size: Float32 checkpoint size (original)" -echo " INT8 Size: Quantized INT8 checkpoint size" -echo " Reduction: Percentage size reduction (target: ~75%)" -echo "" -echo "Target file sizes:" -echo " DQN: < 50 MB (INT8)" -echo " PPO: < 75 MB (INT8)" -echo " MAMBA: < 100 MB (INT8)" -echo " TFT: < 100 MB (INT8)" diff --git a/ml/scripts/profile_memory.sh b/ml/scripts/profile_memory.sh deleted file mode 100755 index 9a77ed2da..000000000 --- a/ml/scripts/profile_memory.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# Memory Profiling Script for ML Training -# Usage: ./profile_memory.sh [EPOCHS] -# -# Example: ./profile_memory.sh dqn test_data/ES_FUT_small.parquet 3 - -set -e - -MODEL=$1 -PARQUET=$2 -EPOCHS=${3:-3} - -if [ -z "$MODEL" ] || [ -z "$PARQUET" ]; then - echo "Usage: $0 [EPOCHS]" - echo " MODEL: dqn, ppo, mamba2, tft" - echo " PARQUET_FILE: path to training data" - echo " EPOCHS: number of epochs (default: 3)" - exit 1 -fi - -echo "==========================================" -echo "Memory Profiling: train_${MODEL}" -echo "Data: $PARQUET" -echo "Epochs: $EPOCHS" -echo "==========================================" -echo - -# Get base directory (foxhunt root) -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -FOXHUNT_ROOT="$( cd "$SCRIPT_DIR/../.." && pwd )" - -cd "$FOXHUNT_ROOT" - -# Check if parquet file exists -if [ ! -f "$PARQUET" ]; then - echo "Error: Parquet file not found: $PARQUET" - exit 1 -fi - -# Run training with memory profiling -echo "Starting profiling run..." -echo - -/usr/bin/time -v cargo run -p ml --example train_${MODEL} --release -- \ - --parquet-file "$PARQUET" \ - --epochs "$EPOCHS" \ - 2>&1 | tee /tmp/profile_${MODEL}_$$.log - -echo -echo "==========================================" -echo "Memory Profile Summary" -echo "==========================================" -grep -E "(Maximum resident|User time|System time|Percent of CPU|Elapsed|Minor.*page faults|Major.*page faults)" /tmp/profile_${MODEL}_$$.log || true - -echo -echo "Full log saved to: /tmp/profile_${MODEL}_$$.log" diff --git a/runpod/QUICK_START.md b/runpod/QUICK_START.md deleted file mode 100644 index ebc6f04cb..000000000 --- a/runpod/QUICK_START.md +++ /dev/null @@ -1,76 +0,0 @@ -# Foxhunt RunPod Module - Quick Start - -## Installation (Required) - -```bash -cd /home/jgrusewski/Work/foxhunt/ml/python/foxhunt_runpod -pip install -r requirements.txt -``` - -## Verify Installation - -```bash -python /home/jgrusewski/Work/foxhunt/ml/python/test_module_import.py -``` - -Expected output: -``` -Testing foxhunt_runpod module imports... -1. Importing main module... - ✓ Version: 1.0.0 -... -✅ All imports successful! -``` - -## Deploy Pod (Example) - -```python -from foxhunt_runpod import RunPodClient, PodMonitor - -# 1. Deploy pod (auto-selects cheapest GPU ≥16GB VRAM) -client = RunPodClient() -pod = client.deploy_pod( - gpu_type="RTX A4000", # Optional: preferred GPU - command="--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50" -) - -# 2. Monitor training (S3 log tailing) -monitor = PodMonitor(pod['id']) -monitor.wait_until_running(timeout=300) # Wait for pod to start -monitor.stream_s3_logs(follow=True) # Stream logs until completion - -# 3. Auto-terminate on completion -monitor.auto_terminate() -``` - -## Run Examples - -```bash -python /home/jgrusewski/Work/foxhunt/ml/python/foxhunt_runpod/example_usage.py -``` - -Select from 5 examples: -1. Deploy and Monitor Training -2. List Available GPUs -3. List All Pods -4. S3 Operations -5. Monitor Existing Pod - -## Configuration - -Configuration is loaded from `/home/jgrusewski/Work/foxhunt/.env.runpod` (already configured). - -## Documentation - -- **Complete Guide**: `README.md` -- **Implementation Details**: `/home/jgrusewski/Work/foxhunt/RUNPOD_PYTHON_MODULE_IMPLEMENTATION.md` -- **Examples**: `example_usage.py` - -## Key Features - -- ✅ S3 log monitoring (NO SSH required) -- ✅ Automatic GPU selection by price -- ✅ Retry logic with exponential backoff -- ✅ Auto-termination on training completion -- ✅ Rich terminal output -- ✅ Type-safe with pydantic diff --git a/runpod/README.md b/runpod/README.md deleted file mode 100644 index a313a541a..000000000 --- a/runpod/README.md +++ /dev/null @@ -1,367 +0,0 @@ -# Foxhunt RunPod Workflow Module - -Production-ready Python module for deploying and monitoring ML training on RunPod GPU infrastructure. - -## Features - -- **Pod Deployment**: REST API with datacenter filtering and automatic GPU selection -- **S3 Log Monitoring**: Byte-range polling (NO SSH required) -- **Auto-Termination**: Detect training completion and terminate pods automatically -- **Rich Output**: Progress tracking with beautiful terminal formatting -- **Retry Logic**: Exponential backoff with configurable retries -- **Type Safety**: Full type hints and pydantic validation - -## Installation - -```bash -cd ml/python/foxhunt_runpod -pip install -r requirements.txt -``` - -## Configuration - -Create `.env.runpod` in project root: - -```env -# RunPod API -RUNPOD_API_KEY=rpa_... - -# S3 Credentials -RUNPOD_S3_ACCESS_KEY=user_... -RUNPOD_S3_SECRET=rps_... -RUNPOD_S3_ENDPOINT=https://s3api-eur-is-1.runpod.io -RUNPOD_S3_REGION=eur-is-1 - -# Network Volume -RUNPOD_VOLUME_ID=se3zdnb5o4 -RUNPOD_VOLUME_MOUNT=/runpod-volume - -# Optional -RUNPOD_CONTAINER_REGISTRY_AUTH_ID=cmh3... -``` - -## Quick Start - -```python -from foxhunt_runpod import RunPodClient, PodMonitor, S3Client - -# 1. Deploy pod (auto-selects cheapest GPU) -client = RunPodClient() -pod = client.deploy_pod( - gpu_type="RTX A4000", # Optional: preferred GPU - command="--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50" -) - -# 2. Monitor training (S3 log tailing) -monitor = PodMonitor(pod['id']) -monitor.wait_until_running(timeout=300) # Wait for pod to start -monitor.stream_s3_logs(follow=True) # Stream logs until completion - -# 3. Auto-terminate on completion -monitor.auto_terminate() -``` - -## API Reference - -### RunPodClient - -```python -client = RunPodClient() - -# Query available GPUs -gpus = client.get_available_gpus(min_vram_gb=16, cloud_type="SECURE") - -# Deploy pod -pod = client.deploy_pod( - gpu_type="RTX A4000", - image="jgrusewski/foxhunt:latest", - command="--epochs 50", - container_disk_gb=50, - env_vars={"KEY": "value"}, - dry_run=False -) - -# Get pod status -status = client.get_pod_status(pod_id) - -# Terminate pod -client.terminate_pod(pod_id) - -# List all pods -pods = client.list_pods() -client.display_pods_table(pods) -``` - -### PodMonitor - -```python -monitor = PodMonitor(pod_id) - -# Wait for pod to start -monitor.wait_until_running(timeout=300) - -# Stream logs from S3 -monitor.stream_s3_logs( - follow=True, # Continue streaming - poll_interval=5, # Check every 5s - max_lines=1000 # Limit output -) - -# Check completion -completed, error = monitor.check_completion() - -# Auto-terminate on completion -monitor.auto_terminate(wait_for_completion=True) - -# Display pod info -monitor.display_pod_info() -``` - -### S3Client - -```python -s3 = S3Client() - -# Upload binary with progress -s3.upload_binary( - local_file=Path("target/release/examples/train_tft_parquet"), - s3_key="binaries/train_tft_parquet", - force=False -) - -# Tail log file (byte-range request) -content, new_position = s3.tail_log_file( - s3_key="logs/training.log", - start_byte=0, - max_bytes=1024*1024 # 1MB chunks -) - -# Download results -files = s3.download_results( - s3_prefix="models/tft/", - local_dir=Path("./models") -) - -# List inventory -binaries = s3.list_binaries() -models = s3.list_models() -``` - -## S3 Log Monitoring Architecture - -The module monitors training progress via S3 log tailing (NO SSH required): - -1. **Log Writing**: Training container writes to `/runpod-volume/logs/training.log` -2. **S3 Sync**: RunPod network volume auto-syncs to S3 bucket -3. **Byte-Range Polling**: Monitor uses HTTP byte-range requests to tail new content -4. **Pattern Detection**: Checks for completion/error patterns in logs - -### Log File Location - -``` -Pod Container: /runpod-volume/logs/training.log - ↓ (auto-synced) -S3 Bucket: s3://se3zdnb5o4/logs/training.log - ↓ (byte-range GET) -PodMonitor: Streams new content every 5s -``` - -### Completion Detection - -Automatically detects training completion by matching patterns: - -**Success Patterns** (configurable): -- "Training complete" -- "Model saved to" -- "✓ Training finished" -- "SUCCESS:" - -**Error Patterns**: -- "CUDA out of memory" -- "RuntimeError:" -- "ERROR:" -- "panic!" - -## Error Handling - -All errors inherit from `RunPodError`: - -```python -from foxhunt_runpod.errors import ( - RunPodError, # Base exception - ConfigurationError, # Invalid config - PodDeploymentError, # Deployment failed - PodNotFoundError, # Pod doesn't exist - PodTimeoutError, # Operation timed out - S3Error, # S3 operation failed - APIError, # RunPod API error - NetworkError, # Network failure -) - -try: - pod = client.deploy_pod() -except PodDeploymentError as e: - print(f"GPU: {e.gpu_type}, Datacenter: {e.datacenter}") -except NetworkError as e: - print(f"Retries: {e.retry_count}") -``` - -## Configuration Options - -All settings can be overridden via environment variables or config object: - -```python -from foxhunt_runpod.config import RunPodConfig - -config = RunPodConfig( - # API settings - api_timeout=60, # Request timeout (seconds) - max_retries=3, # Retry attempts - retry_backoff=2.0, # Exponential backoff multiplier - - # Monitoring settings - log_poll_interval=5, # S3 log polling (seconds) - pod_status_poll_interval=10, # Pod status polling (seconds) - - # Deployment settings - datacenters=["EUR-IS-1"], # Preferred datacenters - cloud_type="SECURE", # SECURE or COMMUNITY - container_disk_gb=50, # Container disk size - min_vram_gb=16, # Minimum GPU VRAM -) - -client = RunPodClient(config) -``` - -## Example Scripts - -### Deploy and Monitor Training - -```python -#!/usr/bin/env python3 -"""Deploy TFT training on RunPod with monitoring.""" - -from foxhunt_runpod import RunPodClient, PodMonitor - -def main(): - # Deploy pod - client = RunPodClient() - pod = client.deploy_pod( - gpu_type="RTX A4000", - command="--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50" - ) - - pod_id = pod['id'] - print(f"Deployed pod: {pod_id}") - - # Monitor training - monitor = PodMonitor(pod_id) - - # Wait for pod to start (5 min timeout) - monitor.wait_until_running(timeout=300) - - # Stream logs until completion - monitor.stream_s3_logs(follow=True) - - # Auto-terminate - monitor.auto_terminate() - -if __name__ == "__main__": - main() -``` - -### List and Terminate Pods - -```python -#!/usr/bin/env python3 -"""List all pods and terminate old ones.""" - -from foxhunt_runpod import RunPodClient -from datetime import datetime, timedelta - -client = RunPodClient() - -# List all pods -pods = client.list_pods() -client.display_pods_table(pods) - -# Terminate pods older than 2 hours -cutoff = datetime.now() - timedelta(hours=2) - -for pod in pods: - created_at = datetime.fromisoformat(pod.get('createdAt', '')) - if created_at < cutoff: - print(f"Terminating old pod: {pod['id']}") - client.terminate_pod(pod['id']) -``` - -## Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Foxhunt RunPod Module │ -├─────────────────────────────────────────────────────────────┤ -│ │ -│ RunPodClient PodMonitor S3Client │ -│ ├─ get_available_gpus ├─ wait_until_running ├─ upload_binary │ -│ ├─ deploy_pod ├─ stream_s3_logs ├─ tail_log_file │ -│ ├─ get_pod_status ├─ check_completion ├─ download_results │ -│ ├─ terminate_pod └─ auto_terminate └─ list_binaries │ -│ └─ list_pods │ -│ │ -│ RunPodConfig Errors │ -│ ├─ pydantic validation ├─ RunPodError │ -│ └─ .env.runpod loader ├─ PodDeploymentError │ -│ ├─ S3Error │ -│ └─ NetworkError │ -└─────────────────────────────────────────────────────────────┘ - │ │ │ - ▼ ▼ ▼ - RunPod REST API RunPod GraphQL RunPod S3 API - (pod management) (GPU queries) (log tailing) -``` - -## Testing - -```bash -# Deploy test pod (dry run) -python -c " -from foxhunt_runpod import RunPodClient -client = RunPodClient() -client.deploy_pod(dry_run=True) -" - -# Query available GPUs -python -c " -from foxhunt_runpod import RunPodClient -client = RunPodClient() -gpus = client.get_available_gpus() -print(f'Found {len(gpus)} GPUs') -" - -# Test S3 connection -python -c " -from foxhunt_runpod import S3Client -s3 = S3Client() -binaries = s3.list_binaries() -print(f'Binaries: {len(binaries)}') -" -``` - -## Production Checklist - -- [x] Type hints for all functions -- [x] Pydantic validation for config -- [x] Retry logic with exponential backoff -- [x] Comprehensive error handling -- [x] Rich terminal output -- [x] S3-based monitoring (NO SSH) -- [x] Automatic pod termination -- [x] Configuration via env files -- [x] Docstrings for all public methods -- [x] GPU selection by price -- [x] Datacenter filtering - -## License - -Proprietary - Foxhunt HFT Trading System diff --git a/runpod/__init__.py b/runpod/__init__.py deleted file mode 100644 index 52fa67084..000000000 --- a/runpod/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Foxhunt RunPod Workflow Module - -Production-ready Python module for deploying and monitoring ML training on RunPod GPU infrastructure. - -Features: -- Pod deployment via REST API with datacenter filtering -- S3-based log monitoring (NO SSH required) -- Automatic pod termination on training completion -- Progress tracking with rich output -- Retry logic with exponential backoff -- Comprehensive error handling - -Usage: - from foxhunt_runpod import RunPodClient, PodMonitor, S3Client - - # Deploy pod - client = RunPodClient() - pod = client.deploy_pod(gpu_type="RTX A4000", command="--epochs 50") - - # Monitor training - monitor = PodMonitor(pod['id']) - monitor.wait_until_running(timeout=300) - monitor.stream_s3_logs(follow=True) - - # Auto-terminate on completion - monitor.auto_terminate() -""" - -from .client import RunPodClient -from .monitor import PodMonitor -from .s3_client import S3Client -from .config import RunPodConfig -from .errors import ( - RunPodError, - PodDeploymentError, - PodNotFoundError, - S3Error, - ConfigurationError, -) - -__version__ = "1.0.0" -__all__ = [ - "RunPodClient", - "PodMonitor", - "S3Client", - "RunPodConfig", - "RunPodError", - "PodDeploymentError", - "PodNotFoundError", - "S3Error", - "ConfigurationError", -] diff --git a/runpod/client.py b/runpod/client.py deleted file mode 100644 index d2b16684f..000000000 --- a/runpod/client.py +++ /dev/null @@ -1,286 +0,0 @@ -""" -RunPod Client - High-level API for pod management -""" - -import os -import sys -import requests -import shlex -from typing import Optional, List, Dict, Any - - -class RunPodClient: - """High-level client for RunPod pod management.""" - - def __init__(self, api_key: str, volume_id: str, registry_auth_id: Optional[str] = None): - """ - Initialize RunPod client. - - Args: - api_key: RunPod API key - volume_id: RunPod network volume ID - registry_auth_id: Docker registry auth ID (optional) - """ - self.api_key = api_key - self.volume_id = volume_id - self.registry_auth_id = registry_auth_id - - self.graphql_endpoint = "https://api.runpod.io/graphql" - self.rest_api_url = "https://rest.runpod.io/v1/pods" - - # EUR-IS-1 datacenter (volume location) - self.datacenters = ['EUR-IS-1'] - - def get_available_gpus(self, min_vram: int = 16) -> List[Dict[str, Any]]: - """ - Query available GPU types with minimum VRAM. - - Args: - min_vram: Minimum VRAM in GB (default: 16) - - Returns: - List of GPU dictionaries sorted by price (cheapest first) - """ - query = """ - { - gpuTypes { - id - displayName - memoryInGb - secureCloud - communityCloud - lowestPrice(input: {gpuCount: 1}) { - uninterruptablePrice - } - } - } - """ - - data = self._query_graphql(query) - if not data: - return [] - - gpu_types = data.get('data', {}).get('gpuTypes', []) - - available_gpus = [] - for gpu in gpu_types: - memory = gpu.get('memoryInGb', 0) - secure_count = gpu.get('secureCloud', 0) - lowest_price = gpu.get('lowestPrice', {}) - price = lowest_price.get('uninterruptablePrice') if lowest_price else None - - if memory >= min_vram and secure_count > 0 and price is not None: - available_gpus.append({ - 'id': gpu.get('id', ''), - 'name': gpu.get('displayName', 'Unknown'), - 'vram': memory, - 'price': float(price), - 'global_available': secure_count - }) - - # Sort by price (cheapest first) - available_gpus.sort(key=lambda x: x['price']) - - return available_gpus - - def deploy_pod( - self, - gpu_id: str, - image: str, - command: Optional[str] = None, - container_disk: int = 50, - ports: Optional[List[str]] = None, - env: Optional[Dict[str, str]] = None, - dry_run: bool = False - ) -> Optional[Dict[str, Any]]: - """ - Deploy a pod using REST API. - - Args: - gpu_id: GPU type ID - image: Docker image name - command: Docker command (optional) - container_disk: Container disk size in GB - ports: Port mappings (e.g., ["8888/http", "22/tcp"]) - env: Environment variables - dry_run: Show plan without deploying - - Returns: - Pod data dictionary if successful, None otherwise - """ - pod_name = "foxhunt-training" - - if ports is None: - ports = ["8888/http", "22/tcp"] - - if env is None: - env = {} - - deployment_payload = { - "cloudType": "SECURE", - "computeType": "GPU", - "dataCenterIds": self.datacenters, - "dataCenterPriority": "availability", - "gpuTypeIds": [gpu_id], - "gpuTypePriority": "availability", - "gpuCount": 1, - "name": pod_name, - "imageName": image, - "containerDiskInGb": container_disk, - "volumeInGb": 0, - "networkVolumeId": self.volume_id, - "volumeMountPath": "/runpod-volume", - "ports": ports, - "env": env, - "interruptible": False, - "minRAMPerGPU": 8, - "minVCPUPerGPU": 2 - } - - if command: - deployment_payload["dockerStartCmd"] = shlex.split(command) - - if self.registry_auth_id: - deployment_payload["containerRegistryAuthId"] = self.registry_auth_id - - if dry_run: - import json - print("\n" + "="*70) - print("DRY RUN: Deployment payload") - print("="*70) - print(json.dumps(deployment_payload, indent=2)) - return None - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}" - } - - try: - response = requests.post( - self.rest_api_url, - json=deployment_payload, - headers=headers, - timeout=60 - ) - - if response.status_code in [200, 201]: - pod_data = response.json() - - if not isinstance(pod_data, dict) or 'id' not in pod_data: - return None - - return pod_data - - elif response.status_code == 400: - try: - error_data = response.json() - error_msg = error_data.get('error', 'Unknown error') - print(f" ⚠️ Deployment failed: {error_msg}") - except ValueError: - print(f" ⚠️ Deployment failed: {response.text[:200]}") - - return None - - else: - print(f" ❌ Unexpected response (status {response.status_code})") - return None - - except requests.exceptions.RequestException as e: - print(f" ⚠️ Deployment failed: {str(e)[:100]}") - return None - - def stop_pod(self, pod_id: str) -> bool: - """ - Stop a running pod. - - Args: - pod_id: Pod ID to stop - - Returns: - True if successful, False otherwise - """ - url = f"{self.rest_api_url}/{pod_id}/stop" - headers = { - "Authorization": f"Bearer {self.api_key}" - } - - try: - response = requests.post(url, headers=headers, timeout=30) - return response.status_code in [200, 204] - except requests.exceptions.RequestException: - return False - - def terminate_pod(self, pod_id: str) -> bool: - """ - Terminate (delete) a pod. - - Args: - pod_id: Pod ID to terminate - - Returns: - True if successful, False otherwise - """ - url = f"{self.rest_api_url}/{pod_id}/terminate" - headers = { - "Authorization": f"Bearer {self.api_key}" - } - - try: - response = requests.post(url, headers=headers, timeout=30) - return response.status_code in [200, 204] - except requests.exceptions.RequestException: - return False - - def get_pod_status(self, pod_id: str) -> Optional[Dict[str, Any]]: - """ - Get pod status. - - Args: - pod_id: Pod ID - - Returns: - Pod status dictionary if successful, None otherwise - """ - url = f"{self.rest_api_url}/{pod_id}" - headers = { - "Authorization": f"Bearer {self.api_key}" - } - - try: - response = requests.get(url, headers=headers, timeout=30) - if response.status_code == 200: - return response.json() - return None - except requests.exceptions.RequestException: - return None - - def _query_graphql(self, query: str, variables: Optional[Dict] = None) -> Optional[Dict]: - """Execute GraphQL query against RunPod API.""" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}" - } - - payload = {"query": query} - if variables: - payload["variables"] = variables - - try: - response = requests.post( - self.graphql_endpoint, - json=payload, - headers=headers, - timeout=30 - ) - response.raise_for_status() - result = response.json() - - if 'errors' in result: - print(f"ERROR: GraphQL errors: {result['errors']}") - return None - - return result - except requests.exceptions.RequestException as e: - print(f"ERROR: Failed to query RunPod GraphQL API: {e}") - return None diff --git a/runpod/config.py b/runpod/config.py deleted file mode 100644 index 1b72eebb5..000000000 --- a/runpod/config.py +++ /dev/null @@ -1,265 +0,0 @@ -""" -Configuration management for RunPod workflow using pydantic-settings. - -Loads configuration from .env.runpod file with validation. -""" - -from pathlib import Path -from typing import Literal - -from pydantic import Field, field_validator -from pydantic_settings import BaseSettings, SettingsConfigDict - -from .errors import ConfigurationError - - -class RunPodConfig(BaseSettings): - """ - RunPod configuration with validation. - - All settings can be overridden via environment variables. - Default configuration file: .env.runpod in project root. - """ - - # RunPod API - runpod_api_key: str = Field( - ..., - description="RunPod API key for pod management", - min_length=32, - ) - - # RunPod S3 Credentials - runpod_s3_access_key: str = Field( - ..., - description="RunPod S3 access key for network volume", - min_length=16, - ) - runpod_s3_secret: str = Field( - ..., - description="RunPod S3 secret key", - min_length=32, - ) - runpod_s3_endpoint: str = Field( - default="https://s3api-eur-is-1.runpod.io", - description="RunPod S3 endpoint URL", - ) - runpod_s3_region: str = Field( - default="eur-is-1", - description="RunPod S3 region", - ) - - # Network Volume - runpod_volume_id: str = Field( - ..., - description="RunPod network volume ID (bucket name)", - min_length=10, - ) - runpod_volume_mount: str = Field( - default="/runpod-volume", - description="Volume mount path in containers", - ) - - # Pod Configuration - runpod_gpu_type: str = Field( - default="NVIDIA RTX 4090", - description="Preferred GPU type for deployment", - ) - runpod_image: str = Field( - default="jgrusewski/foxhunt:latest", - description="Docker image to deploy", - ) - runpod_container_registry_auth_id: str | None = Field( - default=None, - description="Container registry auth ID for private images", - ) - - # Deployment Settings - datacenters: list[str] = Field( - default=["EUR-IS-1"], - description="Preferred datacenters in priority order", - ) - cloud_type: Literal["SECURE", "COMMUNITY"] = Field( - default="SECURE", - description="Cloud type (SECURE or COMMUNITY)", - ) - container_disk_gb: int = Field( - default=50, - ge=10, - le=1000, - description="Container disk size in GB", - ) - min_vram_gb: int = Field( - default=16, - ge=8, - le=80, - description="Minimum GPU VRAM in GB", - ) - - # API Settings - api_timeout: int = Field( - default=60, - ge=10, - le=300, - description="API request timeout in seconds", - ) - max_retries: int = Field( - default=3, - ge=0, - le=10, - description="Maximum API retry attempts", - ) - retry_backoff: float = Field( - default=2.0, - ge=1.0, - le=10.0, - description="Exponential backoff multiplier for retries", - ) - - # Monitoring Settings - log_poll_interval: int = Field( - default=5, - ge=1, - le=60, - description="S3 log polling interval in seconds", - ) - pod_status_poll_interval: int = Field( - default=10, - ge=5, - le=120, - description="Pod status polling interval in seconds", - ) - completion_check_patterns: list[str] = Field( - default=[ - "Training complete", - "Model saved to", - "✓ Training finished", - "SUCCESS:", - ], - description="Patterns to detect training completion in logs", - ) - error_patterns: list[str] = Field( - default=[ - "CUDA out of memory", - "RuntimeError:", - "AssertionError:", - "FAILED:", - "ERROR:", - "panic!", - ], - description="Patterns to detect errors in logs", - ) - - # Paths - log_file_path: str = Field( - default="logs/training.log", - description="Path to training log file in volume (relative to mount)", - ) - models_dir: str = Field( - default="models", - description="Path to models directory in volume (relative to mount)", - ) - - model_config = SettingsConfigDict( - env_file=".env.runpod", - env_file_encoding="utf-8", - case_sensitive=False, - extra="ignore", - ) - - @field_validator("runpod_s3_endpoint") - @classmethod - def validate_endpoint(cls, v: str) -> str: - """Ensure S3 endpoint is a valid HTTPS URL.""" - if not v.startswith("https://"): - raise ValueError("S3 endpoint must use HTTPS") - return v - - @field_validator("datacenters") - @classmethod - def validate_datacenters(cls, v: list[str]) -> list[str]: - """Ensure at least one datacenter is specified.""" - if not v: - raise ValueError("At least one datacenter must be specified") - return v - - @classmethod - def load_from_file(cls, env_file: str | Path = ".env.runpod") -> "RunPodConfig": - """ - Load configuration from a specific env file. - - Args: - env_file: Path to .env file (relative to project root or absolute) - - Returns: - Validated RunPodConfig instance - - Raises: - ConfigurationError: If configuration is invalid or file not found - """ - env_path = Path(env_file) - - # Try project root if relative path - if not env_path.is_absolute(): - # We're in foxhunt/runpod/, go up 1 level to project root - project_root = Path(__file__).parent.parent - env_path = project_root / env_file - - if not env_path.exists(): - raise ConfigurationError( - f"Configuration file not found: {env_path}\n" - f"Create .env.runpod in project root with required credentials." - ) - - try: - # Override model_config to use specific file - config = cls(_env_file=str(env_path)) - return config - except Exception as e: - raise ConfigurationError(f"Failed to load configuration: {e}") from e - - def get_s3_bucket(self) -> str: - """Get S3 bucket name (same as volume ID).""" - return self.runpod_volume_id - - def get_log_s3_key(self) -> str: - """Get S3 key for training log file.""" - return self.log_file_path - - def get_models_s3_prefix(self) -> str: - """Get S3 prefix for models directory.""" - prefix = self.models_dir - if not prefix.endswith("/"): - prefix += "/" - return prefix - - def get_api_headers(self) -> dict[str, str]: - """Get HTTP headers for RunPod API requests.""" - return { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.runpod_api_key}", - } - - -# Global config instance (lazy-loaded) -_config: RunPodConfig | None = None - - -def get_config(reload: bool = False) -> RunPodConfig: - """ - Get global configuration instance. - - Args: - reload: Force reload configuration from file - - Returns: - RunPodConfig instance - - Raises: - ConfigurationError: If configuration cannot be loaded - """ - global _config - - if _config is None or reload: - _config = RunPodConfig.load_from_file() - - return _config diff --git a/runpod/errors.py b/runpod/errors.py deleted file mode 100644 index ed6dc362e..000000000 --- a/runpod/errors.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Custom exceptions for RunPod workflow operations. - -All exceptions inherit from RunPodError for easy catching of all module errors. -""" - - -class RunPodError(Exception): - """Base exception for all RunPod workflow errors.""" - - pass - - -class ConfigurationError(RunPodError): - """Raised when configuration is invalid or missing required values.""" - - def __init__(self, message: str, missing_keys: list[str] | None = None): - super().__init__(message) - self.missing_keys = missing_keys or [] - - -class PodDeploymentError(RunPodError): - """Raised when pod deployment fails.""" - - def __init__(self, message: str, gpu_type: str | None = None, datacenter: str | None = None): - super().__init__(message) - self.gpu_type = gpu_type - self.datacenter = datacenter - - -class PodNotFoundError(RunPodError): - """Raised when a pod cannot be found.""" - - def __init__(self, pod_id: str): - super().__init__(f"Pod not found: {pod_id}") - self.pod_id = pod_id - - -class PodTerminationError(RunPodError): - """Raised when pod termination fails.""" - - def __init__(self, pod_id: str, reason: str): - super().__init__(f"Failed to terminate pod {pod_id}: {reason}") - self.pod_id = pod_id - self.reason = reason - - -class PodTimeoutError(RunPodError): - """Raised when a pod operation times out.""" - - def __init__(self, pod_id: str, operation: str, timeout: int): - super().__init__( - f"Pod {pod_id} {operation} operation timed out after {timeout} seconds" - ) - self.pod_id = pod_id - self.operation = operation - self.timeout = timeout - - -class S3Error(RunPodError): - """Raised when S3 operations fail.""" - - def __init__(self, message: str, bucket: str | None = None, key: str | None = None): - super().__init__(message) - self.bucket = bucket - self.key = key - - -class S3ObjectNotFoundError(S3Error): - """Raised when an S3 object doesn't exist.""" - - def __init__(self, bucket: str, key: str): - super().__init__(f"S3 object not found: s3://{bucket}/{key}", bucket, key) - - -class APIError(RunPodError): - """Raised when RunPod API returns an error.""" - - def __init__(self, message: str, status_code: int | None = None, response: dict | None = None): - super().__init__(message) - self.status_code = status_code - self.response = response - - -class NetworkError(RunPodError): - """Raised when network requests fail.""" - - def __init__(self, message: str, retry_count: int = 0): - super().__init__(message) - self.retry_count = retry_count diff --git a/runpod/example_usage.py b/runpod/example_usage.py deleted file mode 100755 index 9ce70f79a..000000000 --- a/runpod/example_usage.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 -""" -Example usage of foxhunt_runpod module. - -Demonstrates: -- Pod deployment with GPU selection -- S3 log monitoring -- Auto-termination on completion -""" - -import sys -from pathlib import Path - -# Add parent directory to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from foxhunt_runpod import RunPodClient, PodMonitor, S3Client -from foxhunt_runpod.errors import RunPodError - - -def example_1_deploy_and_monitor(): - """Example 1: Deploy pod and monitor training.""" - print("\n" + "=" * 70) - print("EXAMPLE 1: Deploy and Monitor Training") - print("=" * 70 + "\n") - - try: - # Initialize client - client = RunPodClient() - - # Deploy pod with auto GPU selection - print("Deploying pod...") - pod = client.deploy_pod( - gpu_type="RTX A4000", # Preferred GPU (auto-selects if unavailable) - command="--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50", - container_disk_gb=50, - ) - - pod_id = pod["id"] - print(f"\n✅ Pod deployed: {pod_id}") - - # Monitor training - monitor = PodMonitor(pod_id) - - # Wait for pod to start - print("\nWaiting for pod to start...") - monitor.wait_until_running(timeout=300) - - # Stream logs until completion - print("\nStreaming training logs...") - monitor.stream_s3_logs(follow=True) - - # Auto-terminate on completion - print("\nTerminating pod...") - monitor.auto_terminate() - - except RunPodError as e: - print(f"\n❌ Error: {e}") - return False - - return True - - -def example_2_list_gpus(): - """Example 2: List available GPUs.""" - print("\n" + "=" * 70) - print("EXAMPLE 2: List Available GPUs") - print("=" * 70 + "\n") - - try: - client = RunPodClient() - - # Query available GPUs - gpus = client.get_available_gpus(min_vram_gb=16) - - print(f"Found {len(gpus)} GPU type(s):\n") - - for i, gpu in enumerate(gpus, 1): - print(f"{i}. {gpu['name']}") - print(f" VRAM: {gpu['vram']}GB") - print(f" Price: ${gpu['price']:.3f}/hr") - print(f" Available: {gpu['global_available']} (global secure cloud)") - print() - - except RunPodError as e: - print(f"\n❌ Error: {e}") - return False - - return True - - -def example_3_list_pods(): - """Example 3: List all pods.""" - print("\n" + "=" * 70) - print("EXAMPLE 3: List All Pods") - print("=" * 70 + "\n") - - try: - client = RunPodClient() - - # List all pods - pods = client.list_pods() - - if not pods: - print("No pods found") - return True - - # Display in table format - client.display_pods_table(pods) - - except RunPodError as e: - print(f"\n❌ Error: {e}") - return False - - return True - - -def example_4_s3_operations(): - """Example 4: S3 operations (list binaries, models).""" - print("\n" + "=" * 70) - print("EXAMPLE 4: S3 Operations") - print("=" * 70 + "\n") - - try: - s3 = S3Client() - - # List binaries - print("Binaries on S3:") - binaries = s3.list_binaries() - for binary in binaries: - size_mb = binary["size"] / (1024 * 1024) - print(f" - {binary['name']} ({size_mb:.1f} MB)") - - print() - - # List models - print("Models on S3:") - models = s3.list_models() - for model in models: - size_mb = model["size"] / (1024 * 1024) - print(f" - {model['name']} ({size_mb:.1f} MB)") - - except RunPodError as e: - print(f"\n❌ Error: {e}") - return False - - return True - - -def example_5_monitor_existing_pod(): - """Example 5: Monitor an existing pod.""" - print("\n" + "=" * 70) - print("EXAMPLE 5: Monitor Existing Pod") - print("=" * 70 + "\n") - - pod_id = input("Enter pod ID to monitor: ").strip() - - if not pod_id: - print("❌ No pod ID provided") - return False - - try: - monitor = PodMonitor(pod_id) - - # Display pod info - monitor.display_pod_info() - - # Stream logs - print("\nStreaming logs (Ctrl+C to stop)...") - monitor.stream_s3_logs(follow=True) - - except RunPodError as e: - print(f"\n❌ Error: {e}") - return False - - return True - - -def main(): - """Main menu.""" - examples = { - "1": ("Deploy and Monitor Training", example_1_deploy_and_monitor), - "2": ("List Available GPUs", example_2_list_gpus), - "3": ("List All Pods", example_3_list_pods), - "4": ("S3 Operations", example_4_s3_operations), - "5": ("Monitor Existing Pod", example_5_monitor_existing_pod), - } - - print("\n" + "=" * 70) - print("FOXHUNT RUNPOD MODULE - EXAMPLES") - print("=" * 70 + "\n") - - for key, (name, _) in examples.items(): - print(f"{key}. {name}") - - print("q. Quit\n") - - choice = input("Select example: ").strip().lower() - - if choice == "q": - print("Goodbye!") - return - - if choice not in examples: - print(f"❌ Invalid choice: {choice}") - return - - # Run selected example - _, example_func = examples[choice] - success = example_func() - - if success: - print("\n✅ Example completed successfully") - else: - print("\n❌ Example failed") - - -if __name__ == "__main__": - main() diff --git a/runpod/monitor.py b/runpod/monitor.py deleted file mode 100644 index 2b35d406e..000000000 --- a/runpod/monitor.py +++ /dev/null @@ -1,302 +0,0 @@ -""" -Pod monitoring with S3 log tailing. - -Provides status polling, log streaming, completion detection, and auto-termination. -""" - -import re -import time -from rich.console import Console -from rich.live import Live -from rich.text import Text - -from .client import RunPodClient -from .config import RunPodConfig, get_config -from .s3_client import S3Client -from .errors import ( - PodNotFoundError, - PodTimeoutError, - S3ObjectNotFoundError, - RunPodError, -) - -console = Console() - - -class PodMonitor: - """ - Monitor RunPod pods with S3 log tailing. - - NO SSH required - uses S3 byte-range requests to tail logs. - """ - - def __init__( - self, - pod_id: str, - config: RunPodConfig | None = None, - client: RunPodClient | None = None, - s3_client: S3Client | None = None, - ): - """ - Initialize pod monitor. - - Args: - pod_id: Pod ID to monitor - config: RunPodConfig instance (uses global if None) - client: RunPodClient instance (creates if None) - s3_client: S3Client instance (creates if None) - """ - self.pod_id = pod_id - self.config = config or get_config() - self.client = client or RunPodClient( - api_key=self.config.runpod_api_key, - volume_id=self.config.runpod_volume_id, - registry_auth_id=self.config.runpod_container_registry_auth_id - ) - self.s3_client = s3_client or S3Client(self.config) - - # Log tailing state - self.log_position = 0 - self.last_log_check = 0 - - # Status - self.training_complete = False - self.error_detected = False - - def wait_until_running( - self, timeout: int = 300, poll_interval: int | None = None - ) -> bool: - """ - Wait until pod is in RUNNING state. - - Args: - timeout: Maximum wait time in seconds - poll_interval: Polling interval (uses config default if None) - - Returns: - True if pod is running - - Raises: - PodTimeoutError: If pod doesn't start within timeout - PodNotFoundError: If pod doesn't exist - """ - poll_interval = poll_interval or self.config.pod_status_poll_interval - start_time = time.time() - - console.print(f"Waiting for pod {self.pod_id} to start...") - - while True: - elapsed = time.time() - start_time - if elapsed > timeout: - raise PodTimeoutError(self.pod_id, "start", timeout) - - try: - status_data = self.client.get_pod_status(self.pod_id) - runtime_status = status_data.get("runtime", {}).get("status", "UNKNOWN") - - console.print( - f" Status: {runtime_status} (elapsed: {int(elapsed)}s)", end="\r" - ) - - if runtime_status == "RUNNING": - console.print( - f"\n[green]✅ Pod is running! (took {int(elapsed)}s)[/green]" - ) - return True - - if runtime_status == "FAILED": - console.print(f"\n[red]❌ Pod failed to start[/red]") - return False - - except PodNotFoundError: - console.print(f"\n[red]❌ Pod not found: {self.pod_id}[/red]") - raise - - time.sleep(poll_interval) - - def stream_s3_logs( - self, - follow: bool = True, - poll_interval: int | None = None, - max_lines: int | None = None, - ) -> None: - """ - Stream logs from S3 (byte-range tailing). - - Args: - follow: Continue streaming until completion detected - poll_interval: Polling interval (uses config default if None) - max_lines: Maximum lines to display (None for unlimited) - - Raises: - RunPodError: On errors - """ - poll_interval = poll_interval or self.config.log_poll_interval - log_key = self.config.get_log_s3_key() - - console.print(f"\nStreaming logs from s3://{self.s3_client.bucket}/{log_key}") - console.print("(Polling every {poll_interval}s, Ctrl+C to stop)\n") - console.print("-" * 70) - - lines_shown = 0 - - try: - while True: - # Wait for log file to appear - if not self.s3_client.object_exists(log_key): - if not follow: - console.print( - "[yellow]⚠ Log file not found (training may not have started)[/yellow]" - ) - return - - # Still waiting for log file - time.sleep(poll_interval) - continue - - # Tail new content - try: - content, new_position = self.s3_client.tail_log_file( - log_key, start_byte=self.log_position - ) - - if content: - # Decode and print new lines - text = content.decode("utf-8", errors="ignore") - lines = text.splitlines() - - for line in lines: - console.print(line) - lines_shown += 1 - - # Check for completion/error patterns - self._check_line_patterns(line) - - # Max lines limit - if max_lines and lines_shown >= max_lines: - console.print( - f"\n[yellow]Reached max lines ({max_lines})[/yellow]" - ) - return - - self.log_position = new_position - - except S3ObjectNotFoundError: - # Log file was deleted or doesn't exist yet - pass - - # Check completion - if self.training_complete or self.error_detected: - console.print("\n" + "-" * 70) - if self.training_complete: - console.print("[green]✅ Training completed![/green]") - if self.error_detected: - console.print("[red]❌ Error detected in logs[/red]") - return - - if not follow: - return - - time.sleep(poll_interval) - - except KeyboardInterrupt: - console.print("\n\n[yellow]Streaming stopped by user[/yellow]") - - def _check_line_patterns(self, line: str) -> None: - """Check log line for completion/error patterns.""" - # Check completion patterns - for pattern in self.config.completion_check_patterns: - if pattern.lower() in line.lower(): - self.training_complete = True - return - - # Check error patterns - for pattern in self.config.error_patterns: - if pattern in line: - self.error_detected = True - return - - def check_completion(self) -> tuple[bool, bool]: - """ - Check if training has completed or errored. - - Returns: - Tuple of (completed, error_detected) - """ - return self.training_complete, self.error_detected - - def auto_terminate(self, wait_for_completion: bool = True) -> bool: - """ - Automatically terminate pod when training completes. - - Args: - wait_for_completion: Wait for completion before terminating - - Returns: - True if pod was terminated - - Raises: - RunPodError: On errors - """ - if wait_for_completion: - console.print("\nWaiting for training to complete...") - - # Stream logs until completion - self.stream_s3_logs(follow=True) - - # Check status - completed, error = self.check_completion() - - if not completed and not error: - console.print( - "[yellow]⚠ Training not complete, skipping termination[/yellow]" - ) - return False - - # Terminate pod - console.print(f"\nTerminating pod {self.pod_id}...") - try: - self.client.terminate_pod(self.pod_id) - return True - except Exception as e: - console.print(f"[yellow]⚠ Failed to terminate: {e}[/yellow]") - return False - - def get_pod_info(self) -> dict: - """ - Get current pod information. - - Returns: - Pod status dict - """ - try: - return self.client.get_pod_status(self.pod_id) - except PodNotFoundError: - return {} - - def display_pod_info(self) -> None: - """Display current pod information in formatted output.""" - info = self.get_pod_info() - - if not info: - console.print(f"[yellow]Pod {self.pod_id} not found[/yellow]") - return - - console.print("\n" + "=" * 70) - console.print(f"POD INFO: {self.pod_id}") - console.print("=" * 70) - - console.print(f"Status: {info.get('desiredStatus', 'UNKNOWN')}") - - runtime = info.get("runtime", {}) - runtime_status = runtime.get("status", "UNKNOWN") - console.print(f"Runtime: {runtime_status}") - - machine = info.get("machine", {}) - if machine: - gpu_info = machine.get("gpuType", {}) - console.print(f"GPU: {gpu_info.get('displayName', 'N/A')}") - - console.print(f"Cost/hr: ${info.get('costPerHr', 'N/A')}") - - console.print("=" * 70) diff --git a/runpod/requirements.txt b/runpod/requirements.txt deleted file mode 100644 index c6c2e5a4b..000000000 --- a/runpod/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Core dependencies -boto3>=1.40.0 -pydantic>=2.12.0 -pydantic-settings>=2.11.0 -python-dotenv>=1.2.0 -requests>=2.32.0 -rich>=14.2.0 - -# Retry logic -urllib3>=2.5.0 diff --git a/runpod/s3_client.py b/runpod/s3_client.py deleted file mode 100644 index 95f8c2c6b..000000000 --- a/runpod/s3_client.py +++ /dev/null @@ -1,419 +0,0 @@ -""" -S3 client for RunPod network volume operations. - -Provides: -- File uploads with progress tracking -- Log file tailing via byte-range requests -- Model download -- Volume inventory listing -""" - -import hashlib -import time -from pathlib import Path -from typing import Any, Callable - -import boto3 -from botocore.config import Config -from botocore.exceptions import ClientError -from rich.console import Console -from rich.progress import ( - BarColumn, - DownloadColumn, - Progress, - TaskID, - TextColumn, - TimeRemainingColumn, - TransferSpeedColumn, -) - -from .config import RunPodConfig, get_config -from .errors import S3Error, S3ObjectNotFoundError - -console = Console() - - -class S3Client: - """ - S3 client for RunPod network volume operations. - - Handles all S3 interactions with proper retry logic and error handling. - """ - - def __init__(self, config: RunPodConfig | None = None): - """ - Initialize S3 client. - - Args: - config: RunPodConfig instance (uses global config if None) - """ - self.config = config or get_config() - - # Initialize boto3 S3 client - self.s3_client = boto3.client( - "s3", - aws_access_key_id=self.config.runpod_s3_access_key, - aws_secret_access_key=self.config.runpod_s3_secret, - region_name=self.config.runpod_s3_region, - endpoint_url=self.config.runpod_s3_endpoint, - config=Config( - signature_version="s3v4", - s3={"addressing_style": "path"}, - retries={ - "max_attempts": self.config.max_retries, - "mode": "adaptive", - }, - connect_timeout=30, - read_timeout=self.config.api_timeout, - ), - ) - - self.bucket = self.config.get_s3_bucket() - - def compute_md5(self, file_path: Path) -> str: - """ - Compute MD5 checksum of local file. - - Args: - file_path: Path to local file - - Returns: - MD5 hex digest - - Raises: - S3Error: If file cannot be read - """ - try: - md5 = hashlib.md5() - with open(file_path, "rb") as f: - for chunk in iter(lambda: f.read(8192), b""): - md5.update(chunk) - return md5.hexdigest() - except OSError as e: - raise S3Error(f"Failed to read file {file_path}: {e}") from e - - def object_exists(self, s3_key: str) -> bool: - """ - Check if S3 object exists. - - Args: - s3_key: S3 object key - - Returns: - True if object exists, False otherwise - """ - try: - self.s3_client.head_object(Bucket=self.bucket, Key=s3_key) - return True - except ClientError as e: - if e.response["Error"]["Code"] == "404": - return False - raise S3Error(f"Failed to check object existence: {e}", self.bucket, s3_key) from e - - def get_object_size(self, s3_key: str) -> int: - """ - Get size of S3 object in bytes. - - Args: - s3_key: S3 object key - - Returns: - Object size in bytes - - Raises: - S3ObjectNotFoundError: If object doesn't exist - S3Error: On other S3 errors - """ - try: - response = self.s3_client.head_object(Bucket=self.bucket, Key=s3_key) - return response["ContentLength"] - except ClientError as e: - if e.response["Error"]["Code"] == "404": - raise S3ObjectNotFoundError(self.bucket, s3_key) - raise S3Error(f"Failed to get object size: {e}", self.bucket, s3_key) from e - - def needs_upload(self, local_file: Path, s3_key: str) -> tuple[bool, str]: - """ - Check if file needs upload based on MD5 checksum. - - Args: - local_file: Path to local file - s3_key: Destination S3 key - - Returns: - Tuple of (needs_upload, reason) - """ - local_md5 = self.compute_md5(local_file) - - try: - response = self.s3_client.head_object(Bucket=self.bucket, Key=s3_key) - remote_etag = response["ETag"].strip('"') - - if local_md5 == remote_etag: - return False, "up-to-date (checksum match)" - else: - return True, "changed (checksum mismatch)" - except ClientError as e: - if e.response["Error"]["Code"] == "404": - return True, "new file (doesn't exist on S3)" - else: - return True, f"error checking: {e}" - - def upload_binary( - self, - local_file: Path, - s3_key: str, - force: bool = False, - progress_callback: Callable[[int], None] | None = None, - ) -> bool: - """ - Upload binary file with progress tracking. - - Args: - local_file: Path to local file - s3_key: Destination S3 key - force: Skip checksum check and force upload - progress_callback: Optional callback(bytes_transferred) - - Returns: - True if upload succeeded, False otherwise - - Raises: - S3Error: On upload failure - """ - if not local_file.exists(): - raise S3Error(f"Local file not found: {local_file}") - - file_size = local_file.stat().st_size - - # Check if upload needed (unless forced) - if not force: - needs_upload, reason = self.needs_upload(local_file, s3_key) - if not needs_upload: - console.print(f"[green]✓ {s3_key}[/green]: {reason}") - return True - console.print(f"[cyan]↻ {s3_key}[/cyan]: {reason}") - - try: - # Upload with optional progress callback - self.s3_client.upload_file( - str(local_file), - self.bucket, - s3_key, - Callback=progress_callback, - ExtraArgs={"ContentType": "application/octet-stream"}, - ) - - # Verify upload - response = self.s3_client.head_object(Bucket=self.bucket, Key=s3_key) - remote_size = response["ContentLength"] - - if remote_size == file_size: - console.print(f"[green]✓ Uploaded: {s3_key} ({file_size:,} bytes)[/green]") - return True - else: - # Size mismatch - delete and raise error - console.print( - f"[red]✗ Size mismatch: local={file_size}, remote={remote_size}[/red]" - ) - self.s3_client.delete_object(Bucket=self.bucket, Key=s3_key) - raise S3Error( - f"Upload verification failed: size mismatch", self.bucket, s3_key - ) - - except ClientError as e: - raise S3Error(f"Upload failed: {e}", self.bucket, s3_key) from e - - def tail_log_file( - self, s3_key: str, start_byte: int = 0, max_bytes: int = 1024 * 1024 - ) -> tuple[bytes, int]: - """ - Tail log file using byte-range requests. - - Args: - s3_key: S3 key for log file - start_byte: Starting byte position - max_bytes: Maximum bytes to read (default 1MB) - - Returns: - Tuple of (log_content, new_position) - - Raises: - S3ObjectNotFoundError: If log file doesn't exist - S3Error: On other S3 errors - """ - try: - # Get current file size - file_size = self.get_object_size(s3_key) - - # If file hasn't grown, return empty - if file_size <= start_byte: - return b"", start_byte - - # Calculate byte range to fetch - end_byte = min(start_byte + max_bytes - 1, file_size - 1) - - # Fetch byte range - response = self.s3_client.get_object( - Bucket=self.bucket, Key=s3_key, Range=f"bytes={start_byte}-{end_byte}" - ) - - content = response["Body"].read() - new_position = start_byte + len(content) - - return content, new_position - - except ClientError as e: - if e.response["Error"]["Code"] == "NoSuchKey": - raise S3ObjectNotFoundError(self.bucket, s3_key) - raise S3Error(f"Failed to tail log file: {e}", self.bucket, s3_key) from e - - def download_results(self, s3_prefix: str, local_dir: Path) -> list[Path]: - """ - Download all files with given S3 prefix to local directory. - - Args: - s3_prefix: S3 prefix (directory) to download - local_dir: Local destination directory - - Returns: - List of downloaded file paths - - Raises: - S3Error: On download failure - """ - local_dir.mkdir(parents=True, exist_ok=True) - downloaded_files = [] - - try: - # List all objects with prefix - paginator = self.s3_client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=self.bucket, Prefix=s3_prefix) - - objects = [] - for page in pages: - objects.extend(page.get("Contents", [])) - - if not objects: - console.print(f"[yellow]⚠ No files found with prefix: {s3_prefix}[/yellow]") - return downloaded_files - - console.print(f"Found {len(objects)} file(s) to download") - - # Download each object with progress - with Progress( - TextColumn("[progress.description]{task.description}"), - BarColumn(), - DownloadColumn(), - TransferSpeedColumn(), - TimeRemainingColumn(), - console=console, - ) as progress: - for obj in objects: - s3_key = obj["Key"] - file_size = obj["Size"] - - # Skip directory markers - if s3_key.endswith("/"): - continue - - # Determine local path (preserve directory structure) - rel_path = Path(s3_key).relative_to(s3_prefix) - local_file = local_dir / rel_path - local_file.parent.mkdir(parents=True, exist_ok=True) - - # Download with progress - task = progress.add_task(f"Downloading {rel_path}", total=file_size) - - def callback(bytes_transferred): - progress.update(task, completed=bytes_transferred) - - self.s3_client.download_file( - self.bucket, s3_key, str(local_file), Callback=callback - ) - - downloaded_files.append(local_file) - console.print(f"[green]✓ Downloaded: {rel_path}[/green]") - - return downloaded_files - - except ClientError as e: - raise S3Error(f"Download failed: {e}", self.bucket, s3_prefix) from e - - def list_binaries(self) -> list[dict[str, Any]]: - """ - List all binaries in binaries/ directory. - - Returns: - List of dicts with keys: name, size, last_modified - """ - return self._list_prefix("binaries/") - - def list_models(self) -> list[dict[str, Any]]: - """ - List all models in models/ directory. - - Returns: - List of dicts with keys: name, size, last_modified - """ - models_prefix = self.config.get_models_s3_prefix() - return self._list_prefix(models_prefix) - - def _list_prefix(self, prefix: str) -> list[dict[str, Any]]: - """ - List all objects with given prefix. - - Args: - prefix: S3 prefix to list - - Returns: - List of object metadata dicts - """ - try: - response = self.s3_client.list_objects_v2(Bucket=self.bucket, Prefix=prefix) - - objects = [] - for obj in response.get("Contents", []): - # Skip directory markers - if obj["Key"].endswith("/"): - continue - - objects.append( - { - "name": obj["Key"], - "size": obj["Size"], - "last_modified": obj["LastModified"], - } - ) - - return objects - - except ClientError as e: - raise S3Error(f"Failed to list objects: {e}", self.bucket, prefix) from e - - def create_directory(self, s3_key: str) -> bool: - """ - Create directory marker in S3. - - Args: - s3_key: Directory path (should end with /) - - Returns: - True if created or already exists - """ - if not s3_key.endswith("/"): - s3_key += "/" - - try: - # Check if already exists - if self.object_exists(s3_key): - return True - - # Create empty directory marker - self.s3_client.put_object(Bucket=self.bucket, Key=s3_key) - console.print(f"[green]✓ Created directory: {s3_key}[/green]") - return True - - except ClientError as e: - console.print(f"[yellow]⚠ Failed to create directory {s3_key}: {e}[/yellow]") - return False diff --git a/runpod/s3_monitor.py b/runpod/s3_monitor.py deleted file mode 100644 index 5d899519c..000000000 --- a/runpod/s3_monitor.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -S3 Log Monitor - Stream training logs from RunPod S3 -""" - -import boto3 -import time -import re -from typing import Optional, Callable -from datetime import datetime, timedelta - - -class S3LogMonitor: - """Monitor and stream training logs from RunPod S3.""" - - def __init__( - self, - bucket_name: str, - aws_access_key: str, - aws_secret_key: str, - endpoint_url: str = "https://s3api-eur-is-1.runpod.io" - ): - """ - Initialize S3 log monitor. - - Args: - bucket_name: S3 bucket name - aws_access_key: AWS access key ID - aws_secret_key: AWS secret access key - endpoint_url: S3 endpoint URL - """ - self.bucket_name = bucket_name - - self.s3 = boto3.client( - 's3', - aws_access_key_id=aws_access_key, - aws_secret_access_key=aws_secret_key, - endpoint_url=endpoint_url - ) - - def stream_logs( - self, - pod_id: str, - interval: int = 10, - timeout: Optional[int] = None, - completion_callback: Optional[Callable[[str], bool]] = None - ): - """ - Stream logs from S3 in real-time. - - Args: - pod_id: Pod ID to monitor - interval: Polling interval in seconds - timeout: Maximum monitoring time in seconds (None = infinite) - completion_callback: Function to check if training is complete - Returns True if complete, False otherwise - """ - log_key = f"logs/{pod_id}/training.log" - last_position = 0 - start_time = time.time() - - print(f"\n{'='*70}") - print(f"MONITORING POD: {pod_id}") - print(f"{'='*70}") - print(f"Log Key: {log_key}") - print(f"Interval: {interval}s") - if timeout: - print(f"Timeout: {timeout}s ({timeout // 60}m)") - print(f"{'='*70}\n") - - try: - while True: - # Check timeout - if timeout and (time.time() - start_time) > timeout: - print(f"\n⏱️ Monitoring timeout reached ({timeout}s)") - break - - # Try to fetch log file - try: - response = self.s3.get_object( - Bucket=self.bucket_name, - Key=log_key, - Range=f"bytes={last_position}-" - ) - - # Read new content - new_content = response['Body'].read().decode('utf-8', errors='ignore') - - if new_content: - print(new_content, end='') - last_position += len(new_content.encode('utf-8')) - - # Check completion - if completion_callback and completion_callback(new_content): - print(f"\n✅ Training complete detected!") - break - - except self.s3.exceptions.NoSuchKey: - # Log file doesn't exist yet - print(f" [Waiting for logs...]", end='\r') - - except Exception as e: - error_str = str(e) - if "InvalidRange" not in error_str: - print(f"\n ⚠️ Error reading logs: {error_str[:100]}") - - # Wait before next poll - time.sleep(interval) - - except KeyboardInterrupt: - print(f"\n\n⏸️ Monitoring interrupted by user") - - def check_completion(self, log_content: str) -> bool: - """ - Check if training is complete based on log content. - - Args: - log_content: Recent log content - - Returns: - True if training complete, False otherwise - """ - # Common completion patterns - completion_patterns = [ - r"Training complete", - r"Model saved successfully", - r"Checkpoint saved.*final", - r"All epochs completed", - r"✅.*complete", - ] - - for pattern in completion_patterns: - if re.search(pattern, log_content, re.IGNORECASE): - return True - - return False - - def get_recent_logs(self, pod_id: str, lines: int = 50) -> Optional[str]: - """ - Get recent log lines. - - Args: - pod_id: Pod ID - lines: Number of lines to fetch (approximate) - - Returns: - Recent log content or None if not available - """ - log_key = f"logs/{pod_id}/training.log" - - try: - # Get object metadata to find size - response = self.s3.head_object( - Bucket=self.bucket_name, - Key=log_key - ) - - size = response['ContentLength'] - - # Estimate bytes to fetch (assuming ~100 bytes per line) - fetch_bytes = lines * 100 - start = max(0, size - fetch_bytes) - - # Fetch tail of file - response = self.s3.get_object( - Bucket=self.bucket_name, - Key=log_key, - Range=f"bytes={start}-" - ) - - content = response['Body'].read().decode('utf-8', errors='ignore') - - # Return last N lines - all_lines = content.split('\n') - return '\n'.join(all_lines[-lines:]) - - except Exception as e: - print(f" ⚠️ Error fetching logs: {str(e)[:100]}") - return None diff --git a/scripts/EPSILON_FIX3_VALIDATION_README.md b/scripts/EPSILON_FIX3_VALIDATION_README.md deleted file mode 100644 index d071735ff..000000000 --- a/scripts/EPSILON_FIX3_VALIDATION_README.md +++ /dev/null @@ -1,199 +0,0 @@ -# Fix #3 Epsilon Decay Validation Script - -## Overview -Comprehensive validation test script for **Fix #3: Epsilon Decay Range Change** (`[0.990, 0.999]` → `[0.95, 0.99]`). - -**Expected Outcome**: Break 100% HOLD bias, enable action diversity. - -## Script Location -```bash -/home/jgrusewski/Work/foxhunt/scripts/validate_epsilon_fix3.sh -``` - -## Usage - -### Basic Run -```bash -cd /home/jgrusewski/Work/foxhunt -./scripts/validate_epsilon_fix3.sh -``` - -### Expected Runtime -- **Duration**: ~5-10 minutes (5 trials × 10 epochs) -- **GPU**: RTX 3050 Ti (CUDA-accelerated) -- **Output**: `/tmp/ml_training/fix3_validation/` - -## Success Criteria - -The script validates Fix #3 using three criteria: - -### ✅ Criterion 1: At least 1 trial with <90% HOLD -- **Target**: Break the 100% HOLD bias -- **Threshold**: At least 1 trial showing <90% HOLD actions -- **Pass**: `trials_with_diversity >= 1` - -### ✅ Criterion 2: Action diversity >0% -- **Target**: BUY or SELL actions observed -- **Threshold**: min_hold_pct < 100% -- **Pass**: At least one trial shows non-zero BUY/SELL percentage - -### ✅ Criterion 3: Epsilon values varying -- **Target**: Hyperopt explores epsilon_decay space -- **Threshold**: At least 2 unique epsilon values across trials -- **Pass**: `unique_epsilons > 1` - -## Output Files - -### 1. Raw Log File -``` -/tmp/ml_training/fix3_validation/test_YYYYMMDD_HHMMSS.log -``` -Complete hyperopt output including: -- Trial parameters -- Training progress -- Action distributions -- Objective values - -### 2. Summary Report -``` -/tmp/ml_training/fix3_validation/summary_YYYYMMDD_HHMMSS.txt -``` -Structured analysis including: -- Epsilon decay values observed -- Action distribution statistics -- Success criteria validation -- Overall assessment (PASS/FAIL) - -## Example Output - -### Successful Validation -``` -================================================== -Fix #3 Epsilon Decay Validation Summary -================================================== - -Expected Epsilon Decay Range: [0.95, 0.99] -Actual Epsilon Values Observed: - Trial 1: 0.976 - ✅ Within expected range [0.95, 0.99] - Trial 2: 0.982 - ✅ Within expected range [0.95, 0.99] - Trial 3: 0.968 - ✅ Within expected range [0.95, 0.99] - -Action Distribution Analysis: - Trials Analyzed: 5 - Trials with <90% HOLD: 3 - Min HOLD %: 72% - Max HOLD %: 95% - -Success Criteria Validation: -✅ Criterion 1 PASS: At least 1 trial with <90% HOLD (3 trials) -✅ Criterion 2 PASS: Action diversity detected (min HOLD=72%) -✅ Criterion 3 PASS: Epsilon values varying across trials (3 unique values) - -Overall Assessment: -✅ VALIDATION PASSED: Fix #3 successfully breaks 100% HOLD bias - -Key Improvements: - - Action diversity enabled (BUY/SELL actions observed) - - HOLD percentage reduced to 72% (min) - - 3/5 trials show <90% HOLD -``` - -### Failed Validation -``` -Success Criteria Validation: -❌ Criterion 1 FAIL: No trials with <90% HOLD (all trials ≥90% HOLD) -❌ Criterion 2 FAIL: 100% HOLD bias persists - -Overall Assessment: -❌ VALIDATION FAILED: 100% HOLD bias persists - -Possible Issues: - - Epsilon decay range change not effective - - Other hyperparameters overriding epsilon effect - - Training epochs insufficient for exploration -``` - -## Exit Codes - -- `0`: VALIDATION PASSED (all criteria met) -- `1`: VALIDATION FAILED (one or more criteria failed) - -## Configuration - -Default settings (edit script to customize): - -```bash -TRIALS=5 # Number of hyperopt trials -EPOCHS=10 # Training epochs per trial -PARQUET_FILE="test_data/ES_FUT_180d.parquet" # Input data -OUTPUT_DIR="/tmp/ml_training/fix3_validation" # Results directory -``` - -## Dependencies - -- **Rust toolchain**: `cargo` (release mode) -- **CUDA**: RTX 3050 Ti GPU -- **Parquet file**: `test_data/ES_FUT_180d.parquet` -- **Binary**: `ml/examples/hyperopt_dqn_demo.rs` - -## Troubleshooting - -### Error: Parquet file not found -```bash -ERROR: Parquet file not found: test_data/ES_FUT_180d.parquet -``` -**Fix**: Ensure parquet file exists: -```bash -ls -lh test_data/ES_FUT_180d.parquet -``` - -### Error: CUDA not available -**Fix**: Verify GPU access: -```bash -nvidia-smi -cargo build --release -p ml --features cuda -``` - -### No epsilon values extracted -**Possible causes**: -1. Log format changed (check `hyperopt_dqn_demo.rs` output) -2. Trials failed early (check raw log file) -3. Regex parsing issue (verify log manually) - -## Related Files - -- **Fix Implementation**: `/home/jgrusewski/Work/foxhunt/ml/src/hyperopt/adapters/dqn.rs` (lines 44-45) -- **Hyperopt Example**: `/home/jgrusewski/Work/foxhunt/ml/examples/hyperopt_dqn_demo.rs` -- **Training Binary**: `/home/jgrusewski/Work/foxhunt/ml/examples/train_dqn.rs` - -## Quick Test (1 trial) - -For rapid validation (1-2 minutes): - -```bash -cd /home/jgrusewski/Work/foxhunt -cargo run --release -p ml --example hyperopt_dqn_demo --features cuda -- \ - --parquet-file test_data/ES_FUT_180d.parquet --trials 1 --epochs 5 -``` - -Look for: -- `epsilon_decay: 0.9XX` (in range [0.95, 0.99]) -- `Action distribution: BUY=X%, SELL=Y%, HOLD=Z%` (Z < 100%) - -## Next Steps - -1. **Run validation**: `./scripts/validate_epsilon_fix3.sh` -2. **Review summary**: `cat /tmp/ml_training/fix3_validation/summary_*.txt` -3. **If PASS**: Proceed to full hyperopt (50-100 trials) -4. **If FAIL**: Investigate logs, check for conflicting hyperparameters - -## Support - -For issues or questions: -1. Check raw log file for errors -2. Verify CUDA/GPU availability -3. Review hyperopt_dqn_demo.rs output format -4. Consult CLAUDE.md for DQN bug fix context diff --git a/scripts/LOCAL_CI_QUICK_REF.md b/scripts/LOCAL_CI_QUICK_REF.md deleted file mode 100644 index b59afa2ee..000000000 --- a/scripts/LOCAL_CI_QUICK_REF.md +++ /dev/null @@ -1,132 +0,0 @@ -# Local CI/CD Pipeline - Quick Reference - -**Script**: `scripts/local_ci_pipeline.sh` -**Status**: ✅ Production Ready - ---- - -## Quick Commands - -```bash -# Full pipeline (Build + Test + Push) -./scripts/local_ci_pipeline.sh - -# Test only (skip push) -./scripts/local_ci_pipeline.sh --skip-push - -# Dry-run (show commands) -./scripts/local_ci_pipeline.sh --dry-run - -# Verbose output -./scripts/local_ci_pipeline.sh --verbose --skip-push -``` - ---- - -## Pipeline Stages - -| Stage | Emoji | Duration | Description | -|-------|-------|----------|-------------| -| 0. Pre-flight | 🔍 | <5s | Docker, git, Dockerfile checks | -| 1. Build | 🔨 | 2-3 min | Build Docker image | -| 2. Test | 🧪 | 10-20s | GLIBC, CUDA, entrypoint validation | -| 3. Push | 🚀 | 1-5 min | Push to Docker Hub | - ---- - -## Test Checklist - -- ✅ GLIBC 2.35 (Ubuntu 22.04) -- ✅ CUDA 12.4.1 libraries -- ✅ cuDNN 9 libraries -- ✅ Entrypoint scripts -- ✅ Volume mount architecture - ---- - -## Expected Output - -``` -======================================== -🚀 LOCAL CI/CD PIPELINE SIMULATOR -======================================== - -🔍 STAGE 0: PRE-FLIGHT CHECKS -✓ All required commands available -✓ Docker daemon running -⏱ Pre-flight checks completed in 0m 1s - -🔨 STAGE 1: BUILD -✓ Docker image built successfully: 7.73 GB -⏱ Build completed in 0m 17s - -🧪 STAGE 2: TEST -✓ GLIBC 2.35 validated -✓ CUDA libraries validated -✓ Binary GLIBC dependencies validated -✓ Entrypoint scripts validated -⏱ Test completed in 0m 3s - -======================================== -✅ PIPELINE COMPLETE -======================================== -✓ Total pipeline time: 0m 20s -ℹ GitLab CI/CD readiness: ✅ -``` - ---- - -## Troubleshooting - -| Error | Fix | -|-------|-----| -| Docker daemon not running | `sudo systemctl start docker` | -| Docker Hub auth failed | `docker login` | -| GLIBC version mismatch | Check Dockerfile base image | -| CUDA libraries missing | Check cudnn-devel variant | - ---- - -## GitLab CI/CD Integration - -```yaml -# .gitlab-ci.yml -stages: [build, test, push] - -build: - stage: build - script: - - docker build -f Dockerfile.runpod -t jgrusewski/foxhunt:latest . - -test: - stage: test - script: - - docker run --rm --entrypoint /bin/bash jgrusewski/foxhunt:latest -c "ldd --version | grep 2.35" - - docker run --rm --entrypoint /bin/bash jgrusewski/foxhunt:latest -c "ldconfig -p | grep libcublas" - -push: - stage: push - script: - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_TOKEN - - docker push jgrusewski/foxhunt:latest -``` - ---- - -## Performance - -| Metric | Value | -|--------|-------| -| Total time | 4-9 min | -| Image size | 7.73 GB | -| Test time | 10-20s | -| Cost | $0 (local) | - ---- - -## Files - -- Script: `/scripts/local_ci_pipeline.sh` -- Guide: `/LOCAL_CI_PIPELINE_GUIDE.md` -- Validation: `/LOCAL_CI_PIPELINE_VALIDATION.md` -- Dockerfile: `/Dockerfile.runpod` diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 4096c4d04..000000000 --- a/scripts/README.md +++ /dev/null @@ -1,222 +0,0 @@ - -## RunPod Deployment Script - -**Script**: `runpod_deploy.py` - -Automated deployment script for RunPod GPU pods in EUR-IS region (SECURE cloud). - -### Features -- Scans available GPUs with ≥16GB VRAM -- Auto-selects best value GPU (RTX 4090 preferred, then cheapest) -- Supports custom GPU selection -- Dry-run mode for testing -- Automatic network volume attachment - -### Requirements -```bash -pip install requests python-dotenv -``` - -### Configuration -Create `.env.runpod` with: -``` -RUNPOD_API_KEY=your_api_key -RUNPOD_VOLUME_ID=your_volume_id -``` - -### Usage Examples - -```bash -# Auto-select best value GPU (dry run) -./scripts/runpod_deploy.py --dry-run - -# Deploy with default settings (RTX 4090 preferred) -./scripts/runpod_deploy.py - -# Deploy with specific GPU -./scripts/runpod_deploy.py --gpu-type "RTX 3090" - -# Custom image and larger disk -./scripts/runpod_deploy.py \ - --image runpod/pytorch:2.1.0-py3.10-cuda11.8.0-devel \ - --container-disk 100 - -# With custom command -./scripts/runpod_deploy.py --command "jupyter lab --allow-root" -``` - -### Default Configuration -- **Cloud Type**: SECURE (no spot interruptions) -- **Region**: EUR-IS (Iceland - low latency to Europe) -- **Image**: `runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04` -- **Container Disk**: 50GB -- **Network Volume**: Attached from `.env.runpod` -- **Ports**: 8888/http (Jupyter) - -### GPU Selection Logic -1. If `--gpu-type` specified and available → use it -2. Else if RTX 4090 available → use it (best value) -3. Else → use cheapest available GPU - -### Output -``` -✅ POD DEPLOYED SUCCESSFULLY -====================================================================== -Pod ID: abc123-xyz789 -GPU: RTX 4090 (24GB) -Cost: $0.340/hr -Image: runpod/pytorch:2.4.0 -Status: RUNNING -====================================================================== - -📝 NEXT STEPS: -1. Wait 2-3 minutes for pod to initialize -2. Access Jupyter at: https://abc123-8888.proxy.runpod.net -3. SSH access: ssh root@abc123.ssh.runpod.io -4. Monitor pod: https://www.runpod.io/console/pods -``` - -### Cost Warning -The script will display hourly costs. Remember to stop pods when done to avoid unnecessary charges. - ---- - -## Local CI/CD Pipeline Simulator - -**Script**: `local_ci_pipeline.sh` - -Simulates GitLab CI/CD pipeline locally before deployment. Tests Docker image builds and deployments in a safe, local environment. - -### Features -- 3-stage pipeline: Build → Test → Push -- GLIBC 2.35 validation (Ubuntu 22.04) -- CUDA 12.4.1 + cuDNN 9 library checks -- Entrypoint script validation -- Docker Hub push readiness -- Color-coded output with timing -- Dry-run mode for testing -- Exit on first failure (CI/CD behavior) - -### Requirements -```bash -# Docker installed and running -docker info - -# Docker Hub authentication (for push stage) -docker login -``` - -### Usage Examples - -```bash -# Full pipeline (Build + Test + Push) -./scripts/local_ci_pipeline.sh - -# Test build only (skip push) -./scripts/local_ci_pipeline.sh --skip-push - -# Dry-run (show commands without executing) -./scripts/local_ci_pipeline.sh --dry-run - -# Verbose output for debugging -./scripts/local_ci_pipeline.sh --verbose --skip-push -``` - -### Pipeline Stages - -#### Stage 0: Pre-Flight Checks (🔍) -- Docker daemon running -- Docker BuildKit available -- Docker Hub authentication -- Dockerfile exists -- Git repository status - -#### Stage 1: Build (🔨) -- Build Docker image with CUDA 12.4.1 + cuDNN 9 -- Verify image size (~4.8 GB) -- Duration: ~2-3 minutes - -#### Stage 2: Test (🧪) -- GLIBC 2.35 validation -- CUDA libraries (libcuda, libcurand, libcublas, libcudnn) -- nvidia-smi availability (optional) -- Binary GLIBC dependencies -- Entrypoint script validation -- Duration: ~10-20 seconds - -#### Stage 3: Push (🚀) -- Push image to Docker Hub -- Verify authentication -- Warn about PRIVATE repository -- Duration: ~1-5 minutes - -### Output Example - -``` -======================================== -🚀 LOCAL CI/CD PIPELINE SIMULATOR -======================================== - -ℹ Simulating GitLab CI/CD pipeline locally -ℹ Image: jgrusewski/foxhunt:latest - -======================================== -🔍 STAGE 0: PRE-FLIGHT CHECKS -======================================== -✓ All required commands available -✓ Docker daemon running -✓ Docker Hub authenticated -⏱ Pre-flight checks completed in 0m 3s - -======================================== -🔨 STAGE 1: BUILD -======================================== -✓ Docker image built successfully: 4.80 GB -⏱ Build completed in 2m 34s - -======================================== -🧪 STAGE 2: TEST -======================================== -✓ GLIBC 2.35 validated -✓ CUDA libraries validated -⏱ Test completed in 0m 18s - -======================================== -🚀 STAGE 3: PUSH -======================================== -✓ Image pushed successfully -⏱ Push completed in 3m 12s - -======================================== -✅ PIPELINE COMPLETE -======================================== -✓ Total pipeline time: 6m 7s -ℹ GitLab CI/CD readiness: ✅ -``` - -### Troubleshooting - -**Error: Docker daemon not running** -```bash -sudo systemctl start docker -docker info -``` - -**Error: Docker Hub authentication failed** -```bash -docker login -# Enter credentials for jgrusewski account -``` - -**Error: GLIBC version mismatch** -```bash -# Expected: GLIBC 2.35 (Ubuntu 22.04) -docker run --rm jgrusewski/foxhunt:latest ldd --version -``` - -### Documentation -- Full guide: `/LOCAL_CI_PIPELINE_GUIDE.md` -- Dockerfile: `/Dockerfile.runpod` -- Total time: 4-9 minutes (vs. 10-15 min on GitLab) -- Cost: $0 (vs. GitLab CI/CD minutes) - diff --git a/scripts/README_UPLOAD_BINARY.md b/scripts/README_UPLOAD_BINARY.md deleted file mode 100644 index 83c648759..000000000 --- a/scripts/README_UPLOAD_BINARY.md +++ /dev/null @@ -1,266 +0,0 @@ -# upload_binary.py - Quick Binary Upload Tool - -**Status**: ✅ PRODUCTION READY -**Purpose**: Upload Rust release binaries to RunPod S3 for GPU training -**Location**: `scripts/upload_binary.py` - ---- - -## Quick Start - -```bash -# 1. Activate virtual environment -source .venv/bin/activate - -# 2. Upload binary (auto-finds in target/release/examples/) -python3 scripts/upload_binary.py --binary-name hyperopt_mamba2_demo - -# Output: -# 🔍 Step 1: Locating binary -# ✓ Found: /home/jgrusewski/Work/foxhunt/target/release/examples/hyperopt_mamba2_demo -# -# ✅ Step 2: Validating binary -# ✓ Valid executable -# Size: 14.2 MB (14,849,744 bytes) -# -# 📝 Step 3: Generating S3 key -# ✓ S3 key: binaries/hyperopt_mamba2_demo_cuda_20251030_120534 -# -# 📤 Step 5: Uploading binary -# -# Upload Plan: -# Source: /home/.../hyperopt_mamba2_demo -# Destination: s3://se3zdnb5o4/binaries/hyperopt_mamba2_demo_cuda_20251030_120534 -# Size: 14.2 MB (14,849,744 bytes) -# Force: False -# -# ↻ Upload required: File not found in S3 -# -# Uploading hyperopt_mamba2_demo ━━━━━━━━━━ 100% • 14.2 MB • 45.3 MB/s • 0:00:00 -# -# ✅ Upload complete! -# S3 URI: s3://se3zdnb5o4/binaries/hyperopt_mamba2_demo_cuda_20251030_120534 -# -# Next steps: -# 1. Binary available at: /runpod-volume/binaries/hyperopt_mamba2_demo_cuda_20251030_120534 -# 2. Use in deployment: --command '/runpod-volume/binaries/hyperopt_mamba2_demo_cuda_20251030_120534 --epochs 50' -# 3. Verify: aws s3 ls s3://se3zdnb5o4/binaries/hyperopt_mamba2_demo_cuda_20251030_120534 --profile runpod -``` - ---- - -## Common Commands - -```bash -# Upload with auto-find (default) -python3 scripts/upload_binary.py --binary-name hyperopt_mamba2_demo - -# Force overwrite (skip MD5 check) -python3 scripts/upload_binary.py --binary-name hyperopt_tft_demo --force - -# Static name (no timestamp, overwrites) -python3 scripts/upload_binary.py --binary-name hyperopt_dqn_demo --no-timestamp - -# Dry run (validate only, no upload) -python3 scripts/upload_binary.py --binary-name hyperopt_ppo_demo --dry-run - -# Direct path -python3 scripts/upload_binary.py --binary-path ./target/release/examples/custom_binary -``` - ---- - -## Features - -### ✅ Automatic Binary Location -- Searches `target/release/examples/` by name -- Handles build hashes (e.g., `-84b145a77f64618b`) -- Selects most recent if multiple matches - -### ✅ Smart Upload -- **MD5 Checksum**: Skips upload if file unchanged -- **Progress Bar**: Shows transfer speed and ETA -- **Force Mode**: `--force` to always upload - -### ✅ Flexible Naming -- **Timestamped** (default): `hyperopt_mamba2_demo_cuda_20251030_120534` -- **Static** (`--no-timestamp`): `hyperopt_mamba2_demo_cuda` -- **Plain** (`--no-timestamp --no-cuda`): `hyperopt_mamba2_demo` - -### ✅ Validation -- Checks file exists and is executable -- Validates size (warns if < 100KB) -- Returns S3 path for deployment - ---- - -## Options - -| Flag | Description | Example | -|------|-------------|---------| -| `--binary-name NAME` | Auto-find binary | `hyperopt_mamba2_demo` | -| `--binary-path PATH` | Direct path | `./custom_binary` | -| `--force` | Skip checksum, force upload | `--force` | -| `--no-timestamp` | Static name (overwrites) | `--no-timestamp` | -| `--no-cuda` | Omit `_cuda` suffix | `--no-cuda` | -| `--dry-run` | Validate only, no upload | `--dry-run` | - ---- - -## Requirements - -### 1. Virtual Environment -```bash -source .venv/bin/activate -``` - -### 2. Dependencies -```bash -pip install -r foxhunt_runpod/foxhunt_runpod/requirements.txt -``` - -Includes: -- `boto3` - S3 uploads -- `rich` - Progress bars -- `pydantic-settings` - Config validation -- `python-dotenv` - .env loading - -### 3. Configuration (.env.runpod) -```bash -RUNPOD_S3_ACCESS_KEY= -RUNPOD_S3_SECRET= -RUNPOD_VOLUME_ID=se3zdnb5o4 -RUNPOD_S3_ENDPOINT=https://s3api-eur-is-1.runpod.io -RUNPOD_S3_REGION=eur-is-1 -``` - ---- - -## Workflow: Build → Upload → Deploy - -```bash -# Step 1: Build binary -cargo build --release --example hyperopt_mamba2_demo --features cuda - -# Step 2: Upload to S3 -python3 scripts/upload_binary.py --binary-name hyperopt_mamba2_demo -# Returns: /runpod-volume/binaries/hyperopt_mamba2_demo_cuda_20251030_120534 - -# Step 3: Deploy to RunPod -python3 scripts/runpod_deploy.py \ - --gpu-type "RTX A4000" \ - --command "/runpod-volume/binaries/hyperopt_mamba2_demo_cuda_20251030_120534 \ - --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet \ - --trials 50 \ - --timeout 2h" -``` - ---- - -## S3 Organization - -``` -s3://se3zdnb5o4/binaries/ -├── hyperopt_mamba2_demo_cuda_20251030_120000 -├── hyperopt_mamba2_demo_cuda_20251030_143000 (versioned) -├── hyperopt_tft_demo_cuda_20251030_150000 -├── hyperopt_dqn_demo_cuda_20251030_163000 -└── hyperopt_ppo_demo_cuda_20251030_173000 -``` - -### Runpod Volume Mount -All binaries available at: `/runpod-volume/binaries/` - ---- - -## Performance - -| Binary | Size | Upload Time (50 Mbps) | MD5 Check | -|--------|------|----------------------|-----------| -| hyperopt_mamba2_demo | 14.2 MB | ~2.3s | 0s (skip) | -| hyperopt_tft_demo | 21.1 MB | ~3.4s | 0s (skip) | -| hyperopt_dqn_demo | 13.3 MB | ~2.1s | 0s (skip) | -| hyperopt_ppo_demo | 13.0 MB | ~2.1s | 0s (skip) | - -**MD5 Optimization**: If binary unchanged, upload skipped (saves bandwidth) - ---- - -## Troubleshooting - -### "Binary not found" -```bash -# Build first -cargo build --release --example hyperopt_mamba2_demo --features cuda - -# Verify -ls -lh target/release/examples/hyperopt_mamba2_demo -``` - -### "Not running in virtual environment" -```bash -source .venv/bin/activate -``` - -### "Configuration error" -```bash -# Check .env.runpod -cat .env.runpod | grep RUNPOD_S3 - -# Test S3 access -aws s3 ls s3://se3zdnb5o4/binaries/ \ - --profile runpod \ - --endpoint-url https://s3api-eur-is-1.runpod.io -``` - ---- - -## Advanced Usage - -### Upload All Hyperopt Binaries -```bash -for binary in hyperopt_mamba2_demo hyperopt_tft_demo hyperopt_dqn_demo hyperopt_ppo_demo; do - python3 scripts/upload_binary.py --binary-name $binary -done -``` - -### Static Name for Production -```bash -# Always uploads to same path (simplifies deployment) -python3 scripts/upload_binary.py \ - --binary-name hyperopt_mamba2_demo \ - --no-timestamp \ - --force - -# Deploy script can use fixed path ---command "/runpod-volume/binaries/hyperopt_mamba2_demo_cuda" -``` - ---- - -## Documentation - -- **BINARY_UPLOAD_QUICK_REF.md** - Quick reference guide -- **UPLOAD_BINARY_IMPLEMENTATION.md** - Full implementation details -- **RUNPOD_VOLUME_MOUNT_ARCHITECTURE.md** - S3 volume system -- **RUNPOD_DEPLOY_SCRIPT_UPDATE.md** - Deployment workflow - ---- - -## Related Scripts - -- **runpod_deploy.py** - Deploy pods with uploaded binaries -- **monitor_hyperopt.sh** - Monitor training progress -- **check_hyperopt_status.sh** - Check S3 results - ---- - -## Version - -**v1.0.0** (2025-10-30) -- ✅ Auto-locate binaries -- ✅ MD5 checksum validation -- ✅ Progress tracking -- ✅ Timestamped naming -- ✅ Dry run mode -- ✅ foxhunt_runpod integration diff --git a/scripts/README_scan_gpus.md b/scripts/README_scan_gpus.md deleted file mode 100644 index 482555d42..000000000 --- a/scripts/README_scan_gpus.md +++ /dev/null @@ -1,75 +0,0 @@ -# RunPod GPU Scanner - -## Purpose -Query RunPod API to show available SECURE cloud GPUs in the EUR-IS region with ≥16GB VRAM. - -## Requirements -- Python 3.7+ -- python-dotenv (`pip3 install python-dotenv`) -- requests (`pip3 install requests`) -- Valid RunPod API key in `/home/jgrusewski/Work/foxhunt/.env.runpod` - -## Usage - -### Run the script: -```bash -# From foxhunt root directory -./scripts/scan_gpus.py - -# Or with python3 -python3 scripts/scan_gpus.py -``` - -## Output -The script displays: -- GPU name (e.g., RTX 4090, A100 SXM) -- VRAM capacity (GB) -- Price per hour (USD) -- Availability (number of pods) - -Results are sorted by price (cheapest first). - -## Filtering Criteria -- Memory ≥16GB VRAM -- Secure Cloud availability > 0 -- Valid pricing information -- EUR-IS region - -## Example Output -``` -====================================================================== -RUNPOD SECURE CLOUD GPUs (≥16GB VRAM) - EUR-IS REGION -====================================================================== -GPU Name VRAM Price/hr Available ----------------------------------------------------------------------- -RTX A5000 24GB $0.160 True pods -RTX A4000 16GB $0.170 True pods -RTX 4090 24GB $0.340 True pods -... -====================================================================== -Total GPUs found: 24 -``` - -## Top Recommendations for Foxhunt ML Training - -Based on the current scan results (October 2025): - -### Budget Option (FP32 Models) -- **RTX 4090**: $0.34/hr, 24GB VRAM - - Best value for FP32 training (TFT-FP32 fits in ~500MB) - - Ideal for initial deployment and baseline metrics - -### Professional Option (QAT Models) -- **RTX A6000**: $0.33/hr, 48GB VRAM - - Best for QAT training (requires gradient checkpointing) - - Can run multiple models concurrently - -### Enterprise Option (Multi-Model Inference) -- **A100 PCIe/SXM**: $1.19-1.39/hr, 80GB VRAM - - Production-grade for ensemble inference - - Supports 4+ models with headroom - -## Notes -- Prices and availability fluctuate based on demand -- Run this script regularly to find the best deals -- Consider spot instances for training (not shown in this script) diff --git a/scripts/README_test_tli_tuning.md b/scripts/README_test_tli_tuning.md deleted file mode 100644 index 2e682b984..000000000 --- a/scripts/README_test_tli_tuning.md +++ /dev/null @@ -1,599 +0,0 @@ -# TLI Tuning Workflow Test Script - -**Script**: `/home/jgrusewski/Work/foxhunt/scripts/test_tli_tuning.sh` -**Purpose**: End-to-end testing of the TLI hyperparameter tuning workflow -**Version**: 1.0.0 -**Last Updated**: 2025-10-13 - ---- - -## Overview - -This script provides comprehensive testing of the TLI hyperparameter tuning functionality, including: -- Prerequisite validation (services, authentication, data) -- Job submission and tracking -- Status polling with real-time progress -- Best parameter retrieval and export -- Graceful error handling and cleanup - -## Features - -### 1. Prerequisite Checks ✅ -- TLI binary existence and location -- JWT token validation (with masking for security) -- API Gateway health (HTTP + gRPC) -- ML Training Service availability -- Config file creation (auto-generates if missing) -- Test data validation with size reporting -- Optional tool detection (grpcurl, jq) - -### 2. Three Operating Modes - -#### Full Workflow (Default) -```bash -./scripts/test_tli_tuning.sh -``` -**Steps**: -1. Check prerequisites -2. Start tuning job (DQN, 5 trials) -3. Poll status every 5 seconds -4. Display real-time progress with timestamps -5. Retrieve best parameters when complete -6. Export results to YAML file - -**Duration**: ~5-10 minutes (depending on trials) - -#### Quick Test Mode -```bash -./scripts/test_tli_tuning.sh --quick -``` -**Steps**: -1. Check prerequisites -2. Start tuning job -3. Display job ID and monitoring commands -4. Exit immediately (no polling) - -**Duration**: ~10 seconds -**Use Case**: Verify job submission works without waiting - -#### Check-Only Mode -```bash -./scripts/test_tli_tuning.sh --check-only -``` -**Steps**: -1. Validate all prerequisites -2. Report status of services/files -3. Exit with detailed diagnostics - -**Duration**: ~5 seconds -**Use Case**: Pre-flight checks before running tests - -### 3. Customization Options - -```bash -# Custom model type -./scripts/test_tli_tuning.sh --model PPO --trials 10 - -# Environment variable overrides -export POLL_INTERVAL=10 # Poll every 10 seconds -export MAX_WAIT_TIME=600 # Wait up to 10 minutes -export TLI_BIN=/custom/path/tli # Custom TLI location -./scripts/test_tli_tuning.sh -``` - -### 4. Error Handling - -- **Automatic cleanup**: Stops jobs on script failure/interrupt -- **Exit traps**: SIGINT, SIGTERM handled gracefully -- **Detailed diagnostics**: Clear error messages with remediation steps -- **Progress tracking**: Job IDs saved to `~/.foxhunt/test_tuning_job.txt` - -### 5. Output Features - -- **Color-coded status**: ✅ Green (success), ❌ Red (error), ⚠️ Yellow (warning) -- **Real-time progress**: Timestamps, trial progress, elapsed time -- **Progress visualization**: Text-based progress bar -- **Duration tracking**: Total test duration reported -- **Masked credentials**: JWT tokens masked for security - ---- - -## Prerequisites - -### 1. Build TLI Binary -```bash -cargo build --release -p tli -# Binary: target/release/tli -``` - -### 2. Authenticate with TLI -```bash -./target/release/tli auth login --username --password -# Creates: ~/.foxhunt/jwt_token -``` - -### 3. Start Infrastructure Services -```bash -docker-compose up -d -# Services: postgres, redis, vault, influxdb -``` - -### 4. Start API Gateway -```bash -cargo run --release -p api_gateway -# Ports: 50051 (gRPC), 8080 (HTTP health) -``` - -### 5. Start ML Training Service -```bash -cargo run --release -p ml_training_service -# Ports: 50054 (gRPC), 8095 (HTTP health) -``` - -### 6. Verify Health -```bash -./scripts/comprehensive_health_check.sh -# Should show all services healthy -``` - ---- - -## Usage Examples - -### Example 1: Basic Full Workflow Test -```bash -$ ./scripts/test_tli_tuning.sh - -================================================================================ - Prerequisite Checks -================================================================================ - -[STEP] Checking TLI binary... -✓ TLI binary found -ℹ Location: /home/jgrusewski/Work/foxhunt/target/release/tli - -[STEP] Checking JWT token... -✓ JWT token found -ℹ Token (masked): eyJhbGciOiJIUzI1NiIs...WXZ6aGJHVnU= - -[STEP] Checking API Gateway health... -✓ API Gateway is healthy - -[STEP] Checking ML Training Service health... -✓ ML Training Service is healthy - -[STEP] Checking tuning config file... -✓ Config file found -ℹ Path: /home/jgrusewski/Work/foxhunt/test_data/tuning_config.yaml - -[STEP] Checking test data... -✓ Test data found -ℹ Path: /home/jgrusewski/Work/foxhunt/test_data/btcusdt_sample_100.parquet -ℹ Size: 1.2M - -✓ All prerequisites satisfied - -================================================================================ - Starting Tuning Job -================================================================================ - -[STEP] Submitting tuning job to TLI... -ℹ Command: ./target/release/tli tune start --model DQN --trials 5 --config test_data/tuning_config.yaml - -🚀 Starting hyperparameter tuning job... - Model: DQN - Trials: 5 - Config: test_data/tuning_config.yaml - GPU: ❌ Disabled - -✅ Tuning job started successfully! - Job ID: 550e8400-e29b-41d4-a716-446655440000 - Saved to ~/.foxhunt/tuning_jobs.json - -================================================================================ - Polling Job Status -================================================================================ - -ℹ Polling every 5s (max wait: 5m) - -[14:30:05] Status: RUNNING | Progress: 0/5 (0.0%) | Elapsed: 0s -[14:30:10] Status: RUNNING | Progress: 1/5 (20.0%) | Elapsed: 5s -[14:30:15] Status: RUNNING | Progress: 2/5 (40.0%) | Elapsed: 10s -[14:30:20] Status: RUNNING | Progress: 3/5 (60.0%) | Elapsed: 15s -[14:30:25] Status: RUNNING | Progress: 4/5 (80.0%) | Elapsed: 20s -[14:30:30] Status: RUNNING | Progress: 5/5 (100.0%) | Elapsed: 25s - -✅ Job completed successfully! - -📊 Tuning Job Status - Status: TUNING_COMPLETED - Progress: 5/5 trials (100.0%) - [██████████████████████████████████████████████████] 100.0% - -🏆 Best Results So Far - Sharpe Ratio: 2.3456 - Elapsed Time: 30 seconds - -================================================================================ - Retrieving Best Parameters -================================================================================ - -[STEP] Fetching best hyperparameters... - -🏆 Best Performance Metrics - sharpe_ratio: 2.3456 - total_return: 15.6789 - max_drawdown: 8.4321 - -📋 Best Hyperparameters -┌──────────────────┬──────────┬───────────────┐ -│ Parameter │ Value │ Type │ -├──────────────────┼──────────┼───────────────┤ -│ learning_rate │ 0.000512 │ Learning Rate │ -│ batch_size │ 64.000000│ Integer │ -│ gamma │ 0.976543 │ Float │ -│ epsilon_decay │ 0.995678 │ Float │ -└──────────────────┴──────────┴───────────────┘ - -[STEP] Exporting best parameters to file... -✅ Parameters exported to: best_params_550e8400.yaml - -💡 Use these parameters in your training configuration. - -================================================================================ - Test Completed Successfully -================================================================================ - -✅ Full workflow executed without errors -ℹ Total duration: 35s -``` - -### Example 2: Quick Test (No Waiting) -```bash -$ ./scripts/test_tli_tuning.sh --quick - -================================================================================ - Quick TLI Tuning Test (Start Only) -================================================================================ - -[Prerequisites checks...] -✅ All prerequisites satisfied - -================================================================================ - Starting Tuning Job -================================================================================ - -✅ Tuning job started successfully! - Job ID: 550e8400-e29b-41d4-a716-446655440000 - -✅ Job started successfully -ℹ Monitor with: ./target/release/tli tune status --job-id 550e8400-e29b-41d4-a716-446655440000 -ℹ Get results: ./target/release/tli tune best --job-id 550e8400-e29b-41d4-a716-446655440000 -ℹ Stop job: ./target/release/tli tune stop --job-id 550e8400-e29b-41d4-a716-446655440000 -ℹ Total test duration: 8s -``` - -### Example 3: Custom Model and Trials -```bash -$ ./scripts/test_tli_tuning.sh --model MAMBA_2 --trials 10 - -[Prerequisites checks...] -✅ All prerequisites satisfied - -🚀 Starting hyperparameter tuning job... - Model: MAMBA_2 - Trials: 10 - Config: test_data/tuning_config.yaml - GPU: ❌ Disabled - -[Polling progress 0-100%...] -``` - -### Example 4: Check Prerequisites Only -```bash -$ ./scripts/test_tli_tuning.sh --check-only - -================================================================================ - Prerequisite Checks -================================================================================ - -[STEP] Checking TLI binary... -✓ TLI binary found -ℹ Location: /home/jgrusewski/Work/foxhunt/target/release/tli - -[STEP] Checking JWT token... -✗ JWT token not found at: /home/jgrusewski/.foxhunt/jwt_token -ℹ Authenticate with: ./target/release/tli auth login - -[STEP] Checking API Gateway health... -✗ API Gateway not responding at http://localhost:8080/health -ℹ Start services with: docker-compose up -d - -[STEP] Checking ML Training Service health... -✓ ML Training Service is healthy - -✗ Prerequisites check FAILED -ℹ Please resolve issues above and retry -``` - ---- - -## Configuration - -### Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `TLI_BIN` | `target/release/tli` | Path to TLI binary | -| `JWT_TOKEN_FILE` | `~/.foxhunt/jwt_token` | JWT token location | -| `POLL_INTERVAL` | `5` | Status polling interval (seconds) | -| `MAX_WAIT_TIME` | `300` | Maximum wait time (seconds) | - -### Generated Files - -| File | Location | Purpose | -|------|----------|---------| -| JWT Token | `~/.foxhunt/jwt_token` | Authentication | -| Tuning Jobs | `~/.foxhunt/tuning_jobs.json` | Job tracking | -| Test Job ID | `~/.foxhunt/test_tuning_job.txt` | Current test job | -| Minimal Config | `test_data/tuning_config.yaml` | Auto-generated config | -| Best Params | `best_params_.yaml` | Exported parameters | - -### Minimal Config Structure - -The script auto-generates a minimal config if none exists: - -```yaml -# test_data/tuning_config.yaml -tuning: - study_name: "test_study" - storage: "sqlite:///optuna_test.db" - direction: "maximize" # Maximize Sharpe ratio - - search_space: - learning_rate: - type: "float" - low: 0.0001 - high: 0.01 - log: true - - batch_size: - type: "int" - low: 32 - high: 128 - step: 32 - - gamma: - type: "float" - low: 0.90 - high: 0.99 - - epsilon_decay: - type: "float" - low: 0.990 - high: 0.999 - - training: - epochs: 10 - validation_split: 0.2 - early_stopping_patience: 3 - - metrics: - - "sharpe_ratio" - - "total_return" - - "max_drawdown" - - "win_rate" -``` - ---- - -## Troubleshooting - -### Issue: TLI Binary Not Found - -**Error**: -``` -✗ TLI binary not found at: target/release/tli -ℹ Build with: cargo build --release -p tli -``` - -**Solution**: -```bash -cargo build --release -p tli -``` - ---- - -### Issue: JWT Token Missing - -**Error**: -``` -✗ JWT token not found at: ~/.foxhunt/jwt_token -ℹ Authenticate with: ./target/release/tli auth login -``` - -**Solution**: -```bash -./target/release/tli auth login --username admin --password admin123 -``` - -**Note**: Default dev credentials from `docker-compose.yml` - ---- - -### Issue: API Gateway Not Running - -**Error**: -``` -✗ API Gateway not responding at http://localhost:8080/health -ℹ Start services with: docker-compose up -d -``` - -**Solution**: -```bash -# Start infrastructure -docker-compose up -d - -# Start API Gateway -cargo run --release -p api_gateway -``` - ---- - -### Issue: ML Training Service Unavailable - -**Error**: -``` -✗ ML Training Service not responding at http://localhost:8095/health -``` - -**Solution**: -```bash -# Check service status -curl http://localhost:8095/health - -# Start if not running -cargo run --release -p ml_training_service - -# Check logs -docker-compose logs ml_training_service -``` - ---- - -### Issue: Job Times Out - -**Symptom**: Job exceeds MAX_WAIT_TIME (300s default) - -**Solution**: -```bash -# Increase max wait time -export MAX_WAIT_TIME=600 # 10 minutes -./scripts/test_tli_tuning.sh - -# Or reduce trials for faster completion -./scripts/test_tli_tuning.sh --trials 3 -``` - ---- - -### Issue: Config File Not Found - -**Symptom**: Warning about missing config file - -**Behavior**: Script auto-creates minimal config at `test_data/tuning_config.yaml` - -**Verification**: -```bash -cat test_data/tuning_config.yaml -``` - ---- - -## Integration with CI/CD - -### GitHub Actions Example -```yaml -name: Test TLI Tuning -on: [push, pull_request] - -jobs: - test-tli-tuning: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Start infrastructure - run: docker-compose up -d - - - name: Build TLI - run: cargo build --release -p tli - - - name: Start services - run: | - cargo run --release -p api_gateway & - cargo run --release -p ml_training_service & - sleep 10 - - - name: Authenticate TLI - run: | - ./target/release/tli auth login \ - --username admin \ - --password ${{ secrets.TLI_PASSWORD }} - - - name: Run quick test - run: ./scripts/test_tli_tuning.sh --quick - - - name: Check prerequisites - run: ./scripts/test_tli_tuning.sh --check-only -``` - ---- - -## Performance Characteristics - -| Operation | Duration | Notes | -|-----------|----------|-------| -| Prerequisites check | ~5s | Validates 10+ conditions | -| Job submission | ~1-2s | gRPC call to API Gateway | -| Status polling | ~5s/poll | Configurable via `POLL_INTERVAL` | -| Full workflow (5 trials) | ~5-10min | Depends on model complexity | -| Quick test | ~10s | Start job only | - ---- - -## Security Considerations - -1. **JWT Token Masking**: Tokens displayed as `first20...last10` characters -2. **Token Storage**: `~/.foxhunt/jwt_token` (chmod 600 recommended) -3. **Cleanup on Exit**: Jobs stopped automatically on script failure -4. **No Hardcoded Credentials**: All auth via TLI login flow -5. **HTTPS Support**: Change `API_GATEWAY_URL` for production - ---- - -## Future Enhancements - -### Planned Features -1. **Real-time Streaming**: Replace polling with server-side streaming gRPC -2. **Multi-job Testing**: Test multiple concurrent tuning jobs -3. **Performance Metrics**: Track latency, throughput, success rates -4. **GPU Testing**: Add `--gpu` flag validation -5. **Data Validation**: Verify parquet file structure before submission - -### Configuration Improvements -1. **Custom Metrics**: Support custom objective functions -2. **Pruning Strategies**: Test Optuna pruning algorithms -3. **Multi-model Tests**: Iterate over all supported models -4. **Resource Limits**: Memory/CPU constraints testing - ---- - -## Related Documentation - -- **TLI Tune Command**: `/home/jgrusewski/Work/foxhunt/tli/src/commands/tune.rs` -- **API Gateway Proxy**: `/home/jgrusewski/Work/foxhunt/services/api_gateway/src/proxy_handlers.rs` -- **ML Training Service**: `/home/jgrusewski/Work/foxhunt/services/ml_training_service/` -- **CLAUDE.md**: Architecture and development guide -- **TESTING_PLAN.md**: ML testing strategy - ---- - -## Version History - -### 1.0.0 (2025-10-13) -- Initial release -- Full workflow testing (start → poll → results) -- Three operating modes (full, quick, check-only) -- Comprehensive error handling -- Auto-config generation -- JWT token masking -- Real-time progress tracking -- Export best parameters to YAML - ---- - -## License - -Part of the Foxhunt HFT Trading System -Copyright (c) 2025 diff --git a/scripts/README_validate_training.md b/scripts/README_validate_training.md deleted file mode 100644 index a34c3c0dc..000000000 --- a/scripts/README_validate_training.md +++ /dev/null @@ -1,359 +0,0 @@ -# ML Training Validation Script - -**Script**: `validate_training.sh` -**Wave**: 152 Agent 20 -**Dependencies**: Agent 19 (`train_all_models_fixed.sh`) - -## Purpose - -Validates all 4 ML training pipelines by running a quick 2-epoch training session for each model and verifying output files are generated correctly. - -## Models Tested - -1. **DQN** (Deep Q-Network) - Reinforcement learning -2. **PPO** (Proximal Policy Optimization) - RL policy gradient -3. **MAMBA-2** - State space model for sequence prediction -4. **TFT** (Temporal Fusion Transformer) - Multi-horizon forecasting - -## Prerequisites - -### 1. Data Files Required - -The script expects 3-month historical data (downloaded by Agent 19): - -``` -test_data/real/BTC-USD_20231001-20231231_databento_ohlcv-1s.parquet -test_data/real/ETH-USD_20231001-20231231_databento_ohlcv-1s.parquet -``` - -If data is missing, run Agent 19 first: -```bash -./scripts/train_all_models_fixed.sh -``` - -### 2. System Requirements - -- **Cargo**: Rust toolchain -- **Disk Space**: ~500MB for models + logs -- **Memory**: 8GB+ recommended (GPU optional but recommended) -- **Time**: ~10-30 minutes (depends on hardware) - -## Usage - -### Quick Run - -```bash -cd /home/jgrusewski/Work/foxhunt -./scripts/validate_training.sh -``` - -### Expected Output - -``` -======================================== -ML Training Validation Script -Wave 152 Agent 20 -======================================== - -Configuration: - Epochs: 2 - Data: test_data/real/ - Output: test_data/models/ - Models: DQN PPO MAMBA TFT - -Checking prerequisites... -✓ Data files found -✓ Cargo available - -======================================== -Training Phase -======================================== - -Training DQN (2 epochs)... -✓ DQN training completed (120s) - -Training PPO (2 epochs)... -✓ PPO training completed (95s) - -Training MAMBA (2 epochs)... -✓ MAMBA training completed (180s) - -Training TFT (2 epochs)... -✓ TFT training completed (140s) - -======================================== -Validation Phase -======================================== - -✓ DQN model saved: 15M -✓ PPO model saved: 18M -✓ MAMBA model saved: 42M -✓ TFT model saved: 28M - -======================================== -Summary -======================================== - -Training Results: - ✓ DQN: SUCCESS (120s) - ✓ PPO: SUCCESS (95s) - ✓ MAMBA: SUCCESS (180s) - ✓ TFT: SUCCESS (140s) - -Validation Results: - Success: 4/4 models - Failed: 0/4 models - -======================================== -✓ ALL TESTS PASSED -======================================== - -All 4 models trained successfully and saved .safetensors files - -Model files: - - test_data/models/dqn_20251014_011545.safetensors - - test_data/models/ppo_20251014_011547.safetensors - - test_data/models/mamba_20251014_011552.safetensors - - test_data/models/tft_20251014_011555.safetensors -``` - -## Exit Codes - -- **0**: All 4 models trained successfully and saved .safetensors files -- **1**: One or more models failed to train or save output files - -## Output Files - -### Model Files -``` -test_data/models/ -├── dqn_TIMESTAMP.safetensors # DQN model weights -├── ppo_TIMESTAMP.safetensors # PPO model weights -├── mamba_TIMESTAMP.safetensors # MAMBA-2 model weights -└── tft_TIMESTAMP.safetensors # TFT model weights -``` - -### Log Files -``` -test_data/models/ -├── DQN_TIMESTAMP.log # DQN training logs -├── PPO_TIMESTAMP.log # PPO training logs -├── MAMBA_TIMESTAMP.log # MAMBA-2 training logs -└── TFT_TIMESTAMP.log # TFT training logs -``` - -## Configuration - -The script uses the following parameters (hardcoded for quick validation): - -```bash -EPOCHS=2 # Quick validation with 2 epochs -BATCH_SIZE=32 # Standard batch size -LEARNING_RATE=0.001 # Standard learning rate -``` - -To modify for longer training, edit the script: -```bash -# Change EPOCHS at line 18 -EPOCHS=10 # Train for 10 epochs instead -``` - -## Troubleshooting - -### Error: "BTC data not found" - -**Solution**: Run Agent 19 first to download data: -```bash -./scripts/train_all_models_fixed.sh -``` - -### Error: "cargo not found" - -**Solution**: Install Rust toolchain: -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source $HOME/.cargo/env -``` - -### Error: "Model training failed" - -**Solution**: Check the model-specific log file: -```bash -cat test_data/models/DQN_TIMESTAMP.log # Replace with actual timestamp -``` - -Common issues: -- Out of memory: Reduce batch size or use GPU -- CUDA errors: Check GPU availability with `nvidia-smi` -- Data format issues: Re-download data with Agent 19 - -### Training Too Slow - -**GPU Acceleration**: If you have NVIDIA GPU: -```bash -# Check CUDA availability -nvidia-smi - -# Verify CUDA environment -echo $CUDA_HOME -echo $LD_LIBRARY_PATH - -# Rebuild with GPU support -cargo build --release --features cuda -``` - -**CPU Performance**: For CPU-only systems: -- Reduce batch size: Edit script, change `--batch-size 32` to `--batch-size 16` -- Use fewer epochs: Change `EPOCHS=2` to `EPOCHS=1` -- Close other applications to free memory - -## Integration with CI/CD - -### GitHub Actions - -```yaml -name: ML Training Validation - -on: - push: - branches: [main, develop] - pull_request: - branches: [main] - -jobs: - validate-training: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Download test data - run: ./scripts/train_all_models_fixed.sh - - - name: Validate training - run: ./scripts/validate_training.sh - - - name: Upload model artifacts - if: always() - uses: actions/upload-artifact@v3 - with: - name: trained-models - path: test_data/models/*.safetensors -``` - -### GitLab CI - -```yaml -ml-training-validation: - stage: test - script: - - ./scripts/train_all_models_fixed.sh # Download data - - ./scripts/validate_training.sh # Validate training - artifacts: - paths: - - test_data/models/*.safetensors - expire_in: 1 week - only: - - main - - develop -``` - -## Performance Benchmarks - -Typical execution times on different hardware: - -| Hardware | Total Time | DQN | PPO | MAMBA | TFT | -|----------|------------|-----|-----|-------|-----| -| RTX 3090 (GPU) | 8-12 min | 2 min | 1.5 min | 3 min | 2.5 min | -| RTX 3050 Ti (GPU) | 12-18 min | 3 min | 2 min | 5 min | 4 min | -| AMD Ryzen 9 (CPU) | 25-35 min | 6 min | 5 min | 10 min | 8 min | -| Intel i7 (CPU) | 35-50 min | 8 min | 7 min | 15 min | 12 min | - -## Related Scripts - -- **Agent 19**: `train_all_models_fixed.sh` - Full 3-month training (prerequisite) -- **Agent 18**: `train_all_models_full.sh` - Original full training script -- **Agent 17**: `test_dqn_training.sh` - DQN-specific validation - -## Success Criteria - -The script passes if: - -1. ✅ All 4 models train without errors -2. ✅ All 4 models save `.safetensors` files -3. ✅ Model files are non-empty (>1MB each) -4. ✅ No compilation errors -5. ✅ Exit code 0 returned - -The script fails if: - -1. ❌ Any model training crashes -2. ❌ Any model fails to save output -3. ❌ Data files missing -4. ❌ Compilation errors -5. ❌ Exit code 1 returned - -## Architecture Notes - -### Model Types - -The script trains one instance of each model architecture: - -``` -DQN (dqn) → Deep Q-Network for discrete action spaces -PPO (ppo) → Proximal Policy Optimization for continuous control -MAMBA (mamba) → MAMBA-2 state space model for sequences -TFT (tft) → Temporal Fusion Transformer for multi-horizon forecasting -``` - -### Training Pipeline - -``` -1. Load Parquet data → 2. Feature engineering → 3. Train model → 4. Save weights -``` - -Each model uses: -- **Input**: 3-month BTC/USD OHLCV-1s data (7.8M candles) -- **Epochs**: 2 (quick validation) -- **Batch Size**: 32 -- **Learning Rate**: 0.001 -- **Output**: `.safetensors` format (safe serialization) - -### CLI Interface - -The script uses the ML training CLI binary: - -```bash -cargo run --release --bin ml_training_cli -- train-model \ - --model-type {dqn|ppo|mamba|tft} \ - --data-path test_data/real/BTC-USD_20231001-20231231_databento_ohlcv-1s.parquet \ - --output-path test_data/models/MODEL_TIMESTAMP \ - --epochs 2 \ - --batch-size 32 \ - --learning-rate 0.001 -``` - -## Future Enhancements - -Potential improvements for future waves: - -1. **Parallel Training**: Train models concurrently (requires 4x memory) -2. **Multi-Asset**: Test with ETH data alongside BTC -3. **Metrics Collection**: Track loss curves, gradients, convergence -4. **Model Comparison**: Compare accuracy across architectures -5. **Hyperparameter Sweep**: Test different learning rates, batch sizes -6. **Checkpointing**: Validate checkpoint save/resume functionality -7. **Distributed Training**: Test multi-GPU training pipelines - -## License - -Part of the Foxhunt HFT Trading System - Internal Use Only - -## Changelog - -- **Wave 152 Agent 20**: Initial creation - - Trains all 4 models (DQN, PPO, MAMBA, TFT) - - 2 epochs each for quick validation - - Verifies .safetensors output files - - Exit 0/1 based on success/failure - - Full logging and summary report diff --git a/scripts/RUNPOD_DEPLOYMENT_SCRIPTS.md b/scripts/RUNPOD_DEPLOYMENT_SCRIPTS.md deleted file mode 100644 index 03d7ac839..000000000 --- a/scripts/RUNPOD_DEPLOYMENT_SCRIPTS.md +++ /dev/null @@ -1,613 +0,0 @@ -# Runpod Deployment Scripts - -This directory contains scripts for deploying Foxhunt ML models to Runpod GPU infrastructure. - -## Quick Start - -```bash -# 1. Set environment variables -export RUNPOD_S3_ENDPOINT=https://s3api-us-ca-1.runpod.io # Your datacenter -export AWS_PROFILE=runpod -export DOCKER_USERNAME=jgrusewski - -# 2. Configure AWS CLI for Runpod -aws configure --profile runpod -# AWS Access Key ID: -# AWS Secret Access Key: -# Default region name: us-east-1 -# Default output format: json - -# 3. Test prerequisites (dry run) -./scripts/runpod_deploy_test.sh - -# 4. Deploy to Runpod -./scripts/runpod_deploy.sh -``` - ---- - -## Scripts Overview - -### `runpod_deploy.sh` - Master Deployment Script - -**Purpose**: Orchestrates the complete Runpod deployment workflow - -**What it does**: -1. ✅ Validates prerequisites (cargo, docker, aws cli, credentials) -2. 🔨 Builds release binaries (5-6 minutes) -3. ☁️ Uploads binaries to Runpod S3 volume (~500 MB) -4. 📊 Uploads test data to Runpod S3 volume (~13 MB) -5. 🔐 Prompts for .env upload (optional, secure confirmation) -6. 🐳 Builds Docker image (~2-3 minutes) -7. 📤 Pushes to Docker Hub (3-5 minutes) -8. 📋 Prints deployment instructions - -**Duration**: ~15-20 minutes total - -**Idempotent**: Yes (safe to re-run, skips already-uploaded files) - -**Usage**: -```bash -export RUNPOD_S3_ENDPOINT=https://s3api-us-ca-1.runpod.io -export AWS_PROFILE=runpod -export DOCKER_USERNAME=jgrusewski -export S3_BUCKET=your-network-volume-id # Optional (default: foxhunt-runpod) - -./scripts/runpod_deploy.sh -``` - -**Output**: -- Uploaded binaries: `s3:///binaries/` (4 files, ~500 MB) -- Uploaded data: `s3:///test_data/` (9 files, ~13 MB) -- Docker image: `jgrusewski/foxhunt:latest` (~2 GB) -- Deployment instructions printed to console - ---- - -### `runpod_deploy_test.sh` - Dry Run Test - -**Purpose**: Validates prerequisites without deploying - -**What it tests**: -1. ✅ Cargo (Rust toolchain) -2. ✅ Docker (daemon running) -3. ✅ AWS CLI (for S3 uploads) -4. ✅ Environment variables (RUNPOD_S3_ENDPOINT, AWS_PROFILE, DOCKER_USERNAME) -5. ✅ AWS profile configuration -6. ✅ Test data files (9 parquet files) -7. ✅ Dockerfile.runpod exists -8. ✅ entrypoint.sh exists and is executable -9. ✅ S3 connectivity (optional) - -**Duration**: ~5 seconds - -**Usage**: -```bash -./scripts/runpod_deploy_test.sh -``` - -**Exit codes**: -- `0`: All tests passed (ready for deployment) -- `>0`: Number of issues found (fix before deploying) - -**Example output**: -``` -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Runpod Deployment Test - Dry Run -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Test 1: Cargo -✓ Cargo installed: 1.83.0 - -Test 2: Docker -✓ Docker running: 27.3.1 - -Test 3: AWS CLI -✓ AWS CLI installed: 2.15.10 - -Test 4: Environment Variables -✓ RUNPOD_S3_ENDPOINT: https://s3api-us-ca-1.runpod.io -✓ AWS_PROFILE: runpod -✓ DOCKER_USERNAME: jgrusewski - -Test 5: AWS Profile Configuration -✓ AWS profile 'runpod' configured -✓ Access Key: abc12345*** - -Test 6: Test Data Files -✓ Found 9 parquet files in /home/user/foxhunt/test_data - - ES_FUT_180d.parquet (2.90 MB) - - NQ_FUT_180d.parquet (4.34 MB) - - 6E_FUT_180d.parquet (2.74 MB) - -Test 7: Dockerfile -✓ Dockerfile.runpod exists - -Test 8: Entrypoint Script -✓ entrypoint.sh exists and is executable - -Test 9: S3 Connectivity (Optional) -✓ S3 connectivity works - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Test Summary -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -✅ All tests passed! Ready for deployment. - -Run deployment with: - ./scripts/runpod_deploy.sh -``` - ---- - -## Prerequisites - -### 1. Rust Toolchain -```bash -# Install Rust -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - -# Verify -cargo --version # Should show 1.70+ -``` - -### 2. Docker -```bash -# Install Docker (Ubuntu/Debian) -sudo apt-get update -sudo apt-get install docker.io -sudo systemctl start docker -sudo systemctl enable docker - -# Add user to docker group (optional, avoids sudo) -sudo usermod -aG docker $USER -newgrp docker - -# Verify -docker --version # Should show 20.10+ -docker info # Should show running -``` - -### 3. AWS CLI -```bash -# Install AWS CLI -pip install awscli - -# Or via apt (Ubuntu/Debian) -sudo apt-get install awscli - -# Verify -aws --version # Should show 2.0+ -``` - -### 4. Runpod Account & Credentials - -#### Create Runpod Account -1. Go to: https://www.runpod.io/console -2. Sign up (requires credit card) -3. Add credits ($10-20 recommended for testing) - -#### Get Runpod Credentials -1. **S3 Endpoint**: Based on datacenter - - US-CA-1: `https://s3api-us-ca-1.runpod.io` - - EU-RO-1: `https://s3api-eu-ro-1.runpod.io` - - Find yours: https://docs.runpod.io/storage/s3-api#endpoint-urls - -2. **User ID** (AWS Access Key): - - Go to: https://www.runpod.io/console/settings - - Copy your User ID (shown at top) - -3. **API Key** (AWS Secret Key): - - Go to: https://www.runpod.io/console/settings - - Click "API Keys" tab - - Create new API key with "Read/Write" permissions - - Copy secret key (only shown once!) - -#### Configure AWS Profile -```bash -# Configure runpod profile -aws configure --profile runpod - -# Enter credentials: -# AWS Access Key ID: -# AWS Secret Access Key: -# Default region name: us-east-1 -# Default output format: json - -# Verify -aws configure list --profile runpod -``` - -#### Create Network Volume -1. Go to: https://www.runpod.io/console/storage -2. Click "Create Network Volume" -3. Name: `foxhunt-runpod` -4. Size: 50 GB (recommended) -5. Datacenter: Same as S3 endpoint (e.g., US-CA-1) -6. Copy Network Volume ID (acts as S3 bucket name) - -### 5. Docker Hub Account -```bash -# Create account: https://hub.docker.com/signup - -# Login -docker login -# Username: jgrusewski -# Password: - -# Verify -docker info | grep Username # Should show your username -``` - -### 6. Environment Variables -```bash -# Add to ~/.bashrc or ~/.zshrc -export RUNPOD_S3_ENDPOINT=https://s3api-us-ca-1.runpod.io # Your datacenter -export AWS_PROFILE=runpod -export DOCKER_USERNAME=jgrusewski -export S3_BUCKET=your-network-volume-id # Optional - -# Reload shell -source ~/.bashrc -``` - ---- - -## Deployment Workflow - -### Step 1: Test Prerequisites -```bash -./scripts/runpod_deploy_test.sh -``` - -**Expected output**: All tests passed ✅ - -**If tests fail**: Follow error messages to install missing tools or configure credentials - -### Step 2: Run Deployment -```bash -./scripts/runpod_deploy.sh -``` - -**Duration**: ~15-20 minutes - -**What happens**: -1. **Validates prerequisites** (~5 seconds) - - Checks cargo, docker, aws cli, credentials - - Fails fast if any prerequisite missing - -2. **Builds release binaries** (~5-6 minutes) - ``` - Compiling foxhunt workspace... - ✓ train_tft_parquet (125.32 MB) - ✓ train_mamba2_parquet (118.45 MB) - ✓ train_dqn (89.67 MB) - ✓ train_ppo (92.11 MB) - Total binaries: 425.55 MB - ``` - -3. **Uploads binaries to Runpod S3** (~2-3 minutes) - ``` - ▶ Uploading train_tft_parquet (125.32 MB)... - ✓ train_tft_parquet uploaded - [... 3 more binaries ...] - ✓ Uploaded 4 binaries to Runpod S3 - ``` - -4. **Uploads test data to Runpod S3** (~30 seconds) - ``` - ▶ Uploading ES_FUT_180d.parquet (2.90 MB)... - ✓ ES_FUT_180d.parquet uploaded - [... 8 more files ...] - ✓ Uploaded 9 data files (12.85 MB) - ``` - -5. **Prompts for .env upload** (optional) - ``` - ⚠ The .env file may contain sensitive credentials - Upload .env? (y/N): n - ✓ .env upload skipped (recommended) - ``` - -6. **Builds Docker image** (~2-3 minutes) - ``` - ▶ Building Docker image: jgrusewski/foxhunt:latest - [... Docker build output ...] - ✓ Docker image built in 2m 34s - ✓ Image size: 1.98GB - ``` - -7. **Pushes to Docker Hub** (~3-5 minutes) - ``` - Is your Docker Hub repo PRIVATE? (y/N): y - ▶ Pushing jgrusewski/foxhunt:latest to Docker Hub... - [... Docker push output ...] - ✓ Image pushed in 4m 12s - ``` - -8. **Prints deployment instructions** - ``` - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - READY FOR RUNPOD DEPLOYMENT - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - - 📦 Uploaded Resources: - - Binaries: s3://foxhunt-runpod/binaries/ (425.55 MB) - - Test Data: s3://foxhunt-runpod/test_data/ (12.85 MB) - - Docker Image: jgrusewski/foxhunt:latest (1.98GB) - - 🚀 Deploy on Runpod Console: - [... detailed instructions ...] - ``` - -### Step 3: Deploy on Runpod Console - -1. **Go to Runpod**: https://www.runpod.io/console/pods - -2. **Click "Deploy"** and configure: - - **GPU Configuration**: - - GPU Type: `Tesla V100 16GB` ($0.14-0.39/hr) or `RTX 4090 24GB` ($0.60/hr) - - vCPU: 6-8 cores (recommended) - - RAM: 30GB+ (recommended) - - Container Disk: 20GB minimum - - **Docker Configuration**: - - Docker Image: `jgrusewski/foxhunt:latest` - - Docker Hub Credentials: Required (private repo) - - Username: `jgrusewski` - - Password: Your Docker Hub password - - **Volume Configuration**: - - Volume Path: `/runpod-volume` - - Network Volume: `foxhunt-runpod` (select from dropdown) - - Access Mode: Read/Write - - **Environment Variables**: - - `BINARY_NAME=train_tft_parquet` - - `RUST_LOG=info` - - **Container Arguments** (override CMD): - ``` - --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet - --epochs 50 - --batch-size 32 - --lookback-window 60 - --forecast-horizon 10 - ``` - -3. **Click "Deploy"** and wait ~30 seconds for pod to start - -4. **Monitor logs** in Runpod console: - ``` - ========================================== - Foxhunt HFT - Runpod Volume Mount Training - ========================================== - Configuration: - Binary: train_tft_parquet - Volume Path: /runpod-volume/ - Architecture: Direct volume mount (NO downloads) - - ========================================== - Verifying Runpod Network Volume Mount - ========================================== - ✓ Volume mounted successfully at /runpod-volume/ - - ========================================== - Starting Training... - ========================================== - Epoch 1/50 [██████████] 100% | Loss: 0.0234 - Epoch 2/50 [██████████] 100% | Loss: 0.0189 - [... training continues ...] - Epoch 50/50 [██████████] 100% | Loss: 0.0056 - ✓ Training complete! Model saved to /workspace/models/ - ``` - -5. **Download trained models**: - ```bash - # Via SSH - scp root@:/workspace/models/*.pt ./models/ - - # Via S3 (if models uploaded to volume) - aws s3 sync s3://foxhunt-runpod/models/ ./models/ \ - --endpoint-url https://s3api-us-ca-1.runpod.io \ - --profile runpod - ``` - ---- - -## Troubleshooting - -### Issue: AWS CLI can't connect to Runpod S3 -**Symptoms**: -``` -An error occurred (InvalidAccessKeyId) when calling the ListBuckets operation -``` - -**Fix**: -```bash -# Reconfigure AWS profile -aws configure --profile runpod - -# Verify credentials -aws configure list --profile runpod - -# Test connectivity -aws s3 ls --endpoint-url $RUNPOD_S3_ENDPOINT --profile runpod -``` - ---- - -### Issue: Docker build fails -**Symptoms**: -``` -ERROR: failed to solve: process "/bin/sh -c cargo build --release..." did not complete successfully -``` - -**Fix**: -```bash -# Clean build artifacts -cd /home/jgrusewski/Work/foxhunt -cargo clean - -# Rebuild -cargo build --release --workspace --features cuda - -# Retry Docker build -docker build -f Dockerfile.runpod -t jgrusewski/foxhunt:latest . -``` - ---- - -### Issue: Runpod volume not mounting -**Symptoms** (in pod logs): -``` -ERROR: /runpod-volume directory does not exist -Runpod Network Volume is NOT mounted! -``` - -**Fix**: -1. Stop pod -2. Edit pod configuration -3. Add volume mount: - - Path: `/runpod-volume` - - Network Volume: `foxhunt-runpod` (select from dropdown) -4. Redeploy pod - ---- - -### Issue: Training binary not found -**Symptoms** (in pod logs): -``` -ERROR: Training binary not found: /runpod-volume/binaries/train_tft_parquet -``` - -**Fix**: -```bash -# Re-upload binaries -cd /home/jgrusewski/Work/foxhunt -cargo build --release --workspace --features cuda - -# Upload to S3 -aws s3 cp target/release/examples/train_tft_parquet \ - s3://foxhunt-runpod/binaries/train_tft_parquet \ - --endpoint-url $RUNPOD_S3_ENDPOINT \ - --profile runpod - -# Verify upload -aws s3 ls s3://foxhunt-runpod/binaries/ \ - --endpoint-url $RUNPOD_S3_ENDPOINT \ - --profile runpod -``` - ---- - -### Issue: Out of memory during training -**Symptoms** (in pod logs): -``` -CUDA error: out of memory -``` - -**Fix**: -1. Use smaller batch size: - ``` - --batch-size 16 # Instead of 32 - ``` - -2. Or upgrade GPU: - - V100 16GB → RTX 4090 24GB - - RTX 4090 24GB → A100 40GB - ---- - -## Cost Estimation - -### GPU Pricing (Runpod Community Cloud) -| GPU | VRAM | Price/Hour | Full Training | 100 Runs | -|---|---|---|---|---| -| Tesla V100 16GB | 16GB | $0.14-0.39 | $0.04-0.10 | $4-10 | -| RTX 4090 24GB | 24GB | $0.60 | $0.15 | $15 | -| A100 40GB | 40GB | $1.50 | $0.38 | $38 | - -### Storage Pricing (Runpod Network Volume) -| Resource | Size | Cost | -|---|---|---| -| Network Volume | 50GB | $5/month | -| Binaries | ~500 MB | Included | -| Test Data | ~13 MB | Included | - -### Total Cost (V100 16GB) -- **Initial setup**: $0 (one-time deployment) -- **Single training run**: $0.04-0.10 (~15 minutes) -- **100 training runs**: $4-10 (hyperparameter tuning) -- **Monthly storage**: $5 (Network Volume) - -**Total first month**: ~$15-25 (includes storage + 100 training runs) - ---- - -## Security Best Practices - -1. **Docker Hub Repository**: - - ✅ Set to PRIVATE (jgrusewski/foxhunt) - - ❌ Never set to PUBLIC (contains proprietary code) - - Verify: https://hub.docker.com/repository/docker/jgrusewski/foxhunt/settings - -2. **Runpod API Key**: - - ✅ Store in AWS CLI profile (`~/.aws/credentials`) - - ❌ Never commit to git - - ❌ Never share publicly - - Rotate every 3-6 months - -3. **Environment Files**: - - ✅ Skip .env upload (contains Vault tokens, DB passwords) - - ❌ Only upload if absolutely required - - ❌ Never commit .env to git - -4. **Network Volume Access**: - - ✅ Restrict to your Runpod account only - - ❌ Never share volume ID publicly - - ❌ Never expose binaries/data externally - ---- - -## Performance Benchmarks - -### Training Time (Tesla V100 16GB) -| Model | Epochs | Time | GPU Memory | Cost (@ $0.25/hr) | -|---|---|---|---|---| -| DQN | 20 | 15-20s | 6MB | $0.001 | -| PPO | 20 | 7-10s | 145MB | $0.001 | -| MAMBA-2 | 50 | 2-3min | 164MB | $0.01 | -| TFT-FP32 | 50 | 3-5min | 500MB | $0.02 | -| **Total** | - | **10-15min** | **815MB peak** | **$0.06** | - -### Training Time (RTX 4090 24GB) -| Model | Epochs | Time | GPU Memory | Cost (@ $0.60/hr) | -|---|---|---|---|---| -| DQN | 20 | 10-15s | 6MB | $0.003 | -| PPO | 20 | 5-7s | 145MB | $0.002 | -| MAMBA-2 | 50 | 1-2min | 164MB | $0.02 | -| TFT-FP32 | 50 | 2-3min | 500MB | $0.03 | -| **Total** | - | **5-8min** | **815MB peak** | **$0.10** | - ---- - -## Next Steps - -After successful deployment: - -1. **Validate models**: Download trained models and test locally -2. **Run backtests**: Use `backtesting_service` to validate performance -3. **Paper trading**: Deploy to staging environment for live testing -4. **Production**: Deploy to production after paper trading validation - -See: -- `RUNPOD_DEPLOYMENT_CHECKLIST.md` - Full deployment checklist -- `WAVE_D_DEPLOYMENT_GUIDE.md` - Wave D production deployment guide -- `CLAUDE.md` - System status and next priorities - ---- - -**Last Updated**: 2025-10-24 -**Status**: ✅ Production Ready (FP32 models only, QAT blocked by 3 P0 issues) diff --git a/scripts/SCRIPTS_INVENTORY.md b/scripts/SCRIPTS_INVENTORY.md deleted file mode 100644 index 6d35a048c..000000000 --- a/scripts/SCRIPTS_INVENTORY.md +++ /dev/null @@ -1,197 +0,0 @@ -# Scripts Inventory - Active Scripts Only -**Last Updated**: 2025-10-30 (Post-Cleanup) -**Total Scripts**: 58 - ---- - -## Quick Reference by Purpose - -### 🚀 Deployment & Infrastructure (8 scripts) -| Script | Purpose | Usage | -|--------|---------|-------| -| `runpod_deploy.py` | Deploy GPU pod to Runpod | `python3 scripts/runpod_deploy.py --gpu-type "RTX A4000"` | -| `upload_binary.py` | Upload binaries to Runpod S3 | `python3 scripts/upload_binary.py ` | -| `monitor_logs.py` | Monitor pod logs | `python3 scripts/monitor_logs.py ` | -| `build_docker_images.sh` | Build multi-stage Docker images | `./scripts/build_docker_images.sh` | -| `build_hyperopt_docker.sh` | Build hyperopt Docker | `./scripts/build_hyperopt_docker.sh` | -| `local_ci_pipeline.sh` | Run local CI/CD | `./scripts/local_ci_pipeline.sh` | -| `start_foxhunt.sh` | Start all services | `./scripts/start_foxhunt.sh` | -| `stop_foxhunt.sh` | Stop all services | `./scripts/stop_foxhunt.sh` | - ---- - -### ✅ Validation & Testing (16 scripts) -| Script | Validates | Usage | -|--------|-----------|-------| -| `validate_auth_enabled.sh` | Auth enabled | `./scripts/validate_auth_enabled.sh` | -| `validate_binary.sh` | Binary integrity | `./scripts/validate_binary.sh` | -| `validate_clippy.sh` | Clippy warnings | `./scripts/validate_clippy.sh` | -| `validate_data_quality.sh` | Data quality | `./scripts/validate_data_quality.sh` | -| `validate_dqn_performance.sh` | DQN performance | `./scripts/validate_dqn_performance.sh` | -| `validate_grpc_endpoints.sh` | gRPC endpoints | `./scripts/validate_grpc_endpoints.sh` | -| `validate_h5_alerting.sh` | H5 alerting | `./scripts/validate_h5_alerting.sh` | -| `validate_jwt_config.sh` | JWT config | `./scripts/validate_jwt_config.sh` | -| `validate_ml_monitoring_metrics.sh` | ML monitoring | `./scripts/validate_ml_monitoring_metrics.sh` | -| `validate-monitoring-performance.sh` | Monitoring performance | `./scripts/validate-monitoring-performance.sh` | -| `validate-performance.py` | System performance | `python3 scripts/validate-performance.py` | -| `validate_ppo_fix.sh` | PPO fixes | `./scripts/validate_ppo_fix.sh` | -| `validate_tft_configs.py` | TFT configs | `python3 scripts/validate_tft_configs.py` | -| `validate_tls_setup.sh` | TLS setup | `./scripts/validate_tls_setup.sh` | -| `validate_training.sh` | Training pipeline | `./scripts/validate_training.sh` | -| `validate_train_script.sh` | Train scripts | `./scripts/validate_train_script.sh` | - ---- - -### 🏥 Health & Diagnostics (5 scripts) -| Script | Purpose | Usage | -|--------|---------|-------| -| `health_check.sh` | Quick health check | `./scripts/health_check.sh` | -| `smoke_test.sh` | Smoke testing | `./scripts/smoke_test.sh` | -| `comprehensive_health_check.sh` | Full system health | `./scripts/comprehensive_health_check.sh` | -| `quick_status.sh` | Quick status | `./scripts/quick_status.sh` | -| `system_resource_monitor.sh` | Resource monitoring | `./scripts/system_resource_monitor.sh` | - ---- - -### 🤖 ML Training & Analysis (7 scripts) -| Script | Purpose | Usage | -|--------|---------|-------| -| `train_all_models_fixed.sh` | Train all models | `./scripts/train_all_models_fixed.sh` | -| `train_tft_production.py` | TFT production training | `python3 scripts/train_tft_production.py` | -| `monitor_hyperopt.sh` | Monitor hyperopt | `./scripts/monitor_hyperopt.sh` | -| `analyze_checkpoints_simple.py` | Checkpoint analysis | `python3 scripts/analyze_checkpoints_simple.py` | -| `compare_checkpoints.py` | Compare checkpoints | `python3 scripts/compare_checkpoints.py` | -| `extract_best_hyperparameters.py` | Extract best params | `python3 scripts/extract_best_hyperparameters.py` | -| `quarterly_retrain.sh` | Quarterly retraining | `./scripts/quarterly_retrain.sh` | - ---- - -### 🔒 Security & Production (7 scripts) -| Script | Purpose | Usage | -|--------|---------|-------| -| `generate-production-secrets.sh` | Generate secrets | `./scripts/generate-production-secrets.sh` | -| `production-security-hardening.sh` | Harden security | `./scripts/production-security-hardening.sh` | -| `security-hardening.sh` | Security config | `./scripts/security-hardening.sh` | -| `setup-docker-secrets.sh` | Docker secrets | `./scripts/setup-docker-secrets.sh` | -| `setup_production_passwords.sh` | Password setup | `./scripts/setup_production_passwords.sh` | -| `export_vault_passwords.sh` | Export Vault passwords | `./scripts/export_vault_passwords.sh` | -| `generate-compliance-report.py` | Compliance report | `python3 scripts/generate-compliance-report.py` | - ---- - -### 🔍 Verification & Setup (7 scripts) -| Script | Purpose | Usage | -|--------|---------|-------| -| `verify_ci_setup.sh` | Verify CI/CD | `./scripts/verify_ci_setup.sh` | -| `verify_ml_dependencies.sh` | Verify ML deps | `./scripts/verify_ml_dependencies.sh` | -| `verify_vault_setup.sh` | Verify Vault | `./scripts/verify_vault_setup.sh` | -| `check_service_binaries.sh` | Check service bins | `./scripts/check_service_binaries.sh` | -| `check_all_available_gpus.py` | Check all GPUs | `python3 scripts/check_all_available_gpus.py` | -| `check_gpu_availability.py` | Check GPU status | `python3 scripts/check_gpu_availability.py` | -| `watch_gpu_availability.sh` | Watch GPU | `./scripts/watch_gpu_availability.sh` | - ---- - -### 🛠️ Utilities (8 scripts) -| Script | Purpose | Usage | -|--------|---------|-------| -| `convert_csv_to_parquet.py` | CSV → Parquet | `python3 scripts/convert_csv_to_parquet.py` | -| `plan_databento_download.sh` | Plan data download | `./scripts/plan_databento_download.sh` | -| `enforce_coverage.sh` | Enforce coverage | `./scripts/enforce_coverage.sh` | -| `run-coverage.sh` | Run coverage | `./scripts/run-coverage.sh` | -| `run_comprehensive_tests.sh` | Comprehensive tests | `./scripts/run_comprehensive_tests.sh` | -| `offline_service_validation.sh` | Offline validation | `./scripts/offline_service_validation.sh` | -| `install_cron.sh` | Install cron jobs | `./scripts/install_cron.sh` | -| `cleanup_deprecated_scripts.sh` | Cleanup scripts | `./scripts/cleanup_deprecated_scripts.sh` | - ---- - -## Common Workflows - -### Deploy to Runpod -```bash -# 1. Build Docker image -./scripts/build_docker_images.sh - -# 2. Deploy pod -python3 scripts/runpod_deploy.py --gpu-type "RTX A4000" - -# 3. Monitor logs -python3 scripts/monitor_logs.py -``` - -### Local Development -```bash -# 1. Start services -./scripts/start_foxhunt.sh - -# 2. Run health checks -./scripts/health_check.sh - -# 3. Run smoke tests -./scripts/smoke_test.sh - -# 4. Stop services -./scripts/stop_foxhunt.sh -``` - -### CI/CD Validation -```bash -# 1. Run local CI pipeline -./scripts/local_ci_pipeline.sh - -# 2. Verify setup -./scripts/verify_ci_setup.sh - -# 3. Validate performance -python3 scripts/validate-performance.py -``` - -### ML Training -```bash -# 1. Verify ML dependencies -./scripts/verify_ml_dependencies.sh - -# 2. Check GPU availability -python3 scripts/check_gpu_availability.py - -# 3. Train all models -./scripts/train_all_models_fixed.sh - -# 4. Monitor training -./scripts/monitor_hyperopt.sh -``` - ---- - -## Deprecated Scripts (Backed Up) - -All deprecated scripts moved to: -``` -/home/jgrusewski/Work/foxhunt/scripts/archive/cleanup_2025_10_30/ -``` - -**Total Deleted**: 56 scripts (49% reduction) - -See `/home/jgrusewski/Work/foxhunt/SCRIPTS_CLEANUP_REPORT_2025_10_30.md` for details. - ---- - -## Maintenance - -### Adding New Scripts -1. Create script in `/scripts/` -2. Make executable: `chmod +x scripts/