9c3d741a08
Move 17 library crates into crates/, CLI binary into bin/fxt, consolidate 10 test crates into testing/, split config crate from deployment config files. Root directory reduced from 38+ to ~17 directories. All Cargo.toml paths and build.rs proto refs updated. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
81 lines
2.3 KiB
TOML
81 lines
2.3 KiB
TOML
# Foxhunt Chaos Engineering Configuration
|
|
# This file configures the nightly chaos engineering tests for the HFT system
|
|
|
|
[general]
|
|
enabled = true
|
|
exclude_weekends = true
|
|
schedule_time = "02:00" # 2:00 AM UTC
|
|
timezone = "UTC"
|
|
max_duration_hours = 3
|
|
retry_on_failure = true
|
|
max_retries = 2
|
|
|
|
[notifications]
|
|
# Slack webhook for alerts (optional)
|
|
# webhook_url = "https://hooks.slack.com/services/YOUR/WEBHOOK/URL"
|
|
|
|
# Report storage
|
|
report_storage_path = "./chaos_reports"
|
|
|
|
[ml_chaos_config]
|
|
ml_service_endpoint = "http://localhost:8080"
|
|
checkpoint_base_path = "/tmp/ml_checkpoints"
|
|
training_timeout_secs = 300
|
|
max_recovery_time_ms = 100 # HFT requirement: sub-100ms recovery
|
|
gpu_memory_threshold_mb = 8192
|
|
|
|
# ML models to test
|
|
model_types = ["tlob", "mamba2", "dqn", "ppo", "liquid", "tft"]
|
|
|
|
[hft_requirements]
|
|
# Maximum latency requirements for HFT system
|
|
max_order_processing_latency_us = 50 # 50 microseconds
|
|
max_market_data_latency_us = 30 # 30 microseconds
|
|
max_risk_calculation_latency_us = 25 # 25 microseconds
|
|
max_recovery_time_ms = 100 # 100 milliseconds recovery
|
|
|
|
[failure_scenarios]
|
|
# Process kill scenarios
|
|
[failure_scenarios.process_kill]
|
|
enabled = true
|
|
signals = ["SIGTERM", "SIGKILL"]
|
|
restart_delay_ms = [1000, 2000, 5000]
|
|
|
|
# Memory pressure scenarios
|
|
[failure_scenarios.memory_pressure]
|
|
enabled = true
|
|
target_mb = [2048, 4096, 8192]
|
|
duration_ms = [10000, 30000, 60000]
|
|
|
|
# Network partition scenarios
|
|
[failure_scenarios.network_partition]
|
|
enabled = true
|
|
target_ports = [8080, 5432, 6379, 9090] # gRPC, PostgreSQL, Redis, Prometheus
|
|
duration_ms = [5000, 15000, 30000]
|
|
|
|
# Disk I/O failure scenarios
|
|
[failure_scenarios.disk_io_failure]
|
|
enabled = true
|
|
target_paths = ["/tmp", "/var/log", "./ml_checkpoints"]
|
|
failure_rate_percent = [10, 30, 50]
|
|
|
|
# CPU throttling scenarios
|
|
[failure_scenarios.cpu_throttle]
|
|
enabled = true
|
|
cpu_limit_percent = [25, 50, 75]
|
|
duration_ms = [15000, 30000, 45000]
|
|
|
|
# GPU resource exhaustion scenarios
|
|
[failure_scenarios.gpu_exhaustion]
|
|
enabled = true
|
|
memory_fill_percent = [80, 90, 95]
|
|
duration_ms = [10000, 20000, 30000]
|
|
|
|
# Database connection failure scenarios
|
|
[failure_scenarios.database_failure]
|
|
enabled = true
|
|
connection_strings = [
|
|
"postgresql://localhost:5432/foxhunt",
|
|
"redis://localhost:6379",
|
|
]
|
|
duration_ms = [5000, 10000, 20000] |