Agent 106: ML health endpoint (HTTP/8095) Agent 107: Redis test fix (serial_test isolation) Agent 108: CLAUDE.md draft update (95-97% → 100%) Agent 109: Prometheus/Grafana setup (31 alerts, 6 dashboards) Agent 110: Deployment docs (9 files + 4 scripts) Agent 111: Security audit prep (0 critical vulnerabilities) Service Health: 4/4 healthy (100%) Tests: 99%+ pass rate Production: ~98% readiness Next: Wave 2 (E2E, load, perf, security validation)
220 lines
7.2 KiB
Plaintext
220 lines
7.2 KiB
Plaintext
# FOXHUNT HFT CRITICAL ALERTING RULES
|
|
# Production monitoring for real money trading
|
|
# Generated for immediate deployment
|
|
|
|
groups:
|
|
- name: hft-critical-latency
|
|
interval: 1s
|
|
rules:
|
|
- alert: HFT_LatencyViolation_Critical
|
|
expr: histogram_quantile(0.99, foxhunt_hft_tick_to_trade_latency_nanos_bucket) > 100000
|
|
for: 0s
|
|
labels:
|
|
severity: critical
|
|
team: trading
|
|
service: hft-engine
|
|
annotations:
|
|
summary: "CRITICAL: Trading latency exceeded 100μs threshold"
|
|
description: "P99 latency is {{ $value | humanize }}ns ({{ printf \"%.1f\" (div $value 1000) }}μs). IMMEDIATE ACTION REQUIRED."
|
|
runbook: "https://docs.foxhunt.io/runbooks/latency-violation"
|
|
|
|
- alert: HFT_LatencyViolation_Warning
|
|
expr: histogram_quantile(0.95, foxhunt_hft_tick_to_trade_latency_nanos_bucket) > 50000
|
|
for: 5s
|
|
labels:
|
|
severity: warning
|
|
team: trading
|
|
service: hft-engine
|
|
annotations:
|
|
summary: "WARNING: Trading latency approaching critical threshold"
|
|
description: "P95 latency is {{ $value | humanize }}ns ({{ printf \"%.1f\" (div $value 1000) }}μs). Monitor closely."
|
|
|
|
- name: hft-financial-limits
|
|
interval: 1s
|
|
rules:
|
|
- alert: HFT_DailyLossLimit_Critical
|
|
expr: foxhunt_daily_pnl_usd < -50000
|
|
for: 0s
|
|
labels:
|
|
severity: critical
|
|
team: risk-management
|
|
service: trading-engine
|
|
annotations:
|
|
summary: "CRITICAL: Daily loss limit exceeded - TRADING HALT"
|
|
description: "Daily P&L is ${{ $value }}. Loss limit of $50,000 exceeded. Trading automatically halted."
|
|
runbook: "https://docs.foxhunt.io/runbooks/loss-limits"
|
|
|
|
- alert: HFT_DailyLossLimit_Warning
|
|
expr: foxhunt_daily_pnl_usd < -10000
|
|
for: 0s
|
|
labels:
|
|
severity: warning
|
|
team: risk-management
|
|
service: trading-engine
|
|
annotations:
|
|
summary: "WARNING: Daily loss approaching limit"
|
|
description: "Daily P&L is ${{ $value }}. Approaching $50,000 loss limit."
|
|
|
|
- alert: HFT_PositionLimit_Critical
|
|
expr: abs(foxhunt_position_size) > 950000
|
|
for: 0s
|
|
labels:
|
|
severity: critical
|
|
team: risk-management
|
|
service: trading-engine
|
|
annotations:
|
|
summary: "CRITICAL: Position size near maximum limit"
|
|
description: "Position size is ${{ $value }}. Near $1M limit. New orders blocked."
|
|
|
|
- name: hft-circuit-breakers
|
|
interval: 1s
|
|
rules:
|
|
- alert: HFT_CircuitBreaker_Triggered
|
|
expr: foxhunt_circuit_breaker_state == 1
|
|
for: 0s
|
|
labels:
|
|
severity: critical
|
|
team: trading
|
|
service: "{{ $labels.service }}"
|
|
annotations:
|
|
summary: "CRITICAL: Circuit breaker triggered for {{ $labels.breaker_type }}"
|
|
description: "Circuit breaker {{ $labels.breaker_type }} is OPEN. Trading may be halted."
|
|
runbook: "https://docs.foxhunt.io/runbooks/circuit-breaker"
|
|
|
|
- alert: HFT_TradingHalt_Active
|
|
expr: foxhunt_trading_halt_active == 1
|
|
for: 0s
|
|
labels:
|
|
severity: critical
|
|
team: trading
|
|
service: trading-engine
|
|
annotations:
|
|
summary: "CRITICAL: Trading halt is active"
|
|
description: "All trading operations are halted. Reason: {{ $labels.halt_reason }}"
|
|
runbook: "https://docs.foxhunt.io/runbooks/trading-halt"
|
|
|
|
- name: hft-market-connectivity
|
|
interval: 5s
|
|
rules:
|
|
- alert: HFT_ExchangeDisconnection
|
|
expr: foxhunt_exchange_connected == 0
|
|
for: 5s
|
|
labels:
|
|
severity: critical
|
|
team: trading
|
|
exchange: "{{ $labels.exchange }}"
|
|
annotations:
|
|
summary: "CRITICAL: Exchange {{ $labels.exchange }} disconnected"
|
|
description: "Connection to {{ $labels.exchange }} lost for >5 seconds. Orders may be affected."
|
|
runbook: "https://docs.foxhunt.io/runbooks/exchange-connectivity"
|
|
|
|
- alert: HFT_MarketDataGap
|
|
expr: increase(foxhunt_market_data_gaps_total[1m]) > 0
|
|
for: 0s
|
|
labels:
|
|
severity: warning
|
|
team: trading
|
|
exchange: "{{ $labels.exchange }}"
|
|
annotations:
|
|
summary: "WARNING: Market data gap detected"
|
|
description: "{{ $value }} market data gaps detected from {{ $labels.exchange }} in last minute."
|
|
|
|
- name: hft-system-health
|
|
interval: 5s
|
|
rules:
|
|
- alert: HFT_SystemHealth_Critical
|
|
expr: foxhunt_cpu_usage_percent > 95
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
team: infrastructure
|
|
service: "{{ $labels.service }}"
|
|
annotations:
|
|
summary: "CRITICAL: High CPU usage on {{ $labels.service }}"
|
|
description: "CPU usage is {{ $value }}% on {{ $labels.service }}. System may be degraded."
|
|
|
|
- alert: HFT_MemoryUsage_Critical
|
|
expr: foxhunt_memory_usage_bytes > 6442450944 # 6GB
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
team: infrastructure
|
|
service: "{{ $labels.service }}"
|
|
annotations:
|
|
summary: "CRITICAL: High memory usage on {{ $labels.service }}"
|
|
description: "Memory usage is {{ humanize $value }} on {{ $labels.service }}."
|
|
|
|
- alert: HFT_GCPause_Critical
|
|
expr: histogram_quantile(0.99, foxhunt_gc_pause_duration_nanos_bucket) > 10000000
|
|
for: 0s
|
|
labels:
|
|
severity: warning
|
|
team: infrastructure
|
|
service: "{{ $labels.service }}"
|
|
annotations:
|
|
summary: "WARNING: Long GC pause detected"
|
|
description: "GC pause P99 is {{ printf \"%.1f\" (div $value 1000000) }}ms on {{ $labels.service }}."
|
|
|
|
- name: hft-order-flow
|
|
interval: 1s
|
|
rules:
|
|
- alert: HFT_OrderRejection_High
|
|
expr: rate(foxhunt_orders_rejected_total[1m]) > 10
|
|
for: 30s
|
|
labels:
|
|
severity: warning
|
|
team: trading
|
|
strategy: "{{ $labels.strategy }}"
|
|
annotations:
|
|
summary: "WARNING: High order rejection rate"
|
|
description: "{{ $value | humanize }} orders/min rejected for strategy {{ $labels.strategy }}."
|
|
|
|
- alert: HFT_FillRate_Low
|
|
expr: (rate(foxhunt_orders_filled_total[5m]) / rate(foxhunt_orders_placed_total[5m])) < 0.8
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
team: trading
|
|
strategy: "{{ $labels.strategy }}"
|
|
annotations:
|
|
summary: "WARNING: Low fill rate detected"
|
|
description: "Fill rate is {{ printf \"%.1f\" (mul $value 100) }}% for strategy {{ $labels.strategy }}."
|
|
|
|
- name: hft-slippage-monitoring
|
|
interval: 5s
|
|
rules:
|
|
- alert: HFT_Slippage_High
|
|
expr: histogram_quantile(0.95, foxhunt_hft_slippage_bps_bucket) > 5
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
team: trading
|
|
symbol: "{{ $labels.symbol }}"
|
|
annotations:
|
|
summary: "WARNING: High slippage detected"
|
|
description: "P95 slippage is {{ $value | humanize }} bps for {{ $labels.symbol }}."
|
|
|
|
- name: hft-risk-metrics
|
|
interval: 5s
|
|
rules:
|
|
- alert: HFT_VaR_Exceeded
|
|
expr: foxhunt_value_at_risk > 100000
|
|
for: 0s
|
|
labels:
|
|
severity: warning
|
|
team: risk-management
|
|
service: risk-management
|
|
annotations:
|
|
summary: "WARNING: Value at Risk exceeded threshold"
|
|
description: "VaR is ${{ $value | humanize }}. Exceeds $100K threshold."
|
|
|
|
- alert: HFT_CorrelationRisk_High
|
|
expr: foxhunt_correlation_risk > 0.8
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
team: risk-management
|
|
service: risk-management
|
|
annotations:
|
|
summary: "WARNING: High correlation risk detected"
|
|
description: "Correlation risk is {{ $value | humanize }}. Portfolio may be overexposed." |