Files
foxhunt/config/prometheus/rules/hft-alerts.yml.disabled
jgrusewski 39c1028502 🚀 Wave 126 Wave 1 Complete: 6 agents deployed - 4/4 services healthy
Agent 106: ML health endpoint (HTTP/8095)
Agent 107: Redis test fix (serial_test isolation)
Agent 108: CLAUDE.md draft update (95-97% → 100%)
Agent 109: Prometheus/Grafana setup (31 alerts, 6 dashboards)
Agent 110: Deployment docs (9 files + 4 scripts)
Agent 111: Security audit prep (0 critical vulnerabilities)

Service Health: 4/4 healthy (100%)
Tests: 99%+ pass rate
Production: ~98% readiness

Next: Wave 2 (E2E, load, perf, security validation)
2025-10-08 00:11:38 +02:00

220 lines
7.2 KiB
Plaintext

# FOXHUNT HFT CRITICAL ALERTING RULES
# Production monitoring for real money trading
# Generated for immediate deployment
groups:
- name: hft-critical-latency
interval: 1s
rules:
- alert: HFT_LatencyViolation_Critical
expr: histogram_quantile(0.99, foxhunt_hft_tick_to_trade_latency_nanos_bucket) > 100000
for: 0s
labels:
severity: critical
team: trading
service: hft-engine
annotations:
summary: "CRITICAL: Trading latency exceeded 100μs threshold"
description: "P99 latency is {{ $value | humanize }}ns ({{ printf \"%.1f\" (div $value 1000) }}μs). IMMEDIATE ACTION REQUIRED."
runbook: "https://docs.foxhunt.io/runbooks/latency-violation"
- alert: HFT_LatencyViolation_Warning
expr: histogram_quantile(0.95, foxhunt_hft_tick_to_trade_latency_nanos_bucket) > 50000
for: 5s
labels:
severity: warning
team: trading
service: hft-engine
annotations:
summary: "WARNING: Trading latency approaching critical threshold"
description: "P95 latency is {{ $value | humanize }}ns ({{ printf \"%.1f\" (div $value 1000) }}μs). Monitor closely."
- name: hft-financial-limits
interval: 1s
rules:
- alert: HFT_DailyLossLimit_Critical
expr: foxhunt_daily_pnl_usd < -50000
for: 0s
labels:
severity: critical
team: risk-management
service: trading-engine
annotations:
summary: "CRITICAL: Daily loss limit exceeded - TRADING HALT"
description: "Daily P&L is ${{ $value }}. Loss limit of $50,000 exceeded. Trading automatically halted."
runbook: "https://docs.foxhunt.io/runbooks/loss-limits"
- alert: HFT_DailyLossLimit_Warning
expr: foxhunt_daily_pnl_usd < -10000
for: 0s
labels:
severity: warning
team: risk-management
service: trading-engine
annotations:
summary: "WARNING: Daily loss approaching limit"
description: "Daily P&L is ${{ $value }}. Approaching $50,000 loss limit."
- alert: HFT_PositionLimit_Critical
expr: abs(foxhunt_position_size) > 950000
for: 0s
labels:
severity: critical
team: risk-management
service: trading-engine
annotations:
summary: "CRITICAL: Position size near maximum limit"
description: "Position size is ${{ $value }}. Near $1M limit. New orders blocked."
- name: hft-circuit-breakers
interval: 1s
rules:
- alert: HFT_CircuitBreaker_Triggered
expr: foxhunt_circuit_breaker_state == 1
for: 0s
labels:
severity: critical
team: trading
service: "{{ $labels.service }}"
annotations:
summary: "CRITICAL: Circuit breaker triggered for {{ $labels.breaker_type }}"
description: "Circuit breaker {{ $labels.breaker_type }} is OPEN. Trading may be halted."
runbook: "https://docs.foxhunt.io/runbooks/circuit-breaker"
- alert: HFT_TradingHalt_Active
expr: foxhunt_trading_halt_active == 1
for: 0s
labels:
severity: critical
team: trading
service: trading-engine
annotations:
summary: "CRITICAL: Trading halt is active"
description: "All trading operations are halted. Reason: {{ $labels.halt_reason }}"
runbook: "https://docs.foxhunt.io/runbooks/trading-halt"
- name: hft-market-connectivity
interval: 5s
rules:
- alert: HFT_ExchangeDisconnection
expr: foxhunt_exchange_connected == 0
for: 5s
labels:
severity: critical
team: trading
exchange: "{{ $labels.exchange }}"
annotations:
summary: "CRITICAL: Exchange {{ $labels.exchange }} disconnected"
description: "Connection to {{ $labels.exchange }} lost for >5 seconds. Orders may be affected."
runbook: "https://docs.foxhunt.io/runbooks/exchange-connectivity"
- alert: HFT_MarketDataGap
expr: increase(foxhunt_market_data_gaps_total[1m]) > 0
for: 0s
labels:
severity: warning
team: trading
exchange: "{{ $labels.exchange }}"
annotations:
summary: "WARNING: Market data gap detected"
description: "{{ $value }} market data gaps detected from {{ $labels.exchange }} in last minute."
- name: hft-system-health
interval: 5s
rules:
- alert: HFT_SystemHealth_Critical
expr: foxhunt_cpu_usage_percent > 95
for: 30s
labels:
severity: critical
team: infrastructure
service: "{{ $labels.service }}"
annotations:
summary: "CRITICAL: High CPU usage on {{ $labels.service }}"
description: "CPU usage is {{ $value }}% on {{ $labels.service }}. System may be degraded."
- alert: HFT_MemoryUsage_Critical
expr: foxhunt_memory_usage_bytes > 6442450944 # 6GB
for: 30s
labels:
severity: critical
team: infrastructure
service: "{{ $labels.service }}"
annotations:
summary: "CRITICAL: High memory usage on {{ $labels.service }}"
description: "Memory usage is {{ humanize $value }} on {{ $labels.service }}."
- alert: HFT_GCPause_Critical
expr: histogram_quantile(0.99, foxhunt_gc_pause_duration_nanos_bucket) > 10000000
for: 0s
labels:
severity: warning
team: infrastructure
service: "{{ $labels.service }}"
annotations:
summary: "WARNING: Long GC pause detected"
description: "GC pause P99 is {{ printf \"%.1f\" (div $value 1000000) }}ms on {{ $labels.service }}."
- name: hft-order-flow
interval: 1s
rules:
- alert: HFT_OrderRejection_High
expr: rate(foxhunt_orders_rejected_total[1m]) > 10
for: 30s
labels:
severity: warning
team: trading
strategy: "{{ $labels.strategy }}"
annotations:
summary: "WARNING: High order rejection rate"
description: "{{ $value | humanize }} orders/min rejected for strategy {{ $labels.strategy }}."
- alert: HFT_FillRate_Low
expr: (rate(foxhunt_orders_filled_total[5m]) / rate(foxhunt_orders_placed_total[5m])) < 0.8
for: 1m
labels:
severity: warning
team: trading
strategy: "{{ $labels.strategy }}"
annotations:
summary: "WARNING: Low fill rate detected"
description: "Fill rate is {{ printf \"%.1f\" (mul $value 100) }}% for strategy {{ $labels.strategy }}."
- name: hft-slippage-monitoring
interval: 5s
rules:
- alert: HFT_Slippage_High
expr: histogram_quantile(0.95, foxhunt_hft_slippage_bps_bucket) > 5
for: 1m
labels:
severity: warning
team: trading
symbol: "{{ $labels.symbol }}"
annotations:
summary: "WARNING: High slippage detected"
description: "P95 slippage is {{ $value | humanize }} bps for {{ $labels.symbol }}."
- name: hft-risk-metrics
interval: 5s
rules:
- alert: HFT_VaR_Exceeded
expr: foxhunt_value_at_risk > 100000
for: 0s
labels:
severity: warning
team: risk-management
service: risk-management
annotations:
summary: "WARNING: Value at Risk exceeded threshold"
description: "VaR is ${{ $value | humanize }}. Exceeds $100K threshold."
- alert: HFT_CorrelationRisk_High
expr: foxhunt_correlation_risk > 0.8
for: 2m
labels:
severity: warning
team: risk-management
service: risk-management
annotations:
summary: "WARNING: High correlation risk detected"
description: "Correlation risk is {{ $value | humanize }}. Portfolio may be overexposed."