From d1c336c2b606026b24f5a2c4ea10b4b4412d6468 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 25 Feb 2026 19:06:05 +0100 Subject: [PATCH] chore: delete obsolete top-level k8s/ and config/k8s/ manifests Superseded by infra/k8s/ which has the current, actively maintained manifests (11 services, databases, GPU taints, tailscale, training). -1,899 lines (10 files) Co-Authored-By: Claude Opus 4.6 --- config/k8s/hpa.yaml | 464 ------------------ k8s/deployments/api-gateway-deployment.yaml | 137 ------ .../backtesting-service-deployment.yaml | 143 ------ .../ml-training-service-deployment.yaml | 153 ------ .../trading-agent-service-deployment.yaml | 137 ------ .../trading-service-deployment.yaml | 137 ------ k8s/foxhunt-complete.yaml | 431 ---------------- k8s/services/api-gateway-service.yaml | 33 -- k8s/services/trading-service.yaml | 29 -- k8s/trading-service.yaml | 233 --------- 10 files changed, 1897 deletions(-) delete mode 100644 config/k8s/hpa.yaml delete mode 100644 k8s/deployments/api-gateway-deployment.yaml delete mode 100644 k8s/deployments/backtesting-service-deployment.yaml delete mode 100644 k8s/deployments/ml-training-service-deployment.yaml delete mode 100644 k8s/deployments/trading-agent-service-deployment.yaml delete mode 100644 k8s/deployments/trading-service-deployment.yaml delete mode 100644 k8s/foxhunt-complete.yaml delete mode 100644 k8s/services/api-gateway-service.yaml delete mode 100644 k8s/services/trading-service.yaml delete mode 100644 k8s/trading-service.yaml diff --git a/config/k8s/hpa.yaml b/config/k8s/hpa.yaml deleted file mode 100644 index 45c5d659b..000000000 --- a/config/k8s/hpa.yaml +++ /dev/null @@ -1,464 +0,0 @@ -# Kubernetes Horizontal Pod Autoscaler (HPA) Manifests -# Version: 1.0 -# Last Updated: 2025-10-07 -# -# These HPA configurations provide: -# - CPU-based autoscaling for all services -# - Custom metrics for advanced scaling (request rate, queue depth) -# - Configurable scale-up/scale-down behavior -# - Min/max replica counts for cost optimization - ---- -# ============================================================================ -# API GATEWAY HPA (Latency-Sensitive, Aggressive Scaling) -# ============================================================================ - -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: api-gateway-hpa - namespace: foxhunt - labels: - app: api-gateway - component: gateway -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: api-gateway - - # Replica limits - minReplicas: 3 # Always maintain 3 for HA (survive 1 node failure + 1 rolling update) - maxReplicas: 20 # Max capacity: 20 instances × 2K req/s = 40K req/s - - # Scaling metrics - metrics: - # CPU-based scaling (primary metric) - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 # Scale when CPU > 70% - - # Memory-based scaling (secondary metric) - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 # Scale when memory > 80% - - # Custom metric: Request rate (requires Prometheus adapter) - - type: Pods - pods: - metric: - name: grpc_requests_per_second - target: - type: AverageValue - averageValue: "2000" # Scale when > 2000 req/s per pod - - # Scaling behavior (aggressive scale-up, slow scale-down) - behavior: - scaleUp: - stabilizationWindowSeconds: 30 # Wait 30s before scaling up - policies: - - type: Percent - value: 100 # Double replicas - periodSeconds: 15 - - type: Pods - value: 4 # Or add 4 pods - periodSeconds: 15 - selectPolicy: Max # Use whichever scales faster - - scaleDown: - stabilizationWindowSeconds: 300 # Wait 5 minutes before scaling down - policies: - - type: Percent - value: 25 # Remove 25% of replicas - periodSeconds: 60 - - type: Pods - value: 1 # Or remove 1 pod - periodSeconds: 60 - selectPolicy: Min # Use whichever scales slower - ---- -# ============================================================================ -# TRADING SERVICE HPA (Latency-Sensitive, Aggressive Scaling) -# ============================================================================ - -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: trading-service-hpa - namespace: foxhunt - labels: - app: trading-service - component: backend -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: trading-service - - # Replica limits - minReplicas: 3 # HA requirement - maxReplicas: 20 # Based on performance testing: 50K orders/sec ÷ 2.5K orders/sec per pod - - # Scaling metrics - metrics: - # CPU-based scaling - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 - - # Memory-based scaling - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 - - # Custom metric: Order submission rate - - type: Pods - pods: - metric: - name: order_submissions_per_second - target: - type: AverageValue - averageValue: "2500" # Scale when > 2500 orders/s per pod - - # Scaling behavior (aggressive scale-up) - behavior: - scaleUp: - stabilizationWindowSeconds: 30 - policies: - - type: Percent - value: 100 - periodSeconds: 15 - - type: Pods - value: 4 - periodSeconds: 15 - selectPolicy: Max - - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 25 - periodSeconds: 60 - - type: Pods - value: 1 - periodSeconds: 60 - selectPolicy: Min - ---- -# ============================================================================ -# BACKTESTING SERVICE HPA (Job Queue Pattern, Conservative Scaling) -# ============================================================================ - -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: backtesting-service-hpa - namespace: foxhunt - labels: - app: backtesting-service - component: backend -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: backtesting-service - - # Replica limits - minReplicas: 2 # Minimum workers for job queue - maxReplicas: 10 # Rarely need more than 10 concurrent backtests - - # Scaling metrics - metrics: - # CPU-based scaling (primary for CPU-intensive backtests) - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 75 # Higher threshold for batch workload - - # Memory-based scaling - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 - - # Custom metric: Redis job queue depth - - type: Pods - pods: - metric: - name: redis_queue_length_backtesting - target: - type: AverageValue - averageValue: "5" # Scale when > 5 jobs per worker - - # Scaling behavior (conservative scale-up, slower scale-down) - behavior: - scaleUp: - stabilizationWindowSeconds: 60 # Wait 60s before scaling up - policies: - - type: Percent - value: 50 # Add 50% more replicas - periodSeconds: 60 - - type: Pods - value: 2 # Or add 2 pods - periodSeconds: 60 - selectPolicy: Min # Use whichever scales slower - - scaleDown: - stabilizationWindowSeconds: 600 # Wait 10 minutes before scaling down - policies: - - type: Percent - value: 25 - periodSeconds: 120 - - type: Pods - value: 1 - periodSeconds: 120 - selectPolicy: Min - ---- -# ============================================================================ -# ML TRAINING SERVICE HPA (MANUAL SCALING RECOMMENDED) -# ============================================================================ - -# NOTE: ML Training Service uses GPU resources, which are expensive and slow to provision. -# Recommendation: Use manual scaling (kubectl scale) or scheduled scaling, NOT auto-scaling. -# -# If auto-scaling is required, use job queue pattern with custom metrics: - -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: ml-training-service-hpa - namespace: foxhunt - labels: - app: ml-training-service - component: backend -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: ml-training-service - - # Replica limits (GPU resources expensive) - minReplicas: 1 # Single instance (GPU idle time acceptable) - maxReplicas: 3 # Limited by GPU cluster size - - # Scaling metrics - metrics: - # GPU utilization (requires NVIDIA DCGM exporter) - - type: Pods - pods: - metric: - name: dcgm_gpu_utilization - target: - type: AverageValue - averageValue: "80" # Scale when GPU > 80% utilized - - # Custom metric: Training job queue depth - - type: Pods - pods: - metric: - name: redis_queue_length_ml_training - target: - type: AverageValue - averageValue: "2" # Scale when > 2 training jobs queued - - # Scaling behavior (very conservative, GPU instances expensive) - behavior: - scaleUp: - stabilizationWindowSeconds: 300 # Wait 5 minutes before scaling up - policies: - - type: Pods - value: 1 # Add 1 pod at a time - periodSeconds: 300 - selectPolicy: Min - - scaleDown: - stabilizationWindowSeconds: 1800 # Wait 30 minutes before scaling down - policies: - - type: Pods - value: 1 - periodSeconds: 300 - selectPolicy: Min - ---- -# ============================================================================ -# CUSTOM METRICS (Prometheus Adapter Configuration) -# ============================================================================ - -# To use custom metrics (grpc_requests_per_second, redis_queue_length, etc.), -# you must install and configure Prometheus Adapter: -# -# 1. Install Prometheus Adapter: -# helm install prometheus-adapter prometheus-community/prometheus-adapter \ -# --namespace monitoring \ -# --values prometheus-adapter-values.yaml -# -# 2. Configure custom metrics (prometheus-adapter-values.yaml): - -# prometheus: -# url: http://prometheus-server.monitoring.svc -# port: 9090 -# -# rules: -# default: false -# custom: -# # gRPC request rate per pod -# - seriesQuery: 'grpc_requests_total{namespace="foxhunt"}' -# resources: -# overrides: -# namespace: {resource: "namespace"} -# pod: {resource: "pod"} -# name: -# matches: "^grpc_requests_total" -# as: "grpc_requests_per_second" -# metricsQuery: 'sum(rate(grpc_requests_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)' -# -# # Redis queue length -# - seriesQuery: 'redis_list_length{namespace="foxhunt",queue="backtesting"}' -# resources: -# overrides: -# namespace: {resource: "namespace"} -# name: -# matches: "^redis_list_length" -# as: "redis_queue_length_backtesting" -# metricsQuery: 'avg(redis_list_length{<<.LabelMatchers>>}) by (<<.GroupBy>>)' -# -# # GPU utilization (requires NVIDIA DCGM exporter) -# - seriesQuery: 'DCGM_FI_DEV_GPU_UTIL{namespace="foxhunt"}' -# resources: -# overrides: -# namespace: {resource: "namespace"} -# pod: {resource: "pod"} -# name: -# matches: "^DCGM_FI_DEV_GPU_UTIL" -# as: "dcgm_gpu_utilization" -# metricsQuery: 'avg(DCGM_FI_DEV_GPU_UTIL{<<.LabelMatchers>>}) by (<<.GroupBy>>)' - -# 3. Verify custom metrics are available: -# kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq . - -# 4. Test HPA with custom metrics: -# kubectl get hpa api-gateway-hpa -o yaml -# # Check "currentMetrics" section for custom metric values - ---- -# ============================================================================ -# TESTING & VALIDATION -# ============================================================================ - -# 1. Deploy HPA manifests: -# kubectl apply -f hpa.yaml -# -# 2. Verify HPA status: -# kubectl get hpa -n foxhunt -# kubectl describe hpa api-gateway-hpa -n foxhunt -# -# 3. Monitor scaling events: -# kubectl get events -n foxhunt --field-selector involvedObject.kind=HorizontalPodAutoscaler -# -# 4. Load test to trigger scaling: -# # Generate load (e.g., with ghz) -# ghz --insecure \ -# --proto api_gateway.proto \ -# --call foxhunt.api_gateway.ApiGateway/SubmitOrder \ -# --rps 10000 \ -# --duration 5m \ -# localhost:50051 -# -# # Watch HPA scale up -# watch kubectl get hpa -n foxhunt -# -# 5. Verify scale-down after load stops: -# # Wait 5-10 minutes after load test ends -# kubectl get hpa api-gateway-hpa -n foxhunt -# # Should scale down to minReplicas - ---- -# ============================================================================ -# TROUBLESHOOTING -# ============================================================================ - -# Issue: HPA shows "unknown" for metrics -# Solution: Ensure metrics-server is installed and running -# kubectl get deployment metrics-server -n kube-system -# kubectl logs -n kube-system deployment/metrics-server -# -# Issue: Custom metrics not available -# Solution: Verify Prometheus Adapter configuration -# kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq . -# kubectl logs -n monitoring deployment/prometheus-adapter -# -# Issue: HPA not scaling up despite high CPU -# Solution: Check stabilizationWindowSeconds (may be waiting) -# kubectl describe hpa -n foxhunt -# # Look for "ScalingLimited" or "BackoffBoth" events -# -# Issue: HPA scaling too aggressively (flapping) -# Solution: Increase stabilizationWindowSeconds or reduce scale-up policies -# -# Issue: Pods evicted due to resource pressure -# Solution: Increase resource requests/limits in Deployment spec -# kubectl describe pod -n foxhunt -# # Look for "Evicted" status - ---- -# ============================================================================ -# PRODUCTION BEST PRACTICES -# ============================================================================ - -# 1. Monitor HPA metrics: -# - Current replicas vs desired replicas -# - Scaling events (up/down) -# - Metric values (CPU, memory, custom) -# -# 2. Alert on scaling issues: -# - HPA unable to scale (resource quotas exceeded) -# - HPA scaling too frequently (flapping) -# - Pods in CrashLoopBackOff after scale-up -# -# 3. Test scaling under load: -# - Run load tests regularly (weekly/monthly) -# - Verify scale-up happens within SLA (< 2 minutes) -# - Verify scale-down doesn't cause service disruption -# -# 4. Cost optimization: -# - Use Reserved Instances for minReplicas (50% savings) -# - Use Spot Instances for auto-scaled replicas (70% savings, non-critical workloads) -# - Monitor auto-scaling costs (CloudWatch/Prometheus) -# -# 5. Resource limits: -# - Set resource requests = expected usage -# - Set resource limits = 2x requests (headroom for bursts) -# - Monitor actual usage and adjust - ---- -# ============================================================================ -# PRODUCTION CHECKLIST -# ============================================================================ - -# [ ] Install metrics-server (for CPU/memory metrics) -# [ ] Install Prometheus Adapter (for custom metrics) -# [ ] Configure custom metrics (request rate, queue depth, GPU util) -# [ ] Deploy HPA manifests (kubectl apply -f hpa.yaml) -# [ ] Verify HPA status (kubectl get hpa) -# [ ] Run load test to validate scaling behavior -# [ ] Configure Grafana dashboards (HPA metrics, scaling events) -# [ ] Configure alerts (scaling issues, resource pressure) -# [ ] Document scaling thresholds and replica limits -# [ ] Review and adjust HPA configuration quarterly diff --git a/k8s/deployments/api-gateway-deployment.yaml b/k8s/deployments/api-gateway-deployment.yaml deleted file mode 100644 index 0780dc3ee..000000000 --- a/k8s/deployments/api-gateway-deployment.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: api-gateway - namespace: foxhunt - labels: - app: api-gateway - component: gateway - tier: frontend - version: v1 -spec: - replicas: 3 - revisionHistoryLimit: 10 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - selector: - matchLabels: - app: api-gateway - template: - metadata: - labels: - app: api-gateway - component: gateway - tier: frontend - version: v1 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9091" - prometheus.io/path: "/metrics" - spec: - serviceAccountName: foxhunt-api-gateway - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: api-gateway - image: foxhunt/api-gateway:latest - imagePullPolicy: Always - ports: - - name: grpc - containerPort: 50051 - protocol: TCP - - name: metrics - containerPort: 9091 - protocol: TCP - - name: health - containerPort: 8080 - protocol: TCP - env: - - name: API_GATEWAY_HOST - value: "0.0.0.0" - - name: API_GATEWAY_PORT - value: "50051" - - name: METRICS_PORT - value: "9091" - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: jwt-secret - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: database-url - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: redis-url - - name: RUST_LOG - value: "info,api_gateway=debug" - - name: RUST_BACKTRACE - value: "1" - resources: - requests: - cpu: "1000m" - memory: "512Mi" - limits: - cpu: "2000m" - memory: "1Gi" - livenessProbe: - httpGet: - path: /health/liveness - port: 8080 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /health/readiness - port: 8080 - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 3 - startupProbe: - httpGet: - path: /health/startup - port: 8080 - initialDelaySeconds: 0 - periodSeconds: 5 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 30 - volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - - name: logs - mountPath: /app/logs - volumes: - - name: config - configMap: - name: api-gateway-config - - name: logs - emptyDir: {} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - api-gateway - topologyKey: kubernetes.io/hostname diff --git a/k8s/deployments/backtesting-service-deployment.yaml b/k8s/deployments/backtesting-service-deployment.yaml deleted file mode 100644 index b15e087ab..000000000 --- a/k8s/deployments/backtesting-service-deployment.yaml +++ /dev/null @@ -1,143 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: backtesting-service - namespace: foxhunt - labels: - app: backtesting-service - component: backtesting - tier: backend - version: v1 -spec: - replicas: 2 - revisionHistoryLimit: 10 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - selector: - matchLabels: - app: backtesting-service - template: - metadata: - labels: - app: backtesting-service - component: backtesting - tier: backend - version: v1 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9093" - prometheus.io/path: "/metrics" - spec: - serviceAccountName: foxhunt-backtesting-service - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: backtesting-service - image: foxhunt/backtesting-service:latest - imagePullPolicy: Always - ports: - - name: grpc - containerPort: 50053 - protocol: TCP - - name: metrics - containerPort: 9093 - protocol: TCP - - name: health - containerPort: 8082 - protocol: TCP - env: - - name: BACKTESTING_SERVICE_HOST - value: "0.0.0.0" - - name: BACKTESTING_SERVICE_PORT - value: "50053" - - name: METRICS_PORT - value: "9093" - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: database-url - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: redis-url - - name: VAULT_ADDR - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: vault-addr - - name: VAULT_TOKEN - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: vault-token - - name: RUST_LOG - value: "info,backtesting_service=debug" - - name: RUST_BACKTRACE - value: "1" - resources: - requests: - cpu: "1000m" - memory: "1Gi" - limits: - cpu: "2000m" - memory: "2Gi" - livenessProbe: - httpGet: - path: /health - port: 8082 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8082 - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 3 - volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - - name: logs - mountPath: /app/logs - - name: data - mountPath: /app/data - - name: results - mountPath: /app/results - volumes: - - name: config - configMap: - name: backtesting-service-config - - name: logs - emptyDir: {} - - name: data - persistentVolumeClaim: - claimName: backtesting-data-pvc - - name: results - persistentVolumeClaim: - claimName: backtesting-results-pvc - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - backtesting-service - topologyKey: kubernetes.io/hostname diff --git a/k8s/deployments/ml-training-service-deployment.yaml b/k8s/deployments/ml-training-service-deployment.yaml deleted file mode 100644 index 110887df3..000000000 --- a/k8s/deployments/ml-training-service-deployment.yaml +++ /dev/null @@ -1,153 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ml-training-service - namespace: foxhunt - labels: - app: ml-training-service - component: ml-training - tier: backend - version: v1 -spec: - replicas: 1 - revisionHistoryLimit: 10 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - selector: - matchLabels: - app: ml-training-service - template: - metadata: - labels: - app: ml-training-service - component: ml-training - tier: backend - version: v1 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9094" - prometheus.io/path: "/metrics" - spec: - serviceAccountName: foxhunt-ml-training-service - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: ml-training-service - image: foxhunt/ml-training-service:latest - imagePullPolicy: Always - ports: - - name: grpc - containerPort: 50054 - protocol: TCP - - name: metrics - containerPort: 9094 - protocol: TCP - - name: health - containerPort: 8095 - protocol: TCP - env: - - name: ML_TRAINING_SERVICE_HOST - value: "0.0.0.0" - - name: ML_TRAINING_SERVICE_PORT - value: "50054" - - name: METRICS_PORT - value: "9094" - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: database-url - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: redis-url - - name: VAULT_ADDR - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: vault-addr - - name: VAULT_TOKEN - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: vault-token - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: aws-secret-access-key - - name: AWS_REGION - value: "us-east-1" - - name: S3_MODEL_BUCKET - value: "foxhunt-models" - - name: RUST_LOG - value: "info,ml_training_service=debug" - - name: RUST_BACKTRACE - value: "1" - resources: - requests: - cpu: "2000m" - memory: "4Gi" - nvidia.com/gpu: 1 - limits: - cpu: "4000m" - memory: "8Gi" - nvidia.com/gpu: 1 - livenessProbe: - httpGet: - path: /health - port: 8095 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8095 - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 3 - volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - - name: logs - mountPath: /app/logs - - name: models - mountPath: /app/models - - name: checkpoints - mountPath: /app/checkpoints - volumes: - - name: config - configMap: - name: ml-training-service-config - - name: logs - emptyDir: {} - - name: models - persistentVolumeClaim: - claimName: ml-models-pvc - - name: checkpoints - persistentVolumeClaim: - claimName: ml-checkpoints-pvc - nodeSelector: - nvidia.com/gpu: "true" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule diff --git a/k8s/deployments/trading-agent-service-deployment.yaml b/k8s/deployments/trading-agent-service-deployment.yaml deleted file mode 100644 index 02390bb4a..000000000 --- a/k8s/deployments/trading-agent-service-deployment.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trading-agent-service - namespace: foxhunt - labels: - app: trading-agent-service - component: trading-agent - tier: backend - version: v1 -spec: - replicas: 2 - revisionHistoryLimit: 10 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - selector: - matchLabels: - app: trading-agent-service - template: - metadata: - labels: - app: trading-agent-service - component: trading-agent - tier: backend - version: v1 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9095" - prometheus.io/path: "/metrics" - spec: - serviceAccountName: foxhunt-trading-agent-service - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: trading-agent-service - image: foxhunt/trading-agent-service:latest - imagePullPolicy: Always - ports: - - name: grpc - containerPort: 50055 - protocol: TCP - - name: metrics - containerPort: 9095 - protocol: TCP - - name: health - containerPort: 8083 - protocol: TCP - env: - - name: TRADING_AGENT_SERVICE_HOST - value: "0.0.0.0" - - name: TRADING_AGENT_SERVICE_PORT - value: "50055" - - name: METRICS_PORT - value: "9095" - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: database-url - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: redis-url - - name: VAULT_ADDR - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: vault-addr - - name: VAULT_TOKEN - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: vault-token - - name: RUST_LOG - value: "info,trading_agent_service=debug" - - name: RUST_BACKTRACE - value: "1" - resources: - requests: - cpu: "1000m" - memory: "1Gi" - limits: - cpu: "2000m" - memory: "2Gi" - livenessProbe: - httpGet: - path: /health - port: 8083 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8083 - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 3 - volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - - name: logs - mountPath: /app/logs - - name: data - mountPath: /app/data - volumes: - - name: config - configMap: - name: trading-agent-service-config - - name: logs - emptyDir: {} - - name: data - emptyDir: {} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - trading-agent-service - topologyKey: kubernetes.io/hostname diff --git a/k8s/deployments/trading-service-deployment.yaml b/k8s/deployments/trading-service-deployment.yaml deleted file mode 100644 index a9b3957a4..000000000 --- a/k8s/deployments/trading-service-deployment.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trading-service - namespace: foxhunt - labels: - app: trading-service - component: trading - tier: backend - version: v1 -spec: - replicas: 2 - revisionHistoryLimit: 10 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 1 - selector: - matchLabels: - app: trading-service - template: - metadata: - labels: - app: trading-service - component: trading - tier: backend - version: v1 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9092" - prometheus.io/path: "/metrics" - spec: - serviceAccountName: foxhunt-trading-service - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: trading-service - image: foxhunt/trading-service:latest - imagePullPolicy: Always - ports: - - name: grpc - containerPort: 50052 - protocol: TCP - - name: metrics - containerPort: 9092 - protocol: TCP - - name: health - containerPort: 8081 - protocol: TCP - env: - - name: TRADING_SERVICE_HOST - value: "0.0.0.0" - - name: TRADING_SERVICE_PORT - value: "50052" - - name: METRICS_PORT - value: "9092" - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: database-url - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: redis-url - - name: VAULT_ADDR - valueFrom: - configMapKeyRef: - name: foxhunt-config - key: vault-addr - - name: VAULT_TOKEN - valueFrom: - secretKeyRef: - name: foxhunt-secrets - key: vault-token - - name: RUST_LOG - value: "info,trading_service=debug" - - name: RUST_BACKTRACE - value: "1" - resources: - requests: - cpu: "2000m" - memory: "2Gi" - limits: - cpu: "4000m" - memory: "4Gi" - livenessProbe: - httpGet: - path: /health - port: 8081 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: 8081 - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 3 - volumeMounts: - - name: config - mountPath: /app/config - readOnly: true - - name: logs - mountPath: /app/logs - - name: data - mountPath: /app/data - volumes: - - name: config - configMap: - name: trading-service-config - - name: logs - emptyDir: {} - - name: data - emptyDir: {} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - trading-service - topologyKey: kubernetes.io/hostname diff --git a/k8s/foxhunt-complete.yaml b/k8s/foxhunt-complete.yaml deleted file mode 100644 index 5c2cb2c80..000000000 --- a/k8s/foxhunt-complete.yaml +++ /dev/null @@ -1,431 +0,0 @@ -# Foxhunt HFT Trading System - Complete Kubernetes Deployment -# Wave 5 Production Deployment Infrastructure -# -# This manifest includes: -# - Namespace -# - ConfigMaps (3) -# - Secrets (2) -# - PersistentVolumeClaims (5) -# - Services (5) -# - HorizontalPodAutoscalers (3) -# -# Usage: kubectl apply -f k8s/foxhunt-complete.yaml - ---- -# Namespace -apiVersion: v1 -kind: Namespace -metadata: - name: foxhunt - labels: - name: foxhunt - environment: production - ---- -# ConfigMap: Main configuration -apiVersion: v1 -kind: ConfigMap -metadata: - name: foxhunt-config - namespace: foxhunt -data: - redis-url: "redis://redis:6379" - vault-addr: "http://vault:8200" - postgres-host: "postgres" - postgres-port: "5432" - postgres-db: "foxhunt" - ---- -# ConfigMap: Prometheus configuration -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-config - namespace: foxhunt -data: - prometheus.yml: | - global: - scrape_interval: 15s - evaluation_interval: 15s - scrape_configs: - - job_name: 'api-gateway' - static_configs: - - targets: ['api-gateway:9091'] - - job_name: 'trading-service' - static_configs: - - targets: ['trading-service:9092'] - - job_name: 'backtesting-service' - static_configs: - - targets: ['backtesting-service:9093'] - - job_name: 'ml-training-service' - static_configs: - - targets: ['ml-training-service:9094'] - - job_name: 'trading-agent-service' - static_configs: - - targets: ['trading-agent-service:9095'] - ---- -# ConfigMap: Redis configuration -apiVersion: v1 -kind: ConfigMap -metadata: - name: redis-config - namespace: foxhunt -data: - redis.conf: | - maxmemory 256mb - maxmemory-policy allkeys-lru - appendonly yes - save 900 1 - save 300 10 - save 60 10000 - ---- -# Secret: Database credentials (base64 encoded) -# IMPORTANT: Replace with actual values from Vault in production -apiVersion: v1 -kind: Secret -metadata: - name: foxhunt-secrets - namespace: foxhunt -type: Opaque -stringData: - jwt-secret: "REPLACE_WITH_VAULT_VALUE" - database-url: "postgresql://foxhunt:REPLACE_PASSWORD@postgres:5432/foxhunt" - vault-token: "REPLACE_WITH_VAULT_TOKEN" - aws-access-key-id: "REPLACE_WITH_AWS_KEY" - aws-secret-access-key: "REPLACE_WITH_AWS_SECRET" - ---- -# Secret: Vault credentials -apiVersion: v1 -kind: Secret -metadata: - name: vault-secrets - namespace: foxhunt -type: Opaque -stringData: - vault-root-token: "REPLACE_WITH_VAULT_ROOT_TOKEN" - vault-unseal-key: "REPLACE_WITH_VAULT_UNSEAL_KEY" - ---- -# PVC: PostgreSQL data -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgres-pvc - namespace: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Gi - storageClassName: fast-ssd - ---- -# PVC: Redis data -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: redis-pvc - namespace: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: fast-ssd - ---- -# PVC: Prometheus data -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: prometheus-pvc - namespace: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi - storageClassName: standard - ---- -# PVC: ML Models -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ml-models-pvc - namespace: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi - storageClassName: fast-ssd - ---- -# PVC: ML Checkpoints -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ml-checkpoints-pvc - namespace: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Gi - storageClassName: fast-ssd - ---- -# PVC: Backtesting Data -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: backtesting-data-pvc - namespace: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 200Gi - storageClassName: standard - ---- -# PVC: Backtesting Results -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: backtesting-results-pvc - namespace: foxhunt -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi - storageClassName: standard - ---- -# Service: Backtesting Service -apiVersion: v1 -kind: Service -metadata: - name: backtesting-service - namespace: foxhunt - labels: - app: backtesting-service - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9093" -spec: - type: ClusterIP - selector: - app: backtesting-service - ports: - - name: grpc - port: 50053 - targetPort: 50053 - protocol: TCP - - name: metrics - port: 9093 - targetPort: 9093 - protocol: TCP - - name: health - port: 8082 - targetPort: 8082 - protocol: TCP - ---- -# Service: ML Training Service -apiVersion: v1 -kind: Service -metadata: - name: ml-training-service - namespace: foxhunt - labels: - app: ml-training-service - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9094" -spec: - type: ClusterIP - selector: - app: ml-training-service - ports: - - name: grpc - port: 50054 - targetPort: 50054 - protocol: TCP - - name: metrics - port: 9094 - targetPort: 9094 - protocol: TCP - - name: health - port: 8095 - targetPort: 8095 - protocol: TCP - ---- -# Service: Trading Agent Service -apiVersion: v1 -kind: Service -metadata: - name: trading-agent-service - namespace: foxhunt - labels: - app: trading-agent-service - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9095" -spec: - type: ClusterIP - selector: - app: trading-agent-service - ports: - - name: grpc - port: 50055 - targetPort: 50055 - protocol: TCP - - name: metrics - port: 9095 - targetPort: 9095 - protocol: TCP - - name: health - port: 8083 - targetPort: 8083 - protocol: TCP - ---- -# HPA: API Gateway -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: api-gateway-hpa - namespace: foxhunt -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: api-gateway - minReplicas: 3 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 50 - periodSeconds: 60 - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Percent - value: 100 - periodSeconds: 30 - - type: Pods - value: 2 - periodSeconds: 30 - selectPolicy: Max - ---- -# HPA: Trading Service -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: trading-service-hpa - namespace: foxhunt -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: trading-service - minReplicas: 2 - maxReplicas: 5 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 80 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 85 - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Pods - value: 1 - periodSeconds: 120 - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Pods - value: 1 - periodSeconds: 60 - ---- -# HPA: ML Training Service -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: ml-training-service-hpa - namespace: foxhunt -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: ml-training-service - minReplicas: 1 - maxReplicas: 3 - metrics: - - type: Resource - resource: - name: nvidia.com/gpu - target: - type: Utilization - averageUtilization: 90 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 85 - behavior: - scaleDown: - stabilizationWindowSeconds: 600 - policies: - - type: Pods - value: 1 - periodSeconds: 300 - scaleUp: - stabilizationWindowSeconds: 60 - policies: - - type: Pods - value: 1 - periodSeconds: 120 diff --git a/k8s/services/api-gateway-service.yaml b/k8s/services/api-gateway-service.yaml deleted file mode 100644 index 9d779eecd..000000000 --- a/k8s/services/api-gateway-service.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: api-gateway - namespace: foxhunt - labels: - app: api-gateway - component: gateway - tier: frontend - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9091" -spec: - type: LoadBalancer - sessionAffinity: ClientIP - sessionAffinityConfig: - clientIP: - timeoutSeconds: 10800 - selector: - app: api-gateway - ports: - - name: grpc - port: 50051 - targetPort: 50051 - protocol: TCP - - name: metrics - port: 9091 - targetPort: 9091 - protocol: TCP - - name: health - port: 8080 - targetPort: 8080 - protocol: TCP diff --git a/k8s/services/trading-service.yaml b/k8s/services/trading-service.yaml deleted file mode 100644 index 323f72aef..000000000 --- a/k8s/services/trading-service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: trading-service - namespace: foxhunt - labels: - app: trading-service - component: trading - tier: backend - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9092" -spec: - type: ClusterIP - selector: - app: trading-service - ports: - - name: grpc - port: 50052 - targetPort: 50052 - protocol: TCP - - name: metrics - port: 9092 - targetPort: 9092 - protocol: TCP - - name: health - port: 8081 - targetPort: 8081 - protocol: TCP diff --git a/k8s/trading-service.yaml b/k8s/trading-service.yaml deleted file mode 100644 index e3fb91d7a..000000000 --- a/k8s/trading-service.yaml +++ /dev/null @@ -1,233 +0,0 @@ -# ============================================================================= -# FOXHUNT TRADING SERVICE - ULTRA-PERFORMANCE KUBERNETES DEPLOYMENT -# ============================================================================= -# This manifest deploys the Trading Service with specialized performance -# optimizations for 14ns latency requirements - -apiVersion: v1 -kind: Namespace -metadata: - name: foxhunt-trading - labels: - app.kubernetes.io/name: foxhunt - app.kubernetes.io/component: trading - performance-tier: ultra-high - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trading-service - namespace: foxhunt-trading - labels: - app: trading-service - version: v1 - performance-tier: ultra-high -spec: - replicas: 2 # Active-standby for HA - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 # Zero-downtime deployment - selector: - matchLabels: - app: trading-service - template: - metadata: - labels: - app: trading-service - version: v1 - performance-tier: ultra-high - annotations: - # Performance annotations - scheduler.alpha.kubernetes.io/critical-pod: "true" - cluster-autoscaler.kubernetes.io/safe-to-evict: "false" - spec: - # ULTRA-PERFORMANCE NODE AFFINITY - nodeSelector: - performance-tier: ultra-high - workload-type: trading - - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: performance-tier - operator: In - values: ["ultra-high"] - - key: kubernetes.io/arch - operator: In - values: ["amd64"] - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: ["trading-service"] - topologyKey: kubernetes.io/hostname - - # PERFORMANCE-CRITICAL CONFIGURATION - hostNetwork: true # Direct host networking for minimal latency - dnsPolicy: ClusterFirstWithHostNet - - # PRIVILEGED ACCESS FOR HARDWARE OPTIMIZATION - securityContext: - runAsNonRoot: false - runAsUser: 0 - fsGroup: 0 - - # PRIORITY AND PREEMPTION - priorityClassName: system-critical - - # RESOURCE TOPOLOGY - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app: trading-service - - containers: - - name: trading-service - image: foxhunt/trading-service:latest - imagePullPolicy: Always - - # ULTRA-PERFORMANCE SECURITY CONTEXT - securityContext: - privileged: true # Required for RDTSC hardware access - allowPrivilegeEscalation: true - readOnlyRootFilesystem: false - capabilities: - add: - - SYS_NICE # CPU scheduling priority - - SYS_TIME # High-resolution timers - - SYS_RAWIO # Raw I/O operations - - NET_RAW # Raw network access - - IPC_LOCK # Memory locking - - SYS_RESOURCE # Resource limits - - # CPU AFFINITY AND RESOURCE LIMITS - resources: - requests: - memory: "8Gi" - cpu: "4" - hugepages-2Mi: "2Gi" - limits: - memory: "8Gi" # Hard limit to prevent swap - cpu: "4" # Dedicated CPU cores - hugepages-2Mi: "2Gi" - - # PERFORMANCE-CRITICAL ENVIRONMENT - env: - - name: RUST_LOG - value: "error" # Minimal logging for performance - - name: MALLOC_CONF - value: "background_thread:false,dirty_decay_ms:0,muzzy_decay_ms:0" - - name: CPU_AFFINITY_CORES - value: "0,1,2,3" # Pinned to specific cores - - name: GOMAXPROCS - value: "4" - - name: OMP_NUM_THREADS - value: "4" - - # VOLUME MOUNTS FOR PERFORMANCE - volumeMounts: - - name: hugepages - mountPath: /dev/hugepages - - name: proc - mountPath: /host/proc - readOnly: true - - # gRPC PORT - ports: - - containerPort: 50051 - name: grpc - protocol: TCP - - # KUBERNETES PROBES (lightweight for performance) - livenessProbe: - tcpSocket: - port: 50051 - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 3 - - readinessProbe: - tcpSocket: - port: 50051 - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 1 - - # PERFORMANCE-OPTIMIZED VOLUMES - volumes: - - name: hugepages - emptyDir: - medium: HugePages-2Mi - - name: proc - hostPath: - path: /proc - type: Directory - - # SCHEDULING CONSTRAINTS - tolerations: - - key: "performance-tier" - operator: "Equal" - value: "ultra-high" - effect: "NoSchedule" - - key: "node.kubernetes.io/not-ready" - operator: "Exists" - effect: "NoExecute" - tolerationSeconds: 30 - ---- -apiVersion: v1 -kind: Service -metadata: - name: trading-service - namespace: foxhunt-trading - labels: - app: trading-service - annotations: - # Performance service annotations - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp" -spec: - type: ClusterIP # Internal service only - sessionAffinity: ClientIP # Session stickiness for performance - selector: - app: trading-service - ports: - - name: grpc - port: 50051 - targetPort: 50051 - protocol: TCP - ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: trading-service-pdb - namespace: foxhunt-trading -spec: - minAvailable: 1 # Always keep at least one instance running - selector: - matchLabels: - app: trading-service - ---- -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: system-critical -value: 1000000 -globalDefault: false -description: "Ultra-high priority for critical trading services" \ No newline at end of file