# kube-prometheus-stack Helm values — Foxhunt production monitoring
#
# Install:
#   helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
#   helm repo update
#   helm install prometheus-stack prometheus-community/kube-prometheus-stack \
#     --namespace foxhunt \
#     -f infra/k8s/monitoring/prometheus-stack-values.yaml
#
# Upgrade:
#   helm upgrade prometheus-stack prometheus-community/kube-prometheus-stack \
#     --namespace foxhunt \
#     -f infra/k8s/monitoring/prometheus-stack-values.yaml
#
# Prerequisites:
#   kubectl -n foxhunt create secret generic alertmanager-slack \
#     --from-literal=slack-webhook-url='https://hooks.slack.com/services/...'
#
# After install, delete old standalone resources:
#   kubectl -n foxhunt delete deploy prometheus
#   kubectl -n foxhunt delete svc prometheus
#   kubectl -n foxhunt delete cm prometheus-config
#   kubectl -n foxhunt delete pvc prometheus-data
#   kubectl -n foxhunt delete sa prometheus
#   kubectl delete clusterrole foxhunt-prometheus
#   kubectl delete clusterrolebinding foxhunt-prometheus
#   kubectl -n foxhunt delete deploy node-exporter
#   kubectl -n foxhunt delete ds node-exporter  # if DaemonSet
#   kubectl -n foxhunt delete deploy kube-state-metrics

# ─────────────────────────────────────────────
# Prometheus Operator
# ─────────────────────────────────────────────
prometheusOperator:
  enabled: true
  nodeSelector:
    k8s.scaleway.com/pool-name: platform
  resources:
    requests:
      cpu: 50m
      memory: 64Mi
    limits:
      cpu: 200m
      memory: 128Mi

# ─────────────────────────────────────────────
# Prometheus
# ─────────────────────────────────────────────
prometheus:
  prometheusSpec:
    nodeSelector:
      k8s.scaleway.com/pool-name: platform
    podMetadata:
      labels:
        app.kubernetes.io/part-of: foxhunt
    retention: 14d
    retentionSize: 8GB
    replicas: 1
    resources:
      requests:
        cpu: 100m
        memory: 256Mi
      limits:
        cpu: 500m
        memory: 512Mi
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: scw-bssd
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 10Gi

    # Discover ALL ServiceMonitors/PodMonitors/PrometheusRules in the cluster,
    # not just those matching Helm release labels.
    serviceMonitorSelectorNilUsesHelmValues: false
    podMonitorSelectorNilUsesHelmValues: false
    ruleSelectorNilUsesHelmValues: false

    # Scrape configs for targets that don't fit ServiceMonitor/PodMonitor CRDs
    additionalScrapeConfigs:
      # Argo Workflows controller (HTTPS with self-signed cert)
      - job_name: argo-workflows
        scheme: https
        tls_config:
          insecure_skip_verify: true
        static_configs:
          - targets: ['argo-workflows-workflow-controller-metrics.foxhunt.svc.cluster.local:9090']
        relabel_configs:
          - target_label: service
            replacement: argo-workflows

      # Training workflow pods (ephemeral Argo pods on port 9094, Running only)
      - job_name: training-pods
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names: [foxhunt]
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
            action: keep
            regex: training-workflow|compile-and-train
          - source_labels: [__meta_kubernetes_pod_phase]
            action: keep
            regex: Running
          - source_labels: [__meta_kubernetes_pod_ip]
            target_label: __address__
            replacement: ${1}:9094
          - source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_label_workflows_argoproj_io_workflow]
            target_label: workflow

      # QuestDB (ML metrics store)
      - job_name: questdb
        metrics_path: /metrics
        static_configs:
          - targets: ['questdb.foxhunt.svc.cluster.local:9003']
        relabel_configs:
          - target_label: service
            replacement: questdb

      # Pushgateway (training job push metrics)
      - job_name: pushgateway
        honor_labels: true
        static_configs:
          - targets: ['pushgateway.foxhunt.svc.cluster.local:9091']

      # Loki metrics
      - job_name: loki
        static_configs:
          - targets: ['loki.foxhunt.svc.cluster.local:3100']

      # Tempo metrics
      - job_name: tempo
        static_configs:
          - targets: ['tempo.foxhunt.svc.cluster.local:3200']

# ─────────────────────────────────────────────
# AlertManager
# ─────────────────────────────────────────────
alertmanager:
  alertmanagerSpec:
    nodeSelector:
      k8s.scaleway.com/pool-name: platform
    replicas: 1
    resources:
      requests:
        cpu: 25m
        memory: 32Mi
      limits:
        cpu: 100m
        memory: 64Mi
    # Mount K8s Secrets into AlertManager at /etc/alertmanager/secrets/<name>/
    secrets:
      - alertmanager-mattermost

  config:
    global:
      # Mattermost incoming webhook (Slack-compatible API).
      # Reads URL from K8s Secret: alertmanager-mattermost/webhook-url
      # Create with:
      #   kubectl -n foxhunt create secret generic alertmanager-mattermost \
      #     --from-literal=webhook-url='http://mattermost.foxhunt.svc.cluster.local:8065/hooks/<HOOK_ID>'
      slack_api_url_file: /etc/alertmanager/secrets/alertmanager-mattermost/webhook-url

    route:
      receiver: default
      group_by: ['alertname', 'severity', 'service']
      group_wait: 5s
      group_interval: 30s
      repeat_interval: 4h

      routes:
        # CRITICAL trading/risk → immediate Slack + PagerDuty
        - matchers:
            - severity="critical"
            - component=~"trading_engine|risk_management|trading|risk"
          receiver: critical-trading
          group_wait: 0s
          group_interval: 5s
          repeat_interval: 5m
          continue: true

        # CRITICAL infrastructure → Slack critical channel
        - matchers:
            - severity="critical"
          receiver: critical-infra
          group_wait: 0s
          group_interval: 10s
          repeat_interval: 15m

        # HIGH priority → Slack alerts channel
        - matchers:
            - severity="high"
          receiver: high-priority
          group_wait: 10s
          group_interval: 1m
          repeat_interval: 30m

        # WARNING / MEDIUM → Slack digest
        - matchers:
            - severity=~"warning|WARNING|medium"
          receiver: warning-alerts
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 4h

    receivers:
      - name: default
        slack_configs:
          - channel: '#foxhunt-alerts'
            title: '{{ .GroupLabels.alertname }}'
            text: >-
              {{ range .Alerts }}*{{ .Labels.severity | toUpper }}*: {{ .Annotations.summary }}
              {{ end }}
            send_resolved: true

      - name: critical-trading
        slack_configs:
          - channel: '#foxhunt-critical'
            title: 'CRITICAL: {{ .GroupLabels.alertname }}'
            text: >-
              {{ range .Alerts }}
              *Alert*: {{ .Annotations.summary }}
              *Impact*: {{ .Annotations.impact }}
              *Action*: {{ .Annotations.action }}
              {{ end }}
            color: 'danger'
            send_resolved: true

      - name: critical-infra
        slack_configs:
          - channel: '#foxhunt-critical'
            title: 'INFRA CRITICAL: {{ .GroupLabels.alertname }}'
            text: >-
              {{ range .Alerts }}{{ .Annotations.summary }}
              {{ end }}
            color: 'danger'
            send_resolved: true

      - name: high-priority
        slack_configs:
          - channel: '#foxhunt-alerts'
            title: 'HIGH: {{ .GroupLabels.alertname }}'
            text: >-
              {{ range .Alerts }}{{ .Annotations.summary }}
              {{ end }}
            color: 'warning'
            send_resolved: true

      - name: warning-alerts
        slack_configs:
          - channel: '#foxhunt-alerts'
            title: 'WARNING: {{ .GroupLabels.alertname }}'
            text: >-
              {{ range .Alerts }}{{ .Annotations.summary }}
              {{ end }}
            color: '#ff9500'
            send_resolved: true

    inhibit_rules:
      # Trading engine down → suppress all trading alerts
      - source_matchers:
          - alertname="TradingEngineDown"
        target_matchers:
          - component=~"trading_engine|trading"
        equal: ['service']

      # Node down → suppress service-level alerts on that node
      - source_matchers:
          - alertname=~"NodeDown|KubeNodeNotReady"
        target_matchers:
          - severity=~"high|warning|medium"
        equal: ['instance']

      # PostgreSQL down → suppress database alerts
      - source_matchers:
          - alertname="PostgreSQLDown"
        target_matchers:
          - component="database"

# ─────────────────────────────────────────────
# ServiceMonitors
# ─────────────────────────────────────────────
additionalServiceMonitors:
  # All foxhunt microservices (7 services, all expose named port "metrics")
  - name: foxhunt-services
    selector:
      matchLabels:
        app.kubernetes.io/part-of: foxhunt
    namespaceSelector:
      matchNames: [foxhunt]
    endpoints:
      - port: metrics
        interval: 15s

  # DCGM Exporter (GPU metrics from kube-system)
  - name: dcgm-exporter
    selector:
      matchExpressions:
        - key: app.kubernetes.io/name
          operator: In
          values: [dcgm-exporter]
    namespaceSelector:
      matchNames: [kube-system, monitoring, foxhunt]
    endpoints:
      - port: metrics
        interval: 15s

# ─────────────────────────────────────────────
# Grafana (disabled — managed separately via grafana Helm chart)
# ─────────────────────────────────────────────
grafana:
  enabled: false

# ─────────────────────────────────────────────
# Node Exporter (replaces standalone node-exporter.yaml)
# ─────────────────────────────────────────────
prometheus-node-exporter:
  tolerations:
    - effect: NoSchedule
      operator: Exists
  resources:
    requests:
      cpu: 10m
      memory: 16Mi
    limits:
      cpu: 50m
      memory: 32Mi

# ─────────────────────────────────────────────
# kube-state-metrics (replaces standalone kube-state-metrics.yaml)
# ─────────────────────────────────────────────
kube-state-metrics:
  nodeSelector:
    k8s.scaleway.com/pool-name: platform
  resources:
    requests:
      cpu: 10m
      memory: 32Mi
    limits:
      cpu: 50m
      memory: 64Mi

# ─────────────────────────────────────────────
# Default rules (Kubernetes cluster health alerts included out-of-box)
# ─────────────────────────────────────────────
defaultRules:
  create: true
  rules:
    alertmanager: true
    etcd: false            # managed by Scaleway Kapsule
    configReloaders: true
    general: true
    k8sContainerCpuUsageSecondsTotal: true
    k8sContainerMemoryCache: true
    k8sContainerMemoryRss: true
    k8sContainerMemorySwap: true
    k8sContainerMemoryWorkingSetBytes: true
    k8sPodOwner: true
    kubeApiserverAvailability: false  # managed by Scaleway
    kubeApiserverBurnrate: false
    kubeApiserverHistogram: false
    kubeApiserverSlos: false
    kubeControllerManager: false      # managed by Scaleway
    kubelet: false              # kubelet metrics not accessible on Kapsule
    kubeProxy: false                  # managed by Scaleway
    kubePrometheusGeneral: true
    kubePrometheusNodeRecording: true
    kubernetesAbsent: true
    kubernetesApps: true
    kubernetesResources: true
    kubernetesStorage: true
    kubernetesSystem: true
    kubeScheduler: false              # managed by Scaleway
    kubeStateMetrics: true
    network: true
    node: true
    nodeExporterAlerting: true
    nodeExporterRecording: true
    prometheus: true
    prometheusOperator: true

# ─────────────────────────────────────────────
# Disable components managed by Scaleway Kapsule
# ─────────────────────────────────────────────
kubeApiServer:
  enabled: false
kubeControllerManager:
  enabled: false
kubeScheduler:
  enabled: false
kubeProxy:
  enabled: false
kubelet:
  enabled: false               # kubelet metrics not accessible on Kapsule
kubeEtcd:
  enabled: false