f3012611f0
Replace standalone Prometheus/node-exporter/kube-state-metrics with kube-prometheus-stack Helm chart. Add ServiceMonitor CRDs, PrometheusRule CRDs (HFT + broker alerts), comprehensive network policies, and wire AlertManager to Mattermost via incoming webhook. - Delete old standalone prometheus.yaml, node-exporter.yaml, kube-state-metrics.yaml - Add prometheus-stack-values.yaml (Helm values with Scaleway Kapsule tuning) - Add service-monitors.yaml (foxhunt-services + dcgm-exporter ServiceMonitors) - Add prometheus-rules-hft.yaml and prometheus-rules-broker.yaml (PrometheusRule CRDs) - Rewrite prometheus network policy for operator-managed pods (DNS, kube-state-metrics, operator, alertmanager sidecar, monitoring namespace for DCGM) - Add alertmanager network policy (Mattermost egress) - Upgrade Grafana: AlertManager datasource, unified alerting, fix nodeSelector - Fix Tempo: tolerations for GPU nodes, resource tuning Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
140 lines
3.0 KiB
YAML
140 lines
3.0 KiB
YAML
# Grafana Tempo — lightweight distributed tracing (single-binary mode)
|
|
# Receives OTLP traces from services, queryable via Grafana
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: tempo-config
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: tempo
|
|
data:
|
|
tempo.yaml: |
|
|
server:
|
|
http_listen_port: 3200
|
|
distributor:
|
|
receivers:
|
|
otlp:
|
|
protocols:
|
|
grpc:
|
|
endpoint: "0.0.0.0:4317"
|
|
http:
|
|
endpoint: "0.0.0.0:4318"
|
|
storage:
|
|
trace:
|
|
backend: local
|
|
local:
|
|
path: /var/tempo/traces
|
|
wal:
|
|
path: /var/tempo/wal
|
|
compactor:
|
|
compaction:
|
|
block_retention: 168h # 7 days
|
|
metrics_generator:
|
|
registry:
|
|
external_labels:
|
|
source: tempo
|
|
cluster: foxhunt
|
|
storage:
|
|
path: /var/tempo/generator/wal
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: tempo
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: tempo
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: tempo
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: tempo
|
|
spec:
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
securityContext:
|
|
fsGroup: 10001
|
|
runAsUser: 10001
|
|
runAsNonRoot: true
|
|
containers:
|
|
- name: tempo
|
|
image: grafana/tempo:2.7.1
|
|
args: ["-config.file=/etc/tempo/tempo.yaml"]
|
|
ports:
|
|
- containerPort: 3200
|
|
name: http
|
|
- containerPort: 4317
|
|
name: otlp-grpc
|
|
- containerPort: 4318
|
|
name: otlp-http
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/tempo
|
|
- name: data
|
|
mountPath: /var/tempo
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 2Gi
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /ready
|
|
port: 3200
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /ready
|
|
port: 3200
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 15
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: tempo-config
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: tempo-data
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: tempo-data
|
|
namespace: foxhunt
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: scw-bssd
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: tempo
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: tempo
|
|
spec:
|
|
selector:
|
|
app.kubernetes.io/name: tempo
|
|
ports:
|
|
- port: 3200
|
|
targetPort: 3200
|
|
name: http
|
|
- port: 4317
|
|
targetPort: 4317
|
|
name: otlp-grpc
|
|
- port: 4318
|
|
targetPort: 4318
|
|
name: otlp-http
|