- Prometheus→operator: egress port 8443→10250 (actual container port) - Alertmanager ingress: add port 8080 from Prometheus for config-reloader sidecar - Argo controller metrics service: remove part-of:foxhunt label (already scraped via additionalScrapeConfigs with HTTPS; foxhunt-services ServiceMonitor was creating duplicate HTTP target → 400) Result: 25/25 targets UP, 0 DOWN. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
189 lines
4.7 KiB
YAML
189 lines
4.7 KiB
YAML
# Prometheus egress: K8s API (service discovery), foxhunt services, node-exporter,
|
|
# kube-state-metrics, DCGM exporter, Argo workflow controller, pushgateway,
|
|
# Loki, Tempo, QuestDB, and AlertManager.
|
|
# Applies to operator-managed Prometheus (kube-prometheus-stack).
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: prometheus
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: prometheus
|
|
policyTypes:
|
|
- Ingress
|
|
- Egress
|
|
ingress:
|
|
# Grafana queries Prometheus on port 9090
|
|
- from:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: grafana
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9090
|
|
egress:
|
|
# DNS resolution (required for additionalScrapeConfigs with hostnames)
|
|
- to:
|
|
- namespaceSelector: {}
|
|
ports:
|
|
- protocol: UDP
|
|
port: 53
|
|
- protocol: TCP
|
|
port: 53
|
|
|
|
# Kubernetes API server (required for kubernetes_sd_configs service discovery)
|
|
# ClusterIP 10.32.0.1:443 NATs to node IP 172.16.0.0/16:6443
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 10.32.0.0/16
|
|
ports:
|
|
- protocol: TCP
|
|
port: 443
|
|
- protocol: TCP
|
|
port: 6443
|
|
|
|
# Scrape foxhunt service metrics ports (9003, 9090-9098)
|
|
- to:
|
|
- podSelector:
|
|
matchExpressions:
|
|
- key: app.kubernetes.io/part-of
|
|
operator: In
|
|
values: ["foxhunt", "foxhunt-ci"]
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9003
|
|
- protocol: TCP
|
|
port: 9090
|
|
- protocol: TCP
|
|
port: 9091
|
|
- protocol: TCP
|
|
port: 9092
|
|
- protocol: TCP
|
|
port: 9093
|
|
- protocol: TCP
|
|
port: 9094
|
|
- protocol: TCP
|
|
port: 9095
|
|
- protocol: TCP
|
|
port: 9096
|
|
- protocol: TCP
|
|
port: 9097
|
|
- protocol: TCP
|
|
port: 9098
|
|
|
|
# kube-state-metrics (Helm sub-chart, port 8080)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: kube-state-metrics
|
|
ports:
|
|
- protocol: TCP
|
|
port: 8080
|
|
|
|
# Prometheus operator (container port 10250)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator
|
|
ports:
|
|
- protocol: TCP
|
|
port: 10250
|
|
|
|
# Alertmanager (port 9093 for API, port 8080 for config-reloader sidecar)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: alertmanager
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9093
|
|
- protocol: TCP
|
|
port: 8080
|
|
|
|
# Loki metrics (port 3100) and Tempo metrics (port 3200)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: loki
|
|
ports:
|
|
- protocol: TCP
|
|
port: 3100
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: tempo
|
|
ports:
|
|
- protocol: TCP
|
|
port: 3200
|
|
|
|
# QuestDB metrics
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: questdb
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9003
|
|
|
|
# Node-exporter (hostNetwork), kubelet, and K8s API on node IPs
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 172.16.0.0/16
|
|
ports:
|
|
- protocol: TCP
|
|
port: 6443
|
|
- protocol: TCP
|
|
port: 9100
|
|
- protocol: TCP
|
|
port: 9400
|
|
- protocol: TCP
|
|
port: 10250 # kubelet metrics (cAdvisor, resource usage)
|
|
|
|
# kube-system: CoreDNS metrics, kube-state-metrics, node-exporter, DCGM
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: kube-system
|
|
ports:
|
|
- protocol: TCP
|
|
port: 8080
|
|
- protocol: TCP
|
|
port: 8443
|
|
- protocol: TCP
|
|
port: 9100
|
|
- protocol: TCP
|
|
port: 9153 # CoreDNS metrics
|
|
- protocol: TCP
|
|
port: 9400
|
|
|
|
# monitoring namespace: DCGM exporter (GPU metrics)
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: monitoring
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9400
|
|
|
|
# Pushgateway (training pods push metrics here)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: pushgateway
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9091
|
|
|
|
# Argo Workflows controller metrics
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: argo-workflows-workflow-controller
|
|
ports:
|
|
- protocol: TCP
|
|
port: 9090
|