Files
foxhunt/infra/k8s/network-policies/prometheus.yaml
jgrusewski 8ce29df5cd fix(netpol): fix 3 DOWN Prometheus targets — operator port, reloader ingress, argo label
- Prometheus→operator: egress port 8443→10250 (actual container port)
- Alertmanager ingress: add port 8080 from Prometheus for config-reloader sidecar
- Argo controller metrics service: remove part-of:foxhunt label (already scraped
  via additionalScrapeConfigs with HTTPS; foxhunt-services ServiceMonitor was
  creating duplicate HTTP target → 400)

Result: 25/25 targets UP, 0 DOWN.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-14 12:52:07 +01:00

189 lines
4.7 KiB
YAML

# Prometheus egress: K8s API (service discovery), foxhunt services, node-exporter,
# kube-state-metrics, DCGM exporter, Argo workflow controller, pushgateway,
# Loki, Tempo, QuestDB, and AlertManager.
# Applies to operator-managed Prometheus (kube-prometheus-stack).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: prometheus
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
policyTypes:
- Ingress
- Egress
ingress:
# Grafana queries Prometheus on port 9090
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: grafana
ports:
- protocol: TCP
port: 9090
egress:
# DNS resolution (required for additionalScrapeConfigs with hostnames)
- to:
- namespaceSelector: {}
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
# Kubernetes API server (required for kubernetes_sd_configs service discovery)
# ClusterIP 10.32.0.1:443 NATs to node IP 172.16.0.0/16:6443
- to:
- ipBlock:
cidr: 10.32.0.0/16
ports:
- protocol: TCP
port: 443
- protocol: TCP
port: 6443
# Scrape foxhunt service metrics ports (9003, 9090-9098)
- to:
- podSelector:
matchExpressions:
- key: app.kubernetes.io/part-of
operator: In
values: ["foxhunt", "foxhunt-ci"]
ports:
- protocol: TCP
port: 9003
- protocol: TCP
port: 9090
- protocol: TCP
port: 9091
- protocol: TCP
port: 9092
- protocol: TCP
port: 9093
- protocol: TCP
port: 9094
- protocol: TCP
port: 9095
- protocol: TCP
port: 9096
- protocol: TCP
port: 9097
- protocol: TCP
port: 9098
# kube-state-metrics (Helm sub-chart, port 8080)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: kube-state-metrics
ports:
- protocol: TCP
port: 8080
# Prometheus operator (container port 10250)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: kube-prometheus-stack-prometheus-operator
ports:
- protocol: TCP
port: 10250
# Alertmanager (port 9093 for API, port 8080 for config-reloader sidecar)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: alertmanager
ports:
- protocol: TCP
port: 9093
- protocol: TCP
port: 8080
# Loki metrics (port 3100) and Tempo metrics (port 3200)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: loki
ports:
- protocol: TCP
port: 3100
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: tempo
ports:
- protocol: TCP
port: 3200
# QuestDB metrics
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: questdb
ports:
- protocol: TCP
port: 9003
# Node-exporter (hostNetwork), kubelet, and K8s API on node IPs
- to:
- ipBlock:
cidr: 172.16.0.0/16
ports:
- protocol: TCP
port: 6443
- protocol: TCP
port: 9100
- protocol: TCP
port: 9400
- protocol: TCP
port: 10250 # kubelet metrics (cAdvisor, resource usage)
# kube-system: CoreDNS metrics, kube-state-metrics, node-exporter, DCGM
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: TCP
port: 8080
- protocol: TCP
port: 8443
- protocol: TCP
port: 9100
- protocol: TCP
port: 9153 # CoreDNS metrics
- protocol: TCP
port: 9400
# monitoring namespace: DCGM exporter (GPU metrics)
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: monitoring
ports:
- protocol: TCP
port: 9400
# Pushgateway (training pods push metrics here)
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: pushgateway
ports:
- protocol: TCP
port: 9091
# Argo Workflows controller metrics
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: argo-workflows-workflow-controller
ports:
- protocol: TCP
port: 9090