c384ab4b43
Prometheus was configured to scrape pods via prometheus.io/scrape annotations, but all services and training jobs used gitlab.com/ prefix from legacy GitLab-managed Prometheus — causing Prometheus to never discover any foxhunt pods. This resulted in stale/missing metrics on the Grafana training dashboard. 12 files updated across services/, gpu-overlays/, training/, and monitoring/ (node-exporter, dcgm-exporter cleanup). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
106 lines
2.9 KiB
YAML
106 lines
2.9 KiB
YAML
# Prometheus Node Exporter — host-level metrics (CPU, memory, disk, network)
|
|
# Replaces k8s-monitoring chart's node-exporter after Cockpit removal
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: node-exporter
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: node-exporter
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: node-exporter
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: node-exporter
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: node-exporter
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxUnavailable: 1
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: node-exporter
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "9100"
|
|
prometheus.io/path: "/metrics"
|
|
spec:
|
|
serviceAccountName: node-exporter
|
|
hostPID: true
|
|
hostNetwork: true
|
|
tolerations:
|
|
- effect: NoSchedule
|
|
operator: Exists
|
|
containers:
|
|
- name: node-exporter
|
|
image: quay.io/prometheus/node-exporter:v1.9.0
|
|
args:
|
|
- --path.procfs=/host/proc
|
|
- --path.sysfs=/host/sys
|
|
- --path.rootfs=/host/root
|
|
- --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
|
|
- --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
|
|
- --web.listen-address=:9100
|
|
ports:
|
|
- containerPort: 9100
|
|
name: metrics
|
|
hostPort: 9100
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
securityContext:
|
|
readOnlyRootFilesystem: true
|
|
allowPrivilegeEscalation: false
|
|
volumeMounts:
|
|
- name: proc
|
|
mountPath: /host/proc
|
|
readOnly: true
|
|
- name: sys
|
|
mountPath: /host/sys
|
|
readOnly: true
|
|
- name: root
|
|
mountPath: /host/root
|
|
readOnly: true
|
|
mountPropagation: HostToContainer
|
|
volumes:
|
|
- name: proc
|
|
hostPath:
|
|
path: /proc
|
|
- name: sys
|
|
hostPath:
|
|
path: /sys
|
|
- name: root
|
|
hostPath:
|
|
path: /
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: node-exporter
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: node-exporter
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "9100"
|
|
spec:
|
|
type: ClusterIP
|
|
clusterIP: None
|
|
selector:
|
|
app.kubernetes.io/name: node-exporter
|
|
ports:
|
|
- port: 9100
|
|
targetPort: 9100
|
|
name: metrics
|