foxhunt/infra/k8s/monitoring/promtail.yaml

# Promtail — ships container logs to Loki (DaemonSet on all nodes)
apiVersion: v1
kind: ServiceAccount
metadata:
  name: promtail
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: promtail
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: promtail
  labels:
    app.kubernetes.io/name: promtail
rules:
  - apiGroups: [""]
    resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"]
    verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: promtail
  labels:
    app.kubernetes.io/name: promtail
subjects:
  - kind: ServiceAccount
    name: promtail
    namespace: foxhunt
roleRef:
  kind: ClusterRole
  name: promtail
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: promtail-config
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: promtail
data:
  promtail.yaml: |
    server:
      http_listen_port: 9080
    positions:
      filename: /run/promtail/positions.yaml
    clients:
      - url: http://loki.foxhunt.svc.cluster.local:3100/loki/api/v1/push
    scrape_configs:
      # Foxhunt application pods (structured Rust logs)
      - job_name: foxhunt-pods
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace]
            action: keep
            regex: foxhunt
          # Drop noisy infra pods (gitlab internals, prometheus, grafana)
          - source_labels: [__meta_kubernetes_pod_label_app]
            action: drop
            regex: (gitlab|prometheus|grafana|promtail)
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
            action: drop
            regex: (promtail|node-exporter|kube-state-metrics)
          # Map to containerd log path
          - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
            separator: /
            target_label: __path__
            replacement: /var/log/pods/*$1/*.log
          # Standard K8s metadata labels
          - source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_container_name]
            target_label: container
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
            target_label: app
          - source_labels: [__meta_kubernetes_pod_label_foxhunt_job__type]
            target_label: job_type
          - source_labels: [__meta_kubernetes_pod_label_foxhunt_model]
            target_label: model
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node
        pipeline_stages:
          - json:
              expressions:
                level: level
                message: fields.message
                target: target
                timestamp: timestamp
          - labels:
              level:
              target:
          - match:
              selector: '{level=""}'
              stages:
                - regex:
                    expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR)'
                - labels:
                    level:

      # Kapsule system pods (kube-system, kube-node-lease, kube-public)
      - job_name: kapsule-system
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace]
            action: keep
            regex: (kube-system|kube-node-lease|kube-public)
          # Map to containerd log path
          - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
            separator: /
            target_label: __path__
            replacement: /var/log/pods/*$1/*.log
          - source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_container_name]
            target_label: container
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
            target_label: app
          - source_labels: [__meta_kubernetes_pod_label_app]
            target_label: kube_app
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node
        pipeline_stages:
          - match:
              selector: '{namespace="kube-system"}'
              stages:
                - regex:
                    expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)'
                - labels:
                    level:

      # GitLab CI runner logs (foxhunt namespace, runner pods)
      - job_name: gitlab-runners
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace]
            action: keep
            regex: foxhunt
          - source_labels: [__meta_kubernetes_pod_label_app]
            action: keep
            regex: gitlab-runner
          - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
            separator: /
            target_label: __path__
            replacement: /var/log/pods/*$1/*.log
          - source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - source_labels: [__meta_kubernetes_pod_container_name]
            target_label: container
          - replacement: gitlab-runner
            target_label: app
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node
        pipeline_stages:
          - regex:
              expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|WARNING)'
          - labels:
              level:

      # Kubernetes node-level logs (kubelet, containerd)
      - job_name: kapsule-node-logs
        static_configs:
          - targets: [localhost]
            labels:
              job: kapsule-nodes
              __path__: /var/log/syslog
          - targets: [localhost]
            labels:
              job: kapsule-nodes
              __path__: /var/log/kern.log
          - targets: [localhost]
            labels:
              job: kapsule-nodes
              __path__: /var/log/kubelet.log
        relabel_configs:
          - source_labels: [__path__]
            regex: .*/(.+)\.log
            target_label: log_type
        pipeline_stages:
          - regex:
              expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)'
          - labels:
              level:
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: promtail
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: promtail
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: promtail
  template:
    metadata:
      labels:
        app.kubernetes.io/name: promtail
    spec:
      serviceAccountName: promtail
      tolerations:
        - effect: NoSchedule
          operator: Exists
      containers:
        - name: promtail
          image: grafana/promtail:3.4.2
          args: ["-config.file=/etc/promtail/promtail.yaml"]
          ports:
            - containerPort: 9080
              name: http
          volumeMounts:
            - name: config
              mountPath: /etc/promtail
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: containers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: pods
              mountPath: /var/log/pods
              readOnly: true
            - name: run
              mountPath: /run/promtail
          env:
            - name: HOSTNAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              cpu: 200m
              memory: 256Mi
          securityContext:
            readOnlyRootFilesystem: true
            allowPrivilegeEscalation: false
      volumes:
        - name: config
          configMap:
            name: promtail-config
        - name: varlog
          hostPath:
            path: /var/log
        - name: containers
          hostPath:
            path: /var/lib/docker/containers
        - name: pods
          hostPath:
            path: /var/log/pods
        - name: run
          emptyDir: {}