{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Argo Workflows Health", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", phase=\"Running\"}", "instant": true, "legendFormat": "Running", "range": false, "refId": "A" } ], "title": "Pod Status", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 3 }, { "color": "red", "value": 10 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 3, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"})", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Restarts", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 6, "y": 1 }, "id": 4, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[$interval]))", "legendFormat": "", "refId": "A" } ], "title": "CPU Usage", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 9, "y": 1 }, "id": 5, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", resource=\"memory\"})", "legendFormat": "", "refId": "A" } ], "title": "Memory", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 12, "y": 1 }, "id": 6, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"})", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Memory (bytes)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 15, "y": 1 }, "id": 7, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[5m]))", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "CPU Cores", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 1 }, "id": 8, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[5m]))", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Network RX", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 1 }, "id": 9, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[5m]))", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Network TX", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 10, "panels": [], "title": "Workflow Metrics", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 6 }, "id": 11, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Running\"}", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Active Workflows", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 6 }, "id": 12, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "argo_workflows_gauge{status=\"Succeeded\"}", "legendFormat": "", "refId": "A", "instant": true, "range": false } ], "title": "Succeeded", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 8, "y": 6 }, "id": 13, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "argo_workflows_gauge{status=\"Failed\"}", "legendFormat": "", "refId": "A", "instant": true, "range": false } ], "title": "Failed", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 12, "y": 6 }, "id": 14, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "argo_workflows_pods_total_count{phase=\"Pending\"}", "legendFormat": "", "refId": "A", "instant": true, "range": false } ], "title": "Pending Pods", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 16, "y": 6 }, "id": 15, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_workers_busy_count", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Workers Busy", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 20, "y": 6 }, "id": 16, "options": { "colorMode": "value", "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_queue_depth_gauge", "instant": true, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Queue Depth", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 17, "panels": [], "title": "Performance", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 12 }, "id": 18, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Running\"}", "legendFormat": "Running", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Pending\"}", "legendFormat": "Pending", "refId": "B" } ], "title": "Workflow Count", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 12 }, "id": 19, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(argo_workflows_k8s_request_total[$interval])", "legendFormat": "{{verb}} {{kind}}", "refId": "A" } ], "title": "K8s API Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 12 }, "id": 20, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", "legendFormat": "p99", "refId": "A" } ], "title": "Operation Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 20 }, "id": 21, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(argo_workflows_queue_latency_bucket[$interval]))", "legendFormat": "p99", "refId": "A" } ], "title": "Queue Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 20 }, "id": 22, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(argo_workflows_error_count[$interval])", "legendFormat": "errors/s", "refId": "A" } ], "title": "Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 20 }, "id": 23, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_pods_total_count", "legendFormat": "{{status}}", "refId": "A" } ], "title": "Pods Total", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "id": 24, "panels": [], "title": "Go Runtime", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 29 }, "id": 25, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_process.runtime.go.goroutines", "legendFormat": "goroutines", "refId": "A" } ], "title": "Goroutines", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 29 }, "id": 26, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_process.runtime.go.mem.heap_inuse", "legendFormat": "in-use", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_process.runtime.go.mem.heap_alloc", "legendFormat": "alloc", "refId": "B" } ], "title": "Heap Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 29 }, "id": 27, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(argo_workflows_process.runtime.go.gc.pause_total_ns[$interval])", "legendFormat": "ns/s", "refId": "A" } ], "title": "GC Pauses", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, "id": 28, "panels": [], "title": "Resources", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 38 }, "id": 29, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[$interval])", "legendFormat": "{{container}}", "refId": "A" } ], "title": "CPU Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 38 }, "id": 30, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}", "legendFormat": "{{container}}", "refId": "A" } ], "title": "Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 38 }, "id": 31, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[$interval])", "legendFormat": "RX {{pod}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[$interval])", "legendFormat": "TX {{pod}}", "refId": "B" } ], "title": "Network I/O", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, "id": 32, "panels": [], "title": "Logs", "type": "row" }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 47 }, "id": 33, "options": { "dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true }, "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], "title": "Log Stream", "type": "logs" }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 57 }, "id": 34, "options": { "dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true }, "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], "title": "Errors & Warnings", "type": "logs" } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "platform", "argo", "ci" ], "templating": { "list": [ { "auto": false, "current": { "text": "5m", "value": "5m" }, "name": "interval", "options": [ { "selected": false, "text": "1m", "value": "1m" }, { "selected": true, "text": "5m", "value": "5m" }, { "selected": false, "text": "15m", "value": "15m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" } ], "query": "1m,5m,15m,30m,1h", "type": "interval" } ] }, "time": { "from": "now-1h", "to": "now" }, "timezone": "browser", "title": "Argo Workflows", "uid": "plat-argo" }