gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitoring): add per-service and platform dashboards, reorganize folder structure

Browse Source 
- 7 application service dashboards (Services folder): API Gateway, Trading
  Service, Backtesting, Data Acquisition, ML Training, Trading Agent, Web Gateway
- 9 platform dashboards (Platform folder): Prometheus, Grafana, Loki & Promtail,
  Tempo, Redis, PostgreSQL, Argo Workflows, QuestDB, Cluster & Costs
- Cluster & Costs dashboard includes Scaleway node cost estimates (€/hr, €/month)
  for platform, GPU (H100), and CI compile nodes
- Reorganized existing dashboards into 7 folders: CI, Infrastructure, Operations,
  Platform, Services, Trading, Training (dropped "Foxhunt" prefix from titles)
- Rewrote import.sh: removed ConfigMap deployment, switched to API-only imports
  with automatic folder creation and per-dashboard folder mapping
- All 24 dashboards deployed via Grafana API (Postgres-backed)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-14 02:30:52 +01:00
parent 6349e384d2
commit ddf697641a
25 changed files with 27567 additions and 6833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

752
infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json
View File

File diff suppressed because it is too large Load Diff

2443
infra/k8s/monitoring/dashboards/foxhunt-cockpit.json
View File

File diff suppressed because it is too large Load Diff

2347
infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json
View File

File diff suppressed because it is too large Load Diff

1088
infra/k8s/monitoring/dashboards/foxhunt-observability.json
View File

File diff suppressed because it is too large Load Diff

3419
infra/k8s/monitoring/dashboards/foxhunt-trading.json
View File

File diff suppressed because it is too large Load Diff

2313
infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json
View File

File diff suppressed because it is too large Load Diff

783
infra/k8s/monitoring/dashboards/foxhunt-training-logs.json
View File

File diff suppressed because it is too large Load Diff

3423
infra/k8s/monitoring/dashboards/foxhunt-training.json
View File

File diff suppressed because it is too large Load Diff

239
infra/k8s/monitoring/dashboards/import.sh
View File

@@ -1,12 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
# Import Grafana dashboards with dynamic datasource resolution.
# Import all Grafana dashboards via API (Postgres-backed, no ConfigMaps).
#
# Dashboards use ${DS_PROMETHEUS}, ${DS_LOKI}, ${DS_TEMPO} variables.
# This script auto-detects datasource UIDs from the Grafana instance,
# resolves variables, and deploys via ConfigMaps (for sidecar provisioning)
# with API fallback for non-provisioned dashboards.
# This script auto-detects datasource UIDs, resolves variables, and
# imports via the Grafana API. Dashboards are stored in Postgres.
#
# Required environment variables:
# GRAFANA_PASS - Password for admin user
@@ -17,11 +16,9 @@ set -euo pipefail
# GRAFANA_PROM_UID - Override Prometheus datasource UID
# GRAFANA_LOKI_UID - Override Loki datasource UID
# GRAFANA_TEMPO_UID - Override Tempo datasource UID
# KUBE_NAMESPACE - K8s namespace (default: foxhunt)
# SKIP_CONFIGMAP - Set to 1 to skip ConfigMap deploy (API-only mode)
#
# Usage:
# GRAFANA_PASS="Welcome01" ./import.sh
# GRAFANA_PASS="xxx" ./import.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GRAFANA_USER="${GRAFANA_USER:-admin}"
@@ -61,7 +58,7 @@ if [[ -z "$PROM_UID" ]]; then
exit 1
fi
# --- Resolve datasource variables and strip __inputs ---
# --- Resolve datasource variables ---
resolve_dashboard() {
local file="$1"
@@ -83,15 +80,49 @@ print(raw)
"
}
# --- Import a single dashboard ---
# --- Ensure folders exist ---
ensure_folder() {
local title="$1"
local uid
uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/folders" 2>/dev/null | \
python3 -c "
import sys, json
for f in json.load(sys.stdin):
if f['title'] == '${title}':
print(f['uid'])
break
" 2>/dev/null || true)
if [[ -z "$uid" ]]; then
uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" \
-H "Content-Type: application/json" \
-d "{\"title\": \"${title}\"}" \
"${GRAFANA_URL}/api/folders" 2>/dev/null | \
python3 -c "import sys,json; print(json.load(sys.stdin)['uid'])" 2>/dev/null)
echo " Created folder: ${title} (${uid})"
fi
echo "$uid"
}
echo ""
echo "Ensuring folders..."
FOLDER_CI=$(ensure_folder "CI")
FOLDER_INFRA=$(ensure_folder "Infrastructure")
FOLDER_OPS=$(ensure_folder "Operations")
FOLDER_TRADING=$(ensure_folder "Trading")
FOLDER_TRAINING=$(ensure_folder "Training")
FOLDER_SERVICES=$(ensure_folder "Services")
FOLDER_PLATFORM=$(ensure_folder "Platform")
# --- Import a single dashboard to a folder ---
import_dashboard() {
local file="$1"
local folder_uid="$2"
local name
name="$(basename "$file" .json)"
echo "Importing ${name}..."
local resolved
resolved=$(resolve_dashboard "$file")
@@ -99,160 +130,82 @@ import_dashboard() {
response=$(curl -s -w "\n%{http_code}" \
-u "${GRAFANA_USER}:${GRAFANA_PASS}" \
-H "Content-Type: application/json" \
-d "{\"dashboard\": ${resolved}, \"overwrite\": true}" \
-d "{\"dashboard\": ${resolved}, \"folderUid\": \"${folder_uid}\", \"overwrite\": true}" \
"${GRAFANA_URL}/api/dashboards/db")
local http_code
http_code=$(echo "$response" | tail -1)
local body
body=$(echo "$response" | sed '$d')
if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then
echo " OK (${http_code})"
echo " OK ${name}"
else
echo " FAILED (${http_code}): ${body}" >&2
local body
body=$(echo "$response" | sed '$d')
echo " FAIL ${name} (${http_code}): ${body}" >&2
return 1
fi
}
# --- ConfigMap deployment ---
# Dashboard JSON files are grouped into ConfigMaps that Grafana's sidecar
# watches (label grafana_dashboard=1). This is the primary deploy method.
# --- Dashboard → Folder mapping ---
KUBE_NAMESPACE="${KUBE_NAMESPACE:-foxhunt}"
# ConfigMap grouping: name -> list of dashboard files
declare -A CONFIGMAP_GROUPS=(
[grafana-dashboards-cockpit]="foxhunt-cockpit.json"
[grafana-dashboards-trading]="foxhunt-trading.json"
[grafana-dashboards-training]="foxhunt-training.json"
[grafana-dashboards-infrastructure]="foxhunt-infrastructure.json"
[grafana-dashboards-observability]="foxhunt-observability.json"
[grafana-dashboards-ci-logs]="foxhunt-ci-logs.json"
[grafana-dashboards-training-logs]="foxhunt-training-logs.json"
[grafana-dashboards-deep-dive]="foxhunt-training-deep-dive.json"
declare -A DASHBOARD_FOLDERS=(
# Original dashboards
[foxhunt-cockpit]="${FOLDER_OPS}"
[foxhunt-trading]="${FOLDER_TRADING}"
[foxhunt-training]="${FOLDER_TRAINING}"
[foxhunt-infrastructure]="${FOLDER_INFRA}"
[foxhunt-observability]="${FOLDER_INFRA}"
[foxhunt-ci-logs]="${FOLDER_CI}"
[foxhunt-training-logs]="${FOLDER_TRAINING}"
[foxhunt-training-deep-dive]="${FOLDER_TRAINING}"
# Application services
[svc-api-gateway]="${FOLDER_SERVICES}"
[svc-trading-service]="${FOLDER_SERVICES}"
[svc-backtesting]="${FOLDER_SERVICES}"
[svc-data-acquisition]="${FOLDER_SERVICES}"
[svc-ml-training]="${FOLDER_SERVICES}"
[svc-trading-agent]="${FOLDER_SERVICES}"
[svc-web-gateway]="${FOLDER_SERVICES}"
# Platform services
[platform-prometheus]="${FOLDER_PLATFORM}"
[platform-grafana]="${FOLDER_PLATFORM}"
[platform-loki]="${FOLDER_PLATFORM}"
[platform-tempo]="${FOLDER_PLATFORM}"
[platform-redis]="${FOLDER_PLATFORM}"
[platform-postgres]="${FOLDER_PLATFORM}"
[platform-argo]="${FOLDER_PLATFORM}"
[platform-questdb]="${FOLDER_PLATFORM}"
[platform-costs]="${FOLDER_PLATFORM}"
)
# Folder mapping: ConfigMap name -> Grafana folder name (used by sidecar annotation)
declare -A CONFIGMAP_FOLDERS=(
[grafana-dashboards-cockpit]="Foxhunt"
[grafana-dashboards-trading]="Foxhunt"
[grafana-dashboards-training]="Foxhunt"
[grafana-dashboards-infrastructure]="Foxhunt"
[grafana-dashboards-observability]="Foxhunt"
[grafana-dashboards-ci-logs]="Foxhunt"
[grafana-dashboards-training-logs]="Foxhunt"
[grafana-dashboards-deep-dive]="Foxhunt"
)
# --- Deploy all dashboards ---
deploy_configmaps() {
echo ""
echo "Deploying dashboard ConfigMaps..."
local tmp_dir
tmp_dir=$(mktemp -d)
trap "rm -rf $tmp_dir" RETURN
# Resolve all dashboard files to temp dir
for f in "${SCRIPT_DIR}"/*.json; do
resolve_dashboard "$f" > "${tmp_dir}/$(basename "$f")"
done
# Build and apply ConfigMaps
python3 -c "
import json, os
tmp = '${tmp_dir}'
ns = '${KUBE_NAMESPACE}'
groups = ${CONFIGMAP_GROUPS_JSON}
folders = ${CONFIGMAP_FOLDERS_JSON}
items = []
for name, files in groups.items():
cm = {
'apiVersion': 'v1',
'kind': 'ConfigMap',
'metadata': {'name': name, 'namespace': ns, 'labels': {'grafana_dashboard': '1'}, 'annotations': {'grafana_folder': folders.get(name, '')}},
'data': {}
}
for f in files:
path = os.path.join(tmp, f)
if os.path.exists(path):
with open(path) as fh:
cm['data'][f] = fh.read()
items.append(cm)
print(json.dumps({'apiVersion': 'v1', 'kind': 'List', 'items': items}))
" | kubectl apply -f -
}
# Export groups as JSON for python
CONFIGMAP_GROUPS_JSON='{'
first=true
for name in "${!CONFIGMAP_GROUPS[@]}"; do
$first || CONFIGMAP_GROUPS_JSON+=','
files_json="["
ffirst=true
for f in ${CONFIGMAP_GROUPS[${name}]}; do
$ffirst || files_json+=","
files_json+="\"${f}\""
ffirst=false
done
files_json+="]"
CONFIGMAP_GROUPS_JSON+="\"${name}\":${files_json}"
first=false
done
CONFIGMAP_GROUPS_JSON+='}'
export CONFIGMAP_GROUPS_JSON
# Export folders as JSON for python
CONFIGMAP_FOLDERS_JSON='{'
first=true
for name in "${!CONFIGMAP_FOLDERS[@]}"; do
$first || CONFIGMAP_FOLDERS_JSON+=','
CONFIGMAP_FOLDERS_JSON+="\"${name}\":\"${CONFIGMAP_FOLDERS[${name}]}\""
first=false
done
CONFIGMAP_FOLDERS_JSON+='}'
export CONFIGMAP_FOLDERS_JSON
# --- Deploy ---
echo ""
echo "Importing dashboards..."
failed=0
total=0
if [[ "${SKIP_CONFIGMAP:-0}" != "1" ]] && command -v kubectl &>/dev/null; then
deploy_configmaps
echo ""
echo "Restarting Grafana to reload dashboards..."
kubectl rollout restart deployment grafana -n "${KUBE_NAMESPACE}" 2>/dev/null || true
kubectl rollout status deployment grafana -n "${KUBE_NAMESPACE}" --timeout=60s 2>/dev/null || true
else
echo ""
echo "Skipping ConfigMap deploy (kubectl not available or SKIP_CONFIGMAP=1)."
echo "Falling back to API import..."
for dashboard in "${SCRIPT_DIR}"/*.json; do
[[ -f "$dashboard" ]] || continue
name="$(basename "$dashboard" .json)"
folder_uid="${DASHBOARD_FOLDERS[$name]:-}"
for dashboard in "${SCRIPT_DIR}/foxhunt-cockpit.json" \
"${SCRIPT_DIR}/foxhunt-trading.json" \
"${SCRIPT_DIR}/foxhunt-training.json" \
"${SCRIPT_DIR}/foxhunt-infrastructure.json" \
"${SCRIPT_DIR}/foxhunt-observability.json" \
"${SCRIPT_DIR}/foxhunt-ci-logs.json" \
"${SCRIPT_DIR}/foxhunt-training-logs.json" \
"${SCRIPT_DIR}/foxhunt-training-deep-dive.json"; do
if [[ -f "$dashboard" ]]; then
if ! import_dashboard "$dashboard"; then
failed=$((failed + 1))
fi
else
echo "WARNING: ${dashboard} not found, skipping" >&2
fi
done
fi
if [[ -z "$folder_uid" ]]; then
echo " SKIP ${name} (no folder mapping)"
continue
fi
total=$((total + 1))
if ! import_dashboard "$dashboard" "$folder_uid"; then
failed=$((failed + 1))
fi
done
echo ""
if [[ $failed -eq 0 ]]; then
echo "All dashboards deployed successfully."
echo "All ${total} dashboards deployed successfully."
else
echo "ERROR: ${failed} dashboard(s) failed." >&2
echo "ERROR: ${failed}/${total} dashboard(s) failed." >&2
exit 1
fi

1632
infra/k8s/monitoring/dashboards/platform-argo.json Normal file
View File

File diff suppressed because it is too large Load Diff

1350
infra/k8s/monitoring/dashboards/platform-costs.json Normal file
View File

File diff suppressed because it is too large Load Diff

802
infra/k8s/monitoring/dashboards/platform-grafana.json Normal file
View File

@@ -0,0 +1,802 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Grafana Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"grafana.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"grafana.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"grafana.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"}",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 25
},
"id": 16,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} |~ \"(?i)(error|warn|panic|fatal)\"",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"grafana"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "Grafana",
"uid": "plat-grafana"
}

1532
infra/k8s/monitoring/dashboards/platform-loki.json Normal file
View File

File diff suppressed because it is too large Load Diff

1146
infra/k8s/monitoring/dashboards/platform-postgres.json Normal file
View File

File diff suppressed because it is too large Load Diff

946
infra/k8s/monitoring/dashboards/platform-prometheus.json Normal file
View File

@@ -0,0 +1,946 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Prometheus Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "count(up{job!=\"\"} == 1)",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Targets Up",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "count(up{job!=\"\"} == 0) or vector(0)",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Targets Down",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_head_series",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Head Series",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Ingestion Rate",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_wal_storage_size_bytes",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "WAL Size",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_head_chunks",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Head Chunks",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_compactions_total",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Compactions",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "prometheus_tsdb_storage_blocks_bytes",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Storage Size",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Scrape Performance",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "scrape_duration_seconds",
"legendFormat": "{{job}}",
"refId": "A"
}
],
"title": "Scrape Duration",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 15
},
"id": 16,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(prometheus_tsdb_head_samples_appended_total[$interval])",
"legendFormat": "samples/s",
"refId": "A"
}
],
"title": "Samples Ingested",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 17,
"panels": [],
"title": "Target Status",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 24
},
"id": 18,
"options": {
"showHeader": true,
"sortBy": []
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "up",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Target Health",
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 32
},
"id": 19,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 33
},
"id": 20,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"}",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 43
},
"id": 21,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} |~ \"(?i)(error|warn|panic|fatal)\"",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"prometheus"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "Prometheus",
"uid": "plat-prometheus"
}

802
infra/k8s/monitoring/dashboards/platform-questdb.json Normal file
View File

@@ -0,0 +1,802 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "QuestDB Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"questdb.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"questdb.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"questdb.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"}",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 25
},
"id": 16,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} |~ \"(?i)(error|warn|panic|fatal)\"",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"questdb"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "QuestDB",
"uid": "plat-questdb"
}

1315
infra/k8s/monitoring/dashboards/platform-redis.json Normal file
View File

File diff suppressed because it is too large Load Diff

802
infra/k8s/monitoring/dashboards/platform-tempo.json Normal file
View File

@@ -0,0 +1,802 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Tempo Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 25
},
"id": 16,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} |~ \"(?i)(error|warn|panic|fatal)\"",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"platform",
"${DS_TEMPO}"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "Tempo",
"uid": "plat-${DS_TEMPO}"
}

963
infra/k8s/monitoring/dashboards/svc-api-gateway.json Normal file
View File

@@ -0,0 +1,963 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Service Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api.*\", phase=\"Running\"}",
"instant": true,
"legendFormat": "Running",
"range": false,
"refId": "A"
}
],
"title": "Pod Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 3
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 3,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"api.*\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Restarts",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 4,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.7
},
{
"color": "red",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"api.*\", resource=\"memory\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 12,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"})",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Memory (bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 15,
"y": 1
},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "CPU Cores",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network RX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 1
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": {
"calcs": [
"lastNotNull"
]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))",
"instant": true,
"legendFormat": "",
"range": false,
"refId": "A"
}
],
"title": "Network TX",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 10,
"panels": [],
"title": "Resources",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"id": 11,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval])",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"id": 12,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}",
"legendFormat": "{{container}}",
"refId": "A"
}
],
"title": "Memory",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])",
"legendFormat": "RX {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])",
"legendFormat": "TX {{pod}}",
"refId": "B"
}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 14,
"panels": [],
"title": "gRPC Metrics",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 15
},
"id": 15,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(grpc_server_handled_total{grpc_service=~\"foxhunt.*\", pod=~\"api.*\"}[$interval])",
"legendFormat": "{{grpc_method}}",
"refId": "A"
}
],
"title": "gRPC Request Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 15
},
"id": 16,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"api.*\"}[$interval]))",
"legendFormat": "p99",
"refId": "A"
}
],
"title": "gRPC Latency (p99)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 15
},
"id": 17,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(grpc_server_handled_total{grpc_code!=\"OK\", pod=~\"api.*\"}[$interval])",
"legendFormat": "{{grpc_code}}",
"refId": "A"
}
],
"title": "gRPC Errors",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 23
},
"id": 18,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 24
},
"id": 19,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"api.*\"}",
"refId": "A"
}
],
"title": "Log Stream",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 34
},
"id": 20,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} |~ \"(?i)(error|warn|panic|fatal)\"",
"refId": "A"
}
],
"title": "Errors & Warnings",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"service",
"api"
],
"templating": {
"list": [
{
"auto": false,
"current": {
"text": "5m",
"value": "5m"
},
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "API Gateway",
"uid": "svc-api-gateway"
}

1075
infra/k8s/monitoring/dashboards/svc-backtesting.json Normal file
View File

File diff suppressed because it is too large Load Diff

1031
infra/k8s/monitoring/dashboards/svc-data-acquisition.json Normal file
View File

File diff suppressed because it is too large Load Diff

1026
infra/k8s/monitoring/dashboards/svc-ml-training.json Normal file
View File

File diff suppressed because it is too large Load Diff

1075
infra/k8s/monitoring/dashboards/svc-trading-agent.json Normal file
View File

File diff suppressed because it is too large Load Diff

1074
infra/k8s/monitoring/dashboards/svc-trading-service.json Normal file
View File

File diff suppressed because it is too large Load Diff

1022
infra/k8s/monitoring/dashboards/svc-web-gateway.json Normal file
View File

File diff suppressed because it is too large Load Diff