feat(monitoring): add Scaleway infrastructure cockpit dashboard
- Pure-Python scraper replaces shell+Python hybrid (fixes pushgateway RemoteDisconnected crash caused by duplicate volume name labels) - Scraper dynamically discovers all Scaleway resources via API: billing, instances, K8s pools, block volumes, IPs - 40-panel cockpit dashboard with fully dynamic tables (auto-adapts when infra changes — no hardcoded instance/volume names) - Volume table keyed by UUID (not truncated name) to prevent collisions - Label sanitization for Prometheus text format safety - Pushgateway push via PUT + Content-Type: text/plain; version=0.0.4 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -176,6 +176,7 @@ declare -A DASHBOARD_FOLDERS=(
|
||||
[platform-argo]="${FOLDER_PLATFORM}"
|
||||
[platform-questdb]="${FOLDER_PLATFORM}"
|
||||
[platform-costs]="${FOLDER_PLATFORM}"
|
||||
[platform-scaleway]="${FOLDER_PLATFORM}"
|
||||
)
|
||||
|
||||
# --- Deploy all dashboards ---
|
||||
|
||||
2669
infra/k8s/monitoring/dashboards/platform-scaleway.json
Normal file
2669
infra/k8s/monitoring/dashboards/platform-scaleway.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,8 @@ type: Opaque
|
||||
stringData:
|
||||
SCW_SECRET_KEY: "84fa878b-81c2-401a-8aef-22996d30e9ea"
|
||||
SCW_ORG_ID: "a55098a3-2cf6-4f73-b011-fff5a299ed94"
|
||||
SCW_PROJECT_ID: "c293eb98-228d-427d-9b16-f0941f3f2adb"
|
||||
SCW_CLUSTER_ID: "34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
@@ -14,73 +16,180 @@ metadata:
|
||||
name: scaleway-billing-script
|
||||
namespace: foxhunt
|
||||
data:
|
||||
scrape.sh: |
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
scrape.py: |
|
||||
#!/usr/bin/env python3
|
||||
"""Scaleway infrastructure scraper → Prometheus pushgateway."""
|
||||
import json, os, sys, urllib.request, urllib.error, re
|
||||
|
||||
PUSHGATEWAY_URL="${PUSHGATEWAY_URL:-http://pushgateway.foxhunt.svc.cluster.local:9091}"
|
||||
SCW_API="https://api.scaleway.com/billing/v2beta1"
|
||||
def sanitize(s):
|
||||
"""Sanitize label values for Prometheus text format."""
|
||||
return str(s).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
|
||||
|
||||
# Fetch consumptions
|
||||
consumptions=$(wget -qO- \
|
||||
--header="X-Auth-Token: ${SCW_SECRET_KEY}" \
|
||||
"${SCW_API}/consumptions?organization_id=${SCW_ORG_ID}" 2>/dev/null) || {
|
||||
echo "ERROR: Failed to fetch billing data" >&2
|
||||
exit 1
|
||||
}
|
||||
PUSHGATEWAY = os.environ.get("PUSHGATEWAY_URL", "http://pushgateway.foxhunt.svc.cluster.local:9091")
|
||||
SECRET = os.environ["SCW_SECRET_KEY"]
|
||||
ORG = os.environ["SCW_ORG_ID"]
|
||||
PROJECT = os.environ["SCW_PROJECT_ID"]
|
||||
CLUSTER = os.environ["SCW_CLUSTER_ID"]
|
||||
|
||||
# Generate Prometheus metrics
|
||||
metrics=$(echo "$consumptions" | python3 -c "
|
||||
import sys, json
|
||||
SCW_BILLING = "https://api.scaleway.com/billing/v2beta1"
|
||||
SCW_INSTANCE = "https://api.scaleway.com/instance/v1/zones/fr-par-2"
|
||||
SCW_K8S = "https://api.scaleway.com/k8s/v1/regions/fr-par"
|
||||
SCW_BLOCK = "https://api.scaleway.com/block/v1alpha1/zones/fr-par-2"
|
||||
SCW_IPAM = "https://api.scaleway.com/ipam/v1/regions/fr-par"
|
||||
|
||||
data = json.load(sys.stdin)
|
||||
consumptions = data.get('consumptions', [])
|
||||
lines = []
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("X-Auth-Token", SECRET)
|
||||
try:
|
||||
return json.loads(urllib.request.urlopen(req, timeout=30).read())
|
||||
except Exception as e:
|
||||
print(f"WARN: {url}: {e}", file=sys.stderr)
|
||||
return {}
|
||||
|
||||
lines.append('# HELP scaleway_billing_consumption_eur Current month billing consumption in EUR')
|
||||
lines.append('# TYPE scaleway_billing_consumption_eur gauge')
|
||||
# ── Fetch all data (dynamic — auto-discovers any infra changes) ──
|
||||
billing = fetch(f"{SCW_BILLING}/consumptions?organization_id={ORG}")
|
||||
instances = fetch(f"{SCW_INSTANCE}/servers?project={PROJECT}&per_page=50")
|
||||
pools = fetch(f"{SCW_K8S}/clusters/{CLUSTER}/pools")
|
||||
volumes = fetch(f"{SCW_BLOCK}/volumes?project_id={PROJECT}&per_page=100")
|
||||
ips = fetch(f"{SCW_IPAM}/ips?project_id={PROJECT}&is_ipv6=false&per_page=100")
|
||||
|
||||
L = []
|
||||
|
||||
# ── Billing ──
|
||||
total = 0
|
||||
cats = {}
|
||||
products = {}
|
||||
|
||||
for c in consumptions:
|
||||
cat = c.get('category_name', 'other').lower().replace(' ', '_').replace('&', 'and')
|
||||
product = c.get('product_name', 'unknown').lower().replace(' ', '_').replace('-', '_')
|
||||
sku = c.get('sku', '')
|
||||
val = c.get('value', {})
|
||||
amount = int(val.get('units', 0) or 0) + int(val.get('nanos', 0) or 0) / 1e9
|
||||
for c in billing.get("consumptions", []):
|
||||
cat = c.get("category_name", "other").lower().replace(" ", "_").replace("&", "and")
|
||||
product = c.get("product_name", "unknown").lower().replace(" ", "_").replace("-", "_")
|
||||
sku = c.get("sku", "")
|
||||
val = c.get("value", {})
|
||||
amount = int(val.get("units", 0) or 0) + int(val.get("nanos", 0) or 0) / 1e9
|
||||
total += amount
|
||||
cats[cat] = cats.get(cat, 0) + amount
|
||||
products[(cat, product, sku)] = products.get((cat, product, sku), 0) + amount
|
||||
|
||||
# Per-product metric
|
||||
key = (cat, product, sku)
|
||||
products[key] = products.get(key, 0) + amount
|
||||
|
||||
# Total
|
||||
lines.append('# HELP scaleway_billing_total_eur Total current month billing in EUR')
|
||||
lines.append('# TYPE scaleway_billing_total_eur gauge')
|
||||
lines.append(f'scaleway_billing_total_eur {total:.4f}')
|
||||
|
||||
# By category
|
||||
L.append("# HELP scaleway_billing_total_eur Total current month billing in EUR")
|
||||
L.append("# TYPE scaleway_billing_total_eur gauge")
|
||||
L.append(f"scaleway_billing_total_eur {total:.4f}")
|
||||
L.append("# HELP scaleway_billing_consumption_eur Billing by category in EUR")
|
||||
L.append("# TYPE scaleway_billing_consumption_eur gauge")
|
||||
for cat, amt in sorted(cats.items(), key=lambda x: -x[1]):
|
||||
lines.append(f'scaleway_billing_consumption_eur{{category=\"{cat}\"}} {amt:.4f}')
|
||||
|
||||
# Top products (skip tiny amounts)
|
||||
lines.append('# HELP scaleway_billing_product_eur Per-product billing in EUR')
|
||||
lines.append('# TYPE scaleway_billing_product_eur gauge')
|
||||
L.append(f'scaleway_billing_consumption_eur{{category="{sanitize(cat)}"}} {amt:.4f}')
|
||||
L.append("# HELP scaleway_billing_product_eur Billing by product in EUR")
|
||||
L.append("# TYPE scaleway_billing_product_eur gauge")
|
||||
for (cat, product, sku), amt in sorted(products.items(), key=lambda x: -x[1]):
|
||||
if amt >= 0.01:
|
||||
lines.append(f'scaleway_billing_product_eur{{category=\"{cat}\",product=\"{product}\",sku=\"{sku}\"}} {amt:.4f}')
|
||||
L.append(f'scaleway_billing_product_eur{{category="{sanitize(cat)}",product="{sanitize(product)}",sku="{sanitize(sku)}"}} {amt:.4f}')
|
||||
|
||||
print('\n'.join(lines))
|
||||
")
|
||||
# ── Instances (dynamic — auto-discovers new/removed instances) ──
|
||||
L.append("# HELP scaleway_instance_info Instance metadata")
|
||||
L.append("# TYPE scaleway_instance_info gauge")
|
||||
L.append("# HELP scaleway_instance_running Instance running state (1=running)")
|
||||
L.append("# TYPE scaleway_instance_running gauge")
|
||||
type_counts = {}
|
||||
for s in instances.get("servers", []):
|
||||
name = s.get("name", "?")
|
||||
itype = s.get("commercial_type", "?")
|
||||
state = s.get("state", "?")
|
||||
tags = dict(t.split("=", 1) for t in s.get("tags", []) if "=" in t)
|
||||
pool = tags.get("pool-name", "unknown")
|
||||
L.append(f'scaleway_instance_info{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}",state="{sanitize(state)}"}} 1')
|
||||
L.append(f'scaleway_instance_running{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}"}} {1 if state == "running" else 0}')
|
||||
type_counts[itype] = type_counts.get(itype, 0) + 1
|
||||
L.append("# HELP scaleway_instances_total Total instances by type")
|
||||
L.append("# TYPE scaleway_instances_total gauge")
|
||||
for t, c in type_counts.items():
|
||||
L.append(f'scaleway_instances_total{{type="{sanitize(t)}"}} {c}')
|
||||
|
||||
# Push to pushgateway
|
||||
echo "$metrics" | wget -qO- --post-data="$(cat -)" \
|
||||
"${PUSHGATEWAY_URL}/metrics/job/scaleway_billing" 2>/dev/null && \
|
||||
echo "OK: pushed $(echo "$metrics" | grep -c '^scaleway') metrics" || \
|
||||
echo "ERROR: push failed" >&2
|
||||
# ── K8s Pools (dynamic — auto-discovers pool changes/autoscaler config) ──
|
||||
L.append("# HELP scaleway_pool_size Current pool node count")
|
||||
L.append("# TYPE scaleway_pool_size gauge")
|
||||
L.append("# HELP scaleway_pool_min Minimum pool size")
|
||||
L.append("# TYPE scaleway_pool_min gauge")
|
||||
L.append("# HELP scaleway_pool_max Maximum pool size")
|
||||
L.append("# TYPE scaleway_pool_max gauge")
|
||||
L.append("# HELP scaleway_pool_autoscaling Pool autoscaling enabled")
|
||||
L.append("# TYPE scaleway_pool_autoscaling gauge")
|
||||
L.append("# HELP scaleway_pool_root_volume_bytes Pool root volume size")
|
||||
L.append("# TYPE scaleway_pool_root_volume_bytes gauge")
|
||||
for p in pools.get("pools", []):
|
||||
name = p.get("name", "?")
|
||||
ntype = p.get("node_type", "?")
|
||||
status = p.get("status", "?")
|
||||
lb = f'pool="{sanitize(name)}",type="{sanitize(ntype)}",status="{sanitize(status)}"'
|
||||
L.append(f"scaleway_pool_size{{{lb}}} {p.get('size', 0)}")
|
||||
L.append(f"scaleway_pool_min{{{lb}}} {p.get('min_size', 0)}")
|
||||
L.append(f"scaleway_pool_max{{{lb}}} {p.get('max_size', 0)}")
|
||||
L.append(f"scaleway_pool_autoscaling{{{lb}}} {1 if p.get('autoscaling') else 0}")
|
||||
L.append(f"scaleway_pool_root_volume_bytes{{{lb}}} {p.get('root_volume_size', 0)}")
|
||||
|
||||
# ── Block Storage (dynamic — auto-discovers volume additions/removals) ──
|
||||
L.append("# HELP scaleway_volume_size_bytes Volume size in bytes")
|
||||
L.append("# TYPE scaleway_volume_size_bytes gauge")
|
||||
L.append("# HELP scaleway_volume_attached Volume attachment state (1=in_use, 0=available)")
|
||||
L.append("# TYPE scaleway_volume_attached gauge")
|
||||
total_bytes = attached_bytes = detached_bytes = 0
|
||||
vol_count = attached_count = detached_count = 0
|
||||
for v in volumes.get("volumes", []):
|
||||
vid = v.get("id", "?")[:36]
|
||||
name = v.get("name", "?")[:60]
|
||||
size = v.get("size", 0)
|
||||
state = v.get("status", "unknown")
|
||||
attached = 1 if state == "in_use" else 0
|
||||
total_bytes += size; vol_count += 1
|
||||
if attached:
|
||||
attached_bytes += size; attached_count += 1
|
||||
else:
|
||||
detached_bytes += size; detached_count += 1
|
||||
L.append(f'scaleway_volume_size_bytes{{id="{sanitize(vid)}",name="{sanitize(name)}",state="{sanitize(state)}"}} {size}')
|
||||
L.append(f'scaleway_volume_attached{{id="{sanitize(vid)}",name="{sanitize(name)}"}} {attached}')
|
||||
L.append("# HELP scaleway_volumes_total_bytes Total block storage bytes")
|
||||
L.append("# TYPE scaleway_volumes_total_bytes gauge")
|
||||
L.append(f"scaleway_volumes_total_bytes {total_bytes}")
|
||||
L.append("# HELP scaleway_volumes_attached_bytes Attached volume bytes")
|
||||
L.append("# TYPE scaleway_volumes_attached_bytes gauge")
|
||||
L.append(f"scaleway_volumes_attached_bytes {attached_bytes}")
|
||||
L.append("# HELP scaleway_volumes_detached_bytes Detached (wasted) volume bytes")
|
||||
L.append("# TYPE scaleway_volumes_detached_bytes gauge")
|
||||
L.append(f"scaleway_volumes_detached_bytes {detached_bytes}")
|
||||
L.append("# HELP scaleway_volumes_count Volume counts")
|
||||
L.append("# TYPE scaleway_volumes_count gauge")
|
||||
L.append(f'scaleway_volumes_count{{state="total"}} {vol_count}')
|
||||
L.append(f'scaleway_volumes_count{{state="attached"}} {attached_count}')
|
||||
L.append(f'scaleway_volumes_count{{state="detached"}} {detached_count}')
|
||||
|
||||
# ── IPs (dynamic) ──
|
||||
L.append("# HELP scaleway_ips_total Total allocated IPs")
|
||||
L.append("# TYPE scaleway_ips_total gauge")
|
||||
L.append(f'scaleway_ips_total {len(ips.get("ips", []))}')
|
||||
|
||||
# ── Push to pushgateway ──
|
||||
metrics = "\n".join(L) + "\n"
|
||||
count = sum(1 for line in L if line.startswith("scaleway_"))
|
||||
print(f"Generated {count} metrics ({len(metrics)} bytes)", file=sys.stderr)
|
||||
|
||||
# Validate: no empty lines between metrics (pushgateway is strict)
|
||||
metrics = re.sub(r'\n{2,}', '\n', metrics)
|
||||
|
||||
url = f"{PUSHGATEWAY}/metrics/job/scaleway_billing"
|
||||
req = urllib.request.Request(url, data=metrics.encode(), method="PUT")
|
||||
req.add_header("Content-Type", "text/plain; version=0.0.4")
|
||||
req.add_header("Connection", "close")
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=30)
|
||||
print(f"OK: pushed {count} metrics (HTTP {resp.status})")
|
||||
except Exception as e:
|
||||
print(f"WARN: PUT failed ({e}), trying POST...", file=sys.stderr)
|
||||
req2 = urllib.request.Request(url, data=metrics.encode(), method="POST")
|
||||
req2.add_header("Content-Type", "text/plain; version=0.0.4")
|
||||
req2.add_header("Connection", "close")
|
||||
try:
|
||||
resp = urllib.request.urlopen(req2, timeout=30)
|
||||
print(f"OK: pushed {count} metrics via POST (HTTP {resp.status})")
|
||||
except Exception as e2:
|
||||
print(f"ERROR: push failed: PUT={e}, POST={e2}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
@@ -88,7 +197,7 @@ metadata:
|
||||
name: scaleway-billing-scraper
|
||||
namespace: foxhunt
|
||||
spec:
|
||||
schedule: "0 * * * *" # Every hour
|
||||
schedule: "0 * * * *"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
@@ -101,12 +210,12 @@ spec:
|
||||
app: scaleway-billing
|
||||
spec:
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: infra
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: scraper
|
||||
image: python:3.12-alpine
|
||||
command: ["/bin/sh", "/scripts/scrape.sh"]
|
||||
command: ["python3", "/scripts/scrape.py"]
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: scaleway-billing
|
||||
@@ -124,4 +233,7 @@ spec:
|
||||
- name: script
|
||||
configMap:
|
||||
name: scaleway-billing-script
|
||||
items:
|
||||
- key: scrape.py
|
||||
path: scrape.py
|
||||
defaultMode: 0755
|
||||
|
||||
Reference in New Issue
Block a user