dd43316a66
- Pure-Python scraper replaces shell+Python hybrid (fixes pushgateway RemoteDisconnected crash caused by duplicate volume name labels) - Scraper dynamically discovers all Scaleway resources via API: billing, instances, K8s pools, block volumes, IPs - 40-panel cockpit dashboard with fully dynamic tables (auto-adapts when infra changes — no hardcoded instance/volume names) - Volume table keyed by UUID (not truncated name) to prevent collisions - Label sanitization for Prometheus text format safety - Pushgateway push via PUT + Content-Type: text/plain; version=0.0.4 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
240 lines
11 KiB
YAML
240 lines
11 KiB
YAML
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: scaleway-billing
|
|
namespace: foxhunt
|
|
type: Opaque
|
|
stringData:
|
|
SCW_SECRET_KEY: "84fa878b-81c2-401a-8aef-22996d30e9ea"
|
|
SCW_ORG_ID: "a55098a3-2cf6-4f73-b011-fff5a299ed94"
|
|
SCW_PROJECT_ID: "c293eb98-228d-427d-9b16-f0941f3f2adb"
|
|
SCW_CLUSTER_ID: "34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1"
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: scaleway-billing-script
|
|
namespace: foxhunt
|
|
data:
|
|
scrape.py: |
|
|
#!/usr/bin/env python3
|
|
"""Scaleway infrastructure scraper → Prometheus pushgateway."""
|
|
import json, os, sys, urllib.request, urllib.error, re
|
|
|
|
def sanitize(s):
|
|
"""Sanitize label values for Prometheus text format."""
|
|
return str(s).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
|
|
|
|
PUSHGATEWAY = os.environ.get("PUSHGATEWAY_URL", "http://pushgateway.foxhunt.svc.cluster.local:9091")
|
|
SECRET = os.environ["SCW_SECRET_KEY"]
|
|
ORG = os.environ["SCW_ORG_ID"]
|
|
PROJECT = os.environ["SCW_PROJECT_ID"]
|
|
CLUSTER = os.environ["SCW_CLUSTER_ID"]
|
|
|
|
SCW_BILLING = "https://api.scaleway.com/billing/v2beta1"
|
|
SCW_INSTANCE = "https://api.scaleway.com/instance/v1/zones/fr-par-2"
|
|
SCW_K8S = "https://api.scaleway.com/k8s/v1/regions/fr-par"
|
|
SCW_BLOCK = "https://api.scaleway.com/block/v1alpha1/zones/fr-par-2"
|
|
SCW_IPAM = "https://api.scaleway.com/ipam/v1/regions/fr-par"
|
|
|
|
def fetch(url):
|
|
req = urllib.request.Request(url)
|
|
req.add_header("X-Auth-Token", SECRET)
|
|
try:
|
|
return json.loads(urllib.request.urlopen(req, timeout=30).read())
|
|
except Exception as e:
|
|
print(f"WARN: {url}: {e}", file=sys.stderr)
|
|
return {}
|
|
|
|
# ── Fetch all data (dynamic — auto-discovers any infra changes) ──
|
|
billing = fetch(f"{SCW_BILLING}/consumptions?organization_id={ORG}")
|
|
instances = fetch(f"{SCW_INSTANCE}/servers?project={PROJECT}&per_page=50")
|
|
pools = fetch(f"{SCW_K8S}/clusters/{CLUSTER}/pools")
|
|
volumes = fetch(f"{SCW_BLOCK}/volumes?project_id={PROJECT}&per_page=100")
|
|
ips = fetch(f"{SCW_IPAM}/ips?project_id={PROJECT}&is_ipv6=false&per_page=100")
|
|
|
|
L = []
|
|
|
|
# ── Billing ──
|
|
total = 0
|
|
cats = {}
|
|
products = {}
|
|
for c in billing.get("consumptions", []):
|
|
cat = c.get("category_name", "other").lower().replace(" ", "_").replace("&", "and")
|
|
product = c.get("product_name", "unknown").lower().replace(" ", "_").replace("-", "_")
|
|
sku = c.get("sku", "")
|
|
val = c.get("value", {})
|
|
amount = int(val.get("units", 0) or 0) + int(val.get("nanos", 0) or 0) / 1e9
|
|
total += amount
|
|
cats[cat] = cats.get(cat, 0) + amount
|
|
products[(cat, product, sku)] = products.get((cat, product, sku), 0) + amount
|
|
|
|
L.append("# HELP scaleway_billing_total_eur Total current month billing in EUR")
|
|
L.append("# TYPE scaleway_billing_total_eur gauge")
|
|
L.append(f"scaleway_billing_total_eur {total:.4f}")
|
|
L.append("# HELP scaleway_billing_consumption_eur Billing by category in EUR")
|
|
L.append("# TYPE scaleway_billing_consumption_eur gauge")
|
|
for cat, amt in sorted(cats.items(), key=lambda x: -x[1]):
|
|
L.append(f'scaleway_billing_consumption_eur{{category="{sanitize(cat)}"}} {amt:.4f}')
|
|
L.append("# HELP scaleway_billing_product_eur Billing by product in EUR")
|
|
L.append("# TYPE scaleway_billing_product_eur gauge")
|
|
for (cat, product, sku), amt in sorted(products.items(), key=lambda x: -x[1]):
|
|
if amt >= 0.01:
|
|
L.append(f'scaleway_billing_product_eur{{category="{sanitize(cat)}",product="{sanitize(product)}",sku="{sanitize(sku)}"}} {amt:.4f}')
|
|
|
|
# ── Instances (dynamic — auto-discovers new/removed instances) ──
|
|
L.append("# HELP scaleway_instance_info Instance metadata")
|
|
L.append("# TYPE scaleway_instance_info gauge")
|
|
L.append("# HELP scaleway_instance_running Instance running state (1=running)")
|
|
L.append("# TYPE scaleway_instance_running gauge")
|
|
type_counts = {}
|
|
for s in instances.get("servers", []):
|
|
name = s.get("name", "?")
|
|
itype = s.get("commercial_type", "?")
|
|
state = s.get("state", "?")
|
|
tags = dict(t.split("=", 1) for t in s.get("tags", []) if "=" in t)
|
|
pool = tags.get("pool-name", "unknown")
|
|
L.append(f'scaleway_instance_info{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}",state="{sanitize(state)}"}} 1')
|
|
L.append(f'scaleway_instance_running{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}"}} {1 if state == "running" else 0}')
|
|
type_counts[itype] = type_counts.get(itype, 0) + 1
|
|
L.append("# HELP scaleway_instances_total Total instances by type")
|
|
L.append("# TYPE scaleway_instances_total gauge")
|
|
for t, c in type_counts.items():
|
|
L.append(f'scaleway_instances_total{{type="{sanitize(t)}"}} {c}')
|
|
|
|
# ── K8s Pools (dynamic — auto-discovers pool changes/autoscaler config) ──
|
|
L.append("# HELP scaleway_pool_size Current pool node count")
|
|
L.append("# TYPE scaleway_pool_size gauge")
|
|
L.append("# HELP scaleway_pool_min Minimum pool size")
|
|
L.append("# TYPE scaleway_pool_min gauge")
|
|
L.append("# HELP scaleway_pool_max Maximum pool size")
|
|
L.append("# TYPE scaleway_pool_max gauge")
|
|
L.append("# HELP scaleway_pool_autoscaling Pool autoscaling enabled")
|
|
L.append("# TYPE scaleway_pool_autoscaling gauge")
|
|
L.append("# HELP scaleway_pool_root_volume_bytes Pool root volume size")
|
|
L.append("# TYPE scaleway_pool_root_volume_bytes gauge")
|
|
for p in pools.get("pools", []):
|
|
name = p.get("name", "?")
|
|
ntype = p.get("node_type", "?")
|
|
status = p.get("status", "?")
|
|
lb = f'pool="{sanitize(name)}",type="{sanitize(ntype)}",status="{sanitize(status)}"'
|
|
L.append(f"scaleway_pool_size{{{lb}}} {p.get('size', 0)}")
|
|
L.append(f"scaleway_pool_min{{{lb}}} {p.get('min_size', 0)}")
|
|
L.append(f"scaleway_pool_max{{{lb}}} {p.get('max_size', 0)}")
|
|
L.append(f"scaleway_pool_autoscaling{{{lb}}} {1 if p.get('autoscaling') else 0}")
|
|
L.append(f"scaleway_pool_root_volume_bytes{{{lb}}} {p.get('root_volume_size', 0)}")
|
|
|
|
# ── Block Storage (dynamic — auto-discovers volume additions/removals) ──
|
|
L.append("# HELP scaleway_volume_size_bytes Volume size in bytes")
|
|
L.append("# TYPE scaleway_volume_size_bytes gauge")
|
|
L.append("# HELP scaleway_volume_attached Volume attachment state (1=in_use, 0=available)")
|
|
L.append("# TYPE scaleway_volume_attached gauge")
|
|
total_bytes = attached_bytes = detached_bytes = 0
|
|
vol_count = attached_count = detached_count = 0
|
|
for v in volumes.get("volumes", []):
|
|
vid = v.get("id", "?")[:36]
|
|
name = v.get("name", "?")[:60]
|
|
size = v.get("size", 0)
|
|
state = v.get("status", "unknown")
|
|
attached = 1 if state == "in_use" else 0
|
|
total_bytes += size; vol_count += 1
|
|
if attached:
|
|
attached_bytes += size; attached_count += 1
|
|
else:
|
|
detached_bytes += size; detached_count += 1
|
|
L.append(f'scaleway_volume_size_bytes{{id="{sanitize(vid)}",name="{sanitize(name)}",state="{sanitize(state)}"}} {size}')
|
|
L.append(f'scaleway_volume_attached{{id="{sanitize(vid)}",name="{sanitize(name)}"}} {attached}')
|
|
L.append("# HELP scaleway_volumes_total_bytes Total block storage bytes")
|
|
L.append("# TYPE scaleway_volumes_total_bytes gauge")
|
|
L.append(f"scaleway_volumes_total_bytes {total_bytes}")
|
|
L.append("# HELP scaleway_volumes_attached_bytes Attached volume bytes")
|
|
L.append("# TYPE scaleway_volumes_attached_bytes gauge")
|
|
L.append(f"scaleway_volumes_attached_bytes {attached_bytes}")
|
|
L.append("# HELP scaleway_volumes_detached_bytes Detached (wasted) volume bytes")
|
|
L.append("# TYPE scaleway_volumes_detached_bytes gauge")
|
|
L.append(f"scaleway_volumes_detached_bytes {detached_bytes}")
|
|
L.append("# HELP scaleway_volumes_count Volume counts")
|
|
L.append("# TYPE scaleway_volumes_count gauge")
|
|
L.append(f'scaleway_volumes_count{{state="total"}} {vol_count}')
|
|
L.append(f'scaleway_volumes_count{{state="attached"}} {attached_count}')
|
|
L.append(f'scaleway_volumes_count{{state="detached"}} {detached_count}')
|
|
|
|
# ── IPs (dynamic) ──
|
|
L.append("# HELP scaleway_ips_total Total allocated IPs")
|
|
L.append("# TYPE scaleway_ips_total gauge")
|
|
L.append(f'scaleway_ips_total {len(ips.get("ips", []))}')
|
|
|
|
# ── Push to pushgateway ──
|
|
metrics = "\n".join(L) + "\n"
|
|
count = sum(1 for line in L if line.startswith("scaleway_"))
|
|
print(f"Generated {count} metrics ({len(metrics)} bytes)", file=sys.stderr)
|
|
|
|
# Validate: no empty lines between metrics (pushgateway is strict)
|
|
metrics = re.sub(r'\n{2,}', '\n', metrics)
|
|
|
|
url = f"{PUSHGATEWAY}/metrics/job/scaleway_billing"
|
|
req = urllib.request.Request(url, data=metrics.encode(), method="PUT")
|
|
req.add_header("Content-Type", "text/plain; version=0.0.4")
|
|
req.add_header("Connection", "close")
|
|
try:
|
|
resp = urllib.request.urlopen(req, timeout=30)
|
|
print(f"OK: pushed {count} metrics (HTTP {resp.status})")
|
|
except Exception as e:
|
|
print(f"WARN: PUT failed ({e}), trying POST...", file=sys.stderr)
|
|
req2 = urllib.request.Request(url, data=metrics.encode(), method="POST")
|
|
req2.add_header("Content-Type", "text/plain; version=0.0.4")
|
|
req2.add_header("Connection", "close")
|
|
try:
|
|
resp = urllib.request.urlopen(req2, timeout=30)
|
|
print(f"OK: pushed {count} metrics via POST (HTTP {resp.status})")
|
|
except Exception as e2:
|
|
print(f"ERROR: push failed: PUT={e}, POST={e2}", file=sys.stderr)
|
|
sys.exit(1)
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: scaleway-billing-scraper
|
|
namespace: foxhunt
|
|
spec:
|
|
schedule: "0 * * * *"
|
|
successfulJobsHistoryLimit: 1
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 120
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: scaleway-billing
|
|
spec:
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: scraper
|
|
image: python:3.12-alpine
|
|
command: ["python3", "/scripts/scrape.py"]
|
|
envFrom:
|
|
- secretRef:
|
|
name: scaleway-billing
|
|
volumeMounts:
|
|
- name: script
|
|
mountPath: /scripts
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
volumes:
|
|
- name: script
|
|
configMap:
|
|
name: scaleway-billing-script
|
|
items:
|
|
- key: scrape.py
|
|
path: scrape.py
|
|
defaultMode: 0755
|