From dd43316a66840a16b91bb2abdcbb2ee954bfd68d Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 14 Mar 2026 03:09:40 +0100 Subject: [PATCH] feat(monitoring): add Scaleway infrastructure cockpit dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pure-Python scraper replaces shell+Python hybrid (fixes pushgateway RemoteDisconnected crash caused by duplicate volume name labels) - Scraper dynamically discovers all Scaleway resources via API: billing, instances, K8s pools, block volumes, IPs - 40-panel cockpit dashboard with fully dynamic tables (auto-adapts when infra changes — no hardcoded instance/volume names) - Volume table keyed by UUID (not truncated name) to prevent collisions - Label sanitization for Prometheus text format safety - Pushgateway push via PUT + Content-Type: text/plain; version=0.0.4 Co-Authored-By: Claude Opus 4.6 --- infra/k8s/monitoring/dashboards/import.sh | 1 + .../dashboards/platform-scaleway.json | 2669 +++++++++++++++++ infra/k8s/monitoring/scaleway-billing.yaml | 218 +- 3 files changed, 2835 insertions(+), 53 deletions(-) create mode 100644 infra/k8s/monitoring/dashboards/platform-scaleway.json diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh index 54fa1672e..4193e2628 100755 --- a/infra/k8s/monitoring/dashboards/import.sh +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -176,6 +176,7 @@ declare -A DASHBOARD_FOLDERS=( [platform-argo]="${FOLDER_PLATFORM}" [platform-questdb]="${FOLDER_PLATFORM}" [platform-costs]="${FOLDER_PLATFORM}" + [platform-scaleway]="${FOLDER_PLATFORM}" ) # --- Deploy all dashboards --- diff --git a/infra/k8s/monitoring/dashboards/platform-scaleway.json b/infra/k8s/monitoring/dashboards/platform-scaleway.json new file mode 100644 index 000000000..d1ad6efde --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-scaleway.json @@ -0,0 +1,2669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "id": 1, + "panels": [], + "title": "Scaleway Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Total Instances", + "gridPos": { + "x": 0, + "y": 1, + "w": 4, + "h": 4 + }, + "id": 2, + "targets": [ + { + "expr": "count(scaleway_instance_info)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Running", + "gridPos": { + "x": 4, + "y": 1, + "w": 4, + "h": 4 + }, + "id": 3, + "targets": [ + { + "expr": "sum(scaleway_instance_running)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "K8s Pools", + "gridPos": { + "x": 8, + "y": 1, + "w": 4, + "h": 4 + }, + "id": 4, + "targets": [ + { + "expr": "count(scaleway_pool_size)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Block Volumes", + "gridPos": { + "x": 12, + "y": 1, + "w": 3, + "h": 4 + }, + "id": 5, + "targets": [ + { + "expr": "scaleway_volumes_count{state=\"total\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Attached", + "gridPos": { + "x": 15, + "y": 1, + "w": 3, + "h": 4 + }, + "id": 6, + "targets": [ + { + "expr": "scaleway_volumes_count{state=\"attached\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Detached (waste)", + "gridPos": { + "x": 18, + "y": 1, + "w": 3, + "h": 4 + }, + "id": 7, + "targets": [ + { + "expr": "scaleway_volumes_count{state=\"detached\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "IPs", + "gridPos": { + "x": 21, + "y": 1, + "w": 3, + "h": 4 + }, + "id": 8, + "targets": [ + { + "expr": "scaleway_ips_total", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Total Storage", + "gridPos": { + "x": 0, + "y": 5, + "w": 6, + "h": 4 + }, + "id": 9, + "targets": [ + { + "expr": "scaleway_volumes_total_bytes", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "decbytes", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Attached Storage", + "gridPos": { + "x": 6, + "y": 5, + "w": 6, + "h": 4 + }, + "id": 10, + "targets": [ + { + "expr": "scaleway_volumes_attached_bytes", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Detached (wasted)", + "gridPos": { + "x": 12, + "y": 5, + "w": 6, + "h": 4 + }, + "id": 11, + "targets": [ + { + "expr": "scaleway_volumes_detached_bytes", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 200000000000.0 + }, + { + "color": "red", + "value": 500000000000.0 + } + ] + }, + "unit": "decbytes", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "MTD Cost", + "gridPos": { + "x": 18, + "y": 5, + "w": 6, + "h": 4 + }, + "id": 12, + "targets": [ + { + "expr": "scaleway_billing_total_eur", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "collapsed": false, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + }, + "id": 13, + "panels": [], + "title": "Billing (Scaleway API \u2014 Month-to-Date)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Compute", + "gridPos": { + "x": 0, + "y": 10, + "w": 4, + "h": 4 + }, + "id": 14, + "targets": [ + { + "expr": "scaleway_billing_consumption_eur{category=\"compute\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#FF9830", + "value": null + } + ] + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Storage", + "gridPos": { + "x": 4, + "y": 10, + "w": 4, + "h": 4 + }, + "id": 15, + "targets": [ + { + "expr": "scaleway_billing_consumption_eur{category=\"storage\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#73BF69", + "value": null + } + ] + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Network", + "gridPos": { + "x": 8, + "y": 10, + "w": 4, + "h": 4 + }, + "id": 16, + "targets": [ + { + "expr": "scaleway_billing_consumption_eur{category=\"network\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#5794F2", + "value": null + } + ] + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Registry", + "gridPos": { + "x": 12, + "y": 10, + "w": 4, + "h": 4 + }, + "id": 17, + "targets": [ + { + "expr": "scaleway_billing_consumption_eur{category=\"container_registry\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#B877D9", + "value": null + } + ] + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Kubernetes", + "gridPos": { + "x": 16, + "y": 10, + "w": 4, + "h": 4 + }, + "id": 18, + "targets": [ + { + "expr": "scaleway_billing_consumption_eur{category=\"containers\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#FF6E76", + "value": null + } + ] + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Other", + "gridPos": { + "x": 20, + "y": 10, + "w": 4, + "h": 4 + }, + "id": 19, + "targets": [ + { + "expr": "sum(scaleway_billing_consumption_eur) - sum(scaleway_billing_consumption_eur{category=~\"compute|storage|network|container_registry|containers\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-yellow", + "value": null + } + ] + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "table", + "title": "Product Cost Breakdown", + "gridPos": { + "x": 0, + "y": 14, + "w": 12, + "h": 10 + }, + "id": 20, + "targets": [ + { + "expr": "scaleway_billing_product_eur", + "legendFormat": "{{category}} / {{product}}", + "refId": "A", + "instant": true, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "currencyEUR" + }, + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "transformations": [ + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "Value", + "desc": true + } + ] + } + } + ], + "options": { + "showHeader": true, + "sortBy": [], + "cellHeight": "sm" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "piechart", + "title": "Cost by Category", + "gridPos": { + "x": 12, + "y": 14, + "w": 12, + "h": 10 + }, + "id": 21, + "targets": [ + { + "expr": "scaleway_billing_consumption_eur", + "legendFormat": "{{category}}", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "pieType": "donut", + "legend": { + "displayMode": "table", + "placement": "right", + "values": [ + "value", + "percent" + ] + }, + "tooltip": { + "mode": "single" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "timeseries", + "title": "MTD Cost Over Time", + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 7 + }, + "id": 22, + "targets": [ + { + "expr": "scaleway_billing_total_eur", + "legendFormat": "Total MTD", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "timeseries", + "title": "Category Cost Over Time", + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 7 + }, + "id": 23, + "targets": [ + { + "expr": "scaleway_billing_consumption_eur", + "legendFormat": "{{category}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "x": 0, + "y": 31, + "w": 24, + "h": 1 + }, + "id": 24, + "panels": [], + "title": "Compute Instances", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "table", + "title": "All Instances", + "gridPos": { + "x": 0, + "y": 32, + "w": 24, + "h": 8 + }, + "id": 25, + "targets": [ + { + "expr": "scaleway_instance_info", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + }, + { + "expr": "scaleway_instance_running", + "legendFormat": "", + "refId": "B", + "instant": true, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "state" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "running": { + "color": "green", + "text": "RUNNING" + }, + "stopped": { + "color": "red", + "text": "STOPPED" + }, + "stopping": { + "color": "orange", + "text": "STOPPING" + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Running" + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "1": { + "text": "YES", + "color": "green" + }, + "0": { + "text": "NO", + "color": "red" + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "displayName", + "value": "Instance Name" + }, + { + "id": "custom.width", + "value": 280 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "displayName", + "value": "Instance Type" + }, + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pool" + }, + "properties": [ + { + "id": "displayName", + "value": "K8s Pool" + }, + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "transformations": [ + { + "id": "merge", + "options": {} + } + ], + "options": { + "showHeader": true, + "sortBy": [], + "cellHeight": "sm" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "bargauge", + "title": "Instances by Type", + "gridPos": { + "x": 0, + "y": 40, + "w": 24, + "h": 3 + }, + "id": 26, + "targets": [ + { + "expr": "scaleway_instances_total", + "legendFormat": "{{type}}", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true + } + }, + { + "collapsed": false, + "gridPos": { + "x": 0, + "y": 44, + "w": 24, + "h": 1 + }, + "id": 27, + "panels": [], + "title": "Kubernetes Node Pools", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "table", + "title": "All Pools (Autoscaler Config)", + "gridPos": { + "x": 0, + "y": 45, + "w": 24, + "h": 8 + }, + "id": 28, + "targets": [ + { + "expr": "scaleway_pool_size", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + }, + { + "expr": "scaleway_pool_min", + "legendFormat": "", + "refId": "B", + "instant": true, + "format": "table" + }, + { + "expr": "scaleway_pool_max", + "legendFormat": "", + "refId": "C", + "instant": true, + "format": "table" + }, + { + "expr": "scaleway_pool_autoscaling", + "legendFormat": "", + "refId": "D", + "instant": true, + "format": "table" + }, + { + "expr": "scaleway_pool_root_volume_bytes", + "legendFormat": "", + "refId": "E", + "instant": true, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pool" + }, + "properties": [ + { + "id": "displayName", + "value": "Pool Name" + }, + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "displayName", + "value": "Node Type" + }, + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "ready": { + "color": "green", + "text": "READY" + }, + "scaling": { + "color": "yellow", + "text": "SCALING" + }, + "warning": { + "color": "orange", + "text": "WARNING" + } + } + } + ] + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Current Size" + }, + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Min" + }, + { + "id": "custom.width", + "value": 60 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Max" + }, + { + "id": "custom.width", + "value": 60 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "Autoscaling" + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "1": { + "text": "ON", + "color": "green" + }, + "0": { + "text": "OFF", + "color": "red" + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #E" + }, + "properties": [ + { + "id": "displayName", + "value": "Root Volume" + }, + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.width", + "value": 120 + } + ] + } + ] + }, + "transformations": [ + { + "id": "merge", + "options": {} + } + ], + "options": { + "showHeader": true, + "sortBy": [], + "cellHeight": "sm" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "timeseries", + "title": "Pool Sizes Over Time", + "gridPos": { + "x": 0, + "y": 53, + "w": 24, + "h": 7 + }, + "id": 29, + "targets": [ + { + "expr": "scaleway_pool_size", + "legendFormat": "{{pool}} ({{type}})", + "refId": "A" + }, + { + "expr": "scaleway_pool_min", + "legendFormat": "{{pool}} min", + "refId": "B" + }, + { + "expr": "scaleway_pool_max", + "legendFormat": "{{pool}} max", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "x": 0, + "y": 61, + "w": 24, + "h": 1 + }, + "id": 30, + "panels": [], + "title": "Block Storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "table", + "title": "All Block Volumes", + "gridPos": { + "x": 0, + "y": 62, + "w": 24, + "h": 12 + }, + "id": 31, + "targets": [ + { + "expr": "scaleway_volume_size_bytes", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + }, + { + "expr": "scaleway_volume_attached", + "legendFormat": "", + "refId": "B", + "instant": true, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume Name" + }, + { + "id": "custom.width", + "value": 400 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "state" + }, + "properties": [ + { + "id": "displayName", + "value": "State" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "in_use": { + "color": "green", + "text": "IN USE" + }, + "available": { + "color": "red", + "text": "AVAILABLE (detached)" + } + } + } + ] + }, + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Size" + }, + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Attached" + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "1": { + "text": "YES", + "color": "green" + }, + "0": { + "text": "NO \u2014 WASTED", + "color": "red" + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 140 + } + ] + } + ] + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "field": "Value #B", + "desc": false + } + ] + } + } + ], + "options": { + "showHeader": true, + "sortBy": [], + "cellHeight": "sm" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "timeseries", + "title": "Storage Over Time", + "gridPos": { + "x": 0, + "y": 74, + "w": 12, + "h": 7 + }, + "id": 32, + "targets": [ + { + "expr": "scaleway_volumes_total_bytes", + "legendFormat": "Total", + "refId": "A" + }, + { + "expr": "scaleway_volumes_attached_bytes", + "legendFormat": "Attached", + "refId": "B" + }, + { + "expr": "scaleway_volumes_detached_bytes", + "legendFormat": "Detached (waste)", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "unit": "decbytes", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "gauge", + "title": "Storage Waste %", + "gridPos": { + "x": 12, + "y": 74, + "w": 6, + "h": 7 + }, + "id": 33, + "targets": [ + { + "expr": "scaleway_volumes_detached_bytes / scaleway_volumes_total_bytes", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Volumes Count", + "gridPos": { + "x": 18, + "y": 74, + "w": 6, + "h": 7 + }, + "id": 34, + "targets": [ + { + "expr": "scaleway_volumes_count", + "legendFormat": "{{state}}", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name", + "justifyMode": "auto" + } + }, + { + "collapsed": false, + "gridPos": { + "x": 0, + "y": 82, + "w": 24, + "h": 1 + }, + "id": 35, + "panels": [], + "title": "Network & IPs", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "stat", + "title": "Total IPv4 IPs", + "gridPos": { + "x": 0, + "y": 83, + "w": 6, + "h": 4 + }, + "id": 36, + "targets": [ + { + "expr": "scaleway_ips_total", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "timeseries", + "title": "IP Count Over Time", + "gridPos": { + "x": 6, + "y": 83, + "w": 18, + "h": 4 + }, + "id": 37, + "targets": [ + { + "expr": "scaleway_ips_total", + "legendFormat": "IPv4 IPs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "mode": "none" + } + }, + "unit": "short", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "collapsed": false, + "gridPos": { + "x": 0, + "y": 87, + "w": 24, + "h": 1 + }, + "id": 38, + "panels": [], + "title": "Projected Costs (Node-Based Estimates)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "timeseries", + "title": "Estimated Hourly Cost", + "gridPos": { + "x": 0, + "y": 88, + "w": 12, + "h": 7 + }, + "id": 39, + "targets": [ + { + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"platform\"}) * 0.569", + "legendFormat": "Platform (DEV1-L)", + "refId": "A" + }, + { + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gpu\"}) * 5.08", + "legendFormat": "GPU (H100)", + "refId": "B" + }, + { + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=~\"ci-.*\"}) * 0.569", + "legendFormat": "CI Nodes", + "refId": "C" + }, + { + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"infra\"}) * 0.142", + "legendFormat": "Infra (DEV1-M)", + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "type": "timeseries", + "title": "Estimated Monthly Cost", + "gridPos": { + "x": 12, + "y": 88, + "w": 12, + "h": 7 + }, + "id": 40, + "targets": [ + { + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"platform\"}) * 0.569 * 730", + "legendFormat": "Platform", + "refId": "A" + }, + { + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gpu\"}) * 5.08 * 730", + "legendFormat": "GPU (H100)", + "refId": "B" + }, + { + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=~\"ci-.*\"}) * 0.569 * 730", + "legendFormat": "CI Nodes", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "pointSize": 5, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "currencyEUR", + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ], + "schemaVersion": 39, + "tags": [ + "platform", + "scaleway", + "costs", + "infrastructure" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Scaleway Infrastructure Cockpit", + "uid": "scaleway-cockpit", + "version": 1, + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/scaleway-billing.yaml b/infra/k8s/monitoring/scaleway-billing.yaml index 71648753c..af7b1f975 100644 --- a/infra/k8s/monitoring/scaleway-billing.yaml +++ b/infra/k8s/monitoring/scaleway-billing.yaml @@ -7,6 +7,8 @@ type: Opaque stringData: SCW_SECRET_KEY: "84fa878b-81c2-401a-8aef-22996d30e9ea" SCW_ORG_ID: "a55098a3-2cf6-4f73-b011-fff5a299ed94" + SCW_PROJECT_ID: "c293eb98-228d-427d-9b16-f0941f3f2adb" + SCW_CLUSTER_ID: "34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1" --- apiVersion: v1 kind: ConfigMap @@ -14,73 +16,180 @@ metadata: name: scaleway-billing-script namespace: foxhunt data: - scrape.sh: | - #!/bin/sh - set -eu + scrape.py: | + #!/usr/bin/env python3 + """Scaleway infrastructure scraper → Prometheus pushgateway.""" + import json, os, sys, urllib.request, urllib.error, re - PUSHGATEWAY_URL="${PUSHGATEWAY_URL:-http://pushgateway.foxhunt.svc.cluster.local:9091}" - SCW_API="https://api.scaleway.com/billing/v2beta1" + def sanitize(s): + """Sanitize label values for Prometheus text format.""" + return str(s).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") - # Fetch consumptions - consumptions=$(wget -qO- \ - --header="X-Auth-Token: ${SCW_SECRET_KEY}" \ - "${SCW_API}/consumptions?organization_id=${SCW_ORG_ID}" 2>/dev/null) || { - echo "ERROR: Failed to fetch billing data" >&2 - exit 1 - } + PUSHGATEWAY = os.environ.get("PUSHGATEWAY_URL", "http://pushgateway.foxhunt.svc.cluster.local:9091") + SECRET = os.environ["SCW_SECRET_KEY"] + ORG = os.environ["SCW_ORG_ID"] + PROJECT = os.environ["SCW_PROJECT_ID"] + CLUSTER = os.environ["SCW_CLUSTER_ID"] - # Generate Prometheus metrics - metrics=$(echo "$consumptions" | python3 -c " - import sys, json + SCW_BILLING = "https://api.scaleway.com/billing/v2beta1" + SCW_INSTANCE = "https://api.scaleway.com/instance/v1/zones/fr-par-2" + SCW_K8S = "https://api.scaleway.com/k8s/v1/regions/fr-par" + SCW_BLOCK = "https://api.scaleway.com/block/v1alpha1/zones/fr-par-2" + SCW_IPAM = "https://api.scaleway.com/ipam/v1/regions/fr-par" - data = json.load(sys.stdin) - consumptions = data.get('consumptions', []) - lines = [] + def fetch(url): + req = urllib.request.Request(url) + req.add_header("X-Auth-Token", SECRET) + try: + return json.loads(urllib.request.urlopen(req, timeout=30).read()) + except Exception as e: + print(f"WARN: {url}: {e}", file=sys.stderr) + return {} - lines.append('# HELP scaleway_billing_consumption_eur Current month billing consumption in EUR') - lines.append('# TYPE scaleway_billing_consumption_eur gauge') + # ── Fetch all data (dynamic — auto-discovers any infra changes) ── + billing = fetch(f"{SCW_BILLING}/consumptions?organization_id={ORG}") + instances = fetch(f"{SCW_INSTANCE}/servers?project={PROJECT}&per_page=50") + pools = fetch(f"{SCW_K8S}/clusters/{CLUSTER}/pools") + volumes = fetch(f"{SCW_BLOCK}/volumes?project_id={PROJECT}&per_page=100") + ips = fetch(f"{SCW_IPAM}/ips?project_id={PROJECT}&is_ipv6=false&per_page=100") + L = [] + + # ── Billing ── total = 0 cats = {} products = {} - - for c in consumptions: - cat = c.get('category_name', 'other').lower().replace(' ', '_').replace('&', 'and') - product = c.get('product_name', 'unknown').lower().replace(' ', '_').replace('-', '_') - sku = c.get('sku', '') - val = c.get('value', {}) - amount = int(val.get('units', 0) or 0) + int(val.get('nanos', 0) or 0) / 1e9 + for c in billing.get("consumptions", []): + cat = c.get("category_name", "other").lower().replace(" ", "_").replace("&", "and") + product = c.get("product_name", "unknown").lower().replace(" ", "_").replace("-", "_") + sku = c.get("sku", "") + val = c.get("value", {}) + amount = int(val.get("units", 0) or 0) + int(val.get("nanos", 0) or 0) / 1e9 total += amount cats[cat] = cats.get(cat, 0) + amount + products[(cat, product, sku)] = products.get((cat, product, sku), 0) + amount - # Per-product metric - key = (cat, product, sku) - products[key] = products.get(key, 0) + amount - - # Total - lines.append('# HELP scaleway_billing_total_eur Total current month billing in EUR') - lines.append('# TYPE scaleway_billing_total_eur gauge') - lines.append(f'scaleway_billing_total_eur {total:.4f}') - - # By category + L.append("# HELP scaleway_billing_total_eur Total current month billing in EUR") + L.append("# TYPE scaleway_billing_total_eur gauge") + L.append(f"scaleway_billing_total_eur {total:.4f}") + L.append("# HELP scaleway_billing_consumption_eur Billing by category in EUR") + L.append("# TYPE scaleway_billing_consumption_eur gauge") for cat, amt in sorted(cats.items(), key=lambda x: -x[1]): - lines.append(f'scaleway_billing_consumption_eur{{category=\"{cat}\"}} {amt:.4f}') - - # Top products (skip tiny amounts) - lines.append('# HELP scaleway_billing_product_eur Per-product billing in EUR') - lines.append('# TYPE scaleway_billing_product_eur gauge') + L.append(f'scaleway_billing_consumption_eur{{category="{sanitize(cat)}"}} {amt:.4f}') + L.append("# HELP scaleway_billing_product_eur Billing by product in EUR") + L.append("# TYPE scaleway_billing_product_eur gauge") for (cat, product, sku), amt in sorted(products.items(), key=lambda x: -x[1]): if amt >= 0.01: - lines.append(f'scaleway_billing_product_eur{{category=\"{cat}\",product=\"{product}\",sku=\"{sku}\"}} {amt:.4f}') + L.append(f'scaleway_billing_product_eur{{category="{sanitize(cat)}",product="{sanitize(product)}",sku="{sanitize(sku)}"}} {amt:.4f}') - print('\n'.join(lines)) - ") + # ── Instances (dynamic — auto-discovers new/removed instances) ── + L.append("# HELP scaleway_instance_info Instance metadata") + L.append("# TYPE scaleway_instance_info gauge") + L.append("# HELP scaleway_instance_running Instance running state (1=running)") + L.append("# TYPE scaleway_instance_running gauge") + type_counts = {} + for s in instances.get("servers", []): + name = s.get("name", "?") + itype = s.get("commercial_type", "?") + state = s.get("state", "?") + tags = dict(t.split("=", 1) for t in s.get("tags", []) if "=" in t) + pool = tags.get("pool-name", "unknown") + L.append(f'scaleway_instance_info{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}",state="{sanitize(state)}"}} 1') + L.append(f'scaleway_instance_running{{name="{sanitize(name)}",type="{sanitize(itype)}",pool="{sanitize(pool)}"}} {1 if state == "running" else 0}') + type_counts[itype] = type_counts.get(itype, 0) + 1 + L.append("# HELP scaleway_instances_total Total instances by type") + L.append("# TYPE scaleway_instances_total gauge") + for t, c in type_counts.items(): + L.append(f'scaleway_instances_total{{type="{sanitize(t)}"}} {c}') - # Push to pushgateway - echo "$metrics" | wget -qO- --post-data="$(cat -)" \ - "${PUSHGATEWAY_URL}/metrics/job/scaleway_billing" 2>/dev/null && \ - echo "OK: pushed $(echo "$metrics" | grep -c '^scaleway') metrics" || \ - echo "ERROR: push failed" >&2 + # ── K8s Pools (dynamic — auto-discovers pool changes/autoscaler config) ── + L.append("# HELP scaleway_pool_size Current pool node count") + L.append("# TYPE scaleway_pool_size gauge") + L.append("# HELP scaleway_pool_min Minimum pool size") + L.append("# TYPE scaleway_pool_min gauge") + L.append("# HELP scaleway_pool_max Maximum pool size") + L.append("# TYPE scaleway_pool_max gauge") + L.append("# HELP scaleway_pool_autoscaling Pool autoscaling enabled") + L.append("# TYPE scaleway_pool_autoscaling gauge") + L.append("# HELP scaleway_pool_root_volume_bytes Pool root volume size") + L.append("# TYPE scaleway_pool_root_volume_bytes gauge") + for p in pools.get("pools", []): + name = p.get("name", "?") + ntype = p.get("node_type", "?") + status = p.get("status", "?") + lb = f'pool="{sanitize(name)}",type="{sanitize(ntype)}",status="{sanitize(status)}"' + L.append(f"scaleway_pool_size{{{lb}}} {p.get('size', 0)}") + L.append(f"scaleway_pool_min{{{lb}}} {p.get('min_size', 0)}") + L.append(f"scaleway_pool_max{{{lb}}} {p.get('max_size', 0)}") + L.append(f"scaleway_pool_autoscaling{{{lb}}} {1 if p.get('autoscaling') else 0}") + L.append(f"scaleway_pool_root_volume_bytes{{{lb}}} {p.get('root_volume_size', 0)}") + + # ── Block Storage (dynamic — auto-discovers volume additions/removals) ── + L.append("# HELP scaleway_volume_size_bytes Volume size in bytes") + L.append("# TYPE scaleway_volume_size_bytes gauge") + L.append("# HELP scaleway_volume_attached Volume attachment state (1=in_use, 0=available)") + L.append("# TYPE scaleway_volume_attached gauge") + total_bytes = attached_bytes = detached_bytes = 0 + vol_count = attached_count = detached_count = 0 + for v in volumes.get("volumes", []): + vid = v.get("id", "?")[:36] + name = v.get("name", "?")[:60] + size = v.get("size", 0) + state = v.get("status", "unknown") + attached = 1 if state == "in_use" else 0 + total_bytes += size; vol_count += 1 + if attached: + attached_bytes += size; attached_count += 1 + else: + detached_bytes += size; detached_count += 1 + L.append(f'scaleway_volume_size_bytes{{id="{sanitize(vid)}",name="{sanitize(name)}",state="{sanitize(state)}"}} {size}') + L.append(f'scaleway_volume_attached{{id="{sanitize(vid)}",name="{sanitize(name)}"}} {attached}') + L.append("# HELP scaleway_volumes_total_bytes Total block storage bytes") + L.append("# TYPE scaleway_volumes_total_bytes gauge") + L.append(f"scaleway_volumes_total_bytes {total_bytes}") + L.append("# HELP scaleway_volumes_attached_bytes Attached volume bytes") + L.append("# TYPE scaleway_volumes_attached_bytes gauge") + L.append(f"scaleway_volumes_attached_bytes {attached_bytes}") + L.append("# HELP scaleway_volumes_detached_bytes Detached (wasted) volume bytes") + L.append("# TYPE scaleway_volumes_detached_bytes gauge") + L.append(f"scaleway_volumes_detached_bytes {detached_bytes}") + L.append("# HELP scaleway_volumes_count Volume counts") + L.append("# TYPE scaleway_volumes_count gauge") + L.append(f'scaleway_volumes_count{{state="total"}} {vol_count}') + L.append(f'scaleway_volumes_count{{state="attached"}} {attached_count}') + L.append(f'scaleway_volumes_count{{state="detached"}} {detached_count}') + + # ── IPs (dynamic) ── + L.append("# HELP scaleway_ips_total Total allocated IPs") + L.append("# TYPE scaleway_ips_total gauge") + L.append(f'scaleway_ips_total {len(ips.get("ips", []))}') + + # ── Push to pushgateway ── + metrics = "\n".join(L) + "\n" + count = sum(1 for line in L if line.startswith("scaleway_")) + print(f"Generated {count} metrics ({len(metrics)} bytes)", file=sys.stderr) + + # Validate: no empty lines between metrics (pushgateway is strict) + metrics = re.sub(r'\n{2,}', '\n', metrics) + + url = f"{PUSHGATEWAY}/metrics/job/scaleway_billing" + req = urllib.request.Request(url, data=metrics.encode(), method="PUT") + req.add_header("Content-Type", "text/plain; version=0.0.4") + req.add_header("Connection", "close") + try: + resp = urllib.request.urlopen(req, timeout=30) + print(f"OK: pushed {count} metrics (HTTP {resp.status})") + except Exception as e: + print(f"WARN: PUT failed ({e}), trying POST...", file=sys.stderr) + req2 = urllib.request.Request(url, data=metrics.encode(), method="POST") + req2.add_header("Content-Type", "text/plain; version=0.0.4") + req2.add_header("Connection", "close") + try: + resp = urllib.request.urlopen(req2, timeout=30) + print(f"OK: pushed {count} metrics via POST (HTTP {resp.status})") + except Exception as e2: + print(f"ERROR: push failed: PUT={e}, POST={e2}", file=sys.stderr) + sys.exit(1) --- apiVersion: batch/v1 kind: CronJob @@ -88,7 +197,7 @@ metadata: name: scaleway-billing-scraper namespace: foxhunt spec: - schedule: "0 * * * *" # Every hour + schedule: "0 * * * *" successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 3 jobTemplate: @@ -101,12 +210,12 @@ spec: app: scaleway-billing spec: nodeSelector: - k8s.scaleway.com/pool-name: infra + k8s.scaleway.com/pool-name: platform restartPolicy: OnFailure containers: - name: scraper image: python:3.12-alpine - command: ["/bin/sh", "/scripts/scrape.sh"] + command: ["python3", "/scripts/scrape.py"] envFrom: - secretRef: name: scaleway-billing @@ -124,4 +233,7 @@ spec: - name: script configMap: name: scaleway-billing-script + items: + - key: scrape.py + path: scrape.py defaultMode: 0755