From 903bf4fbf6443f877cfd93777d52ff1771df5857 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 24 Feb 2026 22:33:42 +0100 Subject: [PATCH] infra(ci): scale-to-zero with idle shutdown and auto-start watcher CI runner auto-shuts down after 15min idle (no GITEA-ACTIONS containers). Gitea server polls every 60s for queued runs and starts instance via SCW API. Co-Authored-By: Claude Opus 4.6 --- .../live/production/ci-runner/terragrunt.hcl | 4 + infra/modules/ci-runner/ci-watcher.sh.tpl | 32 ++++++++ infra/modules/ci-runner/cloud-init.yaml.tpl | 61 +++++++++++++++ infra/modules/ci-runner/main.tf | 76 ++++++++++++++++++- infra/modules/ci-runner/variables.tf | 18 +++++ 5 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 infra/modules/ci-runner/ci-watcher.sh.tpl diff --git a/infra/live/production/ci-runner/terragrunt.hcl b/infra/live/production/ci-runner/terragrunt.hcl index 616b326f8..72fbc9f86 100644 --- a/infra/live/production/ci-runner/terragrunt.hcl +++ b/infra/live/production/ci-runner/terragrunt.hcl @@ -9,6 +9,10 @@ terraform { inputs = { instance_type = "GP1-S" # 8 vCPU, 32GB RAM, 300GB disk + # SCW secret key — needed by ci-watcher to start/stop instance via API. + # Reads from the same env var used by the Scaleway provider. + scw_secret_key = get_env("AWS_SECRET_ACCESS_KEY") + # Sensitive — pass at apply time: # terragrunt apply -var 'tailscale_auth_key=tskey-auth-...' -var 'runner_token=...' # Or set TF_VAR_tailscale_auth_key and TF_VAR_runner_token env vars. diff --git a/infra/modules/ci-runner/ci-watcher.sh.tpl b/infra/modules/ci-runner/ci-watcher.sh.tpl new file mode 100644 index 000000000..cfbd00bd3 --- /dev/null +++ b/infra/modules/ci-runner/ci-watcher.sh.tpl @@ -0,0 +1,32 @@ +#!/bin/bash +# Auto-start CI runner when Gitea has queued Action runs. +# Deployed by Terraform, runs via systemd timer every 60s on the Gitea host. +set -euo pipefail + +source /etc/foxhunt/scw-credentials + +# Count waiting runs (status=1) in Gitea DB +WAITING=$(sqlite3 "$GITEA_DB" "SELECT COUNT(*) FROM action_run WHERE status = 1;" 2>/dev/null || echo 0) + +if [ "$WAITING" -eq 0 ]; then + exit 0 +fi + +# Check if CI instance is stopped +STATE=$(curl -sf -H "X-Auth-Token: $SCW_SECRET_KEY" \ + "https://api.scaleway.com/instance/v1/zones/$SCW_CI_ZONE/servers/$SCW_CI_INSTANCE_ID" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['server']['state'])" 2>/dev/null || echo "unknown") + +if [ "$STATE" = "stopped" ] || [ "$STATE" = "stopped in place" ]; then + logger -t ci-watcher "Starting CI runner: $WAITING queued run(s)" + curl -sf -X POST \ + -H "X-Auth-Token: $SCW_SECRET_KEY" \ + -H "Content-Type: application/json" \ + -d '{"action": "poweron"}' \ + "https://api.scaleway.com/instance/v1/zones/$SCW_CI_ZONE/servers/$SCW_CI_INSTANCE_ID/action" \ + > /dev/null +elif [ "$STATE" = "running" ]; then + : # Already running, nothing to do +else + logger -t ci-watcher "CI runner in unexpected state: $STATE (waiting: $WAITING)" +fi diff --git a/infra/modules/ci-runner/cloud-init.yaml.tpl b/infra/modules/ci-runner/cloud-init.yaml.tpl index 8e20a28e5..b8459ade7 100644 --- a/infra/modules/ci-runner/cloud-init.yaml.tpl +++ b/infra/modules/ci-runner/cloud-init.yaml.tpl @@ -54,6 +54,64 @@ write_files: [Install] WantedBy=multi-user.target + - path: /opt/act_runner/idle-shutdown.sh + permissions: '0755' + content: | + #!/bin/bash + # Shutdown instance after ${idle_timeout_minutes} minutes with no running CI jobs. + # Runs via systemd timer every 5 minutes. + IDLE_FILE="/tmp/ci-idle-since" + TIMEOUT_SEC=$((${idle_timeout_minutes} * 60)) + + RUNNING=$(docker ps -q --filter "name=GITEA-ACTIONS" 2>/dev/null | wc -l) + if [ "$RUNNING" -gt 0 ]; then + rm -f "$IDLE_FILE" + exit 0 + fi + + # No jobs running — record when idle started + if [ ! -f "$IDLE_FILE" ]; then + date +%s > "$IDLE_FILE" + logger -t idle-shutdown "CI runner idle, starting countdown" + exit 0 + fi + + IDLE_SINCE=$(cat "$IDLE_FILE") + NOW=$(date +%s) + IDLE_SECS=$((NOW - IDLE_SINCE)) + + if [ "$IDLE_SECS" -ge "$TIMEOUT_SEC" ]; then + logger -t idle-shutdown "Idle for $${IDLE_SECS}s (threshold $${TIMEOUT_SEC}s), shutting down" + # Clean up Docker to free disk space on next boot + docker system prune -af --volumes 2>/dev/null || true + rm -f "$IDLE_FILE" + poweroff + fi + + - path: /etc/systemd/system/ci-idle-shutdown.service + permissions: '0644' + content: | + [Unit] + Description=Check CI runner idle state and shutdown if unused + + [Service] + Type=oneshot + ExecStart=/opt/act_runner/idle-shutdown.sh + + - path: /etc/systemd/system/ci-idle-shutdown.timer + permissions: '0644' + content: | + [Unit] + Description=Periodic idle check for CI runner auto-shutdown + + [Timer] + OnBootSec=10min + OnUnitActiveSec=5min + AccuracySec=1min + + [Install] + WantedBy=timers.target + runcmd: # --- Docker CE --- - install -m 0755 -d /etc/apt/keyrings @@ -102,4 +160,7 @@ runcmd: - systemctl enable act-runner - systemctl start act-runner + # Enable idle auto-shutdown timer + - systemctl enable --now ci-idle-shutdown.timer + - echo "PROVISIONING_COMPLETE" > /var/log/cloud-init-done diff --git a/infra/modules/ci-runner/main.tf b/infra/modules/ci-runner/main.tf index ab6ef48bd..3aa4a6272 100644 --- a/infra/modules/ci-runner/main.tf +++ b/infra/modules/ci-runner/main.tf @@ -29,10 +29,82 @@ resource "scaleway_instance_server" "ci_runner" { instance_name = var.instance_name gitea_url = var.gitea_url runner_token = var.runner_token - runner_labels = var.runner_labels - runner_capacity = var.runner_capacity + runner_labels = var.runner_labels + runner_capacity = var.runner_capacity + idle_timeout_minutes = var.idle_timeout_minutes }) } tags = ["ci", "foxhunt"] } + +# --- Auto-start watcher on Gitea server --- +# Deploys a systemd timer on the Gitea host that polls for queued runs +# and starts the CI instance via Scaleway API when needed. + +resource "terraform_data" "ci_watcher" { + depends_on = [scaleway_instance_server.ci_runner] + + triggers_replace = { + script = filesha256("${path.module}/ci-watcher.sh.tpl") + zone = var.zone + server = scaleway_instance_server.ci_runner.id + } + + connection { + type = "ssh" + host = var.gitea_tailscale_host + user = "root" + } + + # Deploy credentials file + provisioner "remote-exec" { + inline = [ + "mkdir -p /etc/foxhunt /opt/foxhunt", + "cat > /etc/foxhunt/scw-credentials <<'EOF'", + "SCW_SECRET_KEY=${var.scw_secret_key}", + "SCW_CI_INSTANCE_ID=${scaleway_instance_server.ci_runner.id}", + "SCW_CI_ZONE=${var.zone}", + "GITEA_DB=/var/lib/gitea/data/gitea.db", + "EOF", + "chmod 0600 /etc/foxhunt/scw-credentials", + ] + } + + # Deploy watcher script + provisioner "file" { + content = templatefile("${path.module}/ci-watcher.sh.tpl", {}) + destination = "/opt/foxhunt/ci-watcher.sh" + } + + # Deploy systemd units and enable timer + provisioner "remote-exec" { + inline = [ + "chmod +x /opt/foxhunt/ci-watcher.sh", + # Service unit + "cat > /etc/systemd/system/ci-watcher.service <<'EOF'", + "[Unit]", + "Description=Check for queued Gitea Actions runs and start CI runner", + "", + "[Service]", + "Type=oneshot", + "ExecStart=/opt/foxhunt/ci-watcher.sh", + "EOF", + # Timer unit + "cat > /etc/systemd/system/ci-watcher.timer <<'EOF'", + "[Unit]", + "Description=Poll for queued CI runs every 60s", + "", + "[Timer]", + "OnBootSec=30s", + "OnUnitActiveSec=60s", + "AccuracySec=10s", + "", + "[Install]", + "WantedBy=timers.target", + "EOF", + "systemctl daemon-reload", + "systemctl enable --now ci-watcher.timer", + ] + } +} diff --git a/infra/modules/ci-runner/variables.tf b/infra/modules/ci-runner/variables.tf index 4d720f6bc..89bf5dcaf 100644 --- a/infra/modules/ci-runner/variables.tf +++ b/infra/modules/ci-runner/variables.tf @@ -61,3 +61,21 @@ variable "runner_capacity" { type = number default = 1 } + +variable "idle_timeout_minutes" { + description = "Minutes idle before auto-shutdown (0 = disabled)" + type = number + default = 15 +} + +variable "gitea_tailscale_host" { + description = "Tailscale hostname of the Gitea server (for ci-watcher provisioner)" + type = string + default = "vm-fxhnt-git" +} + +variable "scw_secret_key" { + description = "Scaleway secret key (for ci-watcher API calls)" + type = string + sensitive = true +}