infra(ci): scale-to-zero with idle shutdown and auto-start watcher

CI runner auto-shuts down after 15min idle (no GITEA-ACTIONS containers).
Gitea server polls every 60s for queued runs and starts instance via SCW API.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

infra/live/production/ci-runner/terragrunt.hcl

@@ -9,6 +9,10 @@ terraform {
 inputs = {
   instance_type = "GP1-S"   # 8 vCPU, 32GB RAM, 300GB disk
 
+  # SCW secret key — needed by ci-watcher to start/stop instance via API.
+  # Reads from the same env var used by the Scaleway provider.
+  scw_secret_key = get_env("AWS_SECRET_ACCESS_KEY")
+
   # Sensitive — pass at apply time:
   #   terragrunt apply -var 'tailscale_auth_key=tskey-auth-...' -var 'runner_token=...'
   # Or set TF_VAR_tailscale_auth_key and TF_VAR_runner_token env vars.

infra/modules/ci-runner/ci-watcher.sh.tpl

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Auto-start CI runner when Gitea has queued Action runs.
+# Deployed by Terraform, runs via systemd timer every 60s on the Gitea host.
+set -euo pipefail
+
+source /etc/foxhunt/scw-credentials
+
+# Count waiting runs (status=1) in Gitea DB
+WAITING=$(sqlite3 "$GITEA_DB" "SELECT COUNT(*) FROM action_run WHERE status = 1;" 2>/dev/null || echo 0)
+
+if [ "$WAITING" -eq 0 ]; then
+  exit 0
+fi
+
+# Check if CI instance is stopped
+STATE=$(curl -sf -H "X-Auth-Token: $SCW_SECRET_KEY" \
+  "https://api.scaleway.com/instance/v1/zones/$SCW_CI_ZONE/servers/$SCW_CI_INSTANCE_ID" \
+  | python3 -c "import sys,json; print(json.load(sys.stdin)['server']['state'])" 2>/dev/null || echo "unknown")
+
+if [ "$STATE" = "stopped" ] || [ "$STATE" = "stopped in place" ]; then
+  logger -t ci-watcher "Starting CI runner: $WAITING queued run(s)"
+  curl -sf -X POST \
+    -H "X-Auth-Token: $SCW_SECRET_KEY" \
+    -H "Content-Type: application/json" \
+    -d '{"action": "poweron"}' \
+    "https://api.scaleway.com/instance/v1/zones/$SCW_CI_ZONE/servers/$SCW_CI_INSTANCE_ID/action" \
+    > /dev/null
+elif [ "$STATE" = "running" ]; then
+  : # Already running, nothing to do
+else
+  logger -t ci-watcher "CI runner in unexpected state: $STATE (waiting: $WAITING)"
+fi

infra/modules/ci-runner/cloud-init.yaml.tpl

@@ -54,6 +54,64 @@ write_files:
       [Install]
       WantedBy=multi-user.target
 
+  - path: /opt/act_runner/idle-shutdown.sh
+    permissions: '0755'
+    content: |
+      #!/bin/bash
+      # Shutdown instance after ${idle_timeout_minutes} minutes with no running CI jobs.
+      # Runs via systemd timer every 5 minutes.
+      IDLE_FILE="/tmp/ci-idle-since"
+      TIMEOUT_SEC=$((${idle_timeout_minutes} * 60))
+
+      RUNNING=$(docker ps -q --filter "name=GITEA-ACTIONS" 2>/dev/null | wc -l)
+      if [ "$RUNNING" -gt 0 ]; then
+        rm -f "$IDLE_FILE"
+        exit 0
+      fi
+
+      # No jobs running — record when idle started
+      if [ ! -f "$IDLE_FILE" ]; then
+        date +%s > "$IDLE_FILE"
+        logger -t idle-shutdown "CI runner idle, starting countdown"
+        exit 0
+      fi
+
+      IDLE_SINCE=$(cat "$IDLE_FILE")
+      NOW=$(date +%s)
+      IDLE_SECS=$((NOW - IDLE_SINCE))
+
+      if [ "$IDLE_SECS" -ge "$TIMEOUT_SEC" ]; then
+        logger -t idle-shutdown "Idle for $${IDLE_SECS}s (threshold $${TIMEOUT_SEC}s), shutting down"
+        # Clean up Docker to free disk space on next boot
+        docker system prune -af --volumes 2>/dev/null || true
+        rm -f "$IDLE_FILE"
+        poweroff
+      fi
+
+  - path: /etc/systemd/system/ci-idle-shutdown.service
+    permissions: '0644'
+    content: |
+      [Unit]
+      Description=Check CI runner idle state and shutdown if unused
+
+      [Service]
+      Type=oneshot
+      ExecStart=/opt/act_runner/idle-shutdown.sh
+
+  - path: /etc/systemd/system/ci-idle-shutdown.timer
+    permissions: '0644'
+    content: |
+      [Unit]
+      Description=Periodic idle check for CI runner auto-shutdown
+
+      [Timer]
+      OnBootSec=10min
+      OnUnitActiveSec=5min
+      AccuracySec=1min
+
+      [Install]
+      WantedBy=timers.target
+
 runcmd:
   # --- Docker CE ---
   - install -m 0755 -d /etc/apt/keyrings
@@ -102,4 +160,7 @@ runcmd:
   - systemctl enable act-runner
   - systemctl start act-runner
 
+  # Enable idle auto-shutdown timer
+  - systemctl enable --now ci-idle-shutdown.timer
+
   - echo "PROVISIONING_COMPLETE" > /var/log/cloud-init-done

infra/modules/ci-runner/main.tf

@@ -29,10 +29,82 @@ resource "scaleway_instance_server" "ci_runner" {
       instance_name      = var.instance_name
       gitea_url          = var.gitea_url
       runner_token       = var.runner_token
-      runner_labels      = var.runner_labels
-      runner_capacity    = var.runner_capacity
+      runner_labels         = var.runner_labels
+      runner_capacity       = var.runner_capacity
+      idle_timeout_minutes  = var.idle_timeout_minutes
     })
   }
 
   tags = ["ci", "foxhunt"]
 }
+
+# --- Auto-start watcher on Gitea server ---
+# Deploys a systemd timer on the Gitea host that polls for queued runs
+# and starts the CI instance via Scaleway API when needed.
+
+resource "terraform_data" "ci_watcher" {
+  depends_on = [scaleway_instance_server.ci_runner]
+
+  triggers_replace = {
+    script  = filesha256("${path.module}/ci-watcher.sh.tpl")
+    zone    = var.zone
+    server  = scaleway_instance_server.ci_runner.id
+  }
+
+  connection {
+    type = "ssh"
+    host = var.gitea_tailscale_host
+    user = "root"
+  }
+
+  # Deploy credentials file
+  provisioner "remote-exec" {
+    inline = [
+      "mkdir -p /etc/foxhunt /opt/foxhunt",
+      "cat > /etc/foxhunt/scw-credentials <<'EOF'",
+      "SCW_SECRET_KEY=${var.scw_secret_key}",
+      "SCW_CI_INSTANCE_ID=${scaleway_instance_server.ci_runner.id}",
+      "SCW_CI_ZONE=${var.zone}",
+      "GITEA_DB=/var/lib/gitea/data/gitea.db",
+      "EOF",
+      "chmod 0600 /etc/foxhunt/scw-credentials",
+    ]
+  }
+
+  # Deploy watcher script
+  provisioner "file" {
+    content     = templatefile("${path.module}/ci-watcher.sh.tpl", {})
+    destination = "/opt/foxhunt/ci-watcher.sh"
+  }
+
+  # Deploy systemd units and enable timer
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /opt/foxhunt/ci-watcher.sh",
+      # Service unit
+      "cat > /etc/systemd/system/ci-watcher.service <<'EOF'",
+      "[Unit]",
+      "Description=Check for queued Gitea Actions runs and start CI runner",
+      "",
+      "[Service]",
+      "Type=oneshot",
+      "ExecStart=/opt/foxhunt/ci-watcher.sh",
+      "EOF",
+      # Timer unit
+      "cat > /etc/systemd/system/ci-watcher.timer <<'EOF'",
+      "[Unit]",
+      "Description=Poll for queued CI runs every 60s",
+      "",
+      "[Timer]",
+      "OnBootSec=30s",
+      "OnUnitActiveSec=60s",
+      "AccuracySec=10s",
+      "",
+      "[Install]",
+      "WantedBy=timers.target",
+      "EOF",
+      "systemctl daemon-reload",
+      "systemctl enable --now ci-watcher.timer",
+    ]
+  }
+}

infra/modules/ci-runner/variables.tf

@@ -61,3 +61,21 @@ variable "runner_capacity" {
   type        = number
   default     = 1
 }
+
+variable "idle_timeout_minutes" {
+  description = "Minutes idle before auto-shutdown (0 = disabled)"
+  type        = number
+  default     = 15
+}
+
+variable "gitea_tailscale_host" {
+  description = "Tailscale hostname of the Gitea server (for ci-watcher provisioner)"
+  type        = string
+  default     = "vm-fxhnt-git"
+}
+
+variable "scw_secret_key" {
+  description = "Scaleway secret key (for ci-watcher API calls)"
+  type        = string
+  sensitive   = true
+}