From 29bcbce6e4daa21a0ceaefa32e6bd4de648d3f62 Mon Sep 17 00:00:00 2001
From: Adambomb210 <asidwa@outlook.com>
Date: Sun, 15 Nov 2026 14:07:49 -0500
Subject: [PATCH] feat(alastor): ping krater's /healthz/edge every minute

Krater stops a box, or fails a job and destroys its machine, when the
runner on it goes silent. Runners report over Cloudflare, Traefik on
alastor and the tailnet to weft, so when that path is down every runner
looks dead and none is.

A krater-edge-ping timer on alastor requests /healthz/edge on staging,
and on production while its routes are on, every minute over that same
path. Krater records it; when it stops arriving, Krater holds actions on
silent runners and alerts (patchworklabsorg/krater, claude/network-hold).
An HTTPS request rather than an ICMP ping, so it tests what runners use.

alastor builds from jaspermayone/infra: this goes live when that input is
bumped there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 hosts/alastor/edge.nix | 42 ++++++++++++++++++++++++++++++++++++++++++
 tests/krater-weft.nix  |  7 +++++++
 2 files changed, 49 insertions(+)

diff --git a/hosts/alastor/edge.nix b/hosts/alastor/edge.nix
index 93afec1..ddb9af4 100644
--- a/hosts/alastor/edge.nix
+++ b/hosts/alastor/edge.nix
@@ -17,6 +17,17 @@ let
   tailnet = "wildebeest-stargazer.ts.net";
   kraterProduction = config.patchwork.edge.kraterProduction;
 
+  # Krater's every-minute ping (`krater.services.network_watch` in Krater):
+  # an HTTPS request to /healthz/edge over the path its runners report on
+  # (Cloudflare, Traefik here, the tailnet to weft). While it arrives,
+  # runners can reach Krater; when it stops, Krater holds what it would do
+  # about machines whose runners went silent, and alerts. Production only
+  # while its routes are on.
+  kraterPingUrls = [
+    "https://krater-staging.patchworklabs.org/healthz/edge"
+  ]
+  ++ lib.optional kraterProduction "https://krater.patchworklabs.org/healthz/edge";
+
   # <router name> = { backend = "<host>"; hosts = [ "<fqdn>" ... ]; };
   routes = {
     # Not `quilt`/`quilt-staging`: those were the router names of the old
@@ -86,12 +97,43 @@ in
   # Set from the one switch for Krater production
   # (../weft/krater-production.nix). An option only so that the test can turn
   # the routes on: set the switch file, not this.
+  options.patchwork.edge.kraterEdgePing = lib.mkOption {
+    type = lib.types.bool;
+    default = true;
+    description = "Ping Krater's /healthz/edge every minute, so it knows its runners can reach it.";
+  };
+
   options.patchwork.edge.kraterProduction = lib.mkOption {
     type = lib.types.bool;
     default = (import ../weft/krater-production.nix).enable;
     description = "Route krater.patchworklabs.org and krater-storage.patchworklabs.org to weft.";
   };
 
+  config.systemd.services.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing {
+    description = "Tell Krater its runners' path to it works";
+    path = [ pkgs.curl ];
+    serviceConfig = {
+      Type = "oneshot";
+      DynamicUser = true;
+    };
+    # Every URL, even after one fails: staging and production are separate.
+    script = ''
+      status=0
+      for url in ${lib.escapeShellArgs kraterPingUrls}; do
+        curl -fsS -o /dev/null --max-time 20 "$url" || { echo "no answer from $url" >&2; status=1; }
+      done
+      exit $status
+    '';
+  };
+
+  config.systemd.timers.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "minutely";
+      AccuracySec = "5s";
+    };
+  };
+
   config.environment.etc."traefik/conf.d/patchwork.toml".source =
     (pkgs.formats.toml { }).generate "patchwork.toml"
       {
diff --git a/tests/krater-weft.nix b/tests/krater-weft.nix
index abb7a08..eecbe5d 100644
--- a/tests/krater-weft.nix
+++ b/tests/krater-weft.nix
@@ -158,6 +158,13 @@ let
     "production vhost follows the switch file" = (vhosts ? krater-prod) == switchFile;
     "production secret follows the switch file" = (on.age.secrets ? krater-env) == switchFile;
     "edge default follows the switch file" = edge.patchwork.edge.kraterProduction == switchFile;
+    # Krater holds actions on silent runners when this ping stops arriving.
+    "edge pings krater every minute" =
+      edgeOff.systemd.timers.krater-edge-ping.timerConfig.OnCalendar == "minutely"
+      && lib.hasInfix "krater-staging.patchworklabs.org/healthz/edge" edgeOff.systemd.services.krater-edge-ping.script
+      && !(lib.hasInfix "//krater.patchworklabs.org" edgeOff.systemd.services.krater-edge-ping.script);
+    "edge pings production while its routes are on" =
+      lib.hasInfix "https://krater.patchworklabs.org/healthz/edge" edgeProd.systemd.services.krater-edge-ping.script;
     "switch off: krater only on hem" =
       !(schemaOff.central.apps ? krater)
       && schemaOff.central.stagingOnlyApps ? krater