diff --git a/hosts/alastor/edge.nix b/hosts/alastor/edge.nix index 93afec1..ddb9af4 100644 --- a/hosts/alastor/edge.nix +++ b/hosts/alastor/edge.nix @@ -17,6 +17,17 @@ let tailnet = "wildebeest-stargazer.ts.net"; kraterProduction = config.patchwork.edge.kraterProduction; + # Krater's every-minute ping (`krater.services.network_watch` in Krater): + # an HTTPS request to /healthz/edge over the path its runners report on + # (Cloudflare, Traefik here, the tailnet to weft). While it arrives, + # runners can reach Krater; when it stops, Krater holds what it would do + # about machines whose runners went silent, and alerts. Production only + # while its routes are on. + kraterPingUrls = [ + "https://krater-staging.patchworklabs.org/healthz/edge" + ] + ++ lib.optional kraterProduction "https://krater.patchworklabs.org/healthz/edge"; + # = { backend = ""; hosts = [ "" ... ]; }; routes = { # Not `quilt`/`quilt-staging`: those were the router names of the old @@ -86,12 +97,43 @@ in # Set from the one switch for Krater production # (../weft/krater-production.nix). An option only so that the test can turn # the routes on: set the switch file, not this. + options.patchwork.edge.kraterEdgePing = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Ping Krater's /healthz/edge every minute, so it knows its runners can reach it."; + }; + options.patchwork.edge.kraterProduction = lib.mkOption { type = lib.types.bool; default = (import ../weft/krater-production.nix).enable; description = "Route krater.patchworklabs.org and krater-storage.patchworklabs.org to weft."; }; + config.systemd.services.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing { + description = "Tell Krater its runners' path to it works"; + path = [ pkgs.curl ]; + serviceConfig = { + Type = "oneshot"; + DynamicUser = true; + }; + # Every URL, even after one fails: staging and production are separate. + script = '' + status=0 + for url in ${lib.escapeShellArgs kraterPingUrls}; do + curl -fsS -o /dev/null --max-time 20 "$url" || { echo "no answer from $url" >&2; status=1; } + done + exit $status + ''; + }; + + config.systemd.timers.krater-edge-ping = lib.mkIf config.patchwork.edge.kraterEdgePing { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "minutely"; + AccuracySec = "5s"; + }; + }; + config.environment.etc."traefik/conf.d/patchwork.toml".source = (pkgs.formats.toml { }).generate "patchwork.toml" { diff --git a/tests/krater-weft.nix b/tests/krater-weft.nix index abb7a08..eecbe5d 100644 --- a/tests/krater-weft.nix +++ b/tests/krater-weft.nix @@ -158,6 +158,13 @@ let "production vhost follows the switch file" = (vhosts ? krater-prod) == switchFile; "production secret follows the switch file" = (on.age.secrets ? krater-env) == switchFile; "edge default follows the switch file" = edge.patchwork.edge.kraterProduction == switchFile; + # Krater holds actions on silent runners when this ping stops arriving. + "edge pings krater every minute" = + edgeOff.systemd.timers.krater-edge-ping.timerConfig.OnCalendar == "minutely" + && lib.hasInfix "krater-staging.patchworklabs.org/healthz/edge" edgeOff.systemd.services.krater-edge-ping.script + && !(lib.hasInfix "//krater.patchworklabs.org" edgeOff.systemd.services.krater-edge-ping.script); + "edge pings production while its routes are on" = + lib.hasInfix "https://krater.patchworklabs.org/healthz/edge" edgeProd.systemd.services.krater-edge-ping.script; "switch off: krater only on hem" = !(schemaOff.central.apps ? krater) && schemaOff.central.stagingOnlyApps ? krater