From 31bc806d01d867c9d5e73be558b3a41cc226dd11 Mon Sep 17 00:00:00 2001
From: NeilClack <cnclack@pm.me>
Date: Sun, 4 Oct 2026 20:12:25 -0500
Subject: [PATCH] Update preauthkey and join script

---
 configuration.nix | 65 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 60 insertions(+), 5 deletions(-)

diff --git a/configuration.nix b/configuration.nix
index d429dd8..62a9249 100644
--- a/configuration.nix
+++ b/configuration.nix
@@ -13,7 +13,16 @@ let
   myKey = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL4qWNZnYU5JaqR9c2qh6oFZktzVNBOx42PfDaGofCL6 cnclack@pm.me macbook" ];
 
   # Headscale preauth key. Paste/replace here.
-  headscaleAuthKey = "hskey-auth-zZdWiKTwDfTT-tgRKiAHyXabykoARDDyNfYihgVrJQ6y6zK7lwV0FW6xwHTdyWoqa_ZpCf80vgAJC";
+  #
+  # This MUST be a *reusable* key with a long expiry (1 year is good), created
+  # with:
+  #   headscale preauthkeys create --user <user-id> --reusable --expiration 8760h
+  #
+  # A single-use key only ever enrolls one node, which would mean a fresh key
+  # plus a rebuild for every kiosk that joins the fleet. A reusable key lets any
+  # number of kiosks enroll unattended, and the timer below retries the join so
+  # a node that misses its window at boot heals on its own.
+  headscaleAuthKey = "hskey-auth-UfU_eS_sdOu6-CjrqrVdDJFRFsPnaTtK5sFuCiVlffJAS7uQ1wX1_65MNk205WGsF4clOiBD3ZOV-";
 in
 {
   ############################################################
@@ -115,7 +124,9 @@ in
     wantedBy = [ "multi-user.target" ];
     serviceConfig = {
       Type = "oneshot";
-      RemainAfterExit = true;
+      # Deliberately no RemainAfterExit: the unit has to fall back to inactive
+      # so the timer below can trigger it again. The script is idempotent, so
+      # re-running it on an already-joined kiosk is a cheap no-op.
     };
     path = [ pkgs.tailscale pkgs.coreutils pkgs.gnused pkgs.gnugrep ];
     script = ''
@@ -144,7 +155,17 @@ in
         | cut -c1-63)"
       [ -n "$serial" ] || serial="radiantwave-unknown"
 
-      echo "radiantwave-tailscale-auth: joining as '$serial'"
+      # Already joined? Bail out early. This is what makes the periodic
+      # re-invocation from the timer below cheap and safe to run forever.
+      # The JSON is pretty-printed with a space, so allow optional whitespace.
+      state="$(tailscale status --json 2>/dev/null \
+        | sed -n 's/.*"BackendState": *"\([A-Za-z]*\)".*/\1/p')"
+      if [ "$state" = "Running" ]; then
+        echo "radiantwave-tailscale-auth: already joined as '$serial'"
+        exit 0
+      fi
+
+      echo "radiantwave-tailscale-auth: joining as '$serial' (state: ''${state:-unknown})"
       for _ in $(seq 1 30); do
         if tailscale up \
              --auth-key=${pkgs.lib.escapeShellArg headscaleAuthKey} \
@@ -158,11 +179,45 @@ in
         sleep 2
       done
 
-      echo "radiantwave-tailscale-auth: could not join tailnet after retries" >&2
-      exit 1
+      # Best-effort by design: do NOT exit non-zero here.
+      #
+      # This unit is part of the new generation that switch-to-configuration
+      # starts, and *any* failing unit among those aborts the switch with
+      # status 4/NOPERMISSION. While the preauth key was expired that is exactly
+      # what happened: this unit failed, which failed `switch-to-configuration`,
+      # which failed nixos-upgrade.service -- so the tailscale key silently
+      # stopped *all* system upgrades on the fleet every 10 minutes.
+      #
+      # The retry timer below is what actually recovers the join, so a failed
+      # attempt here must never take down the system switch with it.
+      echo "radiantwave-tailscale-auth: could not join tailnet after retries;" >&2
+      echo "radiantwave-tailscale-auth: leaving the join to the retry timer" >&2
+      exit 0
     '';
   };
 
+  # The join only needs to happen once per node, but it fails for reasons
+  # outside our control: the network may not be usable yet at boot, tailscaled
+  # may be restarted later, or the preauth key may be rotated out from under
+  # the fleet. A timer re-runs the (idempotent) join so a kiosk recovers on its
+  # own instead of needing a manual `systemctl restart`.
+  #
+  # This replaces the old behavior where the one-shot unit failed once at boot
+  # and stayed failed forever (Restart=no, NRestarts=0).
+  systemd.timers.radiantwave-tailscale-auth = {
+    description = "Periodically ensure the kiosk is joined to the tailnet";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      # The join is attempted at boot via multi-user.target above; this is the
+      # first retry in case that one lost the race with the network.
+      OnBootSec = "5min";
+      # Measured from when the unit last went inactive, so a slow join doesn't
+      # immediately retrigger.
+      OnUnitInactiveSec = "10min";
+      Unit = "radiantwave-tailscale-auth.service";
+    };
+  };
+
   system.autoUpgrade = {
     enable = true;
     flake = upgradeFlakeRef;