fix: [alert] KubeDeploymentReplicasMismatch (rook-ceph) (#571)

kubernetes/apps/pve/endpointslice.yaml

@@ -16,29 +16,11 @@ ports:
     port: 8006
     protocol: TCP
 endpoints:
-  # jupiter is OUT OF SERVICE as a PVE node: its MS-01 hardware is being
-  # converted to run Talos directly as k8s-1 (see
-  # kubernetes/talos/patches/k8s-1.yaml), and nothing answers the PVE API on
-  # .201 while that is in progress — verified 2026-08-25 from pve-exporter,
-  # which fails with "[Errno 113] Host is unreachable" for .201.
-  #
-  # This slice is manually managed, and Kubernetes health-checks nothing it did
-  # not create — a hardcoded `ready: true` keeps kube-proxy and Traefik load
-  # balancing onto a dead address forever. That silently defeated the whole
-  # point of pointing the pve scrape job at this Service rather than at a node
-  # IP (see the job_name: pve comment in
-  # kubernetes/apps/monitoring/helmrelease-prometheus.yaml): cluster metrics
-  # were meant to survive any one node being down, but instead
-  # up{job="pve"} flapped 0/1 as connections landed on .201, and pve.renner.uno
-  # served an error for the same fraction of requests.
-  #
-  # Marking it not-ready is the supported way to drain a manual endpoint. If
-  # jupiter is ever rebuilt as a PVE member, flip this back to true.
   - addresses:
       - 10.2.40.201
     hostname: jupiter
     conditions:
-      ready: false
+      ready: true
   - addresses:
       - 10.2.40.202
     hostname: neptune

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -11,6 +11,43 @@ spec:
   mon:
     count: 3
     allowMultiplePerNode: false
+  # TEMPORARY — SET BACK TO false (or drop this block) ONCE k8s-1 HAS REJOINED.
+  #
+  # Stops Rook's mon failover loop for the jupiter window. k8s-1 is no longer a
+  # Node object at all, so the health checker takes the "assigned node not
+  # found" path and fails over *immediately* instead of after the 600s
+  # out-of-quorum timeout:
+  #
+  #   op-mon: mon "c" NOT found in quorum and its assigned node "k8s-1" is not
+  #           found, failing over immediately
+  #
+  # The failover can never complete. Mons run hostNetwork on 3300/6789 and
+  # allowMultiplePerNode is false, so with k8s-1 gone the replacement's canary
+  # has nowhere to land ("2 node(s) didn't have free ports for the requested
+  # pod ports"). The operator gives up after ~2.5 min, deletes the canary, and
+  # the 45s health-check interval starts the next attempt. Each cycle churns a
+  # Deployment plus pod, rewrites the mon-endpoints ConfigMap and the ceph-csi
+  # connection/clientProfile CRs, and buries everything else in the operator log.
+  #
+  # This does NOT hide the degradation: daemonHealth.status is untouched, so
+  # MON_DOWN stays in CephCluster.status.ceph and CephHealthWarn still fires,
+  # and rook-ceph-mon-c stays 0/1 Pending so KubeDeploymentReplicasMismatch
+  # keeps firing. Nothing is given up, because there is no node the failover
+  # could have succeeded on anyway.
+  #
+  # It also keeps mon "c" intact for the end of the window: with the checker
+  # live, the moment k8s-1 rejoins the operator races to stand mon-d up there
+  # and drop "c" from the monmap. Off, mon-c reschedules onto k8s-1 and rejoins
+  # with its existing store under /var/lib/rook/mon-c. If that store did not
+  # survive the reinstall, re-enable the checker first — then failover is
+  # exactly what you want.
+  #
+  # Reverts together with the `RE-ADD IT` note under storage.nodes below and
+  # the `RESTORE TO 1` note in kubernetes/apps/immich/machine-learning.yaml.
+  healthCheck:
+    daemonHealth:
+      mon:
+        disabled: true
   mgr:
     count: 2
     allowMultiplePerNode: false

kubernetes/talos/patches/k8s-1.yaml

@@ -1,7 +1,7 @@
 # k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
 # still Proxmox VMs. Every machine-specific selector below differs from theirs
-# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
-# four physical NICs and all three NVMe drives are visible to Talos at once.
+# for that reason: there are no virtio NICs, no PCI-passthrough layer, and both
+# NVMe drives are visible to Talos at once.
 #
 # Networking uses the v1.12 multi-document config rather than the deprecated
 # machine.network.interfaces. That is deliberate: the docs do not define what
@@ -24,12 +24,10 @@ machine:
     # This disables v6 entirely on that link, including link-local.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    # Three NVMe drives are ordinary block devices to Talos here: the Kingston
-    # OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
-    # Ceph OSD, and the retired Crucial P3 that now carries EPHEMERAL (see the
-    # VolumeConfig below). Every one of them is selected by serial. Nothing in
-    # this file may use a broad selector: `disk.transport == 'nvme'` matches all
-    # three, and the failure mode is Talos taking the OSD drive.
+    # Both NVMe drives are now ordinary block devices to Talos: the Kingston
+    # OM8PGP41024Q-A0 system disk and the Samsung PM983 that Rook claims as the
+    # Ceph OSD. Selecting the installer target by serial means it cannot land
+    # on the OSD drive regardless of which one the kernel enumerates first.
     diskSelector:
       serial: 50026B7382B48B86
   certSANs:
@@ -39,39 +37,9 @@ machine:
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
-# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
-#
-# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
-# the PM983 replaced it. /var is exactly the right role for a disk in that
-# state: containerd's image and snapshot store, pod logs and the Ceph mon store
-# are all reconstructible, so the drive is consumable here in a way it is not as
-# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
-# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
-# they absorb another node's pods.
-#
-# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
-# solely when the volume does not yet exist; a config applied after EPHEMERAL
-# has been created on the system disk is silently ignored, and relocating it
-# then requires wiping the disk and booting install media again. So this
-# document has to be present in the very first `apply-config --insecure`.
-#
-# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
-# would match the PM983 as well, and losing that coin flip means Talos formats
-# the OSD drive.
-apiVersion: v1alpha1
-kind: VolumeConfig
-name: EPHEMERAL
-provisioning:
-  diskSelector:
-    match: disk.serial == '2332E86B749A'
-  minSize: 50GiB
-  grow: true
----
 # Physical NICs are aliased by permanent MAC. Both X710 ports bind i40e, so the
 # driver selector k8s-2/k8s-3 use for their single passed-through port would
-# match two links here. MAC matching is unambiguous and survives udev renaming —
-# which is not hypothetical: adding the third NVMe renumbered the PCI buses and
-# moved the second I226 from enp89s0 to enp90s0.
+# match two links here. MAC matching is unambiguous and survives udev renaming.
 apiVersion: v1alpha1
 kind: LinkAliasConfig
 name: lan0
@@ -113,7 +81,7 @@ link: mgmt0
 # that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
 # serve both VM and bare-metal nodes for as long as the cluster is mixed.
 # Nothing else on this machine is called ens19: the physical NICs are
-# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp90s0.
+# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp89s0.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19