kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -195,6 +195,20 @@ spec:
   # Reached from provider: "" rather than directly from host, which Rook's CRD
   # rejects. See the phase A commit.
   #
+  # ONE-TIME STEP THAT THIS FILE CANNOT EXPRESS, recorded because it is not
+  # reproducible from the manifests alone. Removing addressRanges here does
+  # not clear public_network and cluster_network from Ceph's own config
+  # database — the same trap the bluestore comment above describes: `ceph
+  # config` retains a value that is only deleted from this file. During the
+  # phase A rollout the first OSD came up on the pod network, still had
+  # public_network 10.2.40.0/24 in the config database, found no interface in
+  # it and crash-looped on "Failed to pick public address". Both keys were
+  # removed with `ceph config rm global public_network` and the same for
+  # cluster_network; Rook then wrote cluster_network afresh from the
+  # attachment below. Anyone rebuilding this cluster from scratch will not hit
+  # it — the config database starts empty — but anyone repeating a provider
+  # change on a live cluster will.
+  #
   # Bounded by upgradeOSDRequiresHealthyPGs and storage.osdMaxUpdatesInParallel
   # above: one OSD restarts at a time and only once PGs are clean again.
   #

kubernetes/talos/patches/k8s-1.yaml

@@ -117,13 +117,20 @@ parent: enp2s0f0np0
 up: true
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
-# untagged on the wire. Used by Rook for OSD<->OSD replication via
-# CephCluster.spec.network.addressRanges.cluster. Not routed off its own
-# segment, and deliberately has no default route.
+# untagged on the wire. Carries OSD<->OSD replication and nothing else.
+#
+# The node deliberately holds NO address here. Rook attaches this link to the
+# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
+# san-network-attachment.yaml), and a macvlan parent only has to be up — it
+# does not need an address of its own for its children to work.
+#
+# It used to carry 10.99.0.1/29, because reaching this link required
+# CephCluster.spec.network.provider: host, and that put every Ceph daemon on
+# host networking, mons included. Nothing needs it now, and by the rule this
+# segment is built on — Ceph-internal traffic and nothing else — the node has
+# no business on it either.
 apiVersion: v1alpha1
 kind: LinkConfig
 name: enp2s0f1np1
 up: true
 mtu: 9000
-addresses:
-  - address: 10.99.0.1/29

kubernetes/talos/patches/k8s-2.yaml

@@ -89,10 +89,17 @@ up: true
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
 # untagged on the wire. OSD<->OSD replication only. Not routed off its own
 # segment and deliberately has no default route.
+#
+# The node deliberately holds NO address here. Rook attaches this link to the
+# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
+# san-network-attachment.yaml), and a macvlan parent only has to be up — it
+# does not need an address of its own for its children to work.
+#
+# It used to carry 10.99.0.2/29, because reaching this link required
+# CephCluster.spec.network.provider: host, which put every Ceph daemon on
+# host networking, mons included. Nothing needs it now.
 apiVersion: v1alpha1
 kind: LinkConfig
 name: enp2s0f1np1
 up: true
 mtu: 9000
-addresses:
-  - address: 10.99.0.2/29

kubernetes/talos/patches/k8s-3.yaml

@@ -73,10 +73,16 @@ parent: enp2s0f0np0
 up: true
 ---
 # Dedicated Ceph cluster network — see k8s-1.yaml.
+# The node deliberately holds NO address here. Rook attaches this link to the
+# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
+# san-network-attachment.yaml), and a macvlan parent only has to be up — it
+# does not need an address of its own for its children to work.
+#
+# It used to carry 10.99.0.3/29, because reaching this link required
+# CephCluster.spec.network.provider: host, which put every Ceph daemon on
+# host networking, mons included. Nothing needs it now.
 apiVersion: v1alpha1
 kind: LinkConfig
 name: enp2s0f1np1
 up: true
 mtu: 9000
-addresses:
-  - address: 10.99.0.3/29