diff --git a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml index d2507e8c..c7356f65 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml @@ -195,6 +195,20 @@ spec: # Reached from provider: "" rather than directly from host, which Rook's CRD # rejects. See the phase A commit. # + # ONE-TIME STEP THAT THIS FILE CANNOT EXPRESS, recorded because it is not + # reproducible from the manifests alone. Removing addressRanges here does + # not clear public_network and cluster_network from Ceph's own config + # database — the same trap the bluestore comment above describes: `ceph + # config` retains a value that is only deleted from this file. During the + # phase A rollout the first OSD came up on the pod network, still had + # public_network 10.2.40.0/24 in the config database, found no interface in + # it and crash-looped on "Failed to pick public address". Both keys were + # removed with `ceph config rm global public_network` and the same for + # cluster_network; Rook then wrote cluster_network afresh from the + # attachment below. Anyone rebuilding this cluster from scratch will not hit + # it — the config database starts empty — but anyone repeating a provider + # change on a live cluster will. + # # Bounded by upgradeOSDRequiresHealthyPGs and storage.osdMaxUpdatesInParallel # above: one OSD restarts at a time and only once PGs are clean again. # diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index 4f0d2ca6..e9dbb4a6 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -117,13 +117,20 @@ parent: enp2s0f0np0 up: true --- # Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29, -# untagged on the wire. Used by Rook for OSD<->OSD replication via -# CephCluster.spec.network.addressRanges.cluster. Not routed off its own -# segment, and deliberately has no default route. +# untagged on the wire. Carries OSD<->OSD replication and nothing else. +# +# The node deliberately holds NO address here. Rook attaches this link to the +# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/ +# san-network-attachment.yaml), and a macvlan parent only has to be up — it +# does not need an address of its own for its children to work. +# +# It used to carry 10.99.0.1/29, because reaching this link required +# CephCluster.spec.network.provider: host, and that put every Ceph daemon on +# host networking, mons included. Nothing needs it now, and by the rule this +# segment is built on — Ceph-internal traffic and nothing else — the node has +# no business on it either. apiVersion: v1alpha1 kind: LinkConfig name: enp2s0f1np1 up: true mtu: 9000 -addresses: - - address: 10.99.0.1/29 diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index 04ef5461..9a2e076d 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -89,10 +89,17 @@ up: true # Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29, # untagged on the wire. OSD<->OSD replication only. Not routed off its own # segment and deliberately has no default route. +# +# The node deliberately holds NO address here. Rook attaches this link to the +# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/ +# san-network-attachment.yaml), and a macvlan parent only has to be up — it +# does not need an address of its own for its children to work. +# +# It used to carry 10.99.0.2/29, because reaching this link required +# CephCluster.spec.network.provider: host, which put every Ceph daemon on +# host networking, mons included. Nothing needs it now. apiVersion: v1alpha1 kind: LinkConfig name: enp2s0f1np1 up: true mtu: 9000 -addresses: - - address: 10.99.0.2/29 diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index 6ac6ed82..08e3bf4a 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -73,10 +73,16 @@ parent: enp2s0f0np0 up: true --- # Dedicated Ceph cluster network — see k8s-1.yaml. +# The node deliberately holds NO address here. Rook attaches this link to the +# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/ +# san-network-attachment.yaml), and a macvlan parent only has to be up — it +# does not need an address of its own for its children to work. +# +# It used to carry 10.99.0.3/29, because reaching this link required +# CephCluster.spec.network.provider: host, which put every Ceph daemon on +# host networking, mons included. Nothing needs it now. apiVersion: v1alpha1 kind: LinkConfig name: enp2s0f1np1 up: true mtu: 9000 -addresses: - - address: 10.99.0.3/29