atlas-botandClaude Opus 5.5
90a547546e
Upstream deployments/multus-daemonset-thick.yml v4.2.4 -> v4.3.1 adds: - ClusterRole rule: resource.k8s.io resourceclaims, resourceclaims/status, resourceslices; get, list (new DRA client) - terminationGracePeriodSeconds 10 -> 30 - CRD shortName nad Header comment updated to v4.3.1 (v4.3.1 also has an unsuffixed tag; we keep -thick to match the manifest). Refs: REN-722 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
339 lines
13 KiB
YAML
339 lines
13 KiB
YAML
# Vendored from k8snetworkplumbingwg/multus-cni v4.3.1 thick daemonset:
|
|
# https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.3.1/deployments/multus-daemonset-thick.yml
|
|
#
|
|
# Why vendored, not Helm: there is no published Helm repository for this
|
|
# project — the upstream k8snetworkplumbingwg/helm-charts repo only ships
|
|
# charts for local-clone install. Pointing a Flux HelmRepository at
|
|
# https://k8snetworkplumbingwg.github.io/helm-charts/ 404s.
|
|
#
|
|
# Why thick (not thin): the only version-pinned image tag published for
|
|
# v4.2.4 was `v4.2.4-thick` — there was no `v4.2.4-thin` or unsuffixed
|
|
# `v4.2.4` tag (verified against ghcr.io/k8snetworkplumbingwg/multus-cni
|
|
# package versions). The thin variant only shipped under moving tags
|
|
# (snapshot, latest), which we don't want for reproducibility. v4.3.1 also
|
|
# publishes an unsuffixed tag; we keep `-thick` so image and manifest match.
|
|
#
|
|
# Talos compatibility: the daemon chroots into /hostroot to exec CNI
|
|
# plugin binaries from /opt/cni/bin on the host. Talos's rootfs is
|
|
# read-only but /opt/cni/bin is a writable mount that Cilium already
|
|
# uses successfully — so chroot-then-exec is fine.
|
|
---
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata:
|
|
name: network-attachment-definitions.k8s.cni.cncf.io
|
|
spec:
|
|
group: k8s.cni.cncf.io
|
|
scope: Namespaced
|
|
names:
|
|
plural: network-attachment-definitions
|
|
singular: network-attachment-definition
|
|
kind: NetworkAttachmentDefinition
|
|
shortNames:
|
|
- net-attach-def
|
|
- nad
|
|
versions:
|
|
- name: v1
|
|
served: true
|
|
storage: true
|
|
schema:
|
|
openAPIV3Schema:
|
|
description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
|
|
Working Group to express the intent for attaching pods to one or more logical or physical
|
|
networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
|
|
type: object
|
|
properties:
|
|
apiVersion:
|
|
description: 'APIVersion defines the versioned schema of this represen
|
|
tation of an object. Servers should convert recognized schemas to the
|
|
latest internal value, and may reject unrecognized values. More info:
|
|
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
|
|
type: string
|
|
kind:
|
|
description: 'Kind is a string value representing the REST resource this
|
|
object represents. Servers may infer this from the endpoint the client
|
|
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
|
|
type: string
|
|
metadata:
|
|
type: object
|
|
spec:
|
|
description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
|
|
type: object
|
|
properties:
|
|
config:
|
|
description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
|
|
type: string
|
|
---
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: multus
|
|
rules:
|
|
- apiGroups: ["k8s.cni.cncf.io"]
|
|
resources:
|
|
- '*'
|
|
verbs:
|
|
- '*'
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- pods
|
|
- pods/status
|
|
verbs:
|
|
- get
|
|
- list
|
|
- update
|
|
- watch
|
|
- apiGroups:
|
|
- "resource.k8s.io"
|
|
resources:
|
|
- resourceclaims
|
|
- resourceclaims/status
|
|
- resourceslices
|
|
verbs:
|
|
- get
|
|
- list
|
|
- apiGroups:
|
|
- ""
|
|
- events.k8s.io
|
|
resources:
|
|
- events
|
|
verbs:
|
|
- create
|
|
- patch
|
|
- update
|
|
---
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: multus
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: multus
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: multus
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: multus
|
|
namespace: kube-system
|
|
---
|
|
# Multus is the only writer of /etc/cni/net.d (#832). Cilium writes its conflist
|
|
# to /etc/cni/net.d/multus/, a subdirectory the container runtime does not read
|
|
# (libcni skips directories); Multus picks it up there via multusAutoconfigDir
|
|
# and writes 00-multus.conf. Until that file exists the runtime has no network
|
|
# config at all, so no pod can be created with Cilium alone after a reboot.
|
|
#
|
|
# The "-subdir" names date from the node-by-node rollout; renaming would roll
|
|
# Multus again for nothing.
|
|
kind: ConfigMap
|
|
apiVersion: v1
|
|
metadata:
|
|
name: multus-daemon-config-subdir
|
|
namespace: kube-system
|
|
labels:
|
|
tier: node
|
|
app: multus
|
|
data:
|
|
daemon-config.json: |
|
|
{
|
|
"chrootDir": "/hostroot",
|
|
"cniVersion": "0.3.1",
|
|
"logLevel": "verbose",
|
|
"logToStderr": true,
|
|
"cniConfigDir": "/host/etc/cni/net.d",
|
|
"multusAutoconfigDir": "/host/etc/cni/net.d/multus",
|
|
"multusConfigFile": "auto",
|
|
"socketDir": "/host/run/multus/"
|
|
}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: kube-multus-ds-subdir
|
|
namespace: kube-system
|
|
labels:
|
|
tier: node
|
|
app: multus
|
|
name: multus-subdir
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
name: multus-subdir
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
tier: node
|
|
app: multus
|
|
name: multus-subdir
|
|
spec:
|
|
# Upstream ships this DaemonSet with no priority class, so it runs at 0
|
|
# and can preempt nothing. Any event that makes the pod need a fresh
|
|
# scheduling decision — a rollout, a node restart, an eviction — then has
|
|
# to fit it onto a node that may since have filled up, and if it does not
|
|
# fit it simply stays Pending. The node has no multus after that, and
|
|
# nothing reports it: pods still start, they just never get their
|
|
# secondary attachment, and the annotation everything waits on never
|
|
# appears.
|
|
#
|
|
# Seen on a cold rebuild: two of four nodes sat at 99% memory requests
|
|
# when the DaemonSet rolled, both multus pods went Pending, and Rook's
|
|
# network canary hung in Init:0/1 waiting for a network-status annotation
|
|
# that could never come. The CephCluster stayed in Progressing, which is
|
|
# what infrastructure-ceph-cluster gates on, so the whole apps layer
|
|
# stopped behind a scheduling decision three layers away.
|
|
#
|
|
# node-critical, the same class cilium and cni-dhcp-daemon carry, because
|
|
# that is what this is: without multus a node cannot attach a secondary
|
|
# network, and only that node is affected. Nothing cluster-wide fails.
|
|
#
|
|
# This does outrank kube-apiserver, which Talos runs as a static pod at
|
|
# system-cluster-critical (2000000000, against 2000001000 here), and on a
|
|
# control plane that has run out of memory the kubelet will pick the API
|
|
# server as the pod to evict. That was observed twice, at 6GiB and at
|
|
# 8GiB.
|
|
#
|
|
# It is not a reason to demote this DaemonSet. Kubernetes puts the CNI
|
|
# above the control plane deliberately — a node without its CNI cannot run
|
|
# anything at all — and a class name is a statement about what the pod is,
|
|
# not a dial for picking a number. The eviction says the control plane is
|
|
# starved, and that is where it has to be fixed: enough allocatable
|
|
# memory, or kubeReserved/systemReserved so the static pods keep what they
|
|
# need. Demoting this to cluster-critical would only move the victim.
|
|
priorityClassName: system-node-critical
|
|
hostNetwork: true
|
|
hostPID: true
|
|
tolerations:
|
|
- operator: Exists
|
|
effect: NoSchedule
|
|
- operator: Exists
|
|
effect: NoExecute
|
|
serviceAccountName: multus
|
|
containers:
|
|
- name: kube-multus
|
|
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.3.1-thick@sha256:a357a79359b80ddd5f4699117946cacf3b54f5baecf7944535594863e875fc25
|
|
command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
|
|
resources:
|
|
# Bumped from upstream's 100m/50Mi: on a clean rebuild the edge
|
|
# node sees a thundering herd of CNI ADDs (home-assistant,
|
|
# matter-server, otbr, music-assistant-data-seed scheduled together
|
|
# while every other workload is also rolling) all serialised
|
|
# through this one multus-daemon → cilium-cni delegate. With
|
|
# 50Mi the daemon gets SIGKILLed mid-ADD; CRI logs show
|
|
# `multus-shim failed (add): netplugin failed with no error
|
|
# message: signal: killed`, which leaves containerd holding stale
|
|
# sandbox-name reservations and wedges those pods for the rest
|
|
# of the bootstrap window. Bumping to 256Mi limit / 128Mi request
|
|
# keeps the daemon alive through the storm; CPU stays at 100m
|
|
# request, no limit (CNI-ADD is bursty).
|
|
requests:
|
|
cpu: "100m"
|
|
memory: "128Mi"
|
|
limits:
|
|
memory: "256Mi"
|
|
securityContext:
|
|
privileged: true
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
volumeMounts:
|
|
- name: cni
|
|
mountPath: /host/etc/cni/net.d
|
|
# Exists before Multus starts, so it waits for Cilium's conflist
|
|
# instead of exiting on a missing directory.
|
|
- name: cni-cilium-conf
|
|
mountPath: /host/etc/cni/net.d/multus
|
|
# multus-daemon expects that cnibin path must be identical between pod and container host.
|
|
# e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
|
|
# not to any other directory, like '/opt/bin' or '/usr/bin'.
|
|
- name: cnibin
|
|
mountPath: /opt/cni/bin
|
|
- name: host-run
|
|
mountPath: /host/run
|
|
- name: host-var-lib-cni-multus
|
|
mountPath: /var/lib/cni/multus
|
|
- name: host-var-lib-kubelet
|
|
mountPath: /var/lib/kubelet
|
|
mountPropagation: HostToContainer
|
|
- name: host-run-k8s-cni-cncf-io
|
|
mountPath: /run/k8s.cni.cncf.io
|
|
- name: host-run-netns
|
|
mountPath: /run/netns
|
|
mountPropagation: HostToContainer
|
|
- name: multus-daemon-config
|
|
mountPath: /etc/cni/net.d/multus.d
|
|
readOnly: true
|
|
- name: hostroot
|
|
mountPath: /hostroot
|
|
mountPropagation: HostToContainer
|
|
- mountPath: /etc/cni/multus/net.d
|
|
name: multus-conf-dir
|
|
env:
|
|
- name: MULTUS_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
initContainers:
|
|
- name: install-multus-binary
|
|
image: ghcr.io/k8snetworkplumbingwg/multus-cni:v4.3.1-thick@sha256:a357a79359b80ddd5f4699117946cacf3b54f5baecf7944535594863e875fc25
|
|
command:
|
|
- "/usr/src/multus-cni/bin/install_multus"
|
|
- "-d"
|
|
- "/host/opt/cni/bin"
|
|
- "-t"
|
|
- "thick"
|
|
resources:
|
|
requests:
|
|
cpu: "10m"
|
|
memory: "15Mi"
|
|
securityContext:
|
|
privileged: true
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
volumeMounts:
|
|
- name: cnibin
|
|
mountPath: /host/opt/cni/bin
|
|
mountPropagation: Bidirectional
|
|
terminationGracePeriodSeconds: 30
|
|
volumes:
|
|
- name: cni
|
|
hostPath:
|
|
path: /etc/cni/net.d
|
|
- name: cni-cilium-conf
|
|
hostPath:
|
|
path: /etc/cni/net.d/multus
|
|
type: DirectoryOrCreate
|
|
- name: cnibin
|
|
hostPath:
|
|
path: /opt/cni/bin
|
|
- name: hostroot
|
|
hostPath:
|
|
path: /
|
|
- name: multus-daemon-config
|
|
configMap:
|
|
name: multus-daemon-config-subdir
|
|
items:
|
|
- key: daemon-config.json
|
|
path: daemon-config.json
|
|
- name: host-run
|
|
hostPath:
|
|
path: /run
|
|
- name: host-var-lib-cni-multus
|
|
hostPath:
|
|
path: /var/lib/cni/multus
|
|
- name: host-var-lib-kubelet
|
|
hostPath:
|
|
path: /var/lib/kubelet
|
|
- name: host-run-k8s-cni-cncf-io
|
|
hostPath:
|
|
path: /run/k8s.cni.cncf.io
|
|
- name: host-run-netns
|
|
hostPath:
|
|
path: /run/netns/
|
|
- name: multus-conf-dir
|
|
hostPath:
|
|
path: /etc/cni/multus/net.d
|