From 7186e4d0359fc0675e236ef648e9f57572cd5c42 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 12:51:49 +0200 Subject: [PATCH] feat(gpu): iGPU workloads run only where a node declares the iGPU llama-server, llama-embed, wiki-embed (gpu.intel.com/i915) and the wiki behind them are non-essential and bound to the Intel iGPU. On a site without one (Hetzner, restore rehearsal runs 27/28) their pods stayed Pending, llama-server failed its health check and the Kustomizations held kustomize-controller workers; the GPU plugin and exporter DaemonSets sat on every node with nothing to serve. The node patches k8s-1..3 label their machines hardware.renner.uno/intel-gpu=true. The GPU plugin, the exporter, the three llama Deployments and the wiki select on it, and llama-server and wiki are no longer waited on. Elsewhere the DaemonSets have no pods and the Deployments stay unscheduled without blocking anything. Production rollout order: the label must be on the nodes (talosctl patch, no reboot) before these manifests reconcile, or the GPU plugin and the llama/wiki pods are removed or left Pending. Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/llama-server/deployment.yaml | 3 +++ kubernetes/apps/llama-server/embed.yaml | 3 +++ kubernetes/apps/llama-server/wiki-embed.yaml | 3 +++ kubernetes/apps/monitoring/intel-gpu-exporter.yaml | 2 ++ kubernetes/apps/wiki/deployment.yaml | 4 ++++ kubernetes/clusters/homelab/llama-server.yaml | 5 ++++- kubernetes/clusters/homelab/wiki.yaml | 3 ++- kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml | 1 + kubernetes/talos/patches/k8s-1.yaml | 3 +++ kubernetes/talos/patches/k8s-2.yaml | 2 ++ kubernetes/talos/patches/k8s-3.yaml | 2 ++ 11 files changed, 29 insertions(+), 2 deletions(-) diff --git a/kubernetes/apps/llama-server/deployment.yaml b/kubernetes/apps/llama-server/deployment.yaml index 57460f58..051729c0 100644 --- a/kubernetes/apps/llama-server/deployment.yaml +++ b/kubernetes/apps/llama-server/deployment.yaml @@ -44,6 +44,9 @@ spec: labels: app.kubernetes.io/name: llama-server spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/llama-server/embed.yaml b/kubernetes/apps/llama-server/embed.yaml index 5382f32d..9dd13a49 100644 --- a/kubernetes/apps/llama-server/embed.yaml +++ b/kubernetes/apps/llama-server/embed.yaml @@ -34,6 +34,9 @@ spec: labels: app.kubernetes.io/name: llama-embed spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/llama-server/wiki-embed.yaml b/kubernetes/apps/llama-server/wiki-embed.yaml index ba80bf11..405eb1cc 100644 --- a/kubernetes/apps/llama-server/wiki-embed.yaml +++ b/kubernetes/apps/llama-server/wiki-embed.yaml @@ -23,6 +23,9 @@ spec: labels: app.kubernetes.io/name: wiki-embed spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/monitoring/intel-gpu-exporter.yaml b/kubernetes/apps/monitoring/intel-gpu-exporter.yaml index 2ab049e1..5df5d89c 100644 --- a/kubernetes/apps/monitoring/intel-gpu-exporter.yaml +++ b/kubernetes/apps/monitoring/intel-gpu-exporter.yaml @@ -25,6 +25,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: seccompProfile: type: RuntimeDefault diff --git a/kubernetes/apps/wiki/deployment.yaml b/kubernetes/apps/wiki/deployment.yaml index 5c12bf59..970c51a2 100644 --- a/kubernetes/apps/wiki/deployment.yaml +++ b/kubernetes/apps/wiki/deployment.yaml @@ -39,6 +39,10 @@ spec: labels: app.kubernetes.io/name: wiki spec: + # Its embedding backend (wiki-embed) needs the Intel iGPU; without it + # the wiki has nothing to serve, so it runs where the iGPU is declared. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" automountServiceAccountToken: false securityContext: runAsNonRoot: true diff --git a/kubernetes/clusters/homelab/llama-server.yaml b/kubernetes/clusters/homelab/llama-server.yaml index 4165331c..754cb1fc 100644 --- a/kubernetes/clusters/homelab/llama-server.yaml +++ b/kubernetes/clusters/homelab/llama-server.yaml @@ -15,7 +15,10 @@ spec: name: flux-system path: ./kubernetes/apps/llama-server prune: true - wait: true + # Non-essential, and its pods run only on nodes with the Intel iGPU + # (hardware.renner.uno/intel-gpu); on a site without one they stay + # Pending, which must not hold a reconcile worker or any dependent. + wait: false dependsOn: - name: infrastructure - name: infrastructure-crds diff --git a/kubernetes/clusters/homelab/wiki.yaml b/kubernetes/clusters/homelab/wiki.yaml index 2e6c52cd..4b89f96f 100644 --- a/kubernetes/clusters/homelab/wiki.yaml +++ b/kubernetes/clusters/homelab/wiki.yaml @@ -13,7 +13,8 @@ spec: name: flux-system path: ./kubernetes/apps/wiki prune: true - wait: true + # Non-essential and iGPU-bound like llama-server: not waited on. + wait: false dependsOn: - name: infrastructure - name: infrastructure-crds diff --git a/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml b/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml index 556dfe76..fb9cde14 100644 --- a/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml +++ b/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml @@ -119,3 +119,4 @@ spec: type: DirectoryOrCreate nodeSelector: kubernetes.io/arch: amd64 + hardware.renner.uno/intel-gpu: "true" diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index 1c3fead3..0a12f186 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -54,6 +54,9 @@ machine: # this label instead of listing hostnames, so which boxes carry storage is # stated once, here, by the machine that knows. storage.renner.uno/ceph-osd: "true" + # This machine has the Intel iGPU. The GPU plugin and everything that needs + # gpu.intel.com run only on nodes with this label. + hardware.renner.uno/intel-gpu: "true" # Publishes this node's pod0 addresses, declared below in its VLANConfig, so # the bird DaemonSet reads its peers from the node list rather than from a # copy. Annotations, not labels: a label value cannot contain a colon. diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index a5ceabf1..fdc3484e 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -34,6 +34,8 @@ machine: topology.kubernetes.io/zone: saturn # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::12 diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index a26b57e5..050297d4 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -30,6 +30,8 @@ machine: topology.kubernetes.io/zone: neptune # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::13