diff --git a/kubernetes/RUNBOOK-cold-rebuild.md b/kubernetes/RUNBOOK-cold-rebuild.md index e92f020e..04477a72 100644 --- a/kubernetes/RUNBOOK-cold-rebuild.md +++ b/kubernetes/RUNBOOK-cold-rebuild.md @@ -119,9 +119,11 @@ against do not exist, and the node ends up with no address on any interface. Beyond that it carries: - `siderolabs/intel-ucode` — no hypervisor supplies microcode here -- `siderolabs/i915` — the Alder Lake-P Iris Xe. Nothing requests `/dev/dri` - today; present so wanting hardware transcoding later is a config change - rather than another rolling reinstall +- `siderolabs/i915` — the Alder Lake-P Iris Xe. The Intel GPU device plugin + (`infrastructure/intel-gpu-plugin`) turns this into the schedulable resource + `gpu.intel.com/i915`, which `apps/llama-server` requests for LLM inference + and which any hardware-transcoding workload would use too. Without the + extension the plugin finds no `/dev/dri` and advertises nothing - `siderolabs/thunderbolt` — inert until `machine.kernel.modules` requests `thunderbolt`/`thunderbolt_net`, present for the same reason. Note its udev rule auto-authorises any Thunderbolt device attached once those modules load diff --git a/kubernetes/apps/llama-server/configmap.yaml b/kubernetes/apps/llama-server/configmap.yaml new file mode 100644 index 00000000..26340d0e --- /dev/null +++ b/kubernetes/apps/llama-server/configmap.yaml @@ -0,0 +1,44 @@ +--- +# llama-server router presets. Running the server with --models-preset and no +# -m puts it in router mode: every section below is a separately supervised +# model subprocess, all reachable on one port through one OpenAI-compatible +# /v1 API. Requests pick a section by name via the JSON body's "model" field, +# so chat and embeddings are served by a single Deployment and Service. +# +# Keys are llama-server command-line arguments without the leading dashes. +# +# `hf-repo` makes this file the only place a model is named: llama-server +# resolves and downloads the weights itself into LLAMA_CACHE (the models PVC) +# on first load, so changing a model here needs no matching change anywhere +# else. The quant after the colon is optional and case-insensitive. +apiVersion: v1 +kind: ConfigMap +metadata: + name: llama-server-models + namespace: ai +data: + models.ini: | + version = 1 + + [*] + # Every layer runs on the Intel iGPU. Measured on this hardware (i9-12900H, + # Iris Xe 96 EU): full offload matches CPU throughput at ~10.8 tok/s decode + # while leaving the node's 14 cores free for the rest of the cluster. The + # iGPU is here for that isolation, not for a speed-up. + n-gpu-layers = 99 + # Generation is memory-bandwidth-bound, so extra threads buy nothing once + # the layers are on the GPU. Keeps the pod within its CPU limit. + threads = 4 + + [chat] + hf-repo = bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M + ctx-size = 8192 + load-on-startup = true + + [embed] + hf-repo = Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0 + # Serves POST /v1/embeddings. Set per-model rather than globally: the flag + # restricts a model to embeddings and would disable chat on [chat]. + embedding = true + ctx-size = 2048 + load-on-startup = true diff --git a/kubernetes/apps/llama-server/deployment.yaml b/kubernetes/apps/llama-server/deployment.yaml new file mode 100644 index 00000000..6231c0c6 --- /dev/null +++ b/kubernetes/apps/llama-server/deployment.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llama-server + namespace: ai + labels: + app.kubernetes.io/name: llama-server +spec: + replicas: 1 + # One iGPU per node and one RWX volume of weights: a rolling update would + # start a second copy that loads the same models onto the same device. + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: llama-server + template: + metadata: + labels: + app.kubernetes.io/name: llama-server + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + # Makes the CephFS volume group-writable so the initContainer can + # write the downloaded weights. + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: llama-server + image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731 + args: + # No -m: with only --models-preset the server starts in router + # mode and serves every section of the INI behind one /v1 API. + - --models-preset + - /config/models.ini + - --models-max + - "2" + # The server defaults to allowing every CORS origin, which would let + # any page a LAN browser visits call this unauthenticated API from + # the user's machine. Every consumer here talks to it server-side, + # so no cross-origin access is needed; the built-in WebUI is + # same-origin and unaffected. + - --cors-origins + - localhost + env: + # Where the `hf-repo` presets resolve to. Pointing the cache at the + # PVC is what makes the download a one-time cost: the weights are + # written here on first load and reused on every later start. Also + # keeps the server off the home directory it cannot write as UID 1000. + - name: LLAMA_CACHE + value: /models + ports: + - name: http + containerPort: 8080 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: 500m + memory: 12Gi + gpu.intel.com/i915: "1" + limits: + cpu: "6" + memory: 16Gi + gpu.intel.com/i915: "1" + # /health reports 503 until every load-on-startup model is resident. + # On a cold PVC that includes pulling ~5 GiB from Hugging Face before + # either model can load, so the startup budget is deliberately long + # (20 min); liveness only takes over once startup has succeeded. + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + failureThreshold: 120 + readinessProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 6 + volumeMounts: + - name: models + mountPath: /models + - name: config + mountPath: /config + readOnly: true + volumes: + - name: models + persistentVolumeClaim: + claimName: llama-server-models + - name: config + configMap: + name: llama-server-models +--- +apiVersion: v1 +kind: Service +metadata: + name: llama-server + namespace: ai + labels: + app.kubernetes.io/name: llama-server +spec: + selector: + app.kubernetes.io/name: llama-server + ports: + - name: http + port: 8080 + targetPort: http diff --git a/kubernetes/apps/llama-server/ingress.yaml b/kubernetes/apps/llama-server/ingress.yaml new file mode 100644 index 00000000..beeda810 --- /dev/null +++ b/kubernetes/apps/llama-server/ingress.yaml @@ -0,0 +1,28 @@ +--- +# LAN-only: no `dns.renner.uno/public` label, so the Cloudflare external-dns +# instance skips it and only the UDM instance publishes the record. The API is +# unauthenticated, so it must not be reachable from the internet. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llama-server + namespace: ai + annotations: + cert-manager.io/cluster-issuer: letsencrypt-cluster +spec: + ingressClassName: traefik + tls: + - hosts: + - llm.renner.uno + secretName: llama-server-tls + rules: + - host: llm.renner.uno + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: llama-server + port: + number: 8080 diff --git a/kubernetes/apps/llama-server/kustomization.yaml b/kubernetes/apps/llama-server/kustomization.yaml new file mode 100644 index 00000000..0db65f0e --- /dev/null +++ b/kubernetes/apps/llama-server/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - pvc.yaml + - configmap.yaml + - deployment.yaml + - ingress.yaml diff --git a/kubernetes/apps/llama-server/pvc.yaml b/kubernetes/apps/llama-server/pvc.yaml new file mode 100644 index 00000000..bd6a4168 --- /dev/null +++ b/kubernetes/apps/llama-server/pvc.yaml @@ -0,0 +1,21 @@ +--- +# Holds the GGUF weights, downloaded once by the Deployment's initContainer. +# Prune-disabled so retiring this Kustomization does not throw away several +# GiB of model files that would have to be pulled from Hugging Face again. +# +# ReadWriteMany so a second replica or a one-off benchmark Job can mount the +# same weights; the models are read-only at inference time. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: llama-server-models + namespace: ai + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + accessModes: + - ReadWriteMany + storageClassName: ceph-filesystem + resources: + requests: + storage: 40Gi diff --git a/kubernetes/clusters/homelab/llama-server.yaml b/kubernetes/clusters/homelab/llama-server.yaml new file mode 100644 index 00000000..4ecb076c --- /dev/null +++ b/kubernetes/clusters/homelab/llama-server.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: llama-server + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + # On a cold PVC llama-server pulls ~5 GiB of GGUF weights from Hugging Face + # and then loads both models onto the iGPU before /health goes green, so a + # first start is far slower than a normal app's. + timeout: 45m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/apps/llama-server + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster diff --git a/kubernetes/infrastructure/namespaces/namespaces.yaml b/kubernetes/infrastructure/namespaces/namespaces.yaml index da92438e..acfd306d 100644 --- a/kubernetes/infrastructure/namespaces/namespaces.yaml +++ b/kubernetes/infrastructure/namespaces/namespaces.yaml @@ -183,3 +183,15 @@ apiVersion: v1 kind: Namespace metadata: name: woodpecker +--- +# Local LLM inference (kubernetes/apps/llama-server). Declared here, like +# `woodpecker` above, so the `infrastructure` KS creates it before +# `infrastructure-crds` applies the namespace's CiliumNetworkPolicies. +# +# No pod-security labels: the workload needs no hostPath or privilege — the +# Intel GPU device plugin passes the iGPU in through the device-plugin API — +# so the cluster's default `baseline` enforcement is enough. +apiVersion: v1 +kind: Namespace +metadata: + name: ai diff --git a/kubernetes/infrastructure/network-policies/ai-ns.yaml b/kubernetes/infrastructure/network-policies/ai-ns.yaml new file mode 100644 index 00000000..deff7327 --- /dev/null +++ b/kubernetes/infrastructure/network-policies/ai-ns.yaml @@ -0,0 +1,96 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: default-deny-ingress + namespace: ai +spec: + endpointSelector: {} + ingress: + - fromEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": ai + - fromEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": ingress + app.kubernetes.io/name: traefik + - fromEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": monitoring + - fromEntities: + - host + - remote-node + # ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See + # home-assistant-ns.yaml for the rationale (CT-RELATED path not + # passing related ICMP empirically; per-namespace shape, mirroring + # commit 3c517a0's verified-safe pattern with the 66dbe9f type set). + - fromEndpoints: + - matchExpressions: + - key: io.kubernetes.pod.namespace + operator: Exists + icmps: + - fields: + - { type: 0, family: IPv4 } + - { type: 3, family: IPv4 } + - { type: 8, family: IPv4 } + - { type: 11, family: IPv4 } + - { type: 12, family: IPv4 } + - { type: 1, family: IPv6 } + - { type: 2, family: IPv6 } + - { type: 3, family: IPv6 } + - { type: 4, family: IPv6 } + - { type: 128, family: IPv6 } + - { type: 129, family: IPv6 } + - { type: 133, family: IPv6 } + - { type: 134, family: IPv6 } + - { type: 135, family: IPv6 } + - { type: 136, family: IPv6 } + - { type: 143, family: IPv6 } + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": ai + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": ingress + app.kubernetes.io/name: traefik + # rook-ceph: the model weights live on a CephFS RWX volume + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": rook-ceph + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + # host/remote-node: kubelet probes + node-local reach (Ceph mons on host net) + # world: image pulls and llama-server's Hugging Face model download + - toEntities: + - host + - remote-node + - world + # ICMP — egress mirror; see ingress block above. + - toEndpoints: + - matchExpressions: + - key: io.kubernetes.pod.namespace + operator: Exists + icmps: + - fields: + - { type: 0, family: IPv4 } + - { type: 3, family: IPv4 } + - { type: 8, family: IPv4 } + - { type: 11, family: IPv4 } + - { type: 12, family: IPv4 } + - { type: 1, family: IPv6 } + - { type: 2, family: IPv6 } + - { type: 3, family: IPv6 } + - { type: 4, family: IPv6 } + - { type: 128, family: IPv6 } + - { type: 129, family: IPv6 } + - { type: 133, family: IPv6 } + - { type: 134, family: IPv6 } + - { type: 135, family: IPv6 } + - { type: 136, family: IPv6 } + - { type: 143, family: IPv6 } diff --git a/kubernetes/infrastructure/network-policies/kustomization.yaml b/kubernetes/infrastructure/network-policies/kustomization.yaml index 0ef61488..11763e48 100644 --- a/kubernetes/infrastructure/network-policies/kustomization.yaml +++ b/kubernetes/infrastructure/network-policies/kustomization.yaml @@ -24,4 +24,5 @@ resources: - woodpecker-ns.yaml - trmnl-ns.yaml - wordpress-ns.yaml + - ai-ns.yaml - allow-cilium-health.yaml