Compare commits
43
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d82b96a751 | ||
|
|
463cfa7bde | ||
|
|
761bf9ba1e | ||
|
|
7b446194b3 | ||
|
|
9eaafeb92f | ||
|
|
67d506e1ac | ||
|
|
932d07edbe | ||
|
|
1a1205110d | ||
|
|
aed827ae5b | ||
|
|
3abcec5af7 | ||
|
|
b14bf0cc5e | ||
|
|
58444c9862 | ||
|
|
5f3aecb381 | ||
|
|
4ad47b3925 | ||
|
|
2131c7b5f5 | ||
|
|
d35c06d21d | ||
|
|
1bbff9fca4 | ||
|
|
9a521568c0 | ||
|
|
3b3ed4fbc4 | ||
|
|
c40466a3e6 | ||
|
|
ff0b3434c1 | ||
|
|
2cfebc2f8b | ||
|
|
3481e43032 | ||
|
|
98475a47e2 | ||
|
|
6f20955e3b | ||
|
|
92ac021931 | ||
|
|
5bbca6a162 | ||
|
|
edb07a344e | ||
|
|
260f548e67 | ||
|
|
ef6bc24491 | ||
|
|
779f557624 | ||
|
|
1fbefb3c39 | ||
|
|
f97aa077f0 | ||
|
|
8aa662e997 | ||
|
|
45012470c1 | ||
|
|
688f84dc29 | ||
|
|
6335954d6c | ||
|
|
f0701370db | ||
|
|
9131f73d6c | ||
|
|
7e19ed1b59 | ||
|
|
41556cbb16 | ||
|
|
377270375f | ||
|
|
df9019d90e |
@@ -26,8 +26,16 @@ edge_open3e_poll_seconds: 30
|
||||
# entries when added), so the multi-frame read is cheap at a 30 s poll.
|
||||
# Records faults/info events only (NormalOperation, LagDeviceDtcReported,
|
||||
# LegionellaProtectionTemperatureReached) — NOT setpoint or program changes.
|
||||
# onebase.*: the 29 paired ZigBee slots on the One Base gateway (ECU 0x6c3) --
|
||||
# 8 radiator valves, 9 climate sensors, 9 underfloor actuators, 3 floor
|
||||
# thermostats. Per-device room temperature, active setpoint, humidity, valve
|
||||
# opening/demand, battery and RSSI; the local replacement for the ViCare cloud
|
||||
# room sensors. Needs the onebase overlay (files/open3e-onebase.py) -- the stock
|
||||
# codec declares 57 bytes and a paired slot answers with 68.
|
||||
# Only CurrentValues are polled. The room and device registries (1884.., 2084..)
|
||||
# are static configuration and are read on demand, not every 30 s.
|
||||
edge_open3e_datapoints: >-
|
||||
vitocal.259,vitocal.260,vitocal.268,vitocal.269,vitocal.271,vitocal.274,vitocal.286,vitocal.288,vitocal.318,vitocal.321,vitocal.324,vitocal.335,vitocal.336,vitocal.396,vitocal.402,vitocal.403,vitocal.476,vitocal.477,vitocal.543,vitocal.548,vitocal.565,vitocal.1007,vitocal.1043,vitocal.1190,vitocal.1211,vitocal.1391,vitocal.1416,vitocal.1417,vitocal.2320,vitocal.2346,vitocal.2350,vitocal.2351,vitocal.2352,vitocal.2369,vitocal.2370,vitocal.2486,vitocal.2487,vitocal.2488,vitocal.2543,vitocal.2560,vitocal.2622,vitocal.2624,vitocal.2625,vitocal.2735,vitocal.3016,vx3.266,vx3.378,vx3.379,vx3.380,vx3.535,vx3.1603,vx3.1664,vx3.1690,vx3.1801,vx3.1802,vx3.1810,vx3.1828,vx3.1830,vx3.1577,vx3.1578,vx3.1579,vx3.1580,vx3.1581,vx3.1582,vx3.1834,vx3.1836,vx3.2214,vx3.2240,vx3.3085,vx3.3086,vx3.3087,vx3.3088,vx3.3089,vx3.3090,vcmu.320,vcmu.322,vcmu.325,vcmu.355,vcmu.389,vcmu.391,vcmu.1772,vcmu.1773,vcmu.1775,vcmu.1776,vcmu.2256,vcmu.2259,vcmu.2333,vcmu.2334,vcmu.2569,vcmu.2571,vitocal.2791
|
||||
vitocal.259,vitocal.260,vitocal.268,vitocal.269,vitocal.271,vitocal.274,vitocal.286,vitocal.288,vitocal.318,vitocal.321,vitocal.324,vitocal.335,vitocal.336,vitocal.396,vitocal.402,vitocal.403,vitocal.476,vitocal.477,vitocal.543,vitocal.548,vitocal.565,vitocal.1007,vitocal.1043,vitocal.1190,vitocal.1211,vitocal.1391,vitocal.1416,vitocal.1417,vitocal.2320,vitocal.2346,vitocal.2350,vitocal.2351,vitocal.2352,vitocal.2369,vitocal.2370,vitocal.2486,vitocal.2487,vitocal.2488,vitocal.2543,vitocal.2560,vitocal.2622,vitocal.2624,vitocal.2625,vitocal.2735,vitocal.3016,vx3.266,vx3.378,vx3.379,vx3.380,vx3.535,vx3.1603,vx3.1664,vx3.1690,vx3.1801,vx3.1802,vx3.1810,vx3.1828,vx3.1830,vx3.1577,vx3.1578,vx3.1579,vx3.1580,vx3.1581,vx3.1582,vx3.1834,vx3.1836,vx3.2214,vx3.2240,vx3.3085,vx3.3086,vx3.3087,vx3.3088,vx3.3089,vx3.3090,vcmu.320,vcmu.322,vcmu.325,vcmu.355,vcmu.389,vcmu.391,vcmu.1772,vcmu.1773,vcmu.1775,vcmu.1776,vcmu.2256,vcmu.2259,vcmu.2333,vcmu.2334,vcmu.2569,vcmu.2571,vitocal.2791,onebase.2086,onebase.2089,onebase.2092,onebase.2095,onebase.2098,onebase.2101,onebase.2104,onebase.2107,onebase.2110,onebase.2113,onebase.2116,onebase.2119,onebase.2122,onebase.2125,onebase.2128,onebase.2131,onebase.2134,onebase.2137,onebase.2140,onebase.2143,onebase.2262,onebase.2265,onebase.2268,onebase.2271,onebase.2274,onebase.2277,onebase.2280,onebase.2283,onebase.2286
|
||||
# ECU addresses; -dev selects codec set, tx selects the UDS address.
|
||||
# vcmu (refrigerant-circuit controller) has no dedicated codec set —
|
||||
# omitting dpList makes open3e fall back to the generic datapoint list,
|
||||
@@ -38,6 +46,9 @@ edge_open3e_devices:
|
||||
# files/open3e-vx3plus.py) — die Stock-vx3-Liste filtert sie sonst weg.
|
||||
vx3: { tx: "0x6a1", dpList: "/config/vx3plus.py" }
|
||||
vcmu: { tx: "0x68c" }
|
||||
# One Base ZigBee gateway. Overlay re-declares the paired ZigBee slots at
|
||||
# their real 68-byte length (see files/open3e-onebase.py).
|
||||
onebase: { tx: "0x6c3", dpList: "/config/onebase.py" }
|
||||
|
||||
edge_e3oncan_gap_seconds: 10
|
||||
edge_e3oncan_mqtt_format: "{device}_{didNumber:04d}_{didName}"
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# Managed by ansible (roles/heating-can). Datapoint overlay for the One Base
|
||||
# ZigBee gateway on ECU 0x6c3 (BACKENDGATEWAY).
|
||||
#
|
||||
# Upstream declares ZigBee<N>DeviceCurrentValues as 57 bytes. A PAIRED device
|
||||
# slot on this gateway answers with 68: the 57 known bytes plus the parent
|
||||
# device serial and three trailing bytes. udsoncan reads only the declared
|
||||
# length and then treats the next two bytes -- the first half of that serial --
|
||||
# as a foreign data identifier, so every paired slot fails to decode with
|
||||
# "Server returned values for data identifier 0x.... that was not requested".
|
||||
# The bogus identifier differs per device because the serial does.
|
||||
#
|
||||
# This overlay re-declares the 29 paired slots at 68 bytes, reusing upstream's
|
||||
# sub-type list so the per-device-type O3ESwitch (climate sensor / TRV / floor
|
||||
# thermostat / actuator, incl. DeviceTemperatureSetpoint) is inherited rather
|
||||
# than copied.
|
||||
#
|
||||
# EMPTY slots answer with 57 and decode against the stock codec, so they are
|
||||
# left alone and are not polled. A slot that is later unpaired therefore starts
|
||||
# reporting a decode error on its topic instead of a value.
|
||||
#
|
||||
# Verified on ECU 0x6c3, 2026-09-17: 29 paired slots, all 68 bytes.
|
||||
from open3e.Open3Ecodecs import O3EComplexType, RawCodec
|
||||
from open3e.Open3Edatapoints import dataIdentifiers as _master
|
||||
|
||||
# ZigBee<N>DeviceCurrentValues of the paired slots: devices 1-20 run 2086..2143
|
||||
# in steps of 3, devices 21-29 continue at 2262..2286.
|
||||
_PAIRED_CURRENT_VALUES = [2086 + 3 * n for n in range(20)] + [
|
||||
2262 + 3 * n for n in range(9)
|
||||
]
|
||||
|
||||
|
||||
def _with_parent_serial(did):
|
||||
stock = _master["dids"][did]
|
||||
return O3EComplexType(
|
||||
68,
|
||||
stock.id,
|
||||
list(stock.subTypes)
|
||||
+ [
|
||||
RawCodec(8, "ParentDeviceSerialnumber"),
|
||||
RawCodec(2, "Unknown_65_66"),
|
||||
RawCodec(1, "Unknown_67"),
|
||||
],
|
||||
desc=stock.desc,
|
||||
info=stock.info,
|
||||
acc=stock.acc,
|
||||
)
|
||||
|
||||
|
||||
dataIdentifiers = {
|
||||
"dids": {
|
||||
**_master["dids"],
|
||||
**{did: _with_parent_serial(did) for did in _PAIRED_CURRENT_VALUES},
|
||||
}
|
||||
}
|
||||
@@ -129,6 +129,13 @@
|
||||
mode: "0644"
|
||||
notify: Restart heating-can
|
||||
|
||||
- name: Deploy open3e One Base datapoint overlay (68-byte ZigBee slots)
|
||||
ansible.builtin.copy:
|
||||
src: open3e-onebase.py
|
||||
dest: "{{ heating_can_base_dir }}/onebase.py"
|
||||
mode: "0644"
|
||||
notify: Restart heating-can
|
||||
|
||||
# --- Compose project ---------------------------------------------------------
|
||||
- name: Deploy the broker credential env file
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -17,6 +17,7 @@ services:
|
||||
volumes:
|
||||
- ./devices.json:/config/devices.json:ro
|
||||
- ./vx3plus.py:/config/vx3plus.py:ro
|
||||
- ./onebase.py:/config/onebase.py:ro
|
||||
env_file:
|
||||
# Broker credential, 0600 -- see heating-can.env.j2.
|
||||
- heating-can.env
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# Traffic classes on the k8s nodes
|
||||
|
||||
Issue #576. Three of the four classes are separated; management is not.
|
||||
|
||||
## Where each class runs
|
||||
|
||||
| class | link | mechanism |
|
||||
|---|---|---|
|
||||
| Ceph replication | `ceph-san`, VLAN 49 | the whole link is handed to the OSD pod (Multus `host-device`) |
|
||||
| world→pod | `ingress0`, VLAN 47 | BGP session sourced there, so the advertised next hop is that address |
|
||||
| pod→world | `ingress0`, VLAN 47 | routing rule on the pod source prefix → table 47 |
|
||||
| pod↔pod | `pod0`, VLAN 48 | routing rule on pod source **and** destination → table 48 |
|
||||
| management | `mgmt0`, VLAN 40 | not started |
|
||||
|
||||
Hard constraint, set by the owner: inter-node and pod traffic never traverses
|
||||
the UDM. Everything above stays on the 10G switch.
|
||||
|
||||
## Why policy routing rather than moving the node IP
|
||||
|
||||
Cilium reaches a peer's pods at that peer's Kubernetes InternalIP, in native
|
||||
and tunnel mode alike. `--ipv4-node` is overwritten from the Node object, and
|
||||
`nodePort.directRoutingDevice` selects the BPF NodePort device, not a next hop.
|
||||
Both were established by test (#837, #838), not by reading.
|
||||
|
||||
So the InternalIP decides where pod traffic goes — unless the lookup happens in
|
||||
a different table. The rules do that, and nothing about the node's identity
|
||||
moves. The routing decision runs before Cilium's masquerade hook, so a rule
|
||||
matching the pod source sees the pod address, and the masquerade then rewrites
|
||||
it to the egress link's.
|
||||
|
||||
## Rule order
|
||||
|
||||
Lower priority runs first.
|
||||
|
||||
| prio | match | table |
|
||||
|---|---|---|
|
||||
| 980 | src pod, dst pod | 48 |
|
||||
| 981 | src pod, dst pod (v4) | 48 |
|
||||
| 990 | dst `fd00::/8` | main |
|
||||
| 991 | dst `10.0.0.0/8` | main |
|
||||
| 1000 | src pod | 47 |
|
||||
| 1001 | src pod (v4) | 47 |
|
||||
|
||||
980/981 match source **and** destination. Source alone would also catch the
|
||||
node's own traffic to a pod, which would then leave with a `pod0` source — an
|
||||
address `CiliumNode.spec.addresses` does not list, so Cilium classifies it as
|
||||
`world` rather than `host`.
|
||||
|
||||
990/991 are Cilium's `ipv4/ipv6NativeRoutingCIDR`, and they are what keeps
|
||||
cluster-internal traffic out of table 47. Without them a pod's packet to a
|
||||
service matches the source rule, finds only a default route, and leaves for the
|
||||
UDM — measured, in-cluster DNS stopped resolving.
|
||||
|
||||
Traffic to a pod on the same node falls through on its own: table 48 holds only
|
||||
the peers' CIDRs, so the lookup misses and 990/991 send it to main.
|
||||
|
||||
## Where each table comes from
|
||||
|
||||
- **47** — two default routes declared per node in `talos/patches/k8s-*.yaml`.
|
||||
The IPv6 gateway is the UDM's configured address on the segment, not the
|
||||
RA-learned link-local; `accept_ra_defrtr` stays `0` on `ingress0` so the RA
|
||||
default does not land in main.
|
||||
- **48** — filled by the `bird` DaemonSet: an iBGP mesh over `pod0`, one route
|
||||
per peer pod CIDR with the peer's `pod0` address as next hop.
|
||||
|
||||
## The bird mesh
|
||||
|
||||
`kubernetes/infrastructure/bird/`. Each node advertises only its own pod CIDR
|
||||
and installs what it learns into table 48. Cilium's routes in main are
|
||||
untouched, which is why the two coexist.
|
||||
|
||||
No address is written into this repo twice. Each node publishes its `pod0`
|
||||
addresses as `nodeAnnotations`, beside the `VLANConfig` that assigns them; the
|
||||
pod CIDRs come from the Node objects. An init container renders both from the
|
||||
API with `kubectl -o go-template-file`, excludes the node itself with a field
|
||||
selector, and ends with `bird -p` so BIRD's own parser gates startup.
|
||||
|
||||
The node's own CIDR is **originated**, not read back from the kernel. Reading
|
||||
it back made BIRD depend on the shape Cilium installs, and the IPv4 route is
|
||||
rejected outright: its next hop sits inside the prefix it describes, which KRT
|
||||
refuses with `strange next-hop`. The IPv6 route is a device route and passed,
|
||||
so the failure was silent on one family only.
|
||||
|
||||
## Failure behaviour
|
||||
|
||||
Benign at every layer. An empty or absent table 48 makes 980/981 fall through
|
||||
to main, and pod traffic returns to VLAN 40 — unseparated, not broken. The same
|
||||
holds if the DaemonSet cannot start, which is the case during a cold rebuild
|
||||
until Gitea serves its image.
|
||||
|
||||
## Management, the remaining class
|
||||
|
||||
Not started. The levers differ per service and none of them is the node IP:
|
||||
|
||||
| service | can leave the node IP? | lever |
|
||||
|---|---|---|
|
||||
| etcd | yes | `cluster.etcd.advertisedSubnets`; needs a reboot per node |
|
||||
| kube-apiserver endpoint | yes | it is a VIP — `Layer2VIPConfig.link` |
|
||||
| Talos API | n/a | `apid` answers on every node address; a firewall question |
|
||||
| kubelet | no | it *is* the node IP |
|
||||
|
||||
Two approaches were rejected by the owner and are not to be proposed again:
|
||||
moving the node IP and etcd to `pod0` (#838), and relabelling VLAN 40 as the
|
||||
cluster VLAN while moving etcd and the API VIP to a new management VLAN.
|
||||
Replacing the CNI (#839, Antrea) was shelved.
|
||||
|
||||
Note that the reasoning which produced that list also concluded pod↔pod was
|
||||
blocked, which policy routing disproved. Treat it as a record of decisions, not
|
||||
as an exhaustive analysis.
|
||||
|
||||
## Sources
|
||||
|
||||
Cilium v1.20.1 `pkg/datapath/linux/node.go`, `pkg/node/types/node.go`,
|
||||
`pkg/node/sync/local_node_sync.go`; BIRD 2.17 user manual (kernel protocol,
|
||||
`learn all`, `strict bind`, `direct`); Talos v1.13 `RoutingRuleConfig` and
|
||||
`machine.nodeAnnotations`. PRs #831, #833, #835, #836, #837, #842, #852, #854,
|
||||
#855; closed unmerged: #830, #838, #839.
|
||||
@@ -1,194 +0,0 @@
|
||||
# Proposal — separate the four traffic classes onto their own VLANs
|
||||
|
||||
Status: draft. The question this started with is **answered** — from the Cilium
|
||||
source and flag reference, not from a test — and the answer changes the shape
|
||||
of the proposal rather than confirming it.
|
||||
|
||||
Supersedes the framing in issue #576, which described this as "give node
|
||||
management its own NIC, and Ceph public its own VLAN". That was two of four
|
||||
classes and one of them turned out already solved.
|
||||
|
||||
## The four classes
|
||||
|
||||
| class | what it is | where it goes today |
|
||||
|---|---|---|
|
||||
| **Ceph replication** | OSD-to-OSD heartbeat, replication, recovery | VLAN 49, its own X710 port — **separated** |
|
||||
| **cluster-internal** | pod-to-pod east-west, including Ceph client I/O | pod network, VXLAN over `mgmt0` |
|
||||
| **cluster-external** | ingress to LoadBalancer VIPs, egress to LAN and internet | `mgmt0` |
|
||||
| **management** | Talos API, etcd, kube-apiserver, kubelet | `mgmt0` |
|
||||
|
||||
Three of four share one interface. That is not a design decision — until the
|
||||
bare-metal conversion there was one NIC to work with.
|
||||
|
||||
Ceph client I/O living on the pod network is correct rather than a leftover:
|
||||
that traffic *is* cluster-internal. Ceph is done.
|
||||
|
||||
## Principle
|
||||
|
||||
**Each class gets its own VLAN. Which physical interface a VLAN lands on is a
|
||||
separate, changeable decision.**
|
||||
|
||||
Defining classes by VLAN means the separation survives recabling, port failures
|
||||
and capacity changes, and it does not require one port per class. Defining them
|
||||
by interface would freeze a capacity decision into the architecture.
|
||||
|
||||
## The finding that shapes everything else
|
||||
|
||||
The original open question was whether `nodePort.directRoutingDevice` moves the
|
||||
next-hop of Cilium's direct routes, or only selects an interface. It is
|
||||
answerable from the source, and the answer is: **only the interface.**
|
||||
|
||||
`installDirectRoute` takes the next-hop as a parameter:
|
||||
|
||||
```go
|
||||
func installDirectRoute(log *slog.Logger, prefix netip.Prefix, nodeIP net.IP,
|
||||
skipUnreachable bool) (routeSpec *netlink.Route, err error)
|
||||
```
|
||||
|
||||
and `nodeUpdate` supplies it as `newNode.GetNodeIP(false)`. `GetNodeIP` returns,
|
||||
in priority order, `NodeInternalIP` → `NodeExternalIP` → anything else. The
|
||||
same `newIP4` is what tunnel mode uses as the VXLAN endpoint.
|
||||
|
||||
`NodeInternalIP` comes from the Kubernetes Node object. `--ipv4-node` looks like
|
||||
an override but is not one: `initFromConfig()` calls `SetNodeInternalIP` and
|
||||
`initFromK8s()` runs **after** it and overwrites the value. In a Kubernetes
|
||||
cluster there is no path by which Cilium's node IP differs from the kubelet's
|
||||
`InternalIP`.
|
||||
|
||||
`--direct-routing-device` is documented as "Device name used to connect nodes in
|
||||
direct routing mode" — a device, not an address.
|
||||
|
||||
**So cluster-internal traffic is welded to the node IP, and the node IP is the
|
||||
kubelet InternalIP.** Two consequences:
|
||||
|
||||
- The class cannot be moved off whatever VLAN the node IP lives on.
|
||||
- **Routing mode is irrelevant to this problem.** Tunnel mode sources the VXLAN
|
||||
from the node IP; native mode uses it as the direct-route next-hop. Native
|
||||
routing buys nothing here, so the disruptive, non-rolling step this proposal
|
||||
previously contained is simply gone.
|
||||
|
||||
## Consequence: draw the line the other way
|
||||
|
||||
If cluster-internal cannot leave the node IP, then separation has to come from
|
||||
moving *management services* off it — and each one has to be checked
|
||||
individually, because they do not share a mechanism.
|
||||
|
||||
| management service | can it leave the node IP? | lever |
|
||||
|---|---|---|
|
||||
| etcd | **yes** | `cluster.etcd.advertisedSubnets` (+ `listenSubnets`, which defaults to it) |
|
||||
| kube-apiserver endpoint | **yes** | already a VIP — `Layer2VIPConfig` with a `link:` field |
|
||||
| Talos API | **n/a** | `apid` answers on every node address (measured); a client and firewall choice, not a bind setting |
|
||||
| kubelet | **no** | it *is* the node IP, by definition |
|
||||
|
||||
So four VLANs are reachable, but the boundary does not fall where the class
|
||||
names suggest: **kubelet stays with cluster-internal.** That is the honest
|
||||
result, and it is worth stating plainly rather than filing kubelet under
|
||||
management and hoping.
|
||||
|
||||
The remaining classes keep the levers established earlier:
|
||||
|
||||
- **cluster-external ingress** — `CiliumBGPNodeConfigOverride.spec.bgpInstances[].peers[].localAddress`.
|
||||
For eBGP the session source is the next-hop, so pinning the session pins
|
||||
where return traffic arrives. The session to the UDM has been running for
|
||||
months.
|
||||
- **cluster-external egress** — eBPF masquerading follows the output device, so
|
||||
this is a default-route decision plus `enable-masquerade-to-route-source`
|
||||
(today `false`).
|
||||
- **Ceph replication** — Multus macvlan on VLAN 49, done.
|
||||
|
||||
## The general escape hatch
|
||||
|
||||
Ceph is separated because its OSDs got a second interface and stopped using the
|
||||
pod network for replication. That is the *only* mechanism that fully detaches a
|
||||
workload from the node IP, and it generalises: anything that must not ride the
|
||||
node IP gets its own `NetworkAttachmentDefinition`.
|
||||
|
||||
It is also expensive per workload — an attachment, an IPAM pool, a pod spec
|
||||
change. Worth it for Ceph replication. Not worth it for east-west traffic in
|
||||
general.
|
||||
|
||||
## Recommended mapping
|
||||
|
||||
Two ways to arrange this, and they differ only in which VLAN gets relabelled:
|
||||
|
||||
**A — move the node IP to a new cluster VLAN.** VLAN 40 keeps etcd, the API VIP
|
||||
and Talos API. Cleanest match to the class names, but it changes every node's
|
||||
identity, which is the one address the whole cluster is administered through.
|
||||
|
||||
**B (recommended) — leave the node IP on VLAN 40 and move management off it.**
|
||||
VLAN 40 becomes the cluster VLAN; a new VLAN carries etcd and the API VIP.
|
||||
Identical end state, and no node ever changes its InternalIP.
|
||||
|
||||
Under B:
|
||||
|
||||
| VLAN | carries |
|
||||
|---|---|
|
||||
| 49 | Ceph replication |
|
||||
| 40 (relabelled *cluster*) | pod east-west, kubelet |
|
||||
| new *management* | etcd, kube-apiserver VIP, Talos API by convention |
|
||||
| new *external* | LoadBalancer ingress, egress default route |
|
||||
|
||||
The property the user asked for — management carries only management traffic —
|
||||
holds under B. The label moves; the property is what matters.
|
||||
|
||||
## Migration shape
|
||||
|
||||
Every step is rolling and reversible. The routing-mode change is gone.
|
||||
|
||||
1. **New VLANs** on the switch, trunked to the X710. Nothing uses them.
|
||||
2. **Addresses** on the new VLANs via Talos, one node at a time, `--mode=try`.
|
||||
Still unused; this only proves the L2 works.
|
||||
3. **etcd** to the management VLAN via `advertisedSubnets`. One node at a time,
|
||||
waiting for a healthy quorum between each.
|
||||
4. **API VIP** to the management VLAN via `Layer2VIPConfig.link`. DNS for
|
||||
`k8s.renner.uno` follows.
|
||||
5. **cluster-external**: BGP `localAddress`, default route, masquerade flag.
|
||||
Reversible by reverting the override; failure is visible immediately in
|
||||
`cilium-dbg shell -- bgp/peers`.
|
||||
6. Verify VLAN 40 by what is *absent* from it, not by what still works.
|
||||
|
||||
Step 3 is the one that deserves care — etcd is the only step where a mistake
|
||||
costs quorum rather than connectivity.
|
||||
|
||||
## What this does not buy
|
||||
|
||||
Management currently shares a 10 Gb link and is not contended. Nothing observed
|
||||
today is slow because of this. The case for the split is that traffic classes
|
||||
with different failure modes and security postures should not be
|
||||
indistinguishable — not that something is broken.
|
||||
|
||||
## Sources and measurements
|
||||
|
||||
Source, Cilium v1.20.1 (the running version):
|
||||
|
||||
- `pkg/datapath/linux/node.go` — `installDirectRoute(…, nodeIP net.IP, …)`,
|
||||
fed from `newIP4 = newNode.GetNodeIP(false)`; `createDirectRouteSpec` rejects
|
||||
a next-hop that is not directly reachable.
|
||||
- `pkg/node/types/node.go` — `GetNodeIP` prefers `NodeInternalIP`.
|
||||
- `pkg/node/sync/local_node_sync.go` — `initFromConfig()` applies
|
||||
`--ipv4-node`, `initFromK8s()` runs afterwards and overwrites it from the
|
||||
Kubernetes Node.
|
||||
- `cilium-agent` flag reference — `--direct-routing-device`: "Device name used
|
||||
to connect nodes in direct routing mode".
|
||||
|
||||
Talos:
|
||||
|
||||
- Multihoming guide — `cluster.etcd.advertisedSubnets`; `listenSubnets`
|
||||
defaults to it. kube-apiserver and Talos API listen addresses are not
|
||||
covered there.
|
||||
- `kubernetes/talos/patches/k8s-{1,2,3}.yaml` — `Layer2VIPConfig` for
|
||||
`10.2.40.60` with `link: mgmt0`.
|
||||
|
||||
Measured on the cluster:
|
||||
|
||||
- `apid` answered on a node address that is not the InternalIP, so the Talos
|
||||
API is not bound to one address.
|
||||
- k8s-1 currently holds one routable v4 address plus the VIP on `mgmt0`; its
|
||||
InternalIP had changed since the last session, which is why nothing here
|
||||
names an address as fixed.
|
||||
- Cilium runs `routing-mode: tunnel` with `enable-masquerade-to-route-source`
|
||||
unset and no direct-routing device configured.
|
||||
|
||||
Not verified: whether the kube-apiserver's `--advertise-address` follows the
|
||||
node IP independently of the VIP. It does not affect the recommendation, since
|
||||
clients reach the API through the VIP either way.
|
||||
@@ -70,7 +70,7 @@ spec:
|
||||
mountPath: /ca
|
||||
containers:
|
||||
- name: audiobookshelf
|
||||
image: ghcr.io/advplyr/audiobookshelf:2.36.0 # {"$imagepolicy": "flux-system:audiobookshelf"}
|
||||
image: ghcr.io/advplyr/audiobookshelf:2.36.1 # {"$imagepolicy": "flux-system:audiobookshelf"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 80
|
||||
|
||||
@@ -40,7 +40,7 @@ metadata:
|
||||
# so this Service produces no public record — LDAP stays
|
||||
# LAN-internal, which matches the threat model of the cleartext
|
||||
# bind on 389.
|
||||
external-dns.alpha.kubernetes.io/hostname: ldap.renner.uno
|
||||
external-dns.kubernetes.io/hostname: ldap.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -3,3 +3,4 @@ kind: Kustomization
|
||||
resources:
|
||||
- externalsecret.yaml
|
||||
- helmrelease.yaml
|
||||
- networkpolicy.yaml
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# The apps namespace is default-deny-ingress, so the ServiceMonitor added in
|
||||
# monitoring-rules/external-dns-servicemonitor.yaml scraped into a timeout.
|
||||
# The selector matches both releases — the Cloudflare one deployed here and
|
||||
# external-dns-unifi from infrastructure/ — since they share the chart's
|
||||
# app.kubernetes.io/name and land in the same namespace.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: external-dns-allow-metrics-scrape
|
||||
namespace: apps
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: monitoring
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "7979"
|
||||
protocol: TCP
|
||||
@@ -18,7 +18,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
lbipam.cilium.io/ips: "10.2.99.87,fd99::87"
|
||||
external-dns.alpha.kubernetes.io/hostname: ha.renner.uno
|
||||
external-dns.kubernetes.io/hostname: ha.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -95,7 +95,7 @@ spec:
|
||||
memory: 32Mi
|
||||
containers:
|
||||
- name: home-assistant
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.9.2 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
image: ghcr.io/home-assistant/home-assistant:2026.9.3 # {"$imagepolicy": "flux-system:home-assistant"}
|
||||
env:
|
||||
- name: TZ
|
||||
value: Europe/Berlin
|
||||
|
||||
@@ -19,7 +19,7 @@ metadata:
|
||||
# LAN can connect with `mqtt.renner.uno` (matches the broker cert
|
||||
# SAN) instead of the bare IP. No Cloudflare label — kept off the
|
||||
# public DNS; UniFi serves a split-DNS answer for LAN clients.
|
||||
external-dns.alpha.kubernetes.io/hostname: mqtt.renner.uno
|
||||
external-dns.kubernetes.io/hostname: mqtt.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -42,9 +42,11 @@ spec:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
# Seed the libation-config PVC from the bootstrap secret on
|
||||
# first run only — sentinel file gates re-runs so token
|
||||
# refreshes in AccountsSettings.json and DB updates from
|
||||
# subsequent liberate passes are preserved.
|
||||
# first run only — sentinel file gates re-runs so DB updates
|
||||
# from subsequent liberate passes are preserved. Refreshed
|
||||
# Audible tokens are preserved by the container args, not by
|
||||
# this sentinel: the entrypoint copies AccountsSettings.json
|
||||
# into ephemeral /config-internal rather than symlinking it.
|
||||
- name: seed-config
|
||||
image: busybox:1.37.0
|
||||
command: ["/bin/sh", "-eu", "-c"]
|
||||
@@ -66,6 +68,11 @@ spec:
|
||||
install -m 660 /bootstrap/AccountsSettings.json /libation-state/AccountsSettings.json
|
||||
install -m 660 /bootstrap/Settings.json /libation-state/Settings.json
|
||||
install -m 660 /bootstrap/LibationContext.db /libation-state/LibationContext.db
|
||||
# Without this key the seeded tokens cannot be decrypted:
|
||||
# TokenStorageMethod is Encrypted and the container has no
|
||||
# OS secret store, so a missing key file makes the
|
||||
# entrypoint mint a throwaway one on every run.
|
||||
install -m 660 /bootstrap/libation-master.key /libation-state/libation-master.key
|
||||
touch /libation-state/.bootstrapped
|
||||
ls -la /libation-state
|
||||
volumeMounts:
|
||||
@@ -85,7 +92,23 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: libation
|
||||
image: docker.io/rmcrackan/libation:latest
|
||||
# Pinned by digest: Spegel caches mutable tag resolutions, so
|
||||
# :latest stayed on 13.7.11 despite imagePullPolicy Always.
|
||||
image: docker.io/rmcrackan/libation:14.2.1@sha256:6f9d8ba19dc3803dd6ba295ab5a7abfb6ff3bc06286eebf4ecd8ccbe43245888
|
||||
# The entrypoint stages config into ephemeral /config-internal
|
||||
# by copy (only the DB is symlinked to the PVC), so Audible
|
||||
# tokens refreshed during a run are lost at container exit.
|
||||
# Passing args makes the entrypoint set up the environment and
|
||||
# then exec this instead of its own loop; the trap copies the
|
||||
# refreshed file back to the PVC on every exit path.
|
||||
args:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
trap 'cp /config-internal/AccountsSettings.json /libation-state/AccountsSettings.json' EXIT
|
||||
set -e
|
||||
/libation/LibationCli scan
|
||||
/libation/LibationCli liberate
|
||||
env:
|
||||
- name: LIBATION_CONFIG_DIR
|
||||
value: /libation-state
|
||||
|
||||
@@ -6,3 +6,5 @@ resources:
|
||||
- embed.yaml
|
||||
- gate-service.yaml
|
||||
- ingress.yaml
|
||||
- wiki-embed.yaml
|
||||
- wiki-embed-ingress.yaml
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
---
|
||||
# wiki-embed.renner.uno — LAN-only (no dns.renner.uno/public label), consumed by
|
||||
# bin/wiki_rank.py on the workstation and the nightly wiki jobs. Machine client,
|
||||
# so Traefik BasicAuth instead of the browser-only Authentik gate; same pattern
|
||||
# as kubernetes/apps/monitoring/loki-ingress.yaml.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: wiki-embed-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
basicAuth:
|
||||
secret: wiki-embed-basic-auth
|
||||
realm: wiki-embed
|
||||
---
|
||||
# OpenBao key apps/ai/wiki-embed-basic-auth, field `users` = htpasswd line.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wiki-embed-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: wiki-embed-basic-auth
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/ai/wiki-embed-basic-auth
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
traefik.ingress.kubernetes.io/router.middlewares: ai-wiki-embed-basic-auth@kubernetescrd
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- wiki-embed.renner.uno
|
||||
secretName: wiki-embed-tls
|
||||
rules:
|
||||
- host: wiki-embed.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: wiki-embed
|
||||
port:
|
||||
number: 8080
|
||||
@@ -0,0 +1,117 @@
|
||||
---
|
||||
# Embedding endpoint for the ~/Notes wiki ranker (bin/wiki_rank.py). Own
|
||||
# Deployment so wiki traffic never queues behind Paperless on llama-embed.
|
||||
# Same model and PVC as llama-embed, so no second download.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
replicas: 1
|
||||
# Above the 600s startupProbe budget; see embed.yaml.
|
||||
progressDeadlineSeconds: 900
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: wiki-embed
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
||||
args:
|
||||
- --hf-repo
|
||||
- gpustack/bge-m3-GGUF:Q8_0
|
||||
- --embeddings
|
||||
- --alias
|
||||
- bge-m3
|
||||
# bge-m3 is trained for CLS pooling; the GGUF default is not.
|
||||
- --pooling
|
||||
- cls
|
||||
- --n-gpu-layers
|
||||
- "99"
|
||||
- --threads
|
||||
- "2"
|
||||
# wiki_rank.py caps each note at 1500 chars and sends 2 per request.
|
||||
- --batch-size
|
||||
- "1536"
|
||||
- --ubatch-size
|
||||
- "1536"
|
||||
- --ctx-size
|
||||
- "4096"
|
||||
- --no-webui
|
||||
env:
|
||||
- name: LLAMA_CACHE
|
||||
value: /models
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
# Same sizing as llama-embed: 1.6 GiB measured peak, headroom for slots.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 2Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llama-server-models
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: wiki-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -0,0 +1,25 @@
|
||||
# Both external-dns releases (Cloudflare and the UniFi webhook) carry
|
||||
# app.kubernetes.io/name: external-dns and differ only by instance, so one
|
||||
# monitor covers both. jobLabel keeps them apart as separate job values.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: external-dns
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobLabel: app.kubernetes.io/instance
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- apps
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: external-dns
|
||||
endpoints:
|
||||
# The controller's own metrics. Port 8080 on the unifi release is the
|
||||
# webhook sidecar and exports no external_dns_* series.
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: 60s
|
||||
@@ -21,6 +21,14 @@ resources:
|
||||
# alerted if that stopped — HetznerBackupEndpointDown only probes the S3
|
||||
# endpoint, which proves reachability, not that anything was written.
|
||||
- prometheusrule-cnpg-backup.yaml
|
||||
# DNS publication. Nothing scraped external-dns, so it deleted six LAN
|
||||
# records on 2026-09-12 and stayed silent for five days.
|
||||
- prometheusrule-external-dns.yaml
|
||||
- external-dns-servicemonitor.yaml
|
||||
# Outcome check for the same failure: the names themselves, asked of the
|
||||
# UDM. Catches a cause external-dns cannot see, e.g. records removed there.
|
||||
- prometheusrule-dns-resolution.yaml
|
||||
- probes-dns.yaml
|
||||
- alloy-servicemonitor.yaml
|
||||
- cilium-servicemonitor.yaml
|
||||
- cnpg-podmonitors.yaml
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# DNS probes for the names external-dns publishes to the UDM from
|
||||
# LoadBalancer Services. One Probe per name because the dns prober's
|
||||
# query_name is module-scoped; the target is the resolver, queried on both
|
||||
# families so a single-stack blip does not read as a missing record.
|
||||
#
|
||||
# The UDM is queried directly rather than through the cluster resolver:
|
||||
# k8s-gateway caches these with `serve_stale 1h immediate`, which would keep
|
||||
# a deleted record green for an hour.
|
||||
#
|
||||
# `release` is load-bearing — probeSelector matches on it.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ha
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ha
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
# instance is the resolver and identical across these Probes, so the
|
||||
# queried name has to come from a label.
|
||||
labels:
|
||||
dns_name: ha.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-mqtt
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_mqtt
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: mqtt.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ldap
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ldap
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: ldap.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-syslog
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_syslog
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: syslog.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-ftp-scanner
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_ftp_scanner
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: ftp.paperless.renner.uno
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-lan-syncthing
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-dns
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_lan_syncthing
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- "[fd00::1]:53"
|
||||
- "10.2.40.1:53"
|
||||
labels:
|
||||
dns_name: syncthing.paperless.renner.uno
|
||||
@@ -0,0 +1,49 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: dns-resolution
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# ruleSelector matches on this label; without it the rule is created but
|
||||
# never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: dns-resolution.rules
|
||||
rules:
|
||||
# min by dns_name: both resolver families are probed, and the record
|
||||
# is only gone when neither answers. A single-family failure is the
|
||||
# UDM's dnsmasq refusing connections in bursts (k8s-gateway carries
|
||||
# the same note), which is why the window is 30m rather than the 10m
|
||||
# used for HTTP probes.
|
||||
- alert: LanDnsRecordMissing
|
||||
expr: min by (dns_name) (probe_success{job="blackbox-dns"}) == 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.dns_name }} no longer resolves to its LAN address"
|
||||
description: >-
|
||||
The UDM stopped answering for a name external-dns publishes from
|
||||
a LoadBalancer Service, or answered with the public
|
||||
*.renner.uno wildcard instead of the record. Clients that use
|
||||
the name rather than the address lose the service while
|
||||
everything in the cluster stays healthy — nothing else in the
|
||||
stack reports this. Check the external-dns alerts first, then
|
||||
the record on the UDM.
|
||||
|
||||
# Dead-man's-switch, mirroring BlackboxProbesAbsent: without probe
|
||||
# results the rule above cannot fire.
|
||||
- alert: LanDnsProbesAbsent
|
||||
expr: absent(probe_success{job="blackbox-dns"})
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No LAN DNS probe results are reaching Prometheus"
|
||||
description: >-
|
||||
The blackbox-dns Probes stopped being discovered or the exporter
|
||||
rejects the modules. A 400 from /probe yields no probe_* series
|
||||
at all, which looks the same as no Probe — check that every
|
||||
dns_lan_* module the Probes reference exists in the
|
||||
blackbox-exporter config.
|
||||
@@ -0,0 +1,87 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: external-dns
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# ruleSelector matches on this label; without it the rule is created but
|
||||
# never loaded into Prometheus.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: external-dns.rules
|
||||
rules:
|
||||
# The signal that was missing on 2026-09-12: the sources stopped
|
||||
# producing endpoints they had produced before, and policy=sync
|
||||
# deleted the records. 10% below the 24h peak clears a single
|
||||
# retired Ingress (2 of ~48) and catches a whole class of names.
|
||||
- alert: ExternalDNSSourceEndpointsDropped
|
||||
expr: >-
|
||||
external_dns_source_endpoints_total
|
||||
< 0.9 * max_over_time(external_dns_source_endpoints_total[24h])
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is publishing {{ $value }} endpoints, below its 24h peak"
|
||||
description: >-
|
||||
external-dns stopped seeing endpoints it had before. With
|
||||
policy=sync the matching DNS records are deleted, so names stop
|
||||
resolving without anything else failing. Compare
|
||||
external_dns_source_endpoints_total against the last 24h, then
|
||||
check whether the resources still carry a hostname annotation
|
||||
the running version understands — a changed annotation prefix
|
||||
default empties the source silently. Deliberate removals also
|
||||
fire this and resolve themselves 24h later.
|
||||
|
||||
# Source errors mean the Kubernetes side failed; registry errors mean
|
||||
# the provider did. Either way desired state is computed from partial
|
||||
# input, which policy=sync acts on.
|
||||
- alert: ExternalDNSSourceErrors
|
||||
expr: increase(external_dns_source_errors_total[15m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is failing to read its sources"
|
||||
description: >-
|
||||
external-dns could not list the Kubernetes resources it derives
|
||||
records from. Check the pod log and its RBAC.
|
||||
|
||||
- alert: ExternalDNSRegistryErrors
|
||||
expr: increase(external_dns_registry_errors_total[15m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is failing to reach its DNS provider"
|
||||
description: >-
|
||||
external-dns could not read or write records at the provider.
|
||||
For external-dns-unifi that is the UniFi webhook sidecar and the
|
||||
controller behind it; for external-dns it is the Cloudflare API.
|
||||
|
||||
- alert: ExternalDNSDown
|
||||
expr: up{job=~"external-dns.*"} == 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} is not being scraped"
|
||||
description: >-
|
||||
The external-dns instance is down or unreachable. Records already
|
||||
at the provider keep resolving, but nothing reconciles them any
|
||||
more and every other alert in this group is blind.
|
||||
|
||||
# Dead-man's-switch for the group, mirroring BlackboxProbesAbsent: a
|
||||
# dropped ServiceMonitor makes every rule above evaluate to nothing.
|
||||
- alert: ExternalDNSMetricsMissing
|
||||
expr: absent(external_dns_source_endpoints_total)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No external-dns metrics are reaching Prometheus"
|
||||
description: >-
|
||||
Either both external-dns instances are gone or the ServiceMonitor
|
||||
stopped being discovered. Verify it still carries
|
||||
release=kube-prometheus-stack.
|
||||
@@ -232,7 +232,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
io.cilium/lb-ipam-ips: "10.2.99.84,fd99::84"
|
||||
external-dns.alpha.kubernetes.io/hostname: syslog.renner.uno
|
||||
external-dns.kubernetes.io/hostname: syslog.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -72,3 +72,73 @@ spec:
|
||||
tcp:
|
||||
preferred_ip_protocol: ip6
|
||||
ip_protocol_fallback: true
|
||||
# One module per name: the dns prober takes the RESOLVER as its
|
||||
# target and reads query_name from the module, so a name cannot be
|
||||
# passed per scrape. The Probes in monitoring-rules/probes-dns.yaml
|
||||
# point these at the UDM.
|
||||
#
|
||||
# Both answer checks are load-bearing. valid_rcodes alone passes a
|
||||
# deleted record two labels deep (NOERROR, no answer), and the public
|
||||
# *.renner.uno wildcard answers one label deep with a CNAME to
|
||||
# proxy.renner.uno — NOERROR with the WAN proxy behind it. Requiring
|
||||
# an A and rejecting a CNAME separates a real record from both.
|
||||
dns_lan_ha:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ha.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_mqtt:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: mqtt.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_ldap:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ldap.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_syslog:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: syslog.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_ftp_scanner:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: ftp.paperless.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
dns_lan_syncthing:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
query_name: syncthing.paperless.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
|
||||
@@ -55,7 +55,7 @@ spec:
|
||||
# stdlib only; Flux keeps the tag in lockstep via the shared image
|
||||
# policy) — no extra dependency.
|
||||
- name: reconcile-publish-ip
|
||||
image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
command: ["/app/venv/bin/python", "/scripts/reconcile.py"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
@@ -65,7 +65,7 @@ spec:
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: music-assistant
|
||||
image: ghcr.io/music-assistant/server:2.10.3 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
image: ghcr.io/music-assistant/server:2.10.4 # {"$imagepolicy": "flux-system:music-assistant"}
|
||||
env:
|
||||
- name: LOG_LEVEL
|
||||
value: info
|
||||
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
mountPath: /consume
|
||||
containers:
|
||||
- name: vsftpd
|
||||
image: gitea.renner.uno/k.renner/vsftpd:build.1784731469 # {"$imagepolicy": "flux-system:vsftpd"}
|
||||
image: gitea.renner.uno/k.renner/vsftpd:build.1789630783 # {"$imagepolicy": "flux-system:vsftpd"}
|
||||
ports:
|
||||
- name: ftp-ctrl
|
||||
containerPort: 21
|
||||
@@ -70,24 +70,19 @@ spec:
|
||||
- name: consume
|
||||
mountPath: /home/scanner
|
||||
subPath: scanner
|
||||
# FTP has no health verb, so the control port accepting a connection
|
||||
# is what can be asserted. It is enough for the failure that matters:
|
||||
# if vsftpd dies the scanner cannot deposit anything and nothing else
|
||||
# in the chain notices, because paperless only ever sees an empty
|
||||
# consume directory — which is indistinguishable from nobody having
|
||||
# scanned.
|
||||
#
|
||||
# Only the control port is probed. The passive range is negotiated
|
||||
# per transfer and is not listening at rest.
|
||||
# Asserts the control socket is listening without opening an FTP
|
||||
# connection: vsftpd 3.0.5 segfaults its listener on a fraction of
|
||||
# the connections it accepts. The passive range is negotiated per
|
||||
# transfer and is not listening at rest, so it is not probed.
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: ftp-ctrl
|
||||
exec:
|
||||
command: ["sh", "-c", "netstat -lnt | grep -q ':21 '"]
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: ftp-ctrl
|
||||
exec:
|
||||
command: ["sh", "-c", "netstat -lnt | grep -q ':21 '"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -124,7 +119,7 @@ metadata:
|
||||
# label — FTP must not be exposed via the cargo tunnel (clear-text
|
||||
# protocol, public exposure not appropriate). The brother scanner
|
||||
# uses this hostname instead of the LB IP.
|
||||
external-dns.alpha.kubernetes.io/hostname: ftp.paperless.renner.uno
|
||||
external-dns.kubernetes.io/hostname: ftp.paperless.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -25,18 +25,15 @@ resources:
|
||||
configMapGenerator:
|
||||
# vsftpd config for the ftp-scanner deployment. The content-hash name
|
||||
# suffix is intentionally LEFT ENABLED (unlike home-assistant's nginx
|
||||
# config): vsftpd templates pasv_address from PASV_ADDRESS at container
|
||||
# start, so a changed value must roll the pod. With the hash suffix,
|
||||
# editing any value here renames the generated ConfigMap, which mutates
|
||||
# the Deployment's pod template and triggers an automatic rolling
|
||||
# restart. Without it, an IP change (e.g. an LB re-allocation) silently
|
||||
# leaves the running pod advertising the stale PASV address, which
|
||||
# breaks classic-PASV clients (older Brother scanners) while EPSV
|
||||
# clients keep working.
|
||||
# config): vsftpd reads these into vsftpd.conf at container start, so a
|
||||
# changed value must roll the pod, which the renamed ConfigMap does.
|
||||
#
|
||||
# No PASV_ADDRESS: the LB address is declared once, on the Service
|
||||
# (io.cilium/lb-ipam-ips). The image default 0.0.0.0 tells PASV clients
|
||||
# to reuse the control-connection address.
|
||||
- name: ftp-scanner-config
|
||||
namespace: paperless
|
||||
literals:
|
||||
- PASV_ADDRESS=10.2.99.82
|
||||
- PASV_MIN_PORT=21100
|
||||
- PASV_MAX_PORT=21104
|
||||
- FTP_USER=scanner
|
||||
|
||||
@@ -183,7 +183,7 @@ spec:
|
||||
drop: ["ALL"]
|
||||
containers:
|
||||
- name: paperless
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:3.1.3 # {"$imagepolicy": "flux-system:paperless-ngx"}
|
||||
image: ghcr.io/paperless-ngx/paperless-ngx:3.2.0 # {"$imagepolicy": "flux-system:paperless-ngx"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
|
||||
@@ -145,7 +145,7 @@ metadata:
|
||||
lb-pool: bgp
|
||||
annotations:
|
||||
io.cilium/lb-ipam-ips: "10.2.99.85,fd99::85"
|
||||
external-dns.alpha.kubernetes.io/hostname: syncthing.paperless.renner.uno
|
||||
external-dns.kubernetes.io/hostname: syncthing.paperless.renner.uno
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -134,7 +134,7 @@ spec:
|
||||
# is a device-bricking class of risk. Bumps are a reviewed, manual
|
||||
# act — see kubernetes/infrastructure/image-automation/README.md.
|
||||
- name: terminus
|
||||
image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
image: ghcr.io/usetrmnl/terminus:0.74.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 2345
|
||||
@@ -197,7 +197,7 @@ spec:
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
- name: sidekiq
|
||||
image: ghcr.io/usetrmnl/terminus:0.73.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
image: ghcr.io/usetrmnl/terminus:0.74.0 # {"$imagepolicy": "flux-system:terminus"}
|
||||
command: ["bundle", "exec", "sidekiq", "-r", "./config/sidekiq.rb"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
---
|
||||
# ServiceAccount is the OpenBao identity (auth/kubernetes/role/wiki-index); no
|
||||
# Kubernetes API access needed.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
---
|
||||
# Index (~17 MB JSON for 1700 notes). Survives restarts so a pod start does not
|
||||
# re-embed the wiki; a lost volume only costs one full sync (~5 min).
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: wiki-index-data
|
||||
namespace: ai
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
# Embedding index over the Notes repo's wiki/ (ci/images/wiki-index). Syncs from
|
||||
# Gitea on push webhook and every RESYNC_SECONDS, embeds via wiki-embed, serves
|
||||
# POST /rank to ~/Notes/bin/wiki_rank.py through wiki-index.renner.uno.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
serviceAccountName: wiki-index
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: wiki-index
|
||||
image: gitea.renner.uno/gitea_admin/wiki-index:2026-09-20
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: GITEA_URL
|
||||
value: https://gitea.renner.uno
|
||||
- name: NOTES_REPO
|
||||
value: k.renner/notes
|
||||
- name: NOTES_BRANCH
|
||||
value: main
|
||||
- name: EMBED_URL
|
||||
value: http://wiki-embed.ai.svc.cluster.local:8080
|
||||
- name: BAO_ADDR
|
||||
value: http://openbao.openbao.svc.cluster.local:8200
|
||||
- name: BAO_ROLE
|
||||
value: wiki-index
|
||||
- name: BAO_SECRET_PATH
|
||||
value: homelab/data/apps/ai/wiki-index
|
||||
- name: INDEX_PATH
|
||||
value: /data/index.json
|
||||
- name: RESYNC_SECONDS
|
||||
value: "1800"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
# Pure-Python cosine over 1700 x 1024 floats per query; the index itself is ~60 MB in memory.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
# /health is 503 until the first sync produced an index. Cold start with an
|
||||
# empty PVC embeds every note: budget 15 min.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
failureThreshold: 90
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 30
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 60
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 10
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: wiki-index-data
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
# wiki-index.renner.uno — LAN-only (no dns.renner.uno/public label), machine
|
||||
# client (bin/wiki_rank.py), so Traefik BasicAuth like wiki-embed.renner.uno.
|
||||
# Gitea's webhook does not pass here; it targets the ClusterIP Service directly.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: wiki-index-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
basicAuth:
|
||||
secret: wiki-index-basic-auth
|
||||
realm: wiki-index
|
||||
---
|
||||
# OpenBao key apps/ai/wiki-index-basic-auth, field `users` = htpasswd line.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: wiki-index-basic-auth
|
||||
namespace: ai
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: wiki-index-basic-auth
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: apps/ai/wiki-index-basic-auth
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
traefik.ingress.kubernetes.io/router.middlewares: ai-wiki-index-basic-auth@kubernetescrd
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- wiki-index.renner.uno
|
||||
secretName: wiki-index-tls
|
||||
rules:
|
||||
- host: wiki-index.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: wiki-index
|
||||
port:
|
||||
number: 8080
|
||||
@@ -0,0 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- ingress.yaml
|
||||
@@ -21,7 +21,7 @@ spec:
|
||||
# Service; nothing on this path needs the host's network namespace.
|
||||
containers:
|
||||
- name: zone-configurator
|
||||
image: everythingsmarthome/everything-presence-mmwave-configurator:2.2.4 # {"$imagepolicy": "flux-system:mmwave-configurator"}
|
||||
image: everythingsmarthome/everything-presence-mmwave-configurator:2.2.5 # {"$imagepolicy": "flux-system:mmwave-configurator"}
|
||||
env:
|
||||
- name: HA_BASE_URL
|
||||
value: "http://home-assistant:80"
|
||||
|
||||
@@ -610,8 +610,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then
|
||||
# Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which
|
||||
# would not match the amd64 schematic this cluster installs. That leaves the
|
||||
# cx*/cpx*2 families in the German locations.
|
||||
# Today's traffic-class split, not the target one in
|
||||
# PROPOSAL-traffic-class-separation.md: the node address carries management,
|
||||
# Two segments only, unlike the homelab's split in
|
||||
# NETWORK-traffic-classes.md: here the node address carries management,
|
||||
# cluster-internal and external alike, and Ceph replication has its own.
|
||||
#
|
||||
# The order is load-bearing: it decides which kernel interface each segment
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
# First start on an empty PVC embeds every note (~5 min) before /health is 200.
|
||||
timeout: 20m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/apps/wiki-index
|
||||
prune: true
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
# wiki-embed (the embedding backend) lives in the llama-server app.
|
||||
- name: llama-server
|
||||
@@ -6,84 +6,98 @@ include "/etc/bird/defines.conf";
|
||||
router id from "pod0";
|
||||
log stderr all;
|
||||
|
||||
# Mirrors of the kernel main table, filtered to this node's own pod CIDR.
|
||||
# master4/master6 are implicit and hold what BGP exchanges.
|
||||
ipv4 table kmain4;
|
||||
ipv6 table kmain6;
|
||||
|
||||
protocol device { scan time 10; }
|
||||
|
||||
# Two conditions. cilium_host also carries its own host address, which the
|
||||
# length check drops; and Cilium's routes for the OTHER nodes are protocol
|
||||
# kernel too, which the interface check drops. ifname is set for the
|
||||
# gatewayless IPv6 device route as well, so this works in both families.
|
||||
filter own_pod4 {
|
||||
if ifname !~ "cilium_host" then reject;
|
||||
if net.len = 32 then reject;
|
||||
accept;
|
||||
}
|
||||
filter own_pod6 {
|
||||
if ifname !~ "cilium_host" then reject;
|
||||
if net.len = 128 then reject;
|
||||
accept;
|
||||
# Everything a pod may need to reach, in one table, resolved by longest prefix:
|
||||
# the node's attached networks, the peers' pod CIDRs, and a default for the
|
||||
# rest. No rule has to carve out exceptions, because the specific routes win on
|
||||
# their own.
|
||||
#
|
||||
# The table is BIRD's alone. If this daemon stops, it empties, the rule finds
|
||||
# nothing and falls through to main — pod traffic returns to VLAN 40,
|
||||
# unseparated rather than misrouted. That is why the default route lives here
|
||||
# and not in the Talos config: a default that outlived the specific routes
|
||||
# would send pod traffic to the UDM.
|
||||
|
||||
# The node's attached networks, taken from the interfaces rather than written
|
||||
# down. The usual advice is not to export these to the kernel because it
|
||||
# maintains them itself — true of main, not of a table the kernel never fills.
|
||||
protocol direct attached {
|
||||
interface "mgmt0", "ingress0", "pod0";
|
||||
ipv4 { table master4; };
|
||||
ipv6 { table master6; };
|
||||
}
|
||||
|
||||
# learn all, not learn: plain learn skips routes marked proto kernel, which is
|
||||
# what Cilium's are.
|
||||
protocol kernel klearn4 {
|
||||
learn all;
|
||||
scan time 10;
|
||||
ipv4 { table kmain4; import filter own_pod4; export none; };
|
||||
# Pod egress. GW4/GW6 are the UDM on the ingress segment, published as node
|
||||
# annotations beside the address that defines them.
|
||||
protocol static egress4 {
|
||||
ipv4 { table master4; };
|
||||
route 0.0.0.0/0 via GW4;
|
||||
}
|
||||
protocol kernel klearn6 {
|
||||
learn all;
|
||||
scan time 10;
|
||||
ipv6 { table kmain6; import filter own_pod6; export none; };
|
||||
protocol static egress6 {
|
||||
ipv6 { table master6; };
|
||||
route ::/0 via GW6;
|
||||
}
|
||||
|
||||
protocol pipe own4 { table kmain4; peer table master4; export all; import none; }
|
||||
protocol pipe own6 { table kmain6; peer table master6; export all; import none; }
|
||||
|
||||
# Only BGP-learned routes reach table 48. The local CIDR stays in main, where
|
||||
# cilium_host already serves it.
|
||||
filter to_t48 {
|
||||
if source != RTS_BGP then reject;
|
||||
accept;
|
||||
# This node's own pod CIDR, via the interface Cilium serves it on. Originated
|
||||
# rather than read back out of the kernel: reading it back made BIRD's view
|
||||
# depend on the shape of Cilium's route, and the IPv4 one is rejected outright,
|
||||
# its next hop sitting inside the prefix it describes.
|
||||
#
|
||||
# It belongs in the table, not just in the BGP announcement. A reply from a pod
|
||||
# on another node is forwarded traffic with a pod source, so the rule matches it
|
||||
# too; without this route it would find only the default and leave for the UDM.
|
||||
# Measured — cross-node pod traffic, and with it cluster DNS, stopped.
|
||||
protocol static own4 {
|
||||
ipv4 { table master4; };
|
||||
route OWN_POD4 via "cilium_host";
|
||||
}
|
||||
protocol static own6 {
|
||||
ipv6 { table master6; };
|
||||
route OWN_POD6 via "cilium_host";
|
||||
}
|
||||
|
||||
protocol kernel k48v4 {
|
||||
kernel table 48;
|
||||
protocol kernel k4 {
|
||||
kernel table 47;
|
||||
metric 0;
|
||||
ipv4 { table master4; import none; export filter to_t48; };
|
||||
ipv4 { table master4; import none; export all; };
|
||||
}
|
||||
protocol kernel k48v6 {
|
||||
kernel table 48;
|
||||
protocol kernel k6 {
|
||||
kernel table 47;
|
||||
metric 0;
|
||||
ipv6 { table master6; import none; export filter to_t48; };
|
||||
ipv6 { table master6; import none; export all; };
|
||||
}
|
||||
|
||||
# iBGP over pod0. 64514 keeps clear of Cilium's 64513, which peers with the UDM
|
||||
# from ingress0 and is unrelated.
|
||||
#
|
||||
# No prefix filters: the only route this node offers is its own pod CIDR, which
|
||||
# own_pod4/6 already selected, and the only peers are the other cluster nodes.
|
||||
#
|
||||
# direct is explicit because iBGP defaults to multihop, which makes the gateway
|
||||
# recursive — BIRD would try to resolve the peer in the pod RIB. strict bind
|
||||
# keeps the listener off mgmt0 and ingress0.
|
||||
#
|
||||
# Only the node's own CIDR is offered: the export filter drops everything the
|
||||
# static protocols and the direct protocol produce.
|
||||
template bgp pod4 {
|
||||
local as 64514;
|
||||
source address MY4;
|
||||
direct;
|
||||
strict bind;
|
||||
ipv4 { next hop self; import all; export all; };
|
||||
ipv4 {
|
||||
next hop self;
|
||||
import all;
|
||||
export filter { if net = OWN_POD4 then accept; reject; };
|
||||
};
|
||||
}
|
||||
template bgp pod6 {
|
||||
local as 64514;
|
||||
source address MY6;
|
||||
direct;
|
||||
strict bind;
|
||||
ipv6 { next hop self; import all; export all; };
|
||||
ipv6 {
|
||||
next hop self;
|
||||
import all;
|
||||
export filter { if net = OWN_POD6 then accept; reject; };
|
||||
};
|
||||
}
|
||||
|
||||
include "/etc/bird/peers.conf";
|
||||
|
||||
@@ -1,2 +1,9 @@
|
||||
define MY6 = {{ index .metadata.annotations "net.renner.uno/pod0-v6" }};
|
||||
define MY4 = {{ index .metadata.annotations "net.renner.uno/pod0-v4" }};
|
||||
define GW6 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v6" }};
|
||||
define GW4 = {{ index .metadata.annotations "net.renner.uno/egress-gw-v4" }};
|
||||
{{/* podCIDRs follows the cluster's podSubnets order, v4 first. A flip puts an
|
||||
IPv6 prefix into an ipv4 protocol, which fails the parse in the init
|
||||
container rather than passing silently. */}}
|
||||
define OWN_POD4 = {{ index .spec.podCIDRs 0 }};
|
||||
define OWN_POD6 = {{ index .spec.podCIDRs 1 }};
|
||||
|
||||
@@ -126,3 +126,59 @@ spec:
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
---
|
||||
# wiki-embed is the one ai pod Traefik may reach: it sits behind BasicAuth
|
||||
# (apps/llama-server/wiki-embed-ingress.yaml), not behind the Authentik gate.
|
||||
# Scoped to this pod so the default-deny above keeps Traefik off llama-server.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: wiki-embed-from-traefik
|
||||
namespace: ai
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-embed
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": ingress
|
||||
app.kubernetes.io/name: traefik
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8080"
|
||||
protocol: TCP
|
||||
---
|
||||
# wiki-index: Traefik (LAN clients behind BasicAuth) and Gitea (push webhook)
|
||||
# may reach it; it reaches OpenBao for its credentials. Gitea itself is reached
|
||||
# through Traefik, which the namespace egress above already allows.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: wiki-index
|
||||
namespace: ai
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: wiki-index
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": ingress
|
||||
app.kubernetes.io/name: traefik
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": apps
|
||||
app.kubernetes.io/name: gitea
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8080"
|
||||
protocol: TCP
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": openbao
|
||||
app.kubernetes.io/name: openbao
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8200"
|
||||
protocol: TCP
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# OpenBao policy: wiki-index
|
||||
#
|
||||
# The wiki-index Deployment (namespace ai) reads its Gitea token and the
|
||||
# webhook HMAC secret on every use. Read-only on that one path.
|
||||
#
|
||||
# NOT applied by Flux. Apply ONCE with an admin token:
|
||||
#
|
||||
# bao policy write wiki-index kubernetes/infrastructure/openbao/policies/wiki-index.hcl
|
||||
# bao write auth/kubernetes/role/wiki-index \
|
||||
# bound_service_account_names=wiki-index \
|
||||
# bound_service_account_namespaces=ai \
|
||||
# policies=wiki-index ttl=15m
|
||||
#
|
||||
# Secret: homelab/apps/ai/wiki-index, keys `gitea_token` (read:repository on
|
||||
# the notes repo) and `webhook_secret` (the Gitea webhook's secret).
|
||||
#
|
||||
# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
|
||||
|
||||
path "homelab/data/apps/ai/wiki-index" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -14,6 +14,7 @@ spec:
|
||||
# ClusterSecretStore" and the secret never syncs. Keep this list sorted.
|
||||
conditions:
|
||||
- namespaces:
|
||||
- ai # wiki-embed-basic-auth (apps/llama-server/wiki-embed-ingress.yaml)
|
||||
- apps
|
||||
- audiobookshelf
|
||||
- authentik
|
||||
|
||||
@@ -126,6 +126,16 @@ spec:
|
||||
addressRanges:
|
||||
cluster:
|
||||
- 10.99.0.0/29
|
||||
# Reads go to the OSD in the client's own CRUSH location. crushLocationLabels
|
||||
# is unset, so Rook's default list applies; of those the nodes carry
|
||||
# topology.kubernetes.io/zone, one zone per node, matching the zone buckets
|
||||
# the OSDs are created under.
|
||||
#
|
||||
# Passed as a krbd/mount option at map time, so a volume keeps its current
|
||||
# behaviour until it is remounted.
|
||||
csi:
|
||||
readAffinity:
|
||||
enabled: true
|
||||
# Nodes listed explicitly, not useAllNodes: under useAllNodes the operator
|
||||
# re-provisions an OSD on any node whose device still carries a BlueStore
|
||||
# signature, so purging an OSD before pulling its drive is impossible — it
|
||||
|
||||
@@ -188,14 +188,13 @@ addresses:
|
||||
# network's clients selectively, so the client goes rather than the record.
|
||||
# DHCP is off on VLAN 47 for the same reason.
|
||||
#
|
||||
# Both defaults live in table 47, not in main, so only the pod-source rules in
|
||||
# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
|
||||
# reply the node sends from its own addresses.
|
||||
# No route here. Table 47 belongs to bird, which fills it with this node's
|
||||
# attached networks, the peers' pod CIDRs and the egress default, and empties it
|
||||
# if it stops — leaving the rule to fall through to main. A default declared
|
||||
# here would outlive the specific routes and send pod traffic to the UDM.
|
||||
#
|
||||
# The v6 gateway is the UDM's configured ULA on this segment, the same address
|
||||
# BGP peers with. The RA-learned link-local would be the alternative and is not
|
||||
# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
|
||||
# default would land in main, where it does not belong.
|
||||
# accept_ra_defrtr stays 0 on this link so the RA default does not land in main,
|
||||
# where mgmt0 owns it.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ingress0
|
||||
@@ -204,11 +203,6 @@ parent: uplink
|
||||
up: true
|
||||
addresses:
|
||||
- address: 10.2.47.11/24
|
||||
routes:
|
||||
- gateway: fd02:47::1
|
||||
table: "47"
|
||||
- gateway: 10.2.47.1
|
||||
table: "47"
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
|
||||
# wire. OSD-to-OSD replication and nothing else. The udev rule in the machine
|
||||
|
||||
@@ -132,11 +132,6 @@ parent: uplink
|
||||
up: true
|
||||
addresses:
|
||||
- address: 10.2.47.12/24
|
||||
routes:
|
||||
- gateway: fd02:47::1
|
||||
table: "47"
|
||||
- gateway: 10.2.47.1
|
||||
table: "47"
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
|
||||
# wire. OSD-to-OSD replication only, not routed off its own segment.
|
||||
|
||||
@@ -119,11 +119,6 @@ parent: uplink
|
||||
up: true
|
||||
addresses:
|
||||
- address: 10.2.47.13/24
|
||||
routes:
|
||||
- gateway: fd02:47::1
|
||||
table: "47"
|
||||
- gateway: 10.2.47.1
|
||||
table: "47"
|
||||
---
|
||||
# Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
|
||||
# here on purpose; the link is handed whole to the OSD pod (host-device).
|
||||
|
||||
@@ -20,41 +20,32 @@ machine:
|
||||
validSubnets:
|
||||
- 10.2.40.0/24
|
||||
- fd40::/64
|
||||
# The UDM on the ingress segment. Published here rather than as a route
|
||||
# because bird owns table 47 and originates the default from it; this is the
|
||||
# one place the address is stated.
|
||||
nodeAnnotations:
|
||||
net.renner.uno/egress-gw-v6: fd02:47::1
|
||||
net.renner.uno/egress-gw-v4: 10.2.47.1
|
||||
---
|
||||
# Pod egress leaves over ingress0. Matching on the pod source prefix is what
|
||||
# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
|
||||
# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
|
||||
# source onto VLAN 47.
|
||||
# All pod traffic goes to table 47, and the table decides — longest prefix
|
||||
# match, no exceptions to carve out. bird fills it with the node's attached
|
||||
# networks, the peers' pod CIDRs and the egress default, so a packet to a node
|
||||
# on the mgmt segment takes the connected route, a packet to a pod on another
|
||||
# node takes pod0, and anything else takes the default over ingress0.
|
||||
#
|
||||
# The routing decision runs before Cilium's masquerade hook, so the rule sees
|
||||
# the pod address and the masquerade then rewrites it to ingress0's.
|
||||
# Matching the source is what keeps the node's own traffic out: it answers on
|
||||
# mgmt0, and a reply leaving over ingress0 would carry an mgmt0 source onto
|
||||
# VLAN 47. The routing decision runs before Cilium's masquerade hook, so the
|
||||
# rule sees the pod address and the masquerade then rewrites it.
|
||||
#
|
||||
# Table 47 holds nothing but the two default routes, so the destination rules
|
||||
# below have to run first: without them a pod's packet to a service or to a pod
|
||||
# on another node also matches the source rule, finds only the default, and
|
||||
# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
|
||||
#
|
||||
# They send everything inside the cluster and the local networks back to main.
|
||||
# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
|
||||
# is not foreign traffic.
|
||||
# If bird stops, table 47 empties, the lookup misses and the rule falls through
|
||||
# to main — pod traffic returns to VLAN 40, unseparated rather than misrouted.
|
||||
#
|
||||
# Priorities are below main (32766) and clear of the rule Cilium installs at 9.
|
||||
# The two families are separate rule tables, so the priorities need not match;
|
||||
# the document name is the priority and must be unique.
|
||||
apiVersion: v1alpha1
|
||||
kind: RoutingRuleConfig
|
||||
name: "990"
|
||||
dst: fd00::/8
|
||||
table: main
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
kind: RoutingRuleConfig
|
||||
name: "991"
|
||||
dst: 10.0.0.0/8
|
||||
table: main
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
kind: RoutingRuleConfig
|
||||
name: "1000"
|
||||
src: fd10:244::/48
|
||||
table: "47"
|
||||
|
||||
Reference in New Issue
Block a user