Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24ed2485c9 | ||
|
|
ea439b110d | ||
|
|
a1e3b65cbf | ||
|
|
5e01e2f1b5 | ||
|
|
40ef53fa67 | ||
|
|
fbf5c4dcb8 | ||
|
|
112299da49 | ||
|
|
01e843beef | ||
|
|
2fbce12b0b | ||
|
|
d41f4870ad | ||
|
|
7fcaa4b035 | ||
|
|
11bf13883f | ||
|
|
4b502daa98 | ||
|
|
83a20bfa80 | ||
|
|
c1a1a59ecc |
@@ -0,0 +1,18 @@
|
||||
---
|
||||
# Shared configuration for every NTP server (see plays/install-ntp-ha.yml).
|
||||
|
||||
# IPv6 service address -- what ntp.srv.renner.uno resolves to, and the path
|
||||
# clients actually use. The IPv4 VIP is legacy.
|
||||
#
|
||||
# Defined at group level rather than in the play because install-ntp-ha.yml runs
|
||||
# two plays over this group -- withdraw the static assignments fleet-wide, then
|
||||
# hand the address to keepalived -- and both need the same value. Duplicating a
|
||||
# service address across plays is exactly the sort of thing that drifts.
|
||||
ntp_vip6: "fd40::123/128"
|
||||
|
||||
# Link-local address carried as the FIRST virtual_ipaddress of the IPv6 VRRP
|
||||
# instance. RFC 5798 gives an IPv6 virtual router a link-local primary, and
|
||||
# keepalived warns "the first IPv6 VIP address should be link local" without
|
||||
# one. fe80::123 mirrors the service address and cannot collide with a real
|
||||
# NIC's EUI-64 link-local.
|
||||
ntp_vip6_linklocal: "fe80::123/64"
|
||||
@@ -22,12 +22,54 @@
|
||||
# so it stays BACKUP while the PVE nodes exist and wins the VIP by default once
|
||||
# they are gone -- no config change required at cutover.
|
||||
|
||||
- name: Install NTP Server (HA)
|
||||
# Phase 1. Every static assignment of the IPv6 service address must be gone
|
||||
# fleet-wide before ANY host claims it via VRRP.
|
||||
#
|
||||
# DAD is enabled on the VRRP address (deliberately -- under single-master VRRP a
|
||||
# duplicate is a real fault worth detecting). So a static copy on one host and a
|
||||
# VRRP-managed copy on another cannot coexist: the kernel marks the newcomer
|
||||
# `dadfailed deprecated`, and its master then believes it is serving while
|
||||
# carrying an unusable address. That is not hypothetical -- it is what edge-1
|
||||
# did on 2026-08-07 when it took the address while three PVE nodes still held it
|
||||
# statically.
|
||||
#
|
||||
# Deliberately NOT serial: this play must complete everywhere before phase 2
|
||||
# starts anywhere. Doing the withdrawal per host inside phase 2 cannot avoid the
|
||||
# overlap, because with serial: 1 the first host claims the address while the
|
||||
# rest are still static.
|
||||
#
|
||||
# There is a gap between the phases where no host holds the address. It is
|
||||
# bounded by one play's run time and clients simply retry.
|
||||
- name: Withdraw static assignments of the NTP service address
|
||||
hosts: ntp_cluster
|
||||
become: true
|
||||
|
||||
vars:
|
||||
keepalived_ntp_vip6: "{{ ntp_vip6 }}"
|
||||
keepalived_ntp_interface: "{{ ntp_keepalived_interface }}"
|
||||
|
||||
tasks:
|
||||
- name: Withdraw any static assignment of the service address
|
||||
ansible.builtin.include_role:
|
||||
name: keepalived-ntp
|
||||
tasks_from: withdraw-static.yml
|
||||
when: ntp_keepalived_enabled | default(false)
|
||||
|
||||
# Phase 2. Configure chrony and hand the address to keepalived.
|
||||
- name: Install NTP Server (HA)
|
||||
hosts: ntp_cluster
|
||||
become: true
|
||||
# One node at a time. The keepalived role withdraws the statically assigned
|
||||
# IPv6 service address and restarts keepalived so the address is re-taken
|
||||
# under VRRP control; doing that across every NTP server at once would leave
|
||||
# the whole fleet with nobody holding fd40::123 until the elections settle.
|
||||
# Serialised, the remaining members keep answering throughout.
|
||||
serial: 1
|
||||
|
||||
vars:
|
||||
ntp_vip: "10.2.40.4"
|
||||
# ntp_vip6 / ntp_vip6_linklocal come from group_vars/ntp_cluster.yml, so
|
||||
# both plays in this file see one definition.
|
||||
ntp_servers:
|
||||
jupiter:
|
||||
priority: 100
|
||||
@@ -49,6 +91,8 @@
|
||||
when: ntp_keepalived_enabled | default(false)
|
||||
vars:
|
||||
keepalived_ntp_vip: "{{ ntp_vip }}"
|
||||
keepalived_ntp_vip6: "{{ ntp_vip6 }}"
|
||||
keepalived_ntp_vip6_linklocal: "{{ ntp_vip6_linklocal }}"
|
||||
keepalived_ntp_state: "{{ ntp_servers[inventory_hostname].state }}"
|
||||
keepalived_ntp_priority: "{{ ntp_servers[inventory_hostname].priority }}"
|
||||
keepalived_ntp_interface: "{{ ntp_keepalived_interface }}"
|
||||
@@ -90,6 +134,49 @@
|
||||
{% endif %}
|
||||
when: ntp_keepalived_enabled | default(false)
|
||||
|
||||
# The v6 address is the path clients actually use, so it gets the same
|
||||
# check as the v4 VIP. Without this a broken v6 rollout -- the address
|
||||
# never claimed, or -nodad not behaving as expected on a given kernel --
|
||||
# would not surface here at all, only later by reading
|
||||
# /var/log/keepalived-state.log by hand.
|
||||
#
|
||||
# `ip -6 addr show ... to <addr>` prints the address when present and
|
||||
# nothing when absent, so emptiness is the test.
|
||||
#
|
||||
# Presence cannot itself distinguish a VRRP-held address from a static
|
||||
# leftover -- they look identical. It is only meaningful because the
|
||||
# keepalived-ntp role asserts, on both distros, that no static assignment
|
||||
# survives: the play fails before reaching here if one does.
|
||||
- name: Check if this server holds the IPv6 service address
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ip
|
||||
- -6
|
||||
- addr
|
||||
- show
|
||||
- dev
|
||||
- "{{ ntp_keepalived_interface }}"
|
||||
- to
|
||||
- "{{ ntp_vip6 }}"
|
||||
register: vip6_check
|
||||
failed_when: false
|
||||
changed_when: false
|
||||
when: ntp_keepalived_enabled | default(false)
|
||||
|
||||
- name: Display IPv6 service address status
|
||||
ansible.builtin.debug:
|
||||
msg: >
|
||||
{% if vip6_check.stdout | default('') | length > 0 %}
|
||||
This server ({{ inventory_hostname }}) holds the IPv6 service address
|
||||
{{ ntp_vip6 }} -- it is the v6 MASTER and is serving ntp.srv.renner.uno
|
||||
{% else %}
|
||||
This server ({{ inventory_hostname }}) does not hold {{ ntp_vip6 }}.
|
||||
Expected on a v6 BACKUP. If NO host holds it, check_ntp is failing
|
||||
everywhere -- inspect `chronyc tracking` and
|
||||
/var/log/keepalived-ntp-health.log
|
||||
{% endif %}
|
||||
when: ntp_keepalived_enabled | default(false)
|
||||
|
||||
- name: Report hosts serving NTP without participating in the VIP
|
||||
ansible.builtin.debug:
|
||||
msg: >
|
||||
|
||||
@@ -1,9 +1,25 @@
|
||||
---
|
||||
# Chrony NTP Server Role Defaults
|
||||
|
||||
# Networks allowed to use this server as NTP source
|
||||
# Networks allowed to use this server as NTP source.
|
||||
#
|
||||
# BOTH families are required. IPv6 is the primary address family here -- the
|
||||
# service address ntp.srv.renner.uno resolves to fd40::123 -- so an IPv4-only
|
||||
# allow list refuses every client that uses the documented name.
|
||||
#
|
||||
# The hosts carried `allow fd00::/8` for a long time without it ever being in
|
||||
# this file. Applying the role on 2026-08-07 rendered the template over it and
|
||||
# dropped the line on all four NTP servers at once, which took internal NTP
|
||||
# down over IPv6 until it was added here. Same class of untracked host state as
|
||||
# the anycast service address, and the same failure: config that exists only on
|
||||
# the box survives right up until something re-renders it.
|
||||
chrony_allowed_networks:
|
||||
- "10.2.0.0/16"
|
||||
- "fd00::/8"
|
||||
# Link-local. A client on the same segment may source its query from fe80::,
|
||||
# and without this chrony refuses it -- the request arrives and is dropped,
|
||||
# which looks like the server being unreachable rather than a policy refusal.
|
||||
- "fe80::/10"
|
||||
|
||||
# Primary NTP servers (PTB - Physikalisch-Technische Bundesanstalt, Stratum 1)
|
||||
chrony_upstream_servers:
|
||||
@@ -18,6 +34,18 @@ chrony_upstream_pools:
|
||||
# Local stratum if disconnected from upstream
|
||||
chrony_local_stratum: 10
|
||||
|
||||
# Step (rather than slew) the clock when the offset exceeds the threshold, but
|
||||
# only for the first N updates after chronyd starts. Without this, chronyd only
|
||||
# ever slews, and a large offset is never closed in practice: on 2026-08-04
|
||||
# edge-1 rebooted with an RTC ~16 days in the past and chronyd sat synchronised
|
||||
# to ptbtime1 reporting "System time: 1376235s slow" while correcting nothing.
|
||||
# Every outbound TLS handshake then failed x509 NotBefore validation, taking out
|
||||
# the MQTT bridge (heating telemetry) and the Loki log shipper at once.
|
||||
# Limiting to the first few updates keeps steps confined to boot, so a
|
||||
# long-running server is never jumped out from under its workloads.
|
||||
chrony_makestep_threshold: 1.0
|
||||
chrony_makestep_limit: 3
|
||||
|
||||
# Hardware timestamping. Fine on the PVE nodes' Intel NICs; the Wyse 3040
|
||||
# (edge-1) has a Realtek NIC that does not support it, and chronyd logs an
|
||||
# error per interface when asked for it. Overridden to false in
|
||||
|
||||
@@ -14,6 +14,11 @@ pool {{ pool }}
|
||||
# Record the rate at which the system clock gains/losses time
|
||||
driftfile /var/lib/chrony/chrony.drift
|
||||
|
||||
# Step the clock instead of slewing it when the offset is large, limited to the
|
||||
# first few updates after start so only boot-time corrections jump. Without a
|
||||
# makestep directive chronyd slews exclusively and never closes a large gap.
|
||||
makestep {{ chrony_makestep_threshold }} {{ chrony_makestep_limit }}
|
||||
|
||||
# Allow NTP client access from local networks
|
||||
{% for network in chrony_allowed_networks %}
|
||||
allow {{ network }}
|
||||
|
||||
@@ -4,6 +4,29 @@
|
||||
keepalived_ntp_router_id: 4
|
||||
keepalived_ntp_vip: "10.2.40.4"
|
||||
keepalived_ntp_interface: "vmbr1.40"
|
||||
|
||||
# IPv6 service address for NTP (ntp.srv.renner.uno). Empty disables the v6
|
||||
# instance entirely. IPv6 is the primary address family on this network, so
|
||||
# this is the path that actually matters -- the v4 VIP above is legacy.
|
||||
#
|
||||
# Held by keepalived rather than assigned statically by networkd so that it is
|
||||
# gated on check_ntp: present only while this host is fit to serve time, and
|
||||
# withdrawn the moment it is not. A statically configured address is carried
|
||||
# unconditionally from boot, which is how edge-1 came to answer NTP queries on
|
||||
# 2026-08-04 while its clock was ~16 days in the past.
|
||||
#
|
||||
# Needs its own virtual_router_id: it is a distinct VRRP instance, and a VRID
|
||||
# collision with the IPv4 instance on the same interface would have the two
|
||||
# fighting over a single election.
|
||||
keepalived_ntp_vip6: ""
|
||||
keepalived_ntp_router_id6: 6
|
||||
|
||||
# Link-local address for the IPv6 VRRP instance, carried as its FIRST
|
||||
# virtual_ipaddress. RFC 5798 gives an IPv6 virtual router a link-local primary,
|
||||
# and keepalived warns "the first IPv6 VIP address should be link local" without
|
||||
# one. Chosen to mirror the service address rather than derived from the virtual
|
||||
# MAC, since no real NIC generates fe80::123 so there is nothing to collide with.
|
||||
keepalived_ntp_vip6_linklocal: "fe80::123/64"
|
||||
# NOTE: plaintext VRRP auth, unvaulted. Carried over as-is from the original
|
||||
# role; VRRP PASS auth is weak regardless (sent in the clear on the segment),
|
||||
# so this is a documented weakness rather than a secret worth vaulting.
|
||||
|
||||
@@ -1,22 +1,39 @@
|
||||
#!/bin/bash
|
||||
# Keepalived State Change Notification Script
|
||||
#
|
||||
# Two VRRP instances call this: NTP_VIP (IPv4, legacy) passes MASTER/BACKUP/
|
||||
# FAULT, and NTP_VIP6 (IPv6, the primary path) passes the same suffixed with
|
||||
# '6'. They are logged distinctly because the two can legitimately diverge --
|
||||
# a host can hold one family and not the other -- and a log line that does not
|
||||
# say which family it means is useless during an incident.
|
||||
|
||||
TYPE=$1
|
||||
NAME=$(hostname)
|
||||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
log() {
|
||||
echo "$TIMESTAMP: $NAME $1" >> /var/log/keepalived-state.log
|
||||
logger -t keepalived "$NAME $1"
|
||||
}
|
||||
|
||||
case $TYPE in
|
||||
MASTER)
|
||||
echo "$TIMESTAMP: $NAME transitioned to MASTER state (NTP)" >> /var/log/keepalived-state.log
|
||||
logger -t keepalived "$NAME is now NTP MASTER"
|
||||
log "transitioned to MASTER state (NTP, IPv4)"
|
||||
;;
|
||||
BACKUP)
|
||||
echo "$TIMESTAMP: $NAME transitioned to BACKUP state (NTP)" >> /var/log/keepalived-state.log
|
||||
logger -t keepalived "$NAME is now NTP BACKUP"
|
||||
log "transitioned to BACKUP state (NTP, IPv4)"
|
||||
;;
|
||||
FAULT)
|
||||
echo "$TIMESTAMP: $NAME transitioned to FAULT state (NTP)" >> /var/log/keepalived-state.log
|
||||
logger -t keepalived "$NAME is now in NTP FAULT state"
|
||||
log "transitioned to FAULT state (NTP, IPv4)"
|
||||
;;
|
||||
MASTER6)
|
||||
log "transitioned to MASTER state (NTP, IPv6) - now serving ntp.srv.renner.uno"
|
||||
;;
|
||||
BACKUP6)
|
||||
log "transitioned to BACKUP state (NTP, IPv6) - released ntp.srv.renner.uno"
|
||||
;;
|
||||
FAULT6)
|
||||
log "transitioned to FAULT state (NTP, IPv6) - withdrew ntp.srv.renner.uno, unfit to serve time"
|
||||
;;
|
||||
*)
|
||||
echo "$TIMESTAMP: $NAME unknown state: $TYPE" >> /var/log/keepalived-state.log
|
||||
|
||||
@@ -35,6 +35,121 @@
|
||||
state: present
|
||||
when: ansible_facts['os_family'] == 'Archlinux'
|
||||
|
||||
# --- Verify nothing else is assigning the IPv6 service address ---------------
|
||||
#
|
||||
# The removal itself lives in withdraw-static.yml, run as a separate fleet-wide
|
||||
# play before any host reaches this role (see install-ntp-ha.yml). It has to
|
||||
# complete everywhere first: DAD is enabled on the VRRP address, so a static
|
||||
# copy on one host and a VRRP-managed copy on another cannot coexist -- the
|
||||
# kernel marks the newcomer `dadfailed` and its master serves nothing.
|
||||
#
|
||||
# What remains here are the guards. They run on every execution, not just the
|
||||
# migration, and fail the play if any static assignment survives -- which would
|
||||
# both defeat the health gating (a static copy does not disappear when
|
||||
# keepalived decides the host is unfit) and break the VRRP copy via DAD.
|
||||
#
|
||||
# The config file is what gets checked, not the running address, and that is
|
||||
# what makes it safe to run every time: keepalived never writes to these files,
|
||||
# so anything found is by definition static, whereas the live address is
|
||||
# legitimately present on whichever host is master.
|
||||
#
|
||||
# Prefix-agnostic and \b-anchored on purpose. Matching the full
|
||||
# address/prefixlen would fail open against a hand-written assignment using any
|
||||
# other mask.
|
||||
- name: Detect any remaining static assignment of the service address (ifupdown)
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- grep
|
||||
- -rlE
|
||||
- "{{ keepalived_ntp_vip6.split('/')[0] }}\\b"
|
||||
- /etc/network/interfaces
|
||||
- /etc/network/interfaces.d/
|
||||
register: keepalived_ntp_static6_ifupdown
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
- name: Assert ifupdown is not also assigning the service address
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- keepalived_ntp_static6_ifupdown.stdout | default('') | length == 0
|
||||
fail_msg: >-
|
||||
{{ keepalived_ntp_vip6 }} is still assigned statically by ifupdown on
|
||||
this host
|
||||
({{ keepalived_ntp_static6_ifupdown.stdout_lines | default([]) | join(', ') }}).
|
||||
The automatic removal only matches the exact form
|
||||
`up ip -6 addr replace {{ keepalived_ntp_vip6 }} dev <if> nodad`, so a
|
||||
different prefix length or layout is left in place. Handing the address
|
||||
to keepalived while ifupdown also assigns it means it would NOT be
|
||||
withdrawn when check_ntp fails -- the host would keep serving time on an
|
||||
address VRRP believes it has released. Remove the assignment from the
|
||||
file(s) above, then re-run.
|
||||
success_msg: >-
|
||||
No static ifupdown assignment of {{ keepalived_ntp_vip6 }}; safe for
|
||||
keepalived to own it.
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- ansible_facts['os_family'] == 'Debian'
|
||||
|
||||
# On Arch (edge-1) the static assignment lives in a systemd-networkd file owned
|
||||
# by the edge-host role, so this role must not rewrite it -- but it must not
|
||||
# quietly proceed either. install-edge-host.yml is a separate play with no
|
||||
# ordering relationship to this one, so nothing guarantees it has been re-run
|
||||
# to drop the [Address] block. If it has not, edge-1 would end up holding
|
||||
# fd40::123 statically AND running a VRRP instance for the same address:
|
||||
# keepalived would believe it had withdrawn the address while networkd kept it
|
||||
# assigned, and the host would go on answering NTP with a clock the health
|
||||
# check had already rejected.
|
||||
#
|
||||
# That is the precise failure of 2026-08-04, on the host it happened to, so it
|
||||
# fails fast rather than being left to inspection.
|
||||
# Matched on the bare address, not the full address/prefixlen, and anchored
|
||||
# with \b so it cannot also hit a longer address sharing the prefix (fd40::1234).
|
||||
#
|
||||
# Searching for the literal "fd40::123/128" would fail open: a hand-written
|
||||
# [Address] using any other mask -- /64, say -- would not match, the assert
|
||||
# below would pass, and keepalived would be handed an address networkd is
|
||||
# still assigning. Since this check is detect-only, with no delete afterwards
|
||||
# to catch the mistake, a miss here is silent. The Debian path does not share
|
||||
# the problem: `ip -6 addr show ... to <prefix>` matches by range regardless of
|
||||
# the assigned prefix length, and its delete is re-verified by assert.
|
||||
- name: Detect a static assignment of the service address (systemd-networkd)
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- grep
|
||||
- -rlE
|
||||
- "{{ keepalived_ntp_vip6.split('/')[0] }}\\b"
|
||||
- /etc/systemd/network/
|
||||
register: keepalived_ntp_static6_networkd
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- ansible_facts['os_family'] == 'Archlinux'
|
||||
|
||||
- name: Assert networkd is not also assigning the service address
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- keepalived_ntp_static6_networkd.stdout | default('') | length == 0
|
||||
fail_msg: >-
|
||||
{{ keepalived_ntp_vip6 }} is still assigned statically by
|
||||
systemd-networkd on this host
|
||||
({{ keepalived_ntp_static6_networkd.stdout_lines | default([]) | join(', ') }}).
|
||||
Handing the address to keepalived while networkd also assigns it means
|
||||
it would NOT be withdrawn when check_ntp fails -- the host would keep
|
||||
serving time on an address VRRP believes it has released.
|
||||
Re-apply the edge-host role first, whose template no longer carries the
|
||||
[Address] block: ansible-playbook -i inventory/ plays/install-edge-host.yml
|
||||
--limit {{ inventory_hostname }} --tags vlan40
|
||||
success_msg: >-
|
||||
No static networkd assignment of {{ keepalived_ntp_vip6 }}; safe for
|
||||
keepalived to own it.
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- ansible_facts['os_family'] == 'Archlinux'
|
||||
|
||||
- name: Create keepalived configuration
|
||||
ansible.builtin.template:
|
||||
src: keepalived.conf.j2
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
---
|
||||
# Withdraw a static assignment of the IPv6 NTP service address.
|
||||
#
|
||||
# Included by install-ntp-ha.yml as a FIRST play across the whole ntp_cluster,
|
||||
# before any host is given the NTP_VIP6 instance. That ordering is the point of
|
||||
# splitting it out.
|
||||
#
|
||||
# DAD is enabled on the VRRP address (see keepalived.conf.j2), so a static copy
|
||||
# on any host and a VRRP-managed copy on another cannot coexist: the kernel sees
|
||||
# the duplicate and marks the newcomer `dadfailed`, leaving a master that
|
||||
# believes it is serving while carrying an unusable address. Observed on edge-1
|
||||
# on 2026-08-07, where keepalived reported MASTER while the address sat
|
||||
# `dadfailed deprecated tentative` because three PVE nodes still held it
|
||||
# statically.
|
||||
#
|
||||
# Doing the withdrawal per host inside the main role could not avoid that: with
|
||||
# serial: 1 the first host would claim the address via VRRP while the rest were
|
||||
# still static. Fleet-wide first, then claim.
|
||||
#
|
||||
# fd40::123 was assigned by hand on all four servers, in two different ways,
|
||||
# none of it in version control:
|
||||
#
|
||||
# PVE nodes an `up ip -6 addr replace ... nodad` hook in
|
||||
# /etc/network/interfaces
|
||||
# edge-1 an [Address] block in /etc/systemd/network/, owned by the
|
||||
# edge-host role and removed by re-applying it -- not touched
|
||||
# here, since this role must not rewrite another role's file
|
||||
#
|
||||
# There is a service gap between this play and the next, while no host holds
|
||||
# the address. It is bounded by the run time of one play and clients simply
|
||||
# retry; the alternative is the dadfailed state above, which does not resolve
|
||||
# on its own.
|
||||
|
||||
- name: Remove static assignment of the NTP service address (ifupdown)
|
||||
ansible.builtin.lineinfile:
|
||||
path: /etc/network/interfaces
|
||||
regexp: '^\s*up\s+ip\s+-6\s+addr\s+(replace|add)\s+{{ keepalived_ntp_vip6 | regex_escape }}\b'
|
||||
state: absent
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- ansible_facts['os_family'] == 'Debian'
|
||||
register: keepalived_ntp_static6_removed
|
||||
|
||||
# Everything below is gated on the config line having just been removed, so the
|
||||
# runtime delete happens once -- on the migration -- and never again.
|
||||
#
|
||||
# The gate is load-bearing. Once this has landed, the address is legitimately
|
||||
# present on whichever host is the elected v6 master, placed there by
|
||||
# keepalived, and a delete cannot tell that apart from a static assignment (the
|
||||
# address looks identical either way). Ungated, every later run would delete the
|
||||
# master's address and restart keepalived, dropping the service address and
|
||||
# forcing a re-election each time -- the flapping this whole change exists to
|
||||
# prevent.
|
||||
- name: Check whether the service address is still assigned statically
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ip
|
||||
- -6
|
||||
- addr
|
||||
- show
|
||||
- dev
|
||||
- "{{ keepalived_ntp_interface }}"
|
||||
- to
|
||||
- "{{ keepalived_ntp_vip6 }}"
|
||||
register: keepalived_ntp_vip6_present
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- keepalived_ntp_static6_removed.changed | default(false)
|
||||
|
||||
# failed_when: false because the desired end state is "address absent". An
|
||||
# `ip addr del` against an address that has already gone is the outcome this
|
||||
# task wants, not a reason to abort the play for that host.
|
||||
- name: Withdraw the statically assigned service address
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ip
|
||||
- -6
|
||||
- addr
|
||||
- del
|
||||
- "{{ keepalived_ntp_vip6 }}"
|
||||
- dev
|
||||
- "{{ keepalived_ntp_interface }}"
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- keepalived_ntp_static6_removed.changed | default(false)
|
||||
- keepalived_ntp_vip6_present.stdout | default('') | length > 0
|
||||
failed_when: false
|
||||
|
||||
# The withdrawal gets exactly one attempt -- the gate that makes it idempotent
|
||||
# also means it never retries -- and the delete swallows its own errors. So the
|
||||
# result is asserted rather than assumed.
|
||||
#
|
||||
# `ip addr del` matches on address AND prefix length: if a host's static
|
||||
# assignment used a different mask or interface than ntp_vip6 /
|
||||
# keepalived_ntp_interface, the delete is a silent no-op and the address stays
|
||||
# outside VRRP control permanently, with the play reporting success.
|
||||
- name: Confirm the static service address was actually withdrawn
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- ip
|
||||
- -6
|
||||
- addr
|
||||
- show
|
||||
- dev
|
||||
- "{{ keepalived_ntp_interface }}"
|
||||
- to
|
||||
- "{{ keepalived_ntp_vip6 }}"
|
||||
register: keepalived_ntp_vip6_after
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- keepalived_ntp_static6_removed.changed | default(false)
|
||||
|
||||
- name: Assert the service address is no longer assigned outside keepalived
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- keepalived_ntp_vip6_after.stdout | default('') | length == 0
|
||||
fail_msg: >-
|
||||
{{ keepalived_ntp_vip6 }} is still assigned on
|
||||
{{ keepalived_ntp_interface }} after the static assignment was removed.
|
||||
`ip -6 addr del` matches address AND prefix length, so this usually means
|
||||
the on-host assignment differs from ntp_vip6 or uses another interface.
|
||||
Leaving it would both keep the address outside VRRP control and make the
|
||||
VRRP-managed copy fail DAD elsewhere. Inspect with
|
||||
`ip -6 addr show dev {{ keepalived_ntp_interface }}` and
|
||||
`grep -rn {{ keepalived_ntp_vip6 }} /etc/network/ /etc/systemd/network/`.
|
||||
success_msg: >-
|
||||
{{ keepalived_ntp_vip6 }} withdrawn; safe for keepalived to claim it.
|
||||
when:
|
||||
- keepalived_ntp_vip6 | length > 0
|
||||
- keepalived_ntp_static6_removed.changed | default(false)
|
||||
@@ -39,3 +39,72 @@ vrrp_instance NTP_VIP {
|
||||
notify_backup "/usr/local/bin/keepalived-notify.sh BACKUP"
|
||||
notify_fault "/usr/local/bin/keepalived-notify.sh FAULT"
|
||||
}
|
||||
{% if keepalived_ntp_vip6 %}
|
||||
|
||||
# IPv6 service address for NTP (ntp.srv.renner.uno). IPv6 is the primary
|
||||
# address family here, so this instance -- not the IPv4 one above -- is the
|
||||
# path that matters.
|
||||
#
|
||||
# A separate instance is mandatory, not stylistic: keepalived requires that
|
||||
# "all addresses in virtual_ipaddress must be of the same family", so the v6
|
||||
# address cannot be added to NTP_VIP.
|
||||
#
|
||||
# No authentication block: VRRPv3 removed authentication (RFC 5798), and
|
||||
# keepalived always uses version 3 for IPv6 instances. Configuring auth here
|
||||
# is rejected rather than ignored.
|
||||
#
|
||||
# The first address is link-local, and must be. RFC 5798 gives an IPv6 virtual
|
||||
# router a link-local address as its primary, and keepalived warns
|
||||
# "the first IPv6 VIP address should be link local" otherwise. It carries on,
|
||||
# but the instance is then sourcing VRRPv3 from something the protocol does not
|
||||
# expect.
|
||||
#
|
||||
# DAD is deliberately left ON (no -nodad). The flag was inherited from the
|
||||
# previous design, where every NTP server carried fd40::123 simultaneously and
|
||||
# duplicate detection had to be suppressed for that to work at all. Under
|
||||
# single-master VRRP the opposite is true: exactly one host should hold the
|
||||
# address, so a duplicate is a genuine fault and detecting it is the point.
|
||||
#
|
||||
# This does mean the migration cannot leave a static copy anywhere while a
|
||||
# VRRP-managed one exists -- DAD sees the two and marks the newcomer
|
||||
# `dadfailed`, leaving a master that believes it is serving while carrying an
|
||||
# unusable address. Observed on edge-1 on 2026-08-07. install-ntp-ha.yml
|
||||
# therefore withdraws every static assignment fleet-wide in a first play
|
||||
# before any host configures this instance.
|
||||
#
|
||||
# track_script is what makes this health-gated: the address is only held while
|
||||
# check_ntp passes, and is withdrawn as soon as it fails. On withdrawal
|
||||
# keepalived sends unsolicited Neighbour Advertisements so client neighbour
|
||||
# caches are updated immediately -- a statically assigned address cannot do
|
||||
# this, and clients would keep sending to the dead holder until their own
|
||||
# neighbour unreachability detection expired.
|
||||
vrrp_instance NTP_VIP6 {
|
||||
state {{ keepalived_ntp_state }}
|
||||
interface {{ keepalived_ntp_interface }}
|
||||
virtual_router_id {{ keepalived_ntp_router_id6 }}
|
||||
priority {{ keepalived_ntp_priority }}
|
||||
advert_int 1
|
||||
|
||||
virtual_ipaddress {
|
||||
{{ keepalived_ntp_vip6_linklocal }} dev {{ keepalived_ntp_interface }}
|
||||
{{ keepalived_ntp_vip6 }} dev {{ keepalived_ntp_interface }}
|
||||
}
|
||||
|
||||
# weight 0 overrides the script's own weight, and the distinction decides
|
||||
# whether this works at all. A non-zero weight only SUBTRACTS from priority
|
||||
# on failure -- so the last remaining host, having no one to lose the
|
||||
# election to, stays MASTER and keeps serving time with a clock it has
|
||||
# already reported as wrong. Weight 0 instead transitions the instance to
|
||||
# FAULT after <fall> failures, which releases the address unconditionally.
|
||||
#
|
||||
# That is the required behaviour here: a host unfit to serve time must stop
|
||||
# answering, whether or not a replacement exists.
|
||||
track_script {
|
||||
check_ntp weight 0
|
||||
}
|
||||
|
||||
notify_master "/usr/local/bin/keepalived-notify.sh MASTER6"
|
||||
notify_backup "/usr/local/bin/keepalived-notify.sh BACKUP6"
|
||||
notify_fault "/usr/local/bin/keepalived-notify.sh FAULT6"
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:8.8.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
image: redis:8.10.0-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
|
||||
args:
|
||||
- redis-server
|
||||
- --save
|
||||
|
||||
@@ -34,7 +34,7 @@ spec:
|
||||
# Image automation, range 3.x (see image-automation/woodpecker.yaml).
|
||||
# MUST stay version-matched with woodpecker-server (gRPC parity) — both
|
||||
# track the same range/filter so they bump together on each release.
|
||||
image: woodpeckerci/woodpecker-agent:v3.16.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
image: woodpeckerci/woodpecker-agent:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
|
||||
env:
|
||||
- name: WOODPECKER_SERVER
|
||||
value: woodpecker.apps.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
|
||||
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
# minor + patch auto-update within 3.x; only the 4.0 major is held for
|
||||
# a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
|
||||
# -bump hazard is gone). Stays in lockstep with the agent.
|
||||
image: woodpeckerci/woodpecker-server:v3.16.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
image: woodpeckerci/woodpecker-server:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
|
||||
env:
|
||||
- name: WOODPECKER_HOST
|
||||
value: https://ci.renner.uno
|
||||
|
||||
Reference in New Issue
Block a user