fix: [alert] KubeContainerWaiting (home-assistant) (#592)

Reviewed-on: k.renner/homelab#597

.woodpecker/deploy-infrastructure.yml

@@ -131,18 +131,18 @@ steps:
       - |
         # Ensure Talos image exists on all Proxmox nodes via API.
         #
-        # jupiter is NOT in this list: it runs Talos on bare metal and is no
-        # longer a Proxmox node, so the storage API returns nothing for it and
-        # the download-wait loop below would burn its full 60x10s before
-        # giving up. saturn and neptune still host k8s-2/k8s-3 as VMs and
-        # still need the nocloud image; bare-metal nodes boot the metal-amd64
-        # image from their own installed disk instead.
+        # Only nodes still running Proxmox belong in this list. jupiter and
+        # saturn run Talos on bare metal now, so the storage API returns
+        # nothing for them and the download-wait loop below would burn its full
+        # 60x10s per node before giving up. Bare-metal nodes boot the
+        # metal-amd64 image from their own installed disk instead, and the
+        # whole step goes away once neptune is converted too.
         TALOS_VERSION="v1.12.6"
         TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
         TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
         TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
         API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
-        for node in saturn neptune; do
+        for node in neptune; do
           EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
             -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
           if [ "$EXISTS" -gt 0 ]; then

kubernetes/RUNBOOK-baremetal-conversion.md

@@ -172,12 +172,22 @@ use the secureboot installer image too. That is a separate project.
 ## 5. Install
 
 Boot the stick. Maintenance mode DHCPs untagged on every link; on these hosts
-that lands on `192.168.8.x`, which is routable from the workstation.
+that lands on `192.168.8.x`, which is routable from the workstation. It DHCPs on
+*every* link, so one machine answers on several addresses at once — they are the
+same host, not several hosts. `get disks` on each confirms it.
 
-Maintenance mode is effectively write-only — `talosctl get` has no `--insecure`
-and `version` returns "API is not implemented in maintenance mode". You cannot
-inspect disks or links before applying. Verify serials and MACs from the host's
-PVE beforehand, in step 3.
+Maintenance mode does serve `talosctl get`. `--insecure` is a flag on the
+subcommand rather than a global, so it must come *after* `get` — placed before
+it, cobra resolves to the legacy `talosctl disks` command and reports
+`unknown flag: --insecure`:
+
+```sh
+talosctl -n <maintenance-ip> get disks --insecure
+talosctl -n <maintenance-ip> get links --insecure
+```
+
+Verify serials and MACs here, on the machine itself, against the config about to
+be applied.
 
 Build the config from the **live cluster's secrets**, not a fresh
 `talosctl gen config`, or the node forms a new cluster instead of joining:
@@ -202,13 +212,58 @@ Strip the generated `HostnameConfig` document — it collides with
 against a base with no `inlineManifests` (issue #581). Use only the additive
 second document.
 
-Validate, then apply:
+Validate:
 
 ```sh
 talosctl validate --config install.yaml --mode metal
-talosctl apply-config --insecure -n <maintenance-ip> --file install.yaml
 ```
 
+### Install minimal, complete on the running node
+
+Do **not** apply the full per-node config in maintenance mode. A document that
+references an interface which does not exist yet hangs the install: the node
+sits at `STAGE: Installing` indefinitely with `KUBELET n/a`, `APISERVER n/a`,
+and the console scrolling
+
+```
+k8s.NodeApplyController: error getting node: nodes "k8s-N" not found
+```
+
+That message is a symptom of an install that never finished, not a Kubernetes or
+networking fault — the node holds a DHCP address and is reachable throughout. On
+saturn the offenders were the `ens19` VLAN document and the sysctl naming it,
+`net.ipv6.conf.ens19.disable_ipv6`. Three cold installs hung on it; the identical
+config applied cleanly once the node was up. Wiping disks makes no difference —
+it is not leftover state.
+
+**Stage 1** — the smallest config that installs and joins, plus `EPHEMERAL`,
+which is one-shot (section 7) and so cannot be deferred to stage 2:
+
+```
+machine.network.hostname, machine.install.diskSelector, certSANs, nodeLabels
+LinkConfig    <uplink>
+VLANConfig    mgmt0
+DHCPv4Config  mgmt0
+VolumeConfig  EPHEMERAL
+```
+
+```sh
+talosctl apply-config --insecure -n <maintenance-ip> --file install-stage1.yaml
+```
+
+**Stage 2** — once the node is `Ready` and `talosctl etcd members` lists it,
+apply the full config to the running node. `--mode=try` reverts on its own if the
+network config is wrong, so it cannot strand the node; re-apply with
+`--mode=auto` to persist.
+
+```sh
+talosctl -e <node-ip> -n <node-ip> apply-config --mode=try --timeout=3m --file install.yaml
+talosctl -e <node-ip> -n <node-ip> apply-config --mode=auto --file install.yaml
+```
+
+Both applied without a reboot on saturn. Confirm `ens19`, `mgmt0` and the Ceph
+link are up afterwards with `talosctl get links` and `get addresses`.
+
 ## 6. The network config will strand the node if it is wrong
 
 Talos stops running default DHCP operators **for the entire machine** as soon as

kubernetes/apps/immich/machine-learning.yaml

@@ -6,17 +6,14 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-machine-learning
 spec:
-  # TEMPORARY — RESTORE TO 1 ONCE k8s-1 HAS REJOINED.
-  #
-  # Down for the jupiter bare-metal conversion. With one of three nodes out,
-  # the remaining pair have no room: cluster CPU requests exceed two nodes'
-  # capacity, and k8s-3 runs ~5.8 GiB above the kubelet's eviction threshold,
-  # so every pod that does not need to be running is one less thing competing
-  # for both. This one is the best candidate: 500m reserved against a measured
-  # 2m idle draw, and inference bursts hard enough that lowering the request
-  # instead would be the wrong trade. Photo processing pauses; nothing else
-  # depends on it.
-  replicas: 0
+  # Scale this and immich-server to 0 for the duration of any single-node
+  # maintenance — cluster CPU requests exceed what two nodes can hold, and this
+  # pair frees ~1000m against a shortfall of roughly 200m. Stopping them is the
+  # right lever rather than lowering their requests: 500m reserved against a
+  # 2m idle draw looks like waste, but inference bursts hard and the request is
+  # honest about what it needs when someone uploads photos.
+  # See RUNBOOK-baremetal-conversion.md.
+  replicas: 1
   strategy:
     type: RollingUpdate
   selector:

kubernetes/apps/immich/server.yaml

@@ -36,11 +36,10 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-server
 spec:
-  # TEMPORARY — RESTORE TO 1 ONCE k8s-1 HAS REJOINED. See the note on
-  # immich-machine-learning; immich is offline for the jupiter conversion.
-  # Its Postgres cluster is deliberately left running, so nothing needs
-  # restoring beyond these two replica counts.
-  replicas: 0
+  # Scaled to 0 alongside immich-machine-learning during single-node
+  # maintenance; see the note there. The CNPG Postgres cluster stays running,
+  # so recovery is just these two replica counts.
+  replicas: 1
   strategy:
     type: Recreate
   selector:

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -77,6 +77,21 @@ spec:
       # redirects the socket directly to the backend pod instead of sending
       # traffic out to the VIP and back (which fails with L2 announcements).
       lbExternalClusterIP: true
+      # VLAN ids whose tagged frames are passed to the stack instead of being
+      # dropped by Cilium's VLAN filter. Unset, the filter drops every tagged
+      # frame the datapath sees — which is what strands the v30iot macvlan
+      # pods on a converted node: there `ens19` (VLAN 30) and `mgmt0`
+      # (VLAN 40) are VLAN sub-interfaces of the enp2s0f0np0 trunk, so a
+      # macvlan child's DHCP DISCOVER leaves tagged and dies as
+      # DROP_VLAN_FILTER. The CNI DHCP daemon reports that as "no buffer
+      # space available" (a tc drop surfaces to an AF_PACKET sender as
+      # ENOBUFS, not as a timeout) and the pod sits in ContainerCreating
+      # forever — issue #592, predicted in RUNBOOK-baremetal-conversion.md.
+      # Lists only the two VLANs a node terminates; [0] would bypass the
+      # filter for every id. Takes effect on agent restart.
+      # Keep in lockstep with vlan-bpf-bypass in
+      # kubernetes/talos/patches/cilium-cni.yaml and kubernetes/talos/cilium.yaml.
+      vlanBypass: [30, 40]
     cni:
       # Multus (kubernetes/infrastructure/multus) runs alongside Cilium and
       # writes 00-multus.conf to /etc/cni/net.d so kubelet picks multus

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -167,33 +167,37 @@ spec:
   # impossible: the OSD is recreated within a reconcile and starts backfilling
   # onto the drive being replaced.
   #
-  # nvme0n1 on k8s-2/k8s-3 is the NVMe drive PCI-passed-through from saturn and
-  # neptune, and is the only block device those guests can see, so a bare device
-  # name is unambiguous there.
-  #
-  # k8s-1 is bare metal and sees three NVMe drives, so it names its OSD by
-  # /dev/disk/by-id path instead. A bare name would be actively wrong: Talos
-  # enumerates this machine's drives in a different order than Proxmox did, and
-  # nvme0n1 here is the Kingston SYSTEM disk. Rook would probably decline it for
-  # carrying partitions, but that is luck rather than intent. The by-id path is
-  # derived from model and serial and does not depend on enumeration at all.
+  # k8s-1 and k8s-2 are bare metal and see three NVMe drives each, so they name
+  # their OSDs by /dev/disk/by-id path instead. A bare name would be actively
+  # wrong: Talos enumerates these machines' drives in a different order than
+  # Proxmox did, and nvme0n1 there is the Kingston SYSTEM disk. Rook would
+  # probably decline it for carrying partitions, but that is luck rather than
+  # intent. The by-id path is derived from model and serial and does not depend
+  # on enumeration at all.
   #
+  #   k8s-1 (jupiter)
   #   nvme0n1  50026B7382B48B86  Kingston  — Talos system disk
   #   nvme1n1  2332E86B749A      Crucial   — Talos EPHEMERAL (/var)
   #   nvme2n1  S437NX0M400426    PM983     — this OSD
+  #
+  #   k8s-2 (saturn)
+  #   nvme0n1  50026B7382DA5F6E  Kingston  — Talos system disk
+  #   nvme1n1  2332E86B74BC      Crucial   — Talos EPHEMERAL (/var)
+  #   nvme2n1  S437NX0M403161    PM983     — this OSD
   storage:
     useAllNodes: false
     useAllDevices: false
     nodes:
-      - name: k8s-2
-        devices:
-          - name: nvme0n1
-      - name: k8s-3
-        devices:
-          - name: nvme0n1
+      # k8s-3 is absent while neptune's OSD drive is swapped and the node is
+      # converted to bare metal. RE-ADD IT once the PM983 is in and the node has
+      # rejoined, naming the device by /dev/disk/by-id as the other two do —
+      # bare metal sees three NVMe drives and the bare name is not the OSD.
       - name: k8s-1
         devices:
           - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M400426
+      - name: k8s-2
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M403161
     config:
       osdsPerDevice: "1"
   # Storage layer must never be evicted by kubelet under node pressure

kubernetes/infrastructure/rook-ceph-cluster/storageclass.yaml

@@ -87,7 +87,10 @@ metadata:
   name: cephfs
   namespace: rook-ceph
 spec:
-  # TEMPORARY — REVERT TO size 3 / no min_size OVERRIDE ONCE k8s-1 IS BACK.
+  # TEMPORARY — RESTORE TO size 3 / no min_size OVERRIDE ONCE SATURN AND
+  # NEPTUNE HAVE BOTH BEEN CONVERTED TO BARE METAL. It stays at 2/1 across all
+  # three conversions rather than being toggled per node: each change
+  # re-replicates the pool, and the window reopens with the next node anyway.
   #
   # The standing trade for this pool is size 3: it holds ~253 MiB against the
   # data pool's 280 GiB, so it contributes nothing measurable to drive wear —
@@ -96,13 +99,14 @@ spec:
   # quarter of a gigabyte to keep that intact is the right trade in steady
   # state.
   #
-  # It does not hold while one of three hosts is out for the jupiter bare-metal
-  # conversion. size 3 on two remaining hosts leaves every PG undersized at two
-  # copies with min_size 2, so a single further OSD loss drops the pool below
-  # min_size and blocks IO on every CephFS mount cluster-wide. At 2/1 the same
-  # two copies are active+clean instead, and a further loss degrades rather
-  # than blocks — availability bought at the cost of accepting writes onto a
-  # single copy while degraded.
+  # It does not hold while one of three hosts is out. size 3 on two remaining
+  # hosts leaves every PG undersized at two copies with min_size 2, so a single
+  # further OSD loss drops the pool below min_size and blocks IO on every
+  # CephFS mount cluster-wide. At 2/1 the same two copies are active+clean
+  # instead, and a further loss degrades rather than blocks — availability
+  # bought at the cost of accepting writes onto a single copy while degraded.
+  #
+  # See RUNBOOK-baremetal-conversion.md.
   metadataPool:
     replicated:
       size: 2

kubernetes/talos/cilium.yaml

@@ -306,6 +306,12 @@ data:
   # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic
   # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
   bpf-map-dynamic-size-ratio: "0.0025"
+  # VLAN ids whose tagged frames bypass the eBPF datapath rather than being
+  # dropped by the VLAN filter. 30 (v30iot, the macvlan parent) and 40 (mgmt)
+  # are VLAN sub-interfaces of the trunk on a bare-metal node. Keep in lockstep
+  # with patches/cilium-cni.yaml and values.bpf.vlanBypass in
+  # kubernetes/infrastructure/cilium/helmrelease.yaml.
+  vlan-bpf-bypass: "30,40"
   # bpf-policy-map-max specifies the maximum number of entries in endpoint
   # policy map (per endpoint). Keep in lockstep with patches/cilium-cni.yaml and
   # values.bpf.policyMapMax in kubernetes/infrastructure/cilium/helmrelease.yaml.

kubernetes/talos/patches/cilium-cni.yaml

@@ -324,6 +324,13 @@ cluster:
           # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic
           # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
           bpf-map-dynamic-size-ratio: "0.0025"
+          # VLAN ids whose tagged frames bypass the eBPF datapath rather than being
+          # dropped by the VLAN filter. 30 (v30iot, the macvlan parent) and 40 (mgmt)
+          # are VLAN sub-interfaces of the trunk on a bare-metal node, so without this
+          # a cold rebuild comes up with the v30iot macvlan pods unable to DHCP. Keep
+          # in lockstep with kubernetes/talos/cilium.yaml and values.bpf.vlanBypass in
+          # kubernetes/infrastructure/cilium/helmrelease.yaml.
+          vlan-bpf-bypass: "30,40"
           # bpf-policy-map-max specifies the maximum number of entries in endpoint
           # policy map (per endpoint). Keep in lockstep with values.bpf.policyMapMax
           # in kubernetes/infrastructure/cilium/helmrelease.yaml, and with the third

kubernetes/talos/patches/k8s-2.yaml

@@ -1,31 +1,99 @@
+# k8s-2 runs Talos directly on saturn's MS-01 hardware. Structurally identical
+# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
+# serials, MACs and addresses differ. k8s-3 is still a Proxmox VM and keeps the
+# VM-era layout until neptune is converted.
 machine:
   network:
     hostname: k8s-2
-    # Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
-    interfaces:
-      - deviceSelector:
-          busPath: "0000:06:12.0"
-        dhcp: true
-        vip:
-          ip: 10.2.40.60
-      # v30iot: link only, no host addressing — see k8s-1.yaml.
-      - deviceSelector:
-          busPath: "0000:06:13.0"
-        dhcp: false
-      # See k8s-1.yaml for context on the dedicated Ceph cluster network.
-      - deviceSelector:
-          driver: i40e
-        addresses:
-          - 10.99.0.2/29
-        mtu: 9000
   sysctls:
-    # v30iot NIC carries no host addressing — see k8s-1.yaml.
+    # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It is the
+    # macvlan parent for the v30iot pod attachments; host addresses on that L2
+    # make node replies bypass the UDM and break every UDM-DNATed flow.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    disk: /dev/vda
+    # Three NVMe drives are visible on bare metal. Select the installer target
+    # by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
+    # order the kernel enumerates them in — Talos and Proxmox do not agree on
+    # that order.
+    diskSelector:
+      serial: 50026B7382DA5F6E
   certSANs:
     - k8s-2.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: saturn
 cluster:
   allowSchedulingOnControlPlanes: true
+---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
+# drive is at 81% of its rated endurance and is finished as an OSD, but
+# everything EPHEMERAL holds is reconstructible, so it is consumable in this
+# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had.
+#
+# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; applied later it is silently
+# ignored and relocating needs a disk wipe and a boot from install media.
+#
+# Matched by serial deliberately: a broad selector such as
+# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B74BC'
+  minSize: 50GiB
+  grow: true
+---
+# X710 port 0 carries the switch trunk and takes no addressing itself — the
+# port's untagged VLAN is not one this node belongs on. Management and v30iot
+# both ride it as tagged VLANs.
+#
+# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are
+# valid VLAN parents, but a selector matching zero links is skipped silently
+# and Talos disables default DHCP machine-wide as soon as any new-style network
+# document exists — so a silent miss leaves the node with no address on any
+# interface. See RUNBOOK-baremetal-conversion.md.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f0np0
+up: true
+---
+# Management, VLAN 40. Node IP, etcd, the API and Ceph's public network.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: mgmt0
+vlanID: 40
+parent: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: mgmt0
+---
+# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
+# k8s.renner.uno.
+apiVersion: v1alpha1
+kind: Layer2VIPConfig
+name: 10.2.40.60
+link: mgmt0
+---
+# Named ens19 on purpose: the four v30iot macvlan NADs hard-code
+# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that
+# name. VLANConfig.name is free-form, so one set of NADs serves both.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ens19
+vlanID: 30
+parent: enp2s0f0np0
+up: true
+---
+# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
+# untagged on the wire. OSD<->OSD replication only. Not routed off its own
+# segment and deliberately has no default route.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f1np1
+up: true
+mtu: 9000
+addresses:
+  - address: 10.99.0.2/29

opentofu/modules/proxmox_vm/variables.tf

@@ -47,8 +47,13 @@ variable "tags" {
 }
 
 variable "template_name" {
-  type    = string
-  default = "ubuntu-24-04-template"
+  type = string
+  # No default clone source. The ubuntu-24.04-template it used to name was
+  # deleted with saturn's conversion to bare metal, and Proxmox is being
+  # retired, so a default pointing at a template that does not exist would
+  # only fail at apply time. Set it explicitly if a clone source is ever
+  # wanted again.
+  default = null
 }
 
 variable "full_clone" {

opentofu/vm_151_k8s_2.tf

@@ -1,50 +0,0 @@
-module "k8s_2" {
-  source = "./modules/proxmox_vm"
-
-  name        = "k8s-2.srv.renner.uno"
-  target_node = "saturn"
-  vmid        = 151
-  memory      = 49152
-  cores       = 8
-
-  template_name = null
-  cloud_init    = false
-  full_clone    = false
-  bios          = "ovmf"
-  machine       = "q35"
-  boot          = "order=virtio0"
-  agent_enabled = true
-
-  # Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
-  # VLAN from this node (previously only k8s-edge had v30iot, so any
-  # IoT-pool LB lease landing here was unreachable from devices).
-  extra_network_bridges = ["v30iot"]
-
-  disks = [
-    {
-      storage = "local-zfs"
-      size    = "64G"
-      format  = "raw"
-      slot    = "virtio0"
-      file_id = local.talos_image
-    }
-  ]
-
-  hostpci = [
-    {
-      mapping = "osd-ssd"
-    },
-    {
-      mapping = "x710-port1"
-    }
-  ]
-
-  tags            = "kubernetes;talos;controlplane"
-  ssh_public_keys = local.common_ssh_keys
-
-  vm_state = "running"
-
-  # Rebuild pipeline issues `-replace`; skip graceful shutdown so a
-  # non-responsive Talos guest can't hang the destroy for 30 minutes.
-  stop_on_destroy = true
-}

opentofu/vm_605_template.tf

@@ -1,71 +0,0 @@
-resource "proxmox_virtual_environment_vm" "ubuntu_24_04_template" {
-  name      = "ubuntu-24.04-template"
-  node_name = "saturn"
-  vm_id     = 605
-
-  started = false
-  template = true
-  on_boot = false
-
-  keyboard_layout = "en-us"
-
-  bios = "seabios"
-
-  agent {
-    enabled = true
-  }
-
-  cpu {
-    cores = 2
-    type  = "x86-64-v3"
-  }
-
-  memory {
-    dedicated = 2048
-  }
-
-  initialization {
-    datastore_id = "local-zfs"
-
-    ip_config {
-      ipv4 {
-        address = "dhcp"
-      }
-    }
-
-    user_account {
-      username = "ladmin"
-      keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
-      ]
-    }
-  }
-
-  disk {
-    datastore_id = "local-zfs"
-    interface    = "scsi0"
-    size         = 32
-    file_format  = "raw"
-    iothread     = false
-  }
-
-  network_device {
-    bridge = "v40srv"
-    model  = "virtio"
-  }
-
-  serial_device {}
-
-  scsi_hardware = "virtio-scsi-pci"
-
-  lifecycle {
-    ignore_changes = [
-      node_name,
-      started,
-      initialization,
-      disk,
-      cpu,
-      memory
-    ]
-  }
-}