Alerts

alert: GenericBatteryLow
expr: (label_replace(ha_sensor_battery_percent{entity!~".*smoke.*",entity!~"sensor\\..*(deadbolt|lock)_battery_level",friendly_name!="None"},
  "device", "$1", "entity", "sensor\\.(.*?)_battery.*")
  unless on (device) label_replace(ha_entity_available{entity=~"sensor\\..*_battery_(state|cycles|health|thermal_status)"},
  "device", "$1", "entity", "sensor\\.(.*?)_battery.*"))
  < 20
for: 1d
labels:
  severity: warning
annotations:
  description: '{{ $labels.entity }} is at {{ $value | humanize }}% battery'
  summary: '{{ $labels.friendly_name }} battery low'

alert: SmokeDetectorBatteryLow
expr: ha_sensor_battery_percent{entity=~".*smoke.*",friendly_name!="None"}
  < 78
for: 1h
labels:
  severity: warning
annotations:
  description: It will start beeping soon
  summary: '{{ $labels.friendly_name }} battery low'

alert: BatteryDeviceUnavailable
expr: label_replace(ha_entity_available{entity=~"sensor\\..*_battery(_level)?"},
  "device", "$1", "entity", "sensor\\.(.*?)_battery.*")
  == 0 unless on (device) label_replace(ha_entity_available{entity=~"sensor\\..*_battery_(state|cycles|health|thermal_status)"},
  "device", "$1", "entity", "sensor\\.(.*?)_battery.*")
for: 3d
labels:
  severity: warning
annotations:
  description: '{{ $labels.entity }} has been unavailable for three days; dead battery,
    or a stale entity to delete in Home Assistant'
  summary: '{{ $labels.friendly_name }} unavailable'

alert: DeadboltBatteryLow
expr: ha_sensor_battery_percent{entity!="sensor.front_door_lock_battery_level",entity=~"sensor\\..*(deadbolt|lock)_battery_level",friendly_name!="None"}
  < 30
for: 1d
labels:
  severity: warning
annotations:
  description: '{{ $labels.entity }} is at {{ $value | humanize }}%; Schlage deadbolts
    start beeping around 25%'
  summary: '{{ $labels.friendly_name }} low'

alert: FrontDeadboltBatteryLow
expr: max_over_time(ha_sensor_battery_percent{entity="sensor.front_door_lock_battery_level"}[3d])
  == 0
for: 2w
labels:
  severity: warning
annotations:
  description: The front deadbolt has reported 0% battery without a break for over
    two weeks
  summary: Front deadbolt battery low

alert: BgpImportLimitNear
expr: max
  without (state) (bird_protocol_prefix_import_count{job="bird",name="ampr"})
  > 4000
for: 1h
labels:
  severity: warning
annotations:
  description: At 5000 bird blocks further imports from AMPR; raise the import limit
    in /etc/bird/bird.conf
  summary: BGP session {{ $labels.name }} is importing {{ $value }} prefixes, near
    the 5000 limit

alert: BgpPrefixNotAnnounced
expr: max
  without (state) (bird_protocol_prefix_export_count{job="bird",proto="BGP"})
  == 0 and on (name) max without (state) (bird_protocol_up{job="bird",proto="BGP"})
  == 1
for: 30m
labels:
  severity: critical
annotations:
  description: The session is Established but exports 0 routes; check the originate
    static protocol and bond0.44
  summary: '{{ $labels.instance }} is not announcing any prefix on BGP session {{
    $labels.name }}'

alert: BgpSessionDown
expr: max
  without (state) (bird_protocol_up{job="bird",proto="BGP"}) == 0
for: 35m
labels:
  severity: critical
annotations:
  description: bird has not had {{ $labels.name }} Established for 30 minutes; 44net
    routes are stale or gone. Check birdc show protocols all {{ $labels.name }}
  summary: BGP session {{ $labels.name }} on {{ $labels.instance }} is down

alert: BirdUnreachable
expr: bird_socket_query_success{job="bird"}
  == 0
for: 30m
labels:
  severity: critical
annotations:
  description: bird is stopped or its control socket is unreachable, so BGP session
    alerts cannot fire
  summary: bird_exporter on {{ $labels.instance }} cannot query bird

alert: CatsNotFed
expr: esphome_switch_value{id="cat_feeder"}
  == 0
for: 20h
labels:
  severity: critical
annotations:
  summary: Cats have not been fed for nearly a day, according to esphome!

alert: ContainerNotRunning
expr: docker_container_running
  == 0 and on (instance, name) docker_container_info{restart_policy!="no"}
for: 10m
labels:
  severity: warning
annotations:
  description: docker ps -a --filter name={{ $labels.name }}; docker logs --tail 50
    {{ $labels.name }}
  summary: Container {{ $labels.name }} on {{ $labels.instance }} is not running

alert: ContainerRestarting
expr: increase(docker_container_restarts[1h])
  >= 3 or docker_container_restarting == 1
for: 15m
labels:
  severity: warning
annotations:
  description: Restarted by its restart policy repeatedly in the last hour; check
    docker logs {{ $labels.name }}
  summary: Container {{ $labels.name }} on {{ $labels.instance }} is crash-looping

alert: ContainerUnhealthy
expr: docker_container_health_status{status="unhealthy"}
  == 1
for: 10m
labels:
  severity: warning
annotations:
  description: Its Docker healthcheck has failed for 10 minutes; see docker inspect
    {{ $labels.name }} (State.Health.Log)
  summary: Container {{ $labels.name }} on {{ $labels.instance }} is unhealthy

alert: DockerCollectorStale
expr: time()
  - node_textfile_mtime_seconds{file=~".*/docker.prom"} > 900 or docker_collector_success
  == 0
for: 10m
labels:
  severity: warning
annotations:
  description: Container alerts are blind; check systemctl status prometheus-node-exporter-docker.timer
  summary: Docker collector on {{ $labels.instance }} is not updating

alert: DockerDown
expr: node_systemd_unit_state{instance="them",name="docker.service",state="active"}
  == 0
for: 5m
labels:
  severity: critical
annotations:
  description: every container on them (grafana, frigate, immich, ...) is down with
    it
  summary: docker.service on them is not active

alert: EnvoyBackendDown
expr: envoy_cluster_membership_healthy{job="envoy"}
  == 0 and envoy_cluster_membership_total{job="envoy"} > 0
for: 15m
labels:
  severity: warning
annotations:
  description: Envoy health checks to cluster {{ $labels.envoy_cluster_name }} have
    failed for 15 minutes; its site returns 503
  summary: Envoy backend {{ $labels.envoy_cluster_name }} is down

alert: EnvoyDown
expr: probe_success{instance="https://library.fivesevenfive.org/robots.txt",job="blackbox-http"}
  == 0
for: 10m
labels:
  severity: critical
annotations:
  summary: Envoy is down!

alert: BtrfsDeviceErrors
expr: sum
  by (instance, device, type) (increase(node_btrfs_device_errors_total[1h])) >
  0
labels:
  severity: warning
annotations:
  description: See btrfs device stats on the mount, run a scrub and check SMART for
    the disk; clear the counters with btrfs device stats -z once dealt with
  summary: btrfs device {{ $labels.device }} on {{ $labels.instance }} logged new
    {{ $labels.type }} errors in the last hour

alert: BtrfsMetadataLow
expr: (sum
  by (instance, uuid) (node_btrfs_size_bytes{block_group_type="metadata"}
  - node_btrfs_used_bytes{block_group_type="metadata"}) + sum by (instance,
  uuid) (node_btrfs_device_unused_bytes) / on (instance, uuid) (max by (instance,
  uuid) (node_btrfs_allocation_ratio{block_group_type="metadata"} > 0)
  or max by (instance, uuid) (0 * node_btrfs_size_bytes{block_group_type="metadata",mode="raid1c3"}
  + 3) or max by (instance, uuid) (0 * node_btrfs_size_bytes{block_group_type="metadata",mode="raid1c4"}
  + 4))) / 2 ^ 30 < 2
for: 1h
labels:
  severity: warning
annotations:
  description: Free metadata plus unallocated space; at about 0.5 GiB (the global
    reserve) every write fails with ENOSPC even though df shows free space. Free chunks
    with btrfs balance start -dusage=25, or delete snapshots
  summary: btrfs {{ $labels.uuid }} on {{ $labels.instance }} has {{ $value | printf
    "%.1f" }} GiB of metadata headroom

alert: FilesystemLow
expr: node_filesystem_avail_bytes{fstype=~"ext4|btrfs|xfs|vfat",mountpoint!~"/mnt/snapraid/.+"}
  / node_filesystem_size_bytes < 0.1
for: 1h
labels:
  severity: warning
annotations:
  description: Under 10% free for an hour; on btrfs see also BtrfsMetadataLow, since
    df can show free space btrfs cannot allocate
  summary: '{{ $labels.mountpoint }} on {{ $labels.instance }} has {{ $value | humanizePercentage
    }} free'

alert: IpForwardingDisabled
expr: node_netstat_Ip_Forwarding{instance="love"}
  != 1
for: 2m
labels:
  severity: critical
annotations:
  description: The border router has stopped routing IPv4; the whole network is offline.
    Check sysctl net.ipv4.ip_forward
  summary: IPv4 forwarding is disabled on {{ $labels.instance }}

alert: Ipv6ForwardingDisabled
expr: node_ip_forwarding{family="inet6",instance="love"}
  == 0
for: 2m
labels:
  severity: critical
annotations:
  description: The border router has stopped routing IPv6. Check sysctl net.ipv6.conf.all.forwarding
  summary: IPv6 forwarding is disabled on {{ $labels.instance }}

alert: RouteCollectorStale
expr: time()
  - node_textfile_mtime_seconds{file=~".*/routes.prom",instance="love"}
  > 1800
for: 10m
labels:
  severity: warning
annotations:
  description: Route, neighbour and IPv6 forwarding metrics are stale, so Ipv6ForwardingDisabled
    cannot fire; check prometheus-node-exporter-routes.timer
  summary: Routing collector on {{ $labels.instance }} has not updated for 30 minutes

alert: FreezerHot
expr: esphome_sensor_value{id="freezer_temp"}
  > 0
for: 1h
labels:
  severity: critical
annotations:
  summary: The chest freezer is above freezing!

alert: FridgeHot
expr: (ha_sensor_temperature{entity=~".*fridge.*"}
  > -30) > 5
for: 3h
labels:
  severity: critical
annotations:
  summary: The fridge is too warm!

alert: FridgeLow
expr: (ha_sensor_temperature{entity=~".*fridge.*"}
  > -30) < 0
for: 12h
labels:
  severity: critical
annotations:
  summary: The fridge is too cold!

alert: ThemUnreachable
expr: probe_success{instance="198.19.48.7",job="blackbox-ping"}
  == 0
for: 5m
labels:
  severity: critical
annotations:
  description: Home Assistant, Frigate, Grafana, Immich, the Omada controller and
    sundial are all unreachable
  summary: them (198.19.48.7) is not answering ping

alert: FurnaceBlowingColdAutomation
expr: (avg(deriv(ha_sensor_temperature{entity=~"sensor.(hall|living_room|office|bedroom|kids_room)_temperature"}[10m]))
  * on (job) max(ha_sensor_state{entity="sensor.heat_stage"})) < 0
for: 40m
labels:
  severity: automation
annotations:
  description: Calling for heat but it's getting colder!
  summary: Furnace is broken!

alert: FurnaceBlowingColdEmail
expr: (avg(deriv(ha_sensor_temperature{entity=~"sensor.(hall|living_room|office|bedroom|kids_room)_temperature"}[15m]))
  * on (job) max(ha_sensor_state{entity="sensor.heat_stage"})) < 0
for: 1h
labels:
  severity: warning
annotations:
  description: Calling for heat but it's getting colder!
  summary: Furnace is broken!

alert: FurnaceFilterDirty
expr: quantile_over_time(0.5,
  hvac:filter_dp_highfan:pa[1w]) < -40 and count_over_time(hvac:filter_dp_highfan:pa[1w])
  >= 60
for: 1d
labels:
  severity: warning
annotations:
  description: 7-day median of press_diff_pressure while sensor.furnace_watts is over
    550 W. Reset the ecobee filter reminder after replacing it.
  summary: 'Replace the furnace filter: high-fan pressure is {{ $value | printf "%.1f"
    }} Pa (fresh is about -33)'

alert: ChronydDown
expr: chrony_up
  == 0 or up{job=~"chrony.*"} == 0
for: 10m
labels:
  severity: critical
annotations:
  description: chrony_exporter can't reach chronyd, or can't be scraped, on
    {{ $labels.instance }}
  summary: chronyd on {{ $labels.instance }} is down

alert: NtpNotServing
expr: rate(chrony_serverstats_ntp_packets_received_total[15m])
  == 0
for: 15m
labels:
  severity: critical
annotations:
  description: No NTP requests counted for 15 minutes; check the socket bindings and
    firewall
  summary: chronyd on {{ $labels.instance }} has stopped receiving NTP packets

alert: NtpNotStratum1
expr: chrony_tracking_stratum{instance=~"us|tick|sundial"}
  != 1
for: 15m
labels:
  severity: warning
annotations:
  description: Its GPS/PPS refclock is not selected; check chronyc sources on {{ $labels.instance
    }}
  summary: chronyd on {{ $labels.instance }} is at stratum {{ $value }}, not 1

alert: PiaHealthcheckNotRunning
expr: time()
  - node_systemd_timer_last_trigger_seconds{name="pia-healthcheck.timer"}
  > 1800
for: 10m
labels:
  severity: warning
annotations:
  description: The PIA self-repair loop is not running
  summary: pia-healthcheck.timer on {{ $labels.instance }} has not fired for 30 minutes

alert: PiaPortForwardDown
expr: node_systemd_unit_state{name="pia-portforward.service",state="active"}
  == 0 and on (instance) node_systemd_unit_state{name="pia-tunnel.service",state="active"}
  == 1
for: 30m
labels:
  severity: warning
annotations:
  description: The tunnel is up but the forwarded port is not being kept alive; transmission
    is unreachable from outside
  summary: pia-portforward.service on {{ $labels.instance }} is not active

alert: PiaRepairNotWorking
expr: node_systemd_unit_state{name="pia-healthcheck.service",state="failed"}
  == 1
for: 30m
labels:
  severity: critical
annotations:
  description: pia-repair is running but not fixing the tunnel; see journalctl -u
    pia-healthcheck -u pia-repair
  summary: PIA health check on {{ $labels.instance }} has been failing for 30 minutes

alert: PiaTunnelDown
expr: node_systemd_unit_state{name="pia-tunnel.service",state="active"}
  == 0
for: 20m
labels:
  severity: critical
annotations:
  description: The PIA WireGuard tunnel (tun0) has been down for 30 minutes and pia-repair
    has not brought it back
  summary: pia-tunnel.service on {{ $labels.instance }} is not active

alert: PostgresDown
expr: pg_up == 0
for: 5m
labels:
  severity: warning
annotations:
  description: Home Assistant history is not being recorded. pg_isready -h 198.19.48.7;
    sudo journalctl -u postgresql@18-main and /var/log/postgresql/postgresql-18-main.log
  summary: Postgres on {{ $labels.instance }} is refusing connections

alert: ScrapeDown
expr: up == 0
for: 30m
labels:
  severity: warning
annotations:
  description: scrape failures for {{ $labels.instance }} via {{ $labels.job }}
  summary: '{{ $labels.instance }} is down'

alert: DiskTemperatureHigh
expr: smartmon_temperature_celsius_raw_value
  * on (instance, disk) group_left (device_model, serial_number) smartmon_device_info
  > 60
for: 1h
labels:
  severity: warning
annotations:
  description: Above 60°C for an hour; heat causes read errors and shortens drive
    life. Check airflow around the USB enclosures
  summary: '{{ $labels.device_model }} {{ $labels.serial_number }} on {{ $labels.instance
    }} ({{ $labels.disk }}) is at {{ $value | printf "%.0f" }}°C'

alert: SmartFailing
expr: smartmon_device_smart_healthy
  * on (instance, disk) group_left (device_model, serial_number) smartmon_device_info
  == 0
for: 15m
labels:
  severity: critical
annotations:
  description: The drive predicts its own failure; replace it. smartctl -H {{ $labels.disk
    }}
  summary: '{{ $labels.device_model }} {{ $labels.serial_number }} on {{ $labels.instance
    }} ({{ $labels.disk }}) fails its SMART self-assessment'

alert: SmartPendingSectorsIncreased
expr: (last_over_time(smartmon_current_pending_sector_raw_value[1h])
  - min_over_time(smartmon_current_pending_sector_raw_value[1d])) * on (instance,
  disk) group_left (device_model, serial_number) smartmon_device_info > 0
for: 15m
labels:
  severity: warning
annotations:
  description: Unreadable sectors waiting to be rewritten or remapped (SMART 197);
    see the Disk health (SMART) dashboard and smartctl -a {{ $labels.disk }}
  summary: '{{ $labels.device_model }} {{ $labels.serial_number }} on {{ $labels.instance
    }} ({{ $labels.disk }}) has {{ $value | printf "%.0f" }} more pending
    sectors than 24h ago'

alert: SmartReallocatedSectorsIncreased
expr: (last_over_time(smartmon_reallocated_sector_ct_raw_value[1h])
  - min_over_time(smartmon_reallocated_sector_ct_raw_value[1d])) * on (instance, disk)
  group_left (device_model, serial_number) smartmon_device_info > 0
for: 15m
labels:
  severity: warning
annotations:
  description: The drive remapped sectors to spares (SMART 5); see the Disk health
    (SMART) dashboard and smartctl -a {{ $labels.disk }}
  summary: '{{ $labels.device_model }} {{ $labels.serial_number }} on {{ $labels.instance
    }} ({{ $labels.disk }}) reallocated {{ $value | printf "%.0f" }} more
    sectors in 24h'

alert: SmartmonCollectorStale
expr: time()
  - node_textfile_mtime_seconds{file=~".*/smartmon.prom"} > 3600
for: 10m
labels:
  severity: warning
annotations:
  description: Disk health alerts are blind; check systemctl status prometheus-node-exporter-smartmon.timer
  summary: SMART collector on {{ $labels.instance }} has not updated for an hour

alert: SnapraidSyncStale
expr: time()
  - snapraid_sync_last_success_timestamp_seconds{instance="love"} > 5 *
  86400 or absent(snapraid_sync_last_success_timestamp_seconds{instance="love"})
for: 1h
labels:
  severity: warning
annotations:
  description: Parity is going stale. snapraid-sync.timer runs it Mon and Fri; see
    systemctl list-timers snapraid-sync.timer and journalctl -u snapraid-sync
  summary: snapraid has not completed a run on love for 5 days

alert: SslExpiresSoon
expr: envoy_server_days_until_first_cert_expiring
  < 14
for: 6h
labels:
  severity: warning
annotations:
  description: Check certbot cronjob
  summary: SSL certificate expiring in 2 weeks

alert: SubwoofersUnplugged
expr: (esphome_sensor_value{id="subwoofer_power"}
  and on (node) esphome_switch_value{id="subwoofer"} == 1) < 7
for: 5m
labels:
  severity: critical
annotations:
  summary: Subwoofers on but idle power draw is too low!

alert: SunpowerPanelDown
expr: sunpower_pvs_inverter_ac_power_watts
  < 2
for: 1d
labels:
  severity: warning
annotations:
  description: '{{ $labels.device_type }} with id {{ $labels.device_id }} has been
    offline for 1 day.'
  summary: '{{ $labels.device_type }} offline'

alert: SunpowerSupervisorDown
expr: sunpower_pvs_device_state{device_type!="Inverter"}
  == 0
for: 30m
labels:
  severity: critical
annotations:
  description: '{{ $labels.device_type }} with id {{ $labels.device_id }} has been
    offline for 30 minutes'
  summary: '{{ $labels.device_type }} offline'

alert: NetworkdDown
expr: node_systemd_unit_state{instance="love",name="systemd-networkd.service",state="active"}
  == 0
for: 15m
labels:
  severity: critical
annotations:
  description: networkd manages the bond, VLANs and the 44net, PIA and HE tunnels
  summary: systemd-networkd on {{ $labels.instance }} is not active

alert: SystemdUnitFailed
expr: node_systemd_unit_state{instance=~"love|them",state="failed"}
  == 1
for: 30m
labels:
  severity: warning
annotations:
  description: systemctl status {{ $labels.name }}
  summary: '{{ $labels.name }} on {{ $labels.instance }} has been failed for 30 minutes'

alert: ThemCpuBusy
expr: 1
  - avg by (instance) (rate(node_cpu_seconds_total{instance="them",mode="idle"}[10m]))
  > 0.9
for: 30m
labels:
  severity: warning
annotations:
  description: Check Frigate (driveway detects on the 2560x1440 main stream), immich
    and HA on them
  summary: them CPU is {{ $value | humanizePercentage }} busy for 30 minutes

alert: ThemCpuPressure
expr: rate(node_pressure_cpu_waiting_seconds_total{instance="them"}[10m])
  > 0.5
for: 30m
labels:
  severity: warning
annotations:
  summary: Tasks on them are waiting for CPU {{ $value | humanizePercentage }} of
    the time

alert: ThemIoPressure
expr: rate(node_pressure_io_stalled_seconds_total{instance="them"}[10m])
  > 0.5
for: 30m
labels:
  severity: warning
annotations:
  description: Frigate recording, immich jobs or a failing disk; check node_disk_*
    for them
  summary: them is stalled on disk IO {{ $value | humanizePercentage }} of the time

alert: ThemMemoryLow
expr: node_memory_MemAvailable_bytes{instance="them"}
  / node_memory_MemTotal_bytes{instance="them"} < 0.05
for: 15m
labels:
  severity: warning
annotations:
  summary: them has only {{ $value | humanizePercentage }} of memory available

alert: ThemMemoryPressure
expr: rate(node_pressure_memory_stalled_seconds_total{instance="them"}[10m])
  > 0.1
for: 15m
labels:
  severity: warning
annotations:
  summary: them is stalled on memory {{ $value | humanizePercentage }} of the time
    (thrashing)

alert: TransmissionDown
expr: probe_success{job="blackbox-tcp",service="transmission"}
  == 0
for: 30m
labels:
  severity: warning
annotations:
  description: TCP connect to {{ $labels.instance }} (RPC port) has failed for 20
    minutes; check systemctl status transmission-daemon
  summary: transmission-daemon on love is not accepting connections

alert: TunnelbrokerUpdateFailing
expr: tunnelbroker_update_success{instance="love"}
  == 0 or time() - node_textfile_mtime_seconds{file=~".*/tunnelbroker.prom",instance="love"}
  > 1800
for: 30m
labels:
  severity: warning
annotations:
  description: he-ipv6 breaks if the WAN IPv4 changed and HE was not told. See journalctl
    -u tunnelbroker-update
  summary: HE tunnelbroker endpoint update is failing or not running

alert: WireguardCollectorStale
expr: time()
  - node_systemd_timer_last_trigger_seconds{name="prometheus-node-exporter-wireguard.timer"}
  > 1800
for: 10m
labels:
  severity: warning
annotations:
  description: Handshake alerts are blind until prometheus-node-exporter-wireguard.timer
    runs again
  summary: WireGuard collector on {{ $labels.instance }} has not run for 30 minutes

alert: WireguardHandshakeStale
expr: time()
  - max by (instance, interface) (wireguard_latest_handshake_seconds) > 900
for: 15m
labels:
  severity: critical
annotations:
  description: Last handshake was {{ $value | humanizeDuration }} ago; the tunnel
    is not passing traffic. Check wg show {{ $labels.interface }}
  summary: WireGuard tunnel {{ $labels.interface }} on {{ $labels.instance }} has
    no recent handshake

alert: UpsRuntimeLow
expr: nut_battery_runtime_seconds
  < 900
for: 2m
labels:
  severity: critical
annotations:
  description: UPS battery low!
  summary: '{{ $labels.instance }} UPS has <15 minutes of battery remaining'