Alerts

/nix/store/l9dnyxl5i5bpk9ab642w8nlr8d12mxc4-rules-checkrules-checked > hydra
BuildsStuckOverTwoDays (0 active) 
alert: BuildsStuckOverTwoDays
expr: hydra_machine_build_duration_bucket{le="259200"}
  - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0
for: 30m
labels:
  severity: page
annotations:
  summary: https://status.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap
/nix/store/l9dnyxl5i5bpk9ab642w8nlr8d12mxc4-rules-checkrules-checked > scheduled-jobs
ChannelUpdateStuck (0 active) 
alert: ChannelUpdateStuck
expr: sum
  without(state) (node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"activating|failed"})
  == 1
for: 30m
labels:
  severity: page
annotations:
  summary: https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s
HomepageUpdateStuck (0 active) 
alert: HomepageUpdateStuck
expr: sum
  without(state) (node_systemd_unit_state{name=~"^update-homepage.*.service$",state=~"activating|failed"})
  == 1
for: 30m
labels:
  severity: page
annotations:
  summary: https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s
RFC39MaintainerSync (0 active) 
alert: RFC39MaintainerSync
expr: node_systemd_unit_state{name=~"^rfc39-sync.service$",state="failed"}
  == 1
for: 30m
labels:
  severity: page
annotations:
  summary: https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s
/nix/store/l9dnyxl5i5bpk9ab642w8nlr8d12mxc4-rules-checkrules-checked > system
RootPartitionLowDiskSpace (1 active) 
alert: RootPartitionLowDiskSpace
expr: node_filesystem_avail_bytes{mountpoint="/"}
  <= 1e+10
for: 30m
labels:
  severity: page
annotations:
  summary: https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{
    $labels.instance }}
Labels State Active Since Value
alertname="RootPartitionLowDiskSpace" device="/dev/disk/by-label/nixos" fstype="ext4" instance="bastion:9100" job="node" mountpoint="/" role="bastion" severity="page" firing 2020-01-22 03:41:49.428787443 +0000 UTC 4.504100864e+09
RootPartitionLowInodes (0 active) 
alert: RootPartitionLowInodes
expr: node_filesystem_files_free{mountpoint="/"}
  <= 10000
for: 30m
labels:
  severity: page
annotations:
  summary: https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{
    $labels.instance }}