Alerts


/nix/store/8dp0mv5a58w6bvjpsirxqf5ppw4mxwhh-rules-checkrules-checked > hydra
BuildsStuckOverTwoDays (0 active) 
alert: BuildsStuckOverTwoDays
expr: hydra_machine_build_duration_bucket{le="259200"} - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0
for: 30m
labels:
  severity: page
annotations:
  summary: https://monitoring.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap
/nix/store/8dp0mv5a58w6bvjpsirxqf5ppw4mxwhh-rules-checkrules-checked > scheduled-jobs
RFC39MaintainerSync (2 active) 
alert: RFC39MaintainerSync
expr: node_systemd_unit_state{name=~"^rfc39-sync.service$",state="failed"} == 1
for: 30m
labels:
  severity: page
annotations:
  summary: https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s
Labels State Active Since Value
alertname="RFC39MaintainerSync" instance="eris:9100" job="node" name="rfc39-sync.service" role="unknown" severity="page" state="failed" type="oneshot" pending 2021-01-23 07:05:44.682940684 +0000 UTC 1
alertname="RFC39MaintainerSync" facility="ewr1" instance="6f3edfd6.packethost.net" job="packet_nodes" name="rfc39-sync.service" packet_device_id="6f3edfd6-2ab0-4cd3-a0b0-e5c151762efc" packet_device_state="failed" packet_facility="ewr1" packet_plan="baremetal_2a5" plan="baremetal_2a5" role="builder" severity="page" state="failed" type="oneshot" pending 2021-01-23 07:05:44.682940684 +0000 UTC 1
ChannelUpdateStuck (0 active) 
alert: ChannelUpdateStuck
expr: max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1
for: 30m
annotations:
  summary: https://monitoring.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s
/nix/store/8dp0mv5a58w6bvjpsirxqf5ppw4mxwhh-rules-checkrules-checked > system
RootPartitionLowDiskSpace (0 active) 
alert: RootPartitionLowDiskSpace
expr: node_filesystem_avail_bytes{mountpoint="/"} <= 1e+10
for: 30m
labels:
  severity: page
annotations:
  summary: https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}
RootPartitionLowInodes (0 active) 
alert: RootPartitionLowInodes
expr: node_filesystem_files_free{mountpoint="/"} <= 10000
for: 30m
labels:
  severity: page
annotations:
  summary: https://monitoring.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}