From 582bf0d72d96cd098527385eea4957d97def0bec Mon Sep 17 00:00:00 2001
From: Etienne Perot <eperot@google.com>
Date: Wed, 9 Aug 2023 18:18:57 -0700
Subject: [PATCH] Add disk usage monitoring to BuildKite script.

If disk usage of any filesystem approaches or reaches 100%, the agent
will log its disk usage to the console, and then reboot.

PiperOrigin-RevId: 555334222
---
 .buildkite/hooks/pre-command | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 1b939b32a..8ef8a2953 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -1,5 +1,13 @@
 set -euo pipefail
 
+if POSIXLY_CORRECT=true df --local --output=pcent,ipcent,target | grep -vE '/snap/' | grep -qE '9[4-9]%|100%'; then
+  echo "Disk usage has reached critical level, node is bad." >&2
+  sudo df -h --local | sed -r 's/^/[df] /' >&2
+  sleep 3
+  sudo poweroff
+  exit 1
+fi
+
 if [[ "${BUILDKITE_PIPELINE_NAME:-}" == "Pipeline" ]]; then
   # Check that the target branch exists.
   git fetch origin "${BUILDKITE_BRANCH}" || {