Fix apt-get reliability issues.

This is frequently causing the core build scripts to fail. The core ubuntu
distribution will perform an auto-update at first start, which may cause the
lock file to be held. All apt-get commands may be done in a loop in order to
retry to avoid this issue. We may want to consider retrying other pieces, but
for now this should avoid the most frequent cause of build flakes.

PiperOrigin-RevId: 297704789

scripts/build.sh

@@ -17,7 +17,7 @@
 source $(dirname $0)/common.sh
 
 # Install required packages for make_repository.sh et al.
-sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils xz-utils
+apt_install dpkg-sig coreutils apt-utils xz-utils
 
 # Build runsc.
 runsc=$(build -c opt //runsc)

scripts/common.sh

@@ -84,3 +84,17 @@ function install_runsc() {
   # Restart docker to pick up the new runtime configuration.
   sudo systemctl restart docker
 }
+
+# Installs the given packages. Note that the package names should be verified to
+# be correct, otherwise this may result in a loop that spins until time out.
+function apt_install() {
+  while true; do
+    if (sudo apt-get update && sudo apt-get install -y "$@"); then
+      break
+    fi
+    result=$?
+    if [[ $result -ne 100 ]]; then
+      return $result
+    fi
+  done
+}

tools/images/ubuntu1604/10_core.sh

@@ -17,7 +17,20 @@
 set -xeo pipefail
 
 # Install all essential build tools.
-apt-get update && apt-get -y install make git-core build-essential linux-headers-$(uname -r) pkg-config
+while true; do
+  if (apt-get update && apt-get install -y \
+      make \
+      git-core \
+      build-essential \
+      linux-headers-$(uname -r) \
+      pkg-config); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Install a recent go toolchain.
 if ! [[ -d /usr/local/go ]]; then

tools/images/ubuntu1604/20_bazel.sh

@@ -19,7 +19,17 @@ set -xeo pipefail
 declare -r BAZEL_VERSION=2.0.0
 
 # Install bazel dependencies.
-apt-get update && apt-get install -y openjdk-8-jdk-headless unzip
+while true; do
+  if (apt-get update && apt-get install -y \
+      openjdk-8-jdk-headless \
+      unzip); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Use the release installer.
 curl -L -o bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh

tools/images/ubuntu1604/25_docker.sh

@@ -15,12 +15,20 @@
 # limitations under the License.
 
 # Add dependencies.
-apt-get update && apt-get -y install \
-    apt-transport-https \
-    ca-certificates \
-    curl \
-    gnupg-agent \
-    software-properties-common
+while true; do
+  if (apt-get update && apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg-agent \
+      software-properties-common); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Install the key.
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
@@ -32,4 +40,15 @@ add-apt-repository \
    stable"
 
 # Install docker.
-apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io
+while true; do
+  if (apt-get update && apt-get install -y \
+      docker-ce \
+      docker-ce-cli \
+      containerd.io); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done

tools/images/ubuntu1604/30_containerd.sh

@@ -34,7 +34,17 @@ install_helper() {
 }
 
 # Install dependencies for the crictl tests.
-apt-get install -y btrfs-tools libseccomp-dev
+while true; do
+  if (apt-get update && apt-get install -y \
+      btrfs-tools \
+      libseccomp-dev); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # Install containerd & cri-tools.
 GOPATH=$(mktemp -d --tmpdir gopathXXXXX)

tools/images/ubuntu1604/40_kokoro.sh

@@ -23,7 +23,22 @@ declare -r ssh_public_keys=(
 )
 
 # Install dependencies.
-apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip python3-pip zip
+while true; do
+  if (apt-get update && apt-get install -y \
+      rsync \
+      coreutils \
+      python-psutil \
+      qemu-kvm \
+      python-pip \
+      python3-pip \
+      zip); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
 
 # junitparser is used to merge junit xml files.
 pip install junitparser

tools/installers/master.sh

@@ -19,17 +19,16 @@ set -e
 
 curl -fsSL https://gvisor.dev/archive.key | sudo apt-key add -
 add-apt-repository "deb https://storage.googleapis.com/gvisor/releases release main"
+
 while true; do
-  if apt-get update; then
-    apt-get install -y runsc
+  if (apt-get update && apt-get install -y runsc); then
     break
   fi
   result=$?
-  # Check if apt update failed to aquire the file lock.
   if [[ $result -ne 100 ]]; then
     exit $result
   fi
 done
+
 runsc install
 service docker restart
-