diff --git a/gitlab/build-linux b/gitlab/build-linux
index 545601bc..4a8c793b 100755
--- a/gitlab/build-linux
+++ b/gitlab/build-linux
@@ -12,10 +12,13 @@ rm -fr build
 mkdir build
 cd build
 export LD_LIBRARY_PATH=/usr/local/lib
-../configure --enable-demos DXCOMPILER_LIBS="-L/usr/local/lib" CFLAGS="-g -O2 -Werror" CPPFLAGS="-DVKD3D_SHADER_UNSUPPORTED_DXIL" && \
-    make -j$(nproc) && \
+if ../configure --enable-demos DXCOMPILER_LIBS="-L/usr/local/lib" CFLAGS="-g -O2 -Werror" CPPFLAGS="-DVKD3D_SHADER_UNSUPPORTED_DXIL" && \
+        make -j$(nproc) ; then
     make -j$(nproc) check || \
-        touch ../pipeline_failed
+        touch ../tests_failed
+else
+    touch ../build_failed
+fi
 
 mkdir -p ../artifacts/$COMMIT
 rsync -Rr config.log doc/* test-suite.log tests/*.log tests/*/*.log ../artifacts/$COMMIT || true
diff --git a/gitlab/build-mac b/gitlab/build-mac
index 6a795d02..27c70d31 100755
--- a/gitlab/build-mac
+++ b/gitlab/build-mac
@@ -11,10 +11,13 @@ set -Eeuxo pipefail
 rm -fr build
 mkdir build
 cd build
-../configure CFLAGS="-g -O2 -Wno-implicit-fallthrough -Wno-enum-conversion -Werror" && \
-    make -j$(sysctl -n hw.ncpu) && \
+if ../configure CFLAGS="-g -O2 -Wno-implicit-fallthrough -Wno-enum-conversion -Werror" && \
+        make -j$(sysctl -n hw.ncpu) ; then
     make -j$(sysctl -n hw.ncpu) check || \
-        touch ../pipeline_failed
+        touch ../tests_failed
+else
+    touch ../build_failed
+fi
 
 mkdir -p ../artifacts/$COMMIT
 rsync -Rr config.log test-suite.log tests/*.log tests/*/*.log ../artifacts/$COMMIT || true
diff --git a/gitlab/build.yml b/gitlab/build.yml
index fd2b63ff..9695a304 100644
--- a/gitlab/build.yml
+++ b/gitlab/build.yml
@@ -18,7 +18,8 @@
     - cat /proc/meminfo > artifacts/meminfo.txt
     - vulkaninfo > artifacts/vulkaninfo.txt
     - git rebase $CI_MERGE_REQUEST_DIFF_BASE_SHA --exec ./gitlab/build-linux
-    - if [ -f pipeline_failed ] ; then exit 1 ; fi
+    - if [ -f build_failed ] ; then exit 1 ; fi
+    - if [ -f tests_failed ] ; then exit 2 ; fi
   artifacts:
     when: always
     paths:
@@ -34,7 +35,9 @@ build-radv-64:
 
 build-llvmpipe-64:
   extends: .build-linux
-  allow_failure: true
+  allow_failure:
+    exit_codes:
+      - 2
   variables:
     VK_LOADER_DRIVERS_SELECT: 'lvp_*'
     VKD3D_SHADER_CONFIG: 'force_validation'
@@ -50,7 +53,9 @@ build-radv-32:
 
 build-llvmpipe-32:
   extends: .build-linux
-  allow_failure: true
+  allow_failure:
+    exit_codes:
+      - 2
   variables:
     VK_LOADER_DRIVERS_SELECT: 'lvp_*'
     CC: 'gcc -m32'
@@ -121,7 +126,9 @@ build-mac:
   stage: build
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
-  allow_failure: true
+  allow_failure:
+    exit_codes:
+      - 2
   interruptible: true
   needs: []
   dependencies: []
@@ -149,7 +156,8 @@ build-mac:
     - cd ../..
     - export PATH="$PWD/wine/build/tools/widl:$PATH"
     - git rebase $CI_MERGE_REQUEST_DIFF_BASE_SHA --exec ./gitlab/build-mac
-    - if [ -f pipeline_failed ] ; then exit 1 ; fi
+    - if [ -f build_failed ] ; then exit 1 ; fi
+    - if [ -f tests_failed ] ; then exit 2 ; fi
   variables:
     VKD3D_DISABLE_EXTENSIONS: "VK_EXT_descriptor_indexing"
     VKD3D_SHADER_CONFIG: 'force_validation'