diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f046b293f6..b4f21cf03b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -95,7 +95,7 @@ jobs:
 
           brew cask audit ${{ join(matrix.audit_args, ' ') }} '${{ matrix.cask.path }}'
         timeout-minutes: 30
-        if: always()
+        if: always() && !matrix.skip_audit
 
       - name: Gather cask information
         id: info
diff --git a/cmd/lib/generate-matrix.rb b/cmd/lib/generate-matrix.rb
index 3f69f5a223..829018ba2e 100755
--- a/cmd/lib/generate-matrix.rb
+++ b/cmd/lib/generate-matrix.rb
@@ -18,7 +18,12 @@ syntax_job = {
 matrix = [syntax_job]
 
 unless labels.include?("ci-syntax-only")
-  matrix += CiMatrix.generate(tap, labels: labels)
+  cask_jobs = CiMatrix.generate(tap, labels: labels)
+
+  # If casks were changed, skip `audit` for all others.
+  syntax_job[:skip_audit] = true if cask_jobs.any?
+
+  matrix += cask_jobs
 end
 
 puts JSON.pretty_generate(matrix)