From 165d72efd2912987b700e5b98f6e04e3cca7c374 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <ryan.gonzalez@collabora.com>
Date: Fri, 21 Jan 2022 16:18:35 -0600
Subject: [PATCH] Add remaining portions of automated compatibility tests

This adds the rest of the compatibility test workflow, in order to
automatically run against the GNU findutils & BFS test suites as part of
standard CI and compare the results to the latest from the 'main'
branch.

Closes #128

Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com>
---
 .github/workflows/compat.yml | 72 ++++++++++++++++++++++++++++++++++++
 util/compare_bfs_result.py   | 25 +++++++++++++
 util/compare_gnu_result.py   | 29 +++++++++++++++
 3 files changed, 126 insertions(+)
 create mode 100644 util/compare_bfs_result.py
 create mode 100644 util/compare_gnu_result.py

diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml
index 962e5cc..124aec8 100644
--- a/.github/workflows/compat.yml
+++ b/.github/workflows/compat.yml
@@ -74,6 +74,42 @@ jobs:
       with:
         name: gnu-result
         path: gnu-result.json
+    - name: Download the result
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        workflow: compat.yml
+        name: gnu-result
+        repo: uutils/findutils
+        branch: main
+        path: dl
+    - name: Download the log
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        workflow: compat.yml
+        name: gnu-test-report
+        repo: uutils/findutils
+        branch: main
+        path: dl
+    - name: Compare failing tests against master
+      shell: bash
+      run: |
+            OLD_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" dl/test-suite.log | sort)
+            NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" findutils.gnu/tests/test-suite.log | sort)
+            for LINE in $OLD_FAILING; do
+              if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then
+                echo "::warning ::Congrats! The gnu test $LINE is now passing!"
+              fi
+            done
+            for LINE in $NEW_FAILING; do
+              if ! grep -Fxq $LINE<<<"$OLD_FAILING"; then
+                echo "::error ::gnu test failed: $LINE. $LINE is passing on 'main'. Maybe you have to rebase?"
+              fi
+            done
+    - name: Compare against main results
+      shell: bash
+      run: |
+        mv dl/gnu-result.json latest-gnu-result.json
+        python findutils/util/compare_gnu_result.py
 
   bfs-tests:
     name: Run BFS tests
@@ -139,3 +175,39 @@ jobs:
       with:
         name: bfs-result
         path: bfs-result.json
+    - name: Download the result
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        workflow: compat.yml
+        name: bfs-result
+        repo: uutils/findutils
+        branch: main
+        path: dl
+    - name: Download the log
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        workflow: compat.yml
+        name: bfs-test-report
+        repo: uutils/findutils
+        branch: main
+        path: dl
+    - name: Compare failing tests against main
+      shell: bash
+      run: |
+        OLD_FAILING=$(sed -n "s/^\([[:print:]]\+\) failed\!/\1/p" dl/tests.log | sort)
+        NEW_FAILING=$(sed -n "s/^\([[:print:]]\+\) failed\!/\1/p" bfs/tests.log | sort)
+        for LINE in $OLD_FAILING; do
+          if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then
+            echo "::warning ::Congrats! The bfs test $LINE is now passing!"
+          fi
+        done
+        for LINE in $NEW_FAILING; do
+          if ! grep -Fxq $LINE<<<"$OLD_FAILING"; then
+            echo "::error ::bfs test failed: $LINE. $LINE is passing on 'main'. Maybe you have to rebase?"
+          fi
+        done
+    - name: Compare against main results
+      shell: bash
+      run: |
+        mv dl/bfs-result.json latest-bfs-result.json
+        python findutils/util/compare_bfs_result.py
diff --git a/util/compare_bfs_result.py b/util/compare_bfs_result.py
new file mode 100644
index 0000000..43780e9
--- /dev/null
+++ b/util/compare_bfs_result.py
@@ -0,0 +1,25 @@
+#!/usr/bin/python
+"""
+Compare the current results to the last results gathered from the main branch to highlight
+if a PR is making the results better/worse
+"""
+
+import json
+import sys
+
+NEW = json.load(open("bfs-result.json"))
+OLD = json.load(open("latest-bfs-result.json"))
+
+# Extract the specific results from the dicts
+last = OLD[list(OLD.keys())[0]]
+current = NEW[list(NEW.keys())[0]]
+
+pass_d = int(current["pass"]) - int(last["pass"])
+fail_d = int(current["fail"]) - int(last["fail"])
+
+# Get an annotation to highlight changes
+print(f"::warning ::Changes from main: PASS {pass_d:+d} / FAIL {fail_d:+d}")
+
+# If results are worse fail the job to draw attention
+if pass_d < 0:
+    sys.exit(1)
diff --git a/util/compare_gnu_result.py b/util/compare_gnu_result.py
new file mode 100644
index 0000000..3f6fb8c
--- /dev/null
+++ b/util/compare_gnu_result.py
@@ -0,0 +1,29 @@
+#!/usr/bin/python
+"""
+Compare the current results to the last results gathered from the main branch to highlight
+if a PR is making the results better/worse
+"""
+
+import json
+import sys
+
+NEW = json.load(open("gnu-result.json"))
+OLD = json.load(open("latest-gnu-result.json"))
+
+# Extract the specific results from the dicts
+last = OLD[list(OLD.keys())[0]]
+current = NEW[list(NEW.keys())[0]]
+
+pass_d = int(current["pass"]) - int(last["pass"])
+fail_d = int(current["fail"]) - int(last["fail"])
+error_d = int(current["error"]) - int(last["error"])
+skip_d = int(current["skip"]) - int(last["skip"])
+
+# Get an annotation to highlight changes
+print(
+    f"::warning ::Changes from main: PASS {pass_d:+d} / FAIL {fail_d:+d} / ERROR {error_d:+d} / SKIP {skip_d:+d} "
+)
+
+# If results are worse fail the job to draw attention
+if pass_d < 0:
+    sys.exit(1)