diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index 5b521b59..b2d1531d 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -16,6 +16,7 @@ permissions:
 jobs:
   compatibility:
     runs-on: ubuntu-latest
+    container: returntocorp/ocaml:alpine
     steps:
     - name: Checkout tree
       uses: actions/checkout@v4
@@ -23,22 +24,22 @@ jobs:
         fetch-depth: 0
         fetch-tags: true
 
-    - name: Set-up OCaml
-      uses: ocaml/setup-ocaml@v2
-      with:
-        ocaml-compiler: 5.1
-        opam-pin: false
-        opam-depext: false
-        dune-cache: true
-
-    - name: install atddiff
-      run: opam install atd
-
     - name: atddiff all supported tags
       id: diff
       shell: bash
       run: |
-        eval $(opam env)
+        set -x
+
+        git config --global --add safe.directory "$(pwd)"
+        
+        # github actions sets HOME=/home/github where we don't have an opam env
+        eval $(HOME=/root opam env)
+        apk add jq
+
+        # check / print version of atddiff
+        atddiff --version
+
+        # run the checks
         echo -ne 'Backwards compatability summary:\n\n```' > summary-00-header.txt
         echo '```' >> summary-20-footer.txt
         ./scripts/check-backwards-compatability.sh | tee summary-10-body.txt
@@ -59,4 +60,4 @@ jobs:
       run: |
         pip install mypy
         rm __init__.py # because dir has a - in it
-        mypy semgrep_output_v1.py
\ No newline at end of file
+        mypy semgrep_output_v1.py
diff --git a/scripts/check-backwards-compatability.sh b/scripts/check-backwards-compatability.sh
index 77cee745..1e525ebe 100755
--- a/scripts/check-backwards-compatability.sh
+++ b/scripts/check-backwards-compatability.sh
@@ -10,13 +10,12 @@
 #    - Diff against origin/main to establish a baseline
 #    - Diff against HEAD
 #    - Diff the two diffs to see if new issues were introduced
-
-set -u # no "-eo pipefail" because we do our own error handling
+set -euo pipefail
 
 minimum="v$(curl -s https://semgrep.dev/api/check-version | jq -r '.versions.minimum')"
 tags=$(git log --simplify-by-decoration --pretty=format:%D "${minimum}^!" origin/main | grep -o 'tag: [^,)]\+' | sed 's/^tag: //' | sort -n)
 
-checked=()
+checked=("dummy")
 errors=0
 for tag in $tags; do
     commit=$(git rev-list -n 1 "$tag")
@@ -26,19 +25,31 @@ for tag in $tags; do
     fi
     checked+=("$commit")
 
+    set +e # do our own error handling for a bit
     echo "Checking backward compatibility of semgrep_output_v1.atd against past version $tag"
-    git difftool -x 'atddiff --backward' -y "$tag" "origin/main" semgrep_output_v1.atd > before.txt
-    git difftool -x 'atddiff --backward' -y "$tag" "HEAD" semgrep_output_v1.atd > after.txt
+    git difftool --trust-exit-code -x 'atddiff --no-locations --backward' -y "$tag" "origin/main" semgrep_output_v1.atd > before.txt
+    ret=$?
+    if [ "$ret" -ge 1 ] && [ "$ret" -le 2 ]; then
+        echo "ERROR: atddiff had an error: $?"
+        cat before.txt
+        exit 1
+    fi
+    git difftool --trust-exit-code -x 'atddiff --no-locations --backward' -y "$tag" "HEAD" semgrep_output_v1.atd > after.txt
+    ret=$?
+    if [ "$ret" -ge 1 ] && [ "$ret" -le 2 ]; then
+        echo "ERROR: atddiff had an error: $?"
+        cat after.txt
+        exit 1
+    fi
 
-    # neccesary because filenames have temp paths and line numbers can change without causing issues
-    expr='s|File "/.*/\(.*.atd\)", line .*$|File "\1", line <removed for diff>|g'
-    diff -u <(sed "$expr" before.txt) <(sed "$expr" after.txt)
+    diff -u <(cat before.txt) <(cat after.txt)
     if [ "$?" -ne 0 ]; then
         echo "ERROR: semgrep_output_v1.atd is not backward compatible with $tag"
         errors=$((errors + 1))
     fi
+    set -e
 done
 
 if [ "$errors" -ne 0 ]; then
     exit 1
-fi
\ No newline at end of file
+fi