script to package grown test and run historical check

README.md

@@ -292,4 +292,28 @@ To run many instances in parallel (16):
 
 ```
 for i in `seq 1 16`; do reduce-new-kills work ${DREDD_EXPERIMENTS_ROOT}/llvm-${LLVM_VERSION}-mutated-build/bin/clang ${DREDD_EXPERIMENTS_ROOT}/csmith & done
+```
+
+# Package grown testsuite
+The following runner verifies that the reduced program is compilable with Clang/GCC under both `-O3` and `-O0` optimization levels. In the case of a miscompilation test case, it checks that the execution output of the binary, compiled by each compiler and optimization mode, produces the same result.
+
+```
+package-tests work ${DREDD_EXPERIMENTS_ROOT}/csmith
+```
+
+To run many instances in parallel (16):
+
+```
+for i in `seq 1 16`; do package-tests work ${DREDD_EXPERIMENTS_ROOT}/csmith & done
+```
+
+# Historical check
+
+Make sure the following package is installed:
+```
+sudo apt install gcc-multilib libncurses5
+```
+
+```
+historical-check work ${LLVM_VERSION} ${DREDD_EXPERIMENTS_ROOT}/csmith/
 ```

dredd_test_runners/historical_check/get_clang_llvm_releases.py

@@ -0,0 +1,99 @@
+import argparse
+import re
+import requests
+import urllib.parse
+
+from pathlib import Path
+from typing import List
+from packaging import version
+
+PRE_GITHUH_RELEASE_URLS = url_pre_github = [
+    "https://releases.llvm.org/7.0.1/clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-18.04.tar.xz",
+    "https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/6.0.0/clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/5.0.2/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/5.0.1/clang+llvm-5.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu16.04.tar.xz",
+    "https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.10.tar.xz",
+    "https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.9.0/clang+llvm-3.9.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.8.1/clang+llvm-3.8.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.7.1/clang+llvm-3.7.1-x86_64-linux-gnu-ubuntu-15.10.tar.xz",
+    "https://releases.llvm.org/3.7.0/clang+llvm-3.7.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.6.2/clang+llvm-3.6.2-x86_64-linux-gnu-ubuntu-15.04.tar.xz",
+    "https://releases.llvm.org/3.6.1/clang+llvm-3.6.1-x86_64-linux-gnu-ubuntu-15.04.tar.xz",
+    "https://releases.llvm.org/3.6.0/clang+llvm-3.6.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.5.2/clang+llvm-3.5.2-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.5.1/clang+llvm-3.5.1-x86_64-linux-gnu.tar.xz",
+    "https://releases.llvm.org/3.5.0/clang+llvm-3.5.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.4.2/clang+llvm-3.4.2-x86_64-linux-gnu-ubuntu-14.04.xz",
+    "https://releases.llvm.org/3.4.1/clang+llvm-3.4.1-x86_64-unknown-ubuntu12.04.tar.xz",
+    "https://releases.llvm.org/3.4/clang+llvm-3.4-x86_64-linux-gnu-ubuntu-13.10.tar.xz",
+    "https://releases.llvm.org/3.3/clang+llvm-3.3-Ubuntu-13.04-x86_64-linux-gnu.tar.bz2",
+    "https://releases.llvm.org/3.2/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz",
+    "https://releases.llvm.org/3.1/clang+llvm-3.1-x86_64-linux-ubuntu_12.04.tar.gz",
+    "https://releases.llvm.org/3.0/clang+llvm-3.0-x86_64-linux-Ubuntu-11_10.tar.gz",
+    "https://releases.llvm.org/2.9/clang+llvm-2.9-x86_64-linux.tar.bz2",
+    "https://releases.llvm.org/2.8/clang+llvm-2.8-x86_64-linux.tar.bz2",
+    "https://releases.llvm.org/2.7/clang+llvm-2.7-x86_64-linux.tar.bz2",
+    "https://releases.llvm.org/2.6/llvm+clang-2.6-x86_64-linux.tar.gz"
+]
+
+def get_clang_llvm_releases(after_version: str) -> List[str]:
+    result : List[str] = []
+
+    page = 0
+    ubuntu_release_pattern = r"^clang\+llvm-(\d+\.\d+\.\d+)-x86_64-linux-gnu-ubuntu-(\d+\.\d+)\.tar\.xz$"
+    while True:
+        response = requests.get(f'https://api.github.com/repos/llvm/llvm-project/releases?page={page}')
+        if len(response.json()) == 0:
+            break
+        if response.status_code != 200:
+            raise Exception(response.json()['message']) 
+
+        for release in response.json():
+            # Skip Pre-release version
+            if release['prerelease']:
+                continue
+
+            # Get release for latest ubuntu version
+            latest_ubuntu_release = ""
+            latest_ubuntu_version = ""
+            for asset in release['assets']:
+                url = urllib.parse.unquote(asset['browser_download_url'])
+                tar_file = url.split('/')[-1]
+                match = re.match(ubuntu_release_pattern, tar_file)
+                if not match:
+                    continue
+
+                # Version smaller than requested versions, return result
+                if version.parse(match.group(1)) <= version.parse(after_version):
+                    return list(reversed(result))
+
+                if latest_ubuntu_version == "" or version.parse(match.group(2)) > version.parse(latest_ubuntu_version):
+                    latest_ubuntu_version = match.group(2)
+                    latest_ubuntu_release = url
+            if latest_ubuntu_release != "":
+                result.append(latest_ubuntu_release)
+        page += 1
+
+    # Continue searching for releases from `PRE_GITHUH_RELEASE_URLS`
+    for release_url in PRE_GITHUH_RELEASE_URLS:
+        release_version = release_url.replace("https://releases.llvm.org/", '').split('/')[0]
+
+        if version.parse(release_version) <= version.parse(after_version):
+            break 
+        result.append(release_url)
+
+    return list(reversed(result))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("after_version", help="Get list of llvm release after this version", type=str)
+    args = parser.parse_args()
+    release_urls = get_clang_llvm_releases(args.after_version)
+    for url in release_urls:
+        print(url)

dredd_test_runners/historical_check/main.py

@@ -0,0 +1,128 @@
+import argparse
+import json
+import sys
+import tempfile
+import subprocess
+import os
+import re
+
+from functools import partial
+from pathlib import Path
+from typing import Dict
+from multiprocessing import Pool
+
+from dredd_test_runners.historical_check.get_clang_llvm_releases import get_clang_llvm_releases
+
+def check_compiler_with_test(csmith_root: Path, compiler_path: Path, test_dir_path: Path) -> bool:
+    # print(f"Checking {compiler_path} on {test_dir_path}")
+    compiler_args = ["-I", f"{csmith_root}/runtime", "-I", f"{csmith_root}/build/runtime", "-pedantic", "-Wall", "-fPIC"]
+
+    test_prog_path = test_dir_path / 'prog.c'
+    if not test_prog_path.exists():
+        print(f"{test_prog_path} doesn't exist")
+        return False
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        proc = subprocess.run([compiler_path, *compiler_args, "-O3", test_prog_path, "-c", "-o", "prog.o"], cwd=tmpdir, capture_output=True)
+        if proc.returncode != 0:
+            print(f'Compilation failed for {compiler_path} with testcase {test_dir_path}:')
+            print(proc.stderr.decode())
+            return False
+
+        reference_output_path = test_dir_path / 'prog.reference_output'
+        is_miscompilation_test = reference_output_path.exists()
+
+        if is_miscompilation_test:
+            proc = subprocess.run(['clang-15', *compiler_args, "prog.o", "-o", "prog.exe"], cwd=tmpdir, capture_output=True)
+            if proc.returncode != 0:
+                print(f'Linking failed for {compiler_path} with testcase {test_dir_path}:')
+                print(proc.stderr.decode())
+                return False
+
+            proc = subprocess.run(['./prog.exe'], cwd=tmpdir, capture_output=True)
+            if proc.returncode != 0:
+                print(f'Execution failed for {compiler_path} with testcase {test_dir_path}:')
+                print(proc.stderr.decode())
+                return False
+
+            with open(reference_output_path, 'rb') as f:
+                reference_output = f.read()
+            if proc.stdout != reference_output:
+                print(f'Comparison failed for {compiler_path} with testcase {test_dir_path}:')
+                return False
+
+            # print(f"Miscompilation check {compiler_path} on {test_dir_path} succeed")
+        # else:
+        #     print(f"Crash check {compiler_path} on {test_dir_path} succeed")
+
+        return True
+
+    print(compiler_path, test_dir_path)
+
+def check_version_with_testsuite(version_url: str, testsuite: Path, csmith_root: Path):
+    version_tar_name = version_url.split('/')[-1]
+    pattern = r'(\.tar\.gz|\.tar\.bz2|\.tar\.xz|\.gz|\.bz2|\.xz)$'
+    version_dir_name = re.sub(pattern, '', version_tar_name)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        proc = subprocess.run(["curl", "-Lo", version_tar_name, version_url], cwd=tmpdir, capture_output=True)
+        if proc.returncode != 0:
+            print(proc.stderr.decode())
+            return
+
+        version_path = Path(tmpdir) / version_dir_name
+        version_path.mkdir()
+
+        proc = subprocess.run(["tar", "-xf", version_tar_name, '-C', version_path, '--strip-components=1'], cwd=tmpdir, capture_output=True)
+        if proc.returncode != 0:
+            print(proc.stderr.decode())
+            return
+
+        compiler_path = version_path / "bin" / "clang"
+
+        if not compiler_path.exists():
+            print(f"Compiler path {compiler_path} doesn't exist.")
+            return 
+
+        test_dirs = testsuite.glob('*')
+        with Pool() as pool:
+            test_result = pool.map(partial(check_compiler_with_test, csmith_root, compiler_path), test_dirs)
+        print(f"RESULT OF {version_dir_name}: {sum(test_result)}/{len(test_result)}")
+        # for test_dir in test_dirs:
+        #     check_compiler_with_test(csmith_root, compiler_path, test_dir)
+
+    return
+
+
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("work_dir",
+                        help="Directory containing test results. It should have subdirectories, 'testsuite'.",
+                        type=Path)
+    parser.add_argument("version",
+                        help="The LLVM version for which the testsuite is being grown from (e.g., 14.0.0).",
+                        type=str)
+    parser.add_argument("csmith_root", help="Path to a checkout of Csmith, assuming that it has been built under "
+                                            "'build' beneath this directory.",
+                        type=Path)
+    args = parser.parse_args()
+
+    testsuite_dir = args.work_dir / "testsuite"
+    if not testsuite_dir.exists() or not testsuite_dir.is_dir():
+        print(f"Error: {str(testsuite_dir)} does not exist.")
+        sys.exit(1)
+
+    future_versions = get_clang_llvm_releases(args.version)
+
+    for version_url in future_versions:
+        check_version_with_testsuite(version_url, testsuite_dir, args.csmith_root)
+
+    pass
+
+if __name__ == '__main__':
+    main()
+
+
+