Merge pull request #514 from S2E/issue/513-nativecompiler

Use system compilers instead of custom ones to build s2e

Author: vitalych

pylint_rc

@@ -9,7 +9,8 @@
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS,TraceEntries_pb2.py
+ignore=CVS
+ignore-patterns=.*_pb2\.py
 
 # Pickle collected data for later comparisons.
 persistent=yes

pyproject.toml

@@ -0,0 +1,81 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "s2e-env"
+description = "A command-line tool for administering S2E environments"
+authors = [
+    {name = "Adrian Herrera", email = "adrian.herrera@epfl.ch"}
+]
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.6"
+classifiers = [
+    "Intended Audience :: Developers",
+    "Environment :: Console",
+    "Programming Language :: Python",
+    "Topic :: Security",
+    "Topic :: Software Development",
+    "Topic :: System",
+]
+dependencies = [
+    # S2E engine requirements
+    "docutils==0.22.2",
+    "pygments==2.19.2",
+    # s2e-env requirements
+    "distro==1.9.0",
+    "enum34==1.1.10",
+    "jinja2==3.1.6",
+    "pefile==2019.4.18",
+    "psutil==7.1.0",
+    "pyelftools @ git+https://github.com/S2E/pyelftools.git#egg=pyelftools-0.24+s2e",
+    "python-magic==0.4.27",
+    "pyyaml==6.0.2",
+    "requests==2.32.5",
+    "sh==2.2.2",
+    "termcolor==3.1.0",
+    "pytrie==0.4.0",
+    "pwntools==4.3.1",
+    "protobuf3-to-dict==0.1.5",
+    "immutables==0.21",
+    "alive_progress==3.3.0",
+    "protobuf==3.20.2",
+    # Dependencies for symchk
+    # TODO: replace with official release when available
+    # "pdbparse @ git+https://github.com/faratech/pdbparse.git@5eeff2bc11a5063f130216f929706631e8302be2",
+    "pefile==2019.4.18",
+    "construct==2.10.70",
+    "patool==1.12",
+    "pyunpack==0.2.2",
+    # FTP server used by image creation
+    "pyftpdlib==2.0.1"
+]
+
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "http://s2e.systems"
+Repository = "https://github.com/S2E/s2e-env.git"
+
+[project.optional-dependencies]
+test = ["pytest"]
+
+[project.scripts]
+s2e = "s2e_env.manage:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+# This will automatically find all packages
+
+[tool.setuptools.package-data]
+s2e_env = [
+    "templates/*",
+    "templates/plugin_creation/*",
+    "dat/*",
+]
+
+[tool.setuptools.dynamic]
+version = {file = "s2e_env/dat/VERSION"}

s2e_env/commands/build.py

@@ -29,12 +29,25 @@
 import sh
 from sh import ErrorReturnCode
 
+from s2e_env import CONSTANTS
 from s2e_env.command import EnvCommand, CommandError
-
+from s2e_env.utils.host import get_os_version
 
 logger = logging.getLogger('build')
 
 
+def _get_clang_version():
+    default = CONSTANTS["clang_versions"]["default"]
+    os_name, os_version = get_os_version()
+    version = CONSTANTS["clang_versions"].get(f"{os_name}-{os_version}", None)
+    if version:
+        logger.info('Using clang-%s to build S2E', version)
+        return version
+
+    logger.info('Using clang-%s to build S2E', default)
+    return default
+
+
 class Command(EnvCommand):
     """
     Builds S2E.
@@ -60,20 +73,53 @@ def add_arguments(self, parser):
                             help='List of S2E components to clean prior to '
                                  'the build process')
 
-    def handle(self, *args, **options):
+    def _build_tools(self, build_dir):
+        makefile = self.env_path('source', 's2e', 'Makefile.tools')
+        if not os.path.isfile(makefile):
+            raise CommandError(f'{makefile} not found')
+
+        try:
+            logger.info('Generating S2E build environment Docker image')
+            docker = sh.Command('docker')
+
+            docker(
+                "build",
+                "--target", "s2e-build-env",
+                "-t", "s2e-build-env",
+                ".",
+                _out=sys.stdout,
+                _err=sys.stderr,
+                _cwd=self.env_path('source', 's2e')
+            )
+
+            logger.info('Building S2E tools in %s', build_dir)
+            docker(
+                "run", "-t", "--rm",
+                "-e", "SYSTEM_CLANG_VERSION=15",
+                "-e", f"S2E_PREFIX={self.install_path()}",
+                "-w", build_dir,
+                "-v", f"{self.env_path()}:{self.env_path()}",
+                "s2e-build-env",
+                "/run_as.sh", os.getuid(), os.getgid(),
+                "make", "-f", makefile, "install",
+                _out=sys.stdout,
+                _err=sys.stderr
+            )
+        except ErrorReturnCode as e:
+            raise CommandError(e) from e
+
+        logger.success('S2E tools built')
+
+    def _build_s2e(self, build_dir, **options):
         # Exit if the makefile doesn't exist
         makefile = self.env_path('source', 'Makefile')
         if not os.path.isfile(makefile):
-            raise CommandError(f'No makefile found in {os.path.dirname(makefile)}')
-
-        # If the build directory doesn't exist, create it
-        build_dir = self.env_path('build')
-        if not os.path.isdir(build_dir):
-            os.mkdir(build_dir)
+            raise CommandError(f'{makefile} not found')
 
         # Set up some environment variables
         env_vars = os.environ.copy()
         env_vars['S2E_PREFIX'] = self.install_path()
+        env_vars['SYSTEM_CLANG_VERSION'] = f"{_get_clang_version()}"
 
         components = options['components']
         self._make = sh.Command('make').bake(directory=build_dir, file=makefile, _env=env_vars)
@@ -96,6 +142,15 @@ def handle(self, *args, **options):
 
         logger.success('S2E built')
 
+    def handle(self, *args, **options):
+        # If the build directory doesn't exist, create it
+        build_dir = self.env_path('build')
+        if not os.path.isdir(build_dir):
+            os.mkdir(build_dir)
+
+        self._build_tools(build_dir)
+        self._build_s2e(build_dir, **options)
+
     def _get_components(self, target):
         lines = self._make(target).strip().split('\n')
         print(lines)

s2e_env/commands/code_coverage/lcov.py

@@ -24,6 +24,7 @@
 
 import logging
 import os
+import re
 import sys
 
 # pylint: disable=no-name-in-module
@@ -93,7 +94,17 @@ def _get_addr_coverage(directory, aggregated_coverage):
     return aggregated_coverage
 
 
-def _save_coverage_info(lcov_path, file_line_info, ignore_missing_files):
+def _is_excluded(file_path, excluded_patterns):
+    if not excluded_patterns:
+        return False
+
+    if any(re.search(pattern, file_path) for pattern in excluded_patterns):
+        return True
+
+    return False
+
+
+def _save_coverage_info(lcov_path, file_line_info, ignore_missing_files, excluded_patterns):
     """
     Save the line coverage information in lcov format.
 
@@ -121,6 +132,10 @@ def _save_coverage_info(lcov_path, file_line_info, ignore_missing_files):
             num_non_zero_lines = 0
             num_instrumented_lines = 0
 
+            if _is_excluded(src_file, excluded_patterns):
+                logger.info('Excluding %s from coverage report', src_file)
+                continue
+
             f.write(f'SF:{src_file}\n')
             for line, count in file_line_info[src_file].items():
                 f.write(f'DA:{line},{count}\n')
@@ -177,14 +192,15 @@ def handle(self, *args, **options):
         for module_path, addr_counts in coverage.items():
             try:
                 cov = options.get('include_covered_files_only', False)
+                excluded_patterns = options.get('exclude_pattern', [])
                 file_line_info = syms.get_coverage(module_path, addr_counts, cov)
 
                 module = os.path.basename(module_path)
 
                 # genhtml will throw an error if there are missing files, so we skip them
                 # if the user enabled html reports.
                 lcov_info_path = os.path.join(lcov_out_dir, module + '.info')
-                _save_coverage_info(lcov_info_path, file_line_info, do_gen_html)
+                _save_coverage_info(lcov_info_path, file_line_info, do_gen_html, excluded_patterns)
 
                 logger.success('Line coverage saved to %s', lcov_info_path)
 

s2e_env/commands/coverage.py

@@ -53,6 +53,10 @@ def add_arguments(self, parser):
                                  help='S2E output directories to consider (default is s2e-last).',
                                  required=False)
 
+        lcov_parser.add_argument('--exclude-pattern', action='append',
+                                 help='Exclude the given pattern from the report. Can be specified multiple times.',
+                                 required=False)
+
         lcov_parser.add_argument('--aggregate-outputs', type=int,
                                  help='Specifies n most recent s2e-out-* folders to aggregate for coverage.',
                                  required=False)

s2e_env/commands/deps.py

@@ -0,0 +1,12 @@
+from s2e_env.command import BaseCommand
+from s2e_env.commands.init import install_dependencies
+
+class Command(BaseCommand):
+    """
+    Installs required system packages to build and run S2E.
+    """
+
+    help = 'Install required system packages for S2E'
+
+    def handle(self, *args, **options):
+        install_dependencies()

s2e_env/commands/init.py

@@ -101,7 +101,7 @@ def _compute_dependencies():
     return all_install_packages
 
 
-def _install_dependencies():
+def install_dependencies():
     """
     Install S2E's dependencies.
 
@@ -122,7 +122,7 @@ def _install_dependencies():
             install_packages.append(package)
 
     install_opts = ['--no-install-recommends']
-    env = {}
+    env = os.environ.copy()
 
     env['DEBIAN_FRONTEND'] = 'noninteractive'
     install_opts = ['-y'] + install_opts
@@ -285,7 +285,8 @@ def add_arguments(self, parser):
         parser.add_argument('dir', nargs='?', default=os.getcwd(),
                             help='The environment directory. Defaults to the '
                                  'current working directory')
-        parser.add_argument('-s', '--skip-dependencies', action='store_true',
+        parser.add_argument('--install-dependencies', action='store_true',
+                            required=False, default=True,
                             help='Skip the dependency install via apt')
         parser.add_argument('-b', '--use-existing-install', required=False,
                             default=None,
@@ -341,8 +342,8 @@ def handle(self, *args, **options):
                 _link_existing_install(env_path, existing_install_path)
             else:
                 # Install S2E's dependencies via apt-get
-                if not options['skip_dependencies']:
-                    _install_dependencies()
+                if options['install_dependencies']:
+                    install_dependencies()
 
                 # Get the source repositories
                 _get_s2e_sources(env_path, options['manifest_branch'])

s2e_env/commands/new_project.py

@@ -56,10 +56,10 @@
 CGC_REGEX = re.compile(r'^CGC 32-bit')
 ELF32_REGEX = re.compile(r'^ELF 32-bit')
 ELF64_REGEX = re.compile(r'^ELF 64-bit')
-DLL32_REGEX = re.compile(r'^PE32 executable \(DLL\)')
-DLL64_REGEX = re.compile(r'^PE32\+ executable \(DLL\)')
-WIN32_DRIVER_REGEX = re.compile(r'^PE32 executable \(native\)')
-WIN64_DRIVER_REGEX = re.compile(r'^PE32\+ executable \(native\)')
+DLL32_REGEX = re.compile(r'^PE32 executable.*?\(DLL\)')
+DLL64_REGEX = re.compile(r'^PE32\+ executable.*?\(DLL\)')
+WIN32_DRIVER_REGEX = re.compile(r'^PE32 executable.*?\(native\)')
+WIN64_DRIVER_REGEX = re.compile(r'^PE32\+ executable.*?\(native\)')
 PE32_REGEX = re.compile(r'^PE32 executable')
 PE64_REGEX = re.compile(r'^PE32\+ executable')
 MSDOS_REGEX = re.compile(r'^MS-DOS executable')