FreeBSD support and config compilation (#15)

* Added support for FreeBSD

* Added darwin -> macos aliases

* Fleshed out arch/os aliases, normalize targets always

* Added aliases for FreeBSD/OSX uname -m arch

* Try harder to divine branch when github lies to us

* Always normalize toolchain platform

* Install docker in ubuntu image for cross compile

* Make sanity tests check for platform normalization via armv8

* Make sure compiler installation happens at package installation time

* Fixed where path shenanigans

* Finally got the courage to just recursively process config trees

* Resolve symlinks when looking up executables

Author: Justin Boswell

.github/actions/release-tag/index.js

@@ -6,7 +6,8 @@ try {
     const parts = github_ref.split('/');
     const branch = parts[parts.length - 1];
     var release_tag = branch;
-    if (branch == 'master') {
+    // GITHUB_REF can be refs/pull/<pr#>/merge for PR openings
+    if (branch == 'master' || branch == 'merge') {
         const spawnSync = require('child_process').spawnSync;
         const result = spawnSync('git', ['describe', '--abbrev=0'], {
             timeout: 2000

.github/docker-images/ubuntu-16-x64/Dockerfile

@@ -20,6 +20,15 @@ RUN apt-get update -qq \
     apt-transport-https \
     && apt-get clean
 
+##############################################################################
+# Docker (for cross compiling via dockcross)
+###############################################################################
+WORKDIR /tmp
+RUN set -ex && curl -fsSL https://get.docker.com -o /tmp/get-docker.sh \
+    && sh /tmp/get-docker.sh \
+    && systemctl enable docker \
+    && echo "service docker start" >> ~/.bashrc
+
 ###############################################################################
 # Python/AWS CLI
 ###############################################################################

.github/workflows/docker-images.yml

@@ -6,6 +6,7 @@ on:
       - '*'
       - '!master'
     paths:
+      - '.github/actions/release-tag/*'
       - '.github/docker-images/**/*'
       - '.github/docker-images/entrypoint.sh'
       - '.github/workflows/docker-images.yml'

.github/workflows/sanity-test.yml

@@ -144,7 +144,7 @@ jobs:
     needs: package
     strategy:
       matrix:
-        arch: [linux-armv6, linux-armv7, linux-arm64]
+        arch: [linux-armv6, linux-armv7, linux-armv8]
 
     steps:
     - name: Checkout Source

README.md

@@ -25,9 +25,17 @@ Usage: ```builder.pyz [build|inspect|<action-name>] [spec] [OPTIONS]```
 * ```--config CONFIG``` - CMake config to use (Debug, Release, RelWithDebInfo, DebugOpt) Default is RelWithDebInfo
 * ```--compiler COMPILER[-VERSION]``` - Use the specified compiler, installing it if necessary
 * ```--platform PLATFORM``` - Platform to cross-compile for (via dockcross, requires docker to be installed)
+  * Valid values are anything that you could get from `uname`.lower() - `uname -m`, e.g. linux-x86_64, linux-x64. See targets below.
 * ```--build-dir DIR``` - Make a new directory to do all the build work in, instead of using the current directory
 * ```--dump-config``` - Dumps the resultant config after merging all available options. Useful for debugging your project configuration.
 
+
+### Supported Targets:
+* linux: x86|i686, x64|x86_64, armv6, armv7, arm64|armv8|aarch64|arm64v8
+* macos|darwin: x64|x86_64
+* windows: x86, x64
+* freebsd: x64
+
 ### Example build
 ```builder.pyz build --project=aws-c-common downstream```
 

builder/__main__.py

@@ -28,7 +28,7 @@
 from project import Project
 from scripts import Scripts
 from toolchain import Toolchain
-from host import current_os, current_host, current_arch, current_platform
+from host import current_os, current_host, current_arch, current_platform, normalize_target
 import data
 
 import api  # force API to load and expose the virtual module
@@ -113,7 +113,7 @@ def inspect_host(env):
     toolchain = Toolchain(spec=spec)
     print('Host Environment:')
     print('  Host: {} {}'.format(spec.host, spec.arch))
-    print('  Default Target: {} {}'.format(spec.target, spec.arch))
+    print('  Target: {} {}'.format(spec.target, spec.arch))
     compiler_path = toolchain.compiler_path()
     if not compiler_path:
         compiler_path = '(Will Install)'
@@ -183,11 +183,15 @@ def parse_args():
         args.command = args.spec
         args.spec = None
 
+    # normalize target
+    if args.target:
+        args.target = normalize_target(args.target)
+
     if args.spec:
         spec = BuildSpec(spec=args.spec, target=args.target)
 
     if args.compiler or args.target:
-        compiler, version = (None, None)
+        compiler, version = ('default', 'default')
         if args.compiler:
             compiler, version = args.compiler.split('-')
         spec = str(spec) if spec else None

builder/actions/install.py

@@ -22,6 +22,7 @@
 from host import current_os, package_tool
 from actions.script import Script
 from toolchain import Toolchain
+from util import list_unique
 
 
 class InstallPackages(Action):
@@ -39,6 +40,7 @@ def run(self, env):
         if not packages:
             return
 
+        packages = list_unique(packages)
         pkg_tool = package_tool()
         print('Installing packages via {}: {}'.format(
             pkg_tool.value, ', '.join(packages)))
@@ -58,7 +60,7 @@ def run(self, env):
         if not InstallPackages.pkg_init_done:
             pkg_setup = config.get('pkg_setup', [])
             if pkg_setup:
-                for cmd in pkg_setup:
+                for cmd in list_unique(pkg_setup):
                     if isinstance(cmd, str):
                         cmd = cmd.split(' ')
                     assert isinstance(cmd, list)
@@ -95,16 +97,21 @@ def run(self, env):
         assert toolchain
 
         # Cross compile with dockcross
-        if toolchain.cross_compile:
+        def _install_cross_compile_toolchain(env):
             print(
                 'Installing cross-compile via dockcross for {}'.format(toolchain.platform))
+            cross_compile_platform = env.config.get(
+                'cross_compile_platform', toolchain.platform)
             result = sh.exec(
-                'docker', 'run', 'dockcross/{}'.format(toolchain.platform), quiet=True)
-            assert result.returncode == 0
+                'docker', 'run', 'dockcross/{}'.format(cross_compile_platform), quiet=True)
             # Strip off any output from docker itself
-            script = '#!' + result.output.partition('#!')[2]
+            output, shebang, script = result.output.partition('#!')
+            script = shebang + script
+            print(output)
+            assert result.returncode == 0
+
             dockcross = os.path.abspath(os.path.join(
-                env.build_dir, 'dockcross-{}'.format(toolchain.platform)))
+                env.build_dir, 'dockcross-{}'.format(cross_compile_platform)))
             Path(dockcross).touch(0o755)
             with open(dockcross, "w+t") as f:
                 f.write(script)
@@ -118,22 +125,8 @@ def run(self, env):
             toolchain.env_file = dockcross_env
             toolchain.shell_env = [
                 dockcross, '-a', '--env-file={}'.format(dockcross_env)]
-            return
-
-        # Compiler is local, or should be, so verify/install and export it
-        compiler = env.spec.compiler
-        version = env.spec.compiler_version
-        if version == 'default':
-            version = None
-
-        # See if the compiler is already installed
-        compiler_path, found_version = Toolchain.find_compiler(
-            compiler, version)
-        if compiler_path:
-            print('Compiler {} {} is already installed ({})'.format(
-                compiler, version, compiler_path))
-            return
 
+        # Expose compiler via environment
         def _export_compiler(_env):
             if current_os() == 'windows':
                 return
@@ -149,5 +142,23 @@ def _export_compiler(_env):
                             print(
                                 'WARNING: Compiler {} could not be found'.format(exe))
 
-        packages = config['compiler_packages']
-        return Script([InstallPackages(packages), _export_compiler])
+        if not toolchain.cross_compile:
+            # Compiler is local, or should be, so verify/install and export it
+            compiler = env.spec.compiler
+            version = env.spec.compiler_version
+            if version == 'default':
+                version = None
+
+            # See if the compiler is already installed
+            compiler_path, found_version = Toolchain.find_compiler(
+                compiler, version)
+            if compiler_path:
+                print('Compiler {} {} is already installed ({})'.format(
+                    compiler, version, compiler_path))
+                return
+
+        packages = list_unique(config.get('compiler_packages', []))
+        after_packages = [_export_compiler]
+        if toolchain.cross_compile:
+            after_packages = [_install_cross_compile_toolchain]
+        return Script([InstallPackages(packages), *after_packages])