Fixes for swift (#116)

* Only export compiler if CC/CXX are not specified

* Reduce what's installed on the swift centos docker image

* Added support for differentiating centos from manylinux

* Trimmed down swift docker images

Author: Justin Boswell

.github/docker-images/swift-5-al2-x64/Dockerfile

@@ -5,24 +5,15 @@ FROM swift:5.4-amazonlinux2
 # Install prereqs
 # any prereqs that appear to be missing are installed on base swift image i.e. tar, git
 ###############################################################################
-RUN yum -y update \
-    && yum -y install \
+RUN yum -y install \
     curl \
     sudo \
     # Python
     python3 \
-    python3-devel \
     python3-pip \
-    make \
-    cmake3 \
-    gcc-c++ \
     openssl-devel \
     && yum clean all \
-    && rm -rf /var/cache/yum \
-    && ln -s /usr/bin/cmake3 /usr/bin/cmake \
-    && ln -s /usr/bin/ctest3 /usr/bin/ctest \
-    && cmake --version \
-    && ctest --version
+    && rm -rf /var/cache/yum
 
 ###############################################################################
 # Python/AWS CLI
@@ -31,28 +22,9 @@ RUN python3 -m pip install --upgrade pip setuptools virtualenv \
     && python3 -m pip install --upgrade awscli \
     && aws --version
 
-###############################################################################
-# Install pre-built OpenSSL
-###############################################################################
-WORKDIR /tmp
-RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypto-1.1.1-linux-x64.tar.gz -o libcrypto.tar.gz \
-    && mkdir /opt/openssl \
-    && tar xzf libcrypto.tar.gz -C /opt/openssl \
-    && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/openssl/lib /opt/openssl/bin/openssl version \
-    && rm -f /tmp/libcrypto.tar.gz
-
-###############################################################################
-# Install pre-built CMake
-###############################################################################
-WORKDIR /tmp
-RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/cmake/cmake-3.13-manylinux1-x64.tar.gz -o cmake.tar.gz \
-    && tar xvzf cmake.tar.gz -C /usr/local \
-    && cmake --version \
-    && rm -f /tmp/cmake.tar.gz
-
 ###############################################################################
 # Install entrypoint
 ###############################################################################
 ADD entrypoint.sh /usr/local/bin/builder
 RUN chmod a+x /usr/local/bin/builder
-ENTRYPOINT ["/usr/local/bin/builder"]
+ENTRYPOINT ["/usr/local/bin/builder"]

.github/docker-images/swift-5-centos-x64/Dockerfile

@@ -5,39 +5,15 @@ FROM swift:5.4-centos7
 # Install prereqs
 # any prereqs that appear to be missing are installed on base swift image i.e. tar, git
 ###############################################################################
-RUN yum -y update \
-    && yum -y install \
+RUN yum -y install \
     sudo \
     # Python
     python3 \
-    python3-devel \
     python3-pip \
-    make \
-    cmake \
-    cmake3 \
     openssl-devel \
-    gcc-c++ \
     && yum clean all \
-    && rm -rf /var/cache/yum 
-###############################################################################
-# Handle cmake on centos properly
-###############################################################################
-RUN sudo alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake 10 \
---slave /usr/local/bin/ctest ctest /usr/bin/ctest \
---slave /usr/local/bin/cpack cpack /usr/bin/cpack \
---slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake \
---family cmake
+    && rm -rf /var/cache/yum
 
-RUN sudo alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 \
---slave /usr/local/bin/ctest ctest /usr/bin/ctest3 \
---slave /usr/local/bin/cpack cpack /usr/bin/cpack3 \
---slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \
---family cmake
-
-RUN ln -s /usr/bin/cmake3 /usr/bin/cmake \
-    ln -s /usr/bin/ctest3 /usr/bin/ctest \
-    cmake --version \
-    && ctest --version
 
 ###############################################################################
 # Python/AWS CLI
@@ -46,20 +22,9 @@ RUN python3 -m pip install --upgrade pip setuptools virtualenv \
     && python3 -m pip install --upgrade awscli \
     && aws --version
 
-###############################################################################
-# Install pre-built OpenSSL
-###############################################################################
-WORKDIR /tmp
-RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypto-1.1.1-linux-x64.tar.gz -o libcrypto.tar.gz \
-    && mkdir /opt/openssl \
-    && tar xzf libcrypto.tar.gz -C /opt/openssl \
-    && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/openssl/lib /opt/openssl/bin/openssl version \
-    && rm -f /tmp/libcrypto.tar.gz
-
-
 ###############################################################################
 # Install entrypoint
 ###############################################################################
 ADD entrypoint.sh /usr/local/bin/builder
 RUN chmod a+x /usr/local/bin/builder
-ENTRYPOINT ["/usr/local/bin/builder"]
+ENTRYPOINT ["/usr/local/bin/builder"]

.github/docker-images/swift-5-ubuntu-16-x64/Dockerfile

@@ -11,9 +11,7 @@ RUN apt-get update -qq \
     curl \
     # Python
     python3 \
-    python3-dev \
     python3-pip \
-    build-essential \
     # For PPAs
     libssl-dev \
     software-properties-common \
@@ -28,15 +26,6 @@ RUN python3 -m pip install setuptools \
     && python3 -m pip install --upgrade awscli \
     && aws --version
 
-###############################################################################
-# Install pre-built CMake
-###############################################################################
-WORKDIR /tmp
-RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/cmake/cmake-3.13-manylinux1-x64.tar.gz -o cmake.tar.gz \
-    && tar xvzf cmake.tar.gz -C /usr/local \
-    && cmake --version \
-    && rm -f /tmp/cmake.tar.gz
-
 ###############################################################################
 # Install entrypoint
 ###############################################################################

builder/actions/install.py

@@ -99,16 +99,19 @@ def export_compiler(compiler, env):
         return
 
     toolchain = env.toolchain
-    cc_path = toolchain.compiler_path()
-    cxx_path = toolchain.cxx_compiler_path()
-    if cc_path:
-        env.shell.setenv('CC', cc_path)
-    else:
-        print('WARNING: C compiler {} could not be found for export'.format(compiler))
-    if cxx_path:
-        env.shell.setenv('CXX', cc_path)
-    else:
-        print('WARNING: CXX compiler {} could not be found for export'.format(compiler))
+    if not env.shell.getenv('CC'):
+        cc_path = toolchain.compiler_path()
+        if cc_path:
+            env.shell.setenv('CC', cc_path)
+        else:
+            print('WARNING: C compiler {} could not be found for export'.format(compiler))
+
+    if not env.shell.getenv('CXX'):
+        cxx_path = toolchain.cxx_compiler_path()
+        if cxx_path:
+            env.shell.setenv('CXX', cc_path)
+        else:
+            print('WARNING: CXX compiler {} could not be found for export'.format(compiler))
 
 
 class InstallCompiler(Action):

builder/core/data.py

@@ -197,6 +197,17 @@ class PKG_TOOLS(Enum):
             'python': "/opt/python/cp39-cp39/bin/python",
         },
     },
+    'centos': {
+        'os': 'linux',
+        'pkg_tool': PKG_TOOLS.YUM,
+        'pkg_update': 'yum update -y',
+        'pkg_install': 'yum install -y',
+        'sudo': False,
+
+        'variables': {
+            'python': 'python3'
+        }
+    },
     'windows': {
         'os': 'windows',
         'variables': {

builder/core/host.py

@@ -7,6 +7,8 @@
 import re
 import sys
 
+from functools import lru_cache
+
 
 def current_os():
     if sys.platform == 'win32':
@@ -61,15 +63,9 @@ def _file_contains(path, search):
     return False
 
 
-# cache the result of this, since it involves a bunch of file I/O
-_current_host = None
-
-
+@lru_cache(1)
 def current_host():
     """ Between sys.platform or linux distro identifiers, determine the specific os """
-    global _current_host
-    if _current_host:
-        return _current_host
 
     def _discover_host():
         platform = current_os()
@@ -79,9 +75,13 @@ def _discover_host():
             if _file_contains('/etc/system-release', 'Bare Metal') or _file_contains('/etc/system-release', 'Amazon Linux AMI'):
                 return 'al2012'
             if _file_contains('/etc/redhat-release', 'CentOS release 5.'):
-                return 'manylinux'
+                if os.path.exists('/opt/python/cp27-cp27m'):
+                    return 'manylinux'
+                return 'centos'
             if _file_contains('/etc/redhat-release', 'CentOS Linux release 7.'):
-                return 'manylinux'
+                if os.path.exists('/opt/python/cp39-cp39'):
+                    return 'manylinux'
+                return 'centos'
             if _file_contains('/etc/lsb-release', 'Ubuntu'):
                 return 'ubuntu'
             if _file_contains('/etc/os-release', 'Debian'):
@@ -93,8 +93,7 @@ def _discover_host():
             return 'linux'
         else:
             return platform
-    _current_host = _discover_host()
-    return _current_host
+    return _discover_host()
 
 
 def package_tool(host=current_host()):

builder/core/shell.py

@@ -82,6 +82,13 @@ def setenv(self, var, value, **kwargs):
         if not self.dryrun:
             os.environ[var] = value
 
+    def getenv(self, var, default=None):
+        """ Get an environment variable """
+        try:
+            return os.environ[var]
+        except:
+            return default
+
     def addpathenv(self, var, path, **kwargs):
         """Add a path to an environment variable"""
         prev = os.getenv(var)
@@ -91,10 +98,6 @@ def addpathenv(self, var, path, **kwargs):
             value = path
         self.setenv(var, value, **kwargs)
 
-    def getenv(self, var):
-        """ Get an environment variable """
-        return os.environ[var]
-
     def pushenv(self, **kwargs):
         """ Store the current environment on a stack, for restoration later """
         if not kwargs.get('quiet', False):