gcc package fixes, maven pre-caching, less docker builds (#13)

Justin Boswell · web-flow · commit aa2d4c69dfb9 · 2020-02-26T16:21:59.000-08:00
* Fixed gcc compiler ending up in packages

* Cut the default timeouts way down on maven operations to hopefully speed up retries on ARM hosts with shitty networking

* ARM targets will pre-cache maven dependencies

* Pull builder from cloudfront, don't rebuild images on builder changes

* Updated README
diff --git a/.github/docker-images/al2-arm64/Dockerfile b/.github/docker-images/al2-arm64/Dockerfile
@@ -18,6 +18,7 @@ RUN yum -y update \
     cmake3 \
     gcc \
     gcc-c++ \
+    maven \
     && yum clean all \
     && rm -rf /var/cache/yum \
     && ln -s /usr/bin/cmake3 /usr/bin/cmake \
@@ -43,9 +44,17 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Maven cache/settings
+###############################################################################
+RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/maven/maven-al2-arm64.tar.gz -o maven.tar.gz \
+    && mkdir -p /root/.m2 \
+    && tar xzf maven.tar.gz -C /root/.m2 \
+    && rm -f /tmp/maven.tar.gz
+ADD maven_settings.xml /root/.m2/settings.xml
+
+###############################################################################
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/al2-arm64/maven_settings.xml b/.github/docker-images/al2-arm64/maven_settings.xml
@@ -0,0 +1,15 @@
+<settings>
+    <servers>
+        <server>
+            <id>central</id>
+            <configuration>
+                <httpConfiguration>
+                <all>
+                    <connectionTimeout>10000</connectionTimeout>
+                    <readTimeout>30000</readTimeout>
+                </all>
+                </httpConfiguration>
+            </configuration>
+        </server>
+    </servers>
+</settings>
diff --git a/.github/docker-images/al2-x64/Dockerfile b/.github/docker-images/al2-x64/Dockerfile
@@ -43,9 +43,8 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/alpine-armv6/Dockerfile b/.github/docker-images/alpine-armv6/Dockerfile
@@ -12,6 +12,7 @@ RUN apk --no-cache add \
     cmake \
     clang \
     perl \
+    maven \
     linux-headers
 
 ###############################################################################
@@ -32,9 +33,13 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Maven settings
+###############################################################################
+ADD maven_settings.xml /root/.m2/settings.xml
+
+###############################################################################
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/alpine-armv6/maven_settings.xml b/.github/docker-images/alpine-armv6/maven_settings.xml
@@ -0,0 +1,15 @@
+<settings>
+    <servers>
+        <server>
+            <id>central</id>
+            <configuration>
+                <httpConfiguration>
+                <all>
+                    <connectionTimeout>10000</connectionTimeout>
+                    <readTimeout>30000</readTimeout>
+                </all>
+                </httpConfiguration>
+            </configuration>
+        </server>
+    </servers>
+</settings>
diff --git a/.github/docker-images/alpine-armv7/Dockerfile b/.github/docker-images/alpine-armv7/Dockerfile
@@ -12,6 +12,7 @@ RUN apk --no-cache add \
     cmake \
     clang \
     perl \
+    maven \
     linux-headers
 
 ###############################################################################
@@ -32,9 +33,13 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Maven settings
+###############################################################################
+ADD maven_settings.xml /root/.m2/settings.xml
+
+###############################################################################
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/alpine-armv7/maven_settings.xml b/.github/docker-images/alpine-armv7/maven_settings.xml
@@ -0,0 +1,15 @@
+<settings>
+    <servers>
+        <server>
+            <id>central</id>
+            <configuration>
+                <httpConfiguration>
+                <all>
+                    <connectionTimeout>10000</connectionTimeout>
+                    <readTimeout>30000</readTimeout>
+                </all>
+                </httpConfiguration>
+            </configuration>
+        </server>
+    </servers>
+</settings>
diff --git a/.github/docker-images/build_cmake.sh b/.github/docker-images/build_cmake.sh
@@ -15,7 +15,7 @@ CMAKE_VERSION=3.13.5
 # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be in env vars to pass to container
 [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ]
 
-if [ ! -e /tmp/aws-crt-${variant}-${arch}-${version}.tar ]; then
+if [ ! -e /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz ]; then
     aws s3 cp s3://aws-crt-builder/_docker/aws-crt-${variant}-${arch}-${version}.tar.gz /tmp
     docker load < /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz
 fi
diff --git a/.github/docker-images/build_libcrypto_1.0.2.sh b/.github/docker-images/build_libcrypto_1.0.2.sh
@@ -13,7 +13,7 @@ libcrypto_version=1.0.2
 # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be in env vars to pass to container
 [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ]
 
-if [ ! -e /tmp/aws-crt-${variant}-${arch}-${version}.tar ]; then
+if [ ! -e /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz ]; then
     aws s3 cp s3://aws-crt-builder/_docker/aws-crt-${variant}-${arch}-${version}.tar.gz /tmp
     docker load < /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz
 fi
diff --git a/.github/docker-images/build_libcrypto_1.1.1.sh b/.github/docker-images/build_libcrypto_1.1.1.sh
@@ -13,7 +13,7 @@ libcrypto_version=1.1.1
 # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be in env vars to pass to container
 [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ]
 
-if [ ! -e /tmp/aws-crt-${variant}-${arch}-${version}.tar ]; then
+if [ ! -e /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz ]; then
     aws s3 cp s3://aws-crt-builder/_docker/aws-crt-${variant}-${arch}-${version}.tar.gz /tmp
     docker load < /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz
 fi
diff --git a/.github/docker-images/cache_maven.sh b/.github/docker-images/cache_maven.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This clones aws-crt-java in the specified container, builds it with maven, then caches maven's respository to S3
+
+set -ex
+
+[ $# -ge 3 ]
+variant=$1
+arch=$2
+version=$3
+
+crt_java_branch=master
+if [ $# -gt 3 ]; then
+    crt_java_branch=$4
+fi
+
+# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be in env vars to pass to container
+[ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ]
+
+if [ ! -e /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz ]; then
+    aws s3 cp s3://aws-crt-builder/_docker/aws-crt-${variant}-${arch}-${version}.tar.gz /tmp
+    docker load < /tmp/aws-crt-${variant}-${arch}-${version}.tar.gz
+fi
+
+container=$(docker run -dit --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint /bin/sh docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${variant}-${arch}:${version})
+docker exec ${container} sh -c "cd /tmp && builder build -p aws-crt-java --branch=${crt_java_branch} || true"
+docker exec ${container} sh -c "tar cvzf /tmp/maven-${variant}-${arch}.tar.gz -C /root/.m2 ."
+docker exec ${container} sh -c "aws s3 cp /tmp/maven-${variant}-${arch}.tar.gz s3://aws-crt-builder/_binaries/maven/maven-${variant}-${arch}.tar.gz"
+docker stop ${container}
diff --git a/.github/docker-images/entrypoint.sh b/.github/docker-images/entrypoint.sh
@@ -1,10 +1,21 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -ex
 
+args=($*)
+
+version=LATEST
+if [ "${args[0]}" == "--version=*" ]; then
+    version=${args[@]:1}
+    version=$(echo $version | cut -f2 -d=)
+fi
+
+# download the version of builder requested
+curl -sSL -o /usr/local/bin/builder.pyz --retry 3 --retry-delay 3 --retry-max-time 30 https://d19elf31gohf1l.cloudfront.net/${version}/builder
 builder=/usr/local/bin/builder.pyz
 chmod a+x $builder
-# on manylinux, use the latest python3
+
+# on manylinux, use the latest python3 via symlink
 if [ -x /opt/python/cp37-cp37m/bin/python ]; then
     ln -s /opt/python/cp37-cp37m/bin/python /usr/local/bin/python3
 fi
diff --git a/.github/docker-images/manylinux1-x64/Dockerfile b/.github/docker-images/manylinux1-x64/Dockerfile
@@ -38,9 +38,8 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/manylinux1-x86/Dockerfile b/.github/docker-images/manylinux1-x86/Dockerfile
@@ -38,9 +38,8 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/manylinux2014-x64/Dockerfile b/.github/docker-images/manylinux2014-x64/Dockerfile
@@ -32,9 +32,8 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/manylinux2014-x86/Dockerfile b/.github/docker-images/manylinux2014-x86/Dockerfile
@@ -37,9 +37,8 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/raspbian-armv6/Dockerfile b/.github/docker-images/raspbian-armv6/Dockerfile
@@ -16,6 +16,7 @@ RUN apt-get update \
     python3-pip \
     build-essential \
     cmake \
+    maven \
     # For PPAs
     software-properties-common \
     apt-transport-https	\
@@ -39,9 +40,17 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Maven cache/settings
+###############################################################################
+RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/maven/maven-raspbian-armv6.tar.gz -o maven.tar.gz \
+    && mkdir -p /root/.m2 \
+    && tar xzf maven.tar.gz -C /root/.m2 \
+    && rm -f /tmp/maven.tar.gz
+ADD maven_settings.xml /root/.m2/settings.xml
+
+###############################################################################
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/raspbian-armv6/maven_settings.xml b/.github/docker-images/raspbian-armv6/maven_settings.xml
@@ -0,0 +1,15 @@
+<settings>
+    <servers>
+        <server>
+            <id>central</id>
+            <configuration>
+                <httpConfiguration>
+                <all>
+                    <connectionTimeout>10000</connectionTimeout>
+                    <readTimeout>30000</readTimeout>
+                </all>
+                </httpConfiguration>
+            </configuration>
+        </server>
+    </servers>
+</settings>
diff --git a/.github/docker-images/raspbian-armv7/Dockerfile b/.github/docker-images/raspbian-armv7/Dockerfile
@@ -16,6 +16,7 @@ RUN apt-get update \
     python3-pip \
     build-essential \
     cmake \
+    maven \
     # For PPAs
     software-properties-common \
     apt-transport-https	\
@@ -39,9 +40,17 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Maven cache/settings
+###############################################################################
+RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/maven/maven-raspbian-armv7.tar.gz -o maven.tar.gz \
+    && mkdir -p /root/.m2 \
+    && tar xzf maven.tar.gz -C /root/.m2 \
+    && rm -f /tmp/maven.tar.gz
+ADD maven_settings.xml /root/.m2/settings.xml
+
+###############################################################################
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/docker-images/raspbian-armv7/maven_settings.xml b/.github/docker-images/raspbian-armv7/maven_settings.xml
@@ -0,0 +1,15 @@
+<settings>
+    <servers>
+        <server>
+            <id>central</id>
+            <configuration>
+                <httpConfiguration>
+                <all>
+                    <connectionTimeout>10000</connectionTimeout>
+                    <readTimeout>30000</readTimeout>
+                </all>
+                </httpConfiguration>
+            </configuration>
+        </server>
+    </servers>
+</settings>
diff --git a/.github/docker-images/ubuntu-16-x64/Dockerfile b/.github/docker-images/ubuntu-16-x64/Dockerfile
@@ -38,9 +38,8 @@ RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/libcrypto/libcrypt
     && rm -f /tmp/libcrypto.tar.gz
 
 ###############################################################################
-# Install builder package
+# Install entrypoint
 ###############################################################################
-ADD builder /usr/local/bin/builder.pyz
 ADD entrypoint.sh /usr/local/bin/builder
-RUN chmod a+x /usr/local/bin/builder.pyz /usr/local/bin/builder
+RUN chmod a+x /usr/local/bin/builder
 ENTRYPOINT ["/usr/local/bin/builder"]
diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
diff --git a/README.md b/README.md
diff --git a/builder/data.py b/builder/data.py
