diff --git a/tools/docker/README.md b/tools/docker/README.md index 384ce326aa37..f584323bcb29 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -16,7 +16,7 @@ docker run -it gcr.io/syzkaller/env To build and push a new version: ``` -docker build -t gcr.io/syzkaller/env tools/docker/env +DOCKER_BUILDKIT=1 docker build -t gcr.io/syzkaller/env tools/docker/env gcloud auth login && gcloud auth configure-docker docker push gcr.io/syzkaller/env ``` diff --git a/tools/docker/env/Dockerfile b/tools/docker/env/Dockerfile index ce993adbb2f2..e97c4656218c 100644 --- a/tools/docker/env/Dockerfile +++ b/tools/docker/env/Dockerfile @@ -3,6 +3,59 @@ # See /tools/docker/README.md for details. +# Build Python2 in a separate container to facilitate caching. +FROM debian:bookworm AS python2-builder + +RUN apt-get update --allow-releaseinfo-change +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -q --no-install-recommends \ + wget gcc make openssl libffi-dev libgdbm-dev libsqlite3-dev libssl-dev zlib1g-dev ca-certificates +RUN wget -O /tmp/Python-2.7.18.tgz 'https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tgz' +RUN cd /tmp/ && tar -zxf Python-2.7.18.tgz +RUN cd /tmp/Python-2.7.18 && ./configure --prefix=/python2/ +RUN cd /tmp/Python-2.7.18 && make -j4 && make altinstall + +# Construct a /syzkaller folder. +FROM debian:bookworm as syzkaller-folder +WORKDIR /syzkaller +RUN apt-get update --allow-releaseinfo-change +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl + +# Pre-create dirs for syz-dock. +# This is necessary to make docker work with the current user, +# otherwise --volume will create these dirs under root and then +# the current user won't have access to them. +RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \ + mkdir -p /syzkaller/.cache && \ + chmod -R 0777 /syzkaller + +# Install OS toolchains from pre-built archives. +# These archives were created with: +# tar -cz --owner=0 --group=0 --mode=go=u -f netbsd-toolchain.tar.gz netbsd/tools netbsd/dest +# tar -cz --owner=0 --group=0 --mode=go=u -f fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \ +# fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \ +# fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ +# fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \ +# fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ +# fuchsia/out/arm64/arm64-shared/*.so +# +# And then uploaded to GCS with: +# gsutil mv gs://syzkaller/GOOS-toolchain.tar.gz gs://syzkaller/GOOS-toolchain.old.tar.gz +# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/ +# gsutil acl ch -g all:R gs://syzkaller/GOOS-toolchain.tar.gz +# +# NetBSD toolchain can be re-built with: +# ./build.sh -j72 -m amd64 -U -T ../tools tools +# ./build.sh -j72 -m amd64 -U -T ../tools -D ../dest distribution +# +# To build root image run: +# docker run -it --rm --privileged --device /dev/loop0 gcr.io/syzkaller/env +# mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc +# create-image.sh -a s390x -d buster + +RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz +RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz + +# Now build the actual syz-env container. FROM debian:bookworm LABEL homepage="https://github.com/google/syzkaller" @@ -31,14 +84,7 @@ RUN curl https://dl.google.com/go/go1.22.7.linux-amd64.tar.gz | tar -C /usr/loca ENV PATH /usr/local/go/bin:/gopath/bin:$PATH ENV GOPATH /gopath -# Pre-create dirs for syz-dock. -# This is necessary to make docker work with the current user, -# otherwise --volume will create these dirs under root and then -# the current user won't have access to them. -RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \ - mkdir -p /syzkaller/.cache && \ - chmod -R 0777 /syzkaller - +# Install clang. RUN apt-get install -y -q gnupg software-properties-common apt-transport-https RUN curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - RUN add-apt-repository "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-15 main" @@ -51,29 +97,12 @@ RUN sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/c RUN sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-15 100 RUN apt autoremove -y -q -# Install OS toolchains from pre-built archives. -# These archives were created with: -# tar -cz --owner=0 --group=0 --mode=go=u -f netbsd-toolchain.tar.gz netbsd/tools netbsd/dest -# tar -cz --owner=0 --group=0 --mode=go=u -f fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \ -# fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \ -# fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ -# fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \ -# fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ -# fuchsia/out/arm64/arm64-shared/*.so -# -# And then uploaded to GCS with: -# gsutil mv gs://syzkaller/GOOS-toolchain.tar.gz gs://syzkaller/GOOS-toolchain.old.tar.gz -# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/ -# gsutil acl ch -g all:R gs://syzkaller/GOOS-toolchain.tar.gz -# -# NetBSD toolchain can be re-built with: -# ./build.sh -j72 -m amd64 -U -T ../tools tools -# ./build.sh -j72 -m amd64 -U -T ../tools -D ../dest distribution -# -# To build root image run: -# docker run -it --rm --privileged --device /dev/loop0 gcr.io/syzkaller/env -# mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc -# create-image.sh -a s390x -d buster +# Install the Spanner emulator. +ARG SPANNER_EMULATOR_VERSION=1.5.28 +RUN mkdir /spanner +RUN curl https://storage.googleapis.com/cloud-spanner-emulator/releases/${SPANNER_EMULATOR_VERSION}/cloud-spanner-emulator_linux_amd64-${SPANNER_EMULATOR_VERSION}.tar.gz | tar -C /spanner -xz +RUN chmod u+x /spanner/gateway_main /spanner/emulator_main +ENV SPANNER_EMULATOR_BIN=/spanner/gateway_main RUN dpkg --add-architecture i386 && \ apt-get update --allow-releaseinfo-change && \ @@ -84,20 +113,16 @@ RUN dpkg --add-architecture i386 && \ apt-get clean autoclean && \ rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* +# Install Python 2.7. +COPY --from=python2-builder /python2/ /usr/local/ +RUN ln -s /usr/local/bin/python2.7 /usr/bin/python2 -RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz -RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz +# Copy the /syzkaller folder and set the toolchain environment variables. +COPY --from=syzkaller-folder /syzkaller/ /syzkaller/ +RUN chmod 0777 /syzkaller ENV SOURCEDIR_FUCHSIA /syzkaller/fuchsia ENV SOURCEDIR_NETBSD /syzkaller/netbsd -# Build Python 2.7 from source. -RUN apt-get install -y -q libsqlite3-dev -RUN wget -O /tmp/Python-2.7.18.tgz 'https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tgz' -RUN cd /tmp/ && tar -zxf Python-2.7.18.tgz -RUN cd /tmp/Python-2.7.18 && ./configure -RUN cd /tmp/Python-2.7.18 && make -j2 && make altinstall -RUN ln -s /usr/local/bin/python2.7 /usr/bin/python2 - # Install node to pass act jobs (https://github.com/nektos/act) RUN apt-get install -y -q nodejs