diff --git a/Dockerfile b/Dockerfile index 558c1b285..9a864e63e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,18 +3,23 @@ # Here we choose Bionic 18.04. FROM ubuntu:bionic +# We want to use the same UID/GID of the external user to avoid permission +# issues. See the user setup at the end of the file. +ARG UID=1000 +ARG GID=1000 + RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - ccache \ - curl \ - ca-certificates \ - build-essential \ - clang \ - python3 \ - git \ - ninja-build \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends \ + ccache \ + curl \ + ca-certificates \ + build-essential \ + clang \ + python3 \ + git \ + ninja-build \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* RUN curl -sSLO https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-x86_64.tar.gz \ && tar xf cmake-3.25.1-linux-x86_64.tar.gz \ @@ -22,3 +27,8 @@ RUN curl -sSLO https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake- && mkdir -p /opt \ && mv cmake-3.25.1-linux-x86_64 /opt/cmake ENV PATH /opt/cmake/bin:$PATH + +RUN groupadd -g ${GID} builder && \ + useradd --create-home --uid ${UID} --gid ${GID} builder +USER builder +WORKDIR /workspace diff --git a/Makefile b/Makefile index b30a62a3b..9c7acffbf 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ ROOT_DIR=${CURDIR} LLVM_PROJ_DIR?=$(ROOT_DIR)/src/llvm-project +VERSION_SCRIPT=python3 ./version.py # Windows needs munging ifeq ($(OS),Windows_NT) @@ -45,8 +46,8 @@ endif TARGETS = wasm32-wasi wasm32-wasip1 wasm32-wasip2 wasm32-wasip1-threads wasm32-wasi-threads # Only the major version is needed for Clang, see https://reviews.llvm.org/D125860. -CLANG_VERSION=$(shell $(BASH) ./llvm_version_major.sh $(LLVM_PROJ_DIR)) -VERSION:=$(shell $(BASH) ./version.sh) +CLANG_VERSION=$(shell $(VERSION_SCRIPT) llvm-major --llvm-dir=$(LLVM_PROJ_DIR)) +VERSION:=$(shell $(VERSION_SCRIPT)) DEBUG_PREFIX_MAP=-fdebug-prefix-map=$(ROOT_DIR)=wasisdk://v$(VERSION) default: build @@ -255,7 +256,11 @@ build/config.BUILT: cp cmake/Platform/WASI.cmake $(BUILD_PREFIX)/share/cmake/Platform touch build/config.BUILT -build: build/llvm.BUILT build/wasi-libc.BUILT build/compiler-rt.BUILT build/libcxx.BUILT build/config.BUILT +build/version.BUILT: + $(VERSION_SCRIPT) dump > $(BUILD_PREFIX)/VERSION + touch build/version.BUILT + +build: build/llvm.BUILT build/wasi-libc.BUILT build/compiler-rt.BUILT build/libcxx.BUILT build/config.BUILT build/version.BUILT strip: build/llvm.BUILT ./strip_symbols.sh $(BUILD_PREFIX)/bin diff --git a/RELEASING.md b/RELEASING.md index d7eaef2f7..c684149be 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,7 +3,7 @@ To publish a new version of `wasi-sdk` as a GitHub release: 1. Tag a commit with an annotated tag. Note that this must be an annotated tag, - not a lightweight tag, so that `version.sh` can use it for calculating the + not a lightweight tag, so that `version.py` can use it for calculating the package version (use `git show wasi-sdk-...` to show other tag messages). Note that you may need to clear the repository cache to avoid problems with cached artifacts [^cache]. diff --git a/deb_from_installation.sh b/deb_from_installation.sh index 2c1596163..2699c819c 100755 --- a/deb_from_installation.sh +++ b/deb_from_installation.sh @@ -18,7 +18,7 @@ fi if [ -n "$2" ]; then VERSION="$2" else - VERSION=`./version.sh` + VERSION=`./version.py` fi if [ -n "$3" ]; then diff --git a/docker_build.sh b/docker_build.sh index 38845277f..050862fac 100755 --- a/docker_build.sh +++ b/docker_build.sh @@ -1,7 +1,18 @@ #!/bin/sh set -ex + echo "Building the docker image" -docker build -t wasi-sdk-builder:latest . +docker build \ + --build-arg UID=$(id -u) --build-arg GID=$(id -g) \ + -t wasi-sdk-builder:latest . + echo "Building the package in docker image" mkdir -p ~/.ccache -docker run --rm -v "$PWD":/workspace -v ~/.ccache:/root/.ccache -e NINJA_FLAGS=-v --workdir /workspace --tmpfs /tmp:exec wasi-sdk-builder:latest make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON +docker run --rm \ + --user $(id -u):$(id -g) \ + -v "$PWD":/workspace:Z \ + -v ~/.ccache:/home/builder/.ccache:Z \ + -e NINJA_FLAGS=-v \ + --tmpfs /tmp:exec \ + wasi-sdk-builder:latest \ + make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON diff --git a/llvm_version_major.sh b/llvm_version_major.sh deleted file mode 100755 index 93fe9b01b..000000000 --- a/llvm_version_major.sh +++ /dev/null @@ -1,4 +0,0 @@ -#/bin/bash -LLVM_PROJ_DIR=${1:-./src/llvm-project} -MAJOR=`(grep "set(LLVM_VERSION_MAJOR" $LLVM_PROJ_DIR/llvm/CMakeLists.txt || grep "set(LLVM_VERSION_MAJOR" $LLVM_PROJ_DIR/cmake/Modules/LLVMVersion.cmake) | awk '{print substr($2, 1, length($2) - 1)}'` -echo $MAJOR diff --git a/tar_from_installation.sh b/tar_from_installation.sh index f90000bf5..7d0943266 100755 --- a/tar_from_installation.sh +++ b/tar_from_installation.sh @@ -10,7 +10,7 @@ fi if [ -n "$2" ]; then VERSION="$2" else - VERSION=`./version.sh` + VERSION=`./version.py` fi if [ -n "$3" ]; then diff --git a/version.py b/version.py new file mode 100755 index 000000000..2d9de6d5b --- /dev/null +++ b/version.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +# This script finds and prints the various versions in this project: wasi-sdk +# itself, LLVM, and the Git revisions of dependencies. +# +# Usage: version [wasi-sdk|llvm|llvm-major|dump] [--llvm-dir=] + +import argparse +import os +import subprocess +import sys + +# The number of characters to use for the abbreviated Git revision. +GIT_REF_LEN = 12 + + +def exec(command, cwd=None): + result = subprocess.run(command, stdout=subprocess.PIPE, + universal_newlines=True, check=True, cwd=cwd) + return result.stdout.strip() + + +def git_commit(dir='.'): + return exec(['git', 'rev-parse', f'--short={GIT_REF_LEN}', 'HEAD'], dir) + + +def parse_git_version(version): + # Parse, e.g.: wasi-sdk-21-0-g317548590b40+m + parts = version.replace('+', '-').split('-') + assert parts.pop(0) == 'wasi' + assert parts.pop(0) == 'sdk' + + major, minor = parts.pop(0), parts.pop(0) + git = None + dirty = False + + if parts: + # Check: git|dirty. + next = parts.pop(0) + if next == 'm': + dirty = True + else: + git = next[1:] + + # Check: dirty. + if parts: + assert parts.pop(0) == 'm', f'expected dirty flag: +m' + dirty = True + + assert not parts, f'unexpected suffixes: {parts}' + return major, minor, git, dirty + + +# Some inline tests to check Git version parsing: +assert parse_git_version( + 'wasi-sdk-21-0-g317548590b40+m') == ('21', '0', '317548590b40', True) +assert parse_git_version('wasi-sdk-21-2+m') == ('21', '2', None, True) +assert parse_git_version( + 'wasi-sdk-23-0-g317548590b40') == ('23', '0', '317548590b40', False) + + +def git_version(): + version = exec(['git', 'describe', '--long', '--candidates=999', + '--match=wasi-sdk-*', '--dirty=+m', f'--abbrev={GIT_REF_LEN}']) + major, minor, git, dirty = parse_git_version(version) + version = f'{major}.{minor}' + if git: + version += f'g{git}' + if dirty: + version += '+m' + return version + + +def parse_cmake_set(line): + return line.split(' ')[1].split(')')[0] + + +def llvm_cmake_version(llvm_dir): + path = f'{llvm_dir}/cmake/Modules/LLVMVersion.cmake' + if not os.path.exists(path): + # Handle older LLVM versions; see #399. + path = f'{llvm_dir}/llvm/CMakeLists.txt' + with open(path) as file: + for line in file: + line = line.strip() + if line.startswith('set(LLVM_VERSION_MAJOR'): + llvm_version_major = parse_cmake_set(line) + elif line.startswith('set(LLVM_VERSION_MINOR'): + llvm_version_minor = parse_cmake_set(line) + elif line.startswith('set(LLVM_VERSION_PATCH'): + llvm_version_patch = parse_cmake_set(line) + return llvm_version_major, llvm_version_minor, llvm_version_patch + + +def main(action, llvm_dir): + if action == 'wasi-sdk': + print(git_version()) + elif action == 'llvm': + major, minor, path = llvm_cmake_version(llvm_dir) + print(f'{major}.{minor}.{path}') + elif action == 'llvm-major': + major, _, _ = llvm_cmake_version(llvm_dir) + print(major) + elif action == 'dump': + print(git_version()) + print(f'wasi-libc: {git_commit("src/wasi-libc")}') + print(f'llvm: {git_commit(llvm_dir)}') + major, minor, path = llvm_cmake_version(llvm_dir) + print(f'llvm-version: {major}.{minor}.{path}') + print(f'config: {git_commit("src/config")}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Print the various kinds of versions in wasi-sdk') + parser.add_argument('action', + choices=['wasi-sdk', 'llvm', 'llvm-major', 'dump'], + nargs='?', + default='wasi-sdk', + help='Which kind of version to print (default: wasi-sdk).') + parser.add_argument('--llvm-dir', + nargs='?', + default='src/llvm-project', + help='Override the location of the LLVM source directory (default: src/llvm-project).') + args = parser.parse_args() + main(args.action, args.llvm_dir) + sys.exit(0) diff --git a/version.sh b/version.sh deleted file mode 100755 index 35c8a908e..000000000 --- a/version.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e -git config --global --add safe.directory "/workspace" -GIT_DESCR=$(git describe --long --candidates=999 --match='wasi-sdk-*' --dirty='+m' --abbrev=12) -GIT_PACKAGE_VERSION=$(echo $GIT_DESCR | perl -ne 'if(/^wasi-sdk-(\d+)-(\d+)-g([0-9a-f]{7,12})([+]m)?$/) { if($2 == 0) { print "$1.$2$4" } else { print "$1.$2g$3$4" } exit } else { print "could not parse git description"; exit 1 }';) -echo $GIT_PACKAGE_VERSION