builder: build wasi-libc inside TinyGo

Instead of relying on a build when TinyGo is being built, do it like all
other libraries when it is needed.

This brings a few benefits:

  * No more running `make wasi-libc` on the command line as a special
    case for WebAssembly. The generic `git submodule update --init` step
    is now enough for wasi-libc.
  * It becomes much easier to customize the build per system. For
    example: include/exclude malloc as needed, disable/enable bulk
    memory operations per target, etc.

Author: aykevl

.circleci/config.yml

@@ -73,14 +73,6 @@ commands:
             - go-cache-v4-{{ checksum "go.mod" }}
       - llvm-source-linux
       - run: go install -tags=llvm<<parameters.llvm>> .
-      - restore_cache:
-          keys:
-            - wasi-libc-sysroot-systemclang-v7
-      - run: make wasi-libc
-      - save_cache:
-          key: wasi-libc-sysroot-systemclang-v7
-          paths:
-            - lib/wasi-libc/sysroot
       - when:
           condition: <<parameters.fmt-check>>
           steps:

.github/workflows/build-macos.yml

@@ -89,15 +89,6 @@ jobs:
         with:
           key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }}
           path: llvm-build
-      - name: Cache wasi-libc sysroot
-        uses: actions/cache@v4
-        id: cache-wasi-libc
-        with:
-          key: wasi-libc-sysroot-${{ matrix.os }}-v1
-          path: lib/wasi-libc/sysroot
-      - name: Build wasi-libc
-        if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
-        run: make wasi-libc
       - name: make gen-device
         run: make -j3 gen-device
       - name: Test TinyGo

.github/workflows/linux.yml

@@ -104,15 +104,6 @@ jobs:
         run: |
           apk add cmake samurai python3
           make binaryen STATIC=1
-      - name: Cache wasi-libc
-        uses: actions/cache@v4
-        id: cache-wasi-libc
-        with:
-          key: wasi-libc-sysroot-linux-alpine-v2
-          path: lib/wasi-libc/sysroot
-      - name: Build wasi-libc
-        if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
-        run: make wasi-libc
       - name: Install fpm
         run: |
           gem install --version 4.0.7 public_suffix
@@ -258,15 +249,6 @@ jobs:
       - name: Build Binaryen
         if: steps.cache-binaryen.outputs.cache-hit != 'true'
         run: make binaryen
-      - name: Cache wasi-libc
-        uses: actions/cache@v4
-        id: cache-wasi-libc
-        with:
-          key: wasi-libc-sysroot-linux-asserts-v6
-          path: lib/wasi-libc/sysroot
-      - name: Build wasi-libc
-        if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
-        run: make wasi-libc
       - run: make gen-device -j4
       - name: Test TinyGo
         run: make ASSERT=1 test

.github/workflows/windows.yml

@@ -91,15 +91,6 @@ jobs:
         with:
           key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }}
           path: llvm-build
-      - name: Cache wasi-libc sysroot
-        uses: actions/cache@v4
-        id: cache-wasi-libc
-        with:
-          key: wasi-libc-sysroot-v5
-          path: lib/wasi-libc/sysroot
-      - name: Build wasi-libc
-        if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
-        run: make wasi-libc
       - name: Cache Go cache
         uses: actions/cache@v4
         with:

GNUmakefile

@@ -259,13 +259,6 @@ build/wasm-opt$(EXE):
 	cp lib/binaryen/bin/wasm-opt$(EXE) build/wasm-opt$(EXE)
 endif
 
-# Build wasi-libc sysroot
-.PHONY: wasi-libc
-wasi-libc: lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a
-lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a:
-	@if [ ! -e lib/wasi-libc/Makefile ]; then echo "Submodules have not been downloaded. Please download them using:\n  git submodule update --init"; exit 1; fi
-	cd lib/wasi-libc && $(MAKE) -j4 EXTRA_CFLAGS="-O2 -g -DNDEBUG -mnontrapping-fptoint -msign-ext" MALLOC_IMPL=none CC="$(CLANG)" AR=$(LLVM_AR) NM=$(LLVM_NM)
-
 # Generate WASI syscall bindings
 WASM_TOOLS_MODULE=go.bytecodealliance.org
 .PHONY: wasi-syscall
@@ -293,7 +286,7 @@ endif
 tinygo: ## Build the TinyGo compiler
 	@if [ ! -f "$(LLVM_BUILDDIR)/bin/llvm-config" ]; then echo "Fetch and build LLVM first by running:"; echo "  $(MAKE) llvm-source"; echo "  $(MAKE) $(LLVM_BUILDDIR)"; exit 1; fi
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOENVFLAGS) $(GO) build -buildmode exe -o build/tinygo$(EXE) -tags "byollvm osusergo" .
-test: wasi-libc check-nodejs-version
+test: check-nodejs-version
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags "byollvm osusergo" $(GOTESTPKGS)
 
 # Standard library packages that pass tests on darwin, linux, wasi, and windows, but take over a minute in wasi
@@ -526,9 +519,9 @@ test-corpus:
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml
 test-corpus-fast:
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus -short . -corpus=testdata/corpus.yaml
-test-corpus-wasi: wasi-libc
+test-corpus-wasi:
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip1
-test-corpus-wasip2: wasi-libc
+test-corpus-wasip2:
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip2
 
 .PHONY: testchdir
@@ -939,7 +932,7 @@ endif
 wasmtest:
 	$(GO) test ./tests/wasm
 
-build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen)
+build/release: tinygo gen-device $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen)
 	@mkdir -p build/release/tinygo/bin
 	@mkdir -p build/release/tinygo/lib/bdwgc
 	@mkdir -p build/release/tinygo/lib/clang/include
@@ -954,7 +947,7 @@ build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN
 	@mkdir -p build/release/tinygo/lib/nrfx
 	@mkdir -p build/release/tinygo/lib/picolibc/newlib/libc
 	@mkdir -p build/release/tinygo/lib/picolibc/newlib/libm
-	@mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers
+	@mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half
 	@mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
 	@mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
 	@mkdir -p build/release/tinygo/lib/wasi-cli/
@@ -1016,15 +1009,36 @@ endif
 	@cp -rp lib/picolibc/newlib/libm/common      build/release/tinygo/lib/picolibc/newlib/libm
 	@cp -rp lib/picolibc/newlib/libm/math        build/release/tinygo/lib/picolibc/newlib/libm
 	@cp -rp lib/picolibc-stdio.c         build/release/tinygo/lib
-	@cp -rp lib/wasi-libc/libc-bottom-half/headers/public           build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers
+	@cp -rp lib/wasi-libc/libc-bottom-half/cloudlibc                build/release/tinygo/lib/wasi-libc/libc-bottom-half
+	@cp -rp lib/wasi-libc/libc-bottom-half/headers                  build/release/tinygo/lib/wasi-libc/libc-bottom-half
+	@cp -rp lib/wasi-libc/libc-bottom-half/sources                  build/release/tinygo/lib/wasi-libc/libc-bottom-half
+	@cp -rp lib/wasi-libc/libc-top-half/headers                     build/release/tinygo/lib/wasi-libc/libc-top-half
 	@cp -rp lib/wasi-libc/libc-top-half/musl/arch/generic           build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
 	@cp -rp lib/wasi-libc/libc-top-half/musl/arch/wasm32            build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
+	@cp -rp lib/wasi-libc/libc-top-half/musl/include                build/release/tinygo/lib/wasi-libc/libc-top-half/musl
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/conf               build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/dirent             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/env                build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/errno              build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/exit               build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/fcntl              build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/fenv               build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
 	@cp -rp lib/wasi-libc/libc-top-half/musl/src/include            build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
 	@cp -rp lib/wasi-libc/libc-top-half/musl/src/internal           build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/legacy             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/locale             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
 	@cp -rp lib/wasi-libc/libc-top-half/musl/src/math               build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/misc               build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/multibyte          build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/network            build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/stat               build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/stdio              build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/stdlib             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
 	@cp -rp lib/wasi-libc/libc-top-half/musl/src/string             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
-	@cp -rp lib/wasi-libc/libc-top-half/musl/include                build/release/tinygo/lib/wasi-libc/libc-top-half/musl
-	@cp -rp lib/wasi-libc/sysroot                                   build/release/tinygo/lib/wasi-libc/sysroot
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/thread             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/time               build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/musl/src/unistd             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@cp -rp lib/wasi-libc/libc-top-half/sources                     build/release/tinygo/lib/wasi-libc/libc-top-half
 	@cp -rp lib/wasi-cli/wit                                        build/release/tinygo/lib/wasi-cli/wit
 	@cp -rp llvm-project/compiler-rt/lib/builtins build/release/tinygo/lib/compiler-rt-builtins
 	@cp -rp llvm-project/compiler-rt/LICENSE.TXT  build/release/tinygo/lib/compiler-rt-builtins

builder/build.go

@@ -14,7 +14,6 @@ import (
 	"fmt"
 	"go/types"
 	"hash/crc32"
-	"io/fs"
 	"math/bits"
 	"os"
 	"os/exec"
@@ -168,11 +167,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
 		defer unlock()
 		libcDependencies = append(libcDependencies, libcJob)
 	case "wasi-libc":
-		path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a")
-		if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) {
-			return BuildResult{}, errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?")
+		libcJob, unlock, err := libWasiLibc.load(config, tmpdir, false)
+		if err != nil {
+			return BuildResult{}, err
 		}
-		libcDependencies = append(libcDependencies, dummyCompileJob(path))
+		defer unlock()
+		libcDependencies = append(libcDependencies, libcJob)
 	case "wasmbuiltins":
 		libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir, false)
 		if err != nil {

builder/library.go

@@ -25,6 +25,9 @@ type Library struct {
 	// cflags returns the C flags specific to this library
 	cflags func(target, headerPath string) []string
 
+	// cflagsForFile returns additional C flags for a particular source file.
+	cflagsForFile func(path string) []string
+
 	// The source directory.
 	sourceDir func() string
 
@@ -227,6 +230,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string, needLibc bool)
 	}
 	for _, path := range paths {
 		// Strip leading "../" parts off the path.
+		path := path
 		cleanpath := path
 		for strings.HasPrefix(cleanpath, "../") {
 			cleanpath = cleanpath[3:]
@@ -240,6 +244,9 @@ func (l *Library) load(config *compileopts.Config, tmpdir string, needLibc bool)
 			run: func(*compileJob) error {
 				var compileArgs []string
 				compileArgs = append(compileArgs, args...)
+				if l.cflagsForFile != nil {
+					compileArgs = append(compileArgs, l.cflagsForFile(path)...)
+				}
 				compileArgs = append(compileArgs, "-o", objpath, srcpath)
 				if config.Options.PrintCommands != nil {
 					config.Options.PrintCommands("clang", compileArgs...)

builder/musl.go

@@ -12,6 +12,45 @@ import (
 	"github.com/tinygo-org/tinygo/goenv"
 )
 
+// Create the alltypes.h file from the appropriate alltypes.h.in files.
+func buildMuslAllTypes(arch, muslDir, outputBitsDir string) error {
+	// Create the file alltypes.h.
+	f, err := os.Create(filepath.Join(outputBitsDir, "alltypes.h"))
+	if err != nil {
+		return err
+	}
+	infiles := []string{
+		filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"),
+		filepath.Join(muslDir, "include", "alltypes.h.in"),
+	}
+	for _, infile := range infiles {
+		data, err := os.ReadFile(infile)
+		if err != nil {
+			return err
+		}
+		lines := strings.Split(string(data), "\n")
+		for _, line := range lines {
+			if strings.HasPrefix(line, "TYPEDEF ") {
+				matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line)
+				value := matches[1]
+				name := matches[2]
+				line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name)
+			}
+			if strings.HasPrefix(line, "STRUCT ") {
+				matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line)
+				name := matches[1]
+				value := matches[2]
+				line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name)
+			}
+			_, err := f.WriteString(line + "\n")
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return f.Close()
+}
+
 var libMusl = Library{
 	name: "musl",
 	makeHeaders: func(target, includeDir string) error {
@@ -24,41 +63,13 @@ var libMusl = Library{
 		arch := compileopts.MuslArchitecture(target)
 		muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "musl")
 
-		// Create the file alltypes.h.
-		f, err := os.Create(filepath.Join(bits, "alltypes.h"))
+		err = buildMuslAllTypes(arch, muslDir, bits)
 		if err != nil {
 			return err
 		}
-		infiles := []string{
-			filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"),
-			filepath.Join(muslDir, "include", "alltypes.h.in"),
-		}
-		for _, infile := range infiles {
-			data, err := os.ReadFile(infile)
-			if err != nil {
-				return err
-			}
-			lines := strings.Split(string(data), "\n")
-			for _, line := range lines {
-				if strings.HasPrefix(line, "TYPEDEF ") {
-					matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line)
-					value := matches[1]
-					name := matches[2]
-					line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name)
-				}
-				if strings.HasPrefix(line, "STRUCT ") {
-					matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line)
-					name := matches[1]
-					value := matches[2]
-					line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name)
-				}
-				f.WriteString(line + "\n")
-			}
-		}
-		f.Close()
 
 		// Create the file syscall.h.
-		f, err = os.Create(filepath.Join(bits, "syscall.h"))
+		f, err := os.Create(filepath.Join(bits, "syscall.h"))
 		if err != nil {
 			return err
 		}