Skip to content

Commit 1181926

Browse files
aykevlaykevl
committed
builder: build wasi-libc inside TinyGo
Instead of relying on a build when TinyGo is being built, do it like all other libraries when it is needed. This brings a few benefits: * No more running `make wasi-libc` on the command line as a special case for WebAssembly. The generic `git submodule update --init` step is now enough for wasi-libc. * It becomes much easier to customize the build per system. For example: include/exclude malloc as needed, disable/enable bulk memory operations per target, etc.
1 parent d0c7456 commit 1181926

File tree

11 files changed

+346
-103
lines changed

11 files changed

+346
-103
lines changed

‎.circleci/config.yml

-8
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,6 @@ commands:
7373
- go-cache-v4-{{ checksum "go.mod" }}
7474
- llvm-source-linux
7575
- run: go install -tags=llvm<<parameters.llvm>> .
76-
- restore_cache:
77-
keys:
78-
- wasi-libc-sysroot-systemclang-v7
79-
- run: make wasi-libc
80-
- save_cache:
81-
key: wasi-libc-sysroot-systemclang-v7
82-
paths:
83-
- lib/wasi-libc/sysroot
8476
- when:
8577
condition: <<parameters.fmt-check>>
8678
steps:

‎.github/workflows/build-macos.yml

-9
Original file line numberDiff line numberDiff line change
@@ -89,15 +89,6 @@ jobs:
8989
with:
9090
key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }}
9191
path: llvm-build
92-
- name: Cache wasi-libc sysroot
93-
uses: actions/cache@v4
94-
id: cache-wasi-libc
95-
with:
96-
key: wasi-libc-sysroot-${{ matrix.os }}-v1
97-
path: lib/wasi-libc/sysroot
98-
- name: Build wasi-libc
99-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
100-
run: make wasi-libc
10192
- name: make gen-device
10293
run: make -j3 gen-device
10394
- name: Test TinyGo

‎.github/workflows/linux.yml

-18
Original file line numberDiff line numberDiff line change
@@ -104,15 +104,6 @@ jobs:
104104
run: |
105105
apk add cmake samurai python3
106106
make binaryen STATIC=1
107-
- name: Cache wasi-libc
108-
uses: actions/cache@v4
109-
id: cache-wasi-libc
110-
with:
111-
key: wasi-libc-sysroot-linux-alpine-v2
112-
path: lib/wasi-libc/sysroot
113-
- name: Build wasi-libc
114-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
115-
run: make wasi-libc
116107
- name: Install fpm
117108
run: |
118109
gem install --version 4.0.7 public_suffix
@@ -258,15 +249,6 @@ jobs:
258249
- name: Build Binaryen
259250
if: steps.cache-binaryen.outputs.cache-hit != 'true'
260251
run: make binaryen
261-
- name: Cache wasi-libc
262-
uses: actions/cache@v4
263-
id: cache-wasi-libc
264-
with:
265-
key: wasi-libc-sysroot-linux-asserts-v6
266-
path: lib/wasi-libc/sysroot
267-
- name: Build wasi-libc
268-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
269-
run: make wasi-libc
270252
- run: make gen-device -j4
271253
- name: Test TinyGo
272254
run: make ASSERT=1 test

‎.github/workflows/windows.yml

-9
Original file line numberDiff line numberDiff line change
@@ -91,15 +91,6 @@ jobs:
9191
with:
9292
key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }}
9393
path: llvm-build
94-
- name: Cache wasi-libc sysroot
95-
uses: actions/cache@v4
96-
id: cache-wasi-libc
97-
with:
98-
key: wasi-libc-sysroot-v5
99-
path: lib/wasi-libc/sysroot
100-
- name: Build wasi-libc
101-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
102-
run: make wasi-libc
10394
- name: Cache Go cache
10495
uses: actions/cache@v4
10596
with:

‎GNUmakefile

+13-13
Original file line numberDiff line numberDiff line change
@@ -259,13 +259,6 @@ build/wasm-opt$(EXE):
259259
cp lib/binaryen/bin/wasm-opt$(EXE) build/wasm-opt$(EXE)
260260
endif
261261

262-
# Build wasi-libc sysroot
263-
.PHONY: wasi-libc
264-
wasi-libc: lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a
265-
lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a:
266-
@if [ ! -e lib/wasi-libc/Makefile ]; then echo "Submodules have not been downloaded. Please download them using:\n git submodule update --init"; exit 1; fi
267-
cd lib/wasi-libc && $(MAKE) -j4 EXTRA_CFLAGS="-O2 -g -DNDEBUG -mnontrapping-fptoint -msign-ext" MALLOC_IMPL=none CC="$(CLANG)" AR=$(LLVM_AR) NM=$(LLVM_NM)
268-
269262
# Generate WASI syscall bindings
270263
WASM_TOOLS_MODULE=go.bytecodealliance.org
271264
.PHONY: wasi-syscall
@@ -293,7 +286,7 @@ endif
293286
tinygo: ## Build the TinyGo compiler
294287
@if [ ! -f "$(LLVM_BUILDDIR)/bin/llvm-config" ]; then echo "Fetch and build LLVM first by running:"; echo " $(MAKE) llvm-source"; echo " $(MAKE) $(LLVM_BUILDDIR)"; exit 1; fi
295288
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOENVFLAGS) $(GO) build -buildmode exe -o build/tinygo$(EXE) -tags "byollvm osusergo" .
296-
test: wasi-libc check-nodejs-version
289+
test: check-nodejs-version
297290
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags "byollvm osusergo" $(GOTESTPKGS)
298291

299292
# Standard library packages that pass tests on darwin, linux, wasi, and windows, but take over a minute in wasi
@@ -526,9 +519,9 @@ test-corpus:
526519
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml
527520
test-corpus-fast:
528521
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus -short . -corpus=testdata/corpus.yaml
529-
test-corpus-wasi: wasi-libc
522+
test-corpus-wasi:
530523
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip1
531-
test-corpus-wasip2: wasi-libc
524+
test-corpus-wasip2:
532525
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip2
533526

534527
.PHONY: testchdir
@@ -939,7 +932,7 @@ endif
939932
wasmtest:
940933
$(GO) test ./tests/wasm
941934

942-
build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen)
935+
build/release: tinygo gen-device $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen)
943936
@mkdir -p build/release/tinygo/bin
944937
@mkdir -p build/release/tinygo/lib/bdwgc
945938
@mkdir -p build/release/tinygo/lib/clang/include
@@ -1016,15 +1009,22 @@ endif
10161009
@cp -rp lib/picolibc/newlib/libm/common build/release/tinygo/lib/picolibc/newlib/libm
10171010
@cp -rp lib/picolibc/newlib/libm/math build/release/tinygo/lib/picolibc/newlib/libm
10181011
@cp -rp lib/picolibc-stdio.c build/release/tinygo/lib
1012+
@cp -rp lib/wasi-libc/libc-bottom-half/cloudlibc build/release/tinygo/lib/wasi-libc/libc-bottom-half
10191013
@cp -rp lib/wasi-libc/libc-bottom-half/headers/public build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers
1014+
@cp -rp lib/wasi-libc/libc-bottom-half/sources build/release/tinygo/lib/wasi-libc/libc-bottom-half
10201015
@cp -rp lib/wasi-libc/libc-top-half/musl/arch/generic build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
10211016
@cp -rp lib/wasi-libc/libc-top-half/musl/arch/wasm32 build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
1017+
@cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl
1018+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/conf build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10221019
@cp -rp lib/wasi-libc/libc-top-half/musl/src/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10231020
@cp -rp lib/wasi-libc/libc-top-half/musl/src/internal build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1021+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/locate build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10241022
@cp -rp lib/wasi-libc/libc-top-half/musl/src/math build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1023+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/multibyte build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1024+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/stdio build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1025+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/stdlib build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10251026
@cp -rp lib/wasi-libc/libc-top-half/musl/src/string build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1026-
@cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl
1027-
@cp -rp lib/wasi-libc/sysroot build/release/tinygo/lib/wasi-libc/sysroot
1027+
@cp -rp lib/wasi-libc/libc-top-half/sources build/release/tinygo/lib/wasi-libc/libc-top-half
10281028
@cp -rp lib/wasi-cli/wit build/release/tinygo/lib/wasi-cli/wit
10291029
@cp -rp llvm-project/compiler-rt/lib/builtins build/release/tinygo/lib/compiler-rt-builtins
10301030
@cp -rp llvm-project/compiler-rt/LICENSE.TXT build/release/tinygo/lib/compiler-rt-builtins

‎builder/build.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"fmt"
1515
"go/types"
1616
"hash/crc32"
17-
"io/fs"
1817
"math/bits"
1918
"os"
2019
"os/exec"
@@ -168,11 +167,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
168167
defer unlock()
169168
libcDependencies = append(libcDependencies, libcJob)
170169
case "wasi-libc":
171-
path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a")
172-
if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) {
173-
return BuildResult{}, errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?")
170+
libcJob, unlock, err := libWasiLibc.load(config, tmpdir, false)
171+
if err != nil {
172+
return BuildResult{}, err
174173
}
175-
libcDependencies = append(libcDependencies, dummyCompileJob(path))
174+
defer unlock()
175+
libcDependencies = append(libcDependencies, libcJob)
176176
case "wasmbuiltins":
177177
libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir, false)
178178
if err != nil {

‎builder/library.go

+7
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ type Library struct {
2525
// cflags returns the C flags specific to this library
2626
cflags func(target, headerPath string) []string
2727

28+
// cflagsForFile returns additional C flags for a particular source file.
29+
cflagsForFile func(path string) []string
30+
2831
// The source directory.
2932
sourceDir func() string
3033

@@ -227,6 +230,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string, needLibc bool)
227230
}
228231
for _, path := range paths {
229232
// Strip leading "../" parts off the path.
233+
path := path
230234
cleanpath := path
231235
for strings.HasPrefix(cleanpath, "../") {
232236
cleanpath = cleanpath[3:]
@@ -240,6 +244,9 @@ func (l *Library) load(config *compileopts.Config, tmpdir string, needLibc bool)
240244
run: func(*compileJob) error {
241245
var compileArgs []string
242246
compileArgs = append(compileArgs, args...)
247+
if l.cflagsForFile != nil {
248+
compileArgs = append(compileArgs, l.cflagsForFile(path)...)
249+
}
243250
compileArgs = append(compileArgs, "-o", objpath, srcpath)
244251
if config.Options.PrintCommands != nil {
245252
config.Options.PrintCommands("clang", compileArgs...)

‎builder/musl.go

+41-30
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,45 @@ import (
1212
"github.com/tinygo-org/tinygo/goenv"
1313
)
1414

15+
// Create the alltypes.h file from the appropriate alltypes.h.in files.
16+
func buildMuslAllTypes(arch, muslDir, outputBitsDir string) error {
17+
// Create the file alltypes.h.
18+
f, err := os.Create(filepath.Join(outputBitsDir, "alltypes.h"))
19+
if err != nil {
20+
return err
21+
}
22+
infiles := []string{
23+
filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"),
24+
filepath.Join(muslDir, "include", "alltypes.h.in"),
25+
}
26+
for _, infile := range infiles {
27+
data, err := os.ReadFile(infile)
28+
if err != nil {
29+
return err
30+
}
31+
lines := strings.Split(string(data), "\n")
32+
for _, line := range lines {
33+
if strings.HasPrefix(line, "TYPEDEF ") {
34+
matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line)
35+
value := matches[1]
36+
name := matches[2]
37+
line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name)
38+
}
39+
if strings.HasPrefix(line, "STRUCT ") {
40+
matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line)
41+
name := matches[1]
42+
value := matches[2]
43+
line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name)
44+
}
45+
_, err := f.WriteString(line + "\n")
46+
if err != nil {
47+
return err
48+
}
49+
}
50+
}
51+
return f.Close()
52+
}
53+
1554
var libMusl = Library{
1655
name: "musl",
1756
makeHeaders: func(target, includeDir string) error {
@@ -24,41 +63,13 @@ var libMusl = Library{
2463
arch := compileopts.MuslArchitecture(target)
2564
muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "musl")
2665

27-
// Create the file alltypes.h.
28-
f, err := os.Create(filepath.Join(bits, "alltypes.h"))
66+
err = buildMuslAllTypes(arch, muslDir, bits)
2967
if err != nil {
3068
return err
3169
}
32-
infiles := []string{
33-
filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"),
34-
filepath.Join(muslDir, "include", "alltypes.h.in"),
35-
}
36-
for _, infile := range infiles {
37-
data, err := os.ReadFile(infile)
38-
if err != nil {
39-
return err
40-
}
41-
lines := strings.Split(string(data), "\n")
42-
for _, line := range lines {
43-
if strings.HasPrefix(line, "TYPEDEF ") {
44-
matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line)
45-
value := matches[1]
46-
name := matches[2]
47-
line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name)
48-
}
49-
if strings.HasPrefix(line, "STRUCT ") {
50-
matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line)
51-
name := matches[1]
52-
value := matches[2]
53-
line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name)
54-
}
55-
f.WriteString(line + "\n")
56-
}
57-
}
58-
f.Close()
5970

6071
// Create the file syscall.h.
61-
f, err = os.Create(filepath.Join(bits, "syscall.h"))
72+
f, err := os.Create(filepath.Join(bits, "syscall.h"))
6273
if err != nil {
6374
return err
6475
}

0 commit comments

Comments
 (0)