Skip to content

cmd/goexports,stdlib: parallelize symbols generation for stdlib #323

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

cmd/goexports,stdlib: parallelize symbols generation for stdlib #323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
68d1d06
cmd/goexports,stdlib: parallelize symbols generation for stdlib
sbinet Aug 1, 2019
d31aadf
cmd/goexports: use sync.WaitGroup instead of errgroup.Group
sbinet Aug 5, 2019
d732705
stdlib: introduce stdlib/internal/genstdlib command to wrap stdlib s…
sbinet Aug 5, 2019
1ba0d6f
stdlib/internal/genstdlib: appease errcheck
sbinet Aug 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 45 additions & 22 deletions cmd/goexports/goexports.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"runtime"
"strconv"
"strings"
"sync"
"text/template"
)

Expand Down Expand Up @@ -300,6 +301,7 @@ func genLicense(fname string) (string, error) {

func main() {
licenseFlag := flag.String("license", "", "path to a LICENSE file")
workersFlag := flag.Int("cpu", -1, "number of packages being wrapped concurrently")

flag.Parse()

Expand All @@ -308,6 +310,11 @@ func main() {
log.Fatalf("missing package path")
}

nworkers := *workersFlag
if nworkers <= 0 {
nworkers = runtime.NumCPU() + 1
}

license, err := genLicense(*licenseFlag)
if err != nil {
log.Fatal(err)
Expand All @@ -319,32 +326,48 @@ func main() {
}
dest := path.Base(dir)

for _, pkg := range flag.Args() {
content, err := genContent(dest, pkg, license)
if err != nil {
log.Fatal(err)
}

var oFile string
if pkg == "syscall" {
goos, arch := os.Getenv("GOOS"), os.Getenv("GOARCH")
oFile = strings.Replace(pkg, "/", "_", -1) + "_" + goos + "_" + arch + ".go"
} else {
oFile = strings.Replace(pkg, "/", "_", -1) + ".go"
}
var (
wg sync.WaitGroup
throttle = make(chan struct{}, nworkers)
)

wg.Add(flag.NArg())
for i := range flag.Args() {
pkg := flag.Arg(i)
go func() {
throttle <- struct{}{}
defer func() {
wg.Done()
<-throttle
}()

content, err := genContent(dest, pkg, license)
if err != nil {
log.Fatalf("could not generate content for package %q: %v", pkg, err)
}

prefix := runtime.Version()
if runtime.Version() != "devel" {
parts := strings.Split(runtime.Version(), ".")
var oFile string
if pkg == "syscall" {
goos, arch := os.Getenv("GOOS"), os.Getenv("GOARCH")
oFile = strings.Replace(pkg, "/", "_", -1) + "_" + goos + "_" + arch + ".go"
} else {
oFile = strings.Replace(pkg, "/", "_", -1) + ".go"
}

prefix = parts[0] + "_" + getMinor(parts[1])
}
prefix := runtime.Version()
if runtime.Version() != "devel" {
parts := strings.Split(runtime.Version(), ".")
prefix = parts[0] + "_" + getMinor(parts[1])
}

err = ioutil.WriteFile(prefix+"_"+oFile, content, 0666)
if err != nil {
log.Fatal(err)
}
err = ioutil.WriteFile(prefix+"_"+oFile, content, 0666)
if err != nil {
log.Fatalf("could not write wrapper file for package %q: %v", pkg, err)
}
}()
}

wg.Wait()
}

func getMinor(part string) string {
Expand Down
174 changes: 174 additions & 0 deletions stdlib/internal/genstdlib/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
// generate symbols to provide access to Go stdlib (https://golang.org/pkg).
package main

import (
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
)

func main() {
tmpdir, err := ioutil.TempDir("", "yaegi-stdlib-")
if err != nil {
log.Fatalf("could not create top-level temporary directory: %v", err)
}
defer func() { _ = os.RemoveAll(tmpdir) }()

export := filepath.Join(tmpdir, "goexports")

out, err := exec.Command("go", "build", "-o", export, "github.com/containous/yaegi/cmd/goexports").CombinedOutput()
if err != nil {
log.Fatalf("could not build goexports command: %v\n%v", err, string(out))
}

out, err = exec.Command(export, pkgs...).CombinedOutput()
if err != nil {
log.Fatalf("could not generate stdlib symbols: %v\n%v", err, string(out))
}
}

// generated from:
// go list std | grep -v internal | grep -v '\.' | grep -v unsafe | grep -v syscall
var pkgs = []string{
"archive/tar",
"archive/zip",
"bufio",
"bytes",
"compress/bzip2",
"compress/flate",
"compress/gzip",
"compress/lzw",
"compress/zlib",
"container/heap",
"container/list",
"container/ring",
"context",
"crypto",
"crypto/aes",
"crypto/cipher",
"crypto/des",
"crypto/dsa",
"crypto/ecdsa",
"crypto/elliptic",
"crypto/hmac",
"crypto/md5",
"crypto/rand",
"crypto/rc4",
"crypto/rsa",
"crypto/sha1",
"crypto/sha256",
"crypto/sha512",
"crypto/subtle",
"crypto/tls",
"crypto/x509",
"crypto/x509/pkix",
"database/sql",
"database/sql/driver",
"debug/dwarf",
"debug/elf",
"debug/gosym",
"debug/macho",
"debug/pe",
"debug/plan9obj",
"encoding",
"encoding/ascii85",
"encoding/asn1",
"encoding/base32",
"encoding/base64",
"encoding/binary",
"encoding/csv",
"encoding/gob",
"encoding/hex",
"encoding/json",
"encoding/pem",
"encoding/xml",
"errors",
"expvar",
"flag",
"fmt",
"go/ast",
"go/build",
"go/constant",
"go/doc",
"go/format",
"go/importer",
"go/parser",
"go/printer",
"go/scanner",
"go/token",
"go/types",
"hash",
"hash/adler32",
"hash/crc32",
"hash/crc64",
"hash/fnv",
"html",
"html/template",
"image",
"image/color",
"image/color/palette",
"image/draw",
"image/gif",
"image/jpeg",
"image/png",
"index/suffixarray",
"io",
"io/ioutil",
"log",
"log/syslog",
"math",
"math/big",
"math/bits",
"math/cmplx",
"math/rand",
"mime",
"mime/multipart",
"mime/quotedprintable",
"net",
"net/http",
"net/http/cgi",
"net/http/cookiejar",
"net/http/fcgi",
"net/http/httptest",
"net/http/httptrace",
"net/http/httputil",
"net/http/pprof",
"net/mail",
"net/rpc",
"net/rpc/jsonrpc",
"net/smtp",
"net/textproto",
"net/url",
"os",
"os/exec",
"os/signal",
"os/user",
"path",
"path/filepath",
"plugin",
"reflect",
"regexp",
"regexp/syntax",
"runtime",
"runtime/debug",
// "runtime/pprof",
// "runtime/trace",
"sort",
"strconv",
"strings",
"sync",
"sync/atomic",
"testing",
"testing/iotest",
"testing/quick",
"text/scanner",
"text/tabwriter",
"text/template",
"text/template/parse",
"time",
"unicode",
"unicode/utf16",
"unicode/utf8",
}
32 changes: 1 addition & 31 deletions stdlib/stdlib.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,34 +15,4 @@ func init() {

// Provide access to go standard library (http://golang.org/pkg/)

//go:generate ../cmd/goexports/goexports archive/tar archive/zip
//go:generate ../cmd/goexports/goexports bufio bytes
//go:generate ../cmd/goexports/goexports compress/bzip2 compress/flate compress/gzip compress/lzw compress/zlib
//go:generate ../cmd/goexports/goexports container/heap container/list container/ring
//go:generate ../cmd/goexports/goexports context crypto crypto/aes crypto/cipher crypto/des crypto/dsa
//go:generate ../cmd/goexports/goexports crypto/ecdsa crypto/elliptic crypto/hmac crypto/md5 crypto/rand
//go:generate ../cmd/goexports/goexports crypto/rc4 crypto/rsa crypto/sha1 crypto/sha256 crypto/sha512
//go:generate ../cmd/goexports/goexports crypto/subtle crypto/tls crypto/x509 crypto/x509/pkix
//go:generate ../cmd/goexports/goexports database/sql database/sql/driver
//go:generate ../cmd/goexports/goexports encoding encoding/ascii85 encoding/asn1 encoding/base32
//go:generate ../cmd/goexports/goexports encoding/base64 encoding/binary encoding/csv encoding/gob
//go:generate ../cmd/goexports/goexports encoding/hex encoding/json encoding/pem encoding/xml
//go:generate ../cmd/goexports/goexports errors expvar flag fmt
//go:generate ../cmd/goexports/goexports go/ast go/build go/constant go/doc go/format go/importer
//go:generate ../cmd/goexports/goexports go/parser go/printer go/scanner go/token go/types
//go:generate ../cmd/goexports/goexports hash hash/adler32 hash/crc32 hash/crc64 hash/fnv
//go:generate ../cmd/goexports/goexports html html/template
//go:generate ../cmd/goexports/goexports image image/color image/color/palette
//go:generate ../cmd/goexports/goexports image/draw image/gif image/jpeg image/png
//go:generate ../cmd/goexports/goexports index/suffixarray io io/ioutil log log/syslog
//go:generate ../cmd/goexports/goexports math math/big math/bits math/cmplx math/rand
//go:generate ../cmd/goexports/goexports mime mime/multipart mime/quotedprintable
//go:generate ../cmd/goexports/goexports net net/http net/http/cgi net/http/cookiejar net/http/fcgi
//go:generate ../cmd/goexports/goexports net/http/httptest net/http/httptrace net/http/httputil
//go:generate ../cmd/goexports/goexports net/mail net/rpc net/rpc/jsonrpc net/smtp net/textproto net/url
//go:generate ../cmd/goexports/goexports os os/exec os/signal os/user
//go:generate ../cmd/goexports/goexports path path/filepath reflect regexp regexp/syntax
//go:generate ../cmd/goexports/goexports runtime runtime/debug
//go:generate ../cmd/goexports/goexports sort strconv strings sync sync/atomic
//go:generate ../cmd/goexports/goexports text/scanner text/tabwriter text/template text/template/parse
//go:generate ../cmd/goexports/goexports time unicode unicode/utf16 unicode/utf8
//go:generate go run ./internal/genstdlib/main.go