Fix dts_lexer: bufio.Scanner: token too long (#1139)

Author: ije

server/cjs_module_lexer.go

@@ -98,19 +98,26 @@ func cjsModuleLexer(b *BuildContext, cjsEntry string) (ret cjsModuleLexerResult,
 		cmd := exec.Command(
 			path.Join(config.WorkDir, "bin/deno"),
 			"run",
+			"--allow-env",
 			"--no-prompt",
 			"--no-config",
 			"--no-lock",
 			"--quiet",
 			js)
-		cmd.Env = append(os.Environ(), "DENO_NO_UPDATE_CHECK=1", "DENO_NO_PACKAGE_JSON=1")
-		data, err = cmd.Output()
+		cmd.Env = []string{"DENO_NO_UPDATE_CHECK=1"}
+		data, err = cmd.CombinedOutput()
 		if err != nil {
+			msg := err.Error()
+			if data != nil {
+				msg = string(data)
+			}
+			err = errors.New("cjsModuleLexer(fallback mode): " + msg)
 			return
 		}
 		var namedExports []string
 		err = json.Unmarshal(data, &namedExports)
 		if err != nil {
+			err = errors.New("cjsModuleLexer(fallback mode): " + err.Error())
 			return
 		}
 		for _, name := range namedExports {

server/dts_lexer.go

@@ -42,6 +42,7 @@ func parseDts(r io.Reader, w *bytes.Buffer, resolve func(specifier string, kind
 	var multiLineComment bool
 	var importOrExportDeclFound bool
 	scanner := bufio.NewScanner(r)
+	scanner.Buffer(nil, 1024*1024)
 	for scanner.Scan() {
 		line, trimedSpaces := trimSpace(scanner.Bytes())
 		w.Write(trimedSpaces)
@@ -99,6 +100,7 @@ func parseDts(r io.Reader, w *bytes.Buffer, resolve func(specifier string, kind
 		} else {
 			var i int
 			stmtScanner := bufio.NewScanner(bytes.NewReader(line))
+			stmtScanner.Buffer(nil, 1024*1024)
 			stmtScanner.Split(splitJSStmt)
 			for stmtScanner.Scan() {
 				if i > 0 {
@@ -205,7 +207,7 @@ func parseDts(r io.Reader, w *bytes.Buffer, resolve func(specifier string, kind
 func splitJSStmt(data []byte, atEOF bool) (advance int, token []byte, err error) {
 	var commentScope bool
 	var stringScope byte
-	for i := 0; i < len(data); i++ {
+	for i := range data {
 		var prev, next byte
 		if i > 0 {
 			prev = data[i-1]

server/loader_implements.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"os"
 	"os/exec"
 	"path"
 	"strconv"
@@ -174,7 +173,7 @@ func resolveVueVersion(npmrc *NpmRC, importMap importmap.ImportMap) (vueVersion
 }
 
 func generateUnoCSS(npmrc *NpmRC, configCSS string, content string) (out *LoaderOutput, err error) {
-	loaderVersion := "0.5.0-beta.3"
+	loaderVersion := "0.5.0"
 	loaderExecPath := path.Join(config.WorkDir, "bin", "unocss-"+loaderVersion)
 
 	err = doOnce(loaderExecPath, func() (err error) {
@@ -282,14 +281,15 @@ func compileUnocssLoader(npmrc *NpmRC, loaderVersion string, loaderExecPath stri
 		"--no-check",
 		"--include=jsr:@std/tar/untar-stream",
 		"--no-prompt",
+		"--allow-env",
 		"--allow-read="+config.WorkDir+"/cache",
 		"--allow-write="+config.WorkDir+"/cache",
 		"--allow-net=registry.npmjs.org,fonts.googleapis.com",
 		"--quiet",
 		"--output", loaderExecPath,
 		path.Join(wd, "loader.js"),
 	)
-	cmd.Env = append(os.Environ(), "DENO_NO_UPDATE_CHECK=1", "DENO_NO_PACKAGE_JSON=1")
+	cmd.Env = []string{"DENO_NO_UPDATE_CHECK=1"}
 	_, err = cmd.Output()
 	if err != nil {
 		err = fmt.Errorf("failed to compile %s: %s", path.Base(loaderExecPath), err.Error())

server/router.go

@@ -1635,8 +1635,8 @@ func esmRouter(db Database, buildStorage storage.Storage, logger *log.Logger) re
 					// then re-build the module
 					key := npmrc.zoneId + ":" + build.Path()
 					db.Delete(key)
-					cacheStore.Delete("lru:" + key)
-					return rex.Status(500, "Storage error")
+					cacheLRU.Remove(key)
+					return rex.Status(500, "Storage error, please try again")
 				}
 				return rex.Status(500, err.Error())
 			}

web/internal/loader.js

@@ -143,7 +143,7 @@ async function unocss(_id, content, config) {
   }
   let uno = once.unoGenerators.get(generatorId);
   if (!uno || uno.configCSS !== config?.css) {
-    uno = import("npm:@esm.sh/unocss@0.5.0-beta.3").then(({ init }) => init({ configCSS: config?.css }));
+    uno = import("npm:@esm.sh/unocss@0.5.0").then(({ init }) => init({ configCSS: config?.css }));
     uno.configCSS = config?.css;
     once.unoGenerators.set(generatorId, uno);
   }