cmd/go: don't run cgo compiler version probe under -toolexec

The -toolexec build flag wraps each toolchain program (compile, asm,
link, ...) in a user-supplied program, e.g. to profile them or swap in
alternates.

To know when to rebuild a cgo package, the go command records the C,
C++, and Fortran compiler versions by probing each compiler:

	cc -### -x c -c -

That probe was run under -toolexec, but the real cgo compiles never
are, so the wrapper saw the probe and none of the actual compiles.

Probe under -toolexec only for gccgo, a Go toolchain compiler treated
like cmd/compile. Leave the C, C++, and Fortran probes unwrapped.

Fixes #64580

Author: harjothkhara

src/cmd/go/internal/work/buildid.go

@@ -193,7 +193,7 @@ func (b *Builder) toolID(name string) string {
 //
 // For these tools we have no -V=full option to dump the build ID,
 // but we can run the tool with -v -### to reliably get the compiler proper
-// and hash that. That will work in the presence of -toolexec.
+// and hash that.
 //
 // In order to get reproducible builds for released compilers, we
 // detect a released compiler by the absence of "experimental" in the
@@ -203,7 +203,14 @@ func (b *Builder) toolID(name string) string {
 // The caller assumes that stat of the exe can be used, combined with the id,
 // to detect changes in the underlying compiler. The returned exe can be empty,
 // which means to rely only on the id.
-func (b *Builder) gccToolID(name, language string) (id, exe string, err error) {
+//
+// useToolexec reports whether the version probe should be run under
+// cfg.BuildToolexec. It is true only for gccgo, which is treated as a Go
+// toolchain compiler (like cmd/compile) and so is probed under -toolexec. It is
+// false for the C, C++, and Fortran compilers driven by cgo: those are not
+// otherwise invoked under -toolexec, so probing them under -toolexec would be
+// inconsistent (see go.dev/issue/64580).
+func (b *Builder) gccToolID(name, language string, useToolexec bool) (id, exe string, err error) {
 	//TODO: Use par.Cache instead of a mutex and a map. See Builder.toolID.
 	key := name + "." + language
 	b.id.Lock()
@@ -218,7 +225,10 @@ func (b *Builder) gccToolID(name, language string) (id, exe string, err error) {
 	// Invoke the driver with -### to see the subcommands and the
 	// version strings. Use -x to set the language. Pretend to
 	// compile an empty file on standard input.
-	cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-")
+	cmdline := str.StringList(name, "-###", "-x", language, "-c", "-")
+	if useToolexec {
+		cmdline = str.StringList(cfg.BuildToolexec, cmdline)
+	}
 	cmd := exec.Command(cmdline[0], cmdline[1:]...)
 	// Force untranslated output so that we see the string "version".
 	cmd.Env = append(os.Environ(), "LC_ALL=C")

src/cmd/go/internal/work/exec.go

@@ -322,15 +322,15 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
 		fmt.Fprintf(h, "CC=%q %q %q %q\n", ccExe, cppflags, cflags, ldflags)
 		// Include the C compiler tool ID so that if the C
 		// compiler changes we rebuild the package.
-		if ccID, _, err := b.gccToolID(ccExe[0], "c"); err == nil {
+		if ccID, _, err := b.gccToolID(ccExe[0], "c", false); err == nil {
 			fmt.Fprintf(h, "CC ID=%q\n", ccID)
 		} else {
 			fmt.Fprintf(h, "CC ID ERROR=%q\n", err)
 		}
 		if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 {
 			cxxExe := b.cxxExe()
 			fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags)
-			if cxxID, _, err := b.gccToolID(cxxExe[0], "c++"); err == nil {
+			if cxxID, _, err := b.gccToolID(cxxExe[0], "c++", false); err == nil {
 				fmt.Fprintf(h, "CXX ID=%q\n", cxxID)
 			} else {
 				fmt.Fprintf(h, "CXX ID ERROR=%q\n", err)
@@ -339,7 +339,7 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
 		if len(p.FFiles) > 0 {
 			fcExe := b.fcExe()
 			fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags)
-			if fcID, _, err := b.gccToolID(fcExe[0], "f95"); err == nil {
+			if fcID, _, err := b.gccToolID(fcExe[0], "f95", false); err == nil {
 				fmt.Fprintf(h, "FC ID=%q\n", fcID)
 			} else {
 				fmt.Fprintf(h, "FC ID ERROR=%q\n", err)
@@ -395,15 +395,15 @@ func (b *Builder) buildActionID(a *Action) cache.ActionID {
 		}
 
 	case "gccgo":
-		id, _, err := b.gccToolID(BuildToolchain.compiler(), "go")
+		id, _, err := b.gccToolID(BuildToolchain.compiler(), "go", true)
 		if err != nil {
 			base.Fatalf("%v", err)
 		}
 		fmt.Fprintf(h, "compile %s %q %q\n", id, forcedGccgoflags, p.Internal.Gccgoflags)
 		fmt.Fprintf(h, "pkgpath %s\n", gccgoPkgpath(p))
 		fmt.Fprintf(h, "ar %q\n", BuildToolchain.(gccgoToolchain).ar())
 		if len(p.SFiles) > 0 {
-			id, _, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp")
+			id, _, _ = b.gccToolID(BuildToolchain.compiler(), "assembler-with-cpp", true)
 			// Ignore error; different assembler versions
 			// are unlikely to make any difference anyhow.
 			fmt.Fprintf(h, "asm %q\n", id)
@@ -1614,7 +1614,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) {
 		// Or external linker settings and flags?
 
 	case "gccgo":
-		id, _, err := b.gccToolID(BuildToolchain.linker(), "go")
+		id, _, err := b.gccToolID(BuildToolchain.linker(), "go", true)
 		if err != nil {
 			base.Fatalf("%v", err)
 		}
@@ -2743,7 +2743,7 @@ func (b *Builder) gccCompilerID(compiler string) (id cache.ActionID, ok bool) {
 	// The first one is the compiler executable we invoke.
 	// The second is the underlying compiler as reported by -v -###
 	// (see b.gccToolID implementation in buildid.go).
-	toolID, exe2, err := b.gccToolID(compiler, "c")
+	toolID, exe2, err := b.gccToolID(compiler, "c", false)
 	if err != nil {
 		return cache.ActionID{}, false
 	}

src/cmd/go/testdata/script/toolexec.txt

@@ -9,6 +9,9 @@ go build ./cmd/mytool
 # TODO(#27628): This should not be necessary.
 env GOCACHE=$WORK/gocache
 
+# mytool records here if it is ever run for a C-family compiler's -### probe.
+env TOOLEXEC_CDRIVER_LOG=$WORK/cdriver.log
+
 # Build the main package with our toolexec program. For each action, it will
 # print the tool's name and the TOOLEXEC_IMPORTPATH value. We expect to compile
 # each package once, and link the main package once.
@@ -24,6 +27,11 @@ stderr -count=1 '^compile'${GOEXE}' TOOLEXEC_IMPORTPATH="test/main/withasm"$'
 stderr -count=1 '^compile'${GOEXE}' TOOLEXEC_IMPORTPATH="test/main"$'
 stderr -count=1 '^link'${GOEXE}' TOOLEXEC_IMPORTPATH="test/main"$'
 
+# The C compiler used by cgo is not run under -toolexec, so the -### probe
+# used to compute its tool ID must not be run under -toolexec either.
+# See go.dev/issue/64580.
+[cgo] ! exists $WORK/cdriver.log
+
 # Test packages are a little bit trickier.
 # We have four variants of test/main, as reported by 'go list -test':
 #
@@ -112,6 +120,23 @@ func main() {
 	toolName := filepath.Base(tool)
 	if len(args) > 0 && args[0] == "-V=full" {
 		// We can't alter the version output.
+	} else if len(args) > 0 && args[0] == "-###" {
+		// The C, C++, and Fortran compilers driven by cgo are probed with
+		// -### to compute their tool ID. They are not otherwise run under
+		// -toolexec, so they must not be probed under -toolexec either.
+		// See go.dev/issue/64580.
+		//
+		// The probe's output is captured by the go command rather than
+		// streamed, so record the invocation in a file the test can inspect.
+		if log := os.Getenv("TOOLEXEC_CDRIVER_LOG"); log != "" {
+			f, err := os.OpenFile(log, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o666)
+			if err != nil {
+				fmt.Fprintln(os.Stderr, err)
+				os.Exit(1)
+			}
+			fmt.Fprintf(f, "%s -###\n", toolName)
+			f.Close()
+		}
 	} else {
 		// Print which tool we're running, and on what package.
 		fmt.Fprintf(os.Stdout, "%s TOOLEXEC_IMPORTPATH=%q\n", toolName, os.Getenv("TOOLEXEC_IMPORTPATH"))