feat: support vendor build (#258)

* feat: support vendor build

* more comment

Author: y1yang0

test/helloworld/go.mod

@@ -6,4 +6,15 @@ replace github.com/alibaba/opentelemetry-go-auto-instrumentation => ../../../ope
 
 replace github.com/alibaba/opentelemetry-go-auto-instrumentation/test/verifier => ../../../opentelemetry-go-auto-instrumentation/test/verifier
 
-require golang.org/x/time v0.5.0
+require (
+	go.opentelemetry.io/otel v1.33.0
+	golang.org/x/time v0.5.0
+)
+
+require (
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel/metric v1.33.0 // indirect
+	go.opentelemetry.io/otel/trace v1.33.0 // indirect
+)

test/helloworld/go.sum

@@ -1,13 +1,25 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts=
-go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc=
-go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w=
-go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ=
-go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc=
-go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
+go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
+go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ=
+go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
+go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s=
+go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

test/helloworld_test.go

@@ -15,6 +15,7 @@
 package test
 
 import (
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"testing"
@@ -55,21 +56,33 @@ func TestRunHelloworld(t *testing.T) {
 	}
 }
 
-// FIXME: Support vendor build mode
-// func TestBuildHelloworldWithVendor1(t *testing.T) {
-// 	UseApp(HelloworldAppName)
-// 	util.RunCmd("go", "mod", "vendor")
-// 	RunGoBuild(t, "-debuglog", "go", "build")
-// }
+func runModVendor(t *testing.T) {
+	cmd := exec.Command("go", "mod", "tidy")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("go mod tidy failed: %v\n%s", err, out)
+	}
+	cmd = exec.Command("go", "mod", "vendor")
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("go mod vendor failed: %v\n%s", err, out)
+	}
+}
 
-// func TestBuildHelloworldWithVendor2(t *testing.T) {
-// 	UseApp(HelloworldAppName)
-// 	util.RunCmd("go", "mod", "vendor")
-// 	RunGoBuild(t, "-debuglog", "go", "build", "-mod=vendor")
-// }
+func TestBuildHelloworldWithVendor1(t *testing.T) {
+	UseApp(HelloworldAppName)
+	runModVendor(t)
+	RunGoBuild(t, "go", "build")
+}
 
-// func TestBuildHelloworldWithVendor3(t *testing.T) {
-// 	UseApp(HelloworldAppName)
-// 	util.RunCmd("go", "mod", "vendor")
-// 	RunGoBuild(t, "-debuglog", "go", "build", "-mod", "vendor")
-// }
+func TestBuildHelloworldWithVendor2(t *testing.T) {
+	UseApp(HelloworldAppName)
+	runModVendor(t)
+	RunGoBuild(t, "go", "build", "-mod=vendor")
+}
+
+func TestBuildHelloworldWithVendor3(t *testing.T) {
+	UseApp(HelloworldAppName)
+	runModVendor(t)
+	RunGoBuild(t, "go", "build", "-mod", "vendor")
+}

tool/preprocess/match.go

@@ -15,9 +15,11 @@
 package preprocess
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/alibaba/opentelemetry-go-auto-instrumentation/pkg"
@@ -35,6 +37,7 @@ const (
 
 type ruleMatcher struct {
 	availableRules map[string][]resource.InstRule
+	moduleVersions map[string]string // vendor used only
 }
 
 func newRuleMatcher() *ruleMatcher {
@@ -172,7 +175,16 @@ func (rm *ruleMatcher) match(cmdArgs []string) *resource.RuleBundle {
 			continue
 		}
 		file := candidate
+
+		// If it's a vendor build, we need to extract the version of the module
+		// from vendor/modules.txt, otherwise we find the version from source
+		// code file path
 		version := shared.ExtractVersion(file)
+		if rm.moduleVersions != nil {
+			if v, ok := rm.moduleVersions[importPath]; ok {
+				version = v
+			}
+		}
 
 		for i := len(availables) - 1; i >= 0; i-- {
 			rule := availables[i]
@@ -280,6 +292,47 @@ func findFlagValue(cmd []string, flag string) string {
 	return ""
 }
 
+func parseVendorModules() (map[string]string, error) {
+	util.Assert(shared.IsVendorBuild(), "why not otherwise")
+	vendorFile := filepath.Join("vendor", "modules.txt")
+	if exist, _ := util.PathExists(vendorFile); !exist {
+		return nil, fmt.Errorf("vendor/modules.txt not found")
+	}
+	// Read the vendor/modules.txt file line by line and parse it in form of
+	// #ImportPath Version
+	file, err := os.Open(vendorFile)
+	if err != nil {
+		return nil, err
+	}
+	defer func(dryRunLog *os.File) {
+		err := dryRunLog.Close()
+		if err != nil {
+			util.Log("Failed to close dry run log file: %v", err)
+		}
+	}(file)
+
+	vendorModules := make(map[string]string)
+	scanner := bufio.NewScanner(file)
+	// 10MB should be enough to accommodate most long line
+	buffer := make([]byte, 0, 10*1024*1024)
+	scanner.Buffer(buffer, cap(buffer))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "# ") {
+			parts := strings.Split(line, " ")
+			if len(parts) == 3 {
+				util.Assert(parts[0] == "#", "sanity check")
+				util.Assert(strings.HasPrefix(parts[2], "v"), "sanity check")
+				vendorModules[parts[1]] = parts[2]
+			}
+		}
+	}
+	if err = scanner.Err(); err != nil {
+		return nil, err
+	}
+	return vendorModules, nil
+}
+
 func runMatch(matcher *ruleMatcher, cmd string, ch chan *resource.RuleBundle) {
 	bundle := matcher.match(shared.SplitCmds(cmd))
 	ch <- bundle
@@ -288,6 +341,20 @@ func runMatch(matcher *ruleMatcher, cmd string, ch chan *resource.RuleBundle) {
 func (dp *DepProcessor) matchRules(compileCmds []string) error {
 	defer util.PhaseTimer("Match")()
 	matcher := newRuleMatcher()
+
+	// If we are in vendor mode, we need to parse the vendor/modules.txt file
+	// to get the version of each module for future matching
+	if dp.vendorBuild {
+		modules, err := parseVendorModules()
+		if err != nil {
+			return fmt.Errorf("failed to parse vendor/modules.txt: %w", err)
+		}
+		if config.GetConf().Verbose {
+			util.Log("Vendor modules: %v", modules)
+		}
+		matcher.moduleVersions = modules
+	}
+
 	// Find used instrumentation rule according to compile commands
 	ch := make(chan *resource.RuleBundle)
 	for _, cmd := range compileCmds {

tool/preprocess/preprocess.go

@@ -108,6 +108,7 @@ type DepProcessor struct {
 	rule2Dir         map[*resource.InstFuncRule]string
 	ruleCache        embed.FS
 	goBuildCmd       []string
+	vendorBuild      bool
 }
 
 func newDepProcessor() *DepProcessor {
@@ -118,6 +119,7 @@ func newDepProcessor() *DepProcessor {
 		importCandidates: nil,
 		rule2Dir:         map[*resource.InstFuncRule]string{},
 		ruleCache:        pkg.ExportRuleCache(),
+		vendorBuild:      shared.IsVendorBuild(),
 	}
 	// There is a tricky, all arguments after the tool itself are saved for
 	// later use, which means the subcommand "go build" are also  included
@@ -473,6 +475,12 @@ func runModTidy() error {
 	return err
 }
 
+func runModVendor() error {
+	out, err := util.RunCmdOutput("go", "mod", "vendor")
+	util.Log("Run go mod vendor: %v", out)
+	return err
+}
+
 func runGoGet(dep string) error {
 	out, err := util.RunCmdOutput("go", "get", dep)
 	util.Log("Run go get %v: %v", dep, out)
@@ -596,15 +604,6 @@ func precheck() error {
 	if err != nil {
 		return fmt.Errorf("failed to check go.mod: %w", err)
 	}
-	// Check if the project is build with vendor mode
-	projRoot, err := shared.GetGoModDir()
-	if err != nil {
-		return fmt.Errorf("failed to get project root: %w", err)
-	}
-	vendor := filepath.Join(projRoot, shared.VendorDir)
-	if exist, _ := util.PathExists(vendor); exist {
-		return fmt.Errorf("vendor mode is not supported")
-	}
 
 	// Check if the build arguments is sane
 	if len(os.Args) < 3 {
@@ -678,6 +677,13 @@ func (dp *DepProcessor) setupDeps() error {
 		return fmt.Errorf("failed to run mod tidy: %w", err)
 	}
 
+	if dp.vendorBuild {
+		err = runModVendor()
+		if err != nil {
+			return fmt.Errorf("failed to run mod vendor: %w", err)
+		}
+	}
+
 	// Run dry build to the build blueprint
 	err = runDryBuild(dp.goBuildCmd)
 	if err != nil {
@@ -766,6 +772,13 @@ func Preprocess() error {
 			return fmt.Errorf("failed to run mod tidy: %w", err)
 		}
 
+		if dp.vendorBuild {
+			err = runModVendor()
+			if err != nil {
+				return fmt.Errorf("failed to run mod vendor: %w", err)
+			}
+		}
+
 		// 	// Retain otel rules and modified user files for debugging
 		dp.saveDebugFiles()
 	}

tool/shared/shared.go

@@ -330,3 +330,15 @@ func SplitCmds(input string) []string {
 	}
 	return args
 }
+
+func IsVendorBuild() bool {
+	projRoot, err := GetGoModDir()
+	if err != nil {
+		return false
+	}
+	vendor := filepath.Join(projRoot, VendorDir)
+	if exist, _ := util.PathExists(vendor); exist {
+		return true
+	}
+	return false
+}