Merge branch 'refs/heads/main' into add-langchain

Author: thinkdb1

README.md

@@ -81,14 +81,14 @@ Also there are several [**documents**](./docs) that you may find useful for eith
 | database/sql  | https://pkg.go.dev/database/sql                | -                     | -                     |
 | echo          | https://github.com/labstack/echo               | v4.0.0                | v4.12.0               |
 | elasticsearch | https://github.com/elastic/go-elasticsearch    | v8.4.0                | v8.15.0               |
-| fasthttp      | https://github.com/valyala/fasthttp            | v1.45.0               | v1.57.0               |
+| fasthttp      | https://github.com/valyala/fasthttp            | v1.45.0               | v1.59.0               |
 | fiber         | https://github.com/gofiber/fiber               | v2.43.0               | v2.52.5               |
 | gin           | https://github.com/gin-gonic/gin               | v1.7.0                | v1.10.0               |
 | gorestful     | https://github.com/emicklei/go-restful         | v3.7.0                | v3.12.1               |
 | go-redis      | https://github.com/redis/go-redis              | v9.0.5                | v9.5.1                |
 | go-redis v8   | https://github.com/redis/go-redis              | v8.11.0               | v8.11.5               |
 | gorm          | https://github.com/go-gorm/gorm                | v1.22.0               | v1.25.9               |
-| grpc          | https://google.golang.org/grpc                 | v1.44.0               | v1.68.2               |
+| grpc          | https://google.golang.org/grpc                 | v1.44.0               | v1.71.0               |
 | hertz         | https://github.com/cloudwego/hertz             | v0.8.0                | v0.9.2                |
 | kitex         | https://github.com/cloudwego/kitex             | v0.5.1                | v0.11.3               |
 | kratos        | https://github.com/go-kratos/kratos            | v2.6.3                | v2.8.2                |
@@ -126,4 +126,20 @@ to engage with us.
 These are only part of the companies using this project, for reference only. If you are using this project, please [add your company here](https://github.com/alibaba/opentelemetry-go-auto-instrumentation/issues/225) to tell us your scenario to make this project better.
 
 - <img src="./docs/alibaba.png" width="80">
-- <img src="./docs/aliyun.png" width="100">
+- <img src="./docs/aliyun.png" width="100">
+
+### Contributors
+
+<a href="https://github.com/alibaba/opentelemetry-go-auto-instrumentation/graphs/contributors">
+  <img alt="contributors" src="https://contrib.rocks/image?repo=alibaba/opentelemetry-go-auto-instrumentation"/>
+</a>
+
+# Star History
+
+[![Star History](https://api.star-history.com/svg?repos=alibaba/opentelemetry-go-auto-instrumentation&type=Date)](https://star-history.com/#alibaba/opentelemetry-go-auto-instrumentation&Date)
+
+<p align="right" style="font-size: 14px; color: #555; margin-top: 20px;">
+    <a href="#readme-top" style="text-decoration: none; color: #007bff; font-weight: bold;">
+        ↑ 返回顶部 ↑
+    </a>
+</p>

test/errors_test.go

@@ -48,12 +48,12 @@ func TestRunErrors(t *testing.T) {
 	}
 	re = regexp.MustCompile(".*OtelOnEnterTrampoline_p1.*")
 	matches = re.FindAllString(text, -1)
-	if len(matches) != 3 {
-		t.Fatalf("expecting 3 matches")
+	if len(matches) != 4 {
+		t.Fatalf("expecting 4 matches")
 	}
 	re = regexp.MustCompile(".*OtelOnExitTrampoline_p2.*")
 	matches = re.FindAllString(text, -1)
-	if len(matches) != 3 {
-		t.Fatalf("expecting 3 matches")
+	if len(matches) != 4 {
+		t.Fatalf("expecting 4 matches")
 	}
 }

test/errorstest/auxiliary/helper.go

@@ -21,8 +21,10 @@ func TestSkip() *int {
 func TestSkip2() int {
 	return 1024
 }
-func p1() {}
-func p2() {}
+func p1() { /*Test*/ }
+func p2() { // Test
+}
+
 func p3(arg1 int, arg2 bool, arg3 float64) (int, bool, float64) {
 	return arg1, arg2, arg3
 }

test/helloworld_test.go

@@ -51,7 +51,7 @@ func TestRunHelloworld(t *testing.T) {
 	ExpectContains(t, stderr, "BYD")
 
 	text := ReadInstrumentLog(t, filepath.Join("fmt", "print.go"))
-	re := regexp.MustCompile(".*OtelOnEnterTrampoline.*OtelOnExitTrampoline.*")
+	re := regexp.MustCompile("//line <generated>:1")
 	matches := re.FindAllString(text, -1)
 	if len(matches) < 1 {
 		t.Fatalf("expecting at least one match")

tool/instrument/inst_func.go

@@ -16,6 +16,7 @@ package instrument
 
 import (
 	"fmt"
+	"go/parser"
 	"path/filepath"
 	"regexp"
 	"sort"
@@ -28,15 +29,7 @@ import (
 )
 
 const (
-	TrampolineJumpIfDesc                 = "/* TRAMPOLINE_JUMP_IF */"
-	TrampolineJumpIfDescRegexp           = "/\\* TRAMPOLINE_JUMP_IF \\*/"
-	TrampolineNoNewlinePlaceholder       = "/* NO_NEWWLINE_PLACEHOLDER */"
-	TrampolineNoNewlinePlaceholderRegexp = "/\\* NO_NEWWLINE_PLACEHOLDER \\*/\n"
-	TrampolineSemicolonPlaceholder       = "/* SEMICOLON_PLACEHOLDER */"
-	TrampolineSemicolonPlaceholderRegexp = "/\\* SEMICOLON_PLACEHOLDER \\*/\n"
-)
-
-const (
+	TJumpLabel         = "/* TRAMPOLINE_JUMP_IF */"
 	OtelAPIFile        = "otel_api.go"
 	OtelTrampolineFile = "otel_trampoline.go"
 )
@@ -54,10 +47,15 @@ func (rp *RuleProcessor) copyOtelApi(pkgName string) error {
 
 func (rp *RuleProcessor) loadAst(filePath string) (*dst.File, error) {
 	file := rp.tryRelocated(filePath)
-	return util.ParseAstFromFile(file)
+	rp.parser = util.NewAstParser()
+	var err error
+	rp.target, err = rp.parser.ParseFile(file, parser.ParseComments)
+	return rp.target, err
 }
 
 func (rp *RuleProcessor) restoreAst(filePath string, root *dst.File) (string, error) {
+	rp.parser = nil
+	rp.target = nil
 	filePath = rp.tryRelocated(filePath)
 	name := filepath.Base(filePath)
 	newFile, err := util.WriteAstToFile(root, filepath.Join(rp.workDir, name))
@@ -87,7 +85,7 @@ func (rp *RuleProcessor) makeName(r *resource.InstFuncRule, onEnter bool) string
 func findJumpPoint(jumpIf *dst.IfStmt) *dst.BlockStmt {
 	// Multiple func rules may apply to the same function, we need to find the
 	// appropriate jump point to insert trampoline jump.
-	if len(jumpIf.Decs.If) == 1 && jumpIf.Decs.If[0] == TrampolineJumpIfDesc {
+	if len(jumpIf.Decs.If) == 1 && jumpIf.Decs.If[0] == TJumpLabel {
 		// Insert trampoline jump within the else block
 		elseBlock := jumpIf.Else.(*dst.BlockStmt)
 		if len(elseBlock.List) > 1 {
@@ -181,32 +179,9 @@ func (rp *RuleProcessor) insertTJump(t *resource.InstFuncRule,
 		ifStmt: tjump,
 		rule:   t,
 	})
-
-	// @@ Unfortunately, dst framework does not support fine-grained space control
-	// i.e. there is no way to generate all above AST into one line code, we have
-	// to manually format it. There we insert OtelNewlineTrampolineHolder anchor
-	// so that we aware of where we should remove trailing newline.
-	//	if .... { /* NO_NEWWLINE_PLACEHOLDER */
-	//	... /* NO_NEWWLINE_PLACEHOLDER */
-	//	} else { /* NO_NEWWLINE_PLACEHOLDER */
-	//	... /* SEMICOLON_PLACEHOLDER */
-	//	} /* NO_NEWWLINE_PLACEHOLDER */
-	//  NEW_LINE
-	{ // then block
-		callExpr := tjump.Body.List[0]
-		callExpr.Decorations().Start.Append(TrampolineNoNewlinePlaceholder)
-		callExpr.Decorations().End.Append(TrampolineSemicolonPlaceholder)
-		retStmt := tjump.Body.List[1]
-		retStmt.Decorations().End.Append(TrampolineNoNewlinePlaceholder)
-	}
-	{ // else block
-		deferStmt := tjump.Else.(*dst.BlockStmt).List[0]
-		deferStmt.Decorations().Start.Append(TrampolineNoNewlinePlaceholder)
-		deferStmt.Decorations().End.Append(TrampolineSemicolonPlaceholder)
-		tjump.Else.Decorations().End.Append(TrampolineNoNewlinePlaceholder)
-		tjump.Decs.If.Append(TrampolineJumpIfDesc) // Anchor label
-	}
-
+	// Add label for trampoline-jump-if. Note that the label will be cleared
+	// during optimization pass, to make it pretty in the generated code
+	tjump.Decs.If.Append(TJumpLabel)
 	// Find if there is already a trampoline-jump-if, insert new tjump if so,
 	// otherwise prepend to block body
 	found := false
@@ -222,10 +197,35 @@ func (rp *RuleProcessor) insertTJump(t *resource.InstFuncRule,
 		}
 	}
 	if !found {
-		// Outmost trampoline-jump-if may follow by user code right after else
-		// block, replacing the trailing newline mandatorily breaks the code,
-		// we need to insert extra new line to make replacement possible
-		tjump.Decorations().After = dst.EmptyLine
+		// Tag the trampoline-jump-if with a special line directive so that
+		// debugger can show the correct line number
+		tjump.Decs.Before = dst.NewLine
+		tjump.Decs.Start.Append("//line <generated>:1")
+		pos := rp.parser.FindPosition(funcDecl.Body)
+		if len(funcDecl.Body.List) > 0 {
+			// It does happens because we may insert raw code snippets at the
+			// function entry. These dynamically generated nodes do not have
+			// corresponding node positions. We need to keep looking downward
+			// until we find a node that contains position information, and then
+			// annotate it with a line directive.
+			for i := 0; i < len(funcDecl.Body.List); i++ {
+				stmt := funcDecl.Body.List[i]
+				pos = rp.parser.FindPosition(stmt)
+				if !pos.IsValid() {
+					continue
+				}
+				tag := fmt.Sprintf("//line %s", pos.String())
+				stmt.Decorations().Before = dst.NewLine
+				stmt.Decorations().Start.Append(tag)
+			}
+		} else {
+			pos = rp.parser.FindPosition(funcDecl.Body)
+			tag := fmt.Sprintf("//line %s", pos.String())
+			empty := util.EmptyStmt()
+			empty.Decs.Before = dst.NewLine
+			empty.Decs.Start.Append(tag)
+			funcDecl.Body.List = append(funcDecl.Body.List, empty)
+		}
 		funcDecl.Body.List = append([]dst.Stmt{tjump}, funcDecl.Body.List...)
 	}
 
@@ -237,38 +237,21 @@ func (rp *RuleProcessor) insertTJump(t *resource.InstFuncRule,
 	return nil
 }
 
-func (rp *RuleProcessor) inliningTJump(filePath string) error {
-	text, err := util.ReadFile(filePath)
-	if err != nil {
-		return err
-	}
-	// Remove trailing newline
-	re := regexp.MustCompile(TrampolineNoNewlinePlaceholderRegexp)
-	text = re.ReplaceAllString(text, " ")
-	// Replace with semicolon
-	re = regexp.MustCompile(TrampolineSemicolonPlaceholderRegexp)
-	text = re.ReplaceAllString(text, ";")
-	// Remove trampoline jump if ideitifiers
-	re = regexp.MustCompile(TrampolineJumpIfDescRegexp)
-	text = re.ReplaceAllString(text, "")
-	// All done, persist to file
-	_, err = util.WriteFile(filePath, text)
-	return err
-}
-
 func (rp *RuleProcessor) insertRaw(r *resource.InstFuncRule, decl *dst.FuncDecl) error {
 	util.Assert(r.OnEnter != "" || r.OnExit != "", "sanity check")
 	if r.OnEnter != "" {
 		// Prepend raw code snippet to function body for onEnter
-		onEnterSnippet, err := util.ParseAstFromSnippet(r.OnEnter)
+		p := util.NewAstParser()
+		onEnterSnippet, err := p.ParseSnippet(r.OnEnter)
 		if err != nil {
 			return err
 		}
 		decl.Body.List = append(onEnterSnippet, decl.Body.List...)
 	}
 	if r.OnExit != "" {
 		// Use defer func(){ raw_code_snippet }() for onExit
-		onExitSnippet, err := util.ParseAstFromSnippet(
+		p := util.NewAstParser()
+		onExitSnippet, err := p.ParseSnippet(
 			fmt.Sprintf("defer func(){ %s }()", r.OnExit),
 		)
 		if err != nil {
@@ -301,8 +284,8 @@ func sortFuncRules(fnRules []*resource.InstFuncRule) []*resource.InstFuncRule {
 
 func (rp *RuleProcessor) writeTrampoline(pkgName string) error {
 	// Prepare trampoline code header
-	code := "package " + pkgName
-	trampoline, err := util.ParseAstFromSource(code)
+	p := util.NewAstParser()
+	trampoline, err := p.ParseSource("package " + pkgName)
 	if err != nil {
 		return err
 	}
@@ -319,6 +302,18 @@ func (rp *RuleProcessor) writeTrampoline(pkgName string) error {
 	return nil
 }
 
+func (rp *RuleProcessor) enableLineDirective(filePath string) error {
+	text, err := util.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+	re := regexp.MustCompile(".*//line ")
+	text = re.ReplaceAllString(text, "//line ")
+	// All done, persist to file
+	_, err = util.WriteFile(filePath, text)
+	return err
+}
+
 func (rp *RuleProcessor) applyFuncRules(bundle *resource.RuleBundle) (err error) {
 	// Nothing to do if no func rules
 	if len(bundle.File2FuncRules) == 0 {
@@ -338,7 +333,6 @@ func (rp *RuleProcessor) applyFuncRules(bundle *resource.RuleBundle) (err error)
 		if err != nil {
 			return err
 		}
-		rp.target = astRoot
 		rp.trampolineJumps = make([]*TJump, 0)
 		for fnName, rules := range fn2rules {
 			for _, decl := range astRoot.Decls {
@@ -378,9 +372,9 @@ func (rp *RuleProcessor) applyFuncRules(bundle *resource.RuleBundle) (err error)
 		if err != nil {
 			return err
 		}
-		// Wait, all above code snippets should be inlined to new line to
-		// avoid potential misbehaviors during debugging
-		err = rp.inliningTJump(filePath)
+		// Line directive must be placed at the beginning of the line, otherwise
+		// it will be ignored by the compiler
+		err = rp.enableLineDirective(filePath)
 		if err != nil {
 			return err
 		}

tool/instrument/instrument.go

@@ -37,7 +37,8 @@ import (
 type RuleProcessor struct {
 	packageName     string
 	workDir         string
-	target          *dst.File // The target file to be instrumented
+	target          *dst.File       // The target file to be instrumented
+	parser          *util.AstParser // The parser for the target file
 	compileArgs     []string
 	rule2Suffix     map[*resource.InstFuncRule]string
 	rawFunc         *dst.FuncDecl