Experimental API generator and example module support

Author: stephanos

cmd/README.md

@@ -20,3 +20,21 @@ For structs we generate implementations of:
 For enumerated types we generate a `FromString` method that instantiates the enum from either the
 old `PascalCase` string we have always supported or the `SCREAMING_SNAKE` string generated by
 protojson.
+
+### experimentgen
+
+This tool generates the small experimental `api-go` side modules used to expose either additive
+service RPCs or wrappers for new fields on existing stable messages under `experimental/<name>`.
+
+Example:
+
+`go run ./cmd/experimentgen -variant example -api-sha <api-commit>`
+
+The command writes to `experimental/<variant>` and clears that target before regenerating it.
+
+Generated modules can contain:
+- `register.go` and `client.go` for additive RPCs such as `Echo`
+- `wrapper.go` for accessors that read and write experimental fields on stable messages via
+  protobuf unknown fields, such as `StartWorkflowExecutionRequest.foo`
+- `enum.go` for additive values on existing stable enums, such as
+  `WORKFLOW_ID_CONFLICT_POLICY_FOO`

cmd/experimentgen/descriptors.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/descriptorpb"
+)
+
+type methodInfo struct {
+	Method       string
+	RequestName  string
+	ResponseName string
+}
+
+type protoFieldInfo struct {
+	Name   string
+	Number int
+}
+
+type messageDef struct {
+	Name   string
+	Fields []protoFieldInfo
+}
+
+type descriptorSnapshot struct {
+	WorkflowServiceMethods map[string]methodInfo
+	WorkflowMessages       map[string]messageDef
+	WorkflowDescriptors    map[string]*descriptorpb.DescriptorProto
+	WorkflowEnums          map[string]map[string]int32
+	WorkflowPackage        string
+	WorkflowRequestFile    *descriptorpb.FileDescriptorProto
+	DescriptorFiles        []*descriptorpb.FileDescriptorProto
+}
+
+func (g generator) loadSnapshot(apiRepo string, sourceSHA string, stableRoot string) (descriptorSnapshot, error) {
+	worktreeDir, err := os.MkdirTemp("", "experimentgen-api-*")
+	if err != nil {
+		return descriptorSnapshot{}, err
+	}
+	cleanup := func() {
+		_ = g.run("", "git", "-C", apiRepo, "worktree", "remove", "--force", worktreeDir)
+		_ = os.RemoveAll(worktreeDir)
+	}
+	defer cleanup()
+
+	if err := g.run("", "git", "-C", apiRepo, "worktree", "add", "--detach", worktreeDir, sourceSHA); err != nil {
+		return descriptorSnapshot{}, err
+	}
+
+	targets := []string{
+		filepath.ToSlash(filepath.Join(stableRoot, "workflowservice/v1/service.proto")),
+		filepath.ToSlash(filepath.Join(stableRoot, "workflowservice/v1/request_response.proto")),
+		filepath.ToSlash(filepath.Join(stableRoot, "enums/v1/workflow.proto")),
+	}
+
+	descPath := filepath.Join(worktreeDir, "experimental-descriptor.pb")
+	args := append([]string{
+		"-I", worktreeDir,
+		"--include_imports",
+		"--descriptor_set_out=" + descPath,
+	}, targets...)
+	if err := g.run(worktreeDir, "protoc", args...); err != nil {
+		return descriptorSnapshot{}, err
+	}
+
+	blob, err := os.ReadFile(descPath)
+	if err != nil {
+		return descriptorSnapshot{}, err
+	}
+	var set descriptorpb.FileDescriptorSet
+	if err := proto.Unmarshal(blob, &set); err != nil {
+		return descriptorSnapshot{}, err
+	}
+	return snapshotFromDescriptorSet(&set, stableRoot)
+}
+
+func snapshotFromDescriptorSet(set *descriptorpb.FileDescriptorSet, stableRoot string) (descriptorSnapshot, error) {
+	snapshot := descriptorSnapshot{
+		WorkflowServiceMethods: make(map[string]methodInfo),
+		WorkflowMessages:       make(map[string]messageDef),
+		WorkflowDescriptors:    make(map[string]*descriptorpb.DescriptorProto),
+		WorkflowEnums:          make(map[string]map[string]int32),
+		DescriptorFiles:        cloneFileDescriptors(set.File),
+	}
+
+	serviceName := filepath.ToSlash(filepath.Join(stableRoot, "workflowservice/v1/service.proto"))
+	requestResponseName := filepath.ToSlash(filepath.Join(stableRoot, "workflowservice/v1/request_response.proto"))
+	enumName := filepath.ToSlash(filepath.Join(stableRoot, "enums/v1/workflow.proto"))
+
+	var serviceFile, requestFile, enumFile *descriptorpb.FileDescriptorProto
+	for _, file := range set.File {
+		switch file.GetName() {
+		case serviceName:
+			serviceFile = file
+		case requestResponseName:
+			requestFile = file
+		case enumName:
+			enumFile = file
+		}
+	}
+	if serviceFile == nil || requestFile == nil || enumFile == nil {
+		return descriptorSnapshot{}, fmt.Errorf("descriptor set missing required files")
+	}
+
+	for _, service := range serviceFile.GetService() {
+		if service.GetName() != "WorkflowService" {
+			continue
+		}
+		for _, method := range service.GetMethod() {
+			name := method.GetName()
+			snapshot.WorkflowServiceMethods[name] = methodInfo{
+				Method:       name,
+				RequestName:  trimDescriptorName(method.GetInputType()),
+				ResponseName: trimDescriptorName(method.GetOutputType()),
+			}
+		}
+	}
+
+	for _, msg := range requestFile.GetMessageType() {
+		snapshot.WorkflowMessages[msg.GetName()] = descriptorMessageDef(msg)
+		snapshot.WorkflowDescriptors[msg.GetName()] = proto.Clone(msg).(*descriptorpb.DescriptorProto)
+	}
+	snapshot.WorkflowPackage = requestFile.GetPackage()
+	snapshot.WorkflowRequestFile = proto.Clone(requestFile).(*descriptorpb.FileDescriptorProto)
+
+	for _, enum := range enumFile.GetEnumType() {
+		values := make(map[string]int32, len(enum.GetValue()))
+		for _, value := range enum.GetValue() {
+			values[value.GetName()] = value.GetNumber()
+		}
+		snapshot.WorkflowEnums[enum.GetName()] = values
+	}
+
+	return snapshot, nil
+}
+
+func descriptorMessageDef(msg *descriptorpb.DescriptorProto) messageDef {
+	fields := make([]protoFieldInfo, 0, len(msg.GetField()))
+	for _, field := range msg.GetField() {
+		fields = append(fields, protoFieldInfo{
+			Name:   field.GetName(),
+			Number: int(field.GetNumber()),
+		})
+	}
+	return messageDef{
+		Name:   msg.GetName(),
+		Fields: fields,
+	}
+}
+
+func trimDescriptorName(name string) string {
+	name = strings.TrimPrefix(name, ".")
+	if idx := strings.LastIndex(name, "."); idx >= 0 {
+		return name[idx+1:]
+	}
+	return name
+}
+
+func cloneFileDescriptors(files []*descriptorpb.FileDescriptorProto) []*descriptorpb.FileDescriptorProto {
+	cloned := make([]*descriptorpb.FileDescriptorProto, 0, len(files))
+	for _, file := range files {
+		cloned = append(cloned, proto.Clone(file).(*descriptorpb.FileDescriptorProto))
+	}
+	return cloned
+}

cmd/experimentgen/diff.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"slices"
+	"strings"
+
+	"go.temporal.io/api/internal/strcase"
+)
+
+const experimentalNumberStart = 1000
+
+type descriptorChanges struct {
+	Methods  []methodInfo
+	Overlays []messageOverlay
+	Enums    []enumInfo
+}
+
+type messageOverlay struct {
+	StableMessage  string
+	OverlayMessage string
+	SourceMessage  string
+	FieldName      string
+	SourceName     string
+	SourceNumber   int
+	FieldNumber    int
+}
+
+type messageOverlayGroup struct {
+	StableMessage  string
+	OverlayMessage string
+	VariableName   string
+	Fields         []messageOverlay
+}
+
+type enumInfo struct {
+	StableEnum   string
+	SourceEnum   string
+	ValueName    string
+	SourceNumber int
+	ValueNumber  int
+}
+
+func collectChanges(base descriptorSnapshot, source descriptorSnapshot) descriptorChanges {
+	changes := descriptorChanges{
+		Methods:  make([]methodInfo, 0),
+		Overlays: make([]messageOverlay, 0),
+		Enums:    make([]enumInfo, 0),
+	}
+
+	for name, method := range source.WorkflowServiceMethods {
+		if _, exists := base.WorkflowServiceMethods[name]; exists {
+			continue
+		}
+		changes.Methods = append(changes.Methods, method)
+	}
+
+	for messageName, sourceMessage := range source.WorkflowMessages {
+		baseMessage, ok := base.WorkflowMessages[messageName]
+		if !ok {
+			continue
+		}
+		usedNumbers := make(map[int]struct{}, len(baseMessage.Fields))
+		baseFields := make(map[string]protoFieldInfo, len(baseMessage.Fields))
+		for _, field := range baseMessage.Fields {
+			usedNumbers[field.Number] = struct{}{}
+			baseFields[field.Name] = field
+		}
+		addedFields := make([]protoFieldInfo, 0, len(sourceMessage.Fields))
+		for _, field := range sourceMessage.Fields {
+			if _, exists := baseFields[field.Name]; exists {
+				continue
+			}
+			addedFields = append(addedFields, field)
+		}
+		slices.SortFunc(addedFields, func(a protoFieldInfo, b protoFieldInfo) int {
+			return a.Number - b.Number
+		})
+		nextNumber := nextAvailableNumber(usedNumbers)
+		for _, field := range addedFields {
+			changes.Overlays = append(changes.Overlays, messageOverlay{
+				StableMessage:  messageName,
+				OverlayMessage: messageName + "Overlay",
+				SourceMessage:  messageName,
+				FieldName:      strcase.ToCamel(field.Name),
+				SourceName:     field.Name,
+				SourceNumber:   field.Number,
+				FieldNumber:    nextNumber,
+			})
+			usedNumbers[nextNumber] = struct{}{}
+			nextNumber = nextAvailableNumber(usedNumbers)
+		}
+	}
+
+	for enumName, sourceValues := range source.WorkflowEnums {
+		baseValues, ok := base.WorkflowEnums[enumName]
+		if !ok {
+			continue
+		}
+		usedNumbers := make(map[int]struct{}, len(baseValues))
+		for _, number := range baseValues {
+			usedNumbers[int(number)] = struct{}{}
+		}
+		names := make([]string, 0, len(sourceValues))
+		for name := range sourceValues {
+			if _, exists := baseValues[name]; exists {
+				continue
+			}
+			names = append(names, name)
+		}
+		slices.Sort(names)
+		nextNumber := nextAvailableNumber(usedNumbers)
+		for _, name := range names {
+			changes.Enums = append(changes.Enums, enumInfo{
+				StableEnum:   enumName,
+				SourceEnum:   enumName,
+				ValueName:    name,
+				SourceNumber: int(sourceValues[name]),
+				ValueNumber:  nextNumber,
+			})
+			usedNumbers[nextNumber] = struct{}{}
+			nextNumber = nextAvailableNumber(usedNumbers)
+		}
+	}
+
+	slices.SortFunc(changes.Methods, func(a methodInfo, b methodInfo) int {
+		return strings.Compare(a.Method, b.Method)
+	})
+	return changes
+}
+
+func nextAvailableNumber(usedNumbers map[int]struct{}) int {
+	for next := experimentalNumberStart; ; next++ {
+		if _, exists := usedNumbers[next]; !exists {
+			return next
+		}
+	}
+}
+
+func groupMessageOverlays(overlays []messageOverlay) []messageOverlayGroup {
+	groups := make([]messageOverlayGroup, 0)
+	groupByMessage := make(map[string]int)
+	for _, overlay := range overlays {
+		idx, ok := groupByMessage[overlay.StableMessage]
+		if !ok {
+			idx = len(groups)
+			groupByMessage[overlay.StableMessage] = idx
+			groups = append(groups, messageOverlayGroup{
+				StableMessage:  overlay.StableMessage,
+				OverlayMessage: overlay.OverlayMessage,
+				VariableName:   lowerFirst(overlay.OverlayMessage),
+			})
+		}
+		groups[idx].Fields = append(groups[idx].Fields, overlay)
+	}
+	return groups
+}
+
+func lowerFirst(s string) string {
+	if s == "" {
+		return ""
+	}
+	return strings.ToLower(s[:1]) + s[1:]
+}