[EXP] GTFS protobuf experiment

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.29.14
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.67
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3
+	github.com/bufbuild/protocompile v0.14.1
 	github.com/deckarep/golang-set/v2 v2.6.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/iancoleman/orderedmap v0.2.0

go.sum

@@ -54,6 +54,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/Xv
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
 github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
 github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw=
+github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -70,8 +72,8 @@ github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqw
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
 github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -182,8 +184,6 @@ golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
 golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/tlpb/gen.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/bufbuild/protocompile"
+	"github.com/interline-io/transitland-lib/tlcli"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+func main() {
+	cmd := tlcli.CobraHelper(&GenGtfsCommand{}, "", "")
+	cmd.Execute()
+}
+
+type GenGtfsCommand struct {
+	Protopath string
+	Outpath   string
+	Command   *cobra.Command
+}
+
+func (cmd *GenGtfsCommand) AddFlags(fl *pflag.FlagSet) {
+}
+
+func (cmd *GenGtfsCommand) HelpDesc() (string, string) {
+	return "Generate GTFS entities", ""
+}
+
+func (cmd *GenGtfsCommand) Parse(args []string) error {
+	fl := tlcli.NewNArgs(args)
+	if fl.NArg() < 2 {
+		return errors.New("<proto> <outppath>")
+	}
+	cmd.Protopath = fl.Arg(0)
+	cmd.Outpath = fl.Arg(1)
+	return nil
+}
+
+func (cmd *GenGtfsCommand) Run() error {
+	compiler := protocompile.Compiler{
+		Resolver: &protocompile.SourceResolver{},
+	}
+	files, err := compiler.Compile(context.Background(), cmd.Protopath)
+	if err != nil {
+		return err
+	}
+	outf, err := os.Create(cmd.Outpath)
+	if err != nil {
+		return err
+	}
+	defer outf.Close()
+
+	// Go
+	outf.WriteString(`package gtfs` + "\n\n")
+	outf.WriteString(`import ( "github.com/interline-io/transitland-lib/tt" )` + "\n\n")
+
+	ttKinds := map[string]string{
+		"Url":       "tt.Url",
+		"Date":      "tt.Date",
+		"Time":      "tt.Time",
+		"Color":     "tt.Color",
+		"Key":       "tt.Key",
+		"Phone":     "tt.Phone",
+		"Email":     "tt.Email",
+		"Reference": "tt.Reference",
+		"Currency":  "tt.Currency",
+		"Language":  "tt.Language",
+		"Int":       "tt.Int",
+		"Bool":      "tt.Bool",
+		"Float":     "tt.Float",
+		"String":    "tt.String",
+		"Timezone":  "tt.Timezone",
+		"Timestamp": "tt.Timestamp",
+		"Seconds":   "tt.Seconds",
+	}
+
+	for _, lf := range files {
+		enums := lf.Enums()
+		for i := 0; i < enums.Len(); i++ {
+			en := enums.Get(i)
+			outf.WriteString(fmt.Sprintf("type %s int32\n\n", en.Name()))
+		}
+		msgs := lf.Messages()
+		for i := 0; i < msgs.Len(); i++ {
+			msg := msgs.Get(i)
+			fields := msg.Fields()
+			if _, ok := ttKinds[string(msg.Name())]; ok {
+				continue
+			}
+			if fields.Len() == 1 && fields.Get(0).Name() == "val" {
+				field := fields.Get(0)
+				outf.WriteString(fmt.Sprintf(
+					"type %s struct { tt.Option[%s] }\n\n",
+					msg.Name(),
+					mapKind(field)),
+				)
+				continue
+			}
+
+			outf.WriteString(fmt.Sprintf("type %s struct {\n", msg.Name()))
+			for j := 0; j < fields.Len(); j++ {
+				field := fields.Get(j)
+				fieldName := toCamelCase(string(field.Name()))
+				fieldKind := mapKind(field)
+				if ttKind, ok := ttKinds[fieldKind]; ok {
+					outf.WriteString(fmt.Sprintf("\t%s %s\n", fieldName, ttKind))
+					continue
+				}
+				switch fieldKind {
+				case "DatabaseEntity":
+					outf.WriteString("\tDatabaseEntity\n")
+				default:
+					outf.WriteString(fmt.Sprintf("\t%s %s\n", fieldName, fieldKind))
+				}
+			}
+			outf.WriteString("}\n\n")
+		}
+	}
+	return nil
+}
+
+func mapKind(field protoreflect.FieldDescriptor) string {
+	fieldKind := field.Kind().String()
+	switch fieldKind {
+	case "enum":
+		fieldKind = string(field.Enum().Name())
+	case "double":
+		fieldKind = "float64"
+	case "float":
+		fieldKind = "float32"
+	}
+	if fmsg := field.Message(); fmsg != nil {
+		fieldKind = string(fmsg.Name())
+	}
+	return fieldKind
+}
+
+func toCamelCase(v string) string {
+	a := strings.Split(v, "_")
+	for i := 0; i < len(a); i++ {
+		s := a[i]
+		if s == "id" {
+			s = "ID"
+		} else {
+			s = strings.ToUpper(s[0:1]) + s[1:]
+		}
+		a[i] = s
+	}
+	return strings.Join(a, "")
+}

internal/tlpb/gen_test.go

@@ -0,0 +1,135 @@
+package main
+
+// func printFirst(v []any) {
+// 	if len(v) == 0 {
+// 		return
+// 	}
+// 	fmt.Println(toJson(v[0]))
+// }
+// func printAll(v []any) {
+// 	for _, ent := range v {
+// 		fmt.Println(toJson(ent))
+// 	}
+// }
+
+// func pbJson(v protoreflect.ProtoMessage) string {
+// 	jj, _ := protojson.Marshal(v)
+// 	return string(jj)
+// }
+
+// func toJson(v any) string {
+// 	jj, _ := json.Marshal(v)
+// 	return string(jj)
+// }
+
+// var TESTFILE = ""
+// var TESTTABLE = ""
+
+// func init() {
+// 	TESTFILE = testpath.RelPath("test/data/external/bart.zip")
+// 	TESTTABLE = "stops.txt"
+// }
+
+//////////////////
+
+// func TestReadPB(t *testing.T) {
+// 	ents, err := ReadPB(TESTFILE)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	for _, ent := range ents {
+// 		fmt.Println(ent)
+// 	}
+// }
+
+// func BenchmarkReadPB(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		ReadPB(TESTFILE)
+// 	}
+// }
+
+// func ReadPB(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := &pb.Stop{}
+// 		if errs := tlcsv.LoadRow(ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }
+
+//////////////////
+
+// func TestReadTT(t *testing.T) {
+// 	ents, err := ReadTT(TESTFILE)
+// 	assert.NoError(t, err)
+// 	printAll(ents)
+// }
+
+// func BenchmarkReadTT(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		a, _ := ReadTT(TESTFILE)
+// 		_ = a
+// 		// printFirst(a)
+// 	}
+// }
+
+// func ReadTT(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := gtfs.Stop{}
+// 		if errs := tlcsv.LoadRow(&ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }
+
+//////////////////
+
+// func TestReadG(t *testing.T) {
+// 	ents, err := ReadG(TESTFILE)
+// 	assert.NoError(t, err)
+// 	printAll(ents)
+// }
+
+// func BenchmarkReadG(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		a, _ := ReadG(TESTFILE)
+// 		_ = a
+// 		// printFirst(a)
+// 	}
+// }
+
+// func ReadG(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := gtfs.Stop{}
+// 		if errs := tlcsv.LoadRow(&ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }