add sql parser

Author: jakeheft

bttest/cmv.go

@@ -1,10 +1,7 @@
 package bttest
 
 import (
-	"fmt"
 	"log"
-	"regexp"
-	"strconv"
 	"strings"
 )
 
@@ -29,92 +26,6 @@ type CMVConfig struct {
 	AppendSourceKey bool `json:"append_source_key,omitempty"`
 }
 
-// ParseCMVConfigFromSQL attempts to extract a CMVConfig from a CMV SQL query.
-// It parses the standard Bigtable CMV SQL pattern:
-//
-//	SELECT SPLIT(_key, '#')[SAFE_OFFSET(n)] AS col, ..., _key AS source_key, family AS family
-//	FROM `table_name`
-//	ORDER BY col1, col2, ..., source_key
-//
-// _key (optionally aliased) in ORDER BY sets AppendSourceKey = true.
-func ParseCMVConfigFromSQL(viewID, query string) (*CMVConfig, error) {
-	cfg := &CMVConfig{
-		ViewID: viewID,
-	}
-
-	fromRe := regexp.MustCompile("(?i)FROM\\s+`([^`]+)`")
-	fromMatch := fromRe.FindStringSubmatch(query)
-	if fromMatch == nil {
-		return nil, fmt.Errorf("could not parse FROM clause in CMV query")
-	}
-	cfg.SourceTable = fromMatch[1]
-
-	// Extract column aliases and their SAFE_OFFSET indices from SELECT.
-	offsetRe := regexp.MustCompile(`SPLIT\(_key,\s*'([^']+)'\)\[SAFE_OFFSET\((\d+)\)\].*?\bAS\s+\w+\)\s+AS\s+(\w+)|SPLIT\(_key,\s*'([^']+)'\)\[SAFE_OFFSET\((\d+)\)\]\s+AS\s+(\w+)`)
-	matches := offsetRe.FindAllStringSubmatch(query, -1)
-	if len(matches) == 0 {
-		return nil, fmt.Errorf("could not parse SPLIT/SAFE_OFFSET expressions in CMV query")
-	}
-
-	colMap := make(map[string]int) // alias → SAFE_OFFSET index
-	for _, m := range matches {
-		if m[1] != "" {
-			cfg.KeySeparator = m[1]
-			idx, _ := strconv.Atoi(m[2])
-			colMap[m[3]] = idx
-		} else {
-			cfg.KeySeparator = m[4]
-			idx, _ := strconv.Atoi(m[5])
-			colMap[m[6]] = idx
-		}
-	}
-
-	// Detect "_key AS <alias>" in SELECT so we can recognise the alias in ORDER BY.
-	var sourceKeyAlias string
-	keyAliasRe := regexp.MustCompile(`(?i)\b_key\s+AS\s+(\w+)`)
-	if m := keyAliasRe.FindStringSubmatch(query); m != nil {
-		sourceKeyAlias = m[1]
-	}
-
-	// Extract ORDER BY columns; _key or its alias sets AppendSourceKey.
-	orderRe := regexp.MustCompile(`(?i)ORDER\s+BY\s+(.+)$`)
-	orderMatch := orderRe.FindStringSubmatch(strings.TrimSpace(query))
-	if orderMatch == nil {
-		return nil, fmt.Errorf("could not parse ORDER BY clause in CMV query")
-	}
-	orderCols := strings.Split(orderMatch[1], ",")
-	for _, col := range orderCols {
-		col = strings.TrimSpace(col)
-		if col == "_key" || (sourceKeyAlias != "" && col == sourceKeyAlias) {
-			cfg.AppendSourceKey = true
-			continue
-		}
-		idx, ok := colMap[col]
-		if !ok {
-			return nil, fmt.Errorf("ORDER BY column %q not found in SELECT", col)
-		}
-		cfg.KeyMapping = append(cfg.KeyMapping, idx)
-	}
-
-	if len(cfg.KeyMapping) == 0 {
-		return nil, fmt.Errorf("no key mapping columns found in ORDER BY")
-	}
-
-	// Extract included column families from SELECT (pattern: family AS family).
-	famRe := regexp.MustCompile(`(?:,\s*)(\w+)\s+AS\s+(\w+)`)
-	famMatches := famRe.FindAllStringSubmatch(query, -1)
-	for _, m := range famMatches {
-		src, alias := m[1], m[2]
-		if src == alias && src != "_key" {
-			if _, isCol := colMap[src]; !isCol {
-				cfg.IncludeFamilies = append(cfg.IncludeFamilies, src)
-			}
-		}
-	}
-
-	return cfg, nil
-}
-
 // cmvRegistry maps plain source table IDs to CMV definitions.
 // Lookups match by table ID suffix against fully-qualified table names.
 type cmvRegistry struct {

bttest/cmv_test.go

@@ -35,23 +35,23 @@ func newTestServerWithCMV(t *testing.T, configs []CMVConfig) *server {
 }
 
 func TestCMVTransformKey(t *testing.T) {
-	// key_mapping [3,4,1,2,0] + append_source_key:
-	// CMV key = parts[3]#parts[4]#parts[1]#parts[2]#parts[0]#<full_source_key>
+	// key_mapping [2,3,1,0] + append_source_key:
+	// CMV key = parts[2]#parts[3]#parts[1]#parts[0]#<full_source_key>
 	inst := &cmvInstance{
 		config: CMVConfig{
-			SourceTable:     "events",
-			ViewID:          "events_by_account",
+			SourceTable:     "sensor_readings",
+			ViewID:          "readings_by_region",
 			KeySeparator:    "#",
-			KeyMapping:      []int{3, 4, 1, 2, 0},
+			KeyMapping:      []int{2, 3, 1, 0},
 			AppendSourceKey: true,
 		},
 	}
 
-	sourceKey := "item-abc#9999999#type-x#region-a#account-42#src-01"
+	sourceKey := "device-1#ts-100#us-east#user-42"
 	got := inst.transformKey(sourceKey)
-	// parts: [0]item-abc [1]9999999 [2]type-x [3]region-a [4]account-42 [5]src-01
-	// mapped: parts[3]#parts[4]#parts[1]#parts[2]#parts[0] + full source key
-	want := "region-a#account-42#9999999#type-x#item-abc#item-abc#9999999#type-x#region-a#account-42#src-01"
+	// parts: [0]device-1 [1]ts-100 [2]us-east [3]user-42
+	// mapped: parts[2]#parts[3]#parts[1]#parts[0] + full source key
+	want := "us-east#user-42#ts-100#device-1#device-1#ts-100#us-east#user-42"
 	assert.Equal(t, want, got)
 }
 
@@ -261,8 +261,8 @@ func TestCMVDropRowRangeAll(t *testing.T) {
 func TestCMVTransformKeyOutOfBounds(t *testing.T) {
 	inst := &cmvInstance{
 		config: CMVConfig{
-			SourceTable:  "events",
-			ViewID:       "events_by_account",
+			SourceTable:  "my_table",
+			ViewID:       "my_view",
 			KeySeparator: "#",
 			KeyMapping:   []int{0, 99}, // index 99 is out of bounds
 		},
@@ -272,32 +272,6 @@ func TestCMVTransformKeyOutOfBounds(t *testing.T) {
 	assert.Equal(t, "only#", got)
 }
 
-func TestParseCMVConfigFromSQL(t *testing.T) {
-	// Standard Bigtable CMV SQL pattern: plain SPLIT (no CAST), _key aliased,
-	// alias appears in ORDER BY to set AppendSourceKey = true.
-	query := `SELECT
-  SPLIT(_key, '#')[SAFE_OFFSET(3)] AS region,
-  SPLIT(_key, '#')[SAFE_OFFSET(4)] AS account_id,
-  SPLIT(_key, '#')[SAFE_OFFSET(1)] AS timestamp,
-  SPLIT(_key, '#')[SAFE_OFFSET(2)] AS type,
-  SPLIT(_key, '#')[SAFE_OFFSET(0)] AS item_id,
-  _key AS src_key,
-  cf1 AS cf1,
-  cf2 AS cf2
-FROM ` + "`events`" + `
-ORDER BY region, account_id, timestamp, type, item_id, src_key`
-
-	cfg, err := ParseCMVConfigFromSQL("events_by_account", query)
-	require.NoError(t, err)
-	assert.Equal(t, "events", cfg.SourceTable)
-	assert.Equal(t, "events_by_account", cfg.ViewID)
-	assert.Equal(t, "#", cfg.KeySeparator)
-	assert.Equal(t, []int{3, 4, 1, 2, 0}, cfg.KeyMapping)
-	assert.True(t, cfg.AppendSourceKey)
-	assert.Contains(t, cfg.IncludeFamilies, "cf1")
-	assert.Contains(t, cfg.IncludeFamilies, "cf2")
-}
-
 func TestCreateMaterializedViewRPC(t *testing.T) {
 	ctx := context.Background()
 	dbFilename := newDBFile(t)
@@ -317,64 +291,60 @@ func TestCreateMaterializedViewRPC(t *testing.T) {
 
 	parent := "projects/test/instances/test"
 	mvSQL := "SELECT\n" +
-		"  SPLIT(_key, '#')[SAFE_OFFSET(3)] AS region,\n" +
-		"  SPLIT(_key, '#')[SAFE_OFFSET(4)] AS account_id,\n" +
+		"  SPLIT(_key, '#')[SAFE_OFFSET(2)] AS region,\n" +
+		"  SPLIT(_key, '#')[SAFE_OFFSET(3)] AS user_id,\n" +
 		"  SPLIT(_key, '#')[SAFE_OFFSET(1)] AS ts,\n" +
-		"  SPLIT(_key, '#')[SAFE_OFFSET(2)] AS typ,\n" +
-		"  SPLIT(_key, '#')[SAFE_OFFSET(0)] AS item_id,\n" +
+		"  SPLIT(_key, '#')[SAFE_OFFSET(0)] AS device_id,\n" +
 		"  _key AS src_key,\n" +
-		"  cf1 AS cf1\n" +
-		"FROM `events`\n" +
-		"ORDER BY region, account_id, ts, typ, item_id, src_key"
+		"  data AS data\n" +
+		"FROM `sensor_readings`\n" +
+		"ORDER BY region, user_id, ts, device_id, src_key"
 
 	op, err := s.CreateMaterializedView(ctx, &btapb.CreateMaterializedViewRequest{
 		Parent:             parent,
-		MaterializedViewId: "events_by_account",
+		MaterializedViewId: "readings_by_region",
 		MaterializedView:   &btapb.MaterializedView{Query: mvSQL},
 	})
 	require.NoError(t, err)
 	assert.True(t, op.Done)
 
-	// GetMaterializedView should return the stored view.
 	mv, err := s.GetMaterializedView(ctx, &btapb.GetMaterializedViewRequest{
-		Name: parent + "/materializedViews/events_by_account",
+		Name: parent + "/materializedViews/readings_by_region",
 	})
 	require.NoError(t, err)
-	assert.Equal(t, "events_by_account", mv.Name[strings.LastIndex(mv.Name, "/")+1:])
+	assert.Equal(t, "readings_by_region", mv.Name[strings.LastIndex(mv.Name, "/")+1:])
 	assert.Equal(t, mvSQL, mv.Query)
 
-	// ListMaterializedViews should include the new view.
 	list, err := s.ListMaterializedViews(ctx, &btapb.ListMaterializedViewsRequest{Parent: parent})
 	require.NoError(t, err)
 	assert.Len(t, list.MaterializedViews, 1)
 
-	// CMV should now fire on writes to the source table.
+	// CMV should fire on writes to the source table.
 	_, err = s.CreateTable(ctx, &btapb.CreateTableRequest{
 		Parent:  parent,
-		TableId: "events",
-		Table:   &btapb.Table{ColumnFamilies: map[string]*btapb.ColumnFamily{"cf1": {}}},
+		TableId: "sensor_readings",
+		Table:   &btapb.Table{ColumnFamilies: map[string]*btapb.ColumnFamily{"data": {}}},
 	})
 	require.NoError(t, err)
 
 	_, err = s.MutateRow(ctx, &btpb.MutateRowRequest{
-		TableName: parent + "/tables/events",
-		RowKey:    []byte("item-abc#9999999#type-x#region-a#account-42"),
+		TableName: parent + "/tables/sensor_readings",
+		RowKey:    []byte("device-1#ts-100#us-east#user-42"),
 		Mutations: []*btpb.Mutation{{
 			Mutation: &btpb.Mutation_SetCell_{SetCell: &btpb.Mutation_SetCell{
-				FamilyName: "cf1", ColumnQualifier: []byte("col1"), Value: []byte("v"),
+				FamilyName: "data", ColumnQualifier: []byte("temp"), Value: []byte("72"),
 			}},
 		}},
 	})
 	require.NoError(t, err)
 
-	// The CMV shadow table should have been created and contain the re-keyed row.
-	fqCMV := parent + "/tables/events_by_account"
+	fqCMV := parent + "/tables/readings_by_region"
 	cmvTbl := s.tables[fqCMV]
 	require.NotNil(t, cmvTbl, "CMV shadow table should have been auto-created")
 
 	// DeleteMaterializedView should remove the view.
 	_, err = s.DeleteMaterializedView(ctx, &btapb.DeleteMaterializedViewRequest{
-		Name: parent + "/materializedViews/events_by_account",
+		Name: parent + "/materializedViews/readings_by_region",
 	})
 	require.NoError(t, err)
 	list, err = s.ListMaterializedViews(ctx, &btapb.ListMaterializedViewsRequest{Parent: parent})