fix(bq2bq): handle BQ dry run error caused by self table SQL reference (#5)

Author: vianhazman

task/bq2bq/main.go

@@ -389,16 +389,30 @@ func (b *BQ2BQ) GenerateDependencies(ctx context.Context, request models.Generat
 		return nil, errors.New("empty sql file")
 	}
 
+	selfTable, err := b.GenerateDestination(ctx, models.GenerateDestinationRequest{
+		Config:  request.Config,
+		Assets:  request.Assets,
+		Project: request.Project,
+	})
+	if err != nil {
+		return response, err
+	}
+
 	// first parse sql statement to find dependencies and ignored tables
-	parsedDependencies, ignoredDependencies, err := b.FindDependenciesWithRegex(ctx, request)
+	parsedDependencies, ignoredDependencies, err := b.FindDependenciesWithRegex(ctx, queryData.Value, selfTable.Destination)
 	if err != nil {
 		return response, err
 	}
 
 	// try to resolve referenced tables directly from BQ APIs
 	response.Dependencies, err = b.FindDependenciesWithRetryableDryRun(timeoutCtx, queryData.Value, svcAcc)
 	if err != nil {
-		return response, err
+		// SQL query with reference to destination table such as DML and self joins will have dependency
+		// cycle on dry run since the table might not be available yet. We check the error from BQ
+		// to ignore if the error message contains destination table not found.
+		if !strings.Contains(err.Error(), fmt.Sprintf("Not found: Table %s was not found", selfTable.Destination)) {
+			return response, err
+		}
 	}
 
 	if len(response.Dependencies) == 0 {
@@ -443,15 +457,6 @@ func (b *BQ2BQ) GenerateDependencies(ctx context.Context, request models.Generat
 		}
 	}
 
-	// before returning remove self
-	selfTable, err := b.GenerateDestination(ctx, models.GenerateDestinationRequest{
-		Config:  request.Config,
-		Assets:  request.Assets,
-		Project: request.Project,
-	})
-	if err != nil {
-		return response, err
-	}
 	response.Dependencies = removeString(response.Dependencies, selfTable.Destination)
 
 	// before returning remove ignored tables
@@ -465,7 +470,7 @@ func (b *BQ2BQ) GenerateDependencies(ctx context.Context, request models.Generat
 
 // FindDependenciesWithRegex look for table patterns in SQL query to find
 // source tables.
-// Config is required to generate destination and avoid cycles
+// Task destination is required to avoid cycles
 //
 // we look for certain patterns in the query source code
 // in particular, we look for the following constructs
@@ -486,29 +491,17 @@ func (b *BQ2BQ) GenerateDependencies(ctx context.Context, request models.Generat
 // they're a single sequence of characters. But on the other hand
 // this also means that otherwise valid reference to "dataset.table"
 // will not be recognised.
-func (b *BQ2BQ) FindDependenciesWithRegex(ctx context.Context, request models.GenerateDependenciesRequest) ([]string, []string, error) {
+func (b *BQ2BQ) FindDependenciesWithRegex(ctx context.Context, queryString string, destination string) ([]string, []string, error) {
 
-	queryData, ok := request.Assets.Get(QueryFileName)
-	if !ok {
-		return nil, nil, errors.New("empty sql file")
-	}
-	queryString := queryData.Value
 	tablesFound := make(map[string]bool)
 	pseudoTables := make(map[string]bool)
 	var tablesIgnored []string
 
 	// we mark destination as a pseudo table to avoid a dependency
 	// cycle. This is for supporting DML queries that may also refer
 	// to themselves.
-	dest, err := b.GenerateDestination(ctx, models.GenerateDestinationRequest{
-		Config:  request.Config,
-		Assets:  request.Assets,
-		Project: request.Project,
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-	pseudoTables[dest.Destination] = true
+
+	pseudoTables[destination] = true
 
 	// remove comments from query
 	matches := queryCommentPatterns.FindAllStringSubmatch(queryString, -1)

task/bq2bq/main_test.go

@@ -357,23 +357,12 @@ func TestBQ2BQ(t *testing.T) {
 								Value: test.Query,
 							},
 						})),
-						Config: models.PluginConfigs{}.FromJobSpec(models.JobSpecConfigs{
-							{
-								Name:  "PROJECT",
-								Value: "proj",
-							},
-							{
-								Name:  "DATASET",
-								Value: "datas",
-							},
-							{
-								Name:  "TABLE",
-								Value: "tab",
-							},
-						}),
 					}
+
+					queryData, _ := data.Assets.Get(QueryFileName)
+					destination := "proj.datas.tab"
 					b2b := &BQ2BQ{}
-					deps, ignored, err := b2b.FindDependenciesWithRegex(context.Background(), data)
+					deps, ignored, err := b2b.FindDependenciesWithRegex(context.Background(), queryData.Value, destination)
 					assert.Nil(t, err)
 					assert.Equal(t, test.Sources, newSet(deps...))
 					assert.Equal(t, test.Ignored, newSet(ignored...))