support glob patterns for filePattern alias matching (#113)

Author: Arnold Trakhtenberg

README.md

@@ -243,14 +243,17 @@ aliases:
 
 #### Search a file for a specific pattern
 
-Specify a file to search, with a regular expression containing a capture group to match aliases. The pattern must contain the templated literal `FLAG_KEY` to be replaced with flag keys.
+Specify a number of files (`paths`) using [glob patterns](https://en.wikipedia.org/wiki/Glob_(programming)) to search. Be as specific as possible with your path globs to minimize the number of files searched for aliases.
 
-Example matching all variable names storing flag keys of the form `MyFlagKey := "my-flag"` in the file `test.go`:
+You must also specify a regular expression (`pattern`) containing a capture group to match aliases. The pattern must contain the the text `FLAG_KEY`, which will be subsituted with flag keys.
+
+Example matching all variable names storing flag keys of the form `MyFlagKey := "my-flag"` in .go files do not end with `_test`:
 
 ```yaml
 aliases:
   - type: filePattern
-    path: test.go
+    paths:
+      - '*[!_test].go'
     pattern: (\w+Key) := "FLAG_KEY"
 ```
 

internal/helpers/helpers.go

@@ -0,0 +1,13 @@
+package helpers
+
+func Dedupe(s []string) []string {
+	keys := make(map[string]struct{}, len(s))
+	ret := make([]string, 0, len(s))
+	for _, entry := range s {
+		if _, value := keys[entry]; !value {
+			keys[entry] = struct{}{}
+			ret = append(ret, entry)
+		}
+	}
+	return ret
+}

internal/options/alias.go

@@ -9,12 +9,15 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/iancoleman/strcase"
-	"github.com/launchdarkly/ld-find-code-refs/internal/validation"
 	"gopkg.in/yaml.v2"
+
+	"github.com/launchdarkly/ld-find-code-refs/internal/helpers"
+	"github.com/launchdarkly/ld-find-code-refs/internal/validation"
 )
 
 // We're using multiple packages for configuration.
@@ -53,7 +56,7 @@ func (a Alias) Generate(flag string) ([]string, error) {
 		ret = []string{strcase.ToDelimited(flag, '.')}
 	case FilePattern:
 		pattern := regexp.MustCompile(strings.ReplaceAll(*a.Pattern, "FLAG_KEY", flag))
-		results := pattern.FindAllStringSubmatch(string(a.FileContents), -1)
+		results := pattern.FindAllStringSubmatch(string(a.AllFileContents), -1)
 		for _, res := range results {
 			if len(res) > 1 {
 				ret = append(ret, res[1:]...)
@@ -108,14 +111,18 @@ const (
 // TODO: can we use OOP?
 type Alias struct {
 	Type AliasType `yaml:"type"`
+	Name string    `yaml:"name"`
 
 	// Literal
 	Flags map[string][]string `yaml:"flags,omitempty"`
 
+	// TODO: Remove this field - it has been superceded by Paths and should be removed
+	Path *string `yaml:"path"`
+
 	// FilePattern
-	Path         *string `yaml:"path,omitempty"`
-	Pattern      *string `yaml:"pattern,omitempty"`
-	FileContents []byte  `yaml:"-"` // data for pattern matching
+	Paths           []string `yaml:"paths,omitempty"`
+	Pattern         *string  `yaml:"pattern,omitempty"`
+	AllFileContents []byte   `yaml:"-"` // data for pattern matching
 
 	// Command
 	Command *string `yaml:"command,omitempty"`
@@ -127,16 +134,15 @@ func (a *Alias) IsValid() error {
 	if err != nil {
 		return err
 	}
-
 	// Validate expected fields
 	switch a.Type {
 	case Literal:
 		if a.Flags == nil {
 			return errors.New("literal aliases must provide an 'flags'")
 		}
 	case FilePattern:
-		if a.Path == nil {
-			return errors.New("filePattern aliases must provide a 'path'")
+		if len(a.Paths) == 0 && a.Path == nil {
+			return errors.New("filePattern aliases must provide at least one path in 'paths'")
 		}
 		if a.Pattern == nil {
 			return errors.New("filePattern aliases must provide a 'pattern'")
@@ -165,8 +171,8 @@ func (a *Alias) IsValid() error {
 			unexpectedField = "flags"
 		}
 	case a.Type != FilePattern:
-		if a.Path != nil {
-			unexpectedField = "path"
+		if len(a.Paths) > 0 {
+			unexpectedField = "paths"
 		}
 		if a.Pattern != nil {
 			unexpectedField = "pattern"
@@ -186,6 +192,62 @@ func (a *Alias) IsValid() error {
 	return nil
 }
 
+func (a *Alias) ProcessFileContent(idx int) error {
+	if a.Type != FilePattern {
+		return nil
+	}
+
+	// TODO: Remove this block with release as the Path field is being removed
+	if a.Path != nil {
+		path := filepath.Join(Dir.Value(), filepath.Dir(*a.Path))
+		absPath, err := validation.NormalizeAndValidatePath(path)
+		if err != nil {
+			return err
+		}
+
+		absFile := filepath.Join(absPath, filepath.Base(*a.Path))
+		if !validation.FileExists(absFile) {
+			return fmt.Errorf("could not find file at path '%s'", absFile)
+		}
+		/* #nosec */
+		data, err := ioutil.ReadFile(absFile)
+		if err != nil {
+			return fmt.Errorf("could not process file at path '%s': %v", absFile, err)
+		}
+		a.AllFileContents = data
+	}
+
+	aliasId := strconv.Itoa(idx)
+	if a.Name != "" {
+		aliasId = a.Name
+	}
+
+	paths := []string{}
+	for _, glob := range a.Paths {
+		absGlob := filepath.Join(Dir.Value(), glob)
+		matches, err := filepath.Glob(absGlob)
+		if err != nil {
+			return fmt.Errorf("filePattern '%s': could not process path glob '%s'", aliasId, absGlob)
+		}
+		paths = append(paths, matches...)
+	}
+	paths = helpers.Dedupe(paths)
+	a.AllFileContents = []byte{}
+
+	for _, path := range paths {
+		if !validation.FileExists(path) {
+			return fmt.Errorf("filePattern '%s': could not find file at path '%s'", aliasId, path)
+		}
+		/* #nosec */
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("filePattern '%s': could not process file at path '%s': %v", aliasId, path, err)
+		}
+		a.AllFileContents = append(a.AllFileContents, data...)
+	}
+	return nil
+}
+
 type YamlOptions struct {
 	Aliases []Alias `yaml:"aliases"`
 }
@@ -225,25 +287,11 @@ func Yaml() (*YamlOptions, error) {
 		return nil, err
 	}
 	for i, a := range o.Aliases {
-		if a.Path != nil {
-			path := filepath.Join(Dir.Value(), filepath.Dir(*a.Path))
-			absPath, err := validation.NormalizeAndValidatePath(path)
-			if err != nil {
-				return nil, err
-			}
-
-			absFile := filepath.Join(absPath, filepath.Base(*a.Path))
-			if !validation.FileExists(absFile) {
-				return nil, fmt.Errorf("could not find file at path '%s'", absFile)
-			}
-			/* #nosec */
-			data, err := ioutil.ReadFile(absFile)
-			if err != nil {
-				return nil, fmt.Errorf("could not process file at path '%s': %v", absFile, err)
-			}
-			a.FileContents = data
-			o.Aliases[i] = a
+		err = a.ProcessFileContent(i)
+		if err != nil {
+			return nil, err
 		}
+		o.Aliases[i] = a
 	}
 	return &o, nil
 }

pkg/coderefs/alias.go

@@ -1,6 +1,7 @@
 package coderefs
 
 import (
+	"github.com/launchdarkly/ld-find-code-refs/internal/helpers"
 	"github.com/launchdarkly/ld-find-code-refs/internal/options"
 )
 
@@ -14,19 +15,7 @@ func generateAliases(flags []string, aliases []options.Alias) (map[string][]stri
 			}
 			ret[flag] = append(ret[flag], flagAliases...)
 		}
-		ret[flag] = dedupe(ret[flag])
+		ret[flag] = helpers.Dedupe(ret[flag])
 	}
 	return ret, nil
 }
-
-func dedupe(s []string) []string {
-	keys := make(map[string]struct{}, len(s))
-	ret := make([]string, 0, len(s))
-	for _, entry := range s {
-		if _, value := keys[entry]; !value {
-			keys[entry] = struct{}{}
-			ret = append(ret, entry)
-		}
-	}
-	return ret
-}

pkg/coderefs/alias_test.go

@@ -113,7 +113,7 @@ func filePattern(flag string) o.Alias {
 	a := alias(o.FilePattern)
 	pattern := "(\\w+)\\s= 'FLAG_KEY'"
 	a.Pattern = &pattern
-	a.FileContents = []byte(fmt.Sprintf("SOME_FLAG = '%s'", flag))
+	a.AllFileContents = []byte(fmt.Sprintf("SOME_FLAG = '%s'", flag))
 	return a
 }
 

pkg/coderefs/coderefs.go

@@ -11,9 +11,9 @@ import (
 	"time"
 
 	"github.com/launchdarkly/ld-find-code-refs/internal/command"
+	"github.com/launchdarkly/ld-find-code-refs/internal/helpers"
 	"github.com/launchdarkly/ld-find-code-refs/internal/ld"
 	"github.com/launchdarkly/ld-find-code-refs/internal/log"
-	"github.com/launchdarkly/ld-find-code-refs/internal/options"
 	o "github.com/launchdarkly/ld-find-code-refs/internal/options"
 	"github.com/launchdarkly/ld-find-code-refs/internal/validation"
 	"github.com/launchdarkly/ld-find-code-refs/internal/version"
@@ -121,7 +121,7 @@ func Scan() {
 		log.Warning.Printf("omitting %d flags with keys less than minimum (%d)", len(omittedFlags), minFlagKeyLen)
 	}
 
-	aliases, err := generateAliases(filteredFlags, options.Aliases)
+	aliases, err := generateAliases(filteredFlags, o.Aliases)
 	if err != nil {
 		log.Error.Fatalf("failed to create flag key aliases: %v", err)
 	}
@@ -497,7 +497,7 @@ func buildHunksForFlag(projKey, flag, path string, flagReferences []*list.Elemen
 			appendToPreviousHunk = false
 		} else {
 			currentHunk.Lines = hunkStringBuilder.String()
-			currentHunk.Aliases = dedupe(currentHunk.Aliases)
+			currentHunk.Aliases = helpers.Dedupe(currentHunk.Aliases)
 			hunks = append(hunks, currentHunk)
 			previousHunk = &hunks[len(hunks)-1]
 		}