Merge pull request #1 from augmentable-dev/parse-git-log

add a git log output parser instead of using go-git

Author: patrickdevivo

cmd/root.go

@@ -1,37 +1,35 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
-	"log"
+	"io/ioutil"
 	"math"
 	"os"
 	"sort"
 	"text/tabwriter"
-	"time"
 
+	"github.com/augmentable-dev/gitpert/pkg/gitlog"
 	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing/object"
-	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/spf13/cobra"
 	"github.com/src-d/enry/v2"
 )
 
 func handleError(err error) {
 	if err != nil {
-		log.Fatalln(err)
+		fmt.Println(err)
+		os.Exit(1)
 	}
 }
 
 var (
 	decayDays int
-	full      bool
 	remote    bool
 )
 
 func init() {
 	rootCmd.Flags().IntVarP(&decayDays, "decay-rate", "d", 30, "determines how long it takes for the impact of a commit to halve, based on how recently the commit was made")
 	rootCmd.Flags().BoolVarP(&remote, "remote", "r", false, "whether or not this is a remote repository")
-	rootCmd.Flags().BoolVarP(&full, "full", "f", false, "include all commits when calculating scores")
 }
 
 var rootCmd = &cobra.Command{
@@ -50,43 +48,32 @@ var rootCmd = &cobra.Command{
 			repoPath = p
 		}
 
-		var repo *git.Repository
-		// if the remote flag is set, clone the repo (using repoPath) into memory
 		if remote {
-			r, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-				URL:          repoPath,
-				SingleBranch: true,
-			})
+			dir, err := ioutil.TempDir("", "gitpert_remote_repo")
 			handleError(err)
-			repo = r
-		} else { // otherwise, open the specified repo
-			r, err := git.PlainOpen(repoPath)
+			defer os.RemoveAll(dir)
+
+			_, err = git.PlainClone(dir, false, &git.CloneOptions{
+				URL: repoPath,
+				// Progress: os.Stdout,
+			})
 			handleError(err)
-			repo = r
+
+			repoPath = dir
 		}
 
-		var fileName *string
+		var fileName string
 		if len(args) > 1 {
-			fileName = &args[1]
+			fileName = args[1]
 		}
 
+		commits, err := gitlog.Exec(context.Background(), repoPath, fileName, []string{})
+		handleError(err)
+
 		// TODO (patrickdevivo) at some point this entire scoring logic should be brought out into a subpackage with some tests
 		// this could also make it possibe for other projects to import the implementation.
 		decayHours := 24 * decayDays
 
-		// this ignores any commits older than 100 half-lives,
-		var since time.Time
-		if !full {
-			since = time.Now().Add(-(time.Duration(decayHours) * time.Hour * 10))
-		}
-		commitIter, err := repo.Log(&git.LogOptions{
-			Order:    git.LogOrderCommitterTime,
-			FileName: fileName,
-			Since:    &since,
-		})
-		handleError(err)
-		defer commitIter.Close()
-
 		type authorAggregate struct {
 			email   string
 			name    string
@@ -96,7 +83,11 @@ var rootCmd = &cobra.Command{
 		}
 		authors := map[string]*authorAggregate{}
 		var authorEmails []string
-		commitIter.ForEach(func(commit *object.Commit) error {
+		var firstCommit *gitlog.Commit
+		for _, commit := range commits {
+			if firstCommit == nil {
+				firstCommit = commit
+			}
 			authorEmail := commit.Author.Email
 			if _, ok := authors[authorEmail]; !ok {
 				authors[authorEmail] = &authorAggregate{
@@ -107,37 +98,24 @@ var rootCmd = &cobra.Command{
 			}
 
 			agg := authors[authorEmail]
-			hoursAgo := time.Now().Sub(commit.Author.When).Hours()
+			hoursAgo := firstCommit.Author.When.Sub(commit.Author.When).Hours()
 			agg.commits++
 
-			// TODO this is a bit hacky, we're absorbing any panics that occur
-			// in particular, it's meant to capture an index out of range error occurring
-			// under some conditions in the underlying git/diff dependency. Maybe another reason to use native git...
-			defer func() {
-				if err := recover(); err != nil {
-					agg.score += math.Exp2(-hoursAgo / float64(decayHours))
-				}
-			}()
-
-			fileStats, err := commit.Stats()
-			handleError(err)
-
 			var additions int
 			var deletions int
-			for _, stat := range fileStats {
+			for file, stat := range commit.Stats {
 				// ignore diffs in vendor files
 				// TODO perhaps it's worth allowing for the user to supply file path patterns be ignored?
-				if enry.IsVendor(stat.Name) {
+				if enry.IsVendor(file) {
 					continue
 				}
-				additions += stat.Addition
-				deletions += stat.Deletion
+				additions += stat.Additions
+				deletions += stat.Deletions
 			}
 			agg.impact += additions + deletions
 
 			agg.score += float64(additions+deletions) * math.Exp2(-hoursAgo/float64(decayHours))
-			return nil
-		})
+		}
 
 		sort.SliceStable(authorEmails, func(i, j int) bool {
 			return authors[authorEmails[j]].score < authors[authorEmails[i]].score

pkg/gitlog/README.md

@@ -0,0 +1,3 @@
+## gitlog
+
+Package for parsing the output of `git log` for the purposes of `gitpert`.

pkg/gitlog/gitlog.go

@@ -0,0 +1,173 @@
+package gitlog
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Commit represents a parsed commit from git log
+type Commit struct {
+	SHA       string
+	Author    Event
+	Committer Event
+	Stats     map[string]Stat
+}
+
+// Event represents the who and when of a commit event
+type Event struct {
+	Name  string
+	Email string
+	When  time.Time
+}
+
+// Stat holds the diff stat of a file
+type Stat struct {
+	Additions int
+	Deletions int
+}
+
+// Result is a list of commits
+type Result []*Commit
+
+func parseLog(reader io.Reader) (Result, error) {
+	scanner := bufio.NewScanner(reader)
+	res := make(Result, 0)
+
+	// line prefixes for the `fuller` formatted output
+	const (
+		commit     = "commit "
+		author     = "Author: "
+		authorDate = "AuthorDate: "
+
+		committer  = "Commit: "
+		commitDate = "CommitDate: "
+	)
+
+	var currentCommit *Commit
+	for scanner.Scan() {
+		line := scanner.Text()
+		switch {
+		case strings.HasPrefix(line, commit):
+			if currentCommit != nil { // if we're seeing a new commit but already have a current commit, we've finished a commit
+				res = append(res, currentCommit)
+			}
+			currentCommit = &Commit{
+				SHA:   strings.TrimPrefix(line, commit),
+				Stats: make(map[string]Stat),
+			}
+		case strings.HasPrefix(line, author):
+			s := strings.TrimPrefix(line, author)
+			spl := strings.Split(s, " ")
+			email := strings.Trim(spl[len(spl)-1], "<>")
+			name := strings.Join(spl[:len(spl)-1], " ")
+			currentCommit.Author.Email = strings.Trim(email, "<>")
+			currentCommit.Author.Name = strings.TrimSpace(name)
+		case strings.HasPrefix(line, authorDate):
+			authorDateString := strings.TrimPrefix(line, authorDate)
+			aD, err := time.Parse(time.RFC3339, authorDateString)
+			if err != nil {
+				return nil, err
+			}
+			currentCommit.Author.When = aD
+		case strings.HasPrefix(line, committer):
+			s := strings.TrimPrefix(line, committer)
+			spl := strings.Split(s, " ")
+			email := strings.Trim(spl[len(spl)-1], "<>")
+			name := strings.Join(spl[:len(spl)-1], " ")
+			currentCommit.Committer.Email = strings.Trim(email, "<>")
+			currentCommit.Committer.Name = strings.TrimSpace(name)
+		case strings.HasPrefix(line, commitDate):
+			commitDateString := strings.TrimPrefix(line, commitDate)
+			cD, err := time.Parse(time.RFC3339, commitDateString)
+			if err != nil {
+				return nil, err
+			}
+			currentCommit.Committer.When = cD
+		case strings.HasPrefix(line, " "): // ignore commit message lines
+		case strings.TrimSpace(line) == "": // ignore empty lines
+		default:
+			s := strings.Split(line, "\t")
+			var additions int
+			var deletions int
+			var err error
+			if s[0] != "-" {
+				additions, err = strconv.Atoi(s[0])
+				if err != nil {
+					return nil, err
+				}
+			}
+			if s[1] != "-" {
+				deletions, err = strconv.Atoi(s[1])
+				if err != nil {
+					return nil, err
+				}
+			}
+			currentCommit.Stats[s[2]] = Stat{
+				Additions: additions,
+				Deletions: deletions,
+			}
+		}
+	}
+	if currentCommit != nil {
+		res = append(res, currentCommit)
+	}
+
+	return res, nil
+}
+
+// Exec runs the git log command
+func Exec(ctx context.Context, repoPath string, filePattern string, additionalFlags []string) (Result, error) {
+	gitPath, err := exec.LookPath("git")
+	if err != nil {
+		return nil, fmt.Errorf("could not find git: %w", err)
+	}
+
+	args := []string{"log"}
+
+	args = append(args, "--numstat", "--format=fuller", "--no-merges", "--no-decorate", "--date=iso8601-strict", "-w")
+	args = append(args, additionalFlags...)
+	if filePattern != "" {
+		args = append(args, filePattern)
+	}
+
+	cmd := exec.CommandContext(ctx, gitPath, args...)
+	cmd.Dir = repoPath
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	res, err := parseLog(stdout)
+	if err != nil {
+		return nil, err
+	}
+
+	errs, err := ioutil.ReadAll(stderr)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		fmt.Println(string(errs))
+		return nil, err
+	}
+
+	return res, nil
+}