Merge branch 'open-sauced:beta' into pull-org

Author: mtfoley

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+@open-sauced/engineering
+

.github/workflows/compliance.yaml

@@ -0,0 +1,15 @@
+name: Compliance
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+permissions:
+  pull-requests: write
+
+jobs:
+  compliance:
+    uses: open-sauced/hot/.github/workflows/compliance.yml@main

CHANGELOG.md

@@ -0,0 +1,27 @@
+# 📦 open-sauced/pizza changelog
+
+[![conventional commits](https://img.shields.io/badge/conventional%20commits-1.0.0-yellow.svg)](https://conventionalcommits.org)
+[![semantic versioning](https://img.shields.io/badge/semantic%20versioning-2.0.0-green.svg)](https://semver.org)
+
+> All notable changes to this project will be documented in this file
+
+## [1.1.0-beta.2](https://github.com/open-sauced/pizza/compare/v1.1.0-beta.1...v1.1.0-beta.2) (2023-10-09)
+
+
+### 🍕 Features
+
+* Ability to turn off sslmode ([#58](https://github.com/open-sauced/pizza/issues/58)) ([b026289](https://github.com/open-sauced/pizza/commit/b0262890b6d74ef85e2d5f57daa73e9ac7f81879))
+
+## [1.1.0-beta.1](https://github.com/open-sauced/pizza/compare/v1.0.1-beta.1...v1.1.0-beta.1) (2023-10-07)
+
+
+### 🍕 Features
+
+* Add our compliance GitHub workflow ([#59](https://github.com/open-sauced/pizza/issues/59)) ([8196306](https://github.com/open-sauced/pizza/commit/819630638474591340c4b5cbb973cca172b30791))
+
+## [1.0.1-beta.1](https://github.com/open-sauced/pizza/compare/v1.0.0...v1.0.1-beta.1) (2023-09-28)
+
+
+### 🐛 Bug Fixes
+
+* Add missing "pizza.sql" file ([#57](https://github.com/open-sauced/pizza/issues/57)) ([66dc680](https://github.com/open-sauced/pizza/commit/66dc6800925da3cf9c8d9c0329b50a05e3f360fe))

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all lint test run local build setup-test-env teardown-test-env
+.PHONY: all lint test run dev local build setup-test-env teardown-test-env
 
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
@@ -19,6 +19,9 @@ test:
 run:
 	go run main.go
 
+dev:
+	go run main.go -ssl-disable
+
 local:
 	go build -o build/pizza-oven main.go
 

hack/pizza.sql

@@ -0,0 +1,67 @@
+-- This file is sourced from https://github.com/open-sauced/api/tree/beta/migrations
+-- and is used only for local development convenience
+
+------------------------------------
+-- Pizza oven "baked" repos table --
+-------------------------------------
+create table if not exists public.baked_repos
+(
+  id bigint not null generated by default as identity ( increment 1 start 1 minvalue 1 maxvalue 9223372036854775807 cache 1 ),
+  clone_url character varying(255) collate pg_catalog."default" not null,
+
+  -- elastic columns
+
+  -- the repo_id field is used to loosely reference an id from the repos table
+  -- but since git repos may have their commits indexed that have not yet been
+  -- indexed into the repos table, there is not direct correlation.
+  -- Joins can happen across the "clone_url" to find repos that have not yet
+  -- synced to baked_repos.
+  repo_id bigint default null,
+
+  -- dynamic columns
+  constraint baked_repos_pkey primary key (id)
+)
+
+tablespace pg_default;
+
+-- indexes for baked repos
+create index if not exists baked_repos_idx_clone_url on baked_repos (clone_url);
+
+-------------------------------------
+-- Pizza oven commit authors table --
+-------------------------------------
+create table if not exists public.commit_authors
+(
+  id bigint not null generated by default as identity ( increment 1 start 1 minvalue 1 maxvalue 9223372036854775807 cache 1 ),
+  commit_author_email character varying(255) collate pg_catalog."default" not null,
+
+  -- dynamic columns
+  constraint commit_authors_pkey primary key (id)
+)
+
+tablespace pg_default;
+
+-- indexes for baked repos
+create index if not exists commit_authors_idx_commit_author_email on commit_authors (commit_author_email);
+
+--------------------------------
+-- Pizza oven indexed commits --
+--------------------------------
+
+create table if not exists public.commits (
+  id bigint not null generated by default as identity ( increment 1 start 1 minvalue 1 maxvalue 9223372036854775807 cache 1 ),
+  baked_repo_id bigint not null references public.baked_repos (id) on delete cascade on update cascade,
+  commit_author_id bigint not null references public.commit_authors(id) on delete cascade on update cascade,
+  commit_hash character varying(255) collate pg_catalog."default" not null,
+  commit_date timestamp with time zone not null default now(),
+
+  -- dynamic columns
+  constraint commits_pkey primary key (id)
+)
+
+tablespace pg_default;
+
+-- psql indexes for commits
+create index if not exists commit_idx_hash on commits (commit_hash);
+create index if not exists commit_idx_date on commits (commit_date);
+

main.go

@@ -20,9 +20,13 @@ func main() {
 	var err error
 
 	// Initialize & parse flags
-	var configPath string
+	var (
+		configPath string
+		sslmode    = "require"
+	)
 	flag.StringVar(&configPath, "config", "", "path to .yaml file config")
 	debugMode := flag.Bool("debug", false, "run in debug mode")
+	disableSSL := flag.Bool("ssl-disable", false, "set 'disable' ssl mode")
 	flag.Parse()
 
 	if *debugMode {
@@ -40,6 +44,11 @@ func main() {
 	sugarLogger := logger.Sugar()
 	sugarLogger.Infof("initiated zap logger with level: %d", sugarLogger.Level())
 
+	if *disableSSL {
+		sslmode = "disable"
+		sugarLogger.Warn("SSL mode is disabled")
+	}
+
 	// Load the environment variables from the .env file
 	err = godotenv.Load()
 	if err != nil {
@@ -60,7 +69,7 @@ func main() {
 	gitProvider := os.Getenv("GIT_PROVIDER")
 
 	// Initialize the database handler
-	pizzaOven := database.NewPizzaOvenDbHandler(databaseHost, databasePort, databaseUser, databasePwd, databaseDbName)
+	pizzaOven := database.NewPizzaOvenDbHandler(databaseHost, databasePort, databaseUser, databasePwd, databaseDbName, sslmode)
 
 	// Initializes configuration using a provided yaml file
 	config := &server.Config{NeverEvictRepos: make(map[string]bool)}

pkg/database/handler.go

@@ -24,8 +24,8 @@ type PizzaOvenDbHandler struct {
 
 // NewPizzaOvenDbHandler builds a PizzaOvenDbHandler based on the provided
 // database connection parameters
-func NewPizzaOvenDbHandler(host, port, user, pwd, dbName string) *PizzaOvenDbHandler {
-	connectString := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=require", host, port, user, pwd, dbName)
+func NewPizzaOvenDbHandler(host, port, user, pwd, dbName, sslmode string) *PizzaOvenDbHandler {
+	connectString := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", host, port, user, pwd, dbName, sslmode)
 
 	// Acquire the *sql.DB instance
 	dbPool, err := sql.Open("postgres", connectString)

pkg/server/server.go

@@ -289,9 +289,11 @@ func (p PizzaOvenServer) processRepository(repoURL string) error {
 			p.Logger.Debugf("No repo found in db. Inserting repo: %s", insight.RepoURLSource)
 			repoID, err = p.PizzaOven.InsertRepository(insight)
 			if err != nil {
+				p.Logger.Errorf("Failed to insert repository %s: %s", insight.RepoURLSource, err.Error())
 				return err
 			}
 		} else {
+			p.Logger.Errorf("Failed to fetch repository ID: %s", err.Error())
 			return err
 		}
 	}
@@ -301,6 +303,7 @@ func (p PizzaOvenServer) processRepository(repoURL string) error {
 	// Use the configured git provider to get the repo
 	providedRepo, err := p.PizzaGitProvider.FetchRepo(insight.RepoURLSource)
 	if err != nil {
+		p.Logger.Error("Failed to fetch repository %s: %s", insight.RepoURLSource, err.Error())
 		return err
 	}
 	defer providedRepo.Done()
@@ -310,12 +313,14 @@ func (p PizzaOvenServer) processRepository(repoURL string) error {
 	p.Logger.Debugf("Inspecting the head of the git repo: %s", insight.RepoURLSource)
 	ref, err := gitRepo.Head()
 	if err != nil {
+		p.Logger.Errorf("Could not find head of the git repo %s: %s", insight.RepoURLSource, err.Error())
 		return err
 	}
 
 	p.Logger.Debugf("Getting last commit in DB: %s", insight.RepoURLSource)
 	latestCommitDate, err := p.PizzaOven.GetLastCommit(repoID)
 	if err != nil {
+		p.Logger.Errorf("Could not fetch the latest commit date in %s: %s", insight.RepoURLSource, err.Error())
 		return err
 	}
 
@@ -334,6 +339,7 @@ func (p PizzaOvenServer) processRepository(repoURL string) error {
 	p.Logger.Debugf("Getting commit iterator with git log options: %v", gitLogOptions)
 	authorIter, err := gitRepo.Log(&gitLogOptions)
 	if err != nil {
+		p.Logger.Errorf("Failed to retrieve commit iterator: %s", err.Error())
 		return err
 	}
 
@@ -346,6 +352,7 @@ func (p PizzaOvenServer) processRepository(repoURL string) error {
 	p.Logger.Debugf("Using temporary db table for commit authors: %s", tmpTableName)
 	authorTxn, authorStmt, err := p.PizzaOven.PrepareBulkAuthorInsert(tmpTableName)
 	if err != nil {
+		p.Logger.Errorf("Failed to prepare the bulk author insert process: %s", err.Error())
 		return err
 	}
 
@@ -381,36 +388,42 @@ func (p PizzaOvenServer) processRepository(repoURL string) error {
 		})
 	})
 	if err != nil {
+		p.Logger.Errorf("Failed to insert author: %s", err.Error())
 		return err
 	}
 
 	// Resolve, execute, and pivot the bulk author transaction
 	err = p.PizzaOven.ResolveTransaction(authorTxn, authorStmt)
 	if err != nil {
+		p.Logger.Errorf("Failed to resolve bulk author transaction: %s", err.Error())
 		return err
 	}
 
 	err = p.PizzaOven.PivotTmpTableToAuthorsTable(tmpTableName)
 	if err != nil {
+		p.Logger.Errorf("Failed to pivot the temporary authors table: %s", err.Error())
 		return err
 	}
 
 	// Re-query the database for author email ids based on the unique list of
 	// author emails that have just been committed
 	authorEmailIDMap, err := p.PizzaOven.GetAuthorIDs(uniqueAuthorEmails)
 	if err != nil {
+		p.Logger.Errorf("Failed to create the author-email/id map: %s", err.Error())
 		return err
 	}
 
 	// Rebuild the iterator from the start using the same options
 	commitIter, err := gitRepo.Log(&gitLogOptions)
 	if err != nil {
+		p.Logger.Errorf("Failed to rebuild the commit iterator: %s", err.Error())
 		return err
 	}
 
 	// Get ready for the commit bulk action
 	commitTxn, commitStmt, err := p.PizzaOven.PrepareBulkCommitInsert()
 	if err != nil {
+		p.Logger.Errorf("Failed to prepare bulk commit insert process: %s", err.Error())
 		return err
 	}
 
@@ -426,18 +439,21 @@ func (p PizzaOvenServer) processRepository(repoURL string) error {
 		p.Logger.Debugf("Inspecting commit: %s %s %s", i.AuthorEmail, i.Hash, i.Date)
 		err = p.PizzaOven.InsertCommit(commitStmt, i, authorEmailIDMap[i.AuthorEmail], repoID)
 		if err != nil {
+			p.Logger.Errorf("Failed to insert commit: %s", err.Error())
 			return err
 		}
 
 		return nil
 	})
 	if err != nil {
+		p.Logger.Errorf("Failed to insert commit: %s", err.Error())
 		return err
 	}
 
 	// Execute and resolve the bulk commit insert
 	err = p.PizzaOven.ResolveTransaction(commitTxn, commitStmt)
 	if err != nil {
+		p.Logger.Errorf("Could not resolve bulk commit insert transaction %v", err.Error())
 		return err
 	}