feat: improve parallelization in github workflows and Makefile (#2436)

The primary goals of this PR are:
* Better parallelization for github workflows and local builds
* Allow easily running the github workflows locally

Changes:
* Makefile now describes dependencies among targets to enable parallel
jobs
* Created `release` target to be called from the release workflow
    * Creates temporary build folders to isolate symlinks
* Created `release-test` target to be called form the test workflow
* Makefile now has a release target which will concurrently build all 4
archives
* Added -j flag to Makefile so it may create multiple jobs
    * This should leverage all cpus 4 cpu machines we will build on
* I've made the arch explicit (GOOS=amd64) for x86 binary for safety
* For now I've preserved the existing targets for "make build" and "make
build-strip", but ideally:
* `make build` only builds using `CGO_ENABLED=0 go build` giving native
arch for local dev.
* `make release` builds all the release binaries, without any env
defined values such as `CGO_ENABLED=0 go build` (it is currently doing
this now)
* I've collapsed the test workflows into one step so they can run
concurrently

One note is that the go tool chain does make use of multiple cpus. But
there should still be measurable benefits on 4 cpu machines with these
changes.

Related:
#2429

---------

Co-authored-by: Chris Stockton <chris.stockton@supabase.io>

Author: cstockton

.github/workflows/release.yml

@@ -99,19 +99,8 @@ jobs:
         run: |
           set -ex
 
-          RELEASE_VERSION=$RELEASE_VERSION make deps
-          RELEASE_VERSION=$RELEASE_VERSION make build build-strip
-
-          ln -s auth gotrue
-          tar -czvf auth-v$RELEASE_VERSION-x86.tar.gz auth gotrue migrations/
-          mv auth-arm64 auth
-          tar -czvf auth-v$RELEASE_VERSION-arm64.tar.gz auth gotrue migrations/
-
-          mv auth-darwin-arm64 auth
-          tar -czvf auth-v$RELEASE_VERSION-darwin-arm64.tar.gz auth gotrue migrations/
-
-          mv auth-arm64-strip auth
-          tar -cf - auth gotrue migrations/ | xz -T0 -9e -C crc64 > auth-v$RELEASE_VERSION-arm64.tar.xz
+          MAKE_JOBS="$(nproc 2>/dev/null || echo 4)"
+          RELEASE_VERSION=$RELEASE_VERSION make -j"$MAKE_JOBS" release
 
       - name: Generate checksums
         if: ${{ steps.release.outputs.release_created == 'true' || steps.release.outputs.prs_created == 'true' }}

.github/workflows/test.yml

@@ -42,28 +42,17 @@ jobs:
           then
             echo 'Make sure to run "gofmt -s -w ." before commit!' && exit 1
           fi
-      - name: Check go vet
-        run: |
-          set -x
-          go vet ./...
-      - name: Run static check
-        run: |
-          set -x
-          make static
-      - name: Check gosec
-        run: |
-          set -x
-          make sec
-      - name: Run govulncheck
-        run: |
-          set -x
-          make vulncheck
       - name: Init Database
         run: psql -f hack/init_postgres.sql postgresql://postgres:root@localhost:5432/postgres
       - name: Run migrations
         run: make migrate_dev
-      - name: Lint and test
-        run: make test
+      - name: Run linters and tests
+        run: |
+          set -x
+
+          MAKE_JOBS="$(nproc 2>/dev/null || echo 4)"
+          make -j"$MAKE_JOBS" release-test
+
       - name: Cleanup coverage
         run: |
           set -x

Makefile

@@ -1,51 +1,98 @@
-.PHONY: all build deps image migrate test vet sec vulncheck format unused
+.PHONY: all build deps image migrate test vet sec vulncheck format unused release
 .PHONY: check-gosec check-govulncheck check-oapi-codegen check-staticcheck
-CHECK_FILES?=./...
+CHECK_FILES ?= ./...
 
 ifdef RELEASE_VERSION
 	VERSION=v$(RELEASE_VERSION)
 else
 	VERSION=$(shell git describe --tags)
 endif
 
-FLAGS=-ldflags "-X github.com/supabase/auth/internal/utilities.Version=$(VERSION)" -buildvcs=false
-
 ifneq ($(shell docker compose version 2>/dev/null),)
-  DOCKER_COMPOSE=docker compose
+	DOCKER_COMPOSE = docker compose
 else
-  DOCKER_COMPOSE=docker-compose
+	DOCKER_COMPOSE = docker-compose
 endif
 
-DEV_DOCKER_COMPOSE:=docker-compose-dev.yml
+DEV_DOCKER_COMPOSE = docker-compose-dev.yml
+
+BUILD_VERSION_PKG = github.com/supabase/auth/internal/utilities
+BUILD_LD_FLAGS = -X $(BUILD_VERSION_PKG).Version=$(VERSION)
+BUILD_CMD = go build \
+	-o $(1) \
+	-buildvcs=false \
+	-ldflags "$(BUILD_LD_FLAGS)$(2)"
+
+RELEASE_TARGETS = x86 arm64 darwin-arm64 arm64-strip
+RELEASE_ARCHIVES = \
+	auth-$(VERSION)-x86.tar.gz \
+	auth-$(VERSION)-arm64.tar.gz \
+	auth-$(VERSION)-darwin-arm64.tar.gz \
+	auth-$(VERSION)-arm64.tar.xz
+
 
 help: ## Show this help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
 all: vet sec static build ## Run the tests and build the binary.
 
-build: deps ## Build the binary.
-	CGO_ENABLED=0 go build $(FLAGS)
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(FLAGS) -o auth-arm64
-	CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build $(FLAGS) -o auth-darwin-arm64
+build: auth auth-arm64 auth-darwin-arm64 ## Build the binaries.
+
+build-strip: auth-arm64-strip ## Build a stripped binary, for which the version file needs to be rewritten.
+
+auth: deps
+	CGO_ENABLED=0 $(call BUILD_CMD,$(@),)
+
+auth-x86: deps
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(call BUILD_CMD,$(@),)
+
+auth-arm64: deps
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(call BUILD_CMD,$(@),)
 
-build-strip: deps ## Build a stripped binary, for which the version file needs to be rewritten.
-	echo "package utilities" > internal/utilities/version.go
-	echo "const Version = \"$(VERSION)\"" >> internal/utilities/version.go
+auth-darwin-arm64: deps
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(call BUILD_CMD,$(@),)
 
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build \
-		$(FLAGS) -ldflags "-s -w" -o auth-arm64-strip
+auth-arm64-strip: deps
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(call BUILD_CMD,$(@), -s)
 
 deps: ## Install dependencies.
 	@go mod download
 	@go mod verify
 
+release-test: \
+	vet \
+	static \
+	sec \
+	vulncheck \
+	test
+
+release: $(RELEASE_ARCHIVES)
+
+auth-$(VERSION)-%.tar.gz: \
+		release-%/auth \
+		release-%/gotrue | migrations
+	tar -C $(<D) -czvf $(@) auth gotrue -C ../ migrations/
+
+auth-$(VERSION)-arm64.tar.xz: \
+		release-arm64-strip/auth \
+		release-arm64-strip/gotrue | migrations
+	tar -C $(<D) -cf - auth gotrue -C ../ migrations/ \
+		| xz -T0 -9e -C crc64 > $(@)
+
+release-%/auth: auth-%
+	mkdir -p $(@D)
+	cp -a $(<) $(@)
+
+release-%/gotrue: release-%/auth
+	ln -sf $(<F) $(@)
+
 migrate_dev: ## Run database migrations for development.
 	hack/migrate.sh postgres
 
 migrate_test: ## Run database migrations for test.
 	hack/migrate.sh postgres
 
-test: build ## Run tests.
+test: auth ## Run tests.
 	go test $(CHECK_FILES) -coverprofile=coverage.out -coverpkg ./... -p 1 -race -v -count=1
 	./hack/coverage.sh
 
@@ -112,3 +159,10 @@ docker-clean: ## Remove the development containers and volumes
 
 format:
 	gofmt -s -w .
+
+clean:
+	rm -rf \
+		$(addprefix release-,$(RELEASE_TARGETS)) \
+		$(addprefix auth-,$(RELEASE_TARGETS)) \
+		$(RELEASE_ARCHIVES) \
+		auth