Merge remote-tracking branch 'upstream/main' into release/v0.1

Author: gabriel-samfira

.github/workflows/integration-tests.yml

@@ -0,0 +1,99 @@
+name: Integration Tests
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Golang
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+
+      - name: Setup LXD
+        uses: canonical/[email protected]
+
+      - name: Build GARM
+        run: make build
+
+      - name: Set up ngrok
+        id: ngrok
+        uses: gabriel-samfira/[email protected]
+        with:
+          ngrok_authtoken: ${{ secrets.NGROK_AUTH_TOKEN }}
+          port: 9997
+          tunnel_type: http
+
+      - name: Setup GARM
+        run: ./test/integration/scripts/setup-garm.sh
+        env:
+          GH_OAUTH_TOKEN: ${{ secrets.GH_OAUTH_TOKEN }}
+          CREDENTIALS_NAME: test-garm-creds
+          GARM_BASE_URL: ${{ steps.ngrok.outputs.tunnel-url }}
+
+      - name: Generate secrets
+        run: |
+          sudo apt-get -qq update && sudo apt-get -qq install -y apg
+
+          GARM_PASSWORD=$(apg -n1 -m32)
+          REPO_WEBHOOK_SECRET=$(apg -n1 -m32)
+          ORG_WEBHOOK_SECRET=$(apg -n1 -m32)
+
+          echo "::add-mask::$GARM_PASSWORD"
+          echo "::add-mask::$REPO_WEBHOOK_SECRET"
+          echo "::add-mask::$ORG_WEBHOOK_SECRET"
+
+          echo "GARM_PASSWORD=$GARM_PASSWORD" >> $GITHUB_ENV
+          echo "REPO_WEBHOOK_SECRET=$REPO_WEBHOOK_SECRET" >> $GITHUB_ENV
+          echo "ORG_WEBHOOK_SECRET=$ORG_WEBHOOK_SECRET" >> $GITHUB_ENV
+
+      - name: Create logs directory
+        if: always()
+        run: sudo mkdir -p /artifacts-logs && sudo chmod 777 /artifacts-logs
+
+      - name: Run integration tests
+        run: |
+          set -o pipefail
+          set -o errexit
+          go run ./test/integration/main.go 2>&1 | tee /artifacts-logs/e2e.log
+        env:
+          GARM_BASE_URL: ${{ steps.ngrok.outputs.tunnel-url }}
+          ORG_NAME: gsamfira
+          REPO_NAME: garm-testing
+          CREDENTIALS_NAME: test-garm-creds
+          WORKFLOW_FILE_NAME: test.yml
+          GH_TOKEN: ${{ secrets.GH_OAUTH_TOKEN }}
+
+      - name: Show GARM logs
+        if: always()
+        run: |
+          sudo systemctl status garm
+          sudo journalctl -u garm --no-pager 2>&1 | tee /artifacts-logs/garm.log
+
+      - name: Upload GARM and e2e logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: garm-logs
+          path: /artifacts-logs
+
+      - name: Cleanup orphan GARM resources via GitHub API
+        if: always()
+        run: |
+          set -o pipefail
+          set -o errexit
+
+          sudo systemctl stop garm
+
+          go run ./test/integration/gh_cleanup/main.go
+        env:
+          GARM_BASE_URL: ${{ steps.ngrok.outputs.tunnel-url }}
+          ORG_NAME: gsamfira
+          REPO_NAME: garm-testing
+          GH_TOKEN: ${{ secrets.GH_OAUTH_TOKEN }}

.gitignore

@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+*.DS_Store
 
 # Test binary, built with `go test -c`
 *.test
@@ -16,3 +17,5 @@ bin/
 # vendor/
 .vscode
 cmd/temp
+build/
+release/

Dockerfile

@@ -10,19 +10,28 @@ ADD . /build/garm
 RUN cd /build/garm && git checkout ${GARM_REF}
 RUN git clone https://github.com/cloudbase/garm-provider-azure /build/garm-provider-azure
 RUN git clone https://github.com/cloudbase/garm-provider-openstack /build/garm-provider-openstack
+RUN git clone https://github.com/cloudbase/garm-provider-lxd /build/garm-provider-lxd
+RUN git clone https://github.com/cloudbase/garm-provider-incus /build/garm-provider-incus
+RUN git clone https://github.com/mercedes-benz/garm-provider-k8s /build/garm-provider-k8s
 
 RUN cd /build/garm && go build -o /bin/garm \
     -tags osusergo,netgo,sqlite_omit_load_extension \
-    -ldflags "-linkmode external -extldflags '-static' -s -w -X main.Version=$(git describe --tags --match='v[0-9]*' --dirty --always)" \
+    -ldflags "-extldflags '-static' -s -w -X main.Version=$(git describe --tags --match='v[0-9]*' --dirty --always)" \
     /build/garm/cmd/garm
 RUN mkdir -p /opt/garm/providers.d
-RUN cd /build/garm-provider-azure && go build -ldflags="-linkmode external -extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-azure .
-RUN cd /build/garm-provider-openstack && go build -ldflags="-linkmode external -extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-openstack .
+RUN cd /build/garm-provider-azure && go build -ldflags="-extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-azure .
+RUN cd /build/garm-provider-openstack && go build -ldflags="-extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-openstack .
+RUN cd /build/garm-provider-lxd && go build -ldflags="-extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-lxd .
+RUN cd /build/garm-provider-incus && go build -ldflags="-extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-incus .
+RUN cd /build/garm-provider-k8s/cmd/garm-provider-k8s && go build -ldflags="-extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-k8s .
 
 FROM scratch
 
 COPY --from=builder /bin/garm /bin/garm
 COPY --from=builder /opt/garm/providers.d/garm-provider-openstack /opt/garm/providers.d/garm-provider-openstack
+COPY --from=builder /opt/garm/providers.d/garm-provider-lxd /opt/garm/providers.d/garm-provider-lxd
+COPY --from=builder /opt/garm/providers.d/garm-provider-incus /opt/garm/providers.d/garm-provider-incus
+COPY --from=builder /opt/garm/providers.d/garm-provider-k8s /opt/garm/providers.d/garm-provider-k8s
 COPY --from=builder /opt/garm/providers.d/garm-provider-azure /opt/garm/providers.d/garm-provider-azure
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 

Dockerfile.build-static

@@ -12,4 +12,6 @@ RUN wget http://musl.cc/aarch64-linux-musl-cross.tgz -O /tmp/aarch64-linux-musl-
 ADD ./scripts/build-static.sh /build-static.sh
 RUN chmod +x /build-static.sh
 
+ADD . /build/garm
+
 CMD ["/bin/sh"]

Makefile

@@ -7,17 +7,26 @@ USER_GROUP=$(shell ((docker --version | grep -q podman) && echo "0" || id -g))
 ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 GOPATH ?= $(shell go env GOPATH)
 VERSION ?= $(shell git describe --tags --match='v[0-9]*' --dirty --always)
+GARM_REF ?= $(shell git rev-parse --abbrev-ref HEAD)
 GO ?= go
 
 
 default: build
 
-.PHONY : build-static test install-lint-deps lint go-test fmt fmtcheck verify-vendor verify
+.PHONY : build-static test install-lint-deps lint go-test fmt fmtcheck verify-vendor verify create-release-files release
 build-static:
 	@echo Building garm
 	docker build --tag $(IMAGE_TAG) -f Dockerfile.build-static .
-	docker run --rm -e USER_ID=$(USER_ID) -e USER_GROUP=$(USER_GROUP) -v $(PWD):/build/garm:z $(IMAGE_TAG) /build-static.sh
-	@echo Binaries are available in $(PWD)/bin
+	docker run --rm -e USER_ID=$(USER_ID) -e GARM_REF=$(GARM_REF) -e USER_GROUP=$(USER_GROUP) -v $(PWD)/build:/build/output:z $(IMAGE_TAG) /build-static.sh
+	@echo Binaries are available in $(PWD)/build
+
+create-release-files:
+	./scripts/make-release.sh
+
+release: build-static create-release-files
+
+clean:
+	@rm -rf ./bin ./build ./release
 
 build:
 	@echo Building garm ${VERSION}

README.md

@@ -4,11 +4,13 @@
 
 Welcome to GARM!
 
-Garm enables you to create and automatically maintain pools of [self-hosted GitHub runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners), with autoscaling that can be used inside your github workflow runs.
+GARM enables you to create and automatically maintain pools of [self-hosted GitHub runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners), with auto-scaling that can be used inside your github workflow runs.
 
 The goal of ```GARM``` is to be simple to set up, simple to configure and simple to use. It is a single binary that can run on any GNU/Linux machine without any other requirements other than the providers it creates the runners in. It is intended to be easy to deploy in any environment and can create runners in any system you can write a provider for. There is no complicated setup process and no extremely complex concepts to understand. Once set up, it's meant to stay out of your way.
 
-Garm supports creating pools on either GitHub itself or on your own deployment of [GitHub Enterprise Server](https://docs.github.com/en/[email protected]/admin/overview/about-github-enterprise-server). For instructions on how to use ```GARM``` with GHE, see the [credentials](/doc/github_credentials.md) section of the documentation.
+GARM supports creating pools on either GitHub itself or on your own deployment of [GitHub Enterprise Server](https://docs.github.com/en/[email protected]/admin/overview/about-github-enterprise-server). For instructions on how to use ```GARM``` with GHE, see the [credentials](/doc/github_credentials.md) section of the documentation.
+
+Through the use of providers, `GARM` can create runners in a variety of environments using the same `GARM` instance. Want to create pools of runners in your OpenStack cloud, your Azure cloud and your Kubernetes cluster? No problem! Just install the appropriate providers, configure them in `GARM` and you're good to go. Create zero-runner pools for instances with high costs (large VMs, GPU enabled instances, etc) and have them spin up on demand, or create large pools of k8s backed runners that can be used for your CI/CD pipelines at a moment's notice. You can mix them up and create pools in any combination of providers or resource allocations you want.
 
 ## Join us on slack
 
@@ -18,14 +20,27 @@ Whether you're running into issues or just want to drop by and say "hi", feel fr
 
 ## Installing
 
+### On virtual or physical machines
+
 Check out the [quickstart](/doc/quickstart.md) document for instructions on how to install ```GARM```. If you'd like to build from source, check out the [building from source](/doc/building_from_source.md) document.
 
+### On Kubernetes
+
+Thanks to the efforts of the amazing folks at @mercedes-benz, GARM can now be integrated into k8s via their operator. Check out the [GARM operator](https://github.com/mercedes-benz/garm-operator/) for more details.
+
+## Supported providers
+
+GARM uses providers to create runners in a particular IaaS. The providers are external executables that GARM calls into to create runners. Before you can create runners, you'll need to install at least one provider.
+
 ## Installing external providers
 
-External providers are binaries that GARM calls into to create runners in a particular IaaS. There are currently two external providers available:
+External providers are binaries that GARM calls into to create runners in a particular IaaS. There are several external providers available:
 
 * [OpenStack](https://github.com/cloudbase/garm-provider-openstack)
 * [Azure](https://github.com/cloudbase/garm-provider-azure)
+* [Kubernetes](https://github.com/mercedes-benz/garm-provider-k8s) - Thanks to the amazing folks at @mercedes-benz for sharing their awesome provider!
+* [LXD](https://github.com/cloudbase/garm-provider-lxd)
+* [Incus](https://github.com/cloudbase/garm-provider-incus)
 
 Follow the instructions in the README of each provider to install them. 
 
@@ -47,8 +62,4 @@ If you would like to optimize the startup time of new instance, take a look at t
 
 ## Write your own provider
 
-The providers are interfaces between ```GARM``` and a particular IaaS in which we spin up GitHub Runners. These providers can be either **native** or **external**. The **native** providers are written in ```Go```, and must implement [the interface defined here](https://github.com/cloudbase/garm/blob/main/runner/common/provider.go#L22-L39). **External** providers can be written in any language, as they are in the form of an external executable that ```GARM``` calls into.
-
-There is currently one **native** provider for [LXD](https://linuxcontainers.org/lxd/) and two **external** providers for [Openstack and Azure](/contrib/providers.d/).
-
-If you want to write your own provider, you can choose to write a native one, or implement an **external** one. The easiest one to write is probably an **external** provider. Please see the [Writing an external provider](/doc/external_provider.md) document for details. Also, feel free to inspect the two available external providers in this repository.
+The providers are interfaces between ```GARM``` and a particular IaaS in which we spin up GitHub Runners. **External** providers can be written in any language, as they are in the form of an external executable that ```GARM``` calls into. Please see the [Writing an external provider](/doc/external_provider.md) document for details. Also, feel free to inspect the two available sample external providers in this repository.

apiserver/controllers/controllers.go

@@ -30,11 +30,16 @@ import (
 	"github.com/cloudbase/garm/runner"
 	wsWriter "github.com/cloudbase/garm/websocket"
 
+	"github.com/gorilla/mux"
 	"github.com/gorilla/websocket"
 	"github.com/pkg/errors"
 )
 
 func NewAPIController(r *runner.Runner, authenticator *auth.Authenticator, hub *wsWriter.Hub) (*APIController, error) {
+	controllerInfo, err := r.GetControllerInfo(auth.GetAdminContext())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get controller info")
+	}
 	return &APIController{
 		r:    r,
 		auth: authenticator,
@@ -43,18 +48,20 @@ func NewAPIController(r *runner.Runner, authenticator *auth.Authenticator, hub *
 			ReadBufferSize:  1024,
 			WriteBufferSize: 16384,
 		},
+		controllerID: controllerInfo.ControllerID.String(),
 	}, nil
 }
 
 type APIController struct {
-	r        *runner.Runner
-	auth     *auth.Authenticator
-	hub      *wsWriter.Hub
-	upgrader websocket.Upgrader
+	r            *runner.Runner
+	auth         *auth.Authenticator
+	hub          *wsWriter.Hub
+	upgrader     websocket.Upgrader
+	controllerID string
 }
 
 func handleError(w http.ResponseWriter, err error) {
-	w.Header().Add("Content-Type", "application/json")
+	w.Header().Set("Content-Type", "application/json")
 	origErr := errors.Cause(err)
 	apiErr := params.APIErrorResponse{
 		Details: origErr.Error(),
@@ -138,7 +145,19 @@ func (a *APIController) handleWorkflowJobEvent(w http.ResponseWriter, r *http.Re
 	labelValues = a.webhookMetricLabelValues("true", "")
 }
 
-func (a *APIController) CatchAll(w http.ResponseWriter, r *http.Request) {
+func (a *APIController) WebhookHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	controllerID, ok := vars["controllerID"]
+	// If the webhook URL includes a controller ID, we validate that it's meant for us. We still
+	// support bare webhook URLs, which are tipically configured manually by the user.
+	// The controllerID suffixed webhook URL is useful when configuring the webhook for an entity
+	// via garm. We cannot tag a webhook URL on github, so there is no way to determine ownership.
+	// Using a controllerID suffix is a simple way to denote ownership.
+	if ok && controllerID != a.controllerID {
+		log.Printf("ignoring webhook meant for controller %s", util.SanitizeLogEntry(controllerID))
+		return
+	}
+
 	headers := r.Header.Clone()
 
 	event := runnerParams.Event(headers.Get("X-Github-Event"))
@@ -195,8 +214,9 @@ func (a *APIController) NotFoundHandler(w http.ResponseWriter, r *http.Request)
 		Details: "Resource not found",
 		Error:   "Not found",
 	}
-	w.WriteHeader(http.StatusNotFound)
+
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusNotFound)
 	if err := json.NewEncoder(w).Encode(apiErr); err != nil {
 		log.Printf("failet to write response: %q", err)
 	}
@@ -377,3 +397,24 @@ func (a *APIController) ListAllJobs(w http.ResponseWriter, r *http.Request) {
 		log.Printf("failed to encode response: %q", err)
 	}
 }
+
+// swagger:route GET /controller-info controllerInfo ControllerInfo
+//
+// Get controller info.
+//
+//	Responses:
+//	  200: ControllerInfo
+//	  409: APIErrorResponse
+func (a *APIController) ControllerInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	info, err := a.r.GetControllerInfo(ctx)
+	if err != nil {
+		handleError(w, err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(info); err != nil {
+		log.Printf("failed to encode response: %q", err)
+	}
+}

apiserver/controllers/instances.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"log"
 	"net/http"
+	"strconv"
 
 	gErrors "github.com/cloudbase/garm-provider-common/errors"
 	"github.com/cloudbase/garm/apiserver/params"
@@ -121,6 +122,12 @@ func (a *APIController) GetInstanceHandler(w http.ResponseWriter, r *http.Reques
 //	    in: path
 //	    required: true
 //
+//	  + name: forceRemove
+//	    description: If true GARM will ignore any provider error when removing the runner and will continue to remove the runner from github and the GARM database.
+//	    type: boolean
+//	    in: query
+//	    required: false
+//
 //	Responses:
 //	  default: APIErrorResponse
 func (a *APIController) DeleteInstanceHandler(w http.ResponseWriter, r *http.Request) {
@@ -138,7 +145,8 @@ func (a *APIController) DeleteInstanceHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if err := a.r.ForceDeleteRunner(ctx, instanceName); err != nil {
+	forceRemove, _ := strconv.ParseBool(r.URL.Query().Get("forceRemove"))
+	if err := a.r.DeleteRunner(ctx, instanceName, forceRemove); err != nil {
 		log.Printf("removing runner: %s", err)
 		handleError(w, err)
 		return
@@ -316,19 +324,3 @@ func (a *APIController) InstanceStatusMessageHandler(w http.ResponseWriter, r *h
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
 }
-
-func (a *APIController) InstanceGithubRegistrationTokenHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
-	token, err := a.r.GetInstanceGithubRegistrationToken(ctx)
-	if err != nil {
-		handleError(w, err)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write([]byte(token)); err != nil {
-		log.Printf("failed to encode response: %q", err)
-	}
-}