fix: failed to collect data from self-hosting gitlab

Author: klesh

plugins/gitlab/gitlab.go

@@ -2,6 +2,9 @@ package main // must be main for plugin entry point
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"strconv"
 
 	"github.com/merico-dev/lake/logger" // A pseudo type for Plugin Interface implementation
 	lakeModels "github.com/merico-dev/lake/models"
@@ -146,3 +149,33 @@ func (plugin Gitlab) ApiResources() map[string]map[string]core.ApiResourceHandle
 
 // Export a variable named PluginEntry for Framework to search and load
 var PluginEntry Gitlab //nolint
+
+// standalone mode for debugging
+func main() {
+	args := os.Args[1:]
+	if len(args) < 1 {
+		panic(fmt.Errorf("Usage: go run ./plugins/gitlab <project_id>"))
+	}
+	projectId, err := strconv.ParseFloat(args[0], 64)
+	if err != nil {
+		panic(fmt.Errorf("error paring board_id: %w", err))
+	}
+
+	PluginEntry.Init()
+	progress := make(chan float32)
+	go func() {
+		PluginEntry.Execute(
+			map[string]interface{}{
+				"projectId": projectId,
+			},
+			progress,
+			context.Background(),
+		)
+		if err != nil {
+			panic(err)
+		}
+	}()
+	for p := range progress {
+		fmt.Println(p)
+	}
+}

plugins/gitlab/tasks/gitlab_api_client.go

@@ -37,7 +37,7 @@ func CreateApiClient() *GitlabApiClient {
 
 type GitlabPaginationHandler func(res *http.Response) error
 
-func getTotal(resourceUriFormat string) (int, int, error) {
+func getTotal(resourceUriFormat string) (totalInt int, rateLimitPerSecond int, err error) {
 	// jsut get the first page of results. The response has a head that tells the total pages
 	page := 0
 	page_size := 1
@@ -53,16 +53,19 @@ func getTotal(resourceUriFormat string) (int, int, error) {
 		return 0, 0, err
 	}
 
-	totalInt := -1
+	totalInt = -1
 	total := res.Header.Get("X-Total")
 	if total != "" {
 		totalInt, err = convertStringToInt(total)
 		if err != nil {
 			return 0, 0, err
 		}
 	}
-
+	rateLimitPerSecond = 100
 	rateRemaining := res.Header.Get("ratelimit-remaining")
+	if rateRemaining == "" {
+		return
+	}
 	date, err := http.ParseTime(res.Header.Get("date"))
 	if err != nil {
 		return 0, 0, err
@@ -75,8 +78,8 @@ func getTotal(resourceUriFormat string) (int, int, error) {
 	if err != nil {
 		return 0, 0, err
 	}
-	rateLimitPerSecond := rateLimitInt / int(rateLimitResetTime.Unix()-date.Unix()) * 9 / 10
-	return totalInt, rateLimitPerSecond, nil
+	rateLimitPerSecond = rateLimitInt / int(rateLimitResetTime.Unix()-date.Unix()) * 9 / 10
+	return
 }
 
 func convertStringToInt(input string) (int, error) {