From 46e4fb9e9e88713c813eb9825c7bc5071bf2c313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Tue, 24 Mar 2026 17:10:03 +0100 Subject: [PATCH 01/11] Fix concurrency error... wrong mutex (fml) --- pkg/store/local.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/store/local.go b/pkg/store/local.go index 0c150ab..bf6b857 100644 --- a/pkg/store/local.go +++ b/pkg/store/local.go @@ -212,9 +212,9 @@ func (l *Local) GetRunner(ctx context.Context, runner *schemas.Runner) error { exists, _ := l.RunnerExists(ctx, runner.Key()) if exists { - l.environmentsMutex.RLock() // Lock the mutex for read-only access + l.runnersMutex.RLock() // Lock the mutex for read-only access *runner = l.runners[runner.Key()] // Retrieve the runner - l.environmentsMutex.RUnlock() // Unlock the mutex + l.runnersMutex.RUnlock() // Unlock the mutex } return nil From 4f69d2c9e32cafbb9d1710c86b6ffb2679adccb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Tue, 24 Mar 2026 17:10:31 +0100 Subject: [PATCH 02/11] Add new local storage tests --- pkg/store/local_test.go | 394 +++++++++++++++++++++++++--------------- 1 file changed, 246 insertions(+), 148 deletions(-) diff --git a/pkg/store/local_test.go b/pkg/store/local_test.go index 3ab9c9f..784df68 100644 --- a/pkg/store/local_test.go +++ b/pkg/store/local_test.go @@ -1,271 +1,369 @@ package store import ( + "context" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" ) +func newTestLocalStore(t *testing.T) *Local { + t.Helper() + + s, ok := NewLocalStore().(*Local) + require.True(t, ok) + + return s +} + +func TestLocalHasExpiredAlwaysFalse(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + + assert.False(t, s.HasProjectExpired(ctx, schemas.ProjectKey("p1"))) + assert.False(t, s.HasEnvExpired(ctx, schemas.EnvironmentKey("e1"))) + assert.False(t, s.HasRunnerExpired(ctx, schemas.RunnerKey("r1"))) + assert.False(t, s.HasRefExpired(ctx, schemas.RefKey("ref1"))) + assert.False(t, s.HasMetricExpired(ctx, schemas.MetricKey("m1"))) +} + func TestLocalProjectFunctions(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + p := schemas.NewProject("foo/bar") - p.OutputSparseStatusMetrics = false + p.Topics = "go,ci" - l := NewLocalStore() - assert.NoError(t, l.SetProject(testCtx, p)) + assert.NoError(t, s.SetProject(ctx, p)) - // Set project - projects, err := l.Projects(testCtx) + projects, err := s.Projects(ctx) assert.NoError(t, err) assert.Contains(t, projects, p.Key()) assert.Equal(t, p, projects[p.Key()]) - // Project exists - exists, err := l.ProjectExists(testCtx, p.Key()) + exists, err := s.ProjectExists(ctx, p.Key()) assert.NoError(t, err) assert.True(t, exists) - // GetProject should succeed - newProject := schemas.NewProject("foo/bar") - assert.NoError(t, l.GetProject(testCtx, &newProject)) - assert.Equal(t, p, newProject) + got := schemas.NewProject("foo/bar") + assert.NoError(t, s.GetProject(ctx, &got)) + assert.Equal(t, p, got) - // Count - count, err := l.ProjectsCount(testCtx) + count, err := s.ProjectsCount(ctx) assert.NoError(t, err) assert.Equal(t, int64(1), count) - // Delete project - assert.NoError(t, l.DelProject(testCtx, p.Key())) - projects, err = l.Projects(testCtx) + assert.NoError(t, s.DelProject(ctx, p.Key())) + + projects, err = s.Projects(ctx) assert.NoError(t, err) assert.NotContains(t, projects, p.Key()) - exists, err = l.ProjectExists(testCtx, p.Key()) + exists, err = s.ProjectExists(ctx, p.Key()) assert.NoError(t, err) assert.False(t, exists) - - // GetProject should not update the var this time - newProject = schemas.NewProject("foo/bar") - assert.NoError(t, l.GetProject(testCtx, &newProject)) - assert.NotEqual(t, p, newProject) } func TestLocalEnvironmentFunctions(t *testing.T) { - environment := schemas.Environment{ - ProjectName: "foo", + s := newTestLocalStore(t) + ctx := context.Background() + + env := schemas.Environment{ + ProjectName: "foo/bar", ID: 1, + Name: "production", + ExternalURL: "https://example.com", + Available: true, } - l := NewLocalStore() - assert.NoError(t, l.SetEnvironment(testCtx, environment)) + assert.NoError(t, s.SetEnvironment(ctx, env)) - // Set project - environments, err := l.Environments(testCtx) + envs, err := s.Environments(ctx) assert.NoError(t, err) - assert.Contains(t, environments, environment.Key()) - assert.Equal(t, environment, environments[environment.Key()]) + assert.Contains(t, envs, env.Key()) + assert.Equal(t, env, envs[env.Key()]) - // Environment exists - exists, err := l.EnvironmentExists(testCtx, environment.Key()) + exists, err := s.EnvironmentExists(ctx, env.Key()) assert.NoError(t, err) assert.True(t, exists) - // GetEnvironment should succeed - newEnvironment := schemas.Environment{ - ProjectName: "foo", - ID: 1, + got := schemas.Environment{ + ProjectName: "foo/bar", + Name: "production", } - assert.NoError(t, l.GetEnvironment(testCtx, &newEnvironment)) - assert.Equal(t, environment, newEnvironment) + assert.NoError(t, s.GetEnvironment(ctx, &got)) + assert.Equal(t, env, got) - // Count - count, err := l.EnvironmentsCount(testCtx) + count, err := s.EnvironmentsCount(ctx) assert.NoError(t, err) assert.Equal(t, int64(1), count) - // Delete Environment - assert.NoError(t, l.DelEnvironment(testCtx, environment.Key())) - environments, err = l.Environments(testCtx) + assert.NoError(t, s.DelEnvironment(ctx, env.Key())) + + envs, err = s.Environments(ctx) assert.NoError(t, err) - assert.NotContains(t, environments, environment.Key()) + assert.NotContains(t, envs, env.Key()) - exists, err = l.EnvironmentExists(testCtx, environment.Key()) + exists, err = s.EnvironmentExists(ctx, env.Key()) assert.NoError(t, err) assert.False(t, exists) +} - // GetEnvironment should not update the var this time - newEnvironment = schemas.Environment{ - ProjectName: "foo", - ID: 1, - ExternalURL: "foo", +func TestLocalRunnerFunctions(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + + runner := schemas.Runner{ + ID: 123, + Description: "shared-runner", + Name: "runner-01", + ProjectName: "foo/bar", + Online: true, + Status: "online", + TagList: []string{"docker", "linux"}, } - assert.NoError(t, l.GetEnvironment(testCtx, &newEnvironment)) - assert.NotEqual(t, environment, newEnvironment) + + assert.NoError(t, s.SetRunner(ctx, runner)) + + runners, err := s.Runners(ctx) + assert.NoError(t, err) + assert.Contains(t, runners, runner.Key()) + assert.Equal(t, runner, runners[runner.Key()]) + + exists, err := s.RunnerExists(ctx, runner.Key()) + assert.NoError(t, err) + assert.True(t, exists) + + got := schemas.Runner{ID: 123} + assert.NoError(t, s.GetRunner(ctx, &got)) + assert.Equal(t, runner, got) + + count, err := s.RunnersCount(ctx) + assert.NoError(t, err) + assert.Equal(t, int64(1), count) + + assert.NoError(t, s.DelRunner(ctx, runner.Key())) + + runners, err = s.Runners(ctx) + assert.NoError(t, err) + assert.NotContains(t, runners, runner.Key()) + + exists, err = s.RunnerExists(ctx, runner.Key()) + assert.NoError(t, err) + assert.False(t, exists) } func TestLocalRefFunctions(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + p := schemas.NewProject("foo/bar") - p.Topics = "salty" - ref := schemas.NewRef( - p, - schemas.RefKindBranch, - "sweet", - ) + p.Topics = "topic1" + + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{ + ID: 100, + Status: "success", + Source: "push", + Variables: `{"foo":"bar"}`, + } - // Set project - l := NewLocalStore() - assert.NoError(t, l.SetRef(testCtx, ref)) + assert.NoError(t, s.SetRef(ctx, ref)) - projectsRefs, err := l.Refs(testCtx) + refs, err := s.Refs(ctx) assert.NoError(t, err) - assert.Contains(t, projectsRefs, ref.Key()) - assert.Equal(t, ref, projectsRefs[ref.Key()]) + assert.Contains(t, refs, ref.Key()) + assert.Equal(t, ref, refs[ref.Key()]) - // Ref exists - exists, err := l.RefExists(testCtx, ref.Key()) + exists, err := s.RefExists(ctx, ref.Key()) assert.NoError(t, err) assert.True(t, exists) - // GetRef should succeed - newRef := schemas.Ref{ - Project: schemas.NewProject("foo/bar"), - Kind: schemas.RefKindBranch, - Name: "sweet", - } - assert.NoError(t, l.GetRef(testCtx, &newRef)) - assert.Equal(t, ref, newRef) + got := schemas.NewRef(p, schemas.RefKindBranch, "main") + assert.NoError(t, s.GetRef(ctx, &got)) + assert.Equal(t, ref, got) - // Count - count, err := l.RefsCount(testCtx) + count, err := s.RefsCount(ctx) assert.NoError(t, err) assert.Equal(t, int64(1), count) - // Delete Ref - assert.NoError(t, l.DelRef(testCtx, ref.Key())) - projectsRefs, err = l.Refs(testCtx) + assert.NoError(t, s.DelRef(ctx, ref.Key())) + + refs, err = s.Refs(ctx) assert.NoError(t, err) - assert.NotContains(t, projectsRefs, ref.Key()) + assert.NotContains(t, refs, ref.Key()) - exists, err = l.RefExists(testCtx, ref.Key()) + exists, err = s.RefExists(ctx, ref.Key()) assert.NoError(t, err) assert.False(t, exists) - - // GetRef should not update the var this time - newRef = schemas.Ref{ - Kind: schemas.RefKindBranch, - Project: schemas.NewProject("foo/bar"), - Name: "sweet", - } - assert.NoError(t, l.GetRef(testCtx, &newRef)) - assert.NotEqual(t, ref, newRef) } func TestLocalMetricFunctions(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + m := schemas.Metric{ Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{ - "foo": "bar", + "project": "foo/bar", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": `{"foo":"bar"}`, + "pipeline_id": "123", + "status": "success", }, - Value: 5, + Value: 99.9, } - l := NewLocalStore() - assert.NoError(t, l.SetMetric(testCtx, m)) + assert.NoError(t, s.SetMetric(ctx, m)) - // Set metric - metrics, err := l.Metrics(testCtx) + metrics, err := s.Metrics(ctx) assert.NoError(t, err) assert.Contains(t, metrics, m.Key()) assert.Equal(t, m, metrics[m.Key()]) - // Metric exists - exists, err := l.MetricExists(testCtx, m.Key()) + exists, err := s.MetricExists(ctx, m.Key()) assert.NoError(t, err) assert.True(t, exists) - // GetMetric should succeed - newMetric := schemas.Metric{ + got := schemas.Metric{ Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{ - "foo": "bar", + "project": "foo/bar", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": `{"foo":"bar"}`, + "pipeline_id": "123", + "status": "success", }, } - assert.NoError(t, l.GetMetric(testCtx, &newMetric)) - assert.Equal(t, m, newMetric) + assert.NoError(t, s.GetMetric(ctx, &got)) + assert.Equal(t, m, got) - // Count - count, err := l.MetricsCount(testCtx) + count, err := s.MetricsCount(ctx) assert.NoError(t, err) assert.Equal(t, int64(1), count) - // Delete Metric - _ = l.DelMetric(testCtx, m.Key()) - metrics, err = l.Metrics(testCtx) + assert.NoError(t, s.DelMetric(ctx, m.Key())) + + metrics, err = s.Metrics(ctx) assert.NoError(t, err) assert.NotContains(t, metrics, m.Key()) - exists, err = l.MetricExists(testCtx, m.Key()) + exists, err = s.MetricExists(ctx, m.Key()) assert.NoError(t, err) assert.False(t, exists) +} - // GetMetric should not update the var this time - newMetric = schemas.Metric{ - Kind: schemas.MetricKindCoverage, - Labels: prometheus.Labels{ - "foo": "bar", - }, +func TestLocalPipelineFunctions(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + + pipeline := schemas.Pipeline{ + ID: 77, + Coverage: 85.5, + Source: "push", + Status: "success", + Variables: `{"foo":"bar"}`, } - assert.NoError(t, l.GetMetric(testCtx, &newMetric)) - assert.NotEqual(t, m, newMetric) + + assert.NoError(t, s.SetPipeline(ctx, pipeline)) + + exists, err := s.PipelineExists(ctx, pipeline.Key()) + assert.NoError(t, err) + assert.True(t, exists) + + got := schemas.Pipeline{ID: 77} + assert.NoError(t, s.GetPipeline(ctx, &got)) + assert.Equal(t, pipeline, got) +} + +func TestLocalPipelineVariablesFunctions(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + + pipeline := schemas.Pipeline{ID: 88} + vars := `{"ENV":"prod"}` + + exists, err := s.PipelineVariablesExists(ctx, pipeline) + assert.NoError(t, err) + assert.False(t, exists) + + value, err := s.GetPipelineVariables(ctx, pipeline) + assert.NoError(t, err) + assert.Equal(t, "", value) + + assert.NoError(t, s.SetPipelineVariables(ctx, pipeline, vars)) + + exists, err = s.PipelineVariablesExists(ctx, pipeline) + assert.NoError(t, err) + assert.True(t, exists) + + value, err = s.GetPipelineVariables(ctx, pipeline) + assert.NoError(t, err) + assert.Equal(t, vars, value) } func TestLocalQueueTask(t *testing.T) { - l := NewLocalStore() - ok, err := l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") - assert.True(t, ok) + s := newTestLocalStore(t) + ctx := context.Background() + + ok, err := s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-1", "") assert.NoError(t, err) + assert.True(t, ok) - ok, err = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") + ok, err = s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-1", "") + assert.NoError(t, err) assert.False(t, ok) +} + +func TestLocalDequeueTaskAndExecutedCount(t *testing.T) { + s := newTestLocalStore(t) + ctx := context.Background() + + ok, err := s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-1", "") assert.NoError(t, err) + assert.True(t, ok) - _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") - ok, err = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") - assert.False(t, ok) + count, err := s.ExecutedTasksCount(ctx) assert.NoError(t, err) -} + assert.Equal(t, uint64(0), count) + + assert.NoError(t, s.DequeueTask(ctx, schemas.TaskTypePullMetrics, "task-1")) + + count, err = s.ExecutedTasksCount(ctx) + assert.NoError(t, err) + assert.Equal(t, uint64(1), count) -func TestLocalDequeueTask(t *testing.T) { - l := NewLocalStore() - _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") - assert.Equal(t, uint64(0), l.(*Local).executedTasksCount) - assert.NoError(t, l.DequeueTask(testCtx, schemas.TaskTypePullMetrics, "foo")) - assert.Equal(t, uint64(1), l.(*Local).executedTasksCount) + queued, err := s.CurrentlyQueuedTasksCount(ctx) + assert.NoError(t, err) + assert.Equal(t, uint64(0), queued) } func TestLocalCurrentlyQueuedTasksCount(t *testing.T) { - l := NewLocalStore() - _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") - _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") - _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "baz", "") + s := newTestLocalStore(t) + ctx := context.Background() + + _, err := s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-1", "") + assert.NoError(t, err) + _, err = s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-2", "") + assert.NoError(t, err) + _, err = s.QueueTask(ctx, schemas.TaskTypeGarbageCollectEnvironments, "task-3", "") + assert.NoError(t, err) - count, _ := l.CurrentlyQueuedTasksCount(testCtx) + count, err := s.CurrentlyQueuedTasksCount(ctx) + assert.NoError(t, err) assert.Equal(t, uint64(3), count) - assert.NoError(t, l.DequeueTask(testCtx, schemas.TaskTypePullMetrics, "foo")) - count, _ = l.CurrentlyQueuedTasksCount(testCtx) - assert.Equal(t, uint64(2), count) -} -func TestLocalExecutedTasksCount(t *testing.T) { - l := NewLocalStore() - _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") - _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") - _ = l.DequeueTask(testCtx, schemas.TaskTypePullMetrics, "foo") - _ = l.DequeueTask(testCtx, schemas.TaskTypePullMetrics, "foo") + assert.NoError(t, s.DequeueTask(ctx, schemas.TaskTypePullMetrics, "task-1")) - count, _ := l.ExecutedTasksCount(testCtx) - assert.Equal(t, uint64(1), count) + count, err = s.CurrentlyQueuedTasksCount(ctx) + assert.NoError(t, err) + assert.Equal(t, uint64(2), count) } From 7ae6a0c0f49e6cec930a5508c3f8d95772c9664f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Wed, 25 Mar 2026 10:51:26 +0100 Subject: [PATCH 03/11] Fix bug, change until to Since --- pkg/ratelimit/ratelimitv2.go | 39 ------------------------------------ pkg/ratelimit/redis.go | 2 +- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 pkg/ratelimit/ratelimitv2.go diff --git a/pkg/ratelimit/ratelimitv2.go b/pkg/ratelimit/ratelimitv2.go deleted file mode 100644 index 7c4657f..0000000 --- a/pkg/ratelimit/ratelimitv2.go +++ /dev/null @@ -1,39 +0,0 @@ -package ratelimit - -import ( - "net/http" // Package for HTTP client and server implementations - "time" // Package for time-related operations - - "golang.org/x/time/rate" // Package for rate limiting -) - -// ThrottledTransport is a custom HTTP transport that implements rate limiting. -type ThrottledTransport struct { - roundTripper http.RoundTripper // The underlying HTTP transport to use for making requests - rateLimiter *rate.Limiter // The rate limiter to control the rate of requests -} - -// RoundTrip implements the RoundTripper interface for ThrottledTransport. -// It ensures that requests are made according to the rate limit. -func (t *ThrottledTransport) RoundTrip(req *http.Request) (*http.Response, error) { - // Wait until the rate limiter allows the request to proceed - err := t.rateLimiter.Wait(req.Context()) - if err != nil { - // Return an error if the rate limiter fails - return nil, err - } - - // Use the underlying round tripper to make the HTTP request - return t.roundTripper.RoundTrip(req) -} - -// NewThrottledTransport creates a new ThrottledTransport with the specified rate limit. -func NewThrottledTransport(limitPeriod time.Duration, requestCount int, transportWrap http.RoundTripper) http.RoundTripper { - // Create and return a new ThrottledTransport with the specified rate limit - // Example usage: client := &http.Client{Transport: NewThrottledTransport(10*time.Second, 60, http.DefaultTransport)} - // This allows 60 requests every 10 seconds. - return &ThrottledTransport{ - roundTripper: transportWrap, // The underlying transport to use for HTTP requests - rateLimiter: rate.NewLimiter(rate.Every(limitPeriod), requestCount), // Create a new rate limiter - } -} diff --git a/pkg/ratelimit/redis.go b/pkg/ratelimit/redis.go index 712afbd..b622b40 100644 --- a/pkg/ratelimit/redis.go +++ b/pkg/ratelimit/redis.go @@ -58,5 +58,5 @@ func (r Redis) Take(ctx context.Context) time.Duration { } // Return the duration taken to allow the request - return time.Until(start) + return time.Since(start) } From de0b741ba16acdb3984f64ecdfc6024e8131ea8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Wed, 25 Mar 2026 13:55:58 +0100 Subject: [PATCH 04/11] Fix pagination bug --- pkg/gitlab/jobs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/gitlab/jobs.go b/pkg/gitlab/jobs.go index e102c2d..dac41dc 100644 --- a/pkg/gitlab/jobs.go +++ b/pkg/gitlab/jobs.go @@ -369,6 +369,8 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo goGitlab.WithContext(ctx), goGitlab.WithKeysetPaginationParameters(resp.NextLink), } + } else { + opt.Page = resp.NextPage } } From fc851e85217852df186b84b913304dcb2e19e888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Wed, 25 Mar 2026 13:56:48 +0100 Subject: [PATCH 05/11] Fix exit before the runners struct is assigned --- pkg/gitlab/runners.go | 85 ++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 58 deletions(-) diff --git a/pkg/gitlab/runners.go b/pkg/gitlab/runners.go index b6a7946..875dd06 100644 --- a/pkg/gitlab/runners.go +++ b/pkg/gitlab/runners.go @@ -138,36 +138,14 @@ func (c *Client) GetRunner(ctx context.Context, project string, runnerID int) (r c.requestsRemaining(resp) // Fill runner details from API response - runner.Name = r.Name runner.ID = r.ID - - // Mark environment as available if its state is "available" - /** - if e.State == "available" { - runner.Available = true - } - */ - - // If the runner has not recorded last details, log and return as is - if r.Groups == nil { - log.WithContext(ctx). - WithFields(log.Fields{ - "project-name": project, - "runner-name": r.Name, - "runner-group": r.Groups, - }). - Debug("no Group found for this runner") - return - } - - // Fill with the rest of the last runner details + runner.Name = r.Name runner.Paused = r.Paused runner.Description = r.Description runner.IsShared = r.IsShared runner.RunnerType = r.RunnerType runner.ContactedAt = r.ContactedAt runner.MaintenanceNote = r.MaintenanceNote - runner.Name = r.Name runner.Online = r.Online runner.Status = r.Status runner.Token = r.Token @@ -176,40 +154,31 @@ func (c *Client) GetRunner(ctx context.Context, project string, runnerID int) (r runner.Locked = r.Locked runner.AccessLevel = r.AccessLevel runner.MaximumTimeout = r.MaximumTimeout - runner.Groups = []struct { - ID int - Name string - WebURL string - }(r.Groups) - runner.Projects = []struct { - ID int - Name string - NameWithNamespace string - Path string - PathWithNamespace string - }(r.Projects) - /* - fmt.Printf("===============\nRunner infos:\n"+ - "Runner Paused:%v\n"+ - "Runner Desc:%v\n"+ - "RunnerIsShared:%v\n"+ - "RunnerType:%v\n"+ - "RunnerContact:%v\n"+ - "RunnerMaintenance:%v\n"+ - "RunnerName:%v\n"+ - "RunnerOnline:%v\n"+ - "Runner Status:%v\n"+ - "Runner Token:%v\n"+ - "Runner TagList:%v\n"+ - "Runner Untagged:%v\n"+ - "RunnerLocked:%v\n"+ - "RunnerAccessLevel:%v\n"+ - "RunnerMaxTimeout:%v\n"+ - "Runner Groups:%v\n"+ - "Runner Projects:%v\n"+ - "=====================\n", - r.Paused, r.Description, r.IsShared, r.RunnerType, r.ContactedAt, r.MaintenanceNote, r.Name, r.Online, r.Status, r.Token, r.TagList, r.RunUntagged, r.Locked, r.AccessLevel, r.MaximumTimeout, r.Groups, r.Projects) - */ - // Return the populated Runner struct and nil error + + // If the runner has not recorded last details, log and return as is + if r.Groups == nil { + log.WithContext(ctx). + WithFields(log.Fields{ + "project-name": project, + "runner-name": r.Name, + }). + Debug("no group found for this runner") + } else { + runner.Groups = []struct { + ID int + Name string + WebURL string + }(r.Groups) + } + + if r.Projects != nil { + runner.Projects = []struct { + ID int + Name string + NameWithNamespace string + Path string + PathWithNamespace string + }(r.Projects) + } return } From 27b2b266ecb0cd10cee59afe50a57f71521d322c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Wed, 25 Mar 2026 15:07:50 +0100 Subject: [PATCH 06/11] Improve output for projects and groups --- pkg/controller/runners.go | 62 +++++++++++++++++++++++---------------- pkg/schemas/runners.go | 45 ++++++++++++++++------------ 2 files changed, 63 insertions(+), 44 deletions(-) diff --git a/pkg/controller/runners.go b/pkg/controller/runners.go index 30a0c07..f55128e 100644 --- a/pkg/controller/runners.go +++ b/pkg/controller/runners.go @@ -2,7 +2,6 @@ package controller import ( "context" - "encoding/json" "strconv" "strings" @@ -72,12 +71,12 @@ func (c *Controller) UpdateRunner(ctx context.Context, runner *schemas.Runner) e return err } - // Update the local runner fields with the latest data - runner.Paused = pulledRunner.Paused - runner.ContactedAt = pulledRunner.ContactedAt - runner.MaintenanceNote = pulledRunner.MaintenanceNote + pulledRunner.ProjectName = runner.ProjectName + pulledRunner.OutputSparseStatusMetrics = runner.OutputSparseStatusMetrics + *runner = pulledRunner log.WithFields(projectRefLogFields).Info("update runner metrics") + // Save the updated runner back to the store return c.Store.SetRunner(ctx, *runner) } @@ -94,32 +93,45 @@ func (c *Controller) ProcessRunnerMetrics(ctx context.Context, runner schemas.Ru } // Initialize labels from the reference default labels and add job-specific labels - groups := runner.Groups - GroupsOut, err := json.Marshal(groups) - if err != nil { - return nil + groupNames := make([]string, 0, len(runner.Groups)) + for _, g := range runner.Groups { + if g.Name != "" { + groupNames = append(groupNames, g.Name) + } } - projects := runner.Projects - projectsOut, err := json.Marshal(projects) - if err != nil { - return nil + + projectNames := make([]string, 0, len(runner.Projects)) + for _, p := range runner.Projects { + switch { + case p.PathWithNamespace != "": + projectNames = append(projectNames, p.PathWithNamespace) + case p.NameWithNamespace != "": + projectNames = append(projectNames, p.NameWithNamespace) + case p.Name != "": + projectNames = append(projectNames, p.Name) + } } + tags := strings.Join(runner.TagList, ",") labels := runner.DefaultLabelsValues() labels["runner_name"] = runner.Name - labels["runner_id"] = strconv.Itoa(runner.ID) // The unique identifier for the environment - labels["is_shared"] = strconv.FormatBool(runner.IsShared) // The kind of the latest deployment's reference - labels["runner_type"] = runner.RunnerType // The name of the latest deployment's reference - labels["online"] = strconv.FormatBool(runner.Online) // The short ID of the current commit - labels["tag_list"] = tags // Placeholder for the latest commit short ID (empty in this context) - labels["active"] = strconv.FormatBool(runner.Paused) // The availability status of the environment - labels["runner_maintenance_note"] = runner.MaintenanceNote // Maintenance note label - labels["contacted_at"] = strconv.FormatInt(runner.ContactedAt.UTC().UnixNano(), 10) // Last contact with gitlab server - labels["status"] = runner.Status // The status of the runner - labels["paused"] = strconv.FormatBool(runner.Paused) // Define if the runner is paused - labels["runner_groups"] = string(GroupsOut) // The groups assigned to this runner - labels["runner_projects"] = string(projectsOut) // The projects assigned to this runner + labels["runner_id"] = strconv.Itoa(runner.ID) // The unique identifier for the environment + labels["is_shared"] = strconv.FormatBool(runner.IsShared) // The kind of the latest deployment's reference + labels["runner_type"] = runner.RunnerType // The name of the latest deployment's reference + labels["online"] = strconv.FormatBool(runner.Online) // The short ID of the current commit + labels["tag_list"] = tags // Placeholder for the latest commit short ID (empty in this context) + labels["active"] = strconv.FormatBool(runner.Paused) // The availability status of the environment + labels["runner_maintenance_note"] = runner.MaintenanceNote // Maintenance note label + labels["status"] = runner.Status // The status of the runner + labels["paused"] = strconv.FormatBool(runner.Paused) // Define if the runner is paused + labels["runner_groups"] = strings.Join(groupNames, ",") // The groups assigned to this runner + labels["runner_projects"] = strings.Join(projectNames, ",") // The projects assigned to this runner + if runner.ContactedAt != nil { + labels["contacted_at"] = strconv.FormatInt(runner.ContactedAt.UTC().UnixNano(), 10) // Last contact with gitlab server + } else { + labels["contacted_at"] = "" + } // Log trace info indicating that job metrics are being processed log.WithFields(projectRefLogFields).Info("processing runner metrics") diff --git a/pkg/schemas/runners.go b/pkg/schemas/runners.go index a6da3e3..05b4446 100644 --- a/pkg/schemas/runners.go +++ b/pkg/schemas/runners.go @@ -1,7 +1,6 @@ package schemas import ( - "encoding/json" "fmt" "hash/crc32" "strconv" @@ -71,30 +70,38 @@ func (r Runner) InformationLabelsValues() (v map[string]string) { v = r.DefaultLabelsValues() // Marshal Groups and projects - groups := r.Groups - GroupsOut, err := json.Marshal(groups) - if err != nil { - return nil + groupNames := make([]string, 0, len(r.Groups)) + for _, g := range r.Groups { + if g.Name != "" { + groupNames = append(groupNames, g.Name) + } } - projects := r.Projects - projectsOut, err := json.Marshal(projects) - if err != nil { - return nil + + projectNames := make([]string, 0, len(r.Projects)) + for _, p := range r.Projects { + switch { + case p.PathWithNamespace != "": + projectNames = append(projectNames, p.PathWithNamespace) + case p.NameWithNamespace != "": + projectNames = append(projectNames, p.NameWithNamespace) + case p.Name != "": + projectNames = append(projectNames, p.Name) + } } tags := strings.Join(r.TagList, ",") // Add additional detailed label values - v["runner_name"] = r.Name // The name of the runner - v["runner_id"] = strconv.Itoa(r.ID) // The unique identifier for the environment - v["is_shared"] = strconv.FormatBool(r.IsShared) // The kind of the latest deployment's reference - v["runner_type"] = r.RunnerType // The name of the latest deployment's reference - v["online"] = strconv.FormatBool(r.Online) // The short ID of the current commit - v["tag_list"] = tags // Placeholder for the latest commit short ID (empty in this context) - v["active"] = strconv.FormatBool(r.Paused) // The availability status of the environment - v["status"] = r.Status // The status of the runner - v["runner_groups"] = string(GroupsOut) // The groups assigned to this runner - v["runner_projects"] = string(projectsOut) // The projects assigned to this runner + v["runner_name"] = r.Name // The name of the runner + v["runner_id"] = strconv.Itoa(r.ID) // The unique identifier for the environment + v["is_shared"] = strconv.FormatBool(r.IsShared) // The kind of the latest deployment's reference + v["runner_type"] = r.RunnerType // The name of the latest deployment's reference + v["online"] = strconv.FormatBool(r.Online) // The short ID of the current commit + v["tag_list"] = tags // Placeholder for the latest commit short ID (empty in this context) + v["active"] = strconv.FormatBool(r.Paused) // The availability status of the environment + v["status"] = r.Status // The status of the runner + v["runner_groups"] = strings.Join(groupNames, ",") // The groups assigned to this runner + v["runner_projects"] = strings.Join(projectNames, ",") // The projects assigned to this runner fmt.Printf("Runner Labels:\n%v\n", v) From e95c4f28f92dacdd79e3d50b156f2975a3d19f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Thu, 26 Mar 2026 09:53:58 +0100 Subject: [PATCH 07/11] Add new metrics for runners --- pkg/controller/collectors.go | 98 ++++++++++++- pkg/controller/garbage_collector.go | 47 +++--- pkg/controller/metrics.go | 4 + pkg/controller/runners.go | 217 +++++++++++++++++++++------- pkg/schemas/metric.go | 50 +++++-- 5 files changed, 331 insertions(+), 85 deletions(-) diff --git a/pkg/controller/collectors.go b/pkg/controller/collectors.go index f65ff26..cead581 100644 --- a/pkg/controller/collectors.go +++ b/pkg/controller/collectors.go @@ -42,10 +42,42 @@ var ( "success", "failed", "canceled", "skipped", "manual", "scheduled", "error", "success_with_warnings", } - // runnerLabels defines labels for metrics related to runner deploy. + // runnerLabels defines labels for the global runner information metric. + // This metric is intended to expose one logical series per runner ID. runnerLabels = []string{ - "project", "runner_description", "runner_name", "runner_id", "is_shared", "runner_type", "runner_projects", - "online", "tag_list", "active", "status", "runner_groups", "runner_maintenance_note", "contacted_at", "paused", + "runner_id", + "runner_name", + "runner_description", + "is_shared", + "runner_type", + "online", + "active", + "status", + "runner_maintenance_note", + "paused", + } + + // runnerProjectLabels defines labels for the relationship between a runner and a project. + runnerProjectLabels = []string{ + "runner_id", + "project", + } + + // runnerTagLabels defines labels for the relationship between a runner and a tag. + runnerTagLabels = []string{ + "runner_id", + "tag", + } + + // runnerGroupLabels defines labels for the relationship between a runner and a group. + runnerGroupLabels = []string{ + "runner_id", + "group", + } + + // runnerContactedAtLabels defines labels for the runner last contact timestamp metric. + runnerContactedAtLabels = []string{ + "runner_id", } ) @@ -558,20 +590,70 @@ func NewCollectorRunCount() prometheus.Collector { } // NewCollectorRunners returns a new Prometheus gauge collector for the -// metric "gitlab_ci_runners". This metric reports information about your runners. -// -// The labels include the default set of labels plus an additional "runnerLabels" label -// and "runnnerInformationLabels" that describes the runners state. +// metric "gitlab_ci_runners_info". This metric exposes global information +// about a GitLab runner, with one logical series per runner ID. func NewCollectorRunners() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "gitlab_ci_runners_info", - Help: "Status of your runners", + Help: "Information about GitLab runners", }, runnerLabels, ) } +// NewCollectorRunnerContactedAtSeconds returns a new Prometheus gauge collector for the +// metric "gitlab_ci_runner_contacted_at_seconds". This metric stores the last contact +// timestamp of a runner as a Unix timestamp in seconds. +func NewCollectorRunnerContactedAtSeconds() prometheus.Collector { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "gitlab_ci_runner_contacted_at_seconds", + Help: "Unix timestamp in seconds of the last runner contact", + }, + runnerContactedAtLabels, + ) +} + +// NewCollectorRunnerProjectInfo returns a new Prometheus gauge collector for the +// metric "gitlab_ci_runner_project_info". This metric represents the relationship +// between a GitLab runner and a project. +func NewCollectorRunnerProjectInfo() prometheus.Collector { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "gitlab_ci_runner_project_info", + Help: "Relationship between a GitLab runner and a project", + }, + runnerProjectLabels, + ) +} + +// NewCollectorRunnerTagInfo returns a new Prometheus gauge collector for the +// metric "gitlab_ci_runner_tag_info". This metric represents the relationship +// between a GitLab runner and one of its tags. +func NewCollectorRunnerTagInfo() prometheus.Collector { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "gitlab_ci_runner_tag_info", + Help: "Relationship between a GitLab runner and a tag", + }, + runnerTagLabels, + ) +} + +// NewCollectorRunnerGroupInfo returns a new Prometheus gauge collector for the +// metric "gitlab_ci_runner_group_info". This metric represents the relationship +// between a GitLab runner and a group. +func NewCollectorRunnerGroupInfo() prometheus.Collector { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "gitlab_ci_runner_group_info", + Help: "Relationship between a GitLab runner and a group", + }, + runnerGroupLabels, + ) +} + // NewCollectorTestReportTotalTime returns a new Prometheus gauge collector for the // metric "gitlab_ci_pipeline_test_report_total_time". This metric tracks the total // duration, in seconds, of all tests executed in the most recently finished pipeline. diff --git a/pkg/controller/garbage_collector.go b/pkg/controller/garbage_collector.go index 0c26a7c..af47c59 100644 --- a/pkg/controller/garbage_collector.go +++ b/pkg/controller/garbage_collector.go @@ -4,6 +4,7 @@ import ( "context" "reflect" "regexp" + "strconv" "dario.cat/mergo" log "github.com/sirupsen/logrus" @@ -548,23 +549,33 @@ func (c *Controller) GarbageCollectMetrics(ctx context.Context) error { } } - // TODO: => Improve this - Handle metrics related to a Runner. + // Handle metrics related to a runner. if metricLabelRunnerExists { + // runner_id is the canonical identifier for runner-related metrics. + runnerID, convErr := strconv.Atoi(metricLabelRunner) + if convErr != nil { + if err = deleteMetric(ctx, c.Store, m, "invalid-runner-id-label"); err != nil { + return err + } + + log.WithFields(log.Fields{ + "metric-kind": m.Kind, + "metric-labels": m.Labels, + "reason": "invalid-runner-id-label", + }).Info("deleted metric from the store") + + continue + } + runnerKey := schemas.Runner{ - ProjectName: metricLabelProject, - Name: metricLabelRunner, + ID: runnerID, }.Key() runner, runnerExists := storedRunners[runnerKey] - // fmt.Println("Stored Runners and runner exists: ", storedRunners[runnerKey], runnerExists) - // Delete the metric if the runner no longer exists + // Delete the metric if the runner no longer exists. if !runnerExists { - // TODO: => This must be donne directly on store controllers - Redis expiration trick - if err = deleteMetric(ctx, c.Store, m, "deleted metric from the store"); err != nil { - return err - } - if err = c.Store.DelMetric(ctx, k); err != nil { + if err = deleteMetric(ctx, c.Store, m, "non-existent-runner"); err != nil { return err } @@ -578,15 +589,17 @@ func (c *Controller) GarbageCollectMetrics(ctx context.Context) error { } switch m.Kind { - case schemas.MetricKindRunner: - if runner.OutputSparseStatusMetrics && m.Value != 1 { - // TODO: => This must be donne directly on store controllers - Redis expiration trick + case schemas.MetricKindRunner, + schemas.MetricKindRunnerContactedAtSeconds, + schemas.MetricKindRunnerProjectInfo, + schemas.MetricKindRunnerTagInfo, + schemas.MetricKindRunnerGroupInfo: + // Keep runner-related metrics as long as the runner still exists. + // For the global runner info metric, optionally apply sparse cleanup. + if m.Kind == schemas.MetricKindRunner && runner.OutputSparseStatusMetrics && m.Value != 1 { if err = deleteMetric(ctx, c.Store, m, "output-sparse-metrics-enabled-on-runner"); err != nil { return err } - if err = c.Store.DelMetric(ctx, k); err != nil { - return err - } log.WithFields(log.Fields{ "metric-kind": m.Kind, @@ -597,7 +610,7 @@ func (c *Controller) GarbageCollectMetrics(ctx context.Context) error { continue } default: - // Nothing to do + // Nothing to do for other metric kinds. } } diff --git a/pkg/controller/metrics.go b/pkg/controller/metrics.go index 9da6bc8..1343b5b 100644 --- a/pkg/controller/metrics.go +++ b/pkg/controller/metrics.go @@ -82,6 +82,10 @@ func NewRegistry(ctx context.Context) *Registry { schemas.MetricKindTestCaseExecutionTime: NewCollectorTestCaseExecutionTime(), schemas.MetricKindTestCaseStatus: NewCollectorTestCaseStatus(), schemas.MetricKindRunner: NewCollectorRunners(), + schemas.MetricKindRunnerContactedAtSeconds: NewCollectorRunnerContactedAtSeconds(), + schemas.MetricKindRunnerProjectInfo: NewCollectorRunnerProjectInfo(), + schemas.MetricKindRunnerGroupInfo: NewCollectorRunnerGroupInfo(), + schemas.MetricKindRunnerTagInfo: NewCollectorRunnerTagInfo(), }, } diff --git a/pkg/controller/runners.go b/pkg/controller/runners.go index f55128e..d1d7db4 100644 --- a/pkg/controller/runners.go +++ b/pkg/controller/runners.go @@ -2,13 +2,95 @@ package controller import ( "context" + "sort" "strconv" - "strings" "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" log "github.com/sirupsen/logrus" ) +// uniqueSortedNonEmpty removes empty values and duplicates, +// then sorts the remaining values for stable metric labels. +func uniqueSortedNonEmpty(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + + for _, v := range values { + if v == "" { + continue + } + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + + sort.Strings(out) + return out +} + +// runnerGroupNames returns a stable, deduplicated list of group names. +func runnerGroupNames(runner schemas.Runner) []string { + names := make([]string, 0, len(runner.Groups)) + for _, g := range runner.Groups { + names = append(names, g.Name) + } + return uniqueSortedNonEmpty(names) +} + +// runnerProjectNames returns a stable, deduplicated list of project names. +// It prefers PathWithNamespace, then NameWithNamespace, then Name. +func runnerProjectNames(runner schemas.Runner) []string { + names := make([]string, 0, len(runner.Projects)) + for _, p := range runner.Projects { + switch { + case p.PathWithNamespace != "": + names = append(names, p.PathWithNamespace) + case p.NameWithNamespace != "": + names = append(names, p.NameWithNamespace) + case p.Name != "": + names = append(names, p.Name) + } + } + return uniqueSortedNonEmpty(names) +} + +// runnerTagNames returns a stable, deduplicated list of runner tags. +func runnerTagNames(runner schemas.Runner) []string { + return uniqueSortedNonEmpty(runner.TagList) +} + +// deleteRunnerMetrics removes all metrics currently stored for a given runner ID. +// This ensures the next export represents the latest snapshot only. +func (c *Controller) deleteRunnerMetrics(ctx context.Context, runnerID int) error { + metrics, err := c.Store.Metrics(ctx) + if err != nil { + return err + } + + runnerIDStr := strconv.Itoa(runnerID) + + for key, metric := range metrics { + if metric.Labels["runner_id"] != runnerIDStr { + continue + } + + switch metric.Kind { + case schemas.MetricKindRunner, + schemas.MetricKindRunnerContactedAtSeconds, + schemas.MetricKindRunnerProjectInfo, + schemas.MetricKindRunnerTagInfo, + schemas.MetricKindRunnerGroupInfo: + if err := c.Store.DelMetric(ctx, key); err != nil { + return err + } + } + } + + return nil +} + // PullRunnersFromProject fetches the list of runners for a given project from the GitLab API, // then checks if each runner already exists in the local store. // For any runner not found in the store, it updates the store with the new runner, @@ -56,92 +138,129 @@ func (c *Controller) PullRunnersFromProject(ctx context.Context, p schemas.Proje return } -// UpdateRunner fetches the latest state of a given runner from the GitLab API, -// updates the local runner object with the latest details (Paused, Contacted At, and maintenance notes), -// and then saves the updated runner back to the local store. +// UpdateRunner fetches the latest runner details from GitLab and fully refreshes +// the local runner object before storing it. func (c *Controller) UpdateRunner(ctx context.Context, runner *schemas.Runner) error { - // Prepare logging fields with project, job name, and job ID for contextual logging projectRefLogFields := log.Fields{ "runner-id": runner.ID, } - // Retrieve the latest runner data from GitLab pulledRunner, err := c.Gitlab.GetRunner(ctx, runner.ProjectName, runner.ID) if err != nil { return err } + // Preserve local context fields that are not guaranteed to be returned + // by the GitLab API call. pulledRunner.ProjectName = runner.ProjectName pulledRunner.OutputSparseStatusMetrics = runner.OutputSparseStatusMetrics + + // Replace the local runner with the freshly pulled one. *runner = pulledRunner log.WithFields(projectRefLogFields).Info("update runner metrics") - - // Save the updated runner back to the store return c.Store.SetRunner(ctx, *runner) } -// ProcessRunnerMetrics processes metrics for a given runner and updates the store accordingly. +// ProcessRunnerMetrics refreshes a runner from GitLab and exports: +// +// 1. One global runner info metric per runner ID +// 2. One contacted_at metric per runner ID +// 3. One runner/project relation metric per project +// 4. One runner/tag relation metric per tag +// 5. One runner/group relation metric per group +// +// This avoids exporting one runner info series per project. func (c *Controller) ProcessRunnerMetrics(ctx context.Context, runner schemas.Runner) (err error) { - // Prepare logging fields with project, job name, and job ID for contextual logging projectRefLogFields := log.Fields{ "project-name-or-id": runner.ProjectName, "runner-desc": runner.Description, } + + // Refresh runner details before exporting metrics. if err = c.UpdateRunner(ctx, &runner); err != nil { return } - // Initialize labels from the reference default labels and add job-specific labels - groupNames := make([]string, 0, len(runner.Groups)) - for _, g := range runner.Groups { - if g.Name != "" { - groupNames = append(groupNames, g.Name) - } + // Remove previous runner metrics so the store only contains the latest snapshot. + if err = c.deleteRunnerMetrics(ctx, runner.ID); err != nil { + return } - projectNames := make([]string, 0, len(runner.Projects)) - for _, p := range runner.Projects { - switch { - case p.PathWithNamespace != "": - projectNames = append(projectNames, p.PathWithNamespace) - case p.NameWithNamespace != "": - projectNames = append(projectNames, p.NameWithNamespace) - case p.Name != "": - projectNames = append(projectNames, p.Name) - } - } + groupNames := runnerGroupNames(runner) + projectNames := runnerProjectNames(runner) + tagNames := runnerTagNames(runner) - tags := strings.Join(runner.TagList, ",") - - labels := runner.DefaultLabelsValues() - labels["runner_name"] = runner.Name - labels["runner_id"] = strconv.Itoa(runner.ID) // The unique identifier for the environment - labels["is_shared"] = strconv.FormatBool(runner.IsShared) // The kind of the latest deployment's reference - labels["runner_type"] = runner.RunnerType // The name of the latest deployment's reference - labels["online"] = strconv.FormatBool(runner.Online) // The short ID of the current commit - labels["tag_list"] = tags // Placeholder for the latest commit short ID (empty in this context) - labels["active"] = strconv.FormatBool(runner.Paused) // The availability status of the environment - labels["runner_maintenance_note"] = runner.MaintenanceNote // Maintenance note label - labels["status"] = runner.Status // The status of the runner - labels["paused"] = strconv.FormatBool(runner.Paused) // Define if the runner is paused - labels["runner_groups"] = strings.Join(groupNames, ",") // The groups assigned to this runner - labels["runner_projects"] = strings.Join(projectNames, ",") // The projects assigned to this runner - if runner.ContactedAt != nil { - labels["contacted_at"] = strconv.FormatInt(runner.ContactedAt.UTC().UnixNano(), 10) // Last contact with gitlab server - } else { - labels["contacted_at"] = "" + runnerID := strconv.Itoa(runner.ID) + + // Export one global runner info metric per runner. + infoLabels := map[string]string{ + "runner_id": runnerID, + "runner_name": runner.Name, + "runner_description": runner.Description, + "is_shared": strconv.FormatBool(runner.IsShared), + "runner_type": runner.RunnerType, + "online": strconv.FormatBool(runner.Online), + "active": strconv.FormatBool(!runner.Paused), + "paused": strconv.FormatBool(runner.Paused), + "status": runner.Status, + "runner_maintenance_note": runner.MaintenanceNote, } - // Log trace info indicating that job metrics are being processed log.WithFields(projectRefLogFields).Info("processing runner metrics") - // Store the size of job artifacts in bytes storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindRunner, - Labels: labels, + Labels: infoLabels, Value: 1, }) + // Export the last contact timestamp as a numeric metric value, not as a label. + if runner.ContactedAt != nil { + storeSetMetric(ctx, c.Store, schemas.Metric{ + Kind: schemas.MetricKindRunnerContactedAtSeconds, + Labels: map[string]string{ + "runner_id": runnerID, + }, + Value: float64(runner.ContactedAt.UTC().Unix()), + }) + } + + // Export one metric per related project. + for _, projectName := range projectNames { + storeSetMetric(ctx, c.Store, schemas.Metric{ + Kind: schemas.MetricKindRunnerProjectInfo, + Labels: map[string]string{ + "runner_id": runnerID, + "project": projectName, + }, + Value: 1, + }) + } + + // Export one metric per runner tag. + for _, tag := range tagNames { + storeSetMetric(ctx, c.Store, schemas.Metric{ + Kind: schemas.MetricKindRunnerTagInfo, + Labels: map[string]string{ + "runner_id": runnerID, + "tag": tag, + }, + Value: 1, + }) + } + + // Export one metric per runner group. + for _, groupName := range groupNames { + storeSetMetric(ctx, c.Store, schemas.Metric{ + Kind: schemas.MetricKindRunnerGroupInfo, + Labels: map[string]string{ + "runner_id": runnerID, + "group": groupName, + }, + Value: 1, + }) + } + return nil } diff --git a/pkg/schemas/metric.go b/pkg/schemas/metric.go index 767f4b3..129dbda 100644 --- a/pkg/schemas/metric.go +++ b/pkg/schemas/metric.go @@ -122,6 +122,18 @@ const ( // MetricKindRunner refers to the runner information. MetricKindRunner + + // MetricKindRunnerContactedAtSeconds refers to the last contact timestamp of a runner. + MetricKindRunnerContactedAtSeconds + + // MetricKindRunnerProjectInfo refers to the relation between a runner and a project. + MetricKindRunnerProjectInfo + + // MetricKindRunnerTagInfo refers to the relation between a runner and a tag. + MetricKindRunnerTagInfo + + // MetricKindRunnerGroupInfo refers to the relation between a runner and a group. + MetricKindRunnerGroupInfo ) // Metric represents a metric with a kind, labels, and a value. @@ -199,20 +211,36 @@ func (m Metric) Key() MetricKey { }) case MetricKindRunner: + // One logical metric series per runner ID. + key += fmt.Sprintf("%v", []string{ + m.Labels["runner_id"], + }) + + case MetricKindRunnerContactedAtSeconds: + // One logical metric series per runner ID. + key += fmt.Sprintf("%v", []string{ + m.Labels["runner_id"], + }) + + case MetricKindRunnerProjectInfo: + // One logical metric series per runner/project pair. key += fmt.Sprintf("%v", []string{ + m.Labels["runner_id"], m.Labels["project"], - m.Labels["kind"], + }) + + case MetricKindRunnerTagInfo: + // One logical metric series per runner/tag pair. + key += fmt.Sprintf("%v", []string{ m.Labels["runner_id"], - m.Labels["runner_description"], - m.Labels["runner_groups"], - m.Labels["runner_projects"], - m.Labels["runner_maintenance_note"], - m.Labels["contacted_at"], - m.Labels["paused"], - m.Labels["runner_type"], - m.Labels["tag_list"], - m.Labels["is_shared"], - m.Labels["active"], + m.Labels["tag"], + }) + + case MetricKindRunnerGroupInfo: + // One logical metric series per runner/group pair. + key += fmt.Sprintf("%v", []string{ + m.Labels["runner_id"], + m.Labels["group"], }) } From 6194d95254f3950edcce2200217e118ca273df92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Thu, 26 Mar 2026 09:54:35 +0100 Subject: [PATCH 08/11] Improve redis connection in schedulers --- pkg/controller/scheduler.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 35b7a99..25991ed 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -504,10 +504,6 @@ func (c *Controller) TaskHandlerGarbageCollectRunners(ctx context.Context) error // - If Scheduled is true, the task is scheduled repeatedly at the configured interval. // // If a Redis client is configured, it also schedules a keepalive task for Redis. -// -// Note: The Redis keepalive scheduling currently happens inside the loop for each task, -// -// which might be more efficient to call just once outside the loop. func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.GarbageCollect) { ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:Schedule") defer span.End() @@ -540,10 +536,11 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G if cfg.Scheduled { c.ScheduleTaskWithTicker(ctx, tt, cfg.IntervalSeconds) } + } - if c.Redis != nil { - c.ScheduleRedisSetKeepalive(ctx) - } + // Start the Redis keepalive loop only once. + if c.Redis != nil { + c.ScheduleRedisSetKeepalive(ctx) } } From b3e9b4ac169938e8ec37fcb4cd49500428fa4fe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Thu, 26 Mar 2026 13:07:51 +0100 Subject: [PATCH 09/11] Improve runners and monitor --- internal/cmd/monitor.go | 4 +++- pkg/controller/runners.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/cmd/monitor.go b/internal/cmd/monitor.go index dadbacf..c5d987e 100644 --- a/internal/cmd/monitor.go +++ b/internal/cmd/monitor.go @@ -6,6 +6,8 @@ import ( monitorUI "github.com/helvethink/gitlab-ci-exporter/pkg/monitor/ui" ) +var startMonitorUI = monitorUI.Start + // Monitor starts the internal monitoring UI. func Monitor(ctx *cli.Context) (int, error) { // Parse global flags from CLI context (e.g., internal monitoring address) @@ -15,7 +17,7 @@ func Monitor(ctx *cli.Context) (int, error) { } // Start the monitoring UI with app version and configured listener address - monitorUI.Start( + startMonitorUI( ctx.App.Version, cfg.InternalMonitoringListenerAddress, ) diff --git a/pkg/controller/runners.go b/pkg/controller/runners.go index d1d7db4..8daacd4 100644 --- a/pkg/controller/runners.go +++ b/pkg/controller/runners.go @@ -85,6 +85,8 @@ func (c *Controller) deleteRunnerMetrics(ctx context.Context, runnerID int) erro if err := c.Store.DelMetric(ctx, key); err != nil { return err } + default: + // nothing happens } } From af07b32a22a4e90282616e9650bccc392ff3d2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Thu, 26 Mar 2026 13:09:35 +0100 Subject: [PATCH 10/11] Regenerate protobuf files --- pkg/monitor/protobuf/monitor.pb.go | 301 ++++++++---------------- pkg/monitor/protobuf/monitor_grpc.pb.go | 86 ++++--- 2 files changed, 142 insertions(+), 245 deletions(-) diff --git a/pkg/monitor/protobuf/monitor.pb.go b/pkg/monitor/protobuf/monitor.pb.go index 54a4a09..098813e 100644 --- a/pkg/monitor/protobuf/monitor.pb.go +++ b/pkg/monitor/protobuf/monitor.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 -// protoc v3.21.0 +// protoc-gen-go v1.36.11 +// protoc v7.34.1 // source: pkg/monitor/protobuf/monitor.proto package protobuf @@ -9,6 +9,7 @@ package protobuf import ( reflect "reflect" sync "sync" + unsafe "unsafe" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -23,18 +24,16 @@ const ( ) type Empty struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Empty) Reset() { *x = Empty{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Empty) String() string { @@ -45,7 +44,7 @@ func (*Empty) ProtoMessage() {} func (x *Empty) ProtoReflect() protoreflect.Message { mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -61,20 +60,17 @@ func (*Empty) Descriptor() ([]byte, []int) { } type Config struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Content string `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` unknownFields protoimpl.UnknownFields - - Content string `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + sizeCache protoimpl.SizeCache } func (x *Config) Reset() { *x = Config{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Config) String() string { @@ -85,7 +81,7 @@ func (*Config) ProtoMessage() {} func (x *Config) ProtoReflect() protoreflect.Message { mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -108,30 +104,27 @@ func (x *Config) GetContent() string { } type Telemetry struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - GitlabApiUsage float64 `protobuf:"fixed64,1,opt,name=gitlab_api_usage,json=gitlabApiUsage,proto3" json:"gitlab_api_usage,omitempty"` - GitlabApiRequestsCount uint64 `protobuf:"varint,2,opt,name=gitlab_api_requests_count,json=gitlabApiRequestsCount,proto3" json:"gitlab_api_requests_count,omitempty"` - GitlabApiRateLimit float64 `protobuf:"fixed64,3,opt,name=gitlab_api_rate_limit,json=gitlabApiRateLimit,proto3" json:"gitlab_api_rate_limit,omitempty"` - GitlabApiLimitRemaining uint64 `protobuf:"varint,4,opt,name=gitlab_api_limit_remaining,json=gitlabApiLimitRemaining,proto3" json:"gitlab_api_limit_remaining,omitempty"` - TasksBufferUsage float64 `protobuf:"fixed64,5,opt,name=tasks_buffer_usage,json=tasksBufferUsage,proto3" json:"tasks_buffer_usage,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + GitlabApiUsage float64 `protobuf:"fixed64,1,opt,name=gitlab_api_usage,json=gitlabApiUsage,proto3" json:"gitlab_api_usage,omitempty"` + GitlabApiRequestsCount uint64 `protobuf:"varint,2,opt,name=gitlab_api_requests_count,json=gitlabApiRequestsCount,proto3" json:"gitlab_api_requests_count,omitempty"` + GitlabApiRateLimit float64 `protobuf:"fixed64,3,opt,name=gitlab_api_rate_limit,json=gitlabApiRateLimit,proto3" json:"gitlab_api_rate_limit,omitempty"` + GitlabApiLimitRemaining uint64 `protobuf:"varint,4,opt,name=gitlab_api_limit_remaining,json=gitlabApiLimitRemaining,proto3" json:"gitlab_api_limit_remaining,omitempty"` + TasksBufferUsage float64 `protobuf:"fixed64,5,opt,name=tasks_buffer_usage,json=tasksBufferUsage,proto3" json:"tasks_buffer_usage,omitempty"` TasksExecutedCount uint64 `protobuf:"varint,6,opt,name=tasks_executed_count,json=tasksExecutedCount,proto3" json:"tasks_executed_count,omitempty"` Projects *Entity `protobuf:"bytes,7,opt,name=projects,proto3" json:"projects,omitempty"` Refs *Entity `protobuf:"bytes,8,opt,name=refs,proto3" json:"refs,omitempty"` Envs *Entity `protobuf:"bytes,9,opt,name=envs,proto3" json:"envs,omitempty"` - Metrics *Entity `protobuf:"bytes,10,opt,name=metrics,proto3" json:"metrics,omitempty"` - Runners *Entity `protobuf:"bytes,11,opt,name=runners,proto3" json:"runners,omitempty"` + Runners *Entity `protobuf:"bytes,10,opt,name=runners,proto3" json:"runners,omitempty"` + Metrics *Entity `protobuf:"bytes,11,opt,name=metrics,proto3" json:"metrics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Telemetry) Reset() { *x = Telemetry{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Telemetry) String() string { @@ -142,7 +135,7 @@ func (*Telemetry) ProtoMessage() {} func (x *Telemetry) ProtoReflect() protoreflect.Message { mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -220,6 +213,13 @@ func (x *Telemetry) GetEnvs() *Entity { return nil } +func (x *Telemetry) GetRunners() *Entity { + if x != nil { + return x.Runners + } + return nil +} + func (x *Telemetry) GetMetrics() *Entity { if x != nil { return x.Metrics @@ -228,24 +228,21 @@ func (x *Telemetry) GetMetrics() *Entity { } type Entity struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Count int64 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` + LastGc *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_gc,json=lastGc,proto3" json:"last_gc,omitempty"` + LastPull *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=last_pull,json=lastPull,proto3" json:"last_pull,omitempty"` + NextGc *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=next_gc,json=nextGc,proto3" json:"next_gc,omitempty"` + NextPull *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=next_pull,json=nextPull,proto3" json:"next_pull,omitempty"` unknownFields protoimpl.UnknownFields - - Count int64 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` - LastGc *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_gc,json=lastGc,proto3" json:"last_gc,omitempty"` - LastPull *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=last_pull,json=lastPull,proto3" json:"last_pull,omitempty"` - NextGc *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=next_gc,json=nextGc,proto3" json:"next_gc,omitempty"` - NextPull *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=next_pull,json=nextPull,proto3" json:"next_pull,omitempty"` + sizeCache protoimpl.SizeCache } func (x *Entity) Reset() { *x = Entity{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Entity) String() string { @@ -256,7 +253,7 @@ func (*Entity) ProtoMessage() {} func (x *Entity) ProtoReflect() protoreflect.Message { mi := &file_pkg_monitor_protobuf_monitor_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -308,117 +305,74 @@ func (x *Entity) GetNextPull() *timestamppb.Timestamp { var File_pkg_monitor_protobuf_monitor_proto protoreflect.FileDescriptor -var file_pkg_monitor_protobuf_monitor_proto_rawDesc = []byte{ - 0x0a, 0x22, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x1a, 0x1f, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x07, - 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x22, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0xe2, 0x03, 0x0a, 0x09, - 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x67, 0x69, 0x74, - 0x6c, 0x61, 0x62, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x0e, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x41, 0x70, 0x69, 0x55, 0x73, - 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x19, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x5f, 0x61, 0x70, - 0x69, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x41, 0x70, - 0x69, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x31, - 0x0a, 0x15, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x72, 0x61, 0x74, - 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x12, 0x67, - 0x69, 0x74, 0x6c, 0x61, 0x62, 0x41, 0x70, 0x69, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, - 0x74, 0x12, 0x3b, 0x0a, 0x1a, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x5f, 0x61, 0x70, 0x69, 0x5f, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x72, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x41, 0x70, 0x69, - 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x2c, - 0x0a, 0x12, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x5f, 0x62, 0x75, 0x66, 0x66, 0x65, 0x72, 0x5f, 0x75, - 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x74, 0x61, 0x73, 0x6b, - 0x73, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x14, - 0x74, 0x61, 0x73, 0x6b, 0x73, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x64, 0x5f, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x12, 0x74, 0x61, 0x73, 0x6b, - 0x73, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2b, - 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0f, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x04, 0x72, - 0x65, 0x66, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, - 0x74, 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x04, 0x72, 0x65, 0x66, 0x73, - 0x12, 0x23, 0x0a, 0x04, 0x65, 0x6e, 0x76, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, - 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, - 0x04, 0x65, 0x6e, 0x76, 0x73, 0x12, 0x29, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, - 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x22, 0xfa, 0x01, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x67, 0x63, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x06, - 0x6c, 0x61, 0x73, 0x74, 0x47, 0x63, 0x12, 0x37, 0x0a, 0x09, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, - 0x75, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x6c, 0x61, 0x73, 0x74, 0x50, 0x75, 0x6c, 0x6c, 0x12, - 0x33, 0x0a, 0x07, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x67, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x06, 0x6e, 0x65, - 0x78, 0x74, 0x47, 0x63, 0x12, 0x37, 0x0a, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x75, 0x6c, - 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x08, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x75, 0x6c, 0x6c, 0x32, 0x71, 0x0a, - 0x07, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x0e, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0f, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x2e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x54, - 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x12, 0x0e, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x6d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x2e, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x22, 0x00, 0x30, 0x01, - 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, - 0x76, 0x69, 0x73, 0x6f, 0x6e, 0x6e, 0x65, 0x61, 0x75, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, - 0x2d, 0x63, 0x69, 0x2d, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2d, 0x65, 0x78, - 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x6f, 0x6e, 0x69, 0x74, - 0x6f, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, -} +const file_pkg_monitor_protobuf_monitor_proto_rawDesc = "" + + "\n" + + "\"pkg/monitor/protobuf/monitor.proto\x12\amonitor\x1a\x1fgoogle/protobuf/timestamp.proto\"\a\n" + + "\x05Empty\"\"\n" + + "\x06Config\x12\x18\n" + + "\acontent\x18\x01 \x01(\tR\acontent\"\x8d\x04\n" + + "\tTelemetry\x12(\n" + + "\x10gitlab_api_usage\x18\x01 \x01(\x01R\x0egitlabApiUsage\x129\n" + + "\x19gitlab_api_requests_count\x18\x02 \x01(\x04R\x16gitlabApiRequestsCount\x121\n" + + "\x15gitlab_api_rate_limit\x18\x03 \x01(\x01R\x12gitlabApiRateLimit\x12;\n" + + "\x1agitlab_api_limit_remaining\x18\x04 \x01(\x04R\x17gitlabApiLimitRemaining\x12,\n" + + "\x12tasks_buffer_usage\x18\x05 \x01(\x01R\x10tasksBufferUsage\x120\n" + + "\x14tasks_executed_count\x18\x06 \x01(\x04R\x12tasksExecutedCount\x12+\n" + + "\bprojects\x18\a \x01(\v2\x0f.monitor.EntityR\bprojects\x12#\n" + + "\x04refs\x18\b \x01(\v2\x0f.monitor.EntityR\x04refs\x12#\n" + + "\x04envs\x18\t \x01(\v2\x0f.monitor.EntityR\x04envs\x12)\n" + + "\arunners\x18\n" + + " \x01(\v2\x0f.monitor.EntityR\arunners\x12)\n" + + "\ametrics\x18\v \x01(\v2\x0f.monitor.EntityR\ametrics\"\xfa\x01\n" + + "\x06Entity\x12\x14\n" + + "\x05count\x18\x01 \x01(\x03R\x05count\x123\n" + + "\alast_gc\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x06lastGc\x127\n" + + "\tlast_pull\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\blastPull\x123\n" + + "\anext_gc\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x06nextGc\x127\n" + + "\tnext_pull\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\bnextPull2q\n" + + "\aMonitor\x12.\n" + + "\tGetConfig\x12\x0e.monitor.Empty\x1a\x0f.monitor.Config\"\x00\x126\n" + + "\fGetTelemetry\x12\x0e.monitor.Empty\x1a\x12.monitor.Telemetry\"\x000\x01B?Z=github.com/helvethink/gitlab-ci-exporter/pkg/monitor/protobufb\x06proto3" var ( file_pkg_monitor_protobuf_monitor_proto_rawDescOnce sync.Once - file_pkg_monitor_protobuf_monitor_proto_rawDescData = file_pkg_monitor_protobuf_monitor_proto_rawDesc + file_pkg_monitor_protobuf_monitor_proto_rawDescData []byte ) func file_pkg_monitor_protobuf_monitor_proto_rawDescGZIP() []byte { file_pkg_monitor_protobuf_monitor_proto_rawDescOnce.Do(func() { - file_pkg_monitor_protobuf_monitor_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_monitor_protobuf_monitor_proto_rawDescData) + file_pkg_monitor_protobuf_monitor_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_pkg_monitor_protobuf_monitor_proto_rawDesc), len(file_pkg_monitor_protobuf_monitor_proto_rawDesc))) }) return file_pkg_monitor_protobuf_monitor_proto_rawDescData } -var ( - file_pkg_monitor_protobuf_monitor_proto_msgTypes = make([]protoimpl.MessageInfo, 4) - file_pkg_monitor_protobuf_monitor_proto_goTypes = []interface{}{ - (*Empty)(nil), // 0: monitor.Empty - (*Config)(nil), // 1: monitor.Config - (*Telemetry)(nil), // 2: monitor.Telemetry - (*Entity)(nil), // 3: monitor.Entity - (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp - } -) - +var file_pkg_monitor_protobuf_monitor_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_pkg_monitor_protobuf_monitor_proto_goTypes = []any{ + (*Empty)(nil), // 0: monitor.Empty + (*Config)(nil), // 1: monitor.Config + (*Telemetry)(nil), // 2: monitor.Telemetry + (*Entity)(nil), // 3: monitor.Entity + (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp +} var file_pkg_monitor_protobuf_monitor_proto_depIdxs = []int32{ 3, // 0: monitor.Telemetry.projects:type_name -> monitor.Entity 3, // 1: monitor.Telemetry.refs:type_name -> monitor.Entity 3, // 2: monitor.Telemetry.envs:type_name -> monitor.Entity - 3, // 3: monitor.Telemetry.metrics:type_name -> monitor.Entity - 4, // 4: monitor.Entity.last_gc:type_name -> google.protobuf.Timestamp - 4, // 5: monitor.Entity.last_pull:type_name -> google.protobuf.Timestamp - 4, // 6: monitor.Entity.next_gc:type_name -> google.protobuf.Timestamp - 4, // 7: monitor.Entity.next_pull:type_name -> google.protobuf.Timestamp - 0, // 8: monitor.Monitor.GetConfig:input_type -> monitor.Empty - 0, // 9: monitor.Monitor.GetTelemetry:input_type -> monitor.Empty - 1, // 10: monitor.Monitor.GetConfig:output_type -> monitor.Config - 2, // 11: monitor.Monitor.GetTelemetry:output_type -> monitor.Telemetry - 10, // [10:12] is the sub-list for method output_type - 8, // [8:10] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 3, // 3: monitor.Telemetry.runners:type_name -> monitor.Entity + 3, // 4: monitor.Telemetry.metrics:type_name -> monitor.Entity + 4, // 5: monitor.Entity.last_gc:type_name -> google.protobuf.Timestamp + 4, // 6: monitor.Entity.last_pull:type_name -> google.protobuf.Timestamp + 4, // 7: monitor.Entity.next_gc:type_name -> google.protobuf.Timestamp + 4, // 8: monitor.Entity.next_pull:type_name -> google.protobuf.Timestamp + 0, // 9: monitor.Monitor.GetConfig:input_type -> monitor.Empty + 0, // 10: monitor.Monitor.GetTelemetry:input_type -> monitor.Empty + 1, // 11: monitor.Monitor.GetConfig:output_type -> monitor.Config + 2, // 12: monitor.Monitor.GetTelemetry:output_type -> monitor.Telemetry + 11, // [11:13] is the sub-list for method output_type + 9, // [9:11] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_pkg_monitor_protobuf_monitor_proto_init() } @@ -426,61 +380,11 @@ func file_pkg_monitor_protobuf_monitor_proto_init() { if File_pkg_monitor_protobuf_monitor_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_pkg_monitor_protobuf_monitor_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Empty); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_monitor_protobuf_monitor_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Config); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_monitor_protobuf_monitor_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Telemetry); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_monitor_protobuf_monitor_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Entity); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_monitor_protobuf_monitor_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_pkg_monitor_protobuf_monitor_proto_rawDesc), len(file_pkg_monitor_protobuf_monitor_proto_rawDesc)), NumEnums: 0, NumMessages: 4, NumExtensions: 0, @@ -491,7 +395,6 @@ func file_pkg_monitor_protobuf_monitor_proto_init() { MessageInfos: file_pkg_monitor_protobuf_monitor_proto_msgTypes, }.Build() File_pkg_monitor_protobuf_monitor_proto = out.File - file_pkg_monitor_protobuf_monitor_proto_rawDesc = nil file_pkg_monitor_protobuf_monitor_proto_goTypes = nil file_pkg_monitor_protobuf_monitor_proto_depIdxs = nil } diff --git a/pkg/monitor/protobuf/monitor_grpc.pb.go b/pkg/monitor/protobuf/monitor_grpc.pb.go index a7e396b..683ce80 100644 --- a/pkg/monitor/protobuf/monitor_grpc.pb.go +++ b/pkg/monitor/protobuf/monitor_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.0 +// - protoc-gen-go-grpc v1.6.1 +// - protoc v7.34.1 // source: pkg/monitor/protobuf/monitor.proto package protobuf @@ -16,15 +16,20 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Monitor_GetConfig_FullMethodName = "/monitor.Monitor/GetConfig" + Monitor_GetTelemetry_FullMethodName = "/monitor.Monitor/GetTelemetry" +) // MonitorClient is the client API for Monitor service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type MonitorClient interface { GetConfig(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Config, error) - GetTelemetry(ctx context.Context, in *Empty, opts ...grpc.CallOption) (Monitor_GetTelemetryClient, error) + GetTelemetry(ctx context.Context, in *Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Telemetry], error) } type monitorClient struct { @@ -36,20 +41,22 @@ func NewMonitorClient(cc grpc.ClientConnInterface) MonitorClient { } func (c *monitorClient) GetConfig(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Config, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Config) - err := c.cc.Invoke(ctx, "/monitor.Monitor/GetConfig", in, out, opts...) + err := c.cc.Invoke(ctx, Monitor_GetConfig_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } -func (c *monitorClient) GetTelemetry(ctx context.Context, in *Empty, opts ...grpc.CallOption) (Monitor_GetTelemetryClient, error) { - stream, err := c.cc.NewStream(ctx, &Monitor_ServiceDesc.Streams[0], "/monitor.Monitor/GetTelemetry", opts...) +func (c *monitorClient) GetTelemetry(ctx context.Context, in *Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Telemetry], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Monitor_ServiceDesc.Streams[0], Monitor_GetTelemetry_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &monitorGetTelemetryClient{stream} + x := &grpc.GenericClientStream[Empty, Telemetry]{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -59,43 +66,33 @@ func (c *monitorClient) GetTelemetry(ctx context.Context, in *Empty, opts ...grp return x, nil } -type Monitor_GetTelemetryClient interface { - Recv() (*Telemetry, error) - grpc.ClientStream -} - -type monitorGetTelemetryClient struct { - grpc.ClientStream -} - -func (x *monitorGetTelemetryClient) Recv() (*Telemetry, error) { - m := new(Telemetry) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Monitor_GetTelemetryClient = grpc.ServerStreamingClient[Telemetry] // MonitorServer is the server API for Monitor service. // All implementations must embed UnimplementedMonitorServer -// for forward compatibility +// for forward compatibility. type MonitorServer interface { GetConfig(context.Context, *Empty) (*Config, error) - GetTelemetry(*Empty, Monitor_GetTelemetryServer) error + GetTelemetry(*Empty, grpc.ServerStreamingServer[Telemetry]) error mustEmbedUnimplementedMonitorServer() } -// UnimplementedMonitorServer must be embedded to have forward compatible implementations. +// UnimplementedMonitorServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. type UnimplementedMonitorServer struct{} func (UnimplementedMonitorServer) GetConfig(context.Context, *Empty) (*Config, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetConfig not implemented") + return nil, status.Error(codes.Unimplemented, "method GetConfig not implemented") } - -func (UnimplementedMonitorServer) GetTelemetry(*Empty, Monitor_GetTelemetryServer) error { - return status.Errorf(codes.Unimplemented, "method GetTelemetry not implemented") +func (UnimplementedMonitorServer) GetTelemetry(*Empty, grpc.ServerStreamingServer[Telemetry]) error { + return status.Error(codes.Unimplemented, "method GetTelemetry not implemented") } func (UnimplementedMonitorServer) mustEmbedUnimplementedMonitorServer() {} +func (UnimplementedMonitorServer) testEmbeddedByValue() {} // UnsafeMonitorServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to MonitorServer will @@ -105,6 +102,13 @@ type UnsafeMonitorServer interface { } func RegisterMonitorServer(s grpc.ServiceRegistrar, srv MonitorServer) { + // If the following call panics, it indicates UnimplementedMonitorServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } s.RegisterService(&Monitor_ServiceDesc, srv) } @@ -118,7 +122,7 @@ func _Monitor_GetConfig_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/monitor.Monitor/GetConfig", + FullMethod: Monitor_GetConfig_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MonitorServer).GetConfig(ctx, req.(*Empty)) @@ -131,21 +135,11 @@ func _Monitor_GetTelemetry_Handler(srv interface{}, stream grpc.ServerStream) er if err := stream.RecvMsg(m); err != nil { return err } - return srv.(MonitorServer).GetTelemetry(m, &monitorGetTelemetryServer{stream}) -} - -type Monitor_GetTelemetryServer interface { - Send(*Telemetry) error - grpc.ServerStream -} - -type monitorGetTelemetryServer struct { - grpc.ServerStream + return srv.(MonitorServer).GetTelemetry(m, &grpc.GenericServerStream[Empty, Telemetry]{ServerStream: stream}) } -func (x *monitorGetTelemetryServer) Send(m *Telemetry) error { - return x.ServerStream.SendMsg(m) -} +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Monitor_GetTelemetryServer = grpc.ServerStreamingServer[Telemetry] // Monitor_ServiceDesc is the grpc.ServiceDesc for Monitor service. // It's only intended for direct use with grpc.RegisterService, From 60357bac855dd53c0ff9ad5840ccdfaef1c32f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Gon=C3=A7alves?= Date: Thu, 26 Mar 2026 13:11:09 +0100 Subject: [PATCH 11/11] Add unit tests --- internal/cmd/monitor_test.go | 83 +++++ internal/collectors/exporter_test.go | 120 +++++++ internal/httpServer/server_test.go | 77 +++++ internal/logging/logger_test.go | 81 +++++ pkg/controller/collectors_test.go | 206 ++++++++++++ pkg/controller/controller_test.go | 124 ++++++++ pkg/controller/environments_test.go | 274 ++++++++++++++++ pkg/controller/garbage_collector_test.go | 261 ++++++++++++++++ pkg/controller/handlers_test.go | 169 ++++++++++ pkg/controller/jobs_test.go | 212 +++++++++++++ pkg/controller/metadata_test.go | 50 +++ pkg/controller/metrics_test.go | 273 ++++++++++++++++ pkg/controller/pipelines_test.go | 241 ++++++++++++++ pkg/controller/projects_test.go | 117 +++++++ pkg/controller/refs_test.go | 115 +++++++ pkg/controller/runners_test.go | 366 ++++++++++++++++++++++ pkg/controller/scheduler_test.go | 46 +++ pkg/controller/store_test.go | 152 +++++++++ pkg/gitlab/branches_test.go | 131 ++++++++ pkg/gitlab/client_test.go | 195 ++++++++++++ pkg/gitlab/environments_test.go | 259 +++++++++++++++ pkg/gitlab/jobs_test.go | 381 +++++++++++++++++++++++ pkg/gitlab/pipelines_test.go | 337 ++++++++++++++++++++ pkg/gitlab/projects_test.go | 215 +++++++++++++ pkg/gitlab/repositories_test.go | 101 ++++++ pkg/gitlab/runners_test.go | 258 +++++++++++++++ pkg/gitlab/tags_test.go | 187 +++++++++++ pkg/gitlab/version_test.go | 102 ++++++ pkg/monitor/client/client_test.go | 49 +++ pkg/monitor/monitor_test.go | 21 ++ pkg/monitor/server/server_test.go | 154 +++++++++ pkg/monitor/ui/ui_test.go | 98 ++++++ pkg/ratelimit/local_test.go | 55 ++++ pkg/ratelimit/ratelimit_test.go | 33 ++ pkg/ratelimit/redis_test.go | 65 ++++ pkg/schemas/metric_test.go | 34 +- pkg/schemas/runners_test.go | 26 +- pkg/schemas/tasks_test.go | 71 +++++ pkg/store/redis_test.go | 3 + pkg/store/store_test.go | 190 ++++++++--- 40 files changed, 5849 insertions(+), 83 deletions(-) create mode 100644 internal/cmd/monitor_test.go create mode 100644 internal/collectors/exporter_test.go create mode 100644 internal/httpServer/server_test.go create mode 100644 internal/logging/logger_test.go create mode 100644 pkg/controller/collectors_test.go create mode 100644 pkg/controller/controller_test.go create mode 100644 pkg/controller/environments_test.go create mode 100644 pkg/controller/garbage_collector_test.go create mode 100644 pkg/controller/handlers_test.go create mode 100644 pkg/controller/jobs_test.go create mode 100644 pkg/controller/metadata_test.go create mode 100644 pkg/controller/metrics_test.go create mode 100644 pkg/controller/pipelines_test.go create mode 100644 pkg/controller/projects_test.go create mode 100644 pkg/controller/refs_test.go create mode 100644 pkg/controller/runners_test.go create mode 100644 pkg/controller/scheduler_test.go create mode 100644 pkg/controller/store_test.go create mode 100644 pkg/gitlab/branches_test.go create mode 100644 pkg/gitlab/client_test.go create mode 100644 pkg/gitlab/environments_test.go create mode 100644 pkg/gitlab/jobs_test.go create mode 100644 pkg/gitlab/pipelines_test.go create mode 100644 pkg/gitlab/projects_test.go create mode 100644 pkg/gitlab/repositories_test.go create mode 100644 pkg/gitlab/runners_test.go create mode 100644 pkg/gitlab/tags_test.go create mode 100644 pkg/gitlab/version_test.go create mode 100644 pkg/monitor/client/client_test.go create mode 100644 pkg/monitor/monitor_test.go create mode 100644 pkg/monitor/server/server_test.go create mode 100644 pkg/monitor/ui/ui_test.go create mode 100644 pkg/ratelimit/local_test.go create mode 100644 pkg/ratelimit/ratelimit_test.go create mode 100644 pkg/ratelimit/redis_test.go create mode 100644 pkg/schemas/tasks_test.go diff --git a/internal/cmd/monitor_test.go b/internal/cmd/monitor_test.go new file mode 100644 index 0000000..51b2e8a --- /dev/null +++ b/internal/cmd/monitor_test.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMonitor(t *testing.T) { + ctx, flags := NewTestContext() + ctx.App.Version = "1.2.3" + flags.String("internal-monitoring-listener-address", "", "") + require.NoError(t, flags.Set("internal-monitoring-listener-address", "http://127.0.0.1:8081")) + + var ( + called bool + gotVersion string + gotListenerURL *url.URL + ) + + previousStart := startMonitorUI + startMonitorUI = func(version string, listenerAddress *url.URL) { + called = true + gotVersion = version + gotListenerURL = listenerAddress + } + t.Cleanup(func() { + startMonitorUI = previousStart + }) + + exitCode, err := Monitor(ctx) + require.NoError(t, err) + assert.Equal(t, 0, exitCode) + assert.True(t, called) + assert.Equal(t, "1.2.3", gotVersion) + require.NotNil(t, gotListenerURL) + assert.Equal(t, "http://127.0.0.1:8081", gotListenerURL.String()) +} + +func TestMonitorReturnsErrorWhenInternalMonitoringAddressIsInvalid(t *testing.T) { + ctx, flags := NewTestContext() + flags.String("internal-monitoring-listener-address", "", "") + require.NoError(t, flags.Set("internal-monitoring-listener-address", "://bad-url")) + + called := false + previousStart := startMonitorUI + startMonitorUI = func(version string, listenerAddress *url.URL) { + called = true + } + t.Cleanup(func() { + startMonitorUI = previousStart + }) + + exitCode, err := Monitor(ctx) + require.Error(t, err) + assert.Equal(t, 1, exitCode) + assert.False(t, called) +} + +func TestMonitorWithoutInternalMonitoringAddress(t *testing.T) { + ctx, flags := NewTestContext() + ctx.App.Version = "dev" + flags.String("internal-monitoring-listener-address", "", "") + + called := false + var gotListenerURL *url.URL + previousStart := startMonitorUI + startMonitorUI = func(version string, listenerAddress *url.URL) { + called = true + gotListenerURL = listenerAddress + } + t.Cleanup(func() { + startMonitorUI = previousStart + }) + + exitCode, err := Monitor(ctx) + require.NoError(t, err) + assert.Equal(t, 0, exitCode) + assert.True(t, called) + assert.Nil(t, gotListenerURL) +} diff --git a/internal/collectors/exporter_test.go b/internal/collectors/exporter_test.go new file mode 100644 index 0000000..1978bc0 --- /dev/null +++ b/internal/collectors/exporter_test.go @@ -0,0 +1,120 @@ +package collectors + +import ( + "io" + "log/slog" + "strings" + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func descString(t *testing.T, desc *prometheus.Desc) string { + t.Helper() + require.NotNil(t, desc) + + return desc.String() +} + +func TestNewMetrics(t *testing.T) { + m := NewMetrics() + + require.NotNil(t, m) + assert.Contains(t, descString(t, m.sampleMetric1), `fqName: "gitlab-ci-exporter_sampleMetric1"`) + assert.Contains(t, descString(t, m.sampleMetric1), `variableLabels: {label1}`) + assert.Contains(t, descString(t, m.sampleMetric2), `fqName: "gitlab-ci-exporter_sampleMetric2"`) + assert.Contains(t, descString(t, m.sampleMetric2), `variableLabels: {label2}`) +} + +func TestNewExporter(t *testing.T) { + settings := &Settings{ + LogLevel: "info", + LogFormat: "text", + MetricsPath: "/metrics", + ListenPort: "8080", + Address: "0.0.0.0", + } + logger := newTestLogger() + + exporter, err := NewExporter(settings, logger) + require.NoError(t, err) + require.NotNil(t, exporter) + require.NotNil(t, exporter.metrics) + assert.Same(t, settings, exporter.Settings) + assert.Same(t, logger, exporter.Logger) +} + +func TestExporterDescribe(t *testing.T) { + exporter, err := NewExporter(&Settings{}, newTestLogger()) + require.NoError(t, err) + + ch := make(chan *prometheus.Desc, 2) + exporter.Describe(ch) + close(ch) + + var descs []string + for desc := range ch { + descs = append(descs, desc.String()) + } + + require.Len(t, descs, 2) + assert.True(t, strings.Contains(descs[0], "sampleMetric1") || strings.Contains(descs[1], "sampleMetric1")) + assert.True(t, strings.Contains(descs[0], "sampleMetric2") || strings.Contains(descs[1], "sampleMetric2")) +} + +func TestExporterCollect(t *testing.T) { + exporter, err := NewExporter(&Settings{}, newTestLogger()) + require.NoError(t, err) + + registry := prometheus.NewRegistry() + require.NoError(t, registry.Register(exporter)) + + families, err := registry.Gather() + require.NoError(t, err) + require.Len(t, families, 2) + + familyByName := make(map[string]*dto.MetricFamily, len(families)) + for _, family := range families { + familyByName[family.GetName()] = family + } + + sample1 := familyByName["gitlab-ci-exporter_sampleMetric1"] + require.NotNil(t, sample1) + require.Len(t, sample1.GetMetric(), 1) + assert.Equal(t, dto.MetricType_GAUGE, sample1.GetType()) + require.Len(t, sample1.GetMetric()[0].GetLabel(), 1) + assert.Equal(t, "label1", sample1.GetMetric()[0].GetLabel()[0].GetName()) + assert.Equal(t, "labelValue", sample1.GetMetric()[0].GetLabel()[0].GetValue()) + assert.GreaterOrEqual(t, sample1.GetMetric()[0].GetGauge().GetValue(), float64(0)) + assert.Less(t, sample1.GetMetric()[0].GetGauge().GetValue(), float64(1)) + + sample2 := familyByName["gitlab-ci-exporter_sampleMetric2"] + require.NotNil(t, sample2) + require.Len(t, sample2.GetMetric(), 1) + assert.Equal(t, dto.MetricType_GAUGE, sample2.GetType()) + require.Len(t, sample2.GetMetric()[0].GetLabel(), 1) + assert.Equal(t, "label2", sample2.GetMetric()[0].GetLabel()[0].GetName()) + assert.Equal(t, "labelValue", sample2.GetMetric()[0].GetLabel()[0].GetValue()) + assert.GreaterOrEqual(t, sample2.GetMetric()[0].GetGauge().GetValue(), float64(0)) + assert.Less(t, sample2.GetMetric()[0].GetGauge().GetValue(), float64(1)) +} + +func TestSampleMetricsReturnValueBetweenZeroAndOne(t *testing.T) { + exporter, err := NewExporter(&Settings{}, newTestLogger()) + require.NoError(t, err) + + sample1 := exporter.sampleMetric1() + sample2 := exporter.sampleMetric2() + + assert.GreaterOrEqual(t, sample1, float64(0)) + assert.Less(t, sample1, float64(1)) + assert.GreaterOrEqual(t, sample2, float64(0)) + assert.Less(t, sample2, float64(1)) +} diff --git a/internal/httpServer/server_test.go b/internal/httpServer/server_test.go new file mode 100644 index 0000000..bb91bd0 --- /dev/null +++ b/internal/httpServer/server_test.go @@ -0,0 +1,77 @@ +package httpServer + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/helvethink/gitlab-ci-exporter/internal/collectors" +) + +func newTestExporter(t *testing.T) *collectors.Exporter { + t.Helper() + + exporter, err := collectors.NewExporter(&collectors.Settings{ + MetricsPath: "metrics", + ListenPort: "9191", + Address: "0.0.0.0", + }, slog.New(slog.NewTextHandler(io.Discard, nil))) + require.NoError(t, err) + + return exporter +} + +func withFreshServeMux(t *testing.T) { + t.Helper() + + previousMux := http.DefaultServeMux + http.DefaultServeMux = http.NewServeMux() + t.Cleanup(func() { + http.DefaultServeMux = previousMux + }) +} + +func TestNewServerSetsExpectedAddr(t *testing.T) { + withFreshServeMux(t) + + server := NewServer(newTestExporter(t)) + + require.NotNil(t, server) + assert.Equal(t, ":9191", server.Addr) + assert.Nil(t, server.Handler) +} + +func TestNewServerServesRootPage(t *testing.T) { + withFreshServeMux(t) + + _ = NewServer(newTestExporter(t)) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + rr := httptest.NewRecorder() + http.DefaultServeMux.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + assert.Contains(t, rr.Body.String(), "Gitlab CI Exporter") + assert.Contains(t, rr.Body.String(), "Metrics at:") + assert.Contains(t, rr.Body.String(), "href='metrics'") + assert.Contains(t, rr.Body.String(), "github.com/Helvethink/gitlab-ci-exporter") +} + +func TestNewServerServesMetricsEndpoint(t *testing.T) { + withFreshServeMux(t) + + _ = NewServer(newTestExporter(t)) + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + rr := httptest.NewRecorder() + http.DefaultServeMux.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + assert.Contains(t, rr.Body.String(), "gitlab_ci_exporter_sampleMetric1") + assert.Contains(t, rr.Body.String(), "gitlab_ci_exporter_sampleMetric2") +} diff --git a/internal/logging/logger_test.go b/internal/logging/logger_test.go new file mode 100644 index 0000000..a94e320 --- /dev/null +++ b/internal/logging/logger_test.go @@ -0,0 +1,81 @@ +package logger + +import ( + "os" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func preserveLogrusState(t *testing.T) { + t.Helper() + + prevLevel := log.GetLevel() + prevFormatter := log.StandardLogger().Formatter + prevReportCaller := log.StandardLogger().ReportCaller + prevOutput := log.StandardLogger().Out + + t.Cleanup(func() { + log.SetLevel(prevLevel) + log.SetFormatter(prevFormatter) + log.SetReportCaller(prevReportCaller) + log.SetOutput(prevOutput) + }) +} + +func TestConfigureTextFormat(t *testing.T) { + preserveLogrusState(t) + + err := Configure(Config{ + Level: "debug", + Format: "text", + ReportCaller: true, + }) + require.NoError(t, err) + + assert.Equal(t, log.DebugLevel, log.GetLevel()) + _, ok := log.StandardLogger().Formatter.(*log.TextFormatter) + assert.True(t, ok) + assert.True(t, log.StandardLogger().ReportCaller) + assert.Same(t, os.Stdout, log.StandardLogger().Out) +} + +func TestConfigureJSONFormat(t *testing.T) { + preserveLogrusState(t) + + err := Configure(Config{ + Level: "info", + Format: "json", + ReportCaller: false, + }) + require.NoError(t, err) + + assert.Equal(t, log.InfoLevel, log.GetLevel()) + _, ok := log.StandardLogger().Formatter.(*log.JSONFormatter) + assert.True(t, ok) + assert.False(t, log.StandardLogger().ReportCaller) + assert.Same(t, os.Stdout, log.StandardLogger().Out) +} + +func TestConfigureInvalidLevelReturnsError(t *testing.T) { + preserveLogrusState(t) + + err := Configure(Config{ + Level: "not-a-level", + Format: "text", + }) + require.Error(t, err) +} + +func TestConfigureInvalidFormatReturnsError(t *testing.T) { + preserveLogrusState(t) + + err := Configure(Config{ + Level: "info", + Format: "xml", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid log format") +} diff --git a/pkg/controller/collectors_test.go b/pkg/controller/collectors_test.go new file mode 100644 index 0000000..55d5589 --- /dev/null +++ b/pkg/controller/collectors_test.go @@ -0,0 +1,206 @@ +package controller + +import ( + "strings" + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func collectorDescString(t *testing.T, collector prometheus.Collector) string { + t.Helper() + + ch := make(chan *prometheus.Desc, 8) + collector.Describe(ch) + close(ch) + + descs := make([]string, 0, len(ch)) + for desc := range ch { + descs = append(descs, desc.String()) + } + + require.NotEmpty(t, descs) + + return strings.Join(descs, "\n") +} + +func metricFamilyByName(t *testing.T, families []*dto.MetricFamily, name string) *dto.MetricFamily { + t.Helper() + + for _, family := range families { + if family.GetName() == name { + return family + } + } + + t.Fatalf("metric family %q not found", name) + return nil +} + +func labelNames(metric *dto.Metric) []string { + names := make([]string, 0, len(metric.GetLabel())) + for _, label := range metric.GetLabel() { + names = append(names, label.GetName()) + } + + return names +} + +func sampleLabelValues(size int) []string { + values := make([]string, 0, size) + for i := 0; i < size; i++ { + values = append(values, "value") + } + + return values +} + +func registerAndCollectMetricFamily(t *testing.T, name string, collector prometheus.Collector, labelCount int) *dto.MetricFamily { + t.Helper() + + registry := prometheus.NewRegistry() + require.NoError(t, registry.Register(collector)) + + switch c := collector.(type) { + case *prometheus.GaugeVec: + c.WithLabelValues(sampleLabelValues(labelCount)...).Set(1) + case *prometheus.CounterVec: + c.WithLabelValues(sampleLabelValues(labelCount)...).Add(1) + default: + t.Fatalf("unsupported collector type %T", collector) + } + + families, err := registry.Gather() + require.NoError(t, err) + + return metricFamilyByName(t, families, name) +} + +func TestCollectorConstructorsReturnExpectedCollectorTypesAndNames(t *testing.T) { + tests := []struct { + name string + newCollector func() prometheus.Collector + metricName string + counter bool + }{ + {"internal queued", NewInternalCollectorCurrentlyQueuedTasksCount, "gcpe_currently_queued_tasks_count", false}, + {"internal envs", NewInternalCollectorEnvironmentsCount, "gcpe_environments_count", false}, + {"internal runners", NewInternalCollectorRunnersCount, "gcpe_runners_count", false}, + {"internal executed", NewInternalCollectorExecutedTasksCount, "gcpe_executed_tasks_count", false}, + {"internal requests count", NewInternalCollectorGitLabAPIRequestsCount, "gcpe_gitlab_api_requests_count", false}, + {"internal requests remaining", NewInternalCollectorGitLabAPIRequestsRemaining, "gcpe_gitlab_api_requests_remaining", false}, + {"internal requests limit", NewInternalCollectorGitLabAPIRequestsLimit, "gcpe_gitlab_api_requests_limit", false}, + {"internal metrics", NewInternalCollectorMetricsCount, "gcpe_metrics_count", false}, + {"internal projects", NewInternalCollectorProjectsCount, "gcpe_projects_count", false}, + {"internal refs", NewInternalCollectorRefsCount, "gcpe_refs_count", false}, + {"coverage", NewCollectorCoverage, "gitlab_ci_pipeline_coverage", false}, + {"duration", NewCollectorDurationSeconds, "gitlab_ci_pipeline_duration_seconds", false}, + {"queued duration", NewCollectorQueuedDurationSeconds, "gitlab_ci_pipeline_queued_duration_seconds", false}, + {"env behind commits", NewCollectorEnvironmentBehindCommitsCount, "gitlab_ci_environment_behind_commits_count", false}, + {"env behind duration", NewCollectorEnvironmentBehindDurationSeconds, "gitlab_ci_environment_behind_duration_seconds", false}, + {"env deployment count", NewCollectorEnvironmentDeploymentCount, "gitlab_ci_environment_deployment_count", true}, + {"env deployment duration", NewCollectorEnvironmentDeploymentDurationSeconds, "gitlab_ci_environment_deployment_duration_seconds", false}, + {"env deployment job id", NewCollectorEnvironmentDeploymentJobID, "gitlab_ci_environment_deployment_job_id", false}, + {"env deployment status", NewCollectorEnvironmentDeploymentStatus, "gitlab_ci_environment_deployment_status", false}, + {"env deployment timestamp", NewCollectorEnvironmentDeploymentTimestamp, "gitlab_ci_environment_deployment_timestamp", false}, + {"env information", NewCollectorEnvironmentInformation, "gitlab_ci_environment_information", false}, + {"pipeline id", NewCollectorID, "gitlab_ci_pipeline_id", false}, + {"job artifact size", NewCollectorJobArtifactSizeBytes, "gitlab_ci_pipeline_job_artifact_size_bytes", false}, + {"job duration", NewCollectorJobDurationSeconds, "gitlab_ci_pipeline_job_duration_seconds", false}, + {"job id", NewCollectorJobID, "gitlab_ci_pipeline_job_id", false}, + {"job queued duration", NewCollectorJobQueuedDurationSeconds, "gitlab_ci_pipeline_job_queued_duration_seconds", false}, + {"job run count", NewCollectorJobRunCount, "gitlab_ci_pipeline_job_run_count", true}, + {"job status", NewCollectorJobStatus, "gitlab_ci_pipeline_job_status", false}, + {"job timestamp", NewCollectorJobTimestamp, "gitlab_ci_pipeline_job_timestamp", false}, + {"pipeline status", NewCollectorStatus, "gitlab_ci_pipeline_status", false}, + {"pipeline timestamp", NewCollectorTimestamp, "gitlab_ci_pipeline_timestamp", false}, + {"pipeline run count", NewCollectorRunCount, "gitlab_ci_pipeline_run_count", true}, + {"runners info", NewCollectorRunners, "gitlab_ci_runners_info", false}, + {"runner contacted at", NewCollectorRunnerContactedAtSeconds, "gitlab_ci_runner_contacted_at_seconds", false}, + {"runner project", NewCollectorRunnerProjectInfo, "gitlab_ci_runner_project_info", false}, + {"runner tag", NewCollectorRunnerTagInfo, "gitlab_ci_runner_tag_info", false}, + {"runner group", NewCollectorRunnerGroupInfo, "gitlab_ci_runner_group_info", false}, + {"test report total time", NewCollectorTestReportTotalTime, "gitlab_ci_pipeline_test_report_total_time", false}, + {"test report total count", NewCollectorTestReportTotalCount, "gitlab_ci_pipeline_test_report_total_count", false}, + {"test report success count", NewCollectorTestReportSuccessCount, "gitlab_ci_pipeline_test_report_success_count", false}, + {"test report failed count", NewCollectorTestReportFailedCount, "gitlab_ci_pipeline_test_report_failed_count", false}, + {"test report skipped count", NewCollectorTestReportSkippedCount, "gitlab_ci_pipeline_test_report_skipped_count", false}, + {"test report error count", NewCollectorTestReportErrorCount, "gitlab_ci_pipeline_test_report_error_count", false}, + {"test suite total time", NewCollectorTestSuiteTotalTime, "gitlab_ci_pipeline_test_suite_total_time", false}, + {"test suite total count", NewCollectorTestSuiteTotalCount, "gitlab_ci_pipeline_test_suite_total_count", false}, + {"test suite success count", NewCollectorTestSuiteSuccessCount, "gitlab_ci_pipeline_test_suite_success_count", false}, + {"test suite failed count", NewCollectorTestSuiteFailedCount, "gitlab_ci_pipeline_test_suite_failed_count", false}, + {"test suite skipped count", NewCollectorTestSuiteSkippedCount, "gitlab_ci_pipeline_test_suite_skipped_count", false}, + {"test suite error count", NewCollectorTestSuiteErrorCount, "gitlab_ci_pipeline_test_suite_error_count", false}, + {"test case execution time", NewCollectorTestCaseExecutionTime, "gitlab_ci_pipeline_test_case_execution_time", false}, + {"test case status", NewCollectorTestCaseStatus, "gitlab_ci_pipeline_test_case_status", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + collector := tt.newCollector() + desc := collectorDescString(t, collector) + + assert.Contains(t, desc, `fqName: "`+tt.metricName+`"`) + + if tt.counter { + _, ok := collector.(*prometheus.CounterVec) + assert.True(t, ok, "expected CounterVec, got %T", collector) + } else { + _, ok := collector.(*prometheus.GaugeVec) + assert.True(t, ok, "expected GaugeVec, got %T", collector) + } + }) + } +} + +func TestCollectorConstructorsExposeExpectedLabels(t *testing.T) { + tests := []struct { + name string + newCollector func() prometheus.Collector + metricName string + expected []string + }{ + { + name: "coverage labels", + newCollector: NewCollectorCoverage, + metricName: "gitlab_ci_pipeline_coverage", + expected: append(append([]string{}, defaultLabels...), pipelineLabels...), + }, + { + name: "environment information labels", + newCollector: NewCollectorEnvironmentInformation, + metricName: "gitlab_ci_environment_information", + expected: append(append([]string{}, environmentLabels...), environmentInformationLabels...), + }, + { + name: "job status labels", + newCollector: NewCollectorJobStatus, + metricName: "gitlab_ci_pipeline_job_status", + expected: append(append([]string{}, defaultLabels...), jobLabels...), + }, + { + name: "runner group labels", + newCollector: NewCollectorRunnerGroupInfo, + metricName: "gitlab_ci_runner_group_info", + expected: append([]string{}, runnerGroupLabels...), + }, + { + name: "test case status labels", + newCollector: NewCollectorTestCaseStatus, + metricName: "gitlab_ci_pipeline_test_case_status", + expected: append(append([]string{}, defaultLabels...), append(testSuiteLabels, append(testCaseLabels, statusLabels...)...)...), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + family := registerAndCollectMetricFamily(t, tt.metricName, tt.newCollector(), len(tt.expected)) + require.Len(t, family.GetMetric(), 1) + assert.ElementsMatch(t, tt.expected, labelNames(family.GetMetric()[0])) + }) + } +} diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go new file mode 100644 index 0000000..730d3c5 --- /dev/null +++ b/pkg/controller/controller_test.go @@ -0,0 +1,124 @@ +package controller + +import ( + "context" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/helvethink/gitlab-ci-exporter/pkg/config" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" + "github.com/helvethink/gitlab-ci-exporter/pkg/store" +) + +func TestRegisterTasksRegistersAllHandlers(t *testing.T) { + ctx := context.Background() + c := &Controller{ + TaskController: NewTaskController(ctx, nil, 10), + } + + c.registerTasks() + + for _, tt := range []schemas.TaskType{ + schemas.TaskTypeGarbageCollectEnvironments, + schemas.TaskTypeGarbageCollectMetrics, + schemas.TaskTypeGarbageCollectProjects, + schemas.TaskTypeGarbageCollectRefs, + schemas.TaskTypeGarbageCollectRunners, + schemas.TaskTypePullEnvironmentMetrics, + schemas.TaskTypePullEnvironmentsFromProject, + schemas.TaskTypePullEnvironmentsFromProjects, + schemas.TaskTypePullMetrics, + schemas.TaskTypePullProject, + schemas.TaskTypePullProjectsFromWildcard, + schemas.TaskTypePullProjectsFromWildcards, + schemas.TaskTypePullRefMetrics, + schemas.TaskTypePullRefsFromProject, + schemas.TaskTypePullRefsFromProjects, + schemas.TaskTypePullRunnersMetrics, + schemas.TaskTypePullRunnersFromProject, + schemas.TaskTypePullRunnersFromProjects, + } { + assert.NotNil(t, c.TaskController.TaskMap.Get(string(tt)), string(tt)) + } +} + +func TestDequeueTaskRemovesTaskFromStore(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + c := &Controller{Store: s} + + ok, err := s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-1", "") + require.NoError(t, err) + require.True(t, ok) + + c.dequeueTask(ctx, schemas.TaskTypePullMetrics, "task-1") + + queued, err := s.CurrentlyQueuedTasksCount(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(0), queued) + + executed, err := s.ExecutedTasksCount(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(1), executed) +} + +func TestConfigureTracingWithoutEndpointReturnsNil(t *testing.T) { + assert.NoError(t, configureTracing(context.Background(), "")) +} + +func TestConfigureGitlabInitializesClient(t *testing.T) { + c := &Controller{} + + err := c.configureGitlab(config.Gitlab{ + URL: "https://gitlab.example.com", + HealthURL: "https://gitlab.example.com/-/health", + Token: "test-token", + EnableTLSVerify: true, + MaximumRequestsPerSecond: 5, + BurstableRequestsPerSecond: 10, + MaximumJobsQueueSize: 10, + EnableHealthCheck: true, + }, "1.2.3") + require.NoError(t, err) + + require.NotNil(t, c.Gitlab) + assert.Equal(t, "https://gitlab.example.com/-/health", c.Gitlab.Readiness.URL) + assert.Contains(t, c.Gitlab.UserAgent, "1.2.3") + assert.NotNil(t, c.Gitlab.RateLimiter) +} + +func TestConfigureRedisWithoutURLSkipsConfiguration(t *testing.T) { + c := &Controller{} + + err := c.configureRedis(context.Background(), &config.Redis{}) + require.NoError(t, err) + assert.Nil(t, c.Redis) +} + +func TestConfigureRedisWithMiniredisConnectsSuccessfully(t *testing.T) { + ctx := context.Background() + mr := miniredis.RunT(t) + + c := &Controller{} + err := c.configureRedis(ctx, &config.Redis{ + URL: "redis://" + mr.Addr(), + }) + require.NoError(t, err) + require.NotNil(t, c.Redis) + + pong, err := c.Redis.Ping(ctx).Result() + require.NoError(t, err) + assert.Equal(t, "PONG", pong) +} + +func TestConfigureRedisWithInvalidURLReturnsError(t *testing.T) { + c := &Controller{} + + err := c.configureRedis(context.Background(), &config.Redis{ + URL: "://bad-url", + }) + require.Error(t, err) +} diff --git a/pkg/controller/environments_test.go b/pkg/controller/environments_test.go new file mode 100644 index 0000000..dc8b235 --- /dev/null +++ b/pkg/controller/environments_test.go @@ -0,0 +1,274 @@ +package controller + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmihailenco/taskq/v4" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +func registerNoopPullEnvironmentMetricsTask(t *testing.T, c *Controller) { + t.Helper() + + _, err := c.TaskController.TaskMap.Register(string(schemas.TaskTypePullEnvironmentMetrics), &taskq.TaskConfig{ + Handler: func(context.Context, schemas.Environment) { + }, + }) + require.NoError(t, err) +} + +func TestUpdateEnvironment(t *testing.T) { + ctx := context.Background() + createdAt := time.Unix(1710000000, 0).UTC() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/projects/group/project/environments/7") + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 7, + "name": "production", + "state": "available", + "external_url": "https://prod.example.com", + "last_deployment": { + "ref": "main", + "created_at": "2024-03-09T16:00:00Z", + "deployable": { + "id": 321, + "duration": 12, + "status": "success", + "tag": false, + "user": {"username": "alice"}, + "commit": {"short_id": "abc123"} + } + } + }`)) + }) + + env := schemas.Environment{ + ProjectName: "group/project", + ID: 7, + Name: "production", + } + + err := c.UpdateEnvironment(ctx, &env) + require.NoError(t, err) + + assert.True(t, env.Available) + assert.Equal(t, "https://prod.example.com", env.ExternalURL) + assert.Equal(t, 321, env.LatestDeployment.JobID) + assert.Equal(t, schemas.RefKindBranch, env.LatestDeployment.RefKind) + assert.Equal(t, "main", env.LatestDeployment.RefName) + assert.Equal(t, "alice", env.LatestDeployment.Username) + assert.Equal(t, "abc123", env.LatestDeployment.CommitShortID) + assert.Equal(t, float64(createdAt.Unix()), env.LatestDeployment.Timestamp) + + storedEnv := schemas.Environment{ + ProjectName: "group/project", + Name: "production", + } + err = c.Store.GetEnvironment(ctx, &storedEnv) + require.NoError(t, err) + assert.Equal(t, env.ExternalURL, storedEnv.ExternalURL) + assert.Equal(t, env.LatestDeployment.JobID, storedEnv.LatestDeployment.JobID) +} + +func TestPullEnvironmentsFromProject(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.Contains(r.URL.Path, "/projects/group/project/environments/7"): + _, _ = w.Write([]byte(`{ + "id": 7, + "name": "production", + "state": "available", + "external_url": "https://prod.example.com", + "last_deployment": { + "ref": "main", + "created_at": "2024-03-09T16:00:00Z", + "deployable": { + "id": 321, + "duration": 12, + "status": "success", + "tag": false + } + } + }`)) + case strings.Contains(r.URL.Path, "/projects/group/project/environments"): + _, _ = w.Write([]byte(`[{"id":7,"name":"production","state":"available"}]`)) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + c.TaskController = NewTaskController(ctx, nil, 10) + c.UUID = uuid.New() + registerNoopPullEnvironmentMetricsTask(t, c) + + p := schemas.NewProject("group/project") + p.Pull.Environments.Enabled = true + p.Pull.Environments.Regexp = ".*" + + err := c.PullEnvironmentsFromProject(ctx, p) + require.NoError(t, err) + + storedEnv := schemas.Environment{ + ProjectName: "group/project", + Name: "production", + } + err = c.Store.GetEnvironment(ctx, &storedEnv) + require.NoError(t, err) + + assert.Equal(t, 7, storedEnv.ID) + assert.True(t, storedEnv.Available) + assert.Equal(t, "https://prod.example.com", storedEnv.ExternalURL) + + queued, err := c.Store.CurrentlyQueuedTasksCount(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(1), queued) +} + +func TestPullEnvironmentMetrics(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.Contains(r.URL.Path, "/projects/group/project/environments/7"): + _, _ = w.Write([]byte(`{ + "id": 7, + "name": "production", + "state": "available", + "external_url": "https://prod.example.com", + "last_deployment": { + "ref": "main", + "created_at": "2024-03-09T16:00:00Z", + "deployable": { + "id": 12, + "duration": 15, + "status": "success", + "tag": false, + "user": {"username": "alice"}, + "commit": {"short_id": "currsha"} + } + } + }`)) + case strings.Contains(r.URL.Path, "/repository/branches/main"): + _, _ = w.Write([]byte(`{ + "name": "main", + "commit": { + "short_id": "newsha", + "committed_date": "2024-03-09T16:05:00Z" + } + }`)) + case strings.Contains(r.URL.Path, "/repository/compare"): + assert.Equal(t, "currsha", r.URL.Query().Get("from")) + assert.Equal(t, "newsha", r.URL.Query().Get("to")) + _, _ = w.Write([]byte(`{ + "commits": [ + {"id": "1"}, + {"id": "2"} + ] + }`)) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + env := schemas.Environment{ + ProjectName: "group/project", + ID: 7, + Name: "production", + OutputSparseStatusMetrics: false, + LatestDeployment: schemas.Deployment{ + JobID: 10, + RefKind: schemas.RefKindBranch, + RefName: "main", + CommitShortID: "oldsha", + Timestamp: float64(time.Unix(1709999900, 0).Unix()), + Status: "running", + }, + } + require.NoError(t, c.Store.SetEnvironment(ctx, env)) + + err := c.PullEnvironmentMetrics(ctx, env) + require.NoError(t, err) + + defaultLabels := env.DefaultLabelsValues() + + behindCommits := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentBehindCommitsCount, + Labels: defaultLabels, + } + require.NoError(t, c.Store.GetMetric(ctx, &behindCommits)) + assert.Equal(t, float64(2), behindCommits.Value) + + behindDuration := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentBehindDurationSeconds, + Labels: defaultLabels, + } + require.NoError(t, c.Store.GetMetric(ctx, &behindDuration)) + assert.Equal(t, float64(300), behindDuration.Value) + + deploymentCount := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentDeploymentCount, + Labels: defaultLabels, + } + require.NoError(t, c.Store.GetMetric(ctx, &deploymentCount)) + assert.Equal(t, float64(1), deploymentCount.Value) + + deploymentDuration := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentDeploymentDurationSeconds, + Labels: defaultLabels, + } + require.NoError(t, c.Store.GetMetric(ctx, &deploymentDuration)) + assert.Equal(t, float64(15), deploymentDuration.Value) + + deploymentJobID := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentDeploymentJobID, + Labels: defaultLabels, + } + require.NoError(t, c.Store.GetMetric(ctx, &deploymentJobID)) + assert.Equal(t, float64(12), deploymentJobID.Value) + + statusMetric := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentDeploymentStatus, + Labels: map[string]string{ + "project": "group/project", + "environment": "production", + "status": "success", + }, + } + require.NoError(t, c.Store.GetMetric(ctx, &statusMetric)) + assert.Equal(t, float64(1), statusMetric.Value) + + infoMetric := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentInformation, + Labels: map[string]string{ + "project": "group/project", + "environment": "production", + "environment_id": "7", + "external_url": "https://prod.example.com", + "kind": "branch", + "ref": "main", + "latest_commit_short_id": "newsha", + "current_commit_short_id": "currsha", + "available": "true", + "username": "alice", + }, + } + require.NoError(t, c.Store.GetMetric(ctx, &infoMetric)) + assert.Equal(t, float64(1), infoMetric.Value) +} diff --git a/pkg/controller/garbage_collector_test.go b/pkg/controller/garbage_collector_test.go new file mode 100644 index 0000000..a00a0a3 --- /dev/null +++ b/pkg/controller/garbage_collector_test.go @@ -0,0 +1,261 @@ +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" + "github.com/helvethink/gitlab-ci-exporter/pkg/store" +) + +func TestDeleteEnv(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + env := schemas.Environment{ + ProjectName: "group/project", + Name: "production", + } + require.NoError(t, s.SetEnvironment(ctx, env)) + + require.NoError(t, deleteEnv(ctx, s, env, "test")) + + exists, err := s.EnvironmentExists(ctx, env.Key()) + require.NoError(t, err) + assert.False(t, exists) +} + +func TestDeleteRunner(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + runner := schemas.Runner{ + ID: 42, + Name: "runner-42", + ProjectName: "group/project", + } + require.NoError(t, s.SetRunner(ctx, runner)) + + require.NoError(t, deleteRunner(ctx, s, runner, "test")) + + exists, err := s.RunnerExists(ctx, runner.Key()) + require.NoError(t, err) + assert.False(t, exists) +} + +func TestDeleteRef(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + ref := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + require.NoError(t, s.SetRef(ctx, ref)) + + require.NoError(t, deleteRef(ctx, s, ref, "test")) + + exists, err := s.RefExists(ctx, ref.Key()) + require.NoError(t, err) + assert.False(t, exists) +} + +func TestDeleteMetric(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + metric := schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 98.5, + } + require.NoError(t, s.SetMetric(ctx, metric)) + + require.NoError(t, deleteMetric(ctx, s, metric, "test")) + + exists, err := s.MetricExists(ctx, metric.Key()) + require.NoError(t, err) + assert.False(t, exists) +} + +func TestGarbageCollectProjectsDeletesUnconfiguredProjects(t *testing.T) { + ctx := context.Background() + c := &Controller{Store: store.NewLocalStore()} + + require.NoError(t, c.Store.SetProject(ctx, schemas.NewProject("group/project"))) + + require.NoError(t, c.GarbageCollectProjects(ctx)) + + count, err := c.Store.ProjectsCount(ctx) + require.NoError(t, err) + assert.Equal(t, int64(0), count) +} + +func TestGarbageCollectEnvironmentsDeletesEnvironmentWithoutProject(t *testing.T) { + ctx := context.Background() + c := &Controller{Store: store.NewLocalStore()} + + env := schemas.Environment{ + ProjectName: "group/project", + Name: "production", + ID: 7, + } + require.NoError(t, c.Store.SetEnvironment(ctx, env)) + + require.NoError(t, c.GarbageCollectEnvironments(ctx)) + + count, err := c.Store.EnvironmentsCount(ctx) + require.NoError(t, err) + assert.Equal(t, int64(0), count) +} + +func TestGarbageCollectRefsDeletesRefWithoutProject(t *testing.T) { + ctx := context.Background() + c := &Controller{Store: store.NewLocalStore()} + + ref := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + require.NoError(t, c.Store.SetRef(ctx, ref)) + + require.NoError(t, c.GarbageCollectRefs(ctx)) + + count, err := c.Store.RefsCount(ctx) + require.NoError(t, err) + assert.Equal(t, int64(0), count) +} + +func TestGarbageCollectRunnersDeletesRunnerWithoutProject(t *testing.T) { + ctx := context.Background() + c := &Controller{Store: store.NewLocalStore()} + + runner := schemas.Runner{ + ID: 42, + Name: "runner-42", + ProjectName: "group/project", + } + require.NoError(t, c.Store.SetRunner(ctx, runner)) + + require.NoError(t, c.GarbageCollectRunners(ctx)) + + count, err := c.Store.RunnersCount(ctx) + require.NoError(t, err) + assert.Equal(t, int64(0), count) +} + +func TestGarbageCollectMetricsDeletesOrphanAndSparseMetrics(t *testing.T) { + ctx := context.Background() + c := &Controller{Store: store.NewLocalStore()} + + refProject := schemas.NewProject("group/project") + refProject.OutputSparseStatusMetrics = true + ref := schemas.NewRef(refProject, schemas.RefKindBranch, "main") + require.NoError(t, c.Store.SetRef(ctx, ref)) + + env := schemas.Environment{ + ProjectName: "group/project", + Name: "production", + OutputSparseStatusMetrics: true, + } + require.NoError(t, c.Store.SetEnvironment(ctx, env)) + + runner := schemas.Runner{ + ID: 42, + Name: "runner-42", + ProjectName: "group/project", + OutputSparseStatusMetrics: true, + } + require.NoError(t, c.Store.SetRunner(ctx, runner)) + + orphanMetric := schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "missing/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 1, + } + require.NoError(t, c.Store.SetMetric(ctx, orphanMetric)) + + sparseRefMetric := schemas.Metric{ + Kind: schemas.MetricKindStatus, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "failed", + }, + Value: 0, + } + require.NoError(t, c.Store.SetMetric(ctx, sparseRefMetric)) + + sparseEnvMetric := schemas.Metric{ + Kind: schemas.MetricKindEnvironmentDeploymentStatus, + Labels: map[string]string{ + "project": "group/project", + "environment": "production", + "status": "failed", + }, + Value: 0, + } + require.NoError(t, c.Store.SetMetric(ctx, sparseEnvMetric)) + + sparseRunnerMetric := schemas.Metric{ + Kind: schemas.MetricKindRunner, + Labels: map[string]string{ + "runner_id": "42", + "runner_name": "runner-42", + "runner_description": "shared-runner", + "is_shared": "true", + "runner_type": "instance_type", + "online": "true", + "active": "true", + "status": "offline", + "runner_maintenance_note": "", + "paused": "false", + }, + Value: 0, + } + require.NoError(t, c.Store.SetMetric(ctx, sparseRunnerMetric)) + + preservedMetric := schemas.Metric{ + Kind: schemas.MetricKindRunnerContactedAtSeconds, + Labels: map[string]string{ + "runner_id": "42", + }, + Value: 1710000000, + } + require.NoError(t, c.Store.SetMetric(ctx, preservedMetric)) + + require.NoError(t, c.GarbageCollectMetrics(ctx)) + + exists, err := c.Store.MetricExists(ctx, orphanMetric.Key()) + require.NoError(t, err) + assert.False(t, exists) + + exists, err = c.Store.MetricExists(ctx, sparseRefMetric.Key()) + require.NoError(t, err) + assert.False(t, exists) + + exists, err = c.Store.MetricExists(ctx, sparseEnvMetric.Key()) + require.NoError(t, err) + assert.False(t, exists) + + exists, err = c.Store.MetricExists(ctx, sparseRunnerMetric.Key()) + require.NoError(t, err) + assert.False(t, exists) + + exists, err = c.Store.MetricExists(ctx, preservedMetric.Key()) + require.NoError(t, err) + assert.True(t, exists) +} diff --git a/pkg/controller/handlers_test.go b/pkg/controller/handlers_test.go new file mode 100644 index 0000000..825bd42 --- /dev/null +++ b/pkg/controller/handlers_test.go @@ -0,0 +1,169 @@ +package controller + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/helvethink/gitlab-ci-exporter/pkg/config" + gitlabclient "github.com/helvethink/gitlab-ci-exporter/pkg/gitlab" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +func TestHealthCheckHandlerWithGitLabReadinessEnabled(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/-/health", r.URL.Path) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + gl, err := gitlabclient.NewClient(gitlabclient.ClientConfig{ + URL: server.URL + "/api/v4", + Token: "test-token", + UserAgentVersion: "test", + ReadinessURL: server.URL + "/-/health", + RateLimiter: noopLimiterControllerRunnersTests{}, + }) + require.NoError(t, err) + gl.RateCounter = ratecounter.NewRateCounter(time.Second) + + c := &Controller{ + Gitlab: gl, + Config: config.Config{ + Gitlab: config.Gitlab{ + EnableHealthCheck: true, + }, + }, + } + + handler := c.HealthCheckHandler(ctx) + + req := httptest.NewRequest(http.MethodGet, "/ready", nil) + rr := httptest.NewRecorder() + handler.ReadyEndpoint(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) +} + +func TestHealthCheckHandlerWithGitLabReadinessDisabled(t *testing.T) { + ctx := context.Background() + c := &Controller{ + Config: config.Config{ + Gitlab: config.Gitlab{ + EnableHealthCheck: false, + }, + }, + } + + handler := c.HealthCheckHandler(ctx) + + req := httptest.NewRequest(http.MethodGet, "/ready", nil) + rr := httptest.NewRecorder() + handler.ReadyEndpoint(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) +} + +func TestMetricsHandlerExportsStoredAndInternalMetrics(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 97.5, + })) + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil).WithContext(ctx) + rr := httptest.NewRecorder() + + c.MetricsHandler(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + assert.Contains(t, rr.Body.String(), "gitlab_ci_pipeline_coverage") + assert.Contains(t, rr.Body.String(), `project="group/project"`) + assert.Contains(t, rr.Body.String(), "97.5") + assert.Contains(t, rr.Body.String(), "gcpe_metrics_count") + assert.Contains(t, rr.Body.String(), "gcpe_projects_count") +} + +func TestWebhookHandlerRejectsInvalidToken(t *testing.T) { + c := &Controller{ + Config: config.Config{ + Server: config.Server{ + Webhook: config.ServerWebhook{ + SecretToken: "expected-secret", + }, + }, + }, + } + + req := httptest.NewRequest(http.MethodPost, "/webhook", strings.NewReader(`{}`)) + req.Header.Set("X-Gitlab-Token", "wrong-secret") + rr := httptest.NewRecorder() + + c.WebhookHandler(rr, req) + + assert.Equal(t, http.StatusForbidden, rr.Code) + assert.JSONEq(t, `{"error":"invalid token"}`, rr.Body.String()) +} + +func TestWebhookHandlerReturnsBadRequestOnEmptyBody(t *testing.T) { + c := &Controller{ + Config: config.Config{ + Server: config.Server{ + Webhook: config.ServerWebhook{ + SecretToken: "expected-secret", + }, + }, + }, + } + + req := httptest.NewRequest(http.MethodPost, "/webhook", http.NoBody) + req.Header.Set("X-Gitlab-Token", "expected-secret") + rr := httptest.NewRecorder() + + c.WebhookHandler(rr, req) + + assert.Equal(t, http.StatusBadRequest, rr.Code) +} + +func TestWebhookHandlerReturnsBadRequestOnInvalidPayload(t *testing.T) { + c := &Controller{ + Config: config.Config{ + Server: config.Server{ + Webhook: config.ServerWebhook{ + SecretToken: "expected-secret", + }, + }, + }, + } + + req := httptest.NewRequest(http.MethodPost, "/webhook", strings.NewReader(`{"broken":`)) + req.Header.Set("X-Gitlab-Token", "expected-secret") + req.Header.Set("X-Gitlab-Event", "Pipeline Hook") + rr := httptest.NewRecorder() + + c.WebhookHandler(rr, req) + + assert.Equal(t, http.StatusBadRequest, rr.Code) +} diff --git a/pkg/controller/jobs_test.go b/pkg/controller/jobs_test.go new file mode 100644 index 0000000..3d3afe4 --- /dev/null +++ b/pkg/controller/jobs_test.go @@ -0,0 +1,212 @@ +package controller + +import ( + "context" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +func TestProcessJobMetricsStoresJobMetrics(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + project := schemas.NewProject("group/project") + project.Pull.Pipeline.Jobs.Enabled = true + project.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true + project.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runner-\d+` + + ref := schemas.NewRef(project, schemas.RefKindBranch, "main") + require.NoError(t, c.Store.SetRef(ctx, ref)) + + job := schemas.Job{ + ID: 101, + Name: "build", + Stage: "build", + Timestamp: 1710000000, + DurationSeconds: 10, + QueuedDurationSeconds: 2, + Status: "success", + PipelineID: 123, + TagList: "docker", + ArtifactSize: 2048, + FailureReason: "", + Runner: schemas.RunnerDesc{ + Description: "shared-runner-42", + }, + } + + c.ProcessJobMetrics(ctx, ref, job) + + storedRef := schemas.NewRef(project, schemas.RefKindBranch, "main") + require.NoError(t, c.Store.GetRef(ctx, &storedRef)) + require.Contains(t, storedRef.LatestJobs, "build") + assert.Equal(t, job, storedRef.LatestJobs["build"]) + + labels := map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "", + "variables": "", + "stage": "build", + "job_name": "build", + "runner_description": `shared-runner-\d+`, + "tag_list": "docker", + "status": "success", + "job_id": "101", + "pipeline_id": "123", + "failure_reason": "", + } + + jobIDMetric := schemas.Metric{ + Kind: schemas.MetricKindJobID, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &jobIDMetric)) + assert.Equal(t, float64(101), jobIDMetric.Value) + + jobRunCountMetric := schemas.Metric{ + Kind: schemas.MetricKindJobRunCount, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &jobRunCountMetric)) + assert.Equal(t, float64(0), jobRunCountMetric.Value) + + jobStatusMetric := schemas.Metric{ + Kind: schemas.MetricKindJobStatus, + Labels: map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "", + "variables": "", + "stage": "build", + "job_name": "build", + "runner_description": `shared-runner-\d+`, + "tag_list": "docker", + "status": "success", + "job_id": "101", + "pipeline_id": "123", + "failure_reason": "", + }, + } + require.NoError(t, c.Store.GetMetric(ctx, &jobStatusMetric)) + assert.Equal(t, float64(1), jobStatusMetric.Value) +} + +func TestProcessJobMetricsDoesNotRewriteIdenticalJob(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + project := schemas.NewProject("group/project") + project.Pull.Pipeline.Jobs.Enabled = true + + job := schemas.Job{ + ID: 101, + Name: "build", + Stage: "build", + Timestamp: 1710000000, + DurationSeconds: 10, + QueuedDurationSeconds: 2, + Status: "success", + PipelineID: 123, + TagList: "docker", + Runner: schemas.RunnerDesc{ + Description: "runner-1", + }, + } + + ref := schemas.NewRef(project, schemas.RefKindBranch, "main") + ref.LatestJobs["build"] = job + require.NoError(t, c.Store.SetRef(ctx, ref)) + + c.ProcessJobMetrics(ctx, ref, job) + + metrics, err := c.Store.Metrics(ctx) + require.NoError(t, err) + assert.Len(t, metrics, 0) +} + +func TestPullRefMostRecentJobsMetricsReturnsEarlyWhenDisabled(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + ref := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + ref.Project.Pull.Pipeline.Jobs.Enabled = false + + err := c.PullRefMostRecentJobsMetrics(ctx, ref) + require.NoError(t, err) +} + +func TestPullRefPipelineJobsMetrics(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + w.Header().Set("X-Total", "1") + + _, _ = w.Write([]byte(`[ + { + "id": 101, + "name": "build", + "stage": "build", + "created_at": "2024-03-09T16:00:00Z", + "duration": 10, + "queued_duration": 1, + "status": "success", + "ref": "main", + "tag_list": ["docker"], + "failure_reason": "", + "pipeline": {"id": 123}, + "artifacts": [{"size": 512}], + "runner": {"description": "runner-1"} + } + ]`)) + }) + + project := schemas.NewProject("group/project") + project.Pull.Pipeline.Jobs.Enabled = true + + ref := schemas.NewRef(project, schemas.RefKindBranch, "main") + ref.LatestPipeline.ID = 123 + require.NoError(t, c.Store.SetRef(ctx, ref)) + + err := c.PullRefPipelineJobsMetrics(ctx, ref) + require.NoError(t, err) + + jobIDMetric := schemas.Metric{ + Kind: schemas.MetricKindJobID, + Labels: map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "", + "variables": "", + "stage": "build", + "job_name": "build", + "runner_description": "runner-1", + "tag_list": "docker", + "status": "success", + "job_id": "101", + "pipeline_id": "123", + "failure_reason": "", + }, + } + require.NoError(t, c.Store.GetMetric(ctx, &jobIDMetric)) + assert.Equal(t, float64(101), jobIDMetric.Value) +} diff --git a/pkg/controller/metadata_test.go b/pkg/controller/metadata_test.go new file mode 100644 index 0000000..48b47fe --- /dev/null +++ b/pkg/controller/metadata_test.go @@ -0,0 +1,50 @@ +package controller + +import ( + "context" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + gitlabclient "github.com/helvethink/gitlab-ci-exporter/pkg/gitlab" +) + +func TestGetGitLabMetadataUpdatesVersion(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v4/metadata", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version":"16.11.2"}`)) + }) + + err := c.GetGitLabMetadata(ctx) + require.NoError(t, err) + assert.Equal(t, "v16.11.2", c.Gitlab.Version().Version) +} + +func TestGetGitLabMetadataKeepsCurrentVersionWhenMetadataVersionIsEmpty(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v4/metadata", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"version":""}`)) + }) + c.Gitlab.UpdateVersion(gitlabclient.NewGitLabVersion("15.9.0")) + + err := c.GetGitLabMetadata(ctx) + require.NoError(t, err) + assert.Equal(t, "v15.9.0", c.Gitlab.Version().Version) +} + +func TestGetGitLabMetadataPropagatesGitLabError(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v4/metadata", r.URL.Path) + http.Error(w, "boom", http.StatusInternalServerError) + }) + + err := c.GetGitLabMetadata(ctx) + require.Error(t, err) +} diff --git a/pkg/controller/metrics_test.go b/pkg/controller/metrics_test.go new file mode 100644 index 0000000..064ee68 --- /dev/null +++ b/pkg/controller/metrics_test.go @@ -0,0 +1,273 @@ +package controller + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + gitlabclient "github.com/helvethink/gitlab-ci-exporter/pkg/gitlab" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" + "github.com/helvethink/gitlab-ci-exporter/pkg/store" +) + +func metricFamilyValue(t *testing.T, family *dto.MetricFamily) float64 { + t.Helper() + require.Len(t, family.GetMetric(), 1) + + switch family.GetType() { + case dto.MetricType_GAUGE: + return family.GetMetric()[0].GetGauge().GetValue() + case dto.MetricType_COUNTER: + return family.GetMetric()[0].GetCounter().GetValue() + default: + t.Fatalf("unsupported metric family type %v", family.GetType()) + return 0 + } +} + +func TestNewRegistryInitializesCollectors(t *testing.T) { + r := NewRegistry(context.Background()) + + require.NotNil(t, r) + require.NotNil(t, r.InternalCollectors.CurrentlyQueuedTasksCount) + require.NotNil(t, r.InternalCollectors.ProjectsCount) + require.NotNil(t, r.GetCollector(schemas.MetricKindCoverage)) + require.NotNil(t, r.GetCollector(schemas.MetricKindRunCount)) + require.NotNil(t, r.GetCollector(schemas.MetricKindRunner)) +} + +func TestRegisterCollectorsReturnsErrorOnDuplicateCollector(t *testing.T) { + collector := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "test_duplicate_collector", Help: "test"}, + []string{}, + ) + + r := &Registry{ + Registry: prometheus.NewRegistry(), + Collectors: RegistryCollectors{ + schemas.MetricKindCoverage: collector, + schemas.MetricKindDurationSeconds: collector, + }, + } + + err := r.RegisterCollectors() + require.Error(t, err) + assert.Contains(t, err.Error(), "could not add provided collector") +} + +func TestExportInternalMetrics(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + + require.NoError(t, s.SetProject(ctx, schemas.NewProject("group/project"))) + require.NoError(t, s.SetEnvironment(ctx, schemas.Environment{ProjectName: "group/project", Name: "production"})) + require.NoError(t, s.SetRef(ctx, schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main"))) + require.NoError(t, s.SetRunner(ctx, schemas.Runner{ID: 42, ProjectName: "group/project"})) + require.NoError(t, s.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 1, + })) + + ok, err := s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-1", "") + require.NoError(t, err) + require.True(t, ok) + ok, err = s.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-2", "") + require.NoError(t, err) + require.True(t, ok) + require.NoError(t, s.DequeueTask(ctx, schemas.TaskTypePullMetrics, "task-1")) + + g := &gitlabclient.Client{ + RequestsRemaining: 17, + RequestsLimit: 50, + } + g.RequestsCounter.Add(9) + + r := NewRegistry(ctx) + require.NoError(t, r.ExportInternalMetrics(ctx, g, s)) + + families, err := r.Gather() + require.NoError(t, err) + + assert.Equal(t, float64(1), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_currently_queued_tasks_count"))) + assert.Equal(t, float64(1), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_executed_tasks_count"))) + assert.Equal(t, float64(1), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_projects_count"))) + assert.Equal(t, float64(1), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_environments_count"))) + assert.Equal(t, float64(1), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_refs_count"))) + assert.Equal(t, float64(1), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_runners_count"))) + assert.Equal(t, float64(1), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_metrics_count"))) + assert.Equal(t, float64(9), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_gitlab_api_requests_count"))) + assert.Equal(t, float64(17), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_gitlab_api_requests_remaining"))) + assert.Equal(t, float64(50), metricFamilyValue(t, metricFamilyByName(t, families, "gcpe_gitlab_api_requests_limit"))) +} + +func TestExportMetricsSetsGaugeAndCounterValues(t *testing.T) { + r := NewRegistry(context.Background()) + + r.ExportMetrics(schemas.Metrics{ + schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 97.5, + }.Key(): { + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 97.5, + }, + schemas.Metric{ + Kind: schemas.MetricKindRunCount, + Labels: map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 3, + }.Key(): { + Kind: schemas.MetricKindRunCount, + Labels: map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 3, + }, + }) + + families, err := r.Gather() + require.NoError(t, err) + + assert.Equal(t, float64(97.5), metricFamilyValue(t, metricFamilyByName(t, families, "gitlab_ci_pipeline_coverage"))) + assert.Equal(t, float64(3), metricFamilyValue(t, metricFamilyByName(t, families, "gitlab_ci_pipeline_run_count"))) +} + +func TestEmitStatusMetricDense(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + + emitStatusMetric(ctx, s, schemas.MetricKindStatus, map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + }, []string{"success", "failed"}, "success", false) + + successMetric := schemas.Metric{ + Kind: schemas.MetricKindStatus, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + } + require.NoError(t, s.GetMetric(ctx, &successMetric)) + assert.Equal(t, float64(1), successMetric.Value) + + failedMetric := schemas.Metric{ + Kind: schemas.MetricKindStatus, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "failed", + }, + } + require.NoError(t, s.GetMetric(ctx, &failedMetric)) + assert.Equal(t, float64(0), failedMetric.Value) +} + +func TestEmitStatusMetricSparse(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + + otherMetric := schemas.Metric{ + Kind: schemas.MetricKindStatus, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "failed", + }, + Value: 1, + } + require.NoError(t, s.SetMetric(ctx, otherMetric)) + + emitStatusMetric(ctx, s, schemas.MetricKindStatus, map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + }, []string{"success", "failed"}, "success", true) + + successMetric := schemas.Metric{ + Kind: schemas.MetricKindStatus, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + } + require.NoError(t, s.GetMetric(ctx, &successMetric)) + assert.Equal(t, float64(1), successMetric.Value) + + exists, err := s.MetricExists(ctx, otherMetric.Key()) + require.NoError(t, err) + assert.False(t, exists) +} diff --git a/pkg/controller/pipelines_test.go b/pkg/controller/pipelines_test.go new file mode 100644 index 0000000..770ed05 --- /dev/null +++ b/pkg/controller/pipelines_test.go @@ -0,0 +1,241 @@ +package controller + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +func TestProcessPipelinesMetricsStoresPipelineMetrics(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines/123")) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": 123, + "coverage": "85.5", + "updated_at": "2024-03-09T16:00:00Z", + "duration": 12, + "queued_duration": 3, + "source": "push", + "status": "success" + }`)) + }) + + ref := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{ID: 122} + + err := c.ProcessPipelinesMetrics(ctx, ref, &goGitlab.PipelineInfo{ID: 123}) + require.NoError(t, err) + + storedPipeline := schemas.Pipeline{ID: 123} + require.NoError(t, c.Store.GetPipeline(ctx, &storedPipeline)) + assert.Equal(t, 85.5, storedPipeline.Coverage) + assert.Equal(t, "success", storedPipeline.Status) + assert.Equal(t, "push", storedPipeline.Source) + + storedRef := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + require.NoError(t, c.Store.GetRef(ctx, &storedRef)) + assert.Equal(t, 123, storedRef.LatestPipeline.ID) + + labels := map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + } + + runCountMetric := schemas.Metric{ + Kind: schemas.MetricKindRunCount, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &runCountMetric)) + assert.Equal(t, float64(1), runCountMetric.Value) + + coverageMetric := schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &coverageMetric)) + assert.Equal(t, 85.5, coverageMetric.Value) + + durationMetric := schemas.Metric{ + Kind: schemas.MetricKindDurationSeconds, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &durationMetric)) + assert.Equal(t, float64(12), durationMetric.Value) + + statusMetric := schemas.Metric{ + Kind: schemas.MetricKindStatus, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &statusMetric)) + assert.Equal(t, float64(1), statusMetric.Value) +} + +func TestProcessTestReportMetricsStoresMetrics(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + ref := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{ + ID: 123, + Source: "push", + } + require.NoError(t, c.Store.SetRef(ctx, ref)) + + report := schemas.TestReport{ + TotalTime: 12.5, + TotalCount: 10, + SuccessCount: 8, + FailedCount: 1, + SkippedCount: 1, + ErrorCount: 0, + } + + c.ProcessTestReportMetrics(ctx, ref, report) + + labels := map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + } + + totalMetric := schemas.Metric{ + Kind: schemas.MetricKindTestReportTotalCount, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &totalMetric)) + assert.Equal(t, float64(10), totalMetric.Value) + + timeMetric := schemas.Metric{ + Kind: schemas.MetricKindTestReportTotalTime, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &timeMetric)) + assert.Equal(t, 12.5, timeMetric.Value) +} + +func TestProcessTestSuiteMetricsStoresMetrics(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + ref := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{Source: "push"} + require.NoError(t, c.Store.SetRef(ctx, ref)) + + suite := schemas.TestSuite{ + Name: "unit", + TotalTime: 4.2, + TotalCount: 5, + SuccessCount: 4, + FailedCount: 1, + SkippedCount: 0, + ErrorCount: 0, + } + + c.ProcessTestSuiteMetrics(ctx, ref, suite) + + labels := map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "test_suite_name": "unit", + } + + totalMetric := schemas.Metric{ + Kind: schemas.MetricKindTestSuiteTotalCount, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &totalMetric)) + assert.Equal(t, float64(5), totalMetric.Value) + + timeMetric := schemas.Metric{ + Kind: schemas.MetricKindTestSuiteTotalTime, + Labels: labels, + } + require.NoError(t, c.Store.GetMetric(ctx, &timeMetric)) + assert.Equal(t, 4.2, timeMetric.Value) +} + +func TestProcessTestCaseMetricsStoresMetrics(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + ref := schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{Source: "push"} + require.NoError(t, c.Store.SetRef(ctx, ref)) + + suite := schemas.TestSuite{Name: "unit"} + testCase := schemas.TestCase{ + Name: "TestCreateUser", + Classname: "service.user", + ExecutionTime: 0.42, + Status: "success", + } + + c.ProcessTestCaseMetrics(ctx, ref, suite, testCase) + + baseLabels := map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "test_suite_name": "unit", + "test_case_name": "TestCreateUser", + "test_case_classname": "service.user", + } + + timeMetric := schemas.Metric{ + Kind: schemas.MetricKindTestCaseExecutionTime, + Labels: baseLabels, + } + require.NoError(t, c.Store.GetMetric(ctx, &timeMetric)) + assert.Equal(t, 0.42, timeMetric.Value) + + statusLabels := map[string]string{ + "project": "group/project", + "topics": "", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "test_suite_name": "unit", + "test_case_name": "TestCreateUser", + "test_case_classname": "service.user", + "status": "success", + } + statusMetric := schemas.Metric{ + Kind: schemas.MetricKindTestCaseStatus, + Labels: statusLabels, + } + require.NoError(t, c.Store.GetMetric(ctx, &statusMetric)) + assert.Equal(t, float64(1), statusMetric.Value) +} diff --git a/pkg/controller/projects_test.go b/pkg/controller/projects_test.go new file mode 100644 index 0000000..72dfb3b --- /dev/null +++ b/pkg/controller/projects_test.go @@ -0,0 +1,117 @@ +package controller + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmihailenco/taskq/v4" + + "github.com/helvethink/gitlab-ci-exporter/pkg/config" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +func registerProjectFollowUpTasks(t *testing.T, c *Controller) { + t.Helper() + + _, err := c.TaskController.TaskMap.Register(string(schemas.TaskTypePullRefsFromProject), &taskq.TaskConfig{ + Handler: func(context.Context, schemas.Project) { + }, + }) + require.NoError(t, err) + + _, err = c.TaskController.TaskMap.Register(string(schemas.TaskTypePullEnvironmentsFromProject), &taskq.TaskConfig{ + Handler: func(context.Context, schemas.Project) { + }, + }) + require.NoError(t, err) + + _, err = c.TaskController.TaskMap.Register(string(schemas.TaskTypePullRunnersFromProject), &taskq.TaskConfig{ + Handler: func(context.Context, schemas.Project) { + }, + }) + require.NoError(t, err) +} + +func TestPullProject(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/projects/group/project") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": 101, + "path_with_namespace": "group/project" + }`)) + }) + c.TaskController = NewTaskController(ctx, nil, 10) + c.UUID = uuid.New() + registerProjectFollowUpTasks(t, c) + + pull := config.ProjectPull{} + pull.Environments.Enabled = true + pull.Refs.Branches.Enabled = true + pull.Runners.Enabled = true + + err := c.PullProject(ctx, "group/project", pull) + require.NoError(t, err) + + storedProject := schemas.NewProject("group/project") + require.NoError(t, c.Store.GetProject(ctx, &storedProject)) + assert.Equal(t, "group/project", storedProject.Name) + assert.True(t, storedProject.Pull.Environments.Enabled) + assert.True(t, storedProject.Pull.Refs.Branches.Enabled) + assert.True(t, storedProject.Pull.Runners.Enabled) + + queued, err := c.Store.CurrentlyQueuedTasksCount(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(3), queued) +} + +func TestPullProjectsFromWildcard(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/groups/platform/projects")) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + {"id": 1, "path_with_namespace": "platform/app-one"}, + {"id": 2, "path_with_namespace": "platform/app-two"} + ]`)) + }) + c.TaskController = NewTaskController(ctx, nil, 10) + c.UUID = uuid.New() + registerProjectFollowUpTasks(t, c) + + wildcard := config.Wildcard{ + Search: "app", + Owner: config.WildcardOwner{ + Kind: "group", + Name: "platform", + }, + } + wildcard.Pull.Environments.Enabled = true + wildcard.Pull.Refs.Branches.Enabled = true + wildcard.Pull.Runners.Enabled = true + + err := c.PullProjectsFromWildcard(ctx, wildcard) + require.NoError(t, err) + + projectOne := schemas.NewProject("platform/app-one") + require.NoError(t, c.Store.GetProject(ctx, &projectOne)) + assert.True(t, projectOne.Pull.Environments.Enabled) + + projectTwo := schemas.NewProject("platform/app-two") + require.NoError(t, c.Store.GetProject(ctx, &projectTwo)) + assert.True(t, projectTwo.Pull.Refs.Branches.Enabled) + + queued, err := c.Store.CurrentlyQueuedTasksCount(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(6), queued) +} diff --git a/pkg/controller/refs_test.go b/pkg/controller/refs_test.go new file mode 100644 index 0000000..658ceba --- /dev/null +++ b/pkg/controller/refs_test.go @@ -0,0 +1,115 @@ +package controller + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmihailenco/taskq/v4" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +func registerNoopPullRefMetricsTask(t *testing.T, c *Controller) { + t.Helper() + + _, err := c.TaskController.TaskMap.Register(string(schemas.TaskTypePullRefMetrics), &taskq.TaskConfig{ + Handler: func(context.Context, schemas.Ref) { + }, + }) + require.NoError(t, err) +} + +func TestGetRefs(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.Contains(r.URL.Path, "/repository/branches"): + _, _ = w.Write([]byte(`[ + {"name":"main"}, + {"name":"develop"} + ]`)) + case strings.Contains(r.URL.Path, "/repository/tags"): + _, _ = w.Write([]byte(`[ + {"name":"v1.0.0"}, + {"name":"v2.0.0"} + ]`)) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + p := schemas.NewProject("group/project") + p.Pull.Refs.Branches.Enabled = true + p.Pull.Refs.Branches.Regexp = "^main$" + p.Pull.Refs.Branches.ExcludeDeleted = true + p.Pull.Refs.Tags.Enabled = true + p.Pull.Refs.Tags.Regexp = "^v1\\." + p.Pull.Refs.Tags.ExcludeDeleted = true + p.Pull.Refs.MergeRequests.Enabled = false + + refs, err := c.GetRefs(ctx, p) + require.NoError(t, err) + require.Len(t, refs, 2) + + _, branchExists := refs[schemas.NewRef(p, schemas.RefKindBranch, "main").Key()] + assert.True(t, branchExists) + + _, tagExists := refs[schemas.NewRef(p, schemas.RefKindTag, "v1.0.0").Key()] + assert.True(t, tagExists) +} + +func TestPullRefsFromProject(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.Contains(r.URL.Path, "/repository/branches"): + _, _ = w.Write([]byte(`[ + {"name":"main"} + ]`)) + case strings.Contains(r.URL.Path, "/repository/tags"): + _, _ = w.Write([]byte(`[ + {"name":"v1.0.0"} + ]`)) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + c.TaskController = NewTaskController(ctx, nil, 10) + c.UUID = uuid.New() + registerNoopPullRefMetricsTask(t, c) + + p := schemas.NewProject("group/project") + p.Pull.Refs.Branches.Enabled = true + p.Pull.Refs.Branches.Regexp = "^main$" + p.Pull.Refs.Branches.ExcludeDeleted = true + p.Pull.Refs.Tags.Enabled = true + p.Pull.Refs.Tags.Regexp = "^v1\\." + p.Pull.Refs.Tags.ExcludeDeleted = true + p.Pull.Refs.MergeRequests.Enabled = false + + err := c.PullRefsFromProject(ctx, p) + require.NoError(t, err) + + storedBranch := schemas.NewRef(p, schemas.RefKindBranch, "main") + require.NoError(t, c.Store.GetRef(ctx, &storedBranch)) + assert.Equal(t, "main", storedBranch.Name) + + storedTag := schemas.NewRef(p, schemas.RefKindTag, "v1.0.0") + require.NoError(t, c.Store.GetRef(ctx, &storedTag)) + assert.Equal(t, "v1.0.0", storedTag.Name) + + queued, err := c.Store.CurrentlyQueuedTasksCount(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(2), queued) +} diff --git a/pkg/controller/runners_test.go b/pkg/controller/runners_test.go new file mode 100644 index 0000000..c74041a --- /dev/null +++ b/pkg/controller/runners_test.go @@ -0,0 +1,366 @@ +package controller + +import ( + "context" + "net/http" + "net/http/httptest" + "sort" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + gitlabclient "github.com/helvethink/gitlab-ci-exporter/pkg/gitlab" + "github.com/helvethink/gitlab-ci-exporter/pkg/ratelimit" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" + "github.com/helvethink/gitlab-ci-exporter/pkg/store" +) + +type noopLimiterControllerRunnersTests struct{} + +func (noopLimiterControllerRunnersTests) Take(ctx context.Context) time.Duration { + return 0 +} + +var _ ratelimit.Limiter = noopLimiterControllerRunnersTests{} + +func newTestRunnerController(t *testing.T, handler http.HandlerFunc) *Controller { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + client := &gitlabclient.Client{ + Client: gl, + RateLimiter: noopLimiterControllerRunnersTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + return &Controller{ + Store: store.NewLocalStore(), + Gitlab: client, + } +} + +func metricsByKind(metrics schemas.Metrics, kind schemas.MetricKind) []schemas.Metric { + out := make([]schemas.Metric, 0) + for _, m := range metrics { + if m.Kind == kind { + out = append(out, m) + } + } + return out +} + +func findMetricByKindAndLabel(metrics schemas.Metrics, kind schemas.MetricKind, labelKey, labelValue string) (schemas.Metric, bool) { + for _, m := range metrics { + if m.Kind == kind && m.Labels[labelKey] == labelValue { + return m, true + } + } + return schemas.Metric{}, false +} + +func TestUniqueSortedNonEmpty(t *testing.T) { + got := uniqueSortedNonEmpty([]string{"beta", "", "alpha", "beta", "gamma", ""}) + assert.Equal(t, []string{"alpha", "beta", "gamma"}, got) +} + +func TestRunnerGroupNames(t *testing.T) { + runner := schemas.Runner{ + Groups: []struct { + ID int + Name string + WebURL string + }{ + {ID: 1, Name: "platform"}, + {ID: 2, Name: "devops"}, + {ID: 3, Name: "platform"}, + {ID: 4, Name: ""}, + }, + } + + got := runnerGroupNames(runner) + assert.Equal(t, []string{"devops", "platform"}, got) +} + +func TestRunnerProjectNames(t *testing.T) { + runner := schemas.Runner{ + Projects: []struct { + ID int + Name string + NameWithNamespace string + Path string + PathWithNamespace string + }{ + {ID: 1, PathWithNamespace: "group-a/proj-a"}, + {ID: 2, NameWithNamespace: "group-b/proj-b"}, + {ID: 3, Name: "proj-c"}, + {ID: 4, PathWithNamespace: "group-a/proj-a"}, + {ID: 5}, + }, + } + + got := runnerProjectNames(runner) + assert.Equal(t, []string{"group-a/proj-a", "group-b/proj-b", "proj-c"}, got) +} + +func TestRunnerTagNames(t *testing.T) { + runner := schemas.Runner{ + TagList: []string{"docker", "linux", "docker", "", "arm64"}, + } + + got := runnerTagNames(runner) + assert.Equal(t, []string{"arm64", "docker", "linux"}, got) +} + +func TestUpdateRunner(t *testing.T) { + ctx := context.Background() + contactedAt := time.Unix(1710000000, 0).UTC() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/runners/101") + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 101, + "name": "runner-101", + "description": "shared-runner", + "paused": false, + "is_shared": true, + "runner_type": "instance_type", + "contacted_at": "2024-03-09T16:00:00Z", + "maintenance_note": "maintenance window", + "online": true, + "status": "online", + "tag_list": ["docker", "linux"], + "groups": [], + "projects": [] + }`)) + }) + + runner := schemas.Runner{ + ProjectName: "group/project", + ID: 101, + OutputSparseStatusMetrics: true, + } + + err := c.UpdateRunner(ctx, &runner) + require.NoError(t, err) + + assert.Equal(t, "group/project", runner.ProjectName) + assert.True(t, runner.OutputSparseStatusMetrics) + assert.Equal(t, "runner-101", runner.Name) + assert.Equal(t, "shared-runner", runner.Description) + assert.True(t, runner.IsShared) + assert.Equal(t, "instance_type", runner.RunnerType) + assert.True(t, runner.Online) + assert.Equal(t, "online", runner.Status) + assert.False(t, runner.Paused) + require.NotNil(t, runner.ContactedAt) + assert.Equal(t, contactedAt.Unix(), runner.ContactedAt.Unix()) + assert.Equal(t, "maintenance window", runner.MaintenanceNote) + + storedRunner := schemas.Runner{ + ProjectName: "group/project", + ID: 101, + } + err = c.Store.GetRunner(ctx, &storedRunner) + require.NoError(t, err) + + assert.Equal(t, "runner-101", storedRunner.Name) + assert.True(t, storedRunner.OutputSparseStatusMetrics) +} + +func TestDeleteRunnerMetrics(t *testing.T) { + ctx := context.Background() + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected HTTP call: %s", r.URL.Path) + }) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindRunner, + Labels: map[string]string{ + "runner_id": "42", + }, + Value: 1, + })) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindRunnerContactedAtSeconds, + Labels: map[string]string{ + "runner_id": "42", + }, + Value: 1710000000, + })) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindRunnerProjectInfo, + Labels: map[string]string{ + "runner_id": "42", + "project": "group/project", + }, + Value: 1, + })) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindRunnerTagInfo, + Labels: map[string]string{ + "runner_id": "999", + "tag": "docker", + }, + Value: 1, + })) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 99, + })) + + err := c.deleteRunnerMetrics(ctx, 42) + require.NoError(t, err) + + metrics, err := c.Store.Metrics(ctx) + require.NoError(t, err) + + assert.Len(t, metricsByKind(metrics, schemas.MetricKindRunner), 0) + assert.Len(t, metricsByKind(metrics, schemas.MetricKindRunnerContactedAtSeconds), 0) + assert.Len(t, metricsByKind(metrics, schemas.MetricKindRunnerProjectInfo), 0) + + remainingTagMetrics := metricsByKind(metrics, schemas.MetricKindRunnerTagInfo) + require.Len(t, remainingTagMetrics, 1) + assert.Equal(t, "999", remainingTagMetrics[0].Labels["runner_id"]) + + remainingCoverage := metricsByKind(metrics, schemas.MetricKindCoverage) + require.Len(t, remainingCoverage, 1) +} + +func TestProcessRunnerMetrics(t *testing.T) { + ctx := context.Background() + + c := newTestRunnerController(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/runners/202") + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 202, + "name": "runner-202", + "description": "linux-runner", + "paused": false, + "is_shared": true, + "runner_type": "instance_type", + "contacted_at": "2024-03-09T16:00:00Z", + "maintenance_note": "maintenance", + "online": true, + "status": "online", + "tag_list": ["docker", "linux", "docker"], + "groups": [ + {"id": 1, "name": "platform", "web_url": "https://gitlab.example.com/groups/platform"}, + {"id": 2, "name": "devops", "web_url": "https://gitlab.example.com/groups/devops"}, + {"id": 3, "name": "platform", "web_url": "https://gitlab.example.com/groups/platform"} + ], + "projects": [ + {"id": 1, "name": "proj-a", "name_with_namespace": "group-a/proj-a", "path": "proj-a", "path_with_namespace": "group-a/proj-a"}, + {"id": 2, "name": "proj-b", "name_with_namespace": "group-b/proj-b", "path": "proj-b", "path_with_namespace": "group-b/proj-b"}, + {"id": 3, "name": "proj-a", "name_with_namespace": "group-a/proj-a", "path": "proj-a", "path_with_namespace": "group-a/proj-a"} + ] + }`)) + }) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindRunnerTagInfo, + Labels: map[string]string{ + "runner_id": "202", + "tag": "old-tag", + }, + Value: 1, + })) + + require.NoError(t, c.Store.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindRunner, + Labels: map[string]string{ + "runner_id": "999", + "runner_name": "other-runner", + "runner_description": "other-desc", + }, + Value: 1, + })) + + inputRunner := schemas.Runner{ + ProjectName: "group/project", + ID: 202, + OutputSparseStatusMetrics: true, + } + + err := c.ProcessRunnerMetrics(ctx, inputRunner) + require.NoError(t, err) + + metrics, err := c.Store.Metrics(ctx) + require.NoError(t, err) + + infoMetric, ok := findMetricByKindAndLabel(metrics, schemas.MetricKindRunner, "runner_id", "202") + require.True(t, ok) + assert.Equal(t, "runner-202", infoMetric.Labels["runner_name"]) + assert.Equal(t, "linux-runner", infoMetric.Labels["runner_description"]) + assert.Equal(t, "true", infoMetric.Labels["is_shared"]) + assert.Equal(t, "instance_type", infoMetric.Labels["runner_type"]) + assert.Equal(t, "true", infoMetric.Labels["online"]) + assert.Equal(t, "true", infoMetric.Labels["active"]) + assert.Equal(t, "false", infoMetric.Labels["paused"]) + assert.Equal(t, "online", infoMetric.Labels["status"]) + assert.Equal(t, "maintenance", infoMetric.Labels["runner_maintenance_note"]) + assert.Equal(t, 1.0, infoMetric.Value) + + contactMetric, ok := findMetricByKindAndLabel(metrics, schemas.MetricKindRunnerContactedAtSeconds, "runner_id", "202") + require.True(t, ok) + assert.Equal(t, float64(1710000000), contactMetric.Value) + + projectMetrics := metricsByKind(metrics, schemas.MetricKindRunnerProjectInfo) + sort.Slice(projectMetrics, func(i, j int) bool { + return projectMetrics[i].Labels["project"] < projectMetrics[j].Labels["project"] + }) + require.Len(t, projectMetrics, 2) + assert.Equal(t, "group-a/proj-a", projectMetrics[0].Labels["project"]) + assert.Equal(t, "group-b/proj-b", projectMetrics[1].Labels["project"]) + + tagMetrics := metricsByKind(metrics, schemas.MetricKindRunnerTagInfo) + sort.Slice(tagMetrics, func(i, j int) bool { + return tagMetrics[i].Labels["tag"] < tagMetrics[j].Labels["tag"] + }) + require.Len(t, tagMetrics, 2) + assert.Equal(t, "docker", tagMetrics[0].Labels["tag"]) + assert.Equal(t, "linux", tagMetrics[1].Labels["tag"]) + + groupMetrics := metricsByKind(metrics, schemas.MetricKindRunnerGroupInfo) + sort.Slice(groupMetrics, func(i, j int) bool { + return groupMetrics[i].Labels["group"] < groupMetrics[j].Labels["group"] + }) + require.Len(t, groupMetrics, 2) + assert.Equal(t, "devops", groupMetrics[0].Labels["group"]) + assert.Equal(t, "platform", groupMetrics[1].Labels["group"]) + + otherRunnerMetric, ok := findMetricByKindAndLabel(metrics, schemas.MetricKindRunner, "runner_id", "999") + require.True(t, ok) + assert.Equal(t, "other-runner", otherRunnerMetric.Labels["runner_name"]) + + _, ok = findMetricByKindAndLabel(metrics, schemas.MetricKindRunnerTagInfo, "tag", "old-tag") + assert.False(t, ok) +} diff --git a/pkg/controller/scheduler_test.go b/pkg/controller/scheduler_test.go new file mode 100644 index 0000000..5314165 --- /dev/null +++ b/pkg/controller/scheduler_test.go @@ -0,0 +1,46 @@ +package controller + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmihailenco/taskq/v4" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" + "github.com/helvethink/gitlab-ci-exporter/pkg/store" +) + +func TestScheduleTaskQueuesTaskOnlyOnce(t *testing.T) { + ctx := context.Background() + var handled atomic.Int32 + + c := &Controller{ + Store: store.NewLocalStore(), + TaskController: NewTaskController(ctx, nil, 10), + UUID: uuid.New(), + } + _, err := c.TaskController.TaskMap.Register(string(schemas.TaskTypePullProject), &taskq.TaskConfig{ + Handler: func(context.Context, string) error { + handled.Add(1) + + return nil + }, + }) + require.NoError(t, err) + + c.ScheduleTask(ctx, schemas.TaskTypePullProject, "group/project", "group/project") + c.ScheduleTask(ctx, schemas.TaskTypePullProject, "group/project", "group/project") + + require.Eventually(t, func() bool { + return handled.Load() == 1 + }, 2*time.Second, 10*time.Millisecond) + + queued, err := c.Store.CurrentlyQueuedTasksCount(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(1), queued) +} diff --git a/pkg/controller/store_test.go b/pkg/controller/store_test.go new file mode 100644 index 0000000..444bd06 --- /dev/null +++ b/pkg/controller/store_test.go @@ -0,0 +1,152 @@ +package controller + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" + "github.com/helvethink/gitlab-ci-exporter/pkg/store" +) + +type metricStoreStub struct { + store.Store + getMetricErr error + setMetricErr error + delMetricErr error +} + +func (s metricStoreStub) GetMetric(ctx context.Context, m *schemas.Metric) error { + if s.getMetricErr != nil { + return s.getMetricErr + } + + return s.Store.GetMetric(ctx, m) +} + +func (s metricStoreStub) SetMetric(ctx context.Context, m schemas.Metric) error { + if s.setMetricErr != nil { + return s.setMetricErr + } + + return s.Store.SetMetric(ctx, m) +} + +func (s metricStoreStub) DelMetric(ctx context.Context, k schemas.MetricKey) error { + if s.delMetricErr != nil { + return s.delMetricErr + } + + return s.Store.DelMetric(ctx, k) +} + +func TestMetricLogFields(t *testing.T) { + m := schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "ref": "main", + }, + } + + fields := metricLogFields(m) + + assert.Equal(t, m.Kind, fields["metric-kind"]) + assert.Equal(t, m.Labels, fields["metric-labels"]) +} + +func TestStoreSetMetricStoresMetric(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + m := schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 98.5, + } + + storeSetMetric(ctx, s, m) + + var got schemas.Metric + got.Kind = m.Kind + got.Labels = m.Labels + + require.NoError(t, s.GetMetric(ctx, &got)) + assert.Equal(t, m.Value, got.Value) +} + +func TestStoreGetMetricLoadsMetric(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + want := schemas.Metric{ + Kind: schemas.MetricKindRunner, + Labels: map[string]string{ + "runner_id": "42", + }, + Value: 1, + } + require.NoError(t, s.SetMetric(ctx, want)) + + got := schemas.Metric{ + Kind: want.Kind, + Labels: map[string]string{ + "runner_id": "42", + }, + } + + storeGetMetric(ctx, s, &got) + + assert.Equal(t, want, got) +} + +func TestStoreDelMetricDeletesMetric(t *testing.T) { + ctx := context.Background() + s := store.NewLocalStore() + m := schemas.Metric{ + Kind: schemas.MetricKindRunnerTagInfo, + Labels: map[string]string{ + "runner_id": "42", + "tag": "docker", + }, + Value: 1, + } + require.NoError(t, s.SetMetric(ctx, m)) + + storeDelMetric(ctx, s, m) + + exists, err := s.MetricExists(ctx, m.Key()) + require.NoError(t, err) + assert.False(t, exists) +} + +func TestStoreMetricHelpersIgnoreStoreErrors(t *testing.T) { + ctx := context.Background() + s := metricStoreStub{ + Store: store.NewLocalStore(), + getMetricErr: errors.New("get failed"), + setMetricErr: errors.New("set failed"), + delMetricErr: errors.New("del failed"), + } + m := schemas.Metric{ + Kind: schemas.MetricKindRunner, + Labels: map[string]string{ + "runner_id": "42", + }, + } + + assert.NotPanics(t, func() { + storeSetMetric(ctx, s, m) + storeGetMetric(ctx, s, &m) + storeDelMetric(ctx, s, m) + }) +} diff --git a/pkg/gitlab/branches_test.go b/pkg/gitlab/branches_test.go new file mode 100644 index 0000000..ce8b3f8 --- /dev/null +++ b/pkg/gitlab/branches_test.go @@ -0,0 +1,131 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +type noopLimiterBranchesTests struct{} + +func (noopLimiterBranchesTests) Take(ctx context.Context) time.Duration { + return 0 +} + +func newTestGitLabClientForBranches(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiterBranchesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestGetProjectBranches_InvalidRegexp(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterBranchesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + p.Pull.Refs.Branches.Regexp = "[" + + refs, err := c.GetProjectBranches(context.Background(), p) + + assert.Error(t, err) + assert.Empty(t, refs) +} + +func TestGetProjectBranches_FiltersAndPaginates(t *testing.T) { + c := newTestGitLabClientForBranches(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/repository/branches")) + w.Header().Set("Content-Type", "application/json") + + page := r.URL.Query().Get("page") + switch page { + case "", "1": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + _, _ = w.Write([]byte(`[ + {"name":"main"}, + {"name":"feature/test"} + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + {"name":"develop"}, + {"name":"release/1.0"} + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + p := schemas.NewProject("group/project") + p.Pull.Refs.Branches.Regexp = `^(main|develop)$` + + refs, err := c.GetProjectBranches(context.Background(), p) + require.NoError(t, err) + + expected1 := schemas.NewRef(p, schemas.RefKindBranch, "main") + expected2 := schemas.NewRef(p, schemas.RefKindBranch, "develop") + + require.Len(t, refs, 2) + assert.Contains(t, refs, expected1.Key()) + assert.Contains(t, refs, expected2.Key()) + assert.Equal(t, expected1, refs[expected1.Key()]) + assert.Equal(t, expected2, refs[expected2.Key()]) +} + +func TestGetBranchLatestCommit(t *testing.T) { + c := newTestGitLabClientForBranches(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/repository/branches/main")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "name": "main", + "commit": { + "short_id": "abc123", + "committed_date": "2024-03-09T16:00:00Z" + } + }`)) + }) + + shortID, ts, err := c.GetBranchLatestCommit(context.Background(), "group/project", "main") + require.NoError(t, err) + + assert.Equal(t, "abc123", shortID) + assert.Equal(t, float64(1710000000), ts) +} + +func TestGetBranchLatestCommit_APIError(t *testing.T) { + c := newTestGitLabClientForBranches(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"boom"}`, http.StatusInternalServerError) + }) + + shortID, ts, err := c.GetBranchLatestCommit(context.Background(), "group/project", "main") + assert.Error(t, err) + assert.Equal(t, "", shortID) + assert.Equal(t, float64(0), ts) +} diff --git a/pkg/gitlab/client_test.go b/pkg/gitlab/client_test.go new file mode 100644 index 0000000..29824f1 --- /dev/null +++ b/pkg/gitlab/client_test.go @@ -0,0 +1,195 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/ratelimit" +) + +type mockLimiter struct { + called int32 +} + +func (m *mockLimiter) Take(ctx context.Context) time.Duration { + atomic.AddInt32(&m.called, 1) + return 0 +} + +var _ ratelimit.Limiter = (*mockLimiter)(nil) + +func TestNewHTTPClient(t *testing.T) { + client := NewHTTPClient(true) + require.NotNil(t, client) + + transport, ok := client.Transport.(*http.Transport) + require.True(t, ok) + require.NotNil(t, transport.TLSClientConfig) + assert.True(t, transport.TLSClientConfig.InsecureSkipVerify) +} + +func TestNewHTTPClient_DisableTLSFalse(t *testing.T) { + client := NewHTTPClient(false) + require.NotNil(t, client) + + transport, ok := client.Transport.(*http.Transport) + require.True(t, ok) + require.NotNil(t, transport.TLSClientConfig) + assert.False(t, transport.TLSClientConfig.InsecureSkipVerify) +} + +func TestNewClient(t *testing.T) { + limiter := &mockLimiter{} + + cfg := ClientConfig{ + URL: "https://gitlab.example.com/api/v4", + Token: "test-token", + UserAgentVersion: "1.2.3", + DisableTLSVerify: true, + ReadinessURL: "https://gitlab.example.com/-/health", + RateLimiter: limiter, + } + + c, err := NewClient(cfg) + require.NoError(t, err) + require.NotNil(t, c) + + require.NotNil(t, c.Client) + assert.Equal(t, "gitlab-ci-pipelines-exporter-1.2.3", c.UserAgent) + assert.Same(t, limiter, c.RateLimiter) + + assert.Equal(t, "https://gitlab.example.com/-/health", c.Readiness.URL) + require.NotNil(t, c.Readiness.HTTPClient) + assert.Equal(t, 5*time.Second, c.Readiness.HTTPClient.Timeout) + + require.NotNil(t, c.RateCounter) +} + +func TestReadinessCheck_OK(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodGet, r.Method) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + c := &Client{} + c.Readiness.URL = server.URL + c.Readiness.HTTPClient = server.Client() + + check := c.ReadinessCheck(context.Background()) + err := check() + assert.NoError(t, err) +} + +func TestReadinessCheck_HTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "not ready", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + + c := &Client{} + c.Readiness.URL = server.URL + c.Readiness.HTTPClient = server.Client() + + check := c.ReadinessCheck(context.Background()) + err := check() + require.Error(t, err) + assert.Contains(t, err.Error(), "HTTP error: 503") +} + +func TestReadinessCheck_NoHTTPClient(t *testing.T) { + c := &Client{} + c.Readiness.URL = "https://gitlab.example.com/-/health" + c.Readiness.HTTPClient = nil + + check := c.ReadinessCheck(context.Background()) + err := check() + require.Error(t, err) + assert.Contains(t, err.Error(), "readiness http client not configured") +} + +func TestRateLimit(t *testing.T) { + limiter := &mockLimiter{} + c := &Client{ + RateLimiter: limiter, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + before := c.RequestsCounter.Load() + c.rateLimit(context.Background()) + + assert.Equal(t, int32(1), atomic.LoadInt32(&limiter.called)) + assert.Equal(t, before+1, c.RequestsCounter.Load()) + assert.Equal(t, int64(1), c.RateCounter.Rate()) +} + +func TestUpdateVersionAndVersion(t *testing.T) { + c := &Client{} + + assert.Equal(t, GitLabVersion{}, c.Version()) + + v := NewGitLabVersion("15.9.0") + c.UpdateVersion(v) + + assert.Equal(t, v, c.Version()) +} + +func TestRequestsRemaining(t *testing.T) { + resp := &goGitlab.Response{ + Response: &http.Response{ + Header: http.Header{ + "Ratelimit-Remaining": []string{"42"}, + "Ratelimit-Limit": []string{"100"}, + }, + }, + } + + c := &Client{} + c.requestsRemaining(resp) + + assert.Equal(t, 42, c.RequestsRemaining) + assert.Equal(t, 100, c.RequestsLimit) +} + +func TestRequestsRemaining_NilResponse(t *testing.T) { + c := &Client{ + RequestsRemaining: 7, + RequestsLimit: 9, + } + + c.requestsRemaining(nil) + + assert.Equal(t, 7, c.RequestsRemaining) + assert.Equal(t, 9, c.RequestsLimit) +} + +func TestRequestsRemaining_InvalidHeaders(t *testing.T) { + resp := &goGitlab.Response{ + Response: &http.Response{ + Header: http.Header{ + "Ratelimit-Remaining": []string{"abc"}, + "Ratelimit-Limit": []string{"def"}, + }, + }, + } + + c := &Client{ + RequestsRemaining: 11, + RequestsLimit: 22, + } + + c.requestsRemaining(resp) + + // strconv.Atoi errors are ignored by production code, so values fall back to zero. + assert.Equal(t, 0, c.RequestsRemaining) + assert.Equal(t, 0, c.RequestsLimit) +} diff --git a/pkg/gitlab/environments_test.go b/pkg/gitlab/environments_test.go new file mode 100644 index 0000000..4919593 --- /dev/null +++ b/pkg/gitlab/environments_test.go @@ -0,0 +1,259 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +type noopLimiterEnvironmentsTests struct{} + +func (noopLimiterEnvironmentsTests) Take(ctx context.Context) time.Duration { + return 0 +} + +func newTestGitLabClientForEnvironments(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiterEnvironmentsTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestGetProjectEnvironments_InvalidRegexp(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterEnvironmentsTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + p.Pull.Environments.Regexp = "[" + + envs, err := c.GetProjectEnvironments(context.Background(), p) + + assert.Error(t, err) + assert.Nil(t, envs) +} + +func TestGetProjectEnvironments_FiltersAndPaginates(t *testing.T) { + c := newTestGitLabClientForEnvironments(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/environments")) + w.Header().Set("Content-Type", "application/json") + + page := r.URL.Query().Get("page") + switch page { + case "", "1": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + _, _ = w.Write([]byte(`[ + {"id":1,"name":"production","state":"available"}, + {"id":2,"name":"review/123","state":"stopped"} + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + {"id":3,"name":"staging","state":"available"} + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + p := schemas.NewProject("group/project") + p.Pull.Environments.Regexp = `^(production|staging)$` + p.OutputSparseStatusMetrics = true + + envs, err := c.GetProjectEnvironments(context.Background(), p) + require.NoError(t, err) + + expected1 := schemas.Environment{ + ProjectName: p.Name, + ID: 1, + Name: "production", + Available: true, + OutputSparseStatusMetrics: true, + } + expected2 := schemas.Environment{ + ProjectName: p.Name, + ID: 3, + Name: "staging", + Available: true, + OutputSparseStatusMetrics: true, + } + + require.Len(t, envs, 2) + assert.Contains(t, envs, expected1.Key()) + assert.Contains(t, envs, expected2.Key()) + assert.Equal(t, expected1, envs[expected1.Key()]) + assert.Equal(t, expected2, envs[expected2.Key()]) +} + +func TestGetProjectEnvironments_ExcludeStopped(t *testing.T) { + c := newTestGitLabClientForEnvironments(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/environments")) + assert.Equal(t, "available", r.URL.Query().Get("states")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id":1,"name":"production","state":"available"} + ]`)) + }) + + p := schemas.NewProject("group/project") + p.Pull.Environments.Regexp = `^production$` + p.Pull.Environments.ExcludeStopped = true + + envs, err := c.GetProjectEnvironments(context.Background(), p) + require.NoError(t, err) + require.Len(t, envs, 1) + + expected := schemas.Environment{ + ProjectName: p.Name, + ID: 1, + Name: "production", + Available: true, + OutputSparseStatusMetrics: true, + } + assert.Contains(t, envs, expected.Key()) + assert.Equal(t, expected, envs[expected.Key()]) +} + +func TestGetEnvironment_NoLastDeployment(t *testing.T) { + c := newTestGitLabClientForEnvironments(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/environments/42")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 42, + "name": "production", + "external_url": "https://example.com", + "state": "available", + "last_deployment": null + }`)) + }) + + env, err := c.GetEnvironment(context.Background(), "group/project", 42) + require.NoError(t, err) + + assert.Equal(t, "group/project", env.ProjectName) + assert.Equal(t, 42, env.ID) + assert.Equal(t, "production", env.Name) + assert.Equal(t, "https://example.com", env.ExternalURL) + assert.True(t, env.Available) + assert.Equal(t, schemas.Deployment{}, env.LatestDeployment) +} + +func TestGetEnvironment_WithLastDeployment_Branch(t *testing.T) { + c := newTestGitLabClientForEnvironments(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/environments/99")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 99, + "name": "staging", + "external_url": "https://staging.example.com", + "state": "available", + "last_deployment": { + "ref": "main", + "created_at": "2024-03-09T16:00:00Z", + "deployable": { + "id": 1234, + "tag": false, + "duration": 12.5, + "status": "success", + "user": { + "username": "jdoe" + }, + "commit": { + "short_id": "abc123" + } + } + } + }`)) + }) + + env, err := c.GetEnvironment(context.Background(), "group/project", 99) + require.NoError(t, err) + + assert.Equal(t, "group/project", env.ProjectName) + assert.Equal(t, 99, env.ID) + assert.Equal(t, "staging", env.Name) + assert.Equal(t, "https://staging.example.com", env.ExternalURL) + assert.True(t, env.Available) + + assert.Equal(t, schemas.RefKindBranch, env.LatestDeployment.RefKind) + assert.Equal(t, "main", env.LatestDeployment.RefName) + assert.Equal(t, 1234, env.LatestDeployment.JobID) + assert.Equal(t, 12.5, env.LatestDeployment.DurationSeconds) + assert.Equal(t, "success", env.LatestDeployment.Status) + assert.Equal(t, "jdoe", env.LatestDeployment.Username) + assert.Equal(t, "abc123", env.LatestDeployment.CommitShortID) + assert.Equal(t, float64(1710000000), env.LatestDeployment.Timestamp) +} + +func TestGetEnvironment_WithLastDeployment_Tag(t *testing.T) { + c := newTestGitLabClientForEnvironments(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/environments/100")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 100, + "name": "release", + "external_url": "https://release.example.com", + "state": "stopped", + "last_deployment": { + "ref": "v1.2.3", + "created_at": "2024-03-10T10:00:00Z", + "deployable": { + "id": 555, + "tag": true, + "duration": 20, + "status": "failed" + } + } + }`)) + }) + + env, err := c.GetEnvironment(context.Background(), "group/project", 100) + require.NoError(t, err) + + assert.Equal(t, "group/project", env.ProjectName) + assert.Equal(t, 100, env.ID) + assert.Equal(t, "release", env.Name) + assert.Equal(t, "https://release.example.com", env.ExternalURL) + assert.False(t, env.Available) + + assert.Equal(t, schemas.RefKindTag, env.LatestDeployment.RefKind) + assert.Equal(t, "v1.2.3", env.LatestDeployment.RefName) + assert.Equal(t, 555, env.LatestDeployment.JobID) + assert.Equal(t, 20.0, env.LatestDeployment.DurationSeconds) + assert.Equal(t, "failed", env.LatestDeployment.Status) + assert.Equal(t, "", env.LatestDeployment.Username) + assert.Equal(t, "", env.LatestDeployment.CommitShortID) + assert.Equal(t, float64(1710064800), env.LatestDeployment.Timestamp) +} diff --git a/pkg/gitlab/jobs_test.go b/pkg/gitlab/jobs_test.go new file mode 100644 index 0000000..8efa1e8 --- /dev/null +++ b/pkg/gitlab/jobs_test.go @@ -0,0 +1,381 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +type noopLimiterJobsTests struct{} + +func (noopLimiterJobsTests) Take(ctx context.Context) time.Duration { + return 0 +} + +func newTestGitLabClientForJobs(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiterJobsTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestListRefPipelineJobs_EmptyLatestPipeline(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterJobsTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + jobs, err := c.ListRefPipelineJobs(context.Background(), ref) + require.NoError(t, err) + assert.Nil(t, jobs) +} + +func TestListPipelineJobs_Paginates(t *testing.T) { + c := newTestGitLabClientForJobs(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines/123/jobs")) + w.Header().Set("Content-Type", "application/json") + + page := r.URL.Query().Get("page") + switch page { + case "", "1": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + w.Header().Set("X-Total", "2") + _, _ = w.Write([]byte(`[ + { + "id": 101, + "name": "build", + "stage": "build", + "created_at": "2024-03-09T16:00:00Z", + "duration": 10, + "queued_duration": 1, + "status": "success", + "ref": "main", + "tag_list": ["docker"], + "failure_reason": "", + "pipeline": {"id": 123}, + "artifacts": [{"size": 100}], + "runner": {"description": "runner-1"} + } + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + w.Header().Set("X-Total", "2") + _, _ = w.Write([]byte(`[ + { + "id": 102, + "name": "test", + "stage": "test", + "created_at": "2024-03-09T16:01:00Z", + "duration": 20, + "queued_duration": 2, + "status": "failed", + "ref": "main", + "tag_list": ["linux"], + "failure_reason": "script_failure", + "pipeline": {"id": 123}, + "artifacts": [{"size": 200}], + "runner": {"description": "runner-2"} + } + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + jobs, err := c.ListPipelineJobs(context.Background(), "group/project", 123) + require.NoError(t, err) + require.Len(t, jobs, 2) + + assert.Equal(t, 101, jobs[0].ID) + assert.Equal(t, "build", jobs[0].Name) + assert.Equal(t, 100.0, jobs[0].ArtifactSize) + + assert.Equal(t, 102, jobs[1].ID) + assert.Equal(t, "test", jobs[1].Name) + assert.Equal(t, "script_failure", jobs[1].FailureReason) +} + +func TestListPipelineBridges_Paginates(t *testing.T) { + c := newTestGitLabClientForJobs(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines/123/bridges")) + w.Header().Set("Content-Type", "application/json") + + page := r.URL.Query().Get("page") + switch page { + case "", "1": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + w.Header().Set("X-Total", "2") + _, _ = w.Write([]byte(`[ + {"id": 1, "name": "bridge-1"} + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + w.Header().Set("X-Total", "2") + _, _ = w.Write([]byte(`[ + {"id": 2, "name": "bridge-2"} + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + bridges, err := c.ListPipelineBridges(context.Background(), "group/project", 123) + require.NoError(t, err) + require.Len(t, bridges, 2) + + assert.Equal(t, 1, bridges[0].ID) + assert.Equal(t, 2, bridges[1].ID) +} + +func TestListPipelineChildJobs(t *testing.T) { + c := newTestGitLabClientForJobs(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.Contains(r.URL.Path, "/pipelines/1000/bridges"): + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + { + "id": 11, + "name": "trigger-child", + "downstream_pipeline": { + "id": 2001, + "project_id": 222 + } + }, + { + "id": 12, + "name": "not-run-yet", + "downstream_pipeline": null + } + ]`)) + + case strings.Contains(r.URL.Path, "/projects/222/pipelines/2001/jobs"): + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + { + "id": 201, + "name": "child-job", + "stage": "test", + "created_at": "2024-03-09T16:00:00Z", + "duration": 12, + "queued_duration": 1, + "status": "success", + "ref": "main", + "tag_list": [], + "failure_reason": "", + "pipeline": {"id": 2001}, + "artifacts": [], + "runner": {"description": "child-runner"} + } + ]`)) + + case strings.Contains(r.URL.Path, "/projects/222/pipelines/2001/bridges"): + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[]`)) + + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + jobs, err := c.ListPipelineChildJobs(context.Background(), "group/project", 1000) + require.NoError(t, err) + require.Len(t, jobs, 1) + + assert.Equal(t, 201, jobs[0].ID) + assert.Equal(t, "child-job", jobs[0].Name) + assert.Equal(t, 2001, jobs[0].PipelineID) +} + +func TestListRefPipelineJobs_WithChildPipelinesEnabled(t *testing.T) { + c := newTestGitLabClientForJobs(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.Contains(r.URL.Path, "/pipelines/1000/jobs"): + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + { + "id": 101, + "name": "parent-job", + "stage": "build", + "created_at": "2024-03-09T16:00:00Z", + "duration": 10, + "queued_duration": 1, + "status": "success", + "ref": "main", + "tag_list": [], + "failure_reason": "", + "pipeline": {"id": 1000}, + "artifacts": [], + "runner": {"description": "runner-parent"} + } + ]`)) + + case strings.Contains(r.URL.Path, "/pipelines/1000/bridges"): + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + { + "id": 11, + "name": "trigger-child", + "downstream_pipeline": { + "id": 2001, + "project_id": 222 + } + } + ]`)) + + case strings.Contains(r.URL.Path, "/projects/222/pipelines/2001/jobs"): + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + { + "id": 201, + "name": "child-job", + "stage": "test", + "created_at": "2024-03-09T16:01:00Z", + "duration": 20, + "queued_duration": 2, + "status": "success", + "ref": "main", + "tag_list": [], + "failure_reason": "", + "pipeline": {"id": 2001}, + "artifacts": [], + "runner": {"description": "runner-child"} + } + ]`)) + + case strings.Contains(r.URL.Path, "/projects/222/pipelines/2001/bridges"): + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[]`)) + + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + p := schemas.NewProject("group/project") + p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = true + + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{ID: 1000} + + jobs, err := c.ListRefPipelineJobs(context.Background(), ref) + require.NoError(t, err) + require.Len(t, jobs, 2) + + assert.Equal(t, "parent-job", jobs[0].Name) + assert.Equal(t, "child-job", jobs[1].Name) +} + +func TestListRefMostRecentJobs_NoJobsInMemory(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterJobsTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + jobs, err := c.ListRefMostRecentJobs(context.Background(), ref) + require.NoError(t, err) + assert.Nil(t, jobs) +} + +func TestListRefMostRecentJobs_FindsAllJobsOnFirstPage(t *testing.T) { + c := newTestGitLabClientForJobs(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/projects/")) + assert.True(t, strings.Contains(r.URL.Path, "/jobs")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + w.Header().Set("X-Total", "2") + + _, _ = w.Write([]byte(`[ + { + "id": 101, + "name": "build", + "stage": "build", + "created_at": "2024-03-09T16:00:00Z", + "duration": 10, + "queued_duration": 1, + "status": "success", + "ref": "main", + "tag_list": [], + "failure_reason": "", + "pipeline": {"id": 1000}, + "artifacts": [], + "runner": {"description": "runner-1"} + }, + { + "id": 102, + "name": "test", + "stage": "test", + "created_at": "2024-03-09T16:01:00Z", + "duration": 20, + "queued_duration": 2, + "status": "failed", + "ref": "main", + "tag_list": [], + "failure_reason": "script_failure", + "pipeline": {"id": 1000}, + "artifacts": [], + "runner": {"description": "runner-2"} + } + ]`)) + }) + + c.version = NewGitLabVersion("15.8.0") + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + ref.LatestJobs = schemas.Jobs{ + "build": {Name: "build"}, + "test": {Name: "test"}, + } + + jobs, err := c.ListRefMostRecentJobs(context.Background(), ref) + require.NoError(t, err) + require.Len(t, jobs, 2) + + assert.Equal(t, "build", jobs[0].Name) + assert.Equal(t, "test", jobs[1].Name) +} diff --git a/pkg/gitlab/pipelines_test.go b/pkg/gitlab/pipelines_test.go new file mode 100644 index 0000000..56d5e75 --- /dev/null +++ b/pkg/gitlab/pipelines_test.go @@ -0,0 +1,337 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +type noopLimiterPipelinesTests struct{} + +func (noopLimiterPipelinesTests) Take(ctx context.Context) time.Duration { + return 0 +} + +func newTestGitLabClientForPipelines(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiterPipelinesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestGetRefPipeline(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines/123")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 123, + "coverage": "87.5", + "updated_at": "2024-03-09T16:00:00Z", + "duration": 120, + "queued_duration": 15, + "source": "push", + "status": "success", + "detailed_status": { + "group": "waiting-for-resource" + } + }`)) + }) + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + got, err := c.GetRefPipeline(context.Background(), ref, 123) + require.NoError(t, err) + + assert.Equal(t, 123, got.ID) + assert.Equal(t, 87.5, got.Coverage) + assert.Equal(t, float64(1710000000), got.Timestamp) + assert.Equal(t, 120.0, got.DurationSeconds) + assert.Equal(t, 15.0, got.QueuedDurationSeconds) + assert.Equal(t, "push", got.Source) + assert.Equal(t, "waiting_for_resource", got.Status) +} + +func TestGetRefPipeline_APIError(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"boom"}`, http.StatusInternalServerError) + }) + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + got, err := c.GetRefPipeline(context.Background(), ref, 123) + assert.Error(t, err) + assert.Equal(t, schemas.Pipeline{}, got) + assert.Contains(t, err.Error(), "could not read content of pipeline") +} + +func TestGetProjectPipelines_DefaultPagination(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines")) + assert.Equal(t, "1", r.URL.Query().Get("page")) + assert.Equal(t, "100", r.URL.Query().Get("per_page")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id": 1, "ref": "main"}, + {"id": 2, "ref": "develop"} + ]`)) + }) + + options := &goGitlab.ListProjectPipelinesOptions{} + pipelines, resp, err := c.GetProjectPipelines(context.Background(), "group/project", options) + require.NoError(t, err) + require.NotNil(t, resp) + + require.Len(t, pipelines, 2) + assert.Equal(t, 1, pipelines[0].ID) + assert.Equal(t, "main", pipelines[0].Ref) + assert.Equal(t, 2, pipelines[1].ID) + assert.Equal(t, "develop", pipelines[1].Ref) +} + +func TestGetProjectPipelines_APIError(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"boom"}`, http.StatusInternalServerError) + }) + + options := &goGitlab.ListProjectPipelinesOptions{} + pipelines, _, err := c.GetProjectPipelines(context.Background(), "group/project", options) + assert.Error(t, err) + assert.Nil(t, pipelines) + assert.Contains(t, err.Error(), "error listing project pipelines") +} + +func TestGetRefPipelineVariablesAsConcatenatedString_EmptyPipeline(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterPipelinesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + got, err := c.GetRefPipelineVariablesAsConcatenatedString(context.Background(), ref, schemas.Pipeline{}) + require.NoError(t, err) + assert.Equal(t, "", got) +} + +func TestGetRefPipelineVariablesAsConcatenatedString_InvalidRegexp(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterPipelinesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + p.Pull.Pipeline.Variables.Regexp = "[" + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + got, err := c.GetRefPipelineVariablesAsConcatenatedString(context.Background(), ref, schemas.Pipeline{ID: 123}) + assert.Error(t, err) + assert.Equal(t, "", got) + assert.Contains(t, err.Error(), "provided filter regex") +} + +func TestGetRefPipelineVariablesAsConcatenatedString_FiltersVariables(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines/123/variables")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`[ + {"key":"FOO","value":"1"}, + {"key":"BAZ","value":"2"}, + {"key":"BAR","value":"3"} + ]`)) + }) + + p := schemas.NewProject("group/project") + p.Pull.Pipeline.Variables.Regexp = `^(FOO|BAR)$` + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + got, err := c.GetRefPipelineVariablesAsConcatenatedString(context.Background(), ref, schemas.Pipeline{ID: 123}) + require.NoError(t, err) + assert.Equal(t, "FOO:1,BAR:3", got) +} + +func TestGetRefsFromPipelines_BranchesMostRecent(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines")) + assert.Equal(t, "branches", r.URL.Query().Get("scope")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id": 1, "ref": "main"}, + {"id": 2, "ref": "feature/test"}, + {"id": 3, "ref": "develop"} + ]`)) + }) + + p := schemas.NewProject("group/project") + p.Pull.Refs.Branches.Regexp = `^(main|develop)$` + p.Pull.Refs.Branches.MostRecent = 1 + p.Pull.Refs.Branches.ExcludeDeleted = false + + refs, err := c.GetRefsFromPipelines(context.Background(), p, schemas.RefKindBranch) + require.NoError(t, err) + + expected := schemas.NewRef(p, schemas.RefKindBranch, "main") + require.Len(t, refs, 1) + assert.Contains(t, refs, expected.Key()) + assert.Equal(t, expected, refs[expected.Key()]) +} + +func TestGetRefsFromPipelines_MergeRequests(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id": 1, "ref": "refs/merge-requests/12/head"}, + {"id": 2, "ref": "refs/merge-requests/99/merge"} + ]`)) + }) + + p := schemas.NewProject("group/project") + p.Pull.Refs.MergeRequests.MostRecent = 2 + + refs, err := c.GetRefsFromPipelines(context.Background(), p, schemas.RefKindMergeRequest) + require.NoError(t, err) + + expected1 := schemas.NewRef(p, schemas.RefKindMergeRequest, "12") + expected2 := schemas.NewRef(p, schemas.RefKindMergeRequest, "99") + + require.Len(t, refs, 2) + assert.Contains(t, refs, expected1.Key()) + assert.Contains(t, refs, expected2.Key()) + assert.Equal(t, expected1, refs[expected1.Key()]) + assert.Equal(t, expected2, refs[expected2.Key()]) +} + +func TestGetRefsFromPipelines_UnsupportedKind(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterPipelinesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + + refs, err := c.GetRefsFromPipelines(context.Background(), p, schemas.RefKind("unsupported")) + assert.Error(t, err) + assert.Empty(t, refs) + assert.Contains(t, err.Error(), "invalid ref kind") +} + +func TestGetRefPipelineTestReport_EmptyLatestPipeline(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterPipelinesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + + report, err := c.GetRefPipelineTestReport(context.Background(), ref) + require.NoError(t, err) + assert.Equal(t, schemas.TestReport{}, report) +} + +func TestGetRefPipelineTestReport_SinglePipeline(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/pipelines/321/test_report")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "total_time": 20, + "total_count": 6, + "success_count": 4, + "failed_count": 1, + "skipped_count": 1, + "error_count": 0, + "test_suites": [ + { + "name": "suite-1", + "total_time": 20, + "total_count": 6, + "success_count": 4, + "failed_count": 1, + "skipped_count": 1, + "error_count": 0, + "test_cases": [ + { + "name": "TestA", + "classname": "suite1", + "execution_time": 0.3, + "status": "success" + } + ] + } + ] + }`)) + }) + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{ID: 321} + + report, err := c.GetRefPipelineTestReport(context.Background(), ref) + require.NoError(t, err) + + assert.Equal(t, 20.0, report.TotalTime) + assert.Equal(t, 6, report.TotalCount) + assert.Equal(t, 4, report.SuccessCount) + assert.Equal(t, 1, report.FailedCount) + assert.Equal(t, 1, report.SkippedCount) + assert.Equal(t, 0, report.ErrorCount) + require.Len(t, report.TestSuites, 1) + assert.Equal(t, "suite-1", report.TestSuites[0].Name) + require.Len(t, report.TestSuites[0].TestCases, 1) + assert.Equal(t, "TestA", report.TestSuites[0].TestCases[0].Name) +} + +func TestGetRefPipelineTestReport_APIError(t *testing.T) { + c := newTestGitLabClientForPipelines(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"boom"}`, http.StatusInternalServerError) + }) + + p := schemas.NewProject("group/project") + ref := schemas.NewRef(p, schemas.RefKindBranch, "main") + ref.LatestPipeline = schemas.Pipeline{ID: 321} + + report, err := c.GetRefPipelineTestReport(context.Background(), ref) + assert.Error(t, err) + assert.Equal(t, schemas.TestReport{}, report) + assert.Contains(t, err.Error(), "could not fetch test report") +} diff --git a/pkg/gitlab/projects_test.go b/pkg/gitlab/projects_test.go new file mode 100644 index 0000000..c2e6617 --- /dev/null +++ b/pkg/gitlab/projects_test.go @@ -0,0 +1,215 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/config" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +type noopLimiterProjectsTests struct{} + +func (noopLimiterProjectsTests) Take(ctx context.Context) time.Duration { + return 0 +} + +func newTestGitLabClientForProjects(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiterProjectsTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestGetProject(t *testing.T) { + c := newTestGitLabClientForProjects(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/projects/")) + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write([]byte(`{ + "id": 101, + "path_with_namespace": "group/project", + "name": "project" + }`)) + }) + + p, err := c.GetProject(context.Background(), "group/project") + require.NoError(t, err) + require.NotNil(t, p) + + assert.Equal(t, 101, p.ID) + assert.Equal(t, "group/project", p.PathWithNamespace) + assert.Equal(t, "project", p.Name) +} + +func TestListProjects_UserOwner(t *testing.T) { + c := newTestGitLabClientForProjects(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/users/john/projects")) + assert.Equal(t, "myapp", r.URL.Query().Get("search")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id":1,"path_with_namespace":"john/myapp-api"}, + {"id":2,"path_with_namespace":"john/myapp-web"}, + {"id":3,"path_with_namespace":"other/myapp-other"} + ]`)) + }) + + w := config.Wildcard{} + w.Search = "myapp" + w.Owner.Kind = "user" + w.Owner.Name = "john" + + projects, err := c.ListProjects(context.Background(), w) + require.NoError(t, err) + + require.Len(t, projects, 2) + assert.Equal(t, "john/myapp-api", projects[0].Name) + assert.Equal(t, "john/myapp-web", projects[1].Name) +} + +func TestListProjects_GroupOwner_Paginates(t *testing.T) { + c := newTestGitLabClientForProjects(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/groups/team/projects")) + w.Header().Set("Content-Type", "application/json") + + page := r.URL.Query().Get("page") + switch page { + case "", "1": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + _, _ = w.Write([]byte(`[ + {"id":1,"path_with_namespace":"team/service-a"}, + {"id":2,"path_with_namespace":"team/service-b"} + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + {"id":3,"path_with_namespace":"team/sub/service-c"} + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + w := config.Wildcard{} + w.Owner.Kind = "group" + w.Owner.Name = "team" + w.Owner.IncludeSubgroups = true + + projects, err := c.ListProjects(context.Background(), w) + require.NoError(t, err) + + require.Len(t, projects, 3) + assert.Equal(t, "team/service-a", projects[0].Name) + assert.Equal(t, "team/service-b", projects[1].Name) + assert.Equal(t, "team/sub/service-c", projects[2].Name) +} + +func TestListProjects_DefaultListsVisibleProjects(t *testing.T) { + c := newTestGitLabClientForProjects(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/projects")) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id":1,"path_with_namespace":"group/project-a"}, + {"id":2,"path_with_namespace":"other/project-b"} + ]`)) + }) + + w := config.Wildcard{} + w.Search = "project" + + projects, err := c.ListProjects(context.Background(), w) + require.NoError(t, err) + + require.Len(t, projects, 2) + assert.Equal(t, "group/project-a", projects[0].Name) + assert.Equal(t, "other/project-b", projects[1].Name) +} + +func TestListProjects_FiltersByOwnerName(t *testing.T) { + c := newTestGitLabClientForProjects(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/groups/team/projects")) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id":1,"path_with_namespace":"team/service-a"}, + {"id":2,"path_with_namespace":"other/service-b"} + ]`)) + }) + + w := config.Wildcard{} + w.Owner.Kind = "group" + w.Owner.Name = "team" + + projects, err := c.ListProjects(context.Background(), w) + require.NoError(t, err) + + require.Len(t, projects, 1) + assert.Equal(t, "team/service-a", projects[0].Name) +} + +func TestListProjects_APIError(t *testing.T) { + c := newTestGitLabClientForProjects(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, `{"message":"boom"}`, http.StatusInternalServerError) + }) + + w := config.Wildcard{} + w.Search = "myapp" + + projects, err := c.ListProjects(context.Background(), w) + assert.Error(t, err) + assert.Empty(t, projects) + assert.Contains(t, err.Error(), "unable to list projects with search pattern") +} + +func TestListProjects_ReturnsSchemaProjects(t *testing.T) { + c := newTestGitLabClientForProjects(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"id":1,"path_with_namespace":"team/service-a"} + ]`)) + }) + + w := config.Wildcard{} + + projects, err := c.ListProjects(context.Background(), w) + require.NoError(t, err) + require.Len(t, projects, 1) + + expected := schemas.NewProject("team/service-a") + assert.Equal(t, expected.Name, projects[0].Name) +} diff --git a/pkg/gitlab/repositories_test.go b/pkg/gitlab/repositories_test.go new file mode 100644 index 0000000..57f8805 --- /dev/null +++ b/pkg/gitlab/repositories_test.go @@ -0,0 +1,101 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" +) + +type noopLimiterRepositoriesTests struct{} + +func (noopLimiterRepositoriesTests) Take(ctx context.Context) time.Duration { + return 0 +} + +func newTestGitLabClientForRepositories(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiterRepositoriesTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestGetCommitCountBetweenRefs(t *testing.T) { + c := newTestGitLabClientForRepositories(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/repository/compare")) + assert.Equal(t, "main", r.URL.Query().Get("from")) + assert.Equal(t, "release", r.URL.Query().Get("to")) + assert.Equal(t, "true", r.URL.Query().Get("straight")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`{ + "commit": null, + "commits": [ + {"id":"111"}, + {"id":"222"}, + {"id":"333"} + ], + "diffs": [] + }`)) + }) + + count, err := c.GetCommitCountBetweenRefs(context.Background(), "group/project", "main", "release") + require.NoError(t, err) + assert.Equal(t, 3, count) +} + +func TestGetCommitCountBetweenRefs_NoCommits(t *testing.T) { + c := newTestGitLabClientForRepositories(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/repository/compare")) + assert.Equal(t, "main", r.URL.Query().Get("from")) + assert.Equal(t, "release", r.URL.Query().Get("to")) + assert.Equal(t, "true", r.URL.Query().Get("straight")) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`{ + "commit": null, + "commits": [], + "diffs": [] + }`)) + }) + + count, err := c.GetCommitCountBetweenRefs(context.Background(), "group/project", "main", "release") + require.NoError(t, err) + assert.Equal(t, 0, count) +} + +func TestGetCommitCountBetweenRefs_APIError(t *testing.T) { + c := newTestGitLabClientForRepositories(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/repository/compare")) + http.Error(w, `{"message":"internal error"}`, http.StatusInternalServerError) + }) + + count, err := c.GetCommitCountBetweenRefs(context.Background(), "group/project", "main", "release") + assert.Error(t, err) + assert.Equal(t, 0, count) +} diff --git a/pkg/gitlab/runners_test.go b/pkg/gitlab/runners_test.go new file mode 100644 index 0000000..6e36da7 --- /dev/null +++ b/pkg/gitlab/runners_test.go @@ -0,0 +1,258 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +type noopLimiterForRunnersTests struct{} + +func (noopLimiterForRunnersTests) Take(ctx context.Context) time.Duration { + return 0 +} + +func newTestGitLabClientForRunners(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiterForRunnersTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestGetProjectRunners_InvalidRegexp(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiterForRunnersTests{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + p.Pull.Runners.Regexp = "[" + + runners, err := c.GetProjectRunners(context.Background(), p) + + assert.Error(t, err) + assert.Nil(t, runners) +} + +func TestGetProjectRunners_FiltersAndPaginates(t *testing.T) { + c := newTestGitLabClientForRunners(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/runners")) + w.Header().Set("Content-Type", "application/json") + + page := r.URL.Query().Get("page") + switch page { + case "", "1": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + _, _ = w.Write([]byte(`[ + {"id":1,"name":"runner-linux-1","description":"Linux runner"}, + {"id":2,"name":"other-runner","description":"Should be filtered out"} + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + {"id":3,"name":"runner-linux-2","description":"Second Linux runner"} + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + p := schemas.NewProject("group/project") + p.Pull.Runners.Regexp = `^runner-linux-` + p.OutputSparseStatusMetrics = true + + runners, err := c.GetProjectRunners(context.Background(), p) + require.NoError(t, err) + + expected1 := schemas.Runner{ + ProjectName: p.Name, + ID: 1, + Name: "runner-linux-1", + Description: "Linux runner", + OutputSparseStatusMetrics: true, + } + expected2 := schemas.Runner{ + ProjectName: p.Name, + ID: 3, + Name: "runner-linux-2", + Description: "Second Linux runner", + OutputSparseStatusMetrics: true, + } + + require.Len(t, runners, 2) + assert.Contains(t, runners, expected1.Key()) + assert.Contains(t, runners, expected2.Key()) + assert.Equal(t, expected1, runners[expected1.Key()]) + assert.Equal(t, expected2, runners[expected2.Key()]) +} + +func TestGetRunner_FullDetails(t *testing.T) { + c := newTestGitLabClientForRunners(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/runners/123")) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`{ + "id": 123, + "name": "runner-123", + "description": "Main shared runner", + "paused": true, + "is_shared": true, + "runner_type": "instance_type", + "contacted_at": "2024-03-09T16:00:00Z", + "maintenance_note": "maintenance window", + "online": true, + "status": "online", + "token": "tok123", + "tag_list": ["docker", "linux"], + "run_untagged": true, + "locked": false, + "access_level": "not_protected", + "maximum_timeout": 3600, + "groups": [ + { + "id": 10, + "name": "group-a", + "web_url": "https://gitlab.example.com/groups/group-a" + } + ], + "projects": [ + { + "id": 20, + "name": "project-a", + "name_with_namespace": "group-a/project-a", + "path": "project-a", + "path_with_namespace": "group-a/project-a" + } + ] + }`)) + }) + + got, err := c.GetRunner(context.Background(), "group/project", 123) + require.NoError(t, err) + + assert.Equal(t, "group/project", got.ProjectName) + assert.Equal(t, 123, got.ID) + assert.Equal(t, "runner-123", got.Name) + assert.Equal(t, "Main shared runner", got.Description) + assert.True(t, got.Paused) + assert.True(t, got.IsShared) + assert.Equal(t, "instance_type", got.RunnerType) + require.NotNil(t, got.ContactedAt) + assert.Equal(t, int64(1710000000), got.ContactedAt.Unix()) + assert.Equal(t, "maintenance window", got.MaintenanceNote) + assert.True(t, got.Online) + assert.Equal(t, "online", got.Status) + assert.Equal(t, "tok123", got.Token) + assert.Equal(t, []string{"docker", "linux"}, got.TagList) + assert.True(t, got.RunUntagged) + assert.False(t, got.Locked) + assert.Equal(t, "not_protected", got.AccessLevel) + assert.Equal(t, 3600, got.MaximumTimeout) + + require.Len(t, got.Groups, 1) + assert.Equal(t, 10, got.Groups[0].ID) + assert.Equal(t, "group-a", got.Groups[0].Name) + assert.Equal(t, "https://gitlab.example.com/groups/group-a", got.Groups[0].WebURL) + + require.Len(t, got.Projects, 1) + assert.Equal(t, 20, got.Projects[0].ID) + assert.Equal(t, "project-a", got.Projects[0].Name) + assert.Equal(t, "group-a/project-a", got.Projects[0].NameWithNamespace) + assert.Equal(t, "project-a", got.Projects[0].Path) + assert.Equal(t, "group-a/project-a", got.Projects[0].PathWithNamespace) +} + +func TestGetRunner_NoGroups_StillReturnsDetails(t *testing.T) { + c := newTestGitLabClientForRunners(t, func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.Contains(r.URL.Path, "/runners/456")) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`{ + "id": 456, + "name": "runner-456", + "description": "Runner without groups", + "paused": false, + "is_shared": false, + "runner_type": "project_type", + "contacted_at": "2024-03-10T10:00:00Z", + "maintenance_note": "", + "online": false, + "status": "offline", + "token": "tok456", + "tag_list": ["shell"], + "run_untagged": false, + "locked": true, + "access_level": "ref_protected", + "maximum_timeout": 1800, + "groups": null, + "projects": [ + { + "id": 30, + "name": "project-b", + "name_with_namespace": "group-b/project-b", + "path": "project-b", + "path_with_namespace": "group-b/project-b" + } + ] + }`)) + }) + + got, err := c.GetRunner(context.Background(), "group/project", 456) + require.NoError(t, err) + + assert.Equal(t, "group/project", got.ProjectName) + assert.Equal(t, 456, got.ID) + assert.Equal(t, "runner-456", got.Name) + assert.Equal(t, "Runner without groups", got.Description) + assert.False(t, got.Paused) + assert.False(t, got.IsShared) + assert.Equal(t, "project_type", got.RunnerType) + require.NotNil(t, got.ContactedAt) + assert.Equal(t, int64(1710064800), got.ContactedAt.Unix()) + assert.Equal(t, "", got.MaintenanceNote) + assert.False(t, got.Online) + assert.Equal(t, "offline", got.Status) + assert.Equal(t, "tok456", got.Token) + assert.Equal(t, []string{"shell"}, got.TagList) + assert.False(t, got.RunUntagged) + assert.True(t, got.Locked) + assert.Equal(t, "ref_protected", got.AccessLevel) + assert.Equal(t, 1800, got.MaximumTimeout) + + assert.Nil(t, got.Groups) + + require.Len(t, got.Projects, 1) + assert.Equal(t, 30, got.Projects[0].ID) + assert.Equal(t, "project-b", got.Projects[0].Name) + assert.Equal(t, "group-b/project-b", got.Projects[0].NameWithNamespace) + assert.Equal(t, "project-b", got.Projects[0].Path) + assert.Equal(t, "group-b/project-b", got.Projects[0].PathWithNamespace) +} diff --git a/pkg/gitlab/tags_test.go b/pkg/gitlab/tags_test.go new file mode 100644 index 0000000..9157e17 --- /dev/null +++ b/pkg/gitlab/tags_test.go @@ -0,0 +1,187 @@ +package gitlab + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + goGitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/helvethink/gitlab-ci-exporter/pkg/ratelimit" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" +) + +type noopLimiter struct{} + +func (noopLimiter) Take(ctx context.Context) time.Duration { + return 0 +} + +var _ ratelimit.Limiter = noopLimiter{} + +func newTestGitLabClientForTags(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + gl, err := goGitlab.NewClient( + "test-token", + goGitlab.WithBaseURL(server.URL+"/api/v4"), + ) + require.NoError(t, err) + + return &Client{ + Client: gl, + RateLimiter: noopLimiter{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } +} + +func TestGetProjectTags_InvalidRegexp(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiter{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + p := schemas.NewProject("group/project") + p.Pull.Refs.Tags.Regexp = "[" + + refs, err := c.GetProjectTags(context.Background(), p) + + assert.Error(t, err) + assert.Empty(t, refs) +} + +func TestGetProjectTags_FiltersAndPaginates(t *testing.T) { + c := newTestGitLabClientForTags(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/repository/tags") + + page := r.URL.Query().Get("page") + w.Header().Set("Content-Type", "application/json") + + switch page { + case "1", "": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + _, _ = w.Write([]byte(`[ + {"name":"v1.0.0","commit":{"short_id":"abc111","committed_date":"2024-03-09T16:00:00Z"}}, + {"name":"dev-snapshot","commit":{"short_id":"def222","committed_date":"2024-03-08T16:00:00Z"}} + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + {"name":"v2.0.0","commit":{"short_id":"ghi333","committed_date":"2024-03-10T16:00:00Z"}}, + {"name":"test-tag","commit":{"short_id":"jkl444","committed_date":"2024-03-07T16:00:00Z"}} + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + p := schemas.NewProject("group/project") + p.Pull.Refs.Tags.Regexp = `^v` + + refs, err := c.GetProjectTags(context.Background(), p) + require.NoError(t, err) + + expected1 := schemas.NewRef(p, schemas.RefKindTag, "v1.0.0") + expected2 := schemas.NewRef(p, schemas.RefKindTag, "v2.0.0") + + assert.Len(t, refs, 2) + assert.Contains(t, refs, expected1.Key()) + assert.Contains(t, refs, expected2.Key()) + assert.Equal(t, expected1, refs[expected1.Key()]) + assert.Equal(t, expected2, refs[expected2.Key()]) +} + +func TestGetProjectMostRecentTagCommit_InvalidRegexp(t *testing.T) { + c := &Client{ + RateLimiter: noopLimiter{}, + RateCounter: ratecounter.NewRateCounter(time.Second), + } + + sha, ts, err := c.GetProjectMostRecentTagCommit(context.Background(), "group/project", "[") + + assert.Error(t, err) + assert.Equal(t, "", sha) + assert.Equal(t, float64(0), ts) +} + +func TestGetProjectMostRecentTagCommit_ReturnsFirstMatchingTag(t *testing.T) { + c := newTestGitLabClientForTags(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/repository/tags") + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"name":"not-a-release","commit":{"short_id":"zzz999","committed_date":"2024-03-08T16:00:00Z"}}, + {"name":"v2.1.0","commit":{"short_id":"abc123","committed_date":"2024-03-09T16:00:00Z"}}, + {"name":"v2.2.0","commit":{"short_id":"def456","committed_date":"2024-03-10T16:00:00Z"}} + ]`)) + }) + + sha, ts, err := c.GetProjectMostRecentTagCommit(context.Background(), "group/project", `^v`) + require.NoError(t, err) + + assert.Equal(t, "abc123", sha) + assert.Equal(t, float64(1710000000), ts) +} + +func TestGetProjectMostRecentTagCommit_PaginatesUntilMatch(t *testing.T) { + c := newTestGitLabClientForTags(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/repository/tags") + + page := r.URL.Query().Get("page") + w.Header().Set("Content-Type", "application/json") + + switch page { + case "1", "": + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "2") + _, _ = w.Write([]byte(`[ + {"name":"snapshot-1","commit":{"short_id":"aaa111","committed_date":"2024-03-08T16:00:00Z"}} + ]`)) + case "2": + w.Header().Set("X-Page", "2") + w.Header().Set("X-Next-Page", "0") + _, _ = w.Write([]byte(`[ + {"name":"v3.0.0","commit":{"short_id":"bbb222","committed_date":"2024-03-11T16:00:00Z"}} + ]`)) + default: + t.Fatalf("unexpected page: %s", page) + } + }) + + sha, ts, err := c.GetProjectMostRecentTagCommit(context.Background(), "group/project", `^v`) + require.NoError(t, err) + + assert.Equal(t, "bbb222", sha) + assert.Equal(t, float64(1710172800), ts) +} + +func TestGetProjectMostRecentTagCommit_NoMatch(t *testing.T) { + c := newTestGitLabClientForTags(t, func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.URL.Path, "/repository/tags") + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Page", "1") + w.Header().Set("X-Next-Page", "0") + + _, _ = w.Write([]byte(`[ + {"name":"snapshot-1","commit":{"short_id":"aaa111","committed_date":"2024-03-08T16:00:00Z"}}, + {"name":"snapshot-2","commit":{"short_id":"bbb222","committed_date":"2024-03-09T16:00:00Z"}} + ]`)) + }) + + sha, ts, err := c.GetProjectMostRecentTagCommit(context.Background(), "group/project", `^v`) + require.NoError(t, err) + assert.Equal(t, "", sha) + assert.Equal(t, float64(0), ts) +} diff --git a/pkg/gitlab/version_test.go b/pkg/gitlab/version_test.go new file mode 100644 index 0000000..36fb7c2 --- /dev/null +++ b/pkg/gitlab/version_test.go @@ -0,0 +1,102 @@ +package gitlab + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewGitLabVersion(t *testing.T) { + tests := []struct { + name string + input string + expected GitLabVersion + }{ + { + name: "empty version", + input: "", + expected: GitLabVersion{Version: ""}, + }, + { + name: "already prefixed", + input: "v15.9.0", + expected: GitLabVersion{Version: "v15.9.0"}, + }, + { + name: "without prefix", + input: "15.9.0", + expected: GitLabVersion{Version: "v15.9.0"}, + }, + { + name: "major minor only", + input: "15.9", + expected: GitLabVersion{Version: "v15.9"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewGitLabVersion(tt.input) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestPipelineJobsKeysetPaginationSupported(t *testing.T) { + tests := []struct { + name string + version string + expected bool + }{ + { + name: "empty version", + version: "", + expected: false, + }, + { + name: "below minimum version", + version: "v15.8.9", + expected: false, + }, + { + name: "exact minimum version", + version: "v15.9.0", + expected: true, + }, + { + name: "greater patch version", + version: "v15.9.1", + expected: true, + }, + { + name: "greater minor version", + version: "v15.10.0", + expected: true, + }, + { + name: "greater major version", + version: "v16.0.0", + expected: true, + }, + { + name: "without v prefix but normalized first", + version: "15.9.0", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := NewGitLabVersion(tt.version) + assert.Equal(t, tt.expected, v.PipelineJobsKeysetPaginationSupported()) + }) + } +} + +func TestPipelineJobsKeysetPaginationSupported_DirectVersionValue(t *testing.T) { + v := GitLabVersion{Version: "v15.8.0"} + assert.False(t, v.PipelineJobsKeysetPaginationSupported()) + + v = GitLabVersion{Version: "v15.9.0"} + assert.True(t, v.PipelineJobsKeysetPaginationSupported()) +} diff --git a/pkg/monitor/client/client_test.go b/pkg/monitor/client/client_test.go new file mode 100644 index 0000000..d6245e1 --- /dev/null +++ b/pkg/monitor/client/client_test.go @@ -0,0 +1,49 @@ +package client + +import ( + "context" + "net" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + pb "github.com/helvethink/gitlab-ci-exporter/pkg/monitor/protobuf" +) + +type testMonitorServer struct { + pb.UnimplementedMonitorServer +} + +func (testMonitorServer) GetConfig(ctx context.Context, _ *pb.Empty) (*pb.Config, error) { + return &pb.Config{Content: "test-config"}, nil +} + +func TestNewClient(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { + _ = lis.Close() + }) + + grpcServer := grpc.NewServer() + pb.RegisterMonitorServer(grpcServer, testMonitorServer{}) + t.Cleanup(grpcServer.Stop) + + go func() { + _ = grpcServer.Serve(lis) + }() + + endpoint, err := url.Parse("dns:///" + lis.Addr().String()) + require.NoError(t, err) + + c := NewClient(context.Background(), endpoint) + require.NotNil(t, c) + require.NotNil(t, c.MonitorClient) + + cfg, err := c.GetConfig(context.Background(), &pb.Empty{}) + require.NoError(t, err) + assert.Equal(t, "test-config", cfg.GetContent()) +} diff --git a/pkg/monitor/monitor_test.go b/pkg/monitor/monitor_test.go new file mode 100644 index 0000000..5542a4b --- /dev/null +++ b/pkg/monitor/monitor_test.go @@ -0,0 +1,21 @@ +package monitor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTaskSchedulingStatus(t *testing.T) { + last := time.Unix(1710000000, 0) + next := last.Add(5 * time.Minute) + + status := TaskSchedulingStatus{ + Last: last, + Next: next, + } + + assert.Equal(t, last, status.Last) + assert.Equal(t, next, status.Next) +} diff --git a/pkg/monitor/server/server_test.go b/pkg/monitor/server/server_test.go new file mode 100644 index 0000000..9e69d01 --- /dev/null +++ b/pkg/monitor/server/server_test.go @@ -0,0 +1,154 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/paulbellamy/ratecounter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/metadata" + + "github.com/helvethink/gitlab-ci-exporter/pkg/config" + "github.com/helvethink/gitlab-ci-exporter/pkg/gitlab" + "github.com/helvethink/gitlab-ci-exporter/pkg/monitor" + pb "github.com/helvethink/gitlab-ci-exporter/pkg/monitor/protobuf" + "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" + "github.com/helvethink/gitlab-ci-exporter/pkg/store" +) + +type telemetryStreamStub struct { + ctx context.Context + cancel context.CancelFunc + sent []*pb.Telemetry +} + +func (s *telemetryStreamStub) Send(tel *pb.Telemetry) error { + s.sent = append(s.sent, tel) + s.cancel() + return nil +} + +func (s *telemetryStreamStub) SetHeader(metadata.MD) error { return nil } +func (s *telemetryStreamStub) SendHeader(metadata.MD) error { return nil } +func (s *telemetryStreamStub) SetTrailer(metadata.MD) {} +func (s *telemetryStreamStub) Context() context.Context { return s.ctx } +func (s *telemetryStreamStub) SendMsg(interface{}) error { return nil } +func (s *telemetryStreamStub) RecvMsg(interface{}) error { return nil } + +func TestNewServer(t *testing.T) { + cfg := config.New() + st := store.NewLocalStore() + tsm := map[schemas.TaskType]*monitor.TaskSchedulingStatus{} + g := &gitlab.Client{} + + s := NewServer(g, cfg, st, tsm) + + require.NotNil(t, s) + assert.Same(t, g, s.gitlabClient) + assert.Equal(t, cfg, s.cfg) + assert.Same(t, st, s.store) + assert.Equal(t, tsm, s.taskSchedulingMonitoring) +} + +func TestServeWithoutInternalMonitoringAddressReturns(t *testing.T) { + s := NewServer(&gitlab.Client{}, config.New(), store.NewLocalStore(), nil) + s.cfg.Global.InternalMonitoringListenerAddress = nil + + assert.NotPanics(t, func() { + s.Serve() + }) +} + +func TestGetConfig(t *testing.T) { + cfg := config.New() + cfg.Gitlab.Token = "secret-token" + cfg.Server.Webhook.SecretToken = "webhook-secret" + + s := NewServer(&gitlab.Client{}, cfg, store.NewLocalStore(), nil) + + got, err := s.GetConfig(context.Background(), &pb.Empty{}) + require.NoError(t, err) + assert.Contains(t, got.GetContent(), "*******") + assert.NotContains(t, got.GetContent(), "secret-token") + assert.NotContains(t, got.GetContent(), "webhook-secret") +} + +func TestGetTelemetry(t *testing.T) { + ctx := context.Background() + st := store.NewLocalStore() + require.NoError(t, st.SetProject(ctx, schemas.NewProject("group/project"))) + require.NoError(t, st.SetEnvironment(ctx, schemas.Environment{ProjectName: "group/project", Name: "production"})) + require.NoError(t, st.SetRunner(ctx, schemas.Runner{ID: 42, ProjectName: "group/project"})) + require.NoError(t, st.SetRef(ctx, schemas.NewRef(schemas.NewProject("group/project"), schemas.RefKindBranch, "main"))) + require.NoError(t, st.SetMetric(ctx, schemas.Metric{ + Kind: schemas.MetricKindCoverage, + Labels: map[string]string{ + "project": "group/project", + "kind": "branch", + "ref": "main", + "source": "push", + "variables": "", + "pipeline_id": "123", + "status": "success", + }, + Value: 1, + })) + ok, err := st.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-1", "") + require.NoError(t, err) + require.True(t, ok) + require.NoError(t, st.DequeueTask(ctx, schemas.TaskTypePullMetrics, "task-1")) + ok, err = st.QueueTask(ctx, schemas.TaskTypePullMetrics, "task-2", "") + require.NoError(t, err) + require.True(t, ok) + + now := time.Unix(1710000000, 0) + tsm := map[schemas.TaskType]*monitor.TaskSchedulingStatus{ + schemas.TaskTypePullProjectsFromWildcards: {Last: now, Next: now.Add(time.Minute)}, + schemas.TaskTypeGarbageCollectProjects: {Last: now.Add(2 * time.Minute), Next: now.Add(3 * time.Minute)}, + schemas.TaskTypePullEnvironmentsFromProjects: {Last: now.Add(4 * time.Minute), Next: now.Add(5 * time.Minute)}, + schemas.TaskTypeGarbageCollectEnvironments: {Last: now.Add(6 * time.Minute), Next: now.Add(7 * time.Minute)}, + schemas.TaskTypePullRunnersFromProjects: {Last: now.Add(8 * time.Minute), Next: now.Add(9 * time.Minute)}, + schemas.TaskTypeGarbageCollectRunners: {Last: now.Add(10 * time.Minute), Next: now.Add(11 * time.Minute)}, + schemas.TaskTypePullRefsFromProjects: {Last: now.Add(12 * time.Minute), Next: now.Add(13 * time.Minute)}, + schemas.TaskTypeGarbageCollectRefs: {Last: now.Add(14 * time.Minute), Next: now.Add(15 * time.Minute)}, + schemas.TaskTypePullMetrics: {Last: now.Add(16 * time.Minute), Next: now.Add(17 * time.Minute)}, + schemas.TaskTypeGarbageCollectMetrics: {Last: now.Add(18 * time.Minute), Next: now.Add(19 * time.Minute)}, + } + + g := &gitlab.Client{ + RateCounter: ratecounter.NewRateCounter(time.Second), + RequestsRemaining: 5, + RequestsLimit: 10, + } + g.RequestsCounter.Add(7) + g.RateCounter.Incr(2) + + cfg := config.New() + cfg.Gitlab.MaximumRequestsPerSecond = 4 + + s := NewServer(g, cfg, st, tsm) + streamCtx, cancel := context.WithCancel(context.Background()) + stream := &telemetryStreamStub{ctx: streamCtx, cancel: cancel} + + err = s.GetTelemetry(&pb.Empty{}, stream) + require.NoError(t, err) + require.Len(t, stream.sent, 1) + + tel := stream.sent[0] + assert.Equal(t, uint64(7), tel.GetGitlabApiRequestsCount()) + assert.Equal(t, uint64(5), tel.GetGitlabApiLimitRemaining()) + assert.Equal(t, uint64(1), tel.GetTasksExecutedCount()) + assert.InDelta(t, 0.5, tel.GetGitlabApiUsage(), 0.001) + assert.InDelta(t, 0.5, tel.GetGitlabApiRateLimit(), 0.001) + assert.InDelta(t, 0.001, tel.GetTasksBufferUsage(), 0.0001) + assert.Equal(t, int64(1), tel.GetProjects().GetCount()) + assert.Equal(t, int64(1), tel.GetEnvs().GetCount()) + assert.Equal(t, int64(1), tel.GetRefs().GetCount()) + assert.Equal(t, int64(1), tel.GetMetrics().GetCount()) + require.NotNil(t, tel.Runners) + assert.Equal(t, int64(1), tel.Runners.GetCount()) + assert.Equal(t, now.Unix(), tel.GetProjects().GetLastPull().AsTime().Unix()) + assert.Equal(t, now.Add(19*time.Minute).Unix(), tel.GetMetrics().GetNextGc().AsTime().Unix()) +} diff --git a/pkg/monitor/ui/ui_test.go b/pkg/monitor/ui/ui_test.go new file mode 100644 index 0000000..6a08a31 --- /dev/null +++ b/pkg/monitor/ui/ui_test.go @@ -0,0 +1,98 @@ +package ui + +import ( + "testing" + "time" + + "github.com/charmbracelet/bubbles/progress" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/timestamppb" + + pb "github.com/helvethink/gitlab-ci-exporter/pkg/monitor/protobuf" +) + +func TestPrettyTimeago(t *testing.T) { + assert.Equal(t, "N/A", prettyTimeago(time.Time{})) + assert.NotEqual(t, "N/A", prettyTimeago(time.Now().Add(-time.Minute))) +} + +func TestRenderEntity(t *testing.T) { + now := time.Unix(1710000000, 0) + + got := renderEntity("Projects", &pb.Entity{ + Count: 3, + LastPull: timestamppb.New(now), + LastGc: timestamppb.New(now.Add(-time.Minute)), + NextPull: timestamppb.New(now.Add(time.Minute)), + NextGc: timestamppb.New(now.Add(2 * time.Minute)), + }) + + assert.Contains(t, got, "Projects") + assert.Contains(t, got, "Total") + assert.Contains(t, got, "3") + assert.Contains(t, got, "Last Pull") + assert.Contains(t, got, "Next GC") +} + +func TestRenderTelemetryViewportWithoutTelemetry(t *testing.T) { + m := &model{} + + assert.Equal(t, "\nloading data..", m.renderTelemetryViewport()) +} + +func TestRenderTelemetryViewport(t *testing.T) { + p := progress.New(progress.WithScaledGradient("#80c904", "#ff9d5c")) + now := time.Unix(1710000000, 0) + + m := &model{ + progress: &p, + telemetry: &pb.Telemetry{ + GitlabApiUsage: 0.5, + GitlabApiRequestsCount: 7, + GitlabApiRateLimit: 0.25, + GitlabApiLimitRemaining: 5, + TasksBufferUsage: 0.1, + TasksExecutedCount: 9, + Projects: &pb.Entity{ + Count: 1, + LastPull: timestamppb.New(now), + LastGc: timestamppb.New(now), + NextPull: timestamppb.New(now), + NextGc: timestamppb.New(now), + }, + Envs: &pb.Entity{ + Count: 2, + LastPull: timestamppb.New(now), + LastGc: timestamppb.New(now), + NextPull: timestamppb.New(now), + NextGc: timestamppb.New(now), + }, + Refs: &pb.Entity{ + Count: 3, + LastPull: timestamppb.New(now), + LastGc: timestamppb.New(now), + NextPull: timestamppb.New(now), + NextGc: timestamppb.New(now), + }, + Metrics: &pb.Entity{ + Count: 4, + LastPull: timestamppb.New(now), + LastGc: timestamppb.New(now), + NextPull: timestamppb.New(now), + NextGc: timestamppb.New(now), + }, + }, + } + + got := m.renderTelemetryViewport() + + assert.Contains(t, got, "GitLab API usage") + assert.Contains(t, got, "GitLab API requests") + assert.Contains(t, got, "Tasks executed") + assert.Contains(t, got, "Projects") + assert.Contains(t, got, "Environments") + assert.Contains(t, got, "Refs") + assert.Contains(t, got, "Metrics") + assert.Contains(t, got, "7") + assert.Contains(t, got, "9") +} diff --git a/pkg/ratelimit/local_test.go b/pkg/ratelimit/local_test.go new file mode 100644 index 0000000..9507503 --- /dev/null +++ b/pkg/ratelimit/local_test.go @@ -0,0 +1,55 @@ +package ratelimit + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewLocalLimiter(t *testing.T) { + l := NewLocalLimiter(10, 2) + + require.NotNil(t, l) + + start := time.Now() + d := l.Take(context.Background()) + elapsed := time.Since(start) + + assert.GreaterOrEqual(t, d, time.Duration(0)) + assert.Less(t, elapsed, 100*time.Millisecond) +} + +func TestLocalTake_RespectsRateLimit(t *testing.T) { + l := NewLocalLimiter(5, 1) // 1 token every 200ms, burst = 1 + ctx := context.Background() + + d1 := l.Take(ctx) + d2 := l.Take(ctx) + + assert.GreaterOrEqual(t, d1, time.Duration(0)) + assert.Less(t, d1, 50*time.Millisecond) + + assert.GreaterOrEqual(t, d2, 150*time.Millisecond) + assert.Less(t, d2, 1*time.Second) +} + +func TestLocalTake_AllowsBurst(t *testing.T) { + l := NewLocalLimiter(5, 2) // burst = 2 + ctx := context.Background() + + d1 := l.Take(ctx) + d2 := l.Take(ctx) + d3 := l.Take(ctx) + + assert.GreaterOrEqual(t, d1, time.Duration(0)) + assert.Less(t, d1, 50*time.Millisecond) + + assert.GreaterOrEqual(t, d2, time.Duration(0)) + assert.Less(t, d2, 50*time.Millisecond) + + assert.GreaterOrEqual(t, d3, 150*time.Millisecond) + assert.Less(t, d3, 1*time.Second) +} diff --git a/pkg/ratelimit/ratelimit_test.go b/pkg/ratelimit/ratelimit_test.go new file mode 100644 index 0000000..5f95d6a --- /dev/null +++ b/pkg/ratelimit/ratelimit_test.go @@ -0,0 +1,33 @@ +package ratelimit + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type mockLimiter struct { + called bool + ctx context.Context + delay time.Duration +} + +func (m *mockLimiter) Take(ctx context.Context) time.Duration { + m.called = true + m.ctx = ctx + return m.delay +} + +func TestTake(t *testing.T) { + ctx := context.Background() + m := &mockLimiter{ + delay: 10 * time.Millisecond, + } + + Take(ctx, m) + + assert.True(t, m.called) + assert.Equal(t, ctx, m.ctx) +} diff --git a/pkg/ratelimit/redis_test.go b/pkg/ratelimit/redis_test.go new file mode 100644 index 0000000..0414aee --- /dev/null +++ b/pkg/ratelimit/redis_test.go @@ -0,0 +1,65 @@ +package ratelimit + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestRedisLimiter(t *testing.T, maxRPS int) *Redis { + t.Helper() + + mr, err := miniredis.Run() + require.NoError(t, err) + + client := redis.NewClient(&redis.Options{ + Addr: mr.Addr(), + }) + + t.Cleanup(func() { + _ = client.Close() + mr.Close() + }) + + l, ok := NewRedisLimiter(client, maxRPS).(Redis) + require.True(t, ok) + + return &l +} + +func TestNewRedisLimiter(t *testing.T) { + l := newTestRedisLimiter(t, 5) + + require.NotNil(t, l) + require.NotNil(t, l.Limiter) + assert.Equal(t, 5, l.MaxRPS) +} + +func TestRedisTake_FirstCallAllowed(t *testing.T) { + l := newTestRedisLimiter(t, 10) + + d := l.Take(context.Background()) + + assert.GreaterOrEqual(t, d, time.Duration(0)) + assert.Less(t, d, 200*time.Millisecond) +} + +func TestRedisTake_SecondCallIsRateLimited(t *testing.T) { + l := newTestRedisLimiter(t, 1) + + ctx := context.Background() + + d1 := l.Take(ctx) + d2 := l.Take(ctx) + + assert.GreaterOrEqual(t, d1, time.Duration(0)) + assert.Less(t, d1, 200*time.Millisecond) + + assert.GreaterOrEqual(t, d2, 900*time.Millisecond) + assert.Less(t, d2, 2*time.Second) +} diff --git a/pkg/schemas/metric_test.go b/pkg/schemas/metric_test.go index eee8d9e..023651c 100644 --- a/pkg/schemas/metric_test.go +++ b/pkg/schemas/metric_test.go @@ -146,40 +146,14 @@ func TestMetricKey_RunnerMetric(t *testing.T) { m := Metric{ Kind: MetricKindRunner, Labels: prometheus.Labels{ - "project": "group/project", - "kind": "runner", - "runner_id": "12", - "runner_description": "shared-runner", - "runner_groups": `[{"id":1,"name":"group1"}]`, - "runner_projects": `[{"id":2,"name":"project1"}]`, - "runner_maintenance_note": "maintenance", - "contacted_at": "2026-03-24T10:00:00Z", - "paused": "false", - "runner_type": "instance_type", - "tag_list": "docker,linux", - "is_shared": "true", - "active": "true", + "runner_id": "101", }, - Value: 1, } - expectedRaw := strconv.Itoa(int(MetricKindRunner)) + fmt.Sprintf("%v", []string{ - "group/project", - "runner", - "12", - "shared-runner", - `[{"id":1,"name":"group1"}]`, - `[{"id":2,"name":"project1"}]`, - "maintenance", - "2026-03-24T10:00:00Z", - "false", - "instance_type", - "docker,linux", - "true", - "true", - }) + expectedRaw := strconv.Itoa(int(MetricKindRunner)) + fmt.Sprintf("%v", []string{"101"}) + expected := MetricKey(strconv.Itoa(int(crc32.ChecksumIEEE([]byte(expectedRaw))))) - assert.Equal(t, checksumKey(expectedRaw), m.Key()) + assert.Equal(t, expected, m.Key()) } func TestMetricKey_ValueDoesNotAffectKey(t *testing.T) { diff --git a/pkg/schemas/runners_test.go b/pkg/schemas/runners_test.go index 7f5087b..05276fa 100644 --- a/pkg/schemas/runners_test.go +++ b/pkg/schemas/runners_test.go @@ -1,7 +1,6 @@ package schemas import ( - "encoding/json" "hash/crc32" "strconv" "strings" @@ -114,12 +113,6 @@ func TestRunnerInformationLabelsValues(t *testing.T) { got := r.InformationLabelsValues() require.NotNil(t, got) - expectedGroups, err := json.Marshal(r.Groups) - require.NoError(t, err) - - expectedProjects, err := json.Marshal(r.Projects) - require.NoError(t, err) - assert.Equal(t, "group/project", got["project"]) assert.Equal(t, "shared-runner-01", got["runner_description"]) assert.Equal(t, "runner-name", got["runner_name"]) @@ -130,31 +123,36 @@ func TestRunnerInformationLabelsValues(t *testing.T) { assert.Equal(t, strings.Join(r.TagList, ","), got["tag_list"]) assert.Equal(t, "true", got["active"]) // matches current implementation: active = Paused assert.Equal(t, "online", got["status"]) - assert.Equal(t, string(expectedGroups), got["runner_groups"]) - assert.Equal(t, string(expectedProjects), got["runner_projects"]) + assert.Equal(t, "group1", got["runner_groups"]) + assert.Equal(t, "group/project1", got["runner_projects"]) } func TestRunnerInformationLabelsValues_EmptySlices(t *testing.T) { r := Runner{ - ID: 7, - Description: "runner-empty", ProjectName: "group/empty", + Description: "runner-empty", + ID: 7, Name: "runner-empty-name", + Online: false, Status: "offline", + IsShared: false, + Paused: false, TagList: []string{}, Projects: nil, Groups: nil, } got := r.InformationLabelsValues() - require.NotNil(t, got) assert.Equal(t, "group/empty", got["project"]) assert.Equal(t, "runner-empty", got["runner_description"]) assert.Equal(t, "runner-empty-name", got["runner_name"]) assert.Equal(t, "7", got["runner_id"]) + assert.Equal(t, "false", got["is_shared"]) + assert.Equal(t, "false", got["online"]) assert.Equal(t, "", got["tag_list"]) + assert.Equal(t, "false", got["active"]) assert.Equal(t, "offline", got["status"]) - assert.Equal(t, "null", got["runner_groups"]) - assert.Equal(t, "null", got["runner_projects"]) + assert.Equal(t, "", got["runner_groups"]) + assert.Equal(t, "", got["runner_projects"]) } diff --git a/pkg/schemas/tasks_test.go b/pkg/schemas/tasks_test.go new file mode 100644 index 0000000..80f9474 --- /dev/null +++ b/pkg/schemas/tasks_test.go @@ -0,0 +1,71 @@ +package schemas + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTaskTypeValues(t *testing.T) { + assert.Equal(t, TaskType("PullProject"), TaskTypePullProject) + assert.Equal(t, TaskType("PullProjectsFromWildcard"), TaskTypePullProjectsFromWildcard) + assert.Equal(t, TaskType("PullProjectsFromWildcards"), TaskTypePullProjectsFromWildcards) + assert.Equal(t, TaskType("PullEnvironmentsFromProject"), TaskTypePullEnvironmentsFromProject) + assert.Equal(t, TaskType("PullEnvironmentsFromProjects"), TaskTypePullEnvironmentsFromProjects) + assert.Equal(t, TaskType("PullEnvironmentMetrics"), TaskTypePullEnvironmentMetrics) + assert.Equal(t, TaskType("PullMetrics"), TaskTypePullMetrics) + assert.Equal(t, TaskType("PullRefsFromProject"), TaskTypePullRefsFromProject) + assert.Equal(t, TaskType("PullRefsFromProjects"), TaskTypePullRefsFromProjects) + assert.Equal(t, TaskType("PullRefMetrics"), TaskTypePullRefMetrics) + assert.Equal(t, TaskType("PullRunnerFromProject"), TaskTypePullRunnersFromProject) + assert.Equal(t, TaskType("PullRunnersFromProjects"), TaskTypePullRunnersFromProjects) + assert.Equal(t, TaskType("PullRunnersMetrics"), TaskTypePullRunnersMetrics) + assert.Equal(t, TaskType("GarbageCollectProjects"), TaskTypeGarbageCollectProjects) + assert.Equal(t, TaskType("GarbageCollectEnvironments"), TaskTypeGarbageCollectEnvironments) + assert.Equal(t, TaskType("GarbageCollectRefs"), TaskTypeGarbageCollectRefs) + assert.Equal(t, TaskType("GarbageCollectMetrics"), TaskTypeGarbageCollectMetrics) + assert.Equal(t, TaskType("GarbageCollectRunners"), TaskTypeGarbageCollectRunners) +} + +func TestTasksMapUsage(t *testing.T) { + tasks := Tasks{ + TaskTypePullMetrics: { + "task-1": nil, + "task-2": nil, + }, + TaskTypeGarbageCollectEnvironments: { + "task-3": nil, + }, + } + + assert.Len(t, tasks, 2) + assert.Contains(t, tasks, TaskTypePullMetrics) + assert.Contains(t, tasks, TaskTypeGarbageCollectEnvironments) + + assert.Len(t, tasks[TaskTypePullMetrics], 2) + assert.Len(t, tasks[TaskTypeGarbageCollectEnvironments], 1) + + _, ok := tasks[TaskTypePullMetrics]["task-1"] + assert.True(t, ok) + + _, ok = tasks[TaskTypeGarbageCollectEnvironments]["task-3"] + assert.True(t, ok) +} + +func TestTasksMapInitialization(t *testing.T) { + var tasks Tasks + assert.Nil(t, tasks) + + tasks = make(Tasks) + assert.NotNil(t, tasks) + assert.Len(t, tasks, 0) + + tasks[TaskTypePullProject] = make(map[string]interface{}) + tasks[TaskTypePullProject]["project-1"] = nil + + assert.Len(t, tasks, 1) + assert.Len(t, tasks[TaskTypePullProject], 1) + + _, ok := tasks[TaskTypePullProject]["project-1"] + assert.True(t, ok) +} diff --git a/pkg/store/redis_test.go b/pkg/store/redis_test.go index 5b8db41..eec4186 100644 --- a/pkg/store/redis_test.go +++ b/pkg/store/redis_test.go @@ -1,6 +1,7 @@ package store import ( + "context" "testing" "time" @@ -13,6 +14,8 @@ import ( "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" ) +var testCtx = context.Background() + func newTestRedisStore(t *testing.T) (mr *miniredis.Miniredis, r *Redis) { mr, err := miniredis.Run() if err != nil { diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 8ed7e07..cc89c0f 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -3,58 +3,170 @@ package store import ( "context" "testing" + "time" + "github.com/alicebob/miniredis/v2" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/helvethink/gitlab-ci-exporter/pkg/config" "github.com/helvethink/gitlab-ci-exporter/pkg/schemas" ) -var testCtx = context.Background() +func newStoreGoTestRedis(t *testing.T) (*miniredis.Miniredis, *Redis) { + t.Helper() -func TestNewLocalStore(t *testing.T) { - expectedValue := &Local{ - projects: make(schemas.Projects), - environments: make(schemas.Environments), - refs: make(schemas.Refs), - runners: make(schemas.Runners), - metrics: make(schemas.Metrics), - pipelines: make(schemas.Pipelines), - pipelineVariables: make(map[schemas.PipelineKey]string), + mr, err := miniredis.Run() + require.NoError(t, err) + + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + + t.Cleanup(func() { + _ = client.Close() + mr.Close() + }) + + return mr, NewRedisStore(client) +} + +func testStoreProjects() config.Projects { + return config.Projects{ + config.NewProject("group/project1"), + config.NewProject("group/project2"), } - assert.Equal(t, expectedValue, NewLocalStore()) +} + +func TestNewLocalStore(t *testing.T) { + s := NewLocalStore() + + l, ok := s.(*Local) + require.True(t, ok) + + assert.NotNil(t, l.projects) + assert.NotNil(t, l.environments) + assert.NotNil(t, l.runners) + assert.NotNil(t, l.refs) + assert.NotNil(t, l.metrics) + assert.NotNil(t, l.pipelines) + assert.NotNil(t, l.pipelineVariables) + + assert.Len(t, l.projects, 0) + assert.Len(t, l.environments, 0) + assert.Len(t, l.runners, 0) + assert.Len(t, l.refs, 0) + assert.Len(t, l.metrics, 0) + assert.Len(t, l.pipelines, 0) + assert.Len(t, l.pipelineVariables, 0) } func TestNewRedisStore(t *testing.T) { - redisClient := redis.NewClient(&redis.Options{}) - redisStore := NewRedisStore(redisClient) - - assert.IsType(t, &Redis{}, redisStore) - assert.Equal(t, redisClient, redisStore.Client) - assert.NotNil(t, redisStore.StoreConfig) // since constructor sets it -} - -func TestNew(t *testing.T) { - localStore := New(testCtx, nil, config.Projects{}) - assert.IsType(t, &Local{}, localStore) - - redisClient := redis.NewClient(&redis.Options{}) - redisStore := NewRedisStore(redisClient) - store := New(testCtx, redisStore, config.Projects{}) - assert.IsType(t, &Redis{}, store) - - localStore = New(testCtx, nil, config.Projects{ - { - Name: "foo", - }, - { - Name: "foo", - }, - { - Name: "bar", - }, + client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"}) + t.Cleanup(func() { + _ = client.Close() }) - count, _ := localStore.ProjectsCount(testCtx) + + r := NewRedisStore(client) + + require.NotNil(t, r) + assert.Same(t, client, r.Client) + require.NotNil(t, r.StoreConfig) + assert.Nil(t, r.StoreConfig.TTLConfig) +} + +func TestNewRedisStore_WithTTLConfig(t *testing.T) { + client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"}) + t.Cleanup(func() { + _ = client.Close() + }) + + ttl := &RedisTTLConfig{ + Project: time.Minute, + Environment: 2 * time.Minute, + Runner: 3 * time.Minute, + Refs: 4 * time.Minute, + Metrics: 5 * time.Minute, + } + + r := NewRedisStore(client, WithTTLConfig(ttl)) + + require.NotNil(t, r) + require.NotNil(t, r.StoreConfig) + require.NotNil(t, r.StoreConfig.TTLConfig) + + assert.Equal(t, time.Minute, r.StoreConfig.TTLConfig.Project) + assert.Equal(t, 2*time.Minute, r.StoreConfig.TTLConfig.Environment) + assert.Equal(t, 3*time.Minute, r.StoreConfig.TTLConfig.Runner) + assert.Equal(t, 4*time.Minute, r.StoreConfig.TTLConfig.Refs) + assert.Equal(t, 5*time.Minute, r.StoreConfig.TTLConfig.Metrics) +} + +func TestNew_WithNilRedis_UsesLocalStoreAndLoadsProjects(t *testing.T) { + ctx := context.Background() + projects := testStoreProjects() + + s := New(ctx, nil, projects) + + _, ok := s.(*Local) + require.True(t, ok) + + count, err := s.ProjectsCount(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2), count) + + for _, p := range projects { + project := schemas.Project{Project: p} + + exists, err := s.ProjectExists(ctx, project.Key()) + require.NoError(t, err) + assert.True(t, exists) + } +} + +func TestNew_WithRedis_UsesRedisStoreAndLoadsProjects(t *testing.T) { + ctx := context.Background() + _, r := newStoreGoTestRedis(t) + projects := testStoreProjects() + + s := New(ctx, r, projects) + + rr, ok := s.(*Redis) + require.True(t, ok) + assert.Same(t, r, rr) + + count, err := s.ProjectsCount(ctx) + require.NoError(t, err) assert.Equal(t, int64(2), count) + + for _, p := range projects { + project := schemas.Project{Project: p} + + exists, err := s.ProjectExists(ctx, project.Key()) + require.NoError(t, err) + assert.True(t, exists) + } +} + +func TestNew_DoesNotOverwriteExistingProjectInRedis(t *testing.T) { + ctx := context.Background() + _, r := newStoreGoTestRedis(t) + + existing := schemas.NewProject("group/project1") + existing.Topics = "keep-me" + require.NoError(t, r.SetProject(ctx, existing)) + + projects := config.Projects{ + config.NewProject("group/project1"), + } + + s := New(ctx, r, projects) + + got := schemas.NewProject("group/project1") + require.NoError(t, s.GetProject(ctx, &got)) + + assert.Equal(t, "keep-me", got.Topics) + + count, err := s.ProjectsCount(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), count) }