Skip to content

Commit 6699550

Browse files
dzemanovdzemanov
authored
fix(scorecard): Handle custom thresholds from scorecard backend (#3293)
* Move handling of custom thresholds to scorecard backend Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Unify default thresholds order for openssf with all other providers Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Update docs Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Add changeset Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Load configured thresholds on startup Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Fix tests Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Validate default provider thresholds on registration Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Uodate docs Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Update threshold docs Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Update tests and docs Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Throw ThresholdConfigFormatError instead of general error Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> * Update docs Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com> --------- Signed-off-by: Dominika Zemanovicova <dzemanov@redhat.com>
1 parent 0239de6 commit 6699550

38 files changed

Lines changed: 588 additions & 429 deletions

‎workspaces/scorecard/.changeset/tired-hoops-happen.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-dependabot': patch
3+
'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-filecheck': patch
4+
'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-sonarqube': patch
5+
'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-openssf': patch
6+
'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-github': patch
7+
'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-jira': patch
8+
'@red-hat-developer-hub/backstage-plugin-scorecard-backend': patch
9+
'@red-hat-developer-hub/backstage-plugin-scorecard-node': patch
10+
---
11+
12+
Custom thresholds for filecheck, openssf, and dependabot are now
13+
configurable. Custom threshold handling has been centralized in
14+
`scorecard-backend`, you can define custom thresholds under
15+
`scorecard.plugins.<providerId>.thresholds`. Provider IDs typically
16+
follow the format `<datasource>.<metric>`.

‎workspaces/scorecard/plugins/scorecard-backend-module-dependabot/src/metricProviders/DependabotMetricProvider.test.ts‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ import { ConfigReader } from '@backstage/config';
1818
import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client';
1919
import type { Entity } from '@backstage/catalog-model';
2020
import { DependabotMetricProvider } from './DependabotMetricProvider';
21-
import { DEPENDABOT_SEVERITY_METRIC } from './DependabotConfig';
21+
import {
22+
DEPENDABOT_SEVERITY_METRIC,
23+
DEPENDABOT_THRESHOLDS,
24+
} from './DependabotConfig';
2225
import { mockServices } from '@backstage/backend-test-utils';
2326

2427
jest.mock('@backstage/catalog-model', () => ({
@@ -105,25 +108,13 @@ describe('DependabotMetricProvider', () => {
105108
});
106109

107110
describe('getMetricThresholds', () => {
108-
it('returns default thresholds when none provided', () => {
111+
it('returns default thresholds', () => {
109112
const provider = new DependabotMetricProvider(
110113
mockConfig,
111114
mockLogger,
112115
'critical',
113116
);
114-
expect(provider.getMetricThresholds()).toBeDefined();
115-
expect(provider.getMetricThresholds().rules).toBeDefined();
116-
});
117-
118-
it('returns custom thresholds when provided', () => {
119-
const custom = { rules: [{ key: 'ok', expression: '<1' }] };
120-
const provider = new DependabotMetricProvider(
121-
mockConfig,
122-
mockLogger,
123-
'critical',
124-
custom,
125-
);
126-
expect(provider.getMetricThresholds()).toEqual(custom);
117+
expect(provider.getMetricThresholds()).toEqual(DEPENDABOT_THRESHOLDS);
127118
});
128119
});
129120

‎workspaces/scorecard/plugins/scorecard-backend-module-dependabot/src/metricProviders/DependabotMetricProvider.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,18 +44,15 @@ const GITHUB_PROJECT_ANNOTATION = 'github.com/project-slug';
4444
*/
4545
export class DependabotMetricProvider implements MetricProvider<'number'> {
4646
private readonly dependabotClient: DependabotClient;
47-
private readonly thresholds: ThresholdConfig;
4847
private readonly severity: DependabotSeverity;
4948

5049
constructor(
5150
config: Config,
5251
logger: LoggerService,
5352
severity: DependabotSeverity,
54-
thresholds?: ThresholdConfig,
5553
) {
5654
this.severity = severity;
5755
this.dependabotClient = new DependabotClient(config, logger);
58-
this.thresholds = thresholds ?? DEPENDABOT_THRESHOLDS;
5956
}
6057

6158
getProviderDatasourceId(): string {
@@ -82,7 +79,7 @@ export class DependabotMetricProvider implements MetricProvider<'number'> {
8279
}
8380

8481
getMetricThresholds(): ThresholdConfig {
85-
return this.thresholds;
82+
return DEPENDABOT_THRESHOLDS;
8683
}
8784

8885
getCatalogFilter(): Record<string, string | symbol | (string | symbol)[]> {

‎workspaces/scorecard/plugins/scorecard-backend-module-dependabot/src/metricProviders/DependabotMetricProviderFactory.test.ts‎

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
createDependabotMetricProviders,
2121
} from './DependabotMetricProviderFactory';
2222
import { mockServices } from '@backstage/backend-test-utils';
23+
import { DEPENDABOT_THRESHOLDS } from './DependabotConfig';
2324

2425
const mockConfig = new ConfigReader({
2526
integrations: { github: [{ host: 'github.com', token: 'test-token' }] },
@@ -36,17 +37,7 @@ describe('createDependabotMetricProvider', () => {
3637
expect(provider.getProviderId()).toBe('dependabot.alerts_high');
3738
expect(provider.getProviderDatasourceId()).toBe('dependabot');
3839
expect(provider.getMetricType()).toBe('number');
39-
});
40-
41-
it('accepts optional thresholds', () => {
42-
const thresholds = { rules: [{ key: 'ok', expression: '<1' }] };
43-
const provider = createDependabotMetricProvider(
44-
mockConfig,
45-
mockLogger,
46-
'critical',
47-
thresholds,
48-
);
49-
expect(provider.getMetricThresholds()).toEqual(thresholds);
40+
expect(provider.getMetricThresholds()).toBe(DEPENDABOT_THRESHOLDS);
5041
});
5142
});
5243

@@ -61,16 +52,4 @@ describe('createDependabotMetricProviders', () => {
6152
'dependabot.alerts_low',
6253
]);
6354
});
64-
65-
it('passes optional thresholds to all providers', () => {
66-
const thresholds = { rules: [{ key: 'custom', expression: '>0' }] };
67-
const providers = createDependabotMetricProviders(
68-
mockConfig,
69-
mockLogger,
70-
thresholds,
71-
);
72-
providers.forEach(p => {
73-
expect(p.getMetricThresholds()).toEqual(thresholds);
74-
});
75-
});
7655
});

‎workspaces/scorecard/plugins/scorecard-backend-module-dependabot/src/metricProviders/DependabotMetricProviderFactory.ts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import { DependabotMetricProvider } from './DependabotMetricProvider';
1717
import { DependabotSeverity, DEPENDABOT_SEVERITIES } from './DependabotConfig';
1818
import { Config } from '@backstage/config';
1919
import { LoggerService } from '@backstage/backend-plugin-api';
20-
import { ThresholdConfig } from '@red-hat-developer-hub/backstage-plugin-scorecard-common';
2120
import { MetricProvider } from '@red-hat-developer-hub/backstage-plugin-scorecard-node';
2221

2322
/**
@@ -27,9 +26,8 @@ export function createDependabotMetricProvider(
2726
config: Config,
2827
logger: LoggerService,
2928
severity: DependabotSeverity,
30-
thresholds?: ThresholdConfig,
3129
): MetricProvider<'number'> {
32-
return new DependabotMetricProvider(config, logger, severity, thresholds);
30+
return new DependabotMetricProvider(config, logger, severity);
3331
}
3432

3533
/**
@@ -38,9 +36,8 @@ export function createDependabotMetricProvider(
3836
export function createDependabotMetricProviders(
3937
config: Config,
4038
logger: LoggerService,
41-
thresholds?: ThresholdConfig,
4239
): MetricProvider<'number'>[] {
4340
return DEPENDABOT_SEVERITIES.map(severity =>
44-
createDependabotMetricProvider(config, logger, severity, thresholds),
41+
createDependabotMetricProvider(config, logger, severity),
4542
);
4643
}

‎workspaces/scorecard/plugins/scorecard-backend-module-filecheck/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,25 @@ Each configured file produces one boolean metric.
104104

105105
You can override the default thresholds via `app-config.yaml`. Check out the detailed explanation of [threshold configuration](../scorecard-backend/docs/thresholds.md).
106106

107+
Example configuration:
108+
109+
```yaml
110+
# app-config.yaml
111+
scorecard:
112+
plugins:
113+
filecheck:
114+
thresholds:
115+
rules:
116+
- key: present
117+
expression: '==true'
118+
icon: scorecardSuccessStatusIcon
119+
color: 'success.main'
120+
- key: absent
121+
expression: '==false'
122+
icon: scorecardErrorStatusIcon
123+
color: 'error.main'
124+
```
125+
107126
## Schedule Configuration
108127

109128
The Scorecard plugin uses Backstage's built-in scheduler service to automatically collect metrics from all registered providers every hour by default. You can change this schedule in the `app-config.yaml` file:

‎workspaces/scorecard/plugins/scorecard-backend-module-filecheck/src/metricProviders/FilecheckMetricProvider.ts‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,10 @@ import {
3030
export class FilecheckMetricProvider implements MetricProvider<'boolean'> {
3131
private readonly client: FilecheckClient;
3232
private readonly filesConfig: FilecheckConfig;
33-
private readonly thresholds: ThresholdConfig;
3433

35-
constructor(
36-
client: FilecheckClient,
37-
filesConfig: FilecheckConfig,
38-
thresholds?: ThresholdConfig,
39-
) {
34+
constructor(client: FilecheckClient, filesConfig: FilecheckConfig) {
4035
this.client = client;
4136
this.filesConfig = filesConfig;
42-
this.thresholds = thresholds ?? DEFAULT_FILECHECK_THRESHOLDS;
4337
}
4438

4539
getProviderDatasourceId(): string {
@@ -73,7 +67,7 @@ export class FilecheckMetricProvider implements MetricProvider<'boolean'> {
7367
}
7468

7569
getMetricThresholds(): ThresholdConfig {
76-
return this.thresholds;
70+
return DEFAULT_FILECHECK_THRESHOLDS;
7771
}
7872

7973
getCatalogFilter(): Record<string, string | symbol | (string | symbol)[]> {

‎workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubOpenPRsProvider.test.ts‎

Lines changed: 1 addition & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -31,56 +31,11 @@ jest.mock('../github/GithubClient');
3131

3232
describe('GithubOpenPRsProvider', () => {
3333
describe('fromConfig', () => {
34-
it('should create provider with default thresholds when no thresholds are configured', () => {
34+
it('should create provider with default thresholds', () => {
3535
const provider = GithubOpenPRsProvider.fromConfig(new ConfigReader({}));
3636

3737
expect(provider.getMetricThresholds()).toEqual(DEFAULT_NUMBER_THRESHOLDS);
3838
});
39-
40-
it('should create provider with custom thresholds when configured', () => {
41-
const customThresholds = {
42-
rules: [
43-
{ key: 'error', expression: '>100' },
44-
{ key: 'warning', expression: '50-100' },
45-
{ key: 'success', expression: '<50' },
46-
],
47-
};
48-
49-
const configWithThresholds = new ConfigReader({
50-
scorecard: {
51-
plugins: {
52-
github: {
53-
open_prs: {
54-
thresholds: customThresholds,
55-
},
56-
},
57-
},
58-
},
59-
});
60-
const provider = GithubOpenPRsProvider.fromConfig(configWithThresholds);
61-
62-
expect(provider.getMetricThresholds()).toEqual(customThresholds);
63-
});
64-
65-
it('should throw error when invalid custom thresholds', () => {
66-
const invalidConfig = new ConfigReader({
67-
scorecard: {
68-
plugins: {
69-
github: {
70-
open_prs: {
71-
thresholds: {
72-
rules: [{ key: 'error', expression: '>!100' }],
73-
},
74-
},
75-
},
76-
},
77-
},
78-
});
79-
80-
expect(() => GithubOpenPRsProvider.fromConfig(invalidConfig)).toThrow(
81-
'Cannot parse "!100" as number from expression: ">!100"',
82-
);
83-
});
8439
});
8540

8641
describe('calculateMetric', () => {

‎workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubOpenPRsProvider.ts‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,15 @@ import {
2222
Metric,
2323
ThresholdConfig,
2424
} from '@red-hat-developer-hub/backstage-plugin-scorecard-common';
25-
import {
26-
getThresholdsFromConfig,
27-
MetricProvider,
28-
} from '@red-hat-developer-hub/backstage-plugin-scorecard-node';
25+
import { MetricProvider } from '@red-hat-developer-hub/backstage-plugin-scorecard-node';
2926
import { GithubClient } from '../github/GithubClient';
3027
import { getRepositoryInformationFromEntity } from '../github/utils';
3128

3229
export class GithubOpenPRsProvider implements MetricProvider<'number'> {
3330
private readonly githubClient: GithubClient;
34-
private readonly thresholds: ThresholdConfig;
3531

36-
private constructor(config: Config, thresholds?: ThresholdConfig) {
32+
private constructor(config: Config) {
3733
this.githubClient = new GithubClient(config);
38-
this.thresholds = thresholds ?? DEFAULT_NUMBER_THRESHOLDS;
3934
}
4035

4136
getProviderDatasourceId(): string {
@@ -62,7 +57,7 @@ export class GithubOpenPRsProvider implements MetricProvider<'number'> {
6257
}
6358

6459
getMetricThresholds(): ThresholdConfig {
65-
return this.thresholds;
60+
return DEFAULT_NUMBER_THRESHOLDS;
6661
}
6762

6863
getCatalogFilter(): Record<string, string | symbol | (string | symbol)[]> {
@@ -72,13 +67,7 @@ export class GithubOpenPRsProvider implements MetricProvider<'number'> {
7267
}
7368

7469
static fromConfig(config: Config): GithubOpenPRsProvider {
75-
const thresholds = getThresholdsFromConfig(
76-
config,
77-
'scorecard.plugins.github.open_prs.thresholds',
78-
'number',
79-
);
80-
81-
return new GithubOpenPRsProvider(config, thresholds);
70+
return new GithubOpenPRsProvider(config);
8271
}
8372

8473
async calculateMetric(entity: Entity): Promise<number> {

‎workspaces/scorecard/plugins/scorecard-backend-module-jira/src/metricProviders/JiraOpenIssuesProvider.test.ts‎

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,13 @@ import type { Entity } from '@backstage/catalog-model';
1919
import {
2020
DEFAULT_NUMBER_THRESHOLDS,
2121
Metric,
22-
ThresholdConfig,
2322
} from '@red-hat-developer-hub/backstage-plugin-scorecard-common';
2423
import { JiraOpenIssuesProvider } from './JiraOpenIssuesProvider';
2524
import { JiraClientFactory } from '../clients/JiraClientFactory';
2625
import { JiraClient } from '../clients/base';
2726
import { mockServices } from '@backstage/backend-test-utils';
2827
import {
2928
newEntityComponent,
30-
newThresholdsConfig,
3129
newMockRootConfig,
3230
} from '../../__fixtures__/testUtils';
3331
import { ScorecardJiraAnnotations } from '../annotations';
@@ -61,8 +59,6 @@ const mockEntity: Entity = newEntityComponent({
6159
[PROJECT_KEY]: 'TEST',
6260
});
6361

64-
const customThresholds: ThresholdConfig = newThresholdsConfig();
65-
6662
const mockAuthOptions = {
6763
discovery: mockServices.discovery(),
6864
auth: mockServices.auth(),
@@ -142,23 +138,13 @@ describe('JiraOpenIssuesProvider', () => {
142138
});
143139

144140
describe('getMetricThresholds', () => {
145-
it('should return default config when no thresholds are configured', () => {
141+
it('should return default provider thresholds', () => {
146142
const provider = JiraOpenIssuesProvider.fromConfig(
147143
mockConfig,
148144
mockAuthOptions,
149145
);
150146
expect(provider.getMetricThresholds()).toEqual(DEFAULT_NUMBER_THRESHOLDS);
151147
});
152-
153-
it('should return custom config when thresholds are configured', () => {
154-
const config = newMockRootConfig({ thresholds: customThresholds });
155-
156-
const provider = JiraOpenIssuesProvider.fromConfig(
157-
config,
158-
mockAuthOptions,
159-
);
160-
expect(provider.getMetricThresholds()).toEqual(customThresholds);
161-
});
162148
});
163149

164150
describe('supportsEntity', () => {
@@ -191,27 +177,6 @@ describe('JiraOpenIssuesProvider', () => {
191177
expect(provider.getMetricThresholds()).toEqual(DEFAULT_NUMBER_THRESHOLDS);
192178
});
193179

194-
it('should create provider with custom config when thresholds are configured', () => {
195-
const config = newMockRootConfig({ thresholds: customThresholds });
196-
197-
const provider = JiraOpenIssuesProvider.fromConfig(
198-
config,
199-
mockAuthOptions,
200-
);
201-
expect(provider.getMetricThresholds()).toEqual(customThresholds);
202-
});
203-
204-
it('should throw an error when invalid thresholds are configured', () => {
205-
const invalidThresholds = {
206-
rules: [{ key: 'invalid', expression: 'bad' }],
207-
};
208-
const config = newMockRootConfig({ thresholds: invalidThresholds });
209-
210-
expect(() =>
211-
JiraOpenIssuesProvider.fromConfig(config, mockAuthOptions),
212-
).toThrow('Invalid thresholds');
213-
});
214-
215180
it('should create provider with proxy connection strategy when proxy path is configured', () => {
216181
JiraOpenIssuesProvider.fromConfig(mockConfig, mockAuthOptions);
217182
expect(mockedProxyConnectionStrategy).toHaveBeenCalledWith(

0 commit comments

Comments
 (0)