Skip to content

Commit 607e6f4

Browse files
Merge remote-tracking branch 'origin/improvement/ZENKO-5202/usage-reporting-route' into improvement/ZENKO-5210
2 parents dc663bb + 0284ea2 commit 607e6f4

File tree

10 files changed

+214
-32
lines changed

10 files changed

+214
-32
lines changed

solution/deps.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ mongodb-connector:
8080
pensieve-api:
8181
sourceRegistry: ghcr.io/scality
8282
image: pensieve-api
83-
tag: 1.9.0
83+
tag: 1.9.1
8484
envsubst: PENSIEVE_API_TAG
8585
rclone:
8686
sourceRegistry: rclone

tests/ctst/common/common.ts

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,10 @@ async function getTopicsOffsets(topics: string[], kafkaAdmin: Admin) {
106106
return offsets;
107107
}
108108

109-
Given('an account', async function (this: Zenko) {
110-
await this.createAccount();
109+
Given('{int} additional accounts', async function (this: Zenko, count: number) {
110+
for (let i = 0; i < count; i++) {
111+
await this.createAccount();
112+
}
111113
});
112114

113115
async function createBucket(world: Zenko, versioning: string, bucketName: string) {
@@ -425,6 +427,12 @@ Then('the API should {string} with {string}', function (this: Zenko, result: str
425427
}
426428
});
427429

430+
Then('the http response code is {int}', function (this: Zenko, expectedStatus: number) {
431+
const response = this.getSaved<{ statusCode: number }>('lastHttpResponse');
432+
assert.strictEqual(response.statusCode, expectedStatus,
433+
`Expected status ${expectedStatus} but got ${response.statusCode}`);
434+
});
435+
428436
Then('the operation finished without error', function (this: Zenko) {
429437
this.useSavedIdentity();
430438
assert.strictEqual(!!this.getResult().err, false);

tests/ctst/common/hooks.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import Zenko from '../world/Zenko';
99
import { CacheHelper, Identity } from 'cli-testing';
1010
import { prepareQuotaScenarios, teardownQuotaScenarios } from 'steps/quotas/quotas';
1111
import { prepareUtilizationScenarios } from 'steps/utilization/utilizationAPI';
12+
import { prepareMetricsScenarios } from './utils';
1213
import { cleanS3Bucket } from './common';
1314
import { cleanAzureContainer, cleanZenkoLocation } from 'steps/azureArchive';
1415
import { displayDebuggingInformation, preparePRA } from 'steps/pra';
@@ -60,6 +61,16 @@ Before({ tags: '@UtilizationAPI', timeout: 1200000 }, async function (scenarioOp
6061
await prepareUtilizationScenarios(this as Zenko, scenarioOptions);
6162
});
6263

64+
Before({ tags: '@PrepareStorageUsageReportingScenarios', timeout: 1200000 }, async function (scenarioOptions) {
65+
await prepareMetricsScenarios(this as Zenko, scenarioOptions, {
66+
versioning: '',
67+
jobNamespace: 'storage-usage-reporting-setup',
68+
jobName: 'end2end-ops-count-items',
69+
objectSize: 200,
70+
objectCount: 3,
71+
});
72+
});
73+
6374
After(async function (this: Zenko, results) {
6475
// Reset any configuration set on the endpoint (ssl, port)
6576
CacheHelper.parameters.ssl = this.parameters.ssl;

tests/ctst/common/utils.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,10 +304,12 @@ export async function cleanupAccount(world: Zenko, accountName: string) {
304304
}
305305
}
306306

307-
interface PrepareScenarioOptions {
307+
export interface PrepareScenarioOptions {
308308
versioning?: string;
309309
jobNamespace?: string;
310310
jobName?: string;
311+
objectSize?: number;
312+
objectCount?: number;
311313
}
312314

313315
/**
@@ -330,7 +332,9 @@ export async function prepareMetricsScenarios(
330332
const {
331333
versioning = '',
332334
jobName = 'end2end-ops-count-items',
333-
jobNamespace = `${featureName}-setup`
335+
jobNamespace = `${featureName}-setup`,
336+
objectSize = 0,
337+
objectCount = 1,
334338
} = options;
335339

336340
if (!fs.existsSync(filePath)) {
@@ -365,7 +369,9 @@ export async function prepareMetricsScenarios(
365369
for (const scenarioId of scenarioIds) {
366370
await world.createAccount(scenarioId, true);
367371
await createBucketWithConfiguration(world, scenarioId, versioning);
368-
await putObject(world);
372+
for (let i = 0; i < objectCount; i++) {
373+
await putObject(world, undefined, undefined, objectSize);
374+
}
369375
output[scenarioId] = Identity.getCurrentCredentials()!;
370376
}
371377

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
Feature: Storage Usage Reporting API
2+
The storage usage reporting API allows authorized Keycloak users to retrieve
3+
aggregated storage usage metrics across all accounts and locations.
4+
5+
@2.14.0
6+
@PreMerge
7+
@StorageUsageReporting
8+
Scenario Outline: Storage usage report access control per persona
9+
Given an identity with the "<persona>" keycloak persona
10+
When the user tries to retrieve the storage usage report
11+
Then the http response code is <expectedStatus>
12+
13+
Examples:
14+
| persona | expectedStatus |
15+
| storage_manager | 200 |
16+
| data_consumer | 403 |
17+
18+
@2.14.0
19+
@PreMerge
20+
@StorageUsageReporting
21+
Scenario: Storage usage report has a valid structure
22+
Given an identity with the "storage_manager" keycloak persona
23+
When the user retrieves the storage usage report
24+
Then the storage usage report response has a valid structure
25+
26+
@2.14.0
27+
@PreMerge
28+
@StorageUsageReporting
29+
Scenario: Storage usage report contains multiple accounts
30+
Given an identity with the "storage_manager" keycloak persona
31+
And 2 additional accounts
32+
When the user retrieves the storage usage report
33+
Then the storage usage report contains the additional accounts
34+
35+
@2.14.0
36+
@PreMerge
37+
@StorageUsageReporting
38+
@PrepareStorageUsageReportingScenarios
39+
Scenario Outline: Storage usage report returns accurate metrics
40+
Given the environment is set up with bucket created, test data uploaded, and count-items ran
41+
And an identity with the "storage_manager" keycloak persona
42+
When the user retrieves the storage usage report
43+
Then the report contains the test account with location "<locationName>"
44+
And the report shows 3 objects and 600 bytes
45+
46+
Examples:
47+
| locationName |
48+
| us-east-1 |

tests/ctst/steps/bucket-policies/common.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -152,14 +152,14 @@ Given('an {string} IAM Policy that {string} with {string} effect for the current
152152
|| identityType === EntityType.DATA_CONSUMER) {
153153
const result = await IAM.attachRolePolicy({
154154
policyArn,
155-
roleName: this.getSaved<string>('identityNameForScenario'),
155+
roleName: this.getSavedIdentity().identityName,
156156
});
157157
assert.ifError(result.stderr || result.err);
158158
}
159159
if (identityType === EntityType.IAM_USER) {
160160
const result = await IAM.attachUserPolicy({
161161
policyArn,
162-
userName: this.getSaved<string>('identityNameForScenario'),
162+
userName: this.getSavedIdentity().identityName,
163163
});
164164
assert.ifError(result.stderr || result.err);
165165
}
@@ -361,13 +361,13 @@ Given('an environment setup for the API', async function (this: Zenko) {
361361
|| identityType === EntityType.DATA_CONSUMER) {
362362
const result = await IAM.attachRolePolicy({
363363
policyArn,
364-
roleName: this.getSaved<string>('identityNameForScenario'),
364+
roleName: this.getSavedIdentity().identityName,
365365
});
366366
assert.ifError(result.stderr || result.err);
367367
} else if (identityType === EntityType.IAM_USER) { // accounts do not have any policy
368368
const result = await IAM.attachUserPolicy({
369369
policyArn,
370-
userName: this.getSaved<string>('identityNameForScenario'),
370+
userName: this.getSavedIdentity().identityName,
371371
});
372372
assert.ifError(result.stderr || result.err);
373373
}
@@ -434,13 +434,13 @@ Given('an environment setup for the API', async function (this: Zenko) {
434434
|| identityType === EntityType.DATA_CONSUMER) {
435435
const result = await IAM.detachRolePolicy({
436436
policyArn,
437-
roleName: this.getSaved<string>('identityNameForScenario'),
437+
roleName: this.getSavedIdentity().identityName,
438438
});
439439
assert.ifError(result.stderr || result.err);
440440
} else if (identityType === EntityType.IAM_USER) { // accounts do not have any policy
441441
const detachResult = await IAM.detachUserPolicy({
442442
policyArn,
443-
userName: this.getSaved<string>('identityNameForScenario'),
443+
userName: this.getSavedIdentity().identityName,
444444
});
445445
assert.ifError(detachResult.stderr || detachResult.err);
446446
}

tests/ctst/steps/iam-policies/common.ts

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,19 @@
11
import { When, Then } from '@cucumber/cucumber';
22
import { strict as assert } from 'assert';
33
import Zenko from '../../world/Zenko';
4-
import { CacheHelper, ClientOptions, Command, Identity, IdentityEnum, VaultAuth } from 'cli-testing';
4+
import { CacheHelper, ClientOptions, Command, Identity, VaultAuth } from 'cli-testing';
55
import { runActionAgainstBucket } from 'steps/utils/utils';
66

77
When('the user tries to perform {string} on the bucket', async function (this: Zenko, action: string) {
88
await runActionAgainstBucket(this, action);
99
});
1010

1111
When('the user tries to perform vault auth {string}', async function (this: Zenko, action: string) {
12+
const lastIdentity = this.getSavedIdentity();
1213
const userCredentials = Identity.getCredentialsForIdentity(
13-
this.getSaved<IdentityEnum>('identityTypeForScenario'),
14-
this.getSaved<string>('identityNameForScenario'),
15-
this.getSaved<string>('accountNameForScenario'),
14+
lastIdentity.identityType,
15+
lastIdentity.identityName,
16+
lastIdentity.accountName,
1617
);
1718

1819
if (!userCredentials) {
@@ -35,7 +36,7 @@ When('the user tries to perform vault auth {string}', async function (this: Zenk
3536
switch (action) {
3637
case 'GetAccountInfo':
3738
this.setResult(await VaultAuth.getAccounts([
38-
this.getSaved<string>('accountNameForScenario') || this.parameters.AccountName,
39+
lastIdentity.accountName || this.parameters.AccountName,
3940
], null, null, {
4041
// @ts-expect-error accountNames is not generated by CTST yet
4142
accountNames: true,
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { Given, When, Then } from '@cucumber/cucumber';
2+
import { strict as assert } from 'assert';
3+
import Zenko from '../../world/Zenko';
4+
import { IdentityEnum } from 'cli-testing';
5+
6+
interface LocationUsage {
7+
bytesTotal: number;
8+
objectsTotal: number;
9+
}
10+
11+
interface ReportingUsageResponse {
12+
isTruncated: boolean;
13+
marker: string | null;
14+
accounts: Record<string, Record<string, LocationUsage>>;
15+
}
16+
17+
Given('an identity with the {string} keycloak persona', function (this: Zenko, persona: string) {
18+
const username = (this.parameters as Record<string, string>)[persona] || persona;
19+
this.addToSaved('keycloakPersona', username);
20+
});
21+
22+
async function fetchStorageUsageReport(world: Zenko) {
23+
const persona = world.getSaved<string>('keycloakPersona');
24+
const result = await world.managementAPIRequest(
25+
'GET',
26+
`/instance/${world.parameters.InstanceID}/reporting/usage`,
27+
{},
28+
{},
29+
persona,
30+
);
31+
world.addToSaved('lastHttpResponse', result);
32+
return result;
33+
}
34+
35+
When('the user tries to retrieve the storage usage report', async function (this: Zenko) {
36+
await fetchStorageUsageReport(this);
37+
});
38+
39+
When('the user retrieves the storage usage report', async function (this: Zenko) {
40+
const result = await fetchStorageUsageReport(this);
41+
assert.strictEqual(result.statusCode, 200,
42+
`Expected status 200 but got ${result.statusCode}`);
43+
});
44+
45+
Then('the storage usage report response has a valid structure', function (this: Zenko) {
46+
const response = this.getSaved<{ statusCode: number; data: ReportingUsageResponse }>(
47+
'lastHttpResponse');
48+
const data = response.data;
49+
assert.strictEqual(typeof data.isTruncated, 'boolean',
50+
'isTruncated should be a boolean');
51+
assert.ok(typeof data.marker === 'string' || data.marker === null,
52+
'marker should be a string or null');
53+
assert.strictEqual(typeof data.accounts, 'object',
54+
'accounts should be an object');
55+
});
56+
57+
Then('the storage usage report contains the additional accounts', async function (this: Zenko) {
58+
const response = this.getSaved<{ statusCode: number; data: ReportingUsageResponse }>(
59+
'lastHttpResponse');
60+
const accountNames = this.getSavedIdentities()
61+
.filter(id => id.identityType === IdentityEnum.ACCOUNT)
62+
.map(id => id.accountName);
63+
for (const accountName of accountNames) {
64+
assert.ok(accountName in response.data.accounts,
65+
`Account ${accountName} should be present in the report`);
66+
}
67+
});
68+
69+
Then('the report contains the test account with location {string}', async function (this: Zenko, locationName: string) {
70+
const response = this.getSaved<{ statusCode: number; data: ReportingUsageResponse }>(
71+
'lastHttpResponse');
72+
const accountName = this.getSaved<string>('accountName');
73+
74+
assert.ok(accountName in response.data.accounts,
75+
`Account ${accountName} should be present in the report`);
76+
77+
const accountData = response.data.accounts[accountName];
78+
assert.ok(locationName in accountData,
79+
`Location ${locationName} should be present for account ${accountName}`);
80+
81+
this.addToSaved('reportedLocationUsage', accountData[locationName]);
82+
});
83+
84+
Then('the report shows {int} objects and {int} bytes', function (this: Zenko, objects: number, bytes: number) {
85+
const usage = this.getSaved<LocationUsage>('reportedLocationUsage');
86+
assert.strictEqual(usage.objectsTotal, objects,
87+
`Expected ${objects} objects but got ${usage.objectsTotal}`);
88+
assert.strictEqual(usage.bytesTotal, bytes,
89+
`Expected ${bytes} bytes but got ${usage.bytesTotal}`);
90+
});

tests/ctst/steps/utils/utils.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,11 @@ export async function deleteFile(path: string) {
3737
return fsp.unlink(path);
3838
}
3939

40-
export async function uploadSetup(world: Zenko, action: string, body?: string) {
40+
export async function uploadSetup(world: Zenko, action: string, body?: string, size?: number) {
4141
if (action !== 'PutObject' && action !== 'UploadPart') {
4242
return;
4343
}
44-
const objectSize = world.getSaved<number>('objectSize') || 0;
44+
const objectSize = size ?? world.getSaved<number>('objectSize') || 0;
4545
if (body || objectSize > 0) {
4646
const tempFileName = `${Utils.randomString()}_${world.getSaved<string>('objectName')}`;
4747
world.addToSaved('tempFileName', `/tmp/${tempFileName}`);
@@ -289,15 +289,15 @@ async function copyObject(world: Zenko, srcObjectName?: string, dstObjectName?:
289289
return result;
290290
}
291291

292-
async function putObject(world: Zenko, objectName?: string, content?: string) {
292+
async function putObject(world: Zenko, objectName?: string, content?: string, objectSize?: number) {
293293
world.resetCommand();
294294
let finalObjectName = objectName;
295295
if (!finalObjectName) {
296296
finalObjectName = `${Utils.randomString()}`;
297297
}
298298
world.addToSaved('objectName', finalObjectName);
299299
world.logger.debug('Adding object', { objectName: finalObjectName });
300-
await uploadSetup(world, 'PutObject', content);
300+
await uploadSetup(world, 'PutObject', content, objectSize);
301301
world.addCommandParameter({ key: finalObjectName });
302302
world.addCommandParameter({ bucket: world.getSaved<string>('bucketName') });
303303
const userMetadata = world.getSaved<string>('userMetadata');

0 commit comments

Comments
 (0)