Skip to content

Commit 31a58c2

Browse files
committed
2 parents cd374d8 + b46b3d4 commit 31a58c2

File tree

14 files changed

+270
-42
lines changed

14 files changed

+270
-42
lines changed

.github/workflows/deploy-documentation.yml

+11-10
Original file line numberDiff line numberDiff line change
@@ -28,31 +28,32 @@ jobs:
2828
- name: Build
2929
working-directory: ./docs
3030
run: npm run build
31-
32-
- name: Upload artifact
31+
32+
- name: Upload static files as artifact
3333
id: upload
3434
uses: actions/upload-pages-artifact@v3
3535
with:
36-
# 👇 Specify build output path
37-
path: ./docs/build
38-
name: github-pages
36+
path: ./docs/build
3937

4038
deploy:
4139
needs: build
4240
runs-on: ubuntu-latest
4341

44-
environment:
45-
name: github-pages
46-
url: ${{ steps.deployment.outputs.page_url }}
47-
4842
permissions:
4943
pages: write
5044
id-token: write
45+
46+
environment:
47+
name: github-pages
48+
url: ${{ steps.deployment.outputs.page_url }}
5149

5250
steps:
51+
- name: Configure Pages
52+
uses: actions/configure-pages@v4
53+
5354
- name: Deploy to GitHub Pages
5455
id: deployment
55-
uses: actions/deploy-pages@v3
56+
uses: actions/deploy-pages@v4
5657

5758
reindex:
5859
needs: deploy

packages/backend/jobs-manager/service/src/modules/auth/constants.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { UnauthorizedException } from '@nestjs/common';
22

33
export const jwtConstants = {
44
secret: process.env['JM_JWT_SECRET'],
5-
expirationTime: '300s',
5+
expirationTime: '12h',
66
};
77

88
export const resetPasswordConstants = {
@@ -11,7 +11,7 @@ export const resetPasswordConstants = {
1111

1212
export const rtConstants = {
1313
secret: process.env['JM_REFRESH_SECRET'],
14-
expirationTime: '25200s',
14+
expirationTime: '5 days',
1515
};
1616

1717
export const orchestratorConstants = {

packages/backend/jobs-manager/service/src/modules/database/database.constants.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,5 @@ export const MONGO_TIMESTAMP_SCHEMA_CONFIG: SchemaOptions = {
1111
},
1212
};
1313

14-
export const detailsLevel = ['full', 'summary', 'number'] as const;
14+
export const detailsLevel = ['extended', 'full', 'summary', 'number'] as const;
1515
export type DetailsLevel = (typeof detailsLevel)[number];

packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.controller.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import { RolesGuard } from '../../../auth/guards/role.guard';
2020
import { ApiKeyStrategy } from '../../../auth/strategies/api-key.strategy';
2121
import { JwtStrategy } from '../../../auth/strategies/jwt.strategy';
2222
import { BatchEditPortsDto, DeleteManyPortsDto, GetPortsDto } from './port.dto';
23-
import { Port, PortDocument } from './port.model';
23+
import { ExtendedPort, Port, PortDocument } from './port.model';
2424
import { PortService } from './port.service';
2525

2626
@Controller('ports')
@@ -32,7 +32,7 @@ export class PortController {
3232
@Get()
3333
async getHostTopTcpPorts(
3434
@Query() dto: GetPortsDto,
35-
): Promise<Port[] | Page<PortDocument>> {
35+
): Promise<Port[] | Page<PortDocument | ExtendedPort>> {
3636
if (dto.detailsLevel === 'summary') throw new HttpNotImplementedException();
3737

3838
const totalRecords = await this.portsService.count(dto);

packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.model.ts

+5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
22
import { Document, Types } from 'mongoose';
33
import { MONGO_TIMESTAMP_SCHEMA_CONFIG } from '../../database.constants';
4+
import { DomainSummary } from '../domain/domain.summary';
45
import { HostSummary, HostSummaryType } from '../host/host.summary';
56

7+
export interface ExtendedPort extends Port {
8+
domains?: DomainSummary[];
9+
}
10+
611
export type PortDocument = Port & Document;
712

813
@Schema(MONGO_TIMESTAMP_SCHEMA_CONFIG)

packages/backend/jobs-manager/service/src/modules/database/reporting/port/port.service.ts

+23-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Injectable, Logger } from '@nestjs/common';
22
import { InjectModel } from '@nestjs/mongoose';
33
import { DeleteResult, UpdateResult } from 'mongodb';
4-
import { FilterQuery, Model, Types } from 'mongoose';
4+
import { FilterQuery, Model, Query, Types } from 'mongoose';
55
import {
66
HttpBadRequestException,
77
HttpNotFoundException,
@@ -12,7 +12,7 @@ import { CorrelationKeyUtils } from '../correlation.utils';
1212
import { Host, HostDocument } from '../host/host.model';
1313
import { WebsiteService } from '../websites/website.service';
1414
import { BatchEditPortsDto, GetPortsDto } from './port.dto';
15-
import { Port, PortDocument } from './port.model';
15+
import { ExtendedPort, Port, PortDocument } from './port.model';
1616

1717
@Injectable()
1818
export class PortService {
@@ -218,7 +218,6 @@ export class PortService {
218218
}
219219

220220
public async getHostPort(hostId: string, portNumber: number) {
221-
console.log(await this.portsModel.findOne());
222221
return await this.portsModel.findOne({
223222
'host.id': { $eq: new Types.ObjectId(hostId) },
224223
port: { $eq: portNumber },
@@ -315,13 +314,13 @@ export class PortService {
315314
page: number = null,
316315
pageSize: number = null,
317316
filter: Partial<GetPortsDto> = null,
318-
): Promise<PortDocument[]> {
317+
): Promise<(PortDocument | ExtendedPort)[]> {
319318
const projection =
320319
filter.detailsLevel === 'number'
321320
? '_id port layer4Protocol correlationKey'
322321
: undefined;
323322

324-
let query;
323+
let query: Query<PortDocument[], any> = null;
325324
if (filter) {
326325
query = this.portsModel.find(await this.buildFilters(filter), projection);
327326
} else {
@@ -331,7 +330,25 @@ export class PortService {
331330
if (page != null && pageSize != null) {
332331
query = query.skip(page * pageSize).limit(pageSize);
333332
}
334-
return await query;
333+
334+
let ports = await query.exec();
335+
if (filter.detailsLevel === 'extended') {
336+
const hostIds: Types.ObjectId[] = ports.map((p) => p.host.id);
337+
const summaries: Pick<HostDocument, '_id' | 'domains'>[] =
338+
await this.hostModel.find({ _id: { $in: hostIds } }, '_id domains');
339+
const extendedPorts: ExtendedPort[] = [];
340+
for (const p of ports) {
341+
extendedPorts.push({
342+
...JSON.parse(JSON.stringify(p)),
343+
domains: summaries.find(
344+
(s) => s._id.toString() === p.host.id.toString(),
345+
)?.domains,
346+
});
347+
}
348+
return extendedPorts;
349+
}
350+
351+
return ports;
335352
}
336353

337354
public async count(filter: GetPortsDto = null) {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
import { getModelToken } from '@nestjs/mongoose';
2+
import { Test, TestingModule } from '@nestjs/testing';
3+
import { randomUUID } from 'crypto';
4+
import { Model } from 'mongoose';
5+
import { AppModule } from '../../app.module';
6+
import { CustomFindingFieldDto } from '../../findings/finding.dto';
7+
import { FindingsService } from '../../findings/findings.service';
8+
import { Job } from '../jobs/models/jobs.model';
9+
import { CustomFinding } from './findings/finding.model';
10+
import { ProjectService } from './project.service';
11+
12+
describe('Project Service Spec', () => {
13+
let moduleFixture: TestingModule;
14+
let findingsService: FindingsService;
15+
let projectService: ProjectService;
16+
let findingsModel: Model<CustomFinding>;
17+
let jobsModel: Model<Job>;
18+
19+
beforeAll(async () => {
20+
moduleFixture = await Test.createTestingModule({
21+
imports: [AppModule],
22+
}).compile();
23+
findingsService = moduleFixture.get(FindingsService);
24+
projectService = moduleFixture.get(ProjectService);
25+
findingsModel = moduleFixture.get<Model<CustomFinding>>(
26+
getModelToken('finding'),
27+
);
28+
jobsModel = moduleFixture.get<Model<Job>>(getModelToken('job'));
29+
});
30+
31+
beforeEach(async () => {
32+
await findingsModel.deleteMany({});
33+
const ids = await projectService.getAllIds();
34+
for (const id of ids) {
35+
await projectService.delete(id);
36+
}
37+
});
38+
39+
afterAll(async () => {
40+
await moduleFixture.close();
41+
});
42+
43+
describe('Cleanup project', () => {
44+
it('Should preserve findings that do not belong to the deleted project', async () => {
45+
// Arrange
46+
const p = await project();
47+
const p2 = await project();
48+
49+
const nonFilteredKey = 'my-finding-1';
50+
const filteredKey = 'my-finding-2';
51+
52+
const j = await jobsModel.create({
53+
projectId: p.id,
54+
task: 'CustomJob',
55+
});
56+
57+
const f1 = await customDomainFinding(p.id, j.id, filteredKey);
58+
const f2 = await customDomainFinding(p2.id, j.id, nonFilteredKey);
59+
60+
// Act
61+
await projectService.delete(p._id.toString());
62+
const findings = await findingsService.getAll(0, 100, {});
63+
64+
// Assert
65+
expect(findings.totalRecords).toBe(1);
66+
expect(findings.items[0].correlationKey).toMatch(
67+
new RegExp(`^project:${p2._id.toString()}`),
68+
);
69+
});
70+
});
71+
72+
async function customDomainFinding(
73+
projectId: string,
74+
jobId: string,
75+
key: string,
76+
domainName: string = 'example.org',
77+
name: string = 'My finding',
78+
fields: Array<CustomFindingFieldDto> = [
79+
{
80+
key: 'my-field',
81+
type: 'text',
82+
label: 'My label',
83+
data: 'My content',
84+
},
85+
],
86+
) {
87+
return await findingsService.save(projectId, jobId, {
88+
key,
89+
name,
90+
fields,
91+
type: 'CustomFinding',
92+
domainName,
93+
});
94+
}
95+
96+
async function project() {
97+
return await projectService.addProject({
98+
name: randomUUID(),
99+
imageType: null,
100+
logo: null,
101+
});
102+
}
103+
});

packages/backend/jobs-manager/service/src/modules/database/reporting/project.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ export class ProjectService {
104104
await this.websiteService.deleteAllForProject(id);
105105
await this.triggerService.deleteAllForProject(id);
106106
await this.findingModel.deleteMany({
107-
projectId: { $eq: new Types.ObjectId(id) },
107+
correlationKey: { $regex: new RegExp(`^project:${id}`) },
108108
});
109109
await this.cronSubscriptionModel.deleteMany({
110110
projectId: { $eq: new Types.ObjectId(id) },

packages/backend/jobs-manager/service/src/modules/findings/findings.service.spec.ts

+65-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ describe('Findings Service Spec', () => {
3535
});
3636

3737
beforeEach(async () => {
38-
findingsModel.deleteMany();
38+
await findingsModel.deleteMany({});
3939
});
4040

4141
afterAll(async () => {
@@ -326,6 +326,70 @@ describe('Findings Service Spec', () => {
326326
expect(endpoint.fields[0].data).toStrictEqual('/example/file.html');
327327
});
328328

329+
describe('Cleanup findings', () => {
330+
beforeEach(() => {
331+
findingsModel.deleteMany({});
332+
});
333+
334+
it('Should preserve findings that are not older than the time limit', async () => {
335+
// Arrange
336+
const c = await project();
337+
338+
const j = await jobsModel.create({
339+
projectId: c.id,
340+
task: 'CustomJob',
341+
});
342+
343+
const nonFilteredKey = 'my-finding-1';
344+
const filteredKey = 'my-finding-2';
345+
346+
const f1 = await customDomainFinding(c.id, j.id, filteredKey);
347+
const f2 = await customDomainFinding(c.id, j.id, nonFilteredKey);
348+
const now = Date.now();
349+
const yearMs = 60 * 60 * 24 * 365 * 1000;
350+
await findingsModel.updateOne(
351+
{ _id: f1._id },
352+
{ created: new Date(now - yearMs + 1) },
353+
);
354+
355+
// Act
356+
await findingsService.cleanup(now, yearMs);
357+
const findings = await findingsService.getAll(0, 100, {});
358+
359+
// Assert
360+
expect(findings.totalRecords).toBe(2);
361+
});
362+
363+
it('Should delete findings that are older than the time limit', async () => {
364+
// Arrange
365+
const c = await project();
366+
367+
const j = await jobsModel.create({
368+
projectId: c.id,
369+
task: 'CustomJob',
370+
});
371+
372+
const nonFilteredKey = 'my-finding-1';
373+
const filteredKey = 'my-finding-2';
374+
375+
const f1 = await customDomainFinding(c.id, j.id, filteredKey);
376+
const f2 = await customDomainFinding(c.id, j.id, nonFilteredKey);
377+
const now = Date.now();
378+
const yearMs = 60 * 60 * 24 * 365 * 1000;
379+
await findingsModel.updateOne(
380+
{ _id: f1._id },
381+
{ created: new Date(now - yearMs - 1) },
382+
);
383+
384+
// Act
385+
await findingsService.cleanup(now, yearMs);
386+
const findings = await findingsService.getAll(0, 100, {});
387+
388+
// Assert
389+
expect(findings.totalRecords).toBe(1);
390+
});
391+
});
392+
329393
async function customDomainFinding(
330394
projectId: string,
331395
jobId: string,

packages/backend/jobs-manager/service/src/modules/findings/findings.service.ts

+10-5
Original file line numberDiff line numberDiff line change
@@ -223,13 +223,18 @@ export class FindingsService {
223223
/**
224224
* Deletes all the findings runs older than `config.jobRunRetentionTimeSeconds`.
225225
*/
226-
public async cleanup(): Promise<void> {
227-
const config = await this.configService.getConfig();
228-
const ttlMilliseconds = config.findingRetentionTimeSeconds * 1000;
229-
const now = Date.now();
226+
public async cleanup(
227+
now: number = Date.now(),
228+
ttlMilliseconds: number | undefined = undefined,
229+
): Promise<void> {
230+
if (ttlMilliseconds === undefined) {
231+
const config = await this.configService.getConfig();
232+
ttlMilliseconds = config.findingRetentionTimeSeconds * 1000;
233+
}
234+
230235
const oldestValidCreationDate = now - ttlMilliseconds;
231236
await this.findingModel.deleteMany({
232-
$or: [{ created: { $lte: new Date(oldestValidCreationDate) } }],
237+
created: { $lte: new Date(oldestValidCreationDate) },
233238
});
234239
}
235240

0 commit comments

Comments
 (0)