Skip to content

Commit f2f64e1

Browse files
vincentlainevincentlaine
committed
API stat commune test perf ⚡
1 parent 422f6e9 commit f2f64e1

9 files changed

Lines changed: 209 additions & 49 deletions

File tree

‎src/data/data.controller.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,4 +83,10 @@ export class DataController {
8383
@Query('departement') departement?: string) {
8484
return this.dataService.departementFindByDate(dateDebut, dateFin, bassinVersant, region, departement);
8585
}
86+
87+
@Get('duree')
88+
@ApiOperation({ summary: 'Récupérer la pondération par commune concernée par des restrictions' })
89+
duree() {
90+
return this.dataService.duree();
91+
}
8692
}

‎src/data/data.module.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,21 @@ import { DataService } from './data.service';
66
import { Departement } from '../zones/entities/departement.entity';
77
import { Region } from '../zones/entities/region.entity';
88
import { BassinVersant } from '../zones/entities/bassin_versant.entity';
9+
import { StatisticCommune } from './entities/statistic_commune.entity';
10+
import { Commune } from '../zones/entities/commune.entity';
911

1012
@Module({
11-
imports: [TypeOrmModule.forFeature([StatisticDepartement, Departement, Region, BassinVersant])],
13+
imports: [TypeOrmModule.forFeature([
14+
StatisticDepartement,
15+
StatisticCommune,
16+
Commune,
17+
Departement,
18+
Region,
19+
BassinVersant,
20+
])],
1221
controllers: [DataController],
1322
providers: [DataService],
23+
exports: [DataService]
1424
})
1525
export class DataModule {
1626
}

‎src/data/data.service.ts‎

Lines changed: 143 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import moment from 'moment';
88
import { Cron, CronExpression } from '@nestjs/schedule';
99
import { Region } from '../zones/entities/region.entity';
1010
import { BassinVersant } from '../zones/entities/bassin_versant.entity';
11+
import { StatisticCommune } from './entities/statistic_commune.entity';
12+
import { Commune } from '../zones/entities/commune.entity';
1113

1214
@Injectable()
1315
export class DataService {
@@ -16,6 +18,8 @@ export class DataService {
1618
data: any;
1719
dataArea: any;
1820
dataDepartement: any;
21+
dataCommune: any;
22+
communes: any[];
1923
departements: any[];
2024
regions: Region[];
2125
bassinsVersants: BassinVersant[];
@@ -26,14 +30,17 @@ export class DataService {
2630

2731
constructor(@InjectRepository(StatisticDepartement)
2832
private readonly statisticDepartementRepository: Repository<StatisticDepartement>,
33+
@InjectRepository(StatisticCommune)
34+
private readonly statisticCommuneRepository: Repository<StatisticCommune>,
35+
@InjectRepository(Commune)
36+
private readonly communeRepository: Repository<Commune>,
2937
@InjectRepository(Departement)
3038
private readonly departementRepository: Repository<Departement>,
3139
@InjectRepository(Region)
3240
private readonly regionRepository: Repository<Region>,
3341
@InjectRepository(BassinVersant)
3442
private readonly bassinVersantRepository: Repository<BassinVersant>,
3543
) {
36-
this.loadData();
3744
}
3845

3946
areaFindByDate(dateDebut?: string, dateFin?: string, bassinVersant?: string, region?: string, departement?: string) {
@@ -151,11 +158,43 @@ export class DataService {
151158
return dataDepartementFiltered;
152159
}
153160

161+
duree() {
162+
return this.dataCommune;
163+
}
164+
154165
@Cron(CronExpression.EVERY_3_HOURS)
155166
async loadData() {
156167
this.logger.log('LOAD DATA');
157-
const statisticsDepartement = await this.statisticDepartementRepository.find({
158-
relations: ['departement'],
168+
await this.loadRefData();
169+
this.logMemoryUsage();
170+
171+
this.data = [];
172+
const endDate = moment();
173+
for (let m = moment(this.beginDate, 'YYYY-MM-DD'); m.diff(endDate, 'days', true) <= 0; m.add(1, 'days')) {
174+
const d = {
175+
date: m.format('YYYY-MM-DD'),
176+
departements: [],
177+
communes: [],
178+
};
179+
this.data.push(d);
180+
}
181+
182+
await this.loadDepartementData();
183+
this.data = [];
184+
185+
await this.loadCommuneData();
186+
}
187+
188+
async loadRefData() {
189+
this.communes = await this.communeRepository.find({
190+
select: {
191+
id: true,
192+
code: true,
193+
nom: true,
194+
},
195+
order: {
196+
code: 'ASC',
197+
},
159198
});
160199
this.departements = (await this.departementRepository
161200
.createQueryBuilder('departement')
@@ -192,45 +231,77 @@ export class DataService {
192231
});
193232
this.fullArea = this.departements.reduce((acc, d) => acc + d.area, 0);
194233
this.metropoleArea = this.departements.filter(d => d.code.length < 3).reduce((acc, d) => acc + d.area, 0);
234+
}
195235

196-
const endDate = moment();
197-
this.data = [];
198-
for (let m = moment(this.beginDate, 'YYYY-MM-DD'); m.diff(endDate, 'days', true) <= 0; m.add(1, 'days')) {
199-
const d = {
200-
date: m.format('YYYY-MM-DD'),
201-
restrictions: [],
202-
};
203-
statisticsDepartement.forEach((statisticDepartement) => {
204-
const restriction = statisticDepartement.restrictions.find(r => r.date === m.format('YYYY-MM-DD'));
205-
if (restriction) {
206-
d.restrictions.push({
207-
...{ departement: statisticDepartement.departement.code },
208-
...restriction,
209-
});
210-
}
211-
});
212-
this.data.push(d);
236+
async loadDepartementData() {
237+
let statisticsDepartement = await this.statisticDepartementRepository.find({
238+
relations: ['departement'],
239+
});
240+
241+
for (const statisticDepartement of statisticsDepartement) {
242+
for (const restriction of statisticDepartement.restrictions) {
243+
const d = this.data.find(x => x.date === restriction.date);
244+
d.departements.push({
245+
...{ departement: statisticDepartement.departement.code },
246+
...restriction,
247+
});
248+
}
213249
}
214250

215251
this.computeDataArea();
252+
this.logMemoryUsage();
253+
216254
this.computeDataDepartement();
255+
this.logMemoryUsage();
256+
257+
for (const d of this.data) {
258+
d.departements = [];
259+
}
217260
}
218261

262+
async loadCommuneData() {
263+
this.logger.log('COMPUTE DATA COMMUNE - BEGIN');
264+
this.dataCommune = [];
265+
const communesCount = await this.statisticCommuneRepository.count();
266+
for (let i = 0; i < communesCount; i = i + 1000) {
267+
for (const d of this.data) {
268+
d.communes = [];
269+
}
270+
let statisticsCommune = await this.statisticCommuneRepository.find({
271+
select: {
272+
id: true,
273+
restrictionsByMonth: true,
274+
commune: {
275+
id: true,
276+
code: true,
277+
},
278+
},
279+
relations: ['commune'],
280+
take: 1000,
281+
skip: i,
282+
});
283+
284+
this.computeDataCommune(statisticsCommune);
285+
}
286+
this.logger.log('COMPUTE DATA COMMUNE - END');
287+
this.logMemoryUsage();
288+
}
219289

220290
computeDataArea() {
221291
this.logger.log('COMPUTE DATA AREA');
222-
this.dataArea = this.data.map(data => {
223-
return {
292+
this.dataArea = [];
293+
for (const data of this.data) {
294+
this.dataArea.push({
224295
date: data.date,
225-
ESO: this.filterRestrictions(data.restrictions, 'SOU', this.fullArea),
226-
ESU: this.filterRestrictions(data.restrictions, 'SUP', this.fullArea),
227-
AEP: this.filterRestrictions(data.restrictions, 'AEP', this.fullArea),
296+
ESO: this.filterRestrictions(data.departements, 'SOU', this.fullArea),
297+
ESU: this.filterRestrictions(data.departements, 'SUP', this.fullArea),
298+
AEP: this.filterRestrictions(data.departements, 'AEP', this.fullArea),
228299
bassinsVersants: this.bassinsVersants.map(b => {
229300
const depFiltered = this.departements.filter(d => b.departements.some(dep => dep.id === d.id));
230301
const area = depFiltered.reduce((acc, d) => acc + d.area, 0);
231-
const restrictions = data.restrictions.filter(r => depFiltered.some(dep => dep.code === r.departement));
302+
const restrictions = data.departements.filter(r => depFiltered.some(dep => dep.code === r.departement));
232303
return {
233-
...b,
304+
id: b.id,
234305
ESO: this.filterRestrictions(restrictions, 'SOU', area),
235306
ESU: this.filterRestrictions(restrictions, 'SUP', area),
236307
AEP: this.filterRestrictions(restrictions, 'AEP', area),
@@ -239,26 +310,26 @@ export class DataService {
239310
regions: this.regions.map(r => {
240311
const depFiltered = this.departements.filter(d => r.departements.some(dep => dep.id === d.id));
241312
const area = depFiltered.reduce((acc, d) => acc + d.area, 0);
242-
const restrictions = data.restrictions.filter(r => depFiltered.some(dep => dep.code === r.departement));
313+
const restrictions = data.departements.filter(r => depFiltered.some(dep => dep.code === r.departement));
243314
return {
244-
...r,
315+
id: r.id,
245316
ESO: this.filterRestrictions(restrictions, 'SOU', area),
246317
ESU: this.filterRestrictions(restrictions, 'SUP', area),
247318
AEP: this.filterRestrictions(restrictions, 'AEP', area),
248319
};
249320
}),
250321
departements: this.departements.map(dep => {
251322
const area = dep.area;
252-
const restrictions = data.restrictions.filter(r => dep.code === r.departement);
323+
const restrictions = data.departements.filter(r => dep.code === r.departement);
253324
return {
254-
...dep,
325+
id: dep.id,
255326
ESO: this.filterRestrictions(restrictions, 'SOU', area),
256327
ESU: this.filterRestrictions(restrictions, 'SUP', area),
257328
AEP: this.filterRestrictions(restrictions, 'AEP', area),
258329
};
259330
}),
260-
};
261-
});
331+
});
332+
}
262333
}
263334

264335
filterRestrictions(restrictions: any, zoneType: string, areaPercentage: number) {
@@ -272,21 +343,36 @@ export class DataService {
272343

273344
computeDataDepartement() {
274345
this.logger.log('COMPUTE DATA DEPARTEMENT');
275-
const dataToReturn = [];
276-
this.data.forEach(d => {
346+
this.dataDepartement = [];
347+
for (const d of this.data) {
277348
const tmp = {
278349
date: d.date,
279350
departements: [],
280351
};
281352
this.departements.forEach(departement => {
282353
tmp.departements.push({
283354
code: departement.code,
284-
niveauGravite: this.findMaxNiveauGravite(d.restrictions, departement.code),
355+
niveauGravite: this.findMaxNiveauGravite(d.departements, departement.code),
285356
});
286357
});
287-
dataToReturn.push(tmp);
288-
});
289-
this.dataDepartement = dataToReturn;
358+
this.dataDepartement.push(tmp);
359+
}
360+
}
361+
362+
computeDataCommune(statisticsCommune) {
363+
const communesFiltered = this.communes.filter(c => statisticsCommune.some(sc => sc.commune.code === c.code));
364+
this.logger.log('COMMUNES FILTERED', communesFiltered.length);
365+
for (const sc of statisticsCommune) {
366+
this.dataCommune.push({
367+
code: sc.commune.code,
368+
restrictions: sc.restrictionsByMonth.map(r => {
369+
return {
370+
d: r.date,
371+
p: r.ponderation,
372+
}
373+
})
374+
})
375+
}
290376
}
291377

292378
findMaxNiveauGravite(restrictions: any[], departementCode: string) {
@@ -344,4 +430,22 @@ export class DataService {
344430
}),
345431
};
346432
}
433+
434+
// TMP
435+
formatMemoryUsage(data) {
436+
return `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
437+
}
438+
439+
logMemoryUsage() {
440+
const memoryData = process.memoryUsage();
441+
442+
const memoryUsage = {
443+
rss: `${this.formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`,
444+
heapTotal: `${this.formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`,
445+
heapUsed: `${this.formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`,
446+
external: `${this.formatMemoryUsage(memoryData.external)} -> V8 external memory`,
447+
};
448+
449+
this.logger.log(memoryUsage);
450+
}
347451
}

‎src/data/entities/statistic_commune.entity.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { BaseEntity, Column, Entity, ManyToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
2+
import { Commune } from '../../zones/entities/commune.entity';
3+
4+
@Entity()
5+
@Unique(['commune'])
6+
export class StatisticCommune extends BaseEntity {
7+
@PrimaryGeneratedColumn()
8+
id: number;
9+
10+
@ManyToOne(() => Commune, (commune) => commune.statisticCommune)
11+
commune: Commune;
12+
13+
@Column('jsonb',{ nullable: true })
14+
restrictions: any[];
15+
16+
@Column('jsonb',{ nullable: true })
17+
restrictionsByMonth: any[];
18+
}

‎src/statistics/statistics.module.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Module } from '@nestjs/common';
1+
import { forwardRef, Module } from '@nestjs/common';
22
import { StatisticsService } from './statistics.service';
33
import { StatisticsController } from './statistics.controller';
44
import { HttpModule } from '@nestjs/axios';
@@ -8,9 +8,15 @@ import { TypeOrmModule } from '@nestjs/typeorm';
88
import { Statistic } from './entities/statistic.entity';
99

1010
@Module({
11-
imports: [TypeOrmModule.forFeature([Statistic]), HttpModule, DepartementsModule, SubscriptionsModule],
11+
imports: [
12+
TypeOrmModule.forFeature([Statistic]),
13+
HttpModule,
14+
DepartementsModule,
15+
forwardRef(() => SubscriptionsModule),
16+
],
1217
controllers: [StatisticsController],
1318
providers: [StatisticsService],
19+
exports: [StatisticsService],
1420
})
1521
export class StatisticsModule {
1622
}

‎src/statistics/statistics.service.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Injectable } from '@nestjs/common';
1+
import { forwardRef, Inject, Injectable } from '@nestjs/common';
22
import process from 'node:process';
33
import { HttpService } from '@nestjs/axios';
44
import { firstValueFrom } from 'rxjs';
@@ -21,8 +21,8 @@ export class StatisticsService {
2121
private readonly statisticRepository: Repository<Statistic>,
2222
private readonly httpService: HttpService,
2323
private readonly departementsService: DepartementsService,
24+
@Inject(forwardRef(() => SubscriptionsService))
2425
private readonly subscriptionsService: SubscriptionsService) {
25-
this.loadStatistics();
2626
}
2727

2828
findAll() {

0 commit comments

Comments
 (0)