perf/dump db

.scalingo/first-deploy.sh

@@ -4,6 +4,12 @@ echo "First deploy"
 
 if [[ $APP =~ "pa-back-staging-pr" ]] ; then
   echo "Dump de la DB de staging"
+
+  if [[ $DATABASE_URL == *"pa_back_pro_817"* ]]; then
+    echo "Error: Target must not be production"
+    exit 1
+  fi
+
   export PATH=$HOME/bin:$PATH
   dbclient-fetcher psql 13
   psql --dbname $DATABASE_URL -c "CREATE extension postgis;"

.scalingo/post-deploy.sh

@@ -1,13 +1,23 @@
 #!/bin/bash
 
+echo "Post deploy"
+
 if [[ $APP =~ "pa-back-staging-pr" ]] ; then
   echo "Dump de la DB de staging"
+
+  if [[ $DATABASE_URL == *"pa_back_pro_817"* ]]; then
+    echo "Error: Target must not be production"
+    exit 1
+  fi
+
   export PATH=$HOME/bin:$PATH
   dbclient-fetcher psql 13
   pg_dump --clean --if-exists --format c --no-owner --no-privileges --no-comments -n 'public' -n 'sequelize' --exclude-schema 'information_schema' --exclude-schema '^pg_*' --exclude-schema 'tiger' --exclude-schema 'tiger_data' --exclude-schema 'topology' --dbname $STAGING_DATABASE_URL --file dump.pgsql
   psql --dbname $DATABASE_URL -c "CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder; CREATE EXTENSION IF NOT EXISTS pg_trgm;"
   pg_restore --clean --if-exists --no-owner --no-privileges --no-comments --dbname $DATABASE_URL dump.pgsql
 fi
 
-yarn migration
+if [[ ! $APP =~ "pa-back-prod-test" ]] ; then
+  yarn migration
+fi
 

package.json

@@ -58,13 +58,15 @@
     "tasks:maj-segments": "IS_WEB=false TASK_NAME=MAJ_SEGMENTS node dist/main",
     "tasks:monitorer-jobs": "IS_WEB=false TASK_NAME=MONITORER_JOBS node dist/main",
     "tasks:suivre-events-milo": "IS_WEB=false TASK_NAME=SUIVRE_FILE_EVENEMENTS_MILO node dist/main",
+    "tasks:dump-perfs": "IS_WEB=false TASK_NAME=DUMP_PERFS node dist/main",
     "tasks:dump-analytics": "IS_WEB=false TASK_NAME=DUMP_ANALYTICS node dist/main",
     "tasks:charger-evenements": "IS_WEB=false TASK_NAME=CHARGER_EVENEMENTS_ANALYTICS node dist/main",
     "tasks:enrichir-evenements": "IS_WEB=false TASK_NAME=ENRICHIR_EVENEMENTS_ANALYTICS node dist/main",
     "tasks:charger-vues": "IS_WEB=false TASK_NAME=CHARGER_LES_VUES_ANALYTICS node dist/main",
     "tasks:initialiser-les-vues": "IS_WEB=false TASK_NAME=INITIALISER_LES_VUES node dist/main",
     "tasks:creer-tables-ae-annuelles": "IS_WEB=false TASK_NAME=CREER_TABLES_AE_ANNUELLES_ANALYTICS node dist/main",
     "tasks:notifier-bonne-alternance": "IS_WEB=false TASK_NAME=NOTIFIER_BONNE_ALTERNANCE node dist/main",
+    "dump-restore-db-perfs": "scripts/perfs/db_dump_restore.sh 2>&1",
     "dump-restore-db": "scripts/analytics/0_db_dump_restore.sh 2>&1",
     "dump-restore-db:local": "dotenv -e .environment yarn dump-restore-db",
     "release:patch": "scripts/release_version_only.sh patch",

scripts/perfs/db_dump_restore.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+if [[ $APP =~ "pa-back-prod-test" ]] ; then
+  echo "Dump de la DB de prod vers la db de perfs"
+
+  if [ -z "$PROD_DATABASE_URL" ]; then
+    echo "error: env var PROD_DATABASE_URL must be set"
+    exit 1
+  fi
+
+  if [ -z "$DATABASE_URL" ]; then
+    echo "error: env var DATABASE_URL must be set"
+    exit 1
+  fi
+
+  if [[ $DATABASE_URL == *"pa_back_pro_817"* ]]; then
+    echo "Error: Target must not be production"
+    exit 1
+  fi
+
+  export PATH=$HOME/bin:$PATH
+  dbclient-fetcher psql 13
+
+  echo "Drop conflicting extensions (if they exist)"
+  psql --dbname $DATABASE_URL -c "DROP EXTENSION IF EXISTS postgis CASCADE;"
+  psql --dbname $DATABASE_URL -c "DROP EXTENSION IF EXISTS postgis_tiger_geocoder CASCADE;"
+  psql --dbname $DATABASE_URL -c "DROP EXTENSION IF EXISTS postgis_topology CASCADE;"
+  psql --dbname $DATABASE_URL -c "DROP EXTENSION IF EXISTS pg_trgm CASCADE;"
+
+  echo "Dump production database (excluding unnecessary schemas)"
+  pg_dump --clean --if-exists --format c --no-owner --no-privileges --no-comments \
+    -n 'public' -n 'sequelize' \
+    --exclude-schema 'information_schema' --exclude-schema '^pg_*' \
+    --exclude-schema 'tiger' --exclude-schema 'tiger_data' --exclude-schema 'topology' \
+    --dbname $PROD_DATABASE_URL --file dump.pgsql
+
+  echo "Restore database dump (WITHOUT extensions yet)"
+  pg_restore --clean --if-exists --no-owner --no-privileges --no-comments --dbname $DATABASE_URL dump.pgsql
+
+  echo "Anonymizing email addresses in jeune table"
+  psql --dbname $DATABASE_URL -c "UPDATE jeune SET email = '[email protected]', nom='test', prenom='test', push_notification_token='plop', id_partenaire='plop' WHERE id != 'e88a3b2a-e994-11ed-a05b-0242ac120003';"
+  psql --dbname $DATABASE_URL -c "UPDATE conseiller SET email = '[email protected]', nom='test', prenom='test', username='plop', id_authentification='plop';"
+  psql --dbname $DATABASE_URL -c "UPDATE rendez_vous SET titre='test', commentaire='plop', modalite='plop', adresse='plop';"
+  psql --dbname $DATABASE_URL -c "UPDATE action SET contenu='test', description='test', qualification_commentaire='plop';"
+
+  echo "Recreate extensions AFTER restore"
+  psql --dbname $DATABASE_URL -c "CREATE EXTENSION postgis;"
+  psql --dbname $DATABASE_URL -c "CREATE EXTENSION postgis_tiger_geocoder CASCADE;"
+  psql --dbname $DATABASE_URL -c "CREATE EXTENSION postgis_topology;"
+  psql --dbname $DATABASE_URL -c "CREATE EXTENSION pg_trgm;"
+
+  echo "Grant correct privileges"
+  psql --dbname $DATABASE_URL -c "GRANT USAGE, CREATE ON SCHEMA public TO pa_back_pro_3082;"
+  psql --dbname $DATABASE_URL -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO pa_back_pro_3082;"
+
+  echo "Adding Data"
+  psql --dbname $DATABASE_URL -c "CREATE TABLE IF NOT EXISTS recherche (
+    id UUID PRIMARY KEY,
+    id_jeune VARCHAR(255) NOT NULL,
+    titre VARCHAR(255) NOT NULL,
+    type VARCHAR(255) NOT NULL,
+    metier VARCHAR(255),
+    localisation VARCHAR(255),
+    criteres JSONB NOT NULL,
+    date_creation TIMESTAMPTZ DEFAULT NOW(),
+    date_derniere_recherche TIMESTAMPTZ DEFAULT NOW(),
+    etat_derniere_recherche VARCHAR(255) DEFAULT 'SUCCES',
+    geometrie GEOMETRY(Polygon, 4326),
+    CONSTRAINT fk_id_jeune FOREIGN KEY (id_jeune) REFERENCES jeune(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE INDEX idx_recherche_id_jeune ON recherche(id_jeune);"
+
+  psql --dbname $DATABASE_URL -c "INSERT INTO recherche (
+    id, id_jeune, titre, type, metier, localisation, criteres, 
+    date_creation, date_derniere_recherche, etat_derniere_recherche, geometrie
+)
+SELECT 
+    gen_random_uuid(), -- Generates a unique UUID for each row
+    'e88a3b2a-e994-11ed-a05b-0242ac120003',
+    'Titre ' || s, 
+    'Type ' || s, 
+    'Metier ' || s, 
+    'Localisation ' || s, 
+    '{}'::jsonb, -- Empty JSON object
+    NOW() - (random() * interval '365 days'), -- Random date within the last year
+    NOW(),
+    'SUCCES',
+    NULL
+FROM generate_series(1, 200000) s;"
+
+fi
+
+rm -f dump.pgsql

src/app.module.ts

@@ -366,6 +366,7 @@ import { GetCJETokenQueryHandler } from './application/queries/get-cje-token.que
 import { NotifierBonneAlternanceJobHandler } from './application/jobs/notifier-bonne-alternance.job.handler.db'
 import { NotifierCampagneJobHandler } from './application/jobs/notifier-campagne.job.handler.db'
 import { GetNotificationsJeuneQueryHandler } from './application/queries/get-notifications-jeune.query.handler.db'
+import { DumpForPerfsJobHandler } from './application/jobs/perfs/dump-for-perfs.job'
 
 export const buildModuleMetadata = (): ModuleMetadata => ({
   imports: [
@@ -820,6 +821,7 @@ export const JobHandlerProviders = [
   SuivreEvenementsMiloCronJobHandler,
   TraiterEvenementMiloJobHandler,
   DumpForAnalyticsJobHandler,
+  DumpForPerfsJobHandler,
   ChargerEvenementsJobHandler,
   NettoyerEvenementsChargesAnalyticsJobHandler,
   EnrichirEvenementsJobHandler,

src/application/jobs/perfs/dump-for-perfs.job.ts

@@ -0,0 +1,45 @@
+import { Inject, Injectable } from '@nestjs/common'
+import { exec } from 'child_process'
+import { promisify } from 'util'
+import { JobHandler } from '../../../building-blocks/types/job-handler'
+import { Planificateur, ProcessJobType } from '../../../domain/planificateur'
+import { SuiviJob, SuiviJobServiceToken } from '../../../domain/suivi-job'
+import { DateService } from '../../../utils/date-service'
+
+@Injectable()
+@ProcessJobType(Planificateur.JobType.DUMP_PERFS)
+export class DumpForPerfsJobHandler extends JobHandler<Planificateur.Job> {
+  constructor(
+    @Inject(SuiviJobServiceToken)
+    suiviJobService: SuiviJob.Service,
+    private dateService: DateService
+  ) {
+    super(Planificateur.JobType.DUMP_PERFS, suiviJobService)
+  }
+
+  async handle(): Promise<SuiviJob> {
+    let erreur
+    const maintenant = this.dateService.now()
+
+    const cmd = 'yarn run dump-restore-db-perfs'
+    const { stdout, stderr } = await promisify(exec)(cmd)
+
+    if (stdout) {
+      this.logger.log(stdout)
+    }
+
+    if (stderr) {
+      this.logger.error(stderr)
+      erreur = stderr
+    }
+
+    return {
+      jobType: this.jobType,
+      nbErreurs: 0,
+      succes: erreur ? false : true,
+      dateExecution: maintenant,
+      tempsExecution: DateService.calculerTempsExecution(maintenant),
+      resultat: {}
+    }
+  }
+}

src/domain/planificateur.ts

@@ -66,6 +66,7 @@ export namespace Planificateur {
     SUIVRE_FILE_EVENEMENTS_MILO = 'SUIVRE_FILE_EVENEMENTS_MILO',
     TRAITER_EVENEMENT_MILO = 'TRAITER_EVENEMENT_MILO',
     DUMP_ANALYTICS = 'DUMP_ANALYTICS',
+    DUMP_PERFS = 'DUMP_PERFS',
     CHARGER_EVENEMENTS_ANALYTICS = 'CHARGER_EVENEMENTS_ANALYTICS',
     NETTOYER_EVENEMENTS_CHARGES_ANALYTICS = 'NETTOYER_EVENEMENTS_CHARGES_ANALYTICS',
     ENRICHIR_EVENEMENTS_ANALYTICS = 'ENRICHIR_EVENEMENTS_ANALYTICS',