fix: revert mapping optimization. Memory problem on bencmark

Author: Nolife999

arc-core/src/main/java/fr/insee/arc/core/service/global/dao/TableOperations.java

@@ -109,28 +109,4 @@ public static String createTableTravail(String tableIn, String tableOut, String
 		return requete.toString();
 	}
 
-	
-	
-	/**
-	 * Create a view AS 
-	 * @param tableIn
-	 * @param viewOut
-	 * @return
-	 */
-	public static GenericPreparedStatementBuilder createViewImage(String tableIn, String viewOut) {
-		GenericPreparedStatementBuilder requete = new GenericPreparedStatementBuilder();
-		requete.append("\n CREATE ");
-		if (!viewOut.contains(".")) {
-			requete.append("TEMPORARY ");
-		}
-		
-		requete.append(
-				"VIEW " + viewOut + " AS ");
-
-		requete.append("\n SELECT * ");
-		requete.append("\n FROM " + tableIn + "; ");
-		return requete;
-		
-	}
-	
 }

arc-core/src/main/java/fr/insee/arc/core/service/global/dao/ThreadOperations.java

@@ -232,7 +232,8 @@ private ArcPreparedStatementBuilder marquageFinal(String tablePil, String tableP
 		requete.append("\n id_norme = b.id_norme, ");
 		requete.append("\n validite = b.validite, ");
 		requete.append("\n periodicite = b.periodicite, ");
-		requete.append("\n date_traitement = clock_timestamp(), ");
+		requete.append("\n date_traitement = to_timestamp('" + new SimpleDateFormat(ArcDateFormat.TIMESTAMP_FORMAT_CONVERSION.getApplicationFormat()).format(new Date()) + "','" + ArcDateFormat.TIMESTAMP_FORMAT_CONVERSION.getDatastoreFormat()
+		+ "'), ");
 		requete.append("\n nb_enr = b.nb_enr, ");
 		requete.append("\n rapport = b.rapport, ");
 		requete.append("\n validite_inf = b.validite_inf, ");

arc-core/src/main/java/fr/insee/arc/core/service/p5mapping/dao/MappingQueries.java

@@ -12,16 +12,11 @@
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Streams;
 
 import fr.insee.arc.core.dataobjects.ArcPreparedStatementBuilder;
 import fr.insee.arc.core.dataobjects.ColumnEnum;
 import fr.insee.arc.core.dataobjects.ViewEnum;
 import fr.insee.arc.core.service.global.bo.JeuDeRegle;
-import fr.insee.arc.core.service.global.dao.TableOperations;
-import fr.insee.arc.core.service.global.thread.ThreadTemporaryTable;
 import fr.insee.arc.core.service.p5mapping.bo.TableMapping;
 import fr.insee.arc.core.service.p5mapping.bo.VariableMapping;
 import fr.insee.arc.core.service.p5mapping.bo.rules.RegleMappingClePrimaire;
@@ -82,11 +77,8 @@ public class MappingQueries implements IConstanteCaractere, IConstanteNumerique
 	private Set<VariableMapping> ensembleVariableMapping;
 	private SortedSet<Integer> ensembleGroupes;
 
-	private static final String IDENTIFIANT_TRIGGER = "id$tg";
 	private static final String PREP_UNION_TABLE = "pu";
 
-	
-	private Map <String, String> insertionTriggers = new HashMap<String, String>();
 
 	private Map<TableMapping, Set<String>> ensembleRubriqueIdentifianteTable;
 	/*
@@ -105,7 +97,6 @@ public class MappingQueries implements IConstanteCaractere, IConstanteNumerique
 	 */
 	private String nomTableModVariableMetier;
 	private String nomTableSource;
-	private String nomViewSource;
 	private String nomTableRegleMapping;
 	private String tableLienIdentifiants;
 
@@ -278,32 +269,6 @@ private void construireListeRubriqueParTable() {
 			this.ensembleRubriqueIdentifianteTable.put(table, s);
 		}
 	}
-
-	private void initializeTrigger(StringBuilder returned) throws ArcException {
-		returned.append("\n CREATE OR REPLACE FUNCTION pg_temp.pu() RETURNS trigger AS $$ BEGIN ");
-	}
-	
-	private void finalizeTrigger(StringBuilder returned) throws ArcException {
-
-		// important to return NULL not to fill memory with pointer
-		returned.append("\n RETURN NULL; END; $$ LANGUAGE plpgsql;");
-		
-		// create a view on a empty image of the controle phase output table
-		// it could avoid some nasty bugs if the instead of clause fails
-		// as in postgres, insert into view triggers an insert on the master table
-		// and we do not want to modify the data from the controle phase output table
-		String tableTrigger = FormatSQL.temporaryTableName(ThreadTemporaryTable.TABLE_MAPPING_DATA_TEMP);
-		this.nomViewSource = FormatSQL.temporaryTableName(ThreadTemporaryTable.VIEW_MAPPING_DATA_TEMP);
-		
-		returned.append(TableOperations.createTableTravail(nomTableSource, tableTrigger, null, true));
-		returned.append(TableOperations.createViewImage(tableTrigger, this.nomViewSource));
-		
-		// apply trigger to view
-		returned.append("\n CREATE TRIGGER pu");
-		returned.append("\n INSTEAD OF INSERT ON "+this.nomViewSource);
-		returned.append("\n FOR EACH ROW EXECUTE FUNCTION pg_temp.pu();");
-	}
-	
 
 	/**
 	 * Construit les objets de type {@link TableMapping} et {@link VariableMapping}
@@ -395,8 +360,9 @@ public String construireTableLienIdentifiants() throws ArcException {
 	 * @return a map with the model table as key and the query that inserts data as value
 	 * @throws ArcException
 	 */
-	public void getQueriesThatInsertDataIntoTemporaryModelTable(String aNomFichier, StringBuilder requete) throws ArcException {
+	public StringBuilder getQueriesThatInsertDataIntoTemporaryModelTable(String aNomFichier) throws ArcException {
 
+		StringBuilder requete = new StringBuilder();
 			for (TableMapping table : this.ensembleTableMapping) {
 
 				requete.append(ModeRequete.NESTLOOP_OFF);
@@ -412,45 +378,9 @@ public void getQueriesThatInsertDataIntoTemporaryModelTable(String aNomFichier,
 				requete.append(ModeRequete.NESTLOOP_ON);
 
 			}
-			
-			
-			// the view trigger is made to split the main table from controle phase in several little tables that contains only the useful columns
-			// it is a huge optimization on large file as the full scan is only made once
-			StringBuilder requeteTrigger = new StringBuilder();
-			createTriggerTables(requeteTrigger);
-			initializeTrigger(requeteTrigger);
-			defineInsertionTrigger(requeteTrigger);
-			finalizeTrigger(requeteTrigger);
-			useTrigger(requeteTrigger);
-			
-			requete.insert(0, requeteTrigger);
-			
-	}
-
-	private void useTrigger(StringBuilder returned) {
-		returned.append("\n INSERT INTO ").append(this.nomViewSource).append(" SELECT * FROM ").append(this.nomTableSource).append(";");
+		return requete;
 	}
 
-	private void defineInsertionTrigger(StringBuilder returned) {
-		insertionTriggers.entrySet().stream().forEach(t -> 
-		{
-			returned.append("\n INSERT INTO ").append(t.getKey());
-			returned.append("\n SELECT ").append(t.getValue()).append("\n FROM ").append("(SELECT NEW.*) e;");
-		}
-		);		
-	}
-
-	private void createTriggerTables(StringBuilder returned) {
-		
-		insertionTriggers.entrySet().stream().forEach(t -> 
-		{
-			returned.append("\n CREATE TEMPORARY TABLE ").append(t.getKey()).append(FormatSQL.WITH_NO_VACUUM);
-			returned.append("\n AS SELECT ").append(t.getValue()).append("\n FROM ").append(this.nomTableSource);
-			returned.append("\n LIMIT 0;");
-		}
-		);
-	}
-	
 	/**
 	 * Delete empty records from model tables
 	 * @param aNomFichier
@@ -474,18 +404,7 @@ public String deleteEmptyRecords(String aNomFichier) throws ArcException {
 		return requeteGlobale.toString().replace(TOKEN_ID_SOURCE, aNomFichier);
 
 	}
-	
-	/**
-	 * Parse expression to identify what source column are used in
-	 * @param expressionSQLCalculMapping
-	 * @return
-	 */
-	private String computeSourceColumnsUsedInExpression (String expressionSQLCalculMapping)
-	{
-		return Streams.concat(regleMappingFactory.getEnsembleNomRubriqueExistante().stream()
-				, regleMappingFactory.getEnsembleIdentifiantRubriqueExistante().stream())
-		.filter(t-> expressionSQLCalculMapping.toLowerCase().contains(t)).collect(Collectors.joining(","));
-	}
+
 
 	private StringBuilder insererTablePrepUnionNew(StringBuilder returned, TableMapping table,
 			Map<String, String> reglesIdentifiantes) throws ArcException {
@@ -504,16 +423,14 @@ private StringBuilder insererTablePrepUnionNew(StringBuilder returned, TableMapp
 			returned.append("\n FROM (");
 			returned.append("\n SELECT ").append(expressionSQLCalculMapping);
 			returned.append("\n FROM ").append(this.tableLienIdentifiants).append(" d ");
-			returned.append("\n ,").append(idTableGroupe(table,groupe)).append(" e ");
+			returned.append("\n ,").append(this.nomTableSource).append(" e ");
 			returned.append("\n WHERE e.id=d.id_table ");
 			returned.append("\n AND ").append(idTableGroupe(table,groupe));
 			returned.append("\n ) a ");
 			returned.append("\n GROUP BY ");
 			table.sqlListeVariables(returned);
 			returned.append(";");
 
-			insertionTriggers.put(idTableGroupe(table,groupe),computeSourceColumnsUsedInExpression(expressionSQLCalculMapping));
-
 			}
 		}
 
@@ -528,7 +445,7 @@ private StringBuilder insererTablePrepUnionNew(StringBuilder returned, TableMapp
 			returned.append("\n FROM (");
 			returned.append("\n SELECT ").append(expressionSQLCalculMapping);
 			returned.append("\n FROM ").append(this.tableLienIdentifiants).append(" d ");
-			returned.append("\n ,").append(idTableNGroupe(table)).append(" e ");
+			returned.append("\n ,").append(this.nomTableSource).append(" e ");
 			returned.append("\n WHERE e.id=d.id_table ");
 			returned.append("\n AND ").append(idTableNGroupe(table));
 			// if table has group, records selected by group must be excluded
@@ -537,9 +454,6 @@ private StringBuilder insererTablePrepUnionNew(StringBuilder returned, TableMapp
 			returned.append("\n GROUP BY ");
 			table.sqlListeVariables(returned);
 			returned.append(";");
-			
-			insertionTriggers.put(idTableNGroupe(table),computeSourceColumnsUsedInExpression(expressionSQLCalculMapping));
-
 		}
 		return returned;
 	}
@@ -770,48 +684,37 @@ private StringBuilder construireTableIdentifiantsFichierCourant(StringBuilder re
 		tableGroups.keySet().stream().forEach(k -> 	blocSelect.append(",\n ").append(k));
 		tableNonGroup.keySet().stream().forEach(k -> blocSelect.append(",\n ").append(k));
 
-		blocSelect.append("\n FROM ").append(idTableNGroupe(IDENTIFIANT_TRIGGER)).append(";");
-		//blocSelect.append("\n FROM ttt;");
+		blocSelect.append("\n FROM ttt;");
 
 		// bloc 2 : on met les variables non groupe et les variables subalternes
 		// identifiantes de groupes
 		alreadyAdded = new HashSet<>();
-		StringBuilder blocTrigger=new StringBuilder(); 
-		
-		blocTrigger.append(ColumnEnum.ID);
-		
+		StringBuilder blocWith=new StringBuilder(); 
+		blocWith.append("\n WITH ttt as materialized ( SELECT id");
 		for (String nomVariable : reglesIdentifiantes.keySet()) {
 			String expressionVariable = reglesIdentifiantes.get(nomVariable);
 			if (nomsVariablesGroupe.get(nomVariable) == null) {
-				blocTrigger.append("\n,").append(expressionVariable).append(" AS ").append(nomVariable);
-			} else
-			{
-				String nomVariableReworked = linkedIds.get(nomsVariablesGroupe.get(nomVariable)).replaceAll(",id\\_[^,]+", "");
-				if (!alreadyAdded
-					.contains(nomVariableReworked)) {
-				blocTrigger.append("\n ").append(nomVariableReworked);
-				alreadyAdded.add(nomVariableReworked);
-				}
+				blocWith.append("\n ," + expressionVariable + " AS " + nomVariable);
+			} else if (!alreadyAdded
+					.contains(linkedIds.get(nomsVariablesGroupe.get(nomVariable)).replaceAll(",id\\_[^,]+", ""))) {
+				blocWith.append("\n "
+						+ linkedIds.get(nomsVariablesGroupe.get(nomVariable)).replaceAll(",id\\_[^,]+", "") + " ");
+				alreadyAdded.add(linkedIds.get(nomsVariablesGroupe.get(nomVariable)).replaceAll(",id\\_[^,]+", ""));
 			}
 		}
-				
-		tableGroups.keySet().stream().forEach(k -> 	{
-				blocTrigger.append("\n , ")
+		
+		tableGroups.keySet().stream().forEach(k -> 	blocWith.append("\n , ")
 				.append(checkIsNotNull(tableGroups.get(k)))
-				.append(" as ").append(k);
-				}
+				.append(" as ").append(k)
 				);
-		tableNonGroup.keySet().stream().forEach(k -> 
-				blocTrigger.append("\n ,")
+		tableNonGroup.keySet().stream().forEach(k -> 	blocWith.append("\n ,")
 				.append(checkIsNotNull(tableNonGroup.get(k)))
 				.append(" as ").append(k)
 				);
 
-		insertionTriggers.put(idTableNGroupe(IDENTIFIANT_TRIGGER),blocTrigger.toString());
-		
-		returned
-		//.append(blocWith)
-		.append(blocSelect);
+		blocWith.append("\n FROM " + this.nomTableSource + " ) ");
+
+		returned.append(blocWith).append(blocSelect);
 
 		return returned;
 	}

arc-core/src/main/java/fr/insee/arc/core/service/p5mapping/thread/ThreadMappingService.java

@@ -150,15 +150,13 @@ private void execute() throws ArcException {
 		requeteMapping.construire();
 
 		ArcPreparedStatementBuilder query = new ArcPreparedStatementBuilder();
-		StringBuilder subQuery = new StringBuilder();
 
-		subQuery.append(requeteMapping.construireTableLienIdentifiants());
+		query.append(requeteMapping.construireTableLienIdentifiants());
 
 		/*
 		 * For each model table, compute a query that inserts the data into a temporary model table.
 		 */
-		requeteMapping.getQueriesThatInsertDataIntoTemporaryModelTable(idSource, subQuery);
-		query.append(subQuery);
+		query.append(requeteMapping.getQueriesThatInsertDataIntoTemporaryModelTable(idSource));
 
 		/*
 		 * Delete empty records if there is not link in children tables

arc-utils/src/main/java/fr/insee/arc/utils/utils/FormatSQL.java

@@ -66,18 +66,6 @@ public static GenericPreparedStatementBuilder dropTable(String... someTables) {
 		}
 		return query;
 	}
-	
-	/**
-	 * query to drop a view in database
-	 * 
-	 * @param tableName
-	 * @return
-	 */
-	public static GenericPreparedStatementBuilder dropView(String someView) {
-		GenericPreparedStatementBuilder query = new GenericPreparedStatementBuilder();
-			query.build(SQL.DROP, SQL.VIEW, SQL.IF_EXISTS, someView, SQL.CASCADE, SQL.END_QUERY, SQL.BR);
-		return query;
-	}
 
 	/**
 	 * query to retrieve