Merge pull request #440 from InseeFr/devZipOutputFolder

zip output folder and encrypt if true

Author: Hajarel-moukh

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/AbstractMainProcessingGenesis.java

@@ -43,6 +43,7 @@ public abstract class AbstractMainProcessingGenesis {
     protected UserInputsGenesis userInputs;
     protected final FileUtilsInterface fileUtilsInterface;
     protected Statement database;
+    @Getter
     protected final KraftwerkExecutionContext kraftwerkExecutionContext;
     /* SPECIFIC VARIABLES */
     @Getter

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessing.java

@@ -42,6 +42,7 @@ public class MainProcessing {
 	List<UserInputsFile> userInputsFileList; // for file by file process
 	@Getter
 	private VtlBindings vtlBindings = new VtlBindings();
+    @Getter
 	private final KraftwerkExecutionContext kraftwerkExecutionContext;
 	private final FileUtilsInterface fileUtilsInterface;
 
@@ -164,7 +165,7 @@ private void writeErrors() {
 
 	private static List<UserInputsFile> getUserInputsFile(UserInputsFile source, boolean fileByFile) throws KraftwerkException {
 		List<UserInputsFile> userInputsFileList = new ArrayList<>();
-		if(Boolean.TRUE.equals(fileByFile)){
+		if(fileByFile){
 			for (String dataMode : source.getModeInputsMap().keySet()) {
 				List<Path> dataFiles = getFilesToProcess(source, dataMode);
 				for (Path dataFile : dataFiles) {

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesisLegacy.java

@@ -1,29 +1,19 @@
 package fr.insee.kraftwerk.api.process;
 
-import fr.insee.bpm.metadata.model.Group;
-import fr.insee.bpm.metadata.model.MetadataModel;
 import fr.insee.kraftwerk.api.client.GenesisClient;
 import fr.insee.kraftwerk.api.configuration.ConfigProperties;
-import fr.insee.kraftwerk.core.data.model.InterrogationId;
 import fr.insee.kraftwerk.core.data.model.Mode;
-import fr.insee.kraftwerk.core.data.model.SurveyUnitUpdateLatest;
 import fr.insee.kraftwerk.core.exceptions.KraftwerkException;
 import fr.insee.kraftwerk.core.utils.KraftwerkExecutionContext;
 import fr.insee.kraftwerk.core.utils.SqlUtils;
 import fr.insee.kraftwerk.core.utils.files.FileUtilsInterface;
-import fr.insee.kraftwerk.core.vtl.VtlBindings;
 import lombok.extern.log4j.Log4j2;
 
 import java.io.IOException;
 import java.nio.file.Path;
 import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 
 @Log4j2
 public class MainProcessingGenesisLegacy extends AbstractMainProcessingGenesis{

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/process/MainProcessingGenesisNew.java

@@ -23,7 +23,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.sql.Connection;
@@ -52,16 +51,16 @@ public MainProcessingGenesisNew(
         super(config,genesisClient,fileUtilsInterface,kraftwerkExecutionContext);
     }
 
-    public void runMain(String questionnaireModelId, int batchSize, Mode dataMode) throws KraftwerkException, IOException {
+    public void runMain(String collectionInstrumentId, int batchSize, Mode dataMode) throws KraftwerkException, IOException {
         log.info("Batch size of interrogations retrieved from Genesis: {}", batchSize);
         String databasePath = ("%s/kraftwerk_temp/%s/db.duckdb".formatted(System.getProperty(JAVA_TMPDIR_PROPERTY),
-                questionnaireModelId));
+                collectionInstrumentId));
         //We delete database at start (in case there is already one)
         SqlUtils.deleteDatabaseFile(databasePath);
-        log.info("Kraftwerk main service started for questionnaire: {} {}", questionnaireModelId, kraftwerkExecutionContext.isWithDDI()
+        log.info("Kraftwerk main service started for questionnaire: {} {}", collectionInstrumentId, kraftwerkExecutionContext.isWithDDI()
                 ? "with DDI": "without DDI");
-        List<Mode> modes = client.getModesByQuestionnaire(questionnaireModelId);
-        init(questionnaireModelId, modes);
+        List<Mode> modes = client.getModesByQuestionnaire(collectionInstrumentId);
+        init(collectionInstrumentId, modes);
         //Try with resources to close database when done
         try (Connection tryDatabase = config.isDuckDbInMemory() ?
                 SqlUtils.openConnection()
@@ -70,7 +69,7 @@ public void runMain(String questionnaireModelId, int batchSize, Mode dataMode) t
                 throw new KraftwerkException(500,"Error during internal database creation");
             }
             this.database = tryDatabase.createStatement();
-            processDataByBatch(questionnaireModelId, batchSize, dataMode);
+            processDataByBatch(collectionInstrumentId, batchSize, dataMode);
             outputFileWriter();
             writeErrors();
             if (!database.isClosed()){database.close();}
@@ -271,12 +270,7 @@ private void moveTempFile(String filename, Path tmpOutputFile) throws KraftwerkE
             log.error("Permission refused to create folder: {} : {}", outDirectory.getParent(), e);
         }
         Path outputPath = outDirectory.resolve(filename);
-        //Encrypt file if requested
-        if(kraftwerkExecutionContext.isWithEncryption()) {
-            InputStream encryptedStream = encryptionUtils.encryptOutputFile(tmpOutputFile, kraftwerkExecutionContext);
-            fileUtilsInterface.writeFile(filename, encryptedStream, true);
-            log.info("File: {} successfully written and encrypted", filename);
-        }
+        kraftwerkExecutionContext.setOutDirectory(outDirectory);
         fileUtilsInterface.moveFile(tmpOutputFile,outputPath.toString());
         log.info("File: {} successfully written", outputPath);
     }
@@ -294,9 +288,7 @@ public String outputFileName(String questionnaireModelId) {
         DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss");
         String timestamp = LocalDateTime.now().format(formatter);
         String fileName = String.format("export_%s_%s", questionnaireModelId, timestamp);
-        return kraftwerkExecutionContext.isWithEncryption() ?
-                fileName + JSON_EXTENSION + ".enc"
-                : fileName + JSON_EXTENSION;
+        return fileName + JSON_EXTENSION;
     }
 
     protected void tmpJsonFileWriter(List<InterrogationId> listIds,

kraftwerk-api/src/main/java/fr/insee/kraftwerk/api/services/MainService.java

@@ -203,17 +203,17 @@ public ResponseEntity<Object> jsonExtraction(
         boolean withDDI = true;
         boolean withEncryption = false;
 
-        MainProcessingGenesisNew mpGenesis = getMainProcessingGenesisByQuestionnaire(withDDI, withEncryption, fileUtilsInterface);
-        try {
-            mpGenesis.runMainJson(collectionInstrumentId, batchSize, dataMode, since);
+		MainProcessingGenesisNew mpGenesis = getMainProcessingGenesisByQuestionnaire(withDDI, withEncryption, fileUtilsInterface);
+		try {
+			mpGenesis.runMainJson(collectionInstrumentId, batchSize, dataMode, since);
             log.info("Data extracted");
-        } catch (KraftwerkException e) {
-            return ResponseEntity.status(e.getStatus()).body(e.getMessage());
-        } catch (IOException e) {
-            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(e.getMessage());
-        }
-        return ResponseEntity.ok(String.format("Data extracted for questionnaireModelId %s",collectionInstrumentId));
-    }
+		} catch (KraftwerkException e) {
+			return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+		} catch (IOException e) {
+			return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(e.getMessage());
+		}
+		return ResponseEntity.ok(String.format("Data extracted for collectionInstrumentId %s",collectionInstrumentId));
+	}
 
     @GetMapping(value ="/json/replay")
     @Operation(operationId = "jsonExtractionReplay",
@@ -250,6 +250,7 @@ public ResponseEntity<Object> jsonExtractionReplay(
         return ResponseEntity.ok(String.format("Data extracted for collectionInstrumentId %s",collectionInstrumentId));
     }
 
+
     @NotNull
     MainProcessingGenesisLegacy getMainProcessingGenesis(boolean withDDI, boolean withEncryption, FileUtilsInterface fileUtilsInterface) {