Skip to content
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
ca19182
Add Endpoint to importCase from s3 directrory
basseche Jan 7, 2026
2665573
fix checkstyle issus
basseche Jan 8, 2026
d6342f1
add copyright
basseche Jan 8, 2026
6581bae
improvements
basseche Jan 8, 2026
045c9af
update param list
basseche Jan 13, 2026
6de831f
Improve memory usage by S3MultiPartFile
basseche Jan 13, 2026
d411a5c
Create temporary file in s3MultiPartFile instead of reading each time…
basseche Jan 14, 2026
773789a
Add test for new Endpoint
basseche Jan 14, 2026
9477539
Merge branch 'main' into Import_Case_From_S3
basseche Jan 14, 2026
e2ae114
add tests
basseche Jan 14, 2026
4251db0
Fix tests
basseche Jan 14, 2026
f1eec67
Increase tests coverage and resolve hotspot security issue
basseche Jan 14, 2026
95a6344
Increase tests coverage 2
basseche Jan 14, 2026
944dea0
Update src/main/java/com/powsybl/caseserver/datasource/utils/S3MultiP…
basseche Jan 15, 2026
3fcade6
Update src/main/java/com/powsybl/caseserver/datasource/utils/S3MultiP…
basseche Jan 16, 2026
30937fa
Review
basseche Jan 16, 2026
3ec46cb
checkstyle
basseche Jan 16, 2026
f41f454
Fix test
basseche Jan 16, 2026
f7df72f
fix tests 2
basseche Jan 16, 2026
8a5dd1b
Review
basseche Jan 21, 2026
124f4e6
Merge branch 'main' into Import_Case_From_S3
basseche Jan 21, 2026
377e51a
changes on S3MultiPartFile
basseche Jan 22, 2026
1a1c065
Update src/main/java/com/powsybl/caseserver/CaseController.java
basseche Jan 22, 2026
9ae6fa2
Update src/main/java/com/powsybl/caseserver/datasource/utils/S3MultiP…
basseche Jan 22, 2026
3842a7c
review
basseche Jan 22, 2026
3d3f244
fix
basseche Jan 22, 2026
f199c41
review
basseche Jan 22, 2026
3b199ab
Add observer
basseche Jan 23, 2026
ac4d82e
modify endpoint
basseche Jan 23, 2026
bfb1e2b
Update src/main/java/com/powsybl/caseserver/CaseController.java
basseche Jan 23, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions src/main/java/com/powsybl/caseserver/CaseController.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,21 @@
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
import java.util.UUID;

import static com.powsybl.caseserver.CaseException.createDirectoryNotFound;
import static com.powsybl.caseserver.Utils.buildHeaders;

/**
* @author Abdelsalem Hedhili <abdelsalem.hedhili at rte-france.com>
* @author Franck Lecuyer <franck.lecuyer at rte-france.com>
Expand Down Expand Up @@ -154,6 +157,26 @@ public ResponseEntity<UUID> duplicateCase(
return ResponseEntity.ok().body(newCaseUuid);
}

@PostMapping(value = "/cases/create", params = {"caseKey", "contentType"})
@Operation(summary = "create a case from converted one stored in folder in s3")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Case created"),
@ApiResponse(responseCode = "404", description = "Source case not found"),
@ApiResponse(responseCode = "500", description = "An error occurred during the case file creation")})
public ResponseEntity<UUID> importCaseFromS3Key(
@RequestParam("caseKey") String caseFolderKey,
@RequestParam("contentType") String contentType,
@RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration,
@RequestParam(value = "withIndexation", required = false, defaultValue = "false") boolean withIndexation) {

try {
UUID uuid = caseService.importCase(caseFolderKey, contentType, withExpiration, withIndexation);
return ResponseEntity.ok().body(uuid);
} catch (IOException e) {
LOGGER.error("Failed to create case from S3 for caseFolderKey: {}", caseFolderKey, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
}
}

@PutMapping(value = "/cases/{caseUuid}/disableExpiration")
@Operation(summary = "disable the case expiration")
@ApiResponses(value = {
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/powsybl/caseserver/Utils.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ private Utils() {
public static final String GZIP_EXTENSION = ".gz";
public static final String GZIP_FORMAT = "gz";
public static final String GZIP_ENCODING = "gzip";
public static final String ZIP_EXTENSION = ".zip";
public static final List<String> COMPRESSION_FORMATS = List.of("bz2", GZIP_FORMAT, "xz", "zst");
public static final List<String> ARCHIVE_FORMATS = List.of("zip", "tar");
public static final String NOT_FOUND = " not found";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/**
* Copyright (c) 2026, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

package com.powsybl.caseserver.datasource.utils;

import org.springframework.web.multipart.MultipartFile;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Set;

/**
* @author Bassel El Cheikh <bassel.el-cheikh_externe at rte-france.com>
*/

public class TmpMultiPartFile implements MultipartFile, Closeable {

private final String name;
private final String contentType;
private Path tempFile;
private long size;

public TmpMultiPartFile(InputStream inputStream, String caseKey, String contentType) throws IOException {
Paths.get(caseKey);
this.name = Path.of(caseKey).getFileName().toString();
this.contentType = contentType;
init(inputStream);
}

private void init(InputStream inputStream) throws IOException {
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
this.tempFile = Files.createTempFile("s3-import-", null, attr);
Files.copy(inputStream, this.tempFile, StandardCopyOption.REPLACE_EXISTING);
this.size = Files.size(this.tempFile);
}

@Override
public String getName() {
return name;
}

@Override
public String getOriginalFilename() {
return getName();
}

@Override
public String getContentType() {
return contentType;
}

@Override
public boolean isEmpty() {
return getSize() == 0;
}

@Override
public long getSize() {
return size;
}

@Override
public byte[] getBytes() throws IOException {
throw new UnsupportedOperationException("Not supported.");
}

@Override
public InputStream getInputStream() throws IOException {
return Files.newInputStream(tempFile);
}

@Override
public void transferTo(File dest) throws IOException, IllegalStateException {
transferTo(dest.toPath());
}

@Override
public void close() throws IOException {
if (tempFile != null) {
Files.deleteIfExists(tempFile);
tempFile = null;
}
}
}
64 changes: 35 additions & 29 deletions src/main/java/com/powsybl/caseserver/service/CaseService.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import com.google.re2j.Pattern;
import com.powsybl.caseserver.CaseException;
import com.powsybl.caseserver.datasource.utils.TmpMultiPartFile;
import com.powsybl.caseserver.dto.CaseInfos;
import com.powsybl.caseserver.elasticsearch.CaseInfosService;
import com.powsybl.caseserver.parsers.FileNameInfos;
Expand All @@ -21,6 +22,7 @@
import com.powsybl.iidm.network.Importer;
import com.powsybl.ws.commons.SecuredTarInputStream;
import com.powsybl.ws.commons.SecuredZipInputStream;
import lombok.Getter;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.utils.FileNameUtils;
Expand Down Expand Up @@ -73,8 +75,10 @@ public class CaseService {
public static final int MAX_ARCHIVE_ENTRIES = 1000;
public static final String DELIMITER = "/";

@Getter
private ComputationManager computationManager = LocalComputationManager.getDefault();

@Getter
@Autowired
private final CaseMetadataRepository caseMetadataRepository;

Expand Down Expand Up @@ -263,6 +267,30 @@ private UUID parseUuidFromKey(String key) {
return UUID.fromString(keyWithoutRootDirectory.substring(0, firstSlash));
}

public Optional<InputStream> getCaseStream(UUID caseUuid) {
try {
return getCaseStream(uuidToKeyWithOriginalFileName(caseUuid));
} catch (CaseException | ResponseStatusException e) {
LOGGER.error(e.getMessage());
return Optional.empty();
}
}

public Optional<InputStream> getCaseStream(String caseFileKey) {
try {
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(caseFileKey)
.build();

ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(getObjectRequest);
return Optional.of(responseInputStream);
} catch (NoSuchKeyException e) {
LOGGER.error("The expected key does not exist in the bucket s3 : {}", caseFileKey);
return Optional.empty();
}
}

private String parseFilenameFromKey(String key) {
String keyWithoutRootDirectory = key.replaceAll(rootDirectory + DELIMITER, "");
int firstSlash = keyWithoutRootDirectory.indexOf(DELIMITER);
Expand Down Expand Up @@ -318,26 +346,6 @@ public String getCaseName(UUID caseUuid) {
return originalFilename;
}

public Optional<InputStream> getCaseStream(UUID caseUuid) {
String caseFileKey = null;
try {
caseFileKey = uuidToKeyWithOriginalFileName(caseUuid);
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(caseFileKey)
.build();

ResponseInputStream<GetObjectResponse> responseInputStream = s3Client.getObject(getObjectRequest);
return Optional.of(responseInputStream);
} catch (NoSuchKeyException e) {
LOGGER.error("The expected key does not exist in the bucket s3 : {}", caseFileKey);
return Optional.empty();
} catch (CaseException | ResponseStatusException e) {
LOGGER.error(e.getMessage());
return Optional.empty();
}
}

public List<CaseInfos> getCases() {
List<CaseInfos> caseInfosList = new ArrayList<>();
CaseInfos caseInfos;
Expand Down Expand Up @@ -456,7 +464,6 @@ public Set<String> listName(UUID caseUuid, String regex) {
public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation, UUID caseUuid) {
String caseName = Objects.requireNonNull(mpf.getOriginalFilename());
validateCaseName(caseName);

String format = withTempCopy(caseUuid, caseName, mpf::transferTo, this::getFormat);
String compressionFormat = FileNameUtils.getExtension(Paths.get(caseName));

Expand Down Expand Up @@ -499,6 +506,13 @@ public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIn
return caseUuid;
}

public UUID importCase(String caseKey, String contentType, boolean withExpiration, boolean withIndexation) throws IOException {
InputStream inputStream = getCaseStream(caseKey).orElseThrow(() -> new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "The expected key does not exist in the bucket s3 : " + caseKey));
try (TmpMultiPartFile mpf = new TmpMultiPartFile(inputStream, caseKey, contentType)) {
return importCase(mpf, withExpiration, withIndexation, UUID.randomUUID());
}
}

private void compressAndUploadToS3(UUID caseUuid, String fileName, String contentType, InputStream inputStream) {
withTempCopy(
caseUuid,
Expand Down Expand Up @@ -660,12 +674,4 @@ public void setComputationManager(ComputationManager computationManager) {
this.computationManager = Objects.requireNonNull(computationManager);
}

public ComputationManager getComputationManager() {
return computationManager;
}

public CaseMetadataRepository getCaseMetadataRepository() {
return caseMetadataRepository;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.powsybl.caseserver.ContextConfigurationWithTestChannel;
import com.powsybl.caseserver.datasource.utils.TmpMultiPartFile;
import com.powsybl.caseserver.dto.CaseInfos;
import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser;
import com.powsybl.caseserver.repository.CaseMetadataEntity;
import com.powsybl.caseserver.repository.CaseMetadataRepository;
import com.powsybl.computation.ComputationManager;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
Expand All @@ -35,6 +37,7 @@
import software.amazon.awssdk.services.s3.model.PutObjectRequest;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
Expand All @@ -45,6 +48,8 @@
import java.util.UUID;
import java.util.zip.GZIPOutputStream;

import static com.powsybl.caseserver.Utils.ZIP_EXTENSION;
import static com.powsybl.caseserver.service.CaseService.DELIMITER;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -841,4 +846,75 @@ void testDuplicate() throws Exception {
assertThrows(ResponseStatusException.class, () -> caseService.duplicateCase(firstCaseUuid, false));
assertNotNull(outputDestination.receive(1000, caseImportDestination));
}

void addZipCaseFile(UUID caseUuid, String folderName, String fileName) throws IOException {
try (InputStream inputStream = CaseControllerTest.class.getResourceAsStream("/" + fileName + ZIP_EXTENSION)) {
if (inputStream != null) {
RequestBody requestBody = RequestBody.fromBytes(inputStream.readAllBytes());
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(caseService.getBucketName())
.key(folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION)
.contentType("application/zip")
.build();
caseService.getS3Client().putObject(putObjectRequest, requestBody);
}
}
}

@Test
void testCreateCase() throws Exception {

UUID caseUuid = UUID.randomUUID();
String folderName = "network_exports";
String fileName = "zippedTestCase";

// create zip case in one folder in bucket
addZipCaseFile(caseUuid, folderName, fileName);

mvc.perform(post("/v1/cases/create")
.param("caseKey", folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION)
.param("contentType", "application/zip"))
.andExpect(status().isOk());

assertNotNull(outputDestination.receive(1000, caseImportDestination));
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can check content of notification,

THEN getObjet from s3 then check file size from resource and file size from s3

}

@Test
void testCreateCaseKo() throws Exception {

UUID caseUuid = UUID.randomUUID();
String folderName = "network_exports";
String fileName = "testCase4";

mvc.perform(post("/v1/cases/create")
.param("caseKey", folderName + DELIMITER + caseUuid + DELIMITER + fileName)
.param("contentType", "application/zip"))
.andExpect(status().isInternalServerError());
Copy link

@thangqp thangqp Jan 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should detail which message error expected and why for example, bad folder name

}

@Test
void testS3MultiPartFile() throws IOException {
UUID caseUuid = UUID.randomUUID();
String folderName = "network_exports";
String fileName = "zippedTestCase";

// create zip case in one folder in bucket
addZipCaseFile(caseUuid, folderName, fileName);

String caseKey = folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION;
InputStream inputStream = caseService.getCaseStream(caseKey).get();
try (TmpMultiPartFile file = new TmpMultiPartFile(inputStream, caseKey, "application/zip")) {
try (InputStream in = CaseControllerTest.class.getResourceAsStream("/" + fileName + ZIP_EXTENSION)) {
assertNotNull(in);
byte[] bytes = in.readAllBytes();
Assertions.assertEquals(bytes.length, file.getSize());
Assertions.assertEquals("application/zip", file.getContentType());
assertFalse(file.isEmpty());
File tmpFile = new File("/tmp/testFile.zip");
file.transferTo(tmpFile);
assertTrue(tmpFile.exists());
tmpFile.delete();
}
}
}
}
Binary file added src/test/resources/zippedTestCase.zip
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should be named as testCase.zip ?

Copy link
Contributor Author

@basseche basseche Jan 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No! If i do that another test will fail in another test file.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, there's a weird behavior with datasources.
But test(2)Case.xiidm was to test a specific bug.
Name it zippedTestCase.zip and it will work

Binary file not shown.