-
Notifications
You must be signed in to change notification settings - Fork 1
Add Endpoint to importCase from s3 directrory #143
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 27 commits
ca19182
2665573
d6342f1
6581bae
045c9af
6de831f
d411a5c
773789a
9477539
e2ae114
4251db0
f1eec67
95a6344
944dea0
3fcade6
30937fa
3ec46cb
f41f454
f7df72f
8a5dd1b
124f4e6
377e51a
1a1c065
9ae6fa2
3842a7c
3d3f244
f199c41
3b199ab
ac4d82e
bfb1e2b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /** | ||
| * Copyright (c) 2026, RTE (http://www.rte-france.com) | ||
| * This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this | ||
| * file, You can obtain one at http://mozilla.org/MPL/2.0/. | ||
| */ | ||
|
|
||
| package com.powsybl.caseserver.datasource.utils; | ||
|
|
||
| import org.springframework.web.multipart.MultipartFile; | ||
|
|
||
| import java.io.Closeable; | ||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.Paths; | ||
| import java.nio.file.StandardCopyOption; | ||
| import java.nio.file.attribute.FileAttribute; | ||
| import java.nio.file.attribute.PosixFilePermission; | ||
| import java.nio.file.attribute.PosixFilePermissions; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * @author Bassel El Cheikh <bassel.el-cheikh_externe at rte-france.com> | ||
| */ | ||
|
|
||
| public class TmpMultiPartFile implements MultipartFile, Closeable { | ||
|
|
||
| private final String name; | ||
| private final String contentType; | ||
| private Path tempFile; | ||
| private long size; | ||
|
|
||
| public TmpMultiPartFile(InputStream inputStream, String caseKey, String contentType) throws IOException { | ||
| Paths.get(caseKey); | ||
| this.name = Path.of(caseKey).getFileName().toString(); | ||
| this.contentType = contentType; | ||
| init(inputStream); | ||
| } | ||
|
|
||
| private void init(InputStream inputStream) throws IOException { | ||
| FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); | ||
| this.tempFile = Files.createTempFile("s3-import-", null, attr); | ||
| Files.copy(inputStream, this.tempFile, StandardCopyOption.REPLACE_EXISTING); | ||
| this.size = Files.size(this.tempFile); | ||
| } | ||
|
|
||
| @Override | ||
| public String getName() { | ||
| return name; | ||
| } | ||
|
|
||
| @Override | ||
| public String getOriginalFilename() { | ||
| return getName(); | ||
| } | ||
|
|
||
| @Override | ||
| public String getContentType() { | ||
| return contentType; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isEmpty() { | ||
| return getSize() == 0; | ||
| } | ||
|
|
||
| @Override | ||
| public long getSize() { | ||
| return size; | ||
| } | ||
|
|
||
| @Override | ||
| public byte[] getBytes() throws IOException { | ||
| throw new UnsupportedOperationException("Not supported."); | ||
| } | ||
|
|
||
| @Override | ||
| public InputStream getInputStream() throws IOException { | ||
| return Files.newInputStream(tempFile); | ||
| } | ||
|
|
||
| @Override | ||
| public void transferTo(File dest) throws IOException, IllegalStateException { | ||
| transferTo(dest.toPath()); | ||
| } | ||
|
|
||
| @Override | ||
| public void close() throws IOException { | ||
| if (tempFile != null) { | ||
| Files.deleteIfExists(tempFile); | ||
| tempFile = null; | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,11 +9,13 @@ | |
| import com.fasterxml.jackson.core.type.TypeReference; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.powsybl.caseserver.ContextConfigurationWithTestChannel; | ||
| import com.powsybl.caseserver.datasource.utils.TmpMultiPartFile; | ||
| import com.powsybl.caseserver.dto.CaseInfos; | ||
| import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser; | ||
| import com.powsybl.caseserver.repository.CaseMetadataEntity; | ||
| import com.powsybl.caseserver.repository.CaseMetadataRepository; | ||
| import com.powsybl.computation.ComputationManager; | ||
| import org.junit.jupiter.api.Assertions; | ||
| import org.junit.jupiter.api.BeforeEach; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.mockito.Mockito; | ||
|
|
@@ -35,6 +37,7 @@ | |
| import software.amazon.awssdk.services.s3.model.PutObjectRequest; | ||
|
|
||
| import java.io.ByteArrayOutputStream; | ||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.time.Instant; | ||
|
|
@@ -45,6 +48,8 @@ | |
| import java.util.UUID; | ||
| import java.util.zip.GZIPOutputStream; | ||
|
|
||
| import static com.powsybl.caseserver.Utils.ZIP_EXTENSION; | ||
| import static com.powsybl.caseserver.service.CaseService.DELIMITER; | ||
| import static org.hamcrest.Matchers.hasSize; | ||
| import static org.hamcrest.Matchers.startsWith; | ||
| import static org.junit.Assert.assertEquals; | ||
|
|
@@ -841,4 +846,75 @@ void testDuplicate() throws Exception { | |
| assertThrows(ResponseStatusException.class, () -> caseService.duplicateCase(firstCaseUuid, false)); | ||
| assertNotNull(outputDestination.receive(1000, caseImportDestination)); | ||
| } | ||
|
|
||
| void addZipCaseFile(UUID caseUuid, String folderName, String fileName) throws IOException { | ||
| try (InputStream inputStream = CaseControllerTest.class.getResourceAsStream("/" + fileName + ZIP_EXTENSION)) { | ||
| if (inputStream != null) { | ||
| RequestBody requestBody = RequestBody.fromBytes(inputStream.readAllBytes()); | ||
| PutObjectRequest putObjectRequest = PutObjectRequest.builder() | ||
| .bucket(caseService.getBucketName()) | ||
| .key(folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION) | ||
| .contentType("application/zip") | ||
| .build(); | ||
| caseService.getS3Client().putObject(putObjectRequest, requestBody); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| void testCreateCase() throws Exception { | ||
|
|
||
| UUID caseUuid = UUID.randomUUID(); | ||
| String folderName = "network_exports"; | ||
| String fileName = "zippedTestCase"; | ||
|
|
||
| // create zip case in one folder in bucket | ||
| addZipCaseFile(caseUuid, folderName, fileName); | ||
|
|
||
| mvc.perform(post("/v1/cases/create") | ||
| .param("caseKey", folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION) | ||
| .param("contentType", "application/zip")) | ||
| .andExpect(status().isOk()); | ||
|
|
||
| assertNotNull(outputDestination.receive(1000, caseImportDestination)); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You can check content of notification, THEN getObjet from s3 then check file size from resource and file size from s3 |
||
| } | ||
|
|
||
| @Test | ||
| void testCreateCaseKo() throws Exception { | ||
|
|
||
| UUID caseUuid = UUID.randomUUID(); | ||
| String folderName = "network_exports"; | ||
| String fileName = "testCase4"; | ||
|
|
||
| mvc.perform(post("/v1/cases/create") | ||
| .param("caseKey", folderName + DELIMITER + caseUuid + DELIMITER + fileName) | ||
| .param("contentType", "application/zip")) | ||
| .andExpect(status().isInternalServerError()); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should detail which message error expected and why for example, bad folder name |
||
| } | ||
|
|
||
| @Test | ||
| void testS3MultiPartFile() throws IOException { | ||
| UUID caseUuid = UUID.randomUUID(); | ||
| String folderName = "network_exports"; | ||
| String fileName = "zippedTestCase"; | ||
|
|
||
| // create zip case in one folder in bucket | ||
| addZipCaseFile(caseUuid, folderName, fileName); | ||
|
|
||
| String caseKey = folderName + DELIMITER + caseUuid + DELIMITER + fileName + ZIP_EXTENSION; | ||
| InputStream inputStream = caseService.getCaseStream(caseKey).get(); | ||
| try (TmpMultiPartFile file = new TmpMultiPartFile(inputStream, caseKey, "application/zip")) { | ||
| try (InputStream in = CaseControllerTest.class.getResourceAsStream("/" + fileName + ZIP_EXTENSION)) { | ||
| assertNotNull(in); | ||
| byte[] bytes = in.readAllBytes(); | ||
| Assertions.assertEquals(bytes.length, file.getSize()); | ||
| Assertions.assertEquals("application/zip", file.getContentType()); | ||
| assertFalse(file.isEmpty()); | ||
| File tmpFile = new File("/tmp/testFile.zip"); | ||
| file.transferTo(tmpFile); | ||
| assertTrue(tmpFile.exists()); | ||
| tmpFile.delete(); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should be named as testCase.zip ?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No! If i do that another test will fail in another test file.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, there's a weird behavior with datasources. |
Uh oh!
There was an error while loading. Please reload this page.