diff --git a/src/integration-test/resources/archives/package.zip b/src/integration-test/resources/archives/package.zip new file mode 100644 index 0000000000..5c9cbbf887 Binary files /dev/null and b/src/integration-test/resources/archives/package.zip differ diff --git a/src/main/java/org/ow2/proactive/catalog/rest/controller/CatalogObjectController.java b/src/main/java/org/ow2/proactive/catalog/rest/controller/CatalogObjectController.java index 5ec7e9e478..2850d70c59 100644 --- a/src/main/java/org/ow2/proactive/catalog/rest/controller/CatalogObjectController.java +++ b/src/main/java/org/ow2/proactive/catalog/rest/controller/CatalogObjectController.java @@ -408,7 +408,6 @@ public ResponseEntity> list( @Parameter(description = "Include only objects whose last commit belong to the given user.") @RequestParam(value = "lastCommitBy", required = false) Optional lastCommitBy, @Parameter(description = "Include only objects whose last commit time is greater than the given EPOCH time.") @RequestParam(value = "lastCommitTimeGreater", required = false) Optional lastCommitTimeGreater, @Parameter(description = "Include only objects whose last commit time is less than the given EPOCH time.") @RequestParam(value = "lastCommitTimeLessThan", required = false) Optional lastCommitTimeLessThan, - @Parameter(description = "Give a list of name separated by comma to get them in an archive", array = @ArraySchema(schema = @Schema())) @RequestParam(value = "listObjectNamesForArchive", required = false) Optional> names, @Parameter(description = "Page number", required = false) @RequestParam(defaultValue = "0", value = "pageNo") int pageNo, @Parameter(description = "Page size", required = false) @RequestParam(defaultValue = MAXVALUE + "", value = "pageSize") int pageSize, @@ -428,74 +427,69 @@ public ResponseEntity> list( objectTagFilter = objectTagFilter.filter(s -> !s.isEmpty()); projectNameFilter = projectNameFilter.filter(s -> !s.isEmpty()); lastCommitBy = lastCommitBy.filter(s -> !s.isEmpty()); - if (names.isPresent()) { - ZipArchiveContent content = catalogObjectService.getCatalogObjectsAsZipArchive(bucketName, names.get()); - return getResponseAsArchive(content, response); - } else { - List metadataList = catalogObjectService.listCatalogObjects(Collections.singletonList(bucketName), - kind, - contentType, - objectNameFilter, - objectTagFilter, - projectNameFilter, - lastCommitBy, - lastCommitTimeGreater, - lastCommitTimeLessThan, - pageNo, - pageSize); - - if (sessionIdRequired && !isPublicBucket.get()) { - // remove all objects that the user shouldn't have access according to the grants specification. - GrantRightsService.removeInaccessibleObjectsInBucket(metadataList, bucketGrants, catalogObjectsGrants); - } + List metadataList = catalogObjectService.listCatalogObjects(Collections.singletonList(bucketName), + kind, + contentType, + objectNameFilter, + objectTagFilter, + projectNameFilter, + lastCommitBy, + lastCommitTimeGreater, + lastCommitTimeLessThan, + pageNo, + pageSize); + + if (sessionIdRequired && !isPublicBucket.get()) { + // remove all objects that the user shouldn't have access according to the grants specification. + GrantRightsService.removeInaccessibleObjectsInBucket(metadataList, bucketGrants, catalogObjectsGrants); + } - Optional userSpecificBucketRights = GrantHelper.filterFirstUserSpecificGrant(bucketGrants) - .map(BucketGrantMetadata::getAccessType); - for (CatalogObjectMetadata catalogObject : metadataList) { - catalogObject.add(LinkUtil.createLink(bucketName, catalogObject.getName())); - catalogObject.add(LinkUtil.createRelativeLink(bucketName, catalogObject.getName())); - if (sessionIdRequired) { - List objectsGrants = GrantHelper.filterObjectGrants(catalogObjectsGrants, - catalogObject.getName()); - catalogObject.setRights(GrantRightsService.getCatalogObjectRights(isPublicBucket.get(), - bucketRights.toString(), - userSpecificBucketRights, - objectsGrants)); - } - } - Optional associatedObjectsByBucketOptional = Optional.empty(); + Optional userSpecificBucketRights = GrantHelper.filterFirstUserSpecificGrant(bucketGrants) + .map(BucketGrantMetadata::getAccessType); + for (CatalogObjectMetadata catalogObject : metadataList) { + catalogObject.add(LinkUtil.createLink(bucketName, catalogObject.getName())); + catalogObject.add(LinkUtil.createRelativeLink(bucketName, catalogObject.getName())); if (sessionIdRequired) { - List associatedObjectsByBucketList = jobPlannerService.getAssociatedObjects(sessionId); - associatedObjectsByBucketOptional = associatedObjectsByBucketList.stream() - .filter(object -> bucketName.equals(object.getBucketName())) - .findFirst(); - if (associatedObjectsByBucketOptional.isPresent()) { - AssociatedObjectsByBucket associatedObjects = associatedObjectsByBucketOptional.get(); - for (CatalogObjectMetadata catalogObject : metadataList) { - AssociatedObject associatedObject = associatedObjects.findAssociatedObject(catalogObject.getName()); - addAssociationStatus(catalogObject, associatedObject); - } + List objectsGrants = GrantHelper.filterObjectGrants(catalogObjectsGrants, + catalogObject.getName()); + catalogObject.setRights(GrantRightsService.getCatalogObjectRights(isPublicBucket.get(), + bucketRights.toString(), + userSpecificBucketRights, + objectsGrants)); + } + } + Optional associatedObjectsByBucketOptional = Optional.empty(); + if (sessionIdRequired) { + List associatedObjectsByBucketList = jobPlannerService.getAssociatedObjects(sessionId); + associatedObjectsByBucketOptional = associatedObjectsByBucketList.stream() + .filter(object -> bucketName.equals(object.getBucketName())) + .findFirst(); + if (associatedObjectsByBucketOptional.isPresent()) { + AssociatedObjectsByBucket associatedObjects = associatedObjectsByBucketOptional.get(); + for (CatalogObjectMetadata catalogObject : metadataList) { + AssociatedObject associatedObject = associatedObjects.findAssociatedObject(catalogObject.getName()); + addAssociationStatus(catalogObject, associatedObject); } } - if (sessionIdRequired && associationStatusFilter.isPresent()) { - if (!associatedObjectsByBucketOptional.isPresent()) { - if (!UNPLANNED.equalsIgnoreCase(associationStatusFilter.get())) { - return ResponseEntity.ok(Collections.emptyList()); - } - // if UNPLANNED and no objects are associated, we return the full list - } else { - AssociatedObjectsByBucket associatedObjectsByBucket = associatedObjectsByBucketOptional.get(); - - metadataList = metadataList.stream() - .filter(metadata -> isAssociatedInJobPlanner(metadata, - associationStatusFilter.get(), - associatedObjectsByBucket)) - .collect(Collectors.toList()); + } + if (sessionIdRequired && associationStatusFilter.isPresent()) { + if (!associatedObjectsByBucketOptional.isPresent()) { + if (!UNPLANNED.equalsIgnoreCase(associationStatusFilter.get())) { + return ResponseEntity.ok(Collections.emptyList()); } + // if UNPLANNED and no objects are associated, we return the full list + } else { + AssociatedObjectsByBucket associatedObjectsByBucket = associatedObjectsByBucketOptional.get(); + + metadataList = metadataList.stream() + .filter(metadata -> isAssociatedInJobPlanner(metadata, + associationStatusFilter.get(), + associatedObjectsByBucket)) + .collect(Collectors.toList()); } - Collections.sort(metadataList); - return ResponseEntity.ok(metadataList); } + Collections.sort(metadataList); + return ResponseEntity.ok(metadataList); } @Operation(summary = "Export catalog objects as a ProActive Package", description = "Export catalog objects as a ProActive Package containing the selected objects or contents of the selected bucket along with a METADATA.json describing the exported objects.
Note: Returns catalog objects metadata associated to the latest revision.") @@ -504,9 +498,10 @@ public ResponseEntity> list( @ApiResponse(responseCode = "401", description = "User not authenticated"), @ApiResponse(responseCode = "403", description = "Permission denied") }) @RequestMapping(value = REQUEST_API_QUERY + "/export", method = GET) - public ResponseEntity> exportAsPackage( + public ResponseEntity> exportCatalogObjects( @Parameter(description = "sessionID") @RequestHeader(value = "sessionID", required = false) String sessionId, @PathVariable String bucketName, + @Parameter(description = "Plain zip instead of a Proactive package") @RequestParam(value = "isPlainZip", required = false, defaultValue = "false") boolean isPlainZip, @Parameter(description = "Give a list of name separated by comma to get them in an archive", array = @ArraySchema(schema = @Schema())) @RequestParam(value = "objectNamesList", required = false) Optional> names, HttpServletResponse response) throws NotAuthenticatedException, AccessDeniedException { @@ -518,7 +513,11 @@ public ResponseEntity> exportAsPackage( ZipArchiveContent content; if (names.isPresent()) { - content = catalogObjectService.getCatalogObjectsAsPackageZipArchive(bucketName, names.get()); + if (isPlainZip) { + content = catalogObjectService.getCatalogObjectsAsZipArchive(bucketName, names.get()); + } else { + content = catalogObjectService.getCatalogObjectsAsPackageZipArchive(bucketName, names.get()); + } } else { List metadataList = catalogObjectService.listCatalogObjects(Collections.singletonList(bucketName), Optional.empty(), @@ -539,11 +538,66 @@ public ResponseEntity> exportAsPackage( List objectNames = metadataList.stream() .map(CatalogObjectMetadata::getName) .collect(Collectors.toList()); - content = catalogObjectService.getCatalogObjectsAsPackageZipArchive(bucketName, objectNames); + if (isPlainZip) { + content = catalogObjectService.getCatalogObjectsAsZipArchive(bucketName, objectNames); + } else { + content = catalogObjectService.getCatalogObjectsAsPackageZipArchive(bucketName, objectNames); + } } return getResponseAsArchive(content, response); } + @Operation(summary = "Import a ProActive package") + @ApiResponses(value = { @ApiResponse(responseCode = "404", description = "Bucket not found"), + @ApiResponse(responseCode = "422", description = "Invalid file content supplied") }) + @RequestMapping(value = REQUEST_API_QUERY + + "/import", consumes = { MediaType.MULTIPART_FORM_DATA_VALUE }, method = POST) + @ResponseStatus(HttpStatus.CREATED) + public CatalogObjectMetadataList importAsPackage( + @Parameter(description = "sessionID", required = true) @RequestHeader(value = "sessionID") String sessionId, + @Parameter(description = "The name of the existing Bucket", required = true, schema = @Schema(pattern = BucketNameValidator.VALID_BUCKET_NAME_PATTERN)) @PathVariable String bucketName, + @Parameter(description = "Project of the package objects") @RequestParam(value = "projectName", required = false, defaultValue = "") Optional projectName, + @Parameter(description = "List of comma separated tags of the objects", schema = @Schema(pattern = TagsValidator.TAGS_PATTERN)) @RequestParam(value = "tags", required = false, defaultValue = "") Optional tags, + @Parameter(description = "The ProActive package zip file", required = true) @RequestPart(value = "file") MultipartFile file) + throws IOException, NotAuthenticatedException, AccessDeniedException { + + // Check Grants + AuthenticatedUser user = null; + String initiator = ANONYMOUS; + if (sessionIdRequired) { + // Check session validation + if (!restApiAccessService.isSessionActive(sessionId)) { + throw new AccessDeniedException("Session id is not active. Please login."); + } + user = restApiAccessService.getUserFromSessionId(sessionId); + if (!AccessTypeHelper.satisfy(grantRightsService.getBucketRights(user, bucketName), write)) { + throw new BucketGrantAccessException(bucketName); + } + } + + String userName = ""; + if (user != null) { + userName = user.getName(); + initiator = userName; + } + CatalogObjectMetadataList catalogObjectMetadataList; + + List catalogObjects = catalogObjectService.createCatalogObjectsFromPackage(bucketName, + userName, + projectName.orElse(""), + tags.orElse(""), + file.getBytes()); + + for (CatalogObjectMetadata catalogObject : catalogObjects) { + catalogObject.add(LinkUtil.createLink(bucketName, catalogObject.getName())); + catalogObject.add(LinkUtil.createRelativeLink(bucketName, catalogObject.getName())); + } + + catalogObjectMetadataList = new CatalogObjectMetadataList(catalogObjects); + log.info(ACTION + initiator + " created new catalog objects from archive inside bucket " + bucketName); + return catalogObjectMetadataList; + } + private void checkBucketGrants(String sessionId, String bucketName, List catalogObjectsGrants, List bucketGrants, StringBuilder bucketRights, AtomicBoolean isPublicBucket) { diff --git a/src/main/java/org/ow2/proactive/catalog/service/CatalogObjectService.java b/src/main/java/org/ow2/proactive/catalog/service/CatalogObjectService.java index a88a923b02..9eac6d587f 100644 --- a/src/main/java/org/ow2/proactive/catalog/service/CatalogObjectService.java +++ b/src/main/java/org/ow2/proactive/catalog/service/CatalogObjectService.java @@ -58,6 +58,7 @@ import org.ow2.proactive.catalog.util.ArchiveManagerHelper; import org.ow2.proactive.catalog.util.ArchiveManagerHelper.FileNameAndContent; import org.ow2.proactive.catalog.util.ArchiveManagerHelper.ZipArchiveContent; +import org.ow2.proactive.catalog.util.PackageMetadataJSONParser; import org.ow2.proactive.catalog.util.RevisionCommitMessageBuilder; import org.ow2.proactive.catalog.util.SeparatorUtility; import org.ow2.proactive.catalog.util.name.validator.KindAndContentTypeValidator; @@ -202,6 +203,47 @@ public List createCatalogObjects(String bucketName, Strin }).collect(Collectors.toList()); } + @Transactional + public List createCatalogObjectsFromPackage(String bucketName, String username, + String projectName, String tags, byte[] zipArchive) { + + PackageMetadataJSONParser.PackageData metadata = ArchiveManagerHelper.extractMetadataObject(bucketName, + zipArchive); + + List objectsList = new ArrayList<>(); + BucketEntity bucketEntity = findBucketByNameAndCheck(bucketName); + + metadata.getCatalog().getObjects().forEach(object -> { + byte[] objectFile = ArchiveManagerHelper.extractObjectByPath(zipArchive, object.getFile()); + CatalogObjectEntity catalogObject = catalogObjectRepository.findOne(new CatalogObjectEntity.CatalogObjectEntityKey(bucketEntity.getId(), + object.getName())); + CatalogObjectMetadata objectMetadata; + if (catalogObject == null) { + objectMetadata = this.createCatalogObject(bucketName, + object.getName(), + projectName, + tags, + object.getMetadata().getKind(), + object.getMetadata().getCommitMessage(), + username, + object.getMetadata().getContentType(), + Collections.emptyList(), + objectFile, + FilenameUtils.getExtension(object.getFile())); + } else { + objectMetadata = this.createCatalogObjectRevision(bucketName, + object.getName(), + projectName, + tags, + object.getMetadata().getCommitMessage(), + username, + objectFile); + } + objectsList.add(objectMetadata); + }); + return objectsList; + } + @Transactional public CatalogObjectMetadata createCatalogObject(String bucketName, String name, String projectName, String tags, String kind, String commitMessage, String username, String contentType, List metadataList, @@ -795,8 +837,7 @@ public List listCatalogObjectRevisions(String bucketName, } @Transactional(readOnly = true) - public CatalogObjectMetadata getCatalogObjectRevision(String bucketName, String name, long commitTime) - throws UnsupportedEncodingException { + public CatalogObjectMetadata getCatalogObjectRevision(String bucketName, String name, long commitTime) { CatalogObjectRevisionEntity revisionEntity = getCatalogObjectRevisionEntityByCommitTime(bucketName, name, commitTime); @@ -805,8 +846,7 @@ public CatalogObjectMetadata getCatalogObjectRevision(String bucketName, String } @Transactional(readOnly = true) - public CatalogRawObject getCatalogObjectRevisionRaw(String bucketName, String name, long commitTime) - throws UnsupportedEncodingException { + public CatalogRawObject getCatalogObjectRevisionRaw(String bucketName, String name, long commitTime) { CatalogObjectRevisionEntity revisionEntity = getCatalogObjectRevisionEntityByCommitTime(bucketName, name, commitTime); diff --git a/src/main/java/org/ow2/proactive/catalog/util/ArchiveManagerHelper.java b/src/main/java/org/ow2/proactive/catalog/util/ArchiveManagerHelper.java index 5a6b3ce285..c33d89ddab 100644 --- a/src/main/java/org/ow2/proactive/catalog/util/ArchiveManagerHelper.java +++ b/src/main/java/org/ow2/proactive/catalog/util/ArchiveManagerHelper.java @@ -25,6 +25,7 @@ */ package org.ow2.proactive.catalog.util; +import static org.ow2.proactive.catalog.util.PackageMetadataJSONParser.readJSONFile; import static org.ow2.proactive.catalog.util.PackageMetadataJSONParser.writeJSONFile; import java.io.ByteArrayInputStream; @@ -38,6 +39,7 @@ import java.util.zip.ZipEntry; import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.Level; import org.ow2.proactive.catalog.repository.entity.CatalogObjectEntity; import org.ow2.proactive.catalog.repository.entity.CatalogObjectRevisionEntity; import org.springframework.beans.factory.annotation.Autowired; @@ -172,7 +174,7 @@ public ByteSource generateMetadataForPackage(String bucketName, .getName(), catalogObjectEntity.getExtension(), catalogObjectEntity.getKind()); - fileNameWithExtension = bucketName + "/resources/catalog/" + fileNameWithExtension; + fileNameWithExtension = "resources/catalog/" + fileNameWithExtension; PackageMetadataJSONParser.CatalogObjectData objectData = new PackageMetadataJSONParser.CatalogObjectData(catalogObjectEntity.getId() .getName(), catalogObjectEntity.getKind(), @@ -184,7 +186,7 @@ public ByteSource generateMetadataForPackage(String bucketName, packageData.getCatalog().getObjects().add(objectData); } try { - return new ByteSource(bucketName + "/METADATA.json", writeJSONFile(packageData)); + return new ByteSource("METADATA.json", writeJSONFile(packageData)); } catch (IOException ioe) { log.error("Could not create a METADATA.json file for the demanded package"); throw new RuntimeException(ioe); @@ -215,8 +217,7 @@ public ZipArchiveContent compressPackageZIP(boolean isPartial, List extractZIP(byte[] byteArrayArchive) { */ private void checkAndAddFileFromZip(List filesList, InputStream in, ZipEntry entry) { String nameZipEntry = FilenameUtils.getName(entry.getName()); - if (!nameZipEntry.isEmpty()) { + if (!nameZipEntry.isEmpty() && !nameZipEntry.equals("METADATA.json")) { filesList.add(process(in, entry)); } } + public static PackageMetadataJSONParser.PackageData extractMetadataObject(String bucketName, + byte[] byteArrayArchive) { + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayArchive)) { + byte[] bytes = ZipUtil.unpackEntry(byteArrayInputStream, "METADATA.json"); + return readJSONFile(bytes); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + public static byte[] extractObjectByPath(byte[] byteArrayArchive, String objectName) { + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayArchive)) { + return ZipUtil.unpackEntry(byteArrayInputStream, objectName); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + /** * Extract ZIP entry into a byte array * @param in entry content diff --git a/src/main/java/org/ow2/proactive/catalog/util/PackageMetadataJSONParser.java b/src/main/java/org/ow2/proactive/catalog/util/PackageMetadataJSONParser.java index a7a0006923..c02048b905 100644 --- a/src/main/java/org/ow2/proactive/catalog/util/PackageMetadataJSONParser.java +++ b/src/main/java/org/ow2/proactive/catalog/util/PackageMetadataJSONParser.java @@ -58,9 +58,14 @@ protected static PackageData createPackageMetadata(String bucketName, String use return new PackageData(bucketName, userGroup); } + protected static PackageData readJSONFile(byte[] jsonFileAsByteArray) throws IOException { + return MAPPER.readValue(jsonFileAsByteArray, PackageMetadataJSONParser.PackageData.class); + } + @Getter @Setter - protected static class PackageData { + @AllArgsConstructor(access = AccessLevel.PUBLIC) + public static class PackageData { private PackageMetaData metadata; @@ -69,13 +74,14 @@ protected static class PackageData { public PackageData(String bucketName, String userGroup) { this.metadata = new PackageMetaData(bucketName); - this.catalog = new BucketData(bucketName, userGroup, new ArrayList()); + this.catalog = new BucketData(bucketName, userGroup, new ArrayList<>()); } } @Getter @Setter + @AllArgsConstructor(access = AccessLevel.PUBLIC) private static class PackageMetaData { private String name; @@ -101,7 +107,7 @@ public PackageMetaData(String name) { @Getter @Setter @AllArgsConstructor(access = AccessLevel.PUBLIC) - protected static class BucketData { + public static class BucketData { private String bucket; @@ -113,7 +119,8 @@ protected static class BucketData { @Getter @Setter - protected static class CatalogObjectData { + @AllArgsConstructor(access = AccessLevel.PUBLIC) + public static class CatalogObjectData { private String name; @@ -131,8 +138,8 @@ public CatalogObjectData(String name, String kind, String commitMessage, String @Getter @Setter - @AllArgsConstructor(access = AccessLevel.PROTECTED) - private static class CatalogObjectMetaData { + @AllArgsConstructor(access = AccessLevel.PUBLIC) + public static class CatalogObjectMetaData { private String kind; diff --git a/src/test/java/org/ow2/proactive/catalog/util/ArchiveManagerHelperTest.java b/src/test/java/org/ow2/proactive/catalog/util/ArchiveManagerHelperTest.java index a85da773e1..9c88cd2ada 100644 --- a/src/test/java/org/ow2/proactive/catalog/util/ArchiveManagerHelperTest.java +++ b/src/test/java/org/ow2/proactive/catalog/util/ArchiveManagerHelperTest.java @@ -148,31 +148,6 @@ public void testCompressZip() throws IOException { assertEquals("workflow_1.xml", actualFiles.get(1).getFileNameWithExtension()); } - @Test - public void testCompressPackageZip() throws IOException { - - assertNull(archiveManager.compressPackageZIP(false, null, "test-empty-bucket")); - - byte[] workflowByteArray0 = convertFromURIToByteArray(XML_FILE_0); - byte[] workflowByteArray1 = convertFromURIToByteArray(XML_FILE_1); - List expectedFiles = new ArrayList<>(); - expectedFiles.add(getCatalogObjectRevisionEntity("workflow_0", workflowByteArray0, "xml")); - expectedFiles.add(getCatalogObjectRevisionEntity("workflow_1", workflowByteArray1, "xml")); - when(rawObjectResponseCreator.getNameWithFileExtension("workflow_0", "xml", null)).thenReturn("workflow_0.xml"); - when(rawObjectResponseCreator.getNameWithFileExtension("workflow_1", "xml", null)).thenReturn("workflow_1.xml"); - //Compress - ZipArchiveContent archive = archiveManager.compressPackageZIP(false, expectedFiles, "test-bucket"); - //Then extract - List actualFiles = archiveManager.extractZIP(archive.getContent()); - assertEquals(3, actualFiles.size()); - - compare(workflowByteArray0, actualFiles.get(1).getContent()); - compare(workflowByteArray1, actualFiles.get(2).getContent()); - assertEquals("METADATA.json", actualFiles.get(0).getFileNameWithExtension()); - assertEquals("workflow_0.xml", actualFiles.get(1).getFileNameWithExtension()); - assertEquals("workflow_1.xml", actualFiles.get(2).getFileNameWithExtension()); - } - @Test public void testExtractZip() throws IOException { assertTrue(archiveManager.extractZIP(null).isEmpty());