diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java index 6918658f994..0d70319717d 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java @@ -30,7 +30,6 @@ import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.commons.pio.Closer; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; @@ -53,12 +52,10 @@ static NodeStore configureSegment(Options options, BlobStore blobStore, Whiteboa FileStoreBuilder builder; if (segmentStoreType == ToolUtils.SegmentStoreType.AZURE) { - final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); SegmentNodeStorePersistence segmentNodeStorePersistence = - ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri, azureStorageCredentialManagerV8); + ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri); File tempDir = Files.createTempDirectory("azure-segment-store").toFile(); closer.register(() -> FileUtils.deleteQuietly(tempDir)); - closer.register(azureStorageCredentialManagerV8); builder = fileStoreBuilder(tempDir).withCustomPersistence(segmentNodeStorePersistence); } else { builder = fileStoreBuilder(new File(pathOrUri)).withMaxFileSize(256); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java index 742576a0662..a3a116ecb58 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java @@ -18,7 +18,6 @@ */ package org.apache.jackrabbit.oak.explorer; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; @@ -39,16 +38,14 @@ public class AzureSegmentStoreExplorerBackend extends AbstractSegmentTarExplorerBackend { private final String path; private SegmentNodeStorePersistence persistence; - private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8; public AzureSegmentStoreExplorerBackend(String path) { this.path = path; - this.azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); } @Override public void open() throws IOException { - this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManagerV8); + this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); try { this.store = fileStoreBuilder(Files.createTempDirectory(getClass().getSimpleName() + "-").toFile()) @@ -63,7 +60,6 @@ public void open() throws IOException { @Override public void close() { super.close(); - azureStorageCredentialManagerV8.close(); } @Override diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java index cf01abbf437..dd495023291 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java @@ -27,7 +27,6 @@ import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.apache.jackrabbit.oak.run.commons.Command; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; @@ -86,22 +85,20 @@ public void execute(String... args) throws Exception { } } else { if (pathOrURI.startsWith("az:")) { - try (AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8()) { - SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI, azureStorageCredentialManagerV8); - ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDirectory(getClass().getSimpleName() + "-").toFile()) - .withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); - statusCode = Diff.builder() - .withPath(pathOrURI) - .withReadOnlyFileStore(store) - .withOutput(out) - .withInterval(interval) - .withIncremental(incremental) - .withFilter(path) - .withIgnoreMissingSegments(ignoreSNFEs) - .withRevisionsProcessor(ToolUtils::readRevisions) - .build() - .run(); - } + SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI); + ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDirectory(getClass().getSimpleName() + "-").toFile()) + .withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); + statusCode = Diff.builder() + .withPath(pathOrURI) + .withReadOnlyFileStore(store) + .withOutput(out) + .withInterval(interval) + .withIncremental(incremental) + .withFilter(path) + .withIgnoreMissingSegments(ignoreSNFEs) + .withRevisionsProcessor(ToolUtils::readRevisions) + .build() + .run(); } else { ReadOnlyFileStore store = fileStoreBuilder(new File(pathOrURI)).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); statusCode = Diff.builder() diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java index 7bf9c2a2749..cc17485b2c0 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java @@ -71,8 +71,7 @@ import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions.Type; import org.apache.jackrabbit.oak.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistenceManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; import org.apache.jackrabbit.oak.segment.file.FileStore; @@ -1128,7 +1127,6 @@ class SegmentStoreFixture implements StoreFixture { class AzureSegmentStoreFixture extends SegmentStoreFixture { private static final String AZURE_DIR = "repository"; private String container; - private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); @Override public NodeStore init(DataStoreBlobStore blobStore, File storeFile) throws Exception { Properties props = AzureDataStoreUtils.getAzureConfig(); @@ -1138,14 +1136,14 @@ class AzureSegmentStoreFixture extends SegmentStoreFixture { container = container + System.currentTimeMillis(); // Create the azure segment container String connectionString = getAzureConnectionString(accessKey, secretKey, container, AZURE_DIR); - AzureUtilitiesV8.cloudBlobDirectoryFrom(connectionString, container, AZURE_DIR); + AzurePersistenceManager.createAzurePersistence(connectionString, null, accessKey, container, AZURE_DIR, false, true); // get the azure uri expected by the command storePath = getAzureUri(accessKey, container, AZURE_DIR); // initialize azure segment for test setup SegmentNodeStorePersistence segmentNodeStorePersistence = - ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath, azureStorageCredentialManagerV8); + ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath); fileStore = fileStoreBuilder(storeFile).withBlobStore(blobStore) .withCustomPersistence(segmentNodeStorePersistence).build(); @@ -1177,7 +1175,6 @@ protected String getAzureConnectionString(String accountName, String secret, Str public void after() { try { AzureDataStoreUtils.deleteContainer(container); - azureStorageCredentialManagerV8.close(); } catch(Exception e) { log.error("Error in cleaning the container {}", container, e); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java index a0f0ddded5a..9a6f75595cc 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java @@ -53,6 +53,10 @@ public class AzurePersistence implements SegmentNodeStorePersistence { protected WriteAccessController writeAccessController = new WriteAccessController(); + public AzurePersistence(BlobContainerClient blobContainerClient, String rootPrefix) { + this(blobContainerClient, blobContainerClient, blobContainerClient, rootPrefix); + } + public AzurePersistence(BlobContainerClient readBlobContainerClient, BlobContainerClient writeBlobContainerClient, BlobContainerClient noRetryBlobContainerClient, String rootPrefix) { this(readBlobContainerClient, writeBlobContainerClient, noRetryBlobContainerClient, rootPrefix, null); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java index 12fee4c31c3..e16179741eb 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java @@ -41,14 +41,18 @@ public class AzurePersistenceManager { private AzurePersistenceManager() { } - public static AzurePersistence createAzurePersistenceFrom(@NotNull String accountName, @NotNull String containerName, @NotNull String rootPrefix, @NotNull Environment environment) throws IOException { + public static AzurePersistence createAzurePersistenceFrom(String accountName, String containerName, String rootPrefix, String sasToken) throws IOException { + return createAzurePersistence(null, sasToken, accountName, containerName, rootPrefix, false, false); + } + + public static AzurePersistence createAzurePersistenceFrom(String accountName, String containerName, String rootPrefix, Environment environment) throws IOException { final String clientId = environment.getVariable(AZURE_CLIENT_ID); final String clientSecret = environment.getVariable(AZURE_CLIENT_SECRET); final String tenantId = environment.getVariable(AZURE_TENANT_ID); if (StringUtils.isNoneBlank(clientId, clientSecret, tenantId)) { try { - return createPersistenceFromServicePrincipalCredentials(accountName, containerName, rootPrefix, clientId, clientSecret, tenantId, false, false); + return createPersistenceFromServicePrincipalCredentials(accountName, containerName, rootPrefix, clientId, clientSecret, tenantId, false, true); } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { log.error("Error occurred while connecting to Azure Storage using service principals: ", e); throw new IllegalArgumentException( @@ -86,6 +90,7 @@ private static AzurePersistence createPersistenceFromAccessKey(Configuration con } private static AzurePersistence createPersistenceFromAccessKey(String accountName, String containerName, String accessKey, String blobEndpoint, String rootPrefix, boolean enableSecondaryLocation, boolean createContainer) throws IOException { + checkIfEmpty(accessKey, "accessKey"); StringBuilder connectionString = new StringBuilder(); connectionString.append("DefaultEndpointsProtocol=https;"); connectionString.append("AccountName=").append(accountName).append(';'); @@ -93,7 +98,7 @@ private static AzurePersistence createPersistenceFromAccessKey(String accountNam if (!StringUtils.isBlank(blobEndpoint)) { connectionString.append("BlobEndpoint=").append(blobEndpoint).append(';'); } - return createAzurePersistence(connectionString.toString(), accountName, containerName, rootPrefix, enableSecondaryLocation, createContainer); + return createAzurePersistence(connectionString.toString(), null, accountName, containerName, rootPrefix, enableSecondaryLocation, createContainer); } @NotNull @@ -114,11 +119,12 @@ private static AzurePersistence createPersistenceFromSasUri(Configuration config @NotNull - private static AzurePersistence createPersistenceFromServicePrincipalCredentials(Configuration configuration) throws IOException { + private static AzurePersistence createPersistenceFromServicePrincipalCredentials(Configuration configuration) { return createPersistenceFromServicePrincipalCredentials(configuration.accountName(), configuration.containerName(), configuration.rootPath(), configuration.clientId(), configuration.clientSecret(), configuration.tenantId(), configuration.enableSecondaryLocation(), true); } - private static AzurePersistence createPersistenceFromServicePrincipalCredentials(String accountName, String containerName, String rootPrefix, String clientId, String clientSecret, String tenantId, boolean enableSecondaryLocation, boolean createContainer) throws IOException { + public static AzurePersistence createPersistenceFromServicePrincipalCredentials(String accountName, String containerName, String rootPrefix, String clientId, String clientSecret, String tenantId, boolean enableSecondaryLocation, boolean createContainer) { + checkArguments(accountName, containerName, rootPrefix); AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy = new AzureHttpRequestLoggingPolicy(); ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder() @@ -146,21 +152,41 @@ private static AzurePersistence createPersistenceFromServicePrincipalCredentials @NotNull private static AzurePersistence createAzurePersistence(String connectionString, Configuration configuration, boolean createContainer) throws IOException { - return createAzurePersistence(connectionString, configuration.accountName(), configuration.containerName(), configuration.rootPath(), configuration.enableSecondaryLocation(), createContainer); + return createAzurePersistence(connectionString, null, configuration.accountName(), configuration.containerName(), configuration.rootPath(), configuration.enableSecondaryLocation(), createContainer); } @NotNull - private static AzurePersistence createAzurePersistence(String connectionString, String accountName, String containerName, String rootPrefix, boolean enableSecondaryLocation, boolean createContainer) throws IOException { + public static AzurePersistence createAzurePersistence(String connectionString, String sasToken, String accountName, String containerName, String rootPrefix, boolean enableSecondaryLocation, boolean createContainer) throws IOException { + if (StringUtils.isBlank(connectionString) && StringUtils.isBlank(sasToken)) { + throw new IllegalArgumentException("Both connectionString and sasToken are not configured. Please configure one of them."); + } + checkArguments(accountName, containerName, rootPrefix); + try { AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy = new AzureHttpRequestLoggingPolicy(); RequestRetryOptions retryOptions = readRequestRetryOptions(enableSecondaryLocation, accountName); - BlobContainerClient blobContainerClient = getBlobContainerClient(accountName, containerName, retryOptions, azureHttpRequestLoggingPolicy, connectionString); + BlobContainerClient blobContainerClient; + if (sasToken != null) { + blobContainerClient = getBlobContainerClientWithSas(accountName, containerName, retryOptions, azureHttpRequestLoggingPolicy, sasToken); + } else { + blobContainerClient = getBlobContainerClient(accountName, containerName, retryOptions, azureHttpRequestLoggingPolicy, connectionString); + } RequestRetryOptions writeRetryOptions = AzureRequestOptions.getRetryOperationsOptimiseForWriteOperations(); - BlobContainerClient writeBlobContainerClient = getBlobContainerClient(accountName, containerName, writeRetryOptions, azureHttpRequestLoggingPolicy, connectionString); + BlobContainerClient writeBlobContainerClient; + if (sasToken != null) { + writeBlobContainerClient = getBlobContainerClientWithSas(accountName, containerName, writeRetryOptions, azureHttpRequestLoggingPolicy, sasToken); + } else { + writeBlobContainerClient = getBlobContainerClient(accountName, containerName, writeRetryOptions, azureHttpRequestLoggingPolicy, connectionString); + } - BlobContainerClient noRetryBlobContainerClient = getBlobContainerClient(accountName, containerName, null, azureHttpRequestLoggingPolicy, connectionString); + BlobContainerClient noRetryBlobContainerClient; + if (sasToken != null) { + noRetryBlobContainerClient = getBlobContainerClientWithSas(accountName, containerName, null, azureHttpRequestLoggingPolicy, sasToken); + } else { + noRetryBlobContainerClient = getBlobContainerClient(accountName, containerName, null, azureHttpRequestLoggingPolicy, connectionString); + } if (createContainer) { blobContainerClient.createIfNotExists(); @@ -174,8 +200,15 @@ private static AzurePersistence createAzurePersistence(String connectionString, } } + private static BlobContainerClient getBlobContainerClientWithSas(String accountName, String containerName, RequestRetryOptions requestRetryOptions, AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy, String sasToken) { + BlobServiceClient blobServiceClient = blobServiceClientBuilder(accountName, requestRetryOptions, azureHttpRequestLoggingPolicy, sasToken) + .buildClient(); + + return blobServiceClient.getBlobContainerClient(containerName); + } + private static BlobContainerClient getBlobContainerClient(String accountName, String containerName, RequestRetryOptions requestRetryOptions, AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy, String connectionString) { - BlobServiceClient blobServiceClient = blobServiceClientBuilder(accountName, requestRetryOptions, azureHttpRequestLoggingPolicy) + BlobServiceClient blobServiceClient = blobServiceClientBuilder(accountName, requestRetryOptions, azureHttpRequestLoggingPolicy, null) .connectionString(connectionString) .buildClient(); @@ -183,15 +216,20 @@ private static BlobContainerClient getBlobContainerClient(String accountName, St } private static BlobContainerClient getBlobContainerClient(String accountName, String containerName, RequestRetryOptions requestRetryOptions, AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy, ClientSecretCredential clientSecretCredential) { - BlobServiceClient blobServiceClient = blobServiceClientBuilder(accountName, requestRetryOptions, azureHttpRequestLoggingPolicy) + BlobServiceClient blobServiceClient = blobServiceClientBuilder(accountName, requestRetryOptions, azureHttpRequestLoggingPolicy, null) .credential(clientSecretCredential) .buildClient(); return blobServiceClient.getBlobContainerClient(containerName); } - private static BlobServiceClientBuilder blobServiceClientBuilder(String accountName, RequestRetryOptions requestRetryOptions, AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy) { - String endpoint = String.format("https://%s.blob.core.windows.net", accountName); + private static BlobServiceClientBuilder blobServiceClientBuilder(String accountName, RequestRetryOptions requestRetryOptions, AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy, String sasToken) { + if (sasToken == null) { + sasToken = ""; + } else { + sasToken = "?" + sasToken; + } + String endpoint = String.format("https://%s.blob.core.windows.net%s", accountName, sasToken); BlobServiceClientBuilder builder = new BlobServiceClientBuilder() .endpoint(endpoint) @@ -221,4 +259,15 @@ private static String normalizePath(@NotNull String rootPath) { return rootPath; } + private static void checkArguments(String accountName, String containerName, String rootPrefix){ + checkIfEmpty(accountName, "Account name"); + checkIfEmpty(containerName, "Container name"); + checkIfEmpty(rootPrefix, "Root prefix"); + } + + private static void checkIfEmpty(String argument, String argumentName) { + if (StringUtils.isEmpty(argument)) { + throw new IllegalArgumentException(String.format("%s must not be empty argument", argumentName)); + } + } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java index f6743edafc5..c3a7349a0e7 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java @@ -60,7 +60,9 @@ public static String getName(AppendBlobClient blob) { public static List getBlobs(BlobContainerClient blobContainerClient, ListBlobsOptions listOptions) { - listOptions.setDetails(new BlobListDetails().setRetrieveMetadata(true)); + if (listOptions != null) { + listOptions.setDetails(new BlobListDetails().setRetrieveMetadata(true)); + } return blobContainerClient.listBlobs(listOptions, null).stream().collect(Collectors.toList()); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java index 486fb5b132a..b6479a66074 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java @@ -16,9 +16,8 @@ */ package org.apache.jackrabbit.oak.segment.azure.tool; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; -import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; +import com.azure.storage.blob.BlobContainerClient; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; @@ -57,6 +56,8 @@ public static class Builder { private String path; + private String rootPrefix; + private String journal; private long debugInterval = Long.MAX_VALUE; @@ -85,7 +86,7 @@ public static class Builder { private Integer persistentCacheSizeGb; - private CloudBlobDirectory cloudBlobDirectory; + private BlobContainerClient blobContainerClient; private Builder() { // Prevent external instantiation. @@ -102,6 +103,17 @@ public Builder withPath(String path) { return this; } + /** + * The root prefix to an existing segment store. This parameter is required. + * + * @param rootPrefix + * @return this builder. + */ + public Builder withRootPrefix(String rootPrefix) { + this.rootPrefix = rootPrefix; + return this; + } + /** * The path to the journal of the segment store. This parameter is * optional. If not provided, the journal in the default location is @@ -278,12 +290,12 @@ public Builder withPersistentCacheSizeGb(Integer persistentCacheSizeGb) { /** * The Azure blob directory to connect to. - * @param cloudBlobDirectory + * @param blobContainerClient * the Azure blob directory. * @return this builder */ - public Builder withCloudBlobDirectory(CloudBlobDirectory cloudBlobDirectory) { - this.cloudBlobDirectory = requireNonNull(cloudBlobDirectory); + public Builder withBlobContainerClient(BlobContainerClient blobContainerClient) { + this.blobContainerClient = requireNonNull(blobContainerClient); return this; } @@ -293,7 +305,7 @@ public Builder withCloudBlobDirectory(CloudBlobDirectory cloudBlobDirectory) { * @return an instance of {@link Runnable}. */ public AzureCheck build() { - if (cloudBlobDirectory == null) { + if (blobContainerClient == null) { requireNonNull(path); } return new AzureCheck(this); @@ -320,6 +332,8 @@ public void afterSegmentRead(File file, long msb, long lsb, int length, long ela private final String path; + private final String rootPrefix; + private final String journal; private final long debugInterval; @@ -348,11 +362,11 @@ public void afterSegmentRead(File file, long msb, long lsb, int length, long ela private final Integer persistentCacheSizeGb; - private final CloudBlobDirectory cloudBlobDirectory; - private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8; + private final BlobContainerClient blobContainerClient; private AzureCheck(Builder builder) { this.path = builder.path; + this.rootPrefix = builder.rootPrefix; this.debugInterval = builder.debugInterval; this.checkHead = builder.checkHead; this.checkBinaries = builder.checkBinaries; @@ -367,8 +381,7 @@ private AzureCheck(Builder builder) { this.failFast = builder.failFast; this.persistentCachePath = builder.persistentCachePath; this.persistentCacheSizeGb = builder.persistentCacheSizeGb; - this.cloudBlobDirectory = builder.cloudBlobDirectory; - this.azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); + this.blobContainerClient = builder.blobContainerClient; } private static Integer revisionsToCheckCount(Integer revisionsCount) { @@ -379,10 +392,10 @@ public int run() { StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor(); SegmentNodeStorePersistence persistence; - if (cloudBlobDirectory != null) { - persistence = new AzurePersistenceV8(cloudBlobDirectory); + if (blobContainerClient != null) { + persistence = new AzurePersistence(blobContainerClient, rootPrefix); } else { - persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManagerV8); + persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); } if (persistentCachePath != null) { @@ -434,8 +447,6 @@ public int run() { } catch (Exception e) { e.printStackTrace(err); return 1; - } finally { - azureStorageCredentialManagerV8.close(); } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java index d150be09361..1a61d9fcb66 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java @@ -19,24 +19,19 @@ import static org.apache.jackrabbit.oak.commons.conditions.Validate.checkArgument; import static java.util.Objects.requireNonNull; import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createArchiveManager; -import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createCloudBlobDirectory; +import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createAzurePersistence; import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.decorateWithCache; import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newFileStore; -import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newSegmentNodeStorePersistence; import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.printableStopwatch; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobListDetails; +import com.azure.storage.blob.models.ListBlobsOptions; import org.apache.jackrabbit.guava.common.base.Stopwatch; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.BlobListingDetails; -import com.microsoft.azure.storage.blob.CloudBlob; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; -import com.microsoft.azure.storage.blob.ListBlobItem; import org.apache.jackrabbit.oak.segment.SegmentCache; -import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; -import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; import org.apache.jackrabbit.oak.segment.file.FileStore; @@ -53,7 +48,6 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.util.Collections; -import java.util.EnumSet; import java.util.List; /** @@ -79,6 +73,10 @@ public static class Builder { private String targetPath; + private String rootPrefix; + + private String targetRootPrefix; + private boolean force; private long gcLogInterval = 150000; @@ -99,9 +97,9 @@ public static class Builder { private int garbageThresholdPercentage; - private CloudBlobDirectory sourceCloudBlobDirectory; + private BlobContainerClient sourceBlobContainerClient; - private CloudBlobDirectory destinationCloudBlobDirectory; + private BlobContainerClient destinationBlobContainerClient; private Builder() { // Prevent external instantiation. @@ -131,6 +129,30 @@ public Builder withTargetPath(String targetPath) { return this; } + /** + * The root directory to an existing segment store. + * + * @param rootPrefix + * the directory to an existing segment store. + * @return the builder + */ + public Builder withRootPrefix(String rootPrefix) { + this.rootPrefix = rootPrefix; + return this; + } + + /** + * The root directory to the target segment store. + * + * @param targetRootPrefix + * the root directory to the target segmen store. + * @return this builder + */ + public Builder withTargetRootPrefix(String targetRootPrefix) { + this.targetRootPrefix = targetRootPrefix; + return this; + } + /** * Whether to fail if run on an older version of the store of force upgrading * its format. @@ -253,13 +275,13 @@ public Builder withGarbageThresholdPercentage(int garbageThresholdPercentage) { return this; } - public Builder withSourceCloudBlobDirectory(CloudBlobDirectory sourceCloudBlobDirectory) { - this.sourceCloudBlobDirectory = requireNonNull(sourceCloudBlobDirectory); + public Builder withSourceBlobContainerClient(BlobContainerClient sourceBlobContainerClient) { + this.sourceBlobContainerClient = requireNonNull(sourceBlobContainerClient); return this; } - public Builder withDestinationCloudBlobDirectory(CloudBlobDirectory destinationCloudBlobDirectory) { - this.destinationCloudBlobDirectory = requireNonNull(destinationCloudBlobDirectory); + public Builder withDestinationBlobContainerClient(BlobContainerClient destinationBlobContainerClient) { + this.destinationBlobContainerClient = requireNonNull(destinationBlobContainerClient); return this; } @@ -269,7 +291,7 @@ public Builder withDestinationCloudBlobDirectory(CloudBlobDirectory destinationC * @return an instance of {@link Runnable}. */ public AzureCompact build() { - if (sourceCloudBlobDirectory == null || destinationCloudBlobDirectory == null) { + if (sourceBlobContainerClient == null || destinationBlobContainerClient == null) { requireNonNull(path); requireNonNull(targetPath); } @@ -283,6 +305,10 @@ public AzureCompact build() { private final String targetPath; + private final String rootPrefix; + + private final String targetRootPrefix; + private final int segmentCacheSize; private final boolean strictVersionCheck; @@ -303,14 +329,15 @@ public AzureCompact build() { private final int garbageThresholdPercentage; - private final CloudBlobDirectory sourceCloudBlobDirectory; + private final BlobContainerClient sourceBlobContainerClient; - private final CloudBlobDirectory destinationCloudBlobDirectory; - private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8; + private final BlobContainerClient destinationBlobContainerClient; private AzureCompact(Builder builder) { this.path = builder.path; this.targetPath = builder.targetPath; + this.rootPrefix = builder.rootPrefix; + this.targetRootPrefix = builder.targetRootPrefix; this.segmentCacheSize = builder.segmentCacheSize; this.strictVersionCheck = !builder.force; this.gcLogInterval = builder.gcLogInterval; @@ -321,22 +348,26 @@ private AzureCompact(Builder builder) { this.persistentCacheSizeGb = builder.persistentCacheSizeGb; this.garbageThresholdGb = builder.garbageThresholdGb; this.garbageThresholdPercentage = builder.garbageThresholdPercentage; - this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory; - this.destinationCloudBlobDirectory = builder.destinationCloudBlobDirectory; - this.azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); + this.sourceBlobContainerClient = builder.sourceBlobContainerClient; + this.destinationBlobContainerClient = builder.destinationBlobContainerClient; } - public int run() throws IOException, StorageException, URISyntaxException { + public int run() throws IOException, URISyntaxException { Stopwatch watch = Stopwatch.createStarted(); SegmentNodeStorePersistence roPersistence; SegmentNodeStorePersistence rwPersistence; - if (sourceCloudBlobDirectory != null && destinationCloudBlobDirectory != null) { - roPersistence = new AzurePersistenceV8(sourceCloudBlobDirectory); - rwPersistence = new AzurePersistenceV8(destinationCloudBlobDirectory); + BlobContainerClient targetContainer; + if (sourceBlobContainerClient != null && destinationBlobContainerClient != null) { + roPersistence = new AzurePersistence(sourceBlobContainerClient, rootPrefix); + rwPersistence = new AzurePersistence(destinationBlobContainerClient, targetRootPrefix); + targetContainer = destinationBlobContainerClient; } else { - roPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path, azureStorageCredentialManagerV8); - rwPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath, azureStorageCredentialManagerV8); + roPersistence = createAzurePersistence(path); + + AzurePersistence rwAzurePersistence = createAzurePersistence(targetPath); + targetContainer = rwAzurePersistence.getReadBlobContainerClient(); + rwPersistence = rwAzurePersistence; } if (persistentCachePath != null) { @@ -348,8 +379,8 @@ public int run() throws IOException, StorageException, URISyntaxException { SegmentArchiveManager roArchiveManager = createArchiveManager(roPersistence); SegmentArchiveManager rwArchiveManager = createArchiveManager(rwPersistence); - System.out.printf("Compacting %s\n", path != null ? path : sourceCloudBlobDirectory.getUri().toString()); - System.out.printf(" to %s\n", targetPath != null ? targetPath : destinationCloudBlobDirectory.getUri().toString()); + System.out.printf("Compacting %s\n", path != null ? path : sourceBlobContainerClient.getBlobContainerUrl()); + System.out.printf(" to %s\n", targetPath != null ? targetPath : destinationBlobContainerClient.getBlobContainerUrl()); System.out.printf(" before\n"); List beforeArchives = Collections.emptyList(); try { @@ -360,14 +391,6 @@ public int run() throws IOException, StorageException, URISyntaxException { printArchives(System.out, beforeArchives); - CloudBlobContainer targetContainer = null; - if (targetPath != null) { - CloudBlobDirectory targetDirectory = createCloudBlobDirectory(targetPath.substring(3), azureStorageCredentialManagerV8); - targetContainer = targetDirectory.getContainer(); - } else { - targetContainer = destinationCloudBlobDirectory.getContainer(); - } - GCGeneration gcGeneration = null; String root = null; @@ -424,8 +447,6 @@ public int run() throws IOException, StorageException, URISyntaxException { long newSize = printTargetRepoSizeInfo(targetContainer); persistGCJournal(rwPersistence, newSize, gcGeneration, root); - // close azure storage credential manager - azureStorageCredentialManagerV8.close(); return 0; } @@ -456,13 +477,14 @@ private boolean isGarbageOverMinimumThreshold(long currentSize, SegmentNodeStore return true; } - private long printTargetRepoSizeInfo(CloudBlobContainer container) { - System.out.printf("Calculating the size of container %s\n", container.getName()); + private long printTargetRepoSizeInfo(BlobContainerClient blobContainerClient) { + System.out.printf("Calculating the size of container %s\n", blobContainerClient.getBlobContainerName()); long size = 0; - for (ListBlobItem i : container.listBlobs(null, true, EnumSet.of(BlobListingDetails.METADATA), null, null)) { - if (i instanceof CloudBlob) { - size += ((CloudBlob) i).getProperties().getLength(); - } + ListBlobsOptions listBlobsOptions = new ListBlobsOptions(); + listBlobsOptions.setDetails(new BlobListDetails().setRetrieveMetadata(true)); + + for (BlobItem blobItem : blobContainerClient.listBlobs(listBlobsOptions, null)) { + size += blobItem.getProperties().getContentLength(); } System.out.printf("The size is: %d MB \n", size / 1024 / 1024); return size; diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java index 706f64885ae..cfed41d3cff 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java @@ -43,7 +43,6 @@ import java.util.concurrent.Future; import org.apache.jackrabbit.oak.commons.Buffer; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; import org.apache.jackrabbit.oak.segment.azure.tool.SegmentStoreMigrator.Segment; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; import org.apache.jackrabbit.oak.segment.azure.util.Retrier; @@ -260,7 +259,6 @@ public SegmentCopy build() { private SegmentNodeStorePersistence destPersistence; private ExecutorService executor = Executors.newFixedThreadPool(READ_THREADS + 1); - private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8; public SegmentCopy(Builder builder) { this.source = builder.source; @@ -273,7 +271,6 @@ public SegmentCopy(Builder builder) { this.maxSizeGb = builder.maxSizeGb; this.outWriter = builder.outWriter; this.errWriter = builder.errWriter; - this.azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); } public int run() { @@ -288,7 +285,7 @@ public int run() { if (flat && destType == SegmentStoreType.TAR) { try { if (srcPersistence == null) { - srcPersistence = newSegmentNodeStorePersistence(srcType, source, azureStorageCredentialManagerV8); + srcPersistence = newSegmentNodeStorePersistence(srcType, source); } SegmentArchiveManager sourceManager = srcPersistence.createArchiveManager(false, false, @@ -366,14 +363,12 @@ public int run() { destination); e.printStackTrace(errWriter); return 1; - } finally { - azureStorageCredentialManagerV8.close(); } } else { try { if (srcPersistence == null || destPersistence == null) { - srcPersistence = newSegmentNodeStorePersistence(srcType, source, azureStorageCredentialManagerV8); - destPersistence = newSegmentNodeStorePersistence(destType, destination, azureStorageCredentialManagerV8); + srcPersistence = newSegmentNodeStorePersistence(srcType, source); + destPersistence = newSegmentNodeStorePersistence(destType, destination); } printMessage(outWriter, "Started segment-copy transfer!"); @@ -397,8 +392,6 @@ public int run() { destination); e.printStackTrace(errWriter); return 1; - } finally { - azureStorageCredentialManagerV8.close(); } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java index 6bba400c16c..404be354193 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java @@ -19,11 +19,9 @@ import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.fetchByteArray; import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.storeDescription; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; - +import com.azure.storage.blob.BlobContainerClient; import org.apache.jackrabbit.oak.commons.Buffer; -import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; import org.apache.jackrabbit.oak.segment.azure.util.Retrier; import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; @@ -45,7 +43,6 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -289,9 +286,9 @@ public Builder withSource(File dir) { return this; } - public Builder withSource(CloudBlobDirectory dir) throws URISyntaxException, StorageException { - this.source = new AzurePersistenceV8(dir); - this.sourceName = storeDescription(SegmentStoreType.AZURE, dir.getContainer().getName() + "/" + dir.getPrefix()); + public Builder withSource(BlobContainerClient blobContainerClient, String rootPrefix) { + this.source = new AzurePersistence(blobContainerClient, rootPrefix); + this.sourceName = storeDescription(SegmentStoreType.AZURE, blobContainerClient.getBlobContainerName() + "/" + rootPrefix); return this; } @@ -313,9 +310,9 @@ public Builder withTarget(File dir) { return this; } - public Builder withTarget(CloudBlobDirectory dir) throws URISyntaxException, StorageException { - this.target = new AzurePersistenceV8(dir); - this.targetName = storeDescription(SegmentStoreType.AZURE, dir.getContainer().getName() + "/" + dir.getPrefix()); + public Builder withTarget(BlobContainerClient blobContainerClient, String rootPrefix) { + this.target = new AzurePersistence(blobContainerClient, rootPrefix); + this.targetName = storeDescription(SegmentStoreType.AZURE, blobContainerClient.getBlobContainerUrl() + "/" + rootPrefix); return this; } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java index 3eab8472154..e41e6f53501 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java @@ -21,27 +21,24 @@ import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE; -import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri; import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions; import java.io.File; import java.io.IOException; import java.io.PrintWriter; -import java.net.URISyntaxException; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.TimeUnit; import org.apache.jackrabbit.oak.commons.Buffer; import org.apache.jackrabbit.oak.commons.collections.ListUtils; -import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistenceManager; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; import org.apache.jackrabbit.oak.segment.file.*; @@ -60,12 +57,8 @@ import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.Iterators; -import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; + import org.apache.jackrabbit.oak.stats.StatisticsProvider; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,15 +123,12 @@ public static SegmentNodeStorePersistence decorateWithCache(SegmentNodeStorePers } public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, - String pathOrUri, - @Nullable AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8) { + String pathOrUri) { SegmentNodeStorePersistence persistence = null; switch (storeType) { case AZURE: - Objects.requireNonNull(azureStorageCredentialManagerV8, "azure storage credentials manager instance cannot be null"); - CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3), azureStorageCredentialManagerV8); - persistence = new AzurePersistenceV8(cloudBlobDirectory); + persistence = createAzurePersistence(pathOrUri.substring(3)); break; default: persistence = new TarPersistence(new File(pathOrUri)); @@ -160,48 +150,43 @@ public static SegmentArchiveManager createArchiveManager(SegmentNodeStorePersist return archiveManager; } - public static CloudBlobDirectory createCloudBlobDirectory(String path, AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8) { - return createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManagerV8); + public static AzurePersistence createAzurePersistence(String path) { + return createAzurePersistence(path, ENVIRONMENT); } - public static CloudBlobDirectory createCloudBlobDirectory(String path, - Environment environment, - AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8) { + public static AzurePersistence createAzurePersistence(String path, + Environment environment) { Map config = parseAzureConfigurationFromUri(path); String accountName = config.get(KEY_ACCOUNT_NAME); - - StorageCredentials credentials; - if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) { - credentials = new StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE)); - } else { - credentials = azureStorageCredentialManagerV8.getStorageCredentialsFromEnvironment(accountName, environment); - } - - String uri = config.get(KEY_STORAGE_URI); + String containerName = config.get(KEY_CONTAINER_NAME); String dir = config.get(KEY_DIR); + AzurePersistence azurePersistence; try { - return AzureUtilitiesV8.cloudBlobDirectoryFrom(credentials, uri, dir); - } catch (URISyntaxException | StorageException e) { + if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) { + azurePersistence = AzurePersistenceManager.createAzurePersistenceFrom(accountName, containerName, dir, config.get(KEY_SHARED_ACCESS_SIGNATURE)); + } else { + azurePersistence = AzurePersistenceManager.createAzurePersistenceFrom(accountName, containerName, dir, environment); + } + } catch (IOException e) { throw new IllegalArgumentException( "Could not connect to the Azure Storage. Please verify the path provided!"); } + return azurePersistence; } public static List readRevisions(String uri) { - try (AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8()) { - SegmentNodeStorePersistence persistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri, azureStorageCredentialManagerV8); - JournalFile journal = persistence.getJournalFile(); - if (journal.exists()) { - try (JournalReader journalReader = new JournalReader(journal)) { - Iterator revisionIterator = Iterators.transform(journalReader, - entry -> entry.getRevision()); - return ListUtils.toList(revisionIterator); - } catch (Exception e) { - log.error("Error while reading from journal file"); - e.printStackTrace(); - } + SegmentNodeStorePersistence persistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri); + JournalFile journal = persistence.getJournalFile(); + if (journal.exists()) { + try (JournalReader journalReader = new JournalReader(journal)) { + Iterator revisionIterator = Iterators.transform(journalReader, + entry -> entry.getRevision()); + return ListUtils.toList(revisionIterator); + } catch (Exception e) { + log.error("Error while reading from journal file"); + e.printStackTrace(); } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java index e2279f9d81b..9ee8f1098fe 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java @@ -147,14 +147,15 @@ public static Map parseAzureConfigurationFromUri(String uriStr) int dotPosHost = host.indexOf("."); String accountName = host.substring(0, dotPosHost); - String container = path.substring(0, lastSlashPosPath); - String storageUri = scheme + "://" + host + container; + String container = path.substring(1, lastSlashPosPath); + String storageUri = scheme + "://" + host + "/" + container; String dir = path.substring(lastSlashPosPath + 1); Map config = new HashMap<>(); config.put(KEY_ACCOUNT_NAME, accountName); config.put(KEY_STORAGE_URI, storageUri); config.put(KEY_DIR, dir); + config.put(KEY_CONTAINER_NAME, container); if (sasToken != null) { config.put(KEY_SHARED_ACCESS_SIGNATURE, sasToken); } diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManagerTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManagerTest.java new file mode 100644 index 00000000000..6909edc07ee --- /dev/null +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManagerTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment.azure; + +import org.junit.Test; + +import static org.junit.Assert.assertThrows; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.osgi.util.converter.Converters; + +import java.util.HashMap; + +public class AzurePersistenceManagerTest { + + @Test + public void bothConnectionStringAndSasTokenNotConfiguredTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence(null, null, "accountName", "containerName", "aem", false, false)); + } + + @Test + public void bothConnectionStringAndSasTokenBlankTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("", "", "accountName", "containerName", "aem", false, false)); + } + + @Test + public void connectionStringBlankAndSasTokenNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("", null, "accountName", "containerName", "aem", false, false)); + } + + @Test + public void connectionStringNullAndSasTokenBlankTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence(null, "", "accountName", "containerName", "aem", false, false)); + } + + @Test + public void accountNameIsNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken", null, "containerName", "aem", false, false)); + } + + @Test + public void accountNameIsEmptyTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken", "", "containerName", "aem", false, false)); + } + + @Test + public void containerNameIsNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken", "accountName", null, "aem", false, false)); + } + + @Test + public void containerNameIsEmptyTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken", "accountName", "", "aem", false, false)); + } + + @Test + public void rootPrefixIsNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken", "accountName", "containerName", null, false, false)); + } + + @Test + public void rootPrefixIsEmptyTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken", "accountName", "containerName", "", false, false)); + } + + @Test + public void servicePrincipalsAccountNameIsNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials(null, "containerName", "rootPrefix", "clientId", "clientSecret", "tenantId", false, false)); + } + + @Test + public void servicePrincipalAccountNameIsEmptyTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("", "containerName", "rootPrefix", "clientId", "clientSecret", "tenantId", false, false)); + } + + @Test + public void servicePrincipalsContainerNameIsNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName", null, "rootPrefix", "clientId", "clientSecret", "tenantId", false, false)); + } + + @Test + public void servicePrincipalsContainerNameIsEmptyTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName", "", "rootPrefix", "clientId", "clientSecret", "tenantId", false, false)); + } + + @Test + public void servicePrincipalsRootPrefixIsNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName", "containerName", null, "clientId", "clientSecret", "tenantId", false, false)); + } + + @Test + public void servicePrincipalsRootPrefixIsEmptyTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName", "containerName", "", "clientId", "clientSecret", "tenantId", false, false)); + } + + @Test + public void createAzurePersistenceAllNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistenceFrom(null, null, null, "sasToken")); + } + + @Test + public void createAzurePersistenceFromEnvAllNullTest() { + Environment env = new Environment(); + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistenceFrom(null, null, null, env)); + } + + @Test + public void createAzurePersistenceFromConfigurationAllNullTest() { + assertThrows(IllegalArgumentException.class, () -> AzurePersistenceManager.createAzurePersistenceFrom(getConfiguration())); + } + + private static Configuration getConfiguration() { + return Converters.standardConverter() + .convert(new HashMap() {{ + put("accountName", null); + put("accessKey", null); + put("connectionURL", "https://accounts.blob.core.windows.net"); + put("sharedAccessSignature", "sharedAccessSignature"); + put("clientId", "clientId"); + put("clientSecret", "clientSecret"); + put("tenantId", "tenantId"); + put("blobEndpoint", "blobEndpoint"); + }}) + .to(Configuration.class); + } + +} \ No newline at end of file diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java index 7d7ada9e3b7..c6e36ab0705 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java @@ -18,18 +18,16 @@ */ package org.apache.jackrabbit.oak.segment.azure.tool; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; -import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; -import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; import org.junit.Test; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_ACCOUNT_NAME; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_ID; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_SECRET; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_TENANT_ID; +import java.io.IOException; + +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; import static org.junit.Assume.assumeNotNull; public class SegmentCopyAzureServicePrincipalToTarTest extends SegmentCopyTestBase { @@ -53,11 +51,7 @@ public void testSegmentCopy() throws Exception { protected SegmentNodeStorePersistence getSrcPersistence() { String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME); String path = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, CONTAINER_NAME, DIR); - CloudBlobDirectory cloudBlobDirectory; - try (AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8()) { - cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManagerV8); - } - return new AzurePersistenceV8(cloudBlobDirectory); + return ToolUtils.createAzurePersistence(path, ENVIRONMENT); } @Override diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java index 05e7d32b9ec..f259939e537 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java @@ -231,7 +231,7 @@ private void checkManifest(SegmentNodeStorePersistence srcPersistence, SegmentNo } protected SegmentNodeStorePersistence getTarPersistence() { - return newSegmentNodeStorePersistence(SegmentStoreType.TAR, folder.getRoot().getAbsolutePath(), null); + return newSegmentNodeStorePersistence(SegmentStoreType.TAR, folder.getRoot().getAbsolutePath()); } protected SegmentNodeStorePersistence getAzurePersistence() throws Exception { diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java index 4fbb51555e2..049cfdb5eee 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java @@ -23,44 +23,30 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.Appender; import ch.qos.logback.core.read.ListAppender; -import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; -import java.net.URISyntaxException; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.jetbrains.annotations.NotNull; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.MockedStatic; import org.slf4j.LoggerFactory; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_ACCOUNT_NAME; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_ID; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_SECRET; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_SECRET_KEY; -import static org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_TENANT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeNotNull; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mockStatic; public class ToolUtilsTest { private static final Environment ENVIRONMENT = new Environment(); @@ -71,75 +57,56 @@ public class ToolUtilsTest { private static final String DEFAULT_ACCOUNT_NAME = "myaccount"; private static final String DEFAULT_CONTAINER_NAME = "oak"; private static final String DEFAULT_REPO_DIR = "repository"; - private static final String DEFAULT_CONTAINER_URL = String.format(CONTAINER_URL_FORMAT, DEFAULT_ACCOUNT_NAME, DEFAULT_CONTAINER_NAME); private static final String DEFAULT_SEGMENT_STORE_PATH = String.format(SEGMENT_STORE_PATH_FORMAT, DEFAULT_ACCOUNT_NAME, DEFAULT_CONTAINER_NAME, DEFAULT_REPO_DIR); public static final String AZURE_SECRET_KEY_WARNING = "AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."; private final TestEnvironment environment = new TestEnvironment(); - private AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8; - - @Before - public void init() { - this.azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); - } - - @After - public void clear() { - this.azureStorageCredentialManagerV8.close(); - } @Test - public void createCloudBlobDirectoryWithAccessKey() { + public void createAzurePersistenceWithAccessKey() { environment.setVariable(AZURE_SECRET_KEY, AzuriteDockerRule.ACCOUNT_KEY); final ListAppender logAppender = subscribeAppender(); - StorageCredentialsAccountAndKey credentials = expectCredentials( - StorageCredentialsAccountAndKey.class, - () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManagerV8), - DEFAULT_CONTAINER_URL - ); + AzurePersistence azurePersistence = ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH, environment); assertTrue(checkLogContainsMessage(AZURE_SECRET_KEY_WARNING, logAppender.list.stream().map(ILoggingEvent::getFormattedMessage).collect(Collectors.toList()))); assertEquals(Level.WARN, logAppender.list.get(0).getLevel()); - assertEquals(DEFAULT_ACCOUNT_NAME, credentials.getAccountName()); - assertEquals(AzuriteDockerRule.ACCOUNT_KEY, credentials.exportBase64EncodedKey()); + assertEquals(DEFAULT_ACCOUNT_NAME, azurePersistence.getReadBlobContainerClient().getAccountName()); + assertEquals(DEFAULT_CONTAINER_NAME, azurePersistence.getReadBlobContainerClient().getBlobContainerName()); unsubscribe(logAppender); } @Test - public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() { + public void createAzurePersistenceFailsWhenAccessKeyNotPresent() { environment.setVariable(AZURE_SECRET_KEY, null); assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManagerV8) + ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH, environment) ); } @Test - public void createCloudBlobDirectoryFailsWhenAccessKeyIsInvalid() { + public void createAzurePersistenceFailsWhenAccessKeyIsInvalid() { environment.setVariable(AZURE_SECRET_KEY, "invalid"); - assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManagerV8) + AzurePersistence azurePersistence = ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH, environment); + + assertThrows(Exception.class, () -> + azurePersistence.getReadBlobContainerClient().exists() ); } @Test - public void createCloudBlobDirectoryWithSasUri() { + public void createAzurePersistenceWithSasUri() { String sasToken = "sig=qL%2Fi%2BP7J6S0sA8Ihc%2BKq75U5uJcnukpfktT2fm1ckXk%3D&se=2022-02-09T11%3A52%3A42Z&sv=2019-02-02&sp=rl&sr=c"; - StorageCredentialsSharedAccessSignature credentials = expectCredentials( - StorageCredentialsSharedAccessSignature.class, - () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken, azureStorageCredentialManagerV8), - DEFAULT_CONTAINER_URL - ); + AzurePersistence azurePersistence = ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken); - assertEquals(sasToken, credentials.getToken()); - assertNull("AccountName should be null when SAS credentials are used", credentials.getAccountName()); + assertEquals(DEFAULT_CONTAINER_NAME, azurePersistence.getReadBlobContainerClient().getBlobContainerName()); } @Test - public void createCloudBlobDirectoryWithServicePrincipal() throws URISyntaxException, StorageException { + public void createAzurePersistenceWithServicePrincipal() { assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)); assumeNotNull(ENVIRONMENT.getVariable(AZURE_TENANT_ID)); assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_ID)); @@ -149,24 +116,9 @@ public void createCloudBlobDirectoryWithServicePrincipal() throws URISyntaxExcep String containerName = "oak"; String segmentStorePath = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, containerName, DEFAULT_REPO_DIR); - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT, azureStorageCredentialManagerV8); - assertNotNull(cloudBlobDirectory); - assertEquals(containerName, cloudBlobDirectory.getContainer().getName()); - } - - private static T expectCredentials(Class clazz, Runnable body, String containerUrl) { - ArgumentCaptor credentialsCaptor = ArgumentCaptor.forClass(clazz); - try (MockedStatic mockedAzureUtilities = mockStatic(AzureUtilitiesV8.class)) { - body.run(); - - mockedAzureUtilities.verify(() -> AzureUtilitiesV8.cloudBlobDirectoryFrom( - credentialsCaptor.capture(), - eq(containerUrl), - eq(DEFAULT_REPO_DIR) - ) - ); - return credentialsCaptor.getValue(); - } + AzurePersistence azurePersistence = ToolUtils.createAzurePersistence(segmentStorePath, ENVIRONMENT); + assertNotNull(azurePersistence); + assertEquals(containerName, azurePersistence.getReadBlobContainerClient().getBlobContainerName()); } private boolean checkLogContainsMessage(String toCheck, List messages) { diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java index 34f6bcc987e..b3535646e34 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java @@ -16,11 +16,9 @@ */ package org.apache.jackrabbit.oak.upgrade.cli.container; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; -import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.file.FileStore; @@ -43,8 +41,7 @@ public class SegmentAzureServicePrincipalNodeStoreContainer implements NodeStore private final BlobStore blobStore; private FileStore fs; private File tmpDir; - private AzurePersistenceV8 azurePersistenceV8; - private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8; + private AzurePersistence azurePersistence; public SegmentAzureServicePrincipalNodeStoreContainer() { this(null); @@ -52,21 +49,20 @@ public SegmentAzureServicePrincipalNodeStoreContainer() { public SegmentAzureServicePrincipalNodeStoreContainer(BlobStore blobStore) { this.blobStore = blobStore; - this.azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8(); } @Override public NodeStore open() throws IOException { try { - azurePersistenceV8 = createAzurePersistence(); + azurePersistence = createAzurePersistence(); } catch (Exception e) { throw new IllegalStateException(e); } tmpDir = Files.createTempDirectory(getClass().getSimpleName() + "-").toFile(); FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(tmpDir) - .withCustomPersistence(azurePersistenceV8).withMemoryMapping(false); + .withCustomPersistence(azurePersistence).withMemoryMapping(false); if (blobStore != null) { builder.withBlobStore(blobStore); } @@ -80,14 +76,13 @@ public NodeStore open() throws IOException { return new FileStoreUtils.NodeStoreWithFileStore(SegmentNodeStoreBuilders.builder(fs).build(), fs); } - private AzurePersistenceV8 createAzurePersistence() { - if (azurePersistenceV8 != null) { - return azurePersistenceV8; + private AzurePersistence createAzurePersistence() { + if (azurePersistence != null) { + return azurePersistence; } - String path = String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilitiesV8.AZURE_ACCOUNT_NAME), + String path = String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), CONTAINER_NAME, DIR); - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManagerV8); - return new AzurePersistenceV8(cloudBlobDirectory); + return ToolUtils.createAzurePersistence(path, ENVIRONMENT); } @Override @@ -99,16 +94,13 @@ public void close() { if (tmpDir != null) { tmpDir.delete(); } - if (azureStorageCredentialManagerV8 != null) { - azureStorageCredentialManagerV8.close(); - } } @Override public void clean() throws IOException { - AzurePersistenceV8 azurePersistenceV8 = createAzurePersistence(); + AzurePersistence azurePersistence = createAzurePersistence(); try { - AzureUtilitiesV8.deleteAllBlobs(azurePersistenceV8.getSegmentstoreDirectory()); + AzureUtilities.deleteAllEntries(azurePersistence.getReadBlobContainerClient(), null); } catch (Exception e) { throw new IOException(e); } @@ -116,7 +108,7 @@ public void clean() throws IOException { @Override public String getDescription() { - return "az:" + String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilitiesV8.AZURE_ACCOUNT_NAME), + return "az:" + String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), CONTAINER_NAME, DIR); } } \ No newline at end of file diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java index 393cd34b283..1114121348b 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java @@ -21,12 +21,12 @@ import com.microsoft.azure.storage.SharedAccessAccountPolicy; import com.microsoft.azure.storage.SharedAccessAccountResourceType; import com.microsoft.azure.storage.SharedAccessAccountService; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.commons.lang3.StringUtils; -import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import org.apache.jackrabbit.oak.commons.pio.Closer; +import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8; -import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.upgrade.cli.CliUtils; @@ -122,7 +122,7 @@ public void testConnectionWithUri_accessKey() throws IOException { assertEquals(1, nodeStore.getFileStore().getSegmentCount()); } finally { closer.close(); - cleanup(uri, azureStorageCredentialManagerV8); + cleanup(uri); } } } @@ -149,16 +149,16 @@ public void testConnectionWithUri_servicePrincipal() throws IOException, Interru assertEquals(1, nodeStore.getFileStore().getSegmentCount()); } finally { closer.close(); - cleanup(uri, azureStorageCredentialManagerV8); + cleanup(uri); } } } - private void cleanup(String uri, AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8) { + private void cleanup(String uri) { uri = uri + "/" + DIR; try { - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT, azureStorageCredentialManagerV8); - AzureUtilitiesV8.deleteAllBlobs(cloudBlobDirectory); + AzurePersistence azurePersistence = ToolUtils.createAzurePersistence(uri, ENVIRONMENT); + AzureUtilities.deleteAllEntries(azurePersistence.getReadBlobContainerClient(), null); } catch (Exception e) { throw new IllegalStateException(e); }