diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs index f866944179..9cd5d32576 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs @@ -35,11 +35,13 @@ namespace Org.Apache.REEF.IO.Tests /// public sealed class TestAzureBlockBlobFileSystemE2E : IDisposable { - private const string SkipMessage = "Fill in credentials before running test"; // Use null to run tests + // Uncomment SkipMessage = null to run tests + private const string SkipMessage = "Fill in credentials before running test"; + // private const string SkipMessage = null; private const string HelloFile = "hello"; - private IFileSystem _fileSystem; - private CloudBlobClient _client; - private CloudBlobContainer _container; + private readonly IFileSystem _fileSystem; + private readonly CloudBlobClient _client; + private readonly CloudBlobContainer _container; public TestAzureBlockBlobFileSystemE2E() { @@ -66,26 +68,22 @@ public void Dispose() private bool CheckBlobExists(ICloudBlob blob) { - var task = blob.ExistsAsync(); - return task.Result; + return blob.ExistsAsync().GetAwaiter().GetResult(); } private bool CheckContainerExists(CloudBlobContainer container) { - var task = container.ExistsAsync(); - return task.Result; + return container.ExistsAsync().GetAwaiter().GetResult(); } private ICloudBlob GetBlobReferenceFromServer(CloudBlobContainer container, string blobName) { - var task = container.GetBlobReferenceFromServerAsync(blobName); - return task.Result; + return container.GetBlobReferenceFromServerAsync(blobName).GetAwaiter().GetResult(); } private string DownloadText(CloudBlockBlob blob) { - var task = blob.DownloadTextAsync(); - return task.Result; + return blob.DownloadTextAsync().GetAwaiter().GetResult(); } [Fact(Skip = SkipMessage)] @@ -161,9 +159,9 @@ public void TestGetChildBlobsInContainerE2E() [Fact(Skip = SkipMessage)] public void TestGetChildContainerInStorageAccountE2E() { - // List containers in the storage account - Uri rootUri = _fileSystem.CreateUriForPath(string.Empty); - ValidateChildren(rootUri, new List { _container.Uri }); + // List container uris in the storage account + var containerUris = _client.ListContainers().Select(container => container.Uri); + ValidateChildren(_client.BaseUri, containerUris); } private void ValidateChildren(Uri storageBlobUri, IEnumerable expectedChildBlobs) @@ -264,6 +262,37 @@ public void TestCopyFromLocalE2E() container.DeleteIfExistsAsync().Wait(); } + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryValidDirectoryLevel1E2E() + { + const string Directory = "dir"; + CreateTempBlobs(Directory); + Assert.True(_fileSystem.IsDirectory(PathToFile(Directory))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryValidDirectoryLevel2E2E() + { + const string Directory = "dir1/dir2"; + CreateTempBlobs(Directory); + Assert.True(_fileSystem.IsDirectory(PathToFile(Directory))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFakeDirectoryE2E() + { + const string Directory = "dir"; + Assert.False(_fileSystem.IsDirectory(PathToFile(Directory))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFileE2E() + { + const string Directory = "dir"; + var blockBlobs = CreateTempBlobs(Directory); + Assert.False(_fileSystem.IsDirectory(blockBlobs.First().Uri)); + } + [Fact(Skip = SkipMessage)] public void TestDeleteDirectoryAtContainerE2E() { @@ -274,16 +303,8 @@ public void TestDeleteDirectoryAtContainerE2E() [Fact(Skip = SkipMessage)] public void TestDeleteDirectoryFirstLevelE2E() { - const string Directory = "dir"; - var blockBlobs = new List(); - for (var i = 0; i < 3; i++) - { - var filePath = Directory + '/' + i; - var blockBlob = _container.GetBlockBlobReference(filePath); - UploadFromString(blockBlob, "hello"); - Assert.True(CheckBlobExists(blockBlob)); - blockBlobs.Add(blockBlob); - } + const string Directory = "dir1"; + var blockBlobs = CreateTempBlobs(Directory); _fileSystem.DeleteDirectory(PathToFile(Directory)); @@ -299,24 +320,11 @@ public void TestDeleteDirectoryFirstLevelE2E() public void TestDeleteDirectorySecondLevelE2E() { const string Directory1 = "dir1"; - const string Directory2 = "dir2"; - var blockBlobs1 = new List(); - var blockBlobs2 = new List(); - for (var i = 0; i < 3; i++) - { - var filePath1 = Directory1 + '/' + i; - var filePath2 = Directory1 + '/' + Directory2 + '/' + i; - var blockBlob1 = _container.GetBlockBlobReference(filePath1); - var blockBlob2 = _container.GetBlockBlobReference(filePath2); - UploadFromString(blockBlob1, "hello"); - UploadFromString(blockBlob2, "hello"); - Assert.True(CheckBlobExists(blockBlob1)); - Assert.True(CheckBlobExists(blockBlob2)); - blockBlobs1.Add(blockBlob1); - blockBlobs2.Add(blockBlob2); - } + const string Directory2 = "dir1/dir2"; + var blockBlobs1 = CreateTempBlobs(Directory1); + var blockBlobs2 = CreateTempBlobs(Directory2); - _fileSystem.DeleteDirectory(PathToFile(Directory1 + '/' + Directory2)); + _fileSystem.DeleteDirectory(PathToFile(Directory2)); foreach (var blockBlob in blockBlobs2) { @@ -331,6 +339,18 @@ public void TestDeleteDirectorySecondLevelE2E() Assert.True(CheckContainerExists(_container)); } + private IEnumerable CreateTempBlobs(string directory, int fileCount = 3) + { + return Enumerable.Range(0, fileCount).Select(i => + { + var filePath = directory + '/' + i; + var blockBlob = _container.GetBlockBlobReference(filePath); + UploadFromString(blockBlob, "hello"); + Assert.True(CheckBlobExists(blockBlob), "Blob does not exist: " + filePath); + return blockBlob; + }); + } + private static void UploadFromString(ICloudBlob blob, string str) { var byteArray = Encoding.UTF8.GetBytes(str); diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs index 131c941f76..6d4841b699 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs @@ -194,6 +194,28 @@ public void TestCreateDirectoryE2E() Assert.True(_adlsClient.CheckExists(dirName)); } + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryValidDirectoryE2E() + { + string dirName = $"/{_defaultFolderName}"; + _adlsClient.CreateDirectory(dirName); + Assert.True(_fileSystem.IsDirectory(PathToFile(dirName))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFakeDirectoryE2E() + { + string fakeDirName = $"/fakeDir"; + Assert.False(_fileSystem.IsDirectory(PathToFile(fakeDirName))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFileE2E() + { + string fileName = UploadFromString(ContentsText); + Assert.False(_fileSystem.IsDirectory(PathToFile(fileName))); + } + [Fact(Skip = SkipMessage)] public void TestDeleteDirectoryE2E() { diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs index 94297a55e8..7872bc8286 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs @@ -33,6 +33,7 @@ namespace Org.Apache.REEF.IO.Tests /// public sealed class TestHadoopFileSystem { + private const string SkipMessage = "These tests need to be run in an environment with HDFS installed."; // Use null to run tests private HadoopFileSystem _fileSystem; private Uri GetTempUri() @@ -56,7 +57,7 @@ public TestHadoopFileSystem() /// /// Creates a temp file locally, uploads it to HDFS and downloads it again. /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestCopyFromLocalAndBack() { var localFile = FileSystemTestUtilities.MakeLocalTempFile(); @@ -77,7 +78,7 @@ public void TestCopyFromLocalAndBack() /// /// Tests whether .Exists() works. /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestExists() { var remoteUri = GetTempUri(); @@ -93,7 +94,7 @@ public void TestExists() /// /// Tests for .GetChildren(). /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestGetChildren() { // Make a directory @@ -129,14 +130,14 @@ public void TestGetChildren() _fileSystem.DeleteDirectory(remoteDirectory); } - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestOpen() { // Open() is not supported by HadoopFileSystem. Use CopyToLocal and open the local file instead. Assert.Throws(() => _fileSystem.Open(GetTempUri())); } - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestCreate() { // Create() is not supported by HadoopFileSystem. Create a local file and use CopyFromLocal instead. @@ -163,10 +164,56 @@ public void CreateUriForPathWithPrefix() Assert.Equal(new Uri(uriString), _fileSystem.CreateUriForPath(uriString)); } + /// + /// Tests whether .IsDirectory() works for valid directory. + /// + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryValidDirectory() + { + // Create directory + var remoteDirUri = GetTempUri(); + _fileSystem.CreateDirectory(remoteDirUri); + + Assert.True(_fileSystem.IsDirectory(remoteDirUri)); + + // Clean up + _fileSystem.DeleteDirectory(remoteDirUri); + } + + /// + /// Tests whether .IsDirectory() works for fake directory. + /// + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFakeDirectory() + { + // Create fake directory uri + var remoteFakeuri = GetTempUri(); + + Assert.False(_fileSystem.IsDirectory(remoteFakeuri)); + } + + /// + /// Tests whether .IsDirectory() works for file. + /// + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFile() + { + // Create file + var remoteFileUri = GetTempUri(); + var localFile = FileSystemTestUtilities.MakeLocalTempFile(); + _fileSystem.CopyFromLocal(localFile, remoteFileUri); + + Assert.False(_fileSystem.IsDirectory(remoteFileUri)); + + // Clean up + _fileSystem.Delete(remoteFileUri); + File.Delete(localFile); + } + /// /// This test is to make sure with the HadoopFileSystemConfiguration, HadoopFileSystem can be injected. /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestHadoopFileSystemConfiguration() { var fileSystemTest = TangFactory.GetTang().NewInjector(HadoopFileSystemConfiguration.ConfigurationModule diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs index 6272cb603a..8e1e226f9a 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs @@ -136,6 +136,29 @@ public void TestCopy() Assert.False(fs.Exists(sourceUri)); } + [Fact] + public void TestIsDirectory() + { + var fs = GetFileSystem(); + // Directory check + var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), "dir")); + fs.CreateDirectory(directoryUri); + Assert.True(fs.IsDirectory(directoryUri), ".IsDirectory() failed on: " + directoryUri); + + // File check + var fileUri = new Uri(directoryUri, "testfile"); + MakeRemoteTestFile(fs, fileUri); + Assert.False(fs.IsDirectory(fileUri), ".IsDirectory() failed on: " + fileUri); + + // Fake directory check + var fakeDirectoryUri = new Uri(Path.Combine(Path.GetTempPath(), "fakeDir")); + Assert.False(fs.IsDirectory(fakeDirectoryUri), ".IsDirectory() failed on: " + fakeDirectoryUri); + + // Clean up + fs.Delete(fileUri); + fs.DeleteDirectory(directoryUri); + } + [Fact] public void TestGetChildren() { diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs index d1004c387c..09f527c989 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs @@ -125,13 +125,32 @@ public void CreateDirectory(Uri directoryUri) { } + /// + /// Checks if uri is a directory uri. + /// + /// uri of the directory/file + /// true if uri is for a directory else false + public bool IsDirectory(Uri uri) + { + var path = uri.AbsolutePath.TrimStart(UrlPathSeparator); + var blobItems = _client.ListBlobsSegmented( + path, + useFlatListing: false, + BlobListingDetails.Metadata, + maxResults: 1, + continuationToken: null, + blobRequestOptions: null, + operationContext: null).Results; + return blobItems.OfType().Any(); + } + /// /// Recursively deletes blobs under a specified "directory URI." /// If only the container is specified, the entire container is deleted. /// public void DeleteDirectory(Uri directoryUri) { - var uriSplit = directoryUri.AbsolutePath.Split(new[] { "/" }, StringSplitOptions.RemoveEmptyEntries); + var uriSplit = directoryUri.AbsolutePath.Split(new[] { UrlPathSeparator }, StringSplitOptions.RemoveEmptyEntries); if (!uriSplit.Any()) { throw new StorageException( @@ -197,7 +216,7 @@ public IEnumerable GetChildren(Uri directoryUri) do { - BlobResultSegment listing = _client.ListBlobsSegmented( + BlobResultSegment listing = _client.ListDirectoryBlobsSegmented( containerName, relativeAddress, useFlatListing: false, diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs index eec480673c..72ed015b84 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs @@ -55,8 +55,7 @@ public Uri BaseUri public ICloudBlob GetBlobReferenceFromServer(Uri blobUri) { - var task = _client.GetBlobReferenceFromServerAsync(blobUri); - return task.Result; + return _client.GetBlobReferenceFromServerAsync(blobUri).GetAwaiter().GetResult(); } public ICloudBlobContainer GetContainerReference(string containerName) @@ -70,6 +69,25 @@ public ICloudBlockBlob GetBlockBlobReference(Uri uri) } public BlobResultSegment ListBlobsSegmented( + string prefix, + bool useFlatListing, + BlobListingDetails blobListingDetails, + int? maxResults, + BlobContinuationToken continuationToken, + BlobRequestOptions blobRequestOptions, + OperationContext operationContext) + { + return _client.ListBlobsSegmentedAsync( + prefix, + useFlatListing, + blobListingDetails, + maxResults, + continuationToken, + blobRequestOptions, + operationContext).GetAwaiter().GetResult(); + } + + public BlobResultSegment ListDirectoryBlobsSegmented( string containerName, string relativeAddress, bool useFlatListing, diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs index e2fbd6161d..53ae2aca14 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs @@ -35,8 +35,7 @@ public AzureCloudBlobContainer(CloudBlobContainer container, BlobRequestOptions public bool CreateIfNotExists() { - var task = _container.CreateIfNotExistsAsync(_requestOptions, null); - return task.Result; + return _container.CreateIfNotExistsAsync(_requestOptions, null).GetAwaiter().GetResult(); } public void DeleteIfExists() diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs index 4d3295e637..43963084fd 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs @@ -39,8 +39,13 @@ public ICloudBlobDirectory GetDirectoryReference(string directoryName) public IEnumerable ListBlobs(bool useFlatListing = false) { - var task = _directory.ListBlobsSegmentedAsync(useFlatListing, BlobListingDetails.All, null, null, null, null); - return task.Result.Results; + return _directory.ListBlobsSegmentedAsync( + useFlatBlobListing: useFlatListing, + BlobListingDetails.All, + maxResults: null, + null, + null, + null).GetAwaiter().GetResult().Results; } } } diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs index c8835687d1..26e9ec9fdf 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs @@ -67,7 +67,7 @@ public Stream Open() { #if REEF_DOTNET_BUILD var task = _blob.OpenReadAsync(null, _requestOptions, null); - return task.Result; + return task.GetAwaiter().GetResult(); #else return _blob.OpenRead(null, _requestOptions, null); #endif @@ -76,8 +76,7 @@ public Stream Open() public Stream Create() { #if REEF_DOTNET_BUILD - var task = _blob.OpenWriteAsync(null, _requestOptions, null); - return task.Result; + return _blob.OpenWriteAsync(null, _requestOptions, null).GetAwaiter().GetResult(); #else return _blob.OpenWrite(null, _requestOptions, null); #endif @@ -85,8 +84,7 @@ public Stream Create() public bool Exists() { - var task = _blob.ExistsAsync(_requestOptions, null); - return task.Result; + return _blob.ExistsAsync(_requestOptions, null).GetAwaiter().GetResult(); } public void Delete() @@ -101,8 +99,7 @@ public void DeleteIfExists() public string StartCopy(Uri source) { - var task = _blob.StartCopyAsync(source, null, null, _requestOptions, null); - return task.Result; + return _blob.StartCopyAsync(source, null, null, _requestOptions, null).GetAwaiter().GetResult(); } public void DownloadToFile(string path, FileMode mode) diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs index f2a43aa7d4..d4ce481177 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs @@ -56,9 +56,21 @@ internal interface ICloudBlobClient ICloudBlockBlob GetBlockBlobReference(Uri uri); /// - /// Paginates a blob listing with container and relative path. + /// Paginates a blob listing with prefix. /// BlobResultSegment ListBlobsSegmented( + string prefix, + bool useFlatListing, + BlobListingDetails blobListingDetails, + int? maxResults, + BlobContinuationToken continuationToken, + BlobRequestOptions blobRequestOptions, + OperationContext operationContext); + + /// + /// Paginates a blob listing with container and relative path. + /// + BlobResultSegment ListDirectoryBlobsSegmented( string containerName, string relativeAddress, bool useFlatListing, diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs index d1ad20aa27..b0f250c0d8 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs @@ -17,8 +17,10 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; +using System.Net; using System.Text; using Microsoft.Azure.DataLake.Store; using Microsoft.Azure.DataLake.Store.FileTransfer; @@ -178,14 +180,34 @@ public void CreateDirectory(Uri directoryUri) } } + /// + /// Checks if uri is a directory uri. + /// + /// uri of the directory/file + /// true if uri is for a directory else false + public bool IsDirectory(Uri uri) + { + try + { + return _adlsClient.GetDirectoryEntry(uri.AbsolutePath).Type == DirectoryEntryType.DIRECTORY; + } + catch (AdlsException e) + { + if (e.HttpStatus == HttpStatusCode.NotFound) + { + return false; + } + throw e; + } + } + /// /// Deletes a directory. /// /// If directory cannot be deleted public void DeleteDirectory(Uri directoryUri) { - bool deleteStatus = Exists(directoryUri) && - _adlsClient.GetDirectoryEntry(directoryUri.AbsolutePath).Type == DirectoryEntryType.DIRECTORY && + bool deleteStatus = IsDirectory(directoryUri) && _adlsClient.DeleteRecursive(directoryUri.AbsolutePath); if (!deleteStatus) { @@ -228,10 +250,10 @@ public Uri CreateUriForPath(string path) } catch (UriFormatException) { - resultUri = new Uri(new Uri(this.UriPrefix), path); + resultUri = new Uri(new Uri(UriPrefix), path); } - if (!resultUri.AbsoluteUri.StartsWith(this.UriPrefix)) + if (!resultUri.AbsoluteUri.StartsWith(UriPrefix, true, CultureInfo.InvariantCulture)) { throw new ArgumentException($"Given URI must begin with valid prefix ({this.UriPrefix})", nameof(path)); } diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs index cd80b629ab..4f6e1cc183 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs @@ -17,7 +17,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Text.RegularExpressions; @@ -152,6 +151,13 @@ public void CreateDirectory(Uri directoryUri) _commandRunner.Run("dfs -mkdir " + directoryUri); } + public bool IsDirectory(Uri uri) + { + // TODO[JIRA REEF - 2039]: HadoopFileSystem .IsDirectory() check needs to work on linux machines. + var stdOut = _commandRunner.Run("dfs -test -d " + uri + "& call echo %^errorlevel%").StdOut; + return stdOut.FirstOrDefault() == "0"; + } + public void DeleteDirectory(Uri directoryUri) { _commandRunner.Run("dfs -rmdir " + directoryUri); diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs index 41e12cdde5..7d3b98318c 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs @@ -90,6 +90,13 @@ public interface IFileSystem /// void CreateDirectory(Uri directoryUri); + /// + /// Checks if uri is a directory uri. + /// + /// uri of the directory/file + /// true if uri is for a directory else false + bool IsDirectory(Uri uri); + /// /// Deletes a directory. /// diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs index 313fdd9fb7..ca5a7efedb 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs @@ -105,6 +105,11 @@ public void CreateDirectory(Uri directoryUri) Directory.CreateDirectory(directoryUri.LocalPath); } + public bool IsDirectory(Uri uri) + { + return Directory.Exists(uri.LocalPath); + } + public void DeleteDirectory(Uri directoryUri) { Directory.Delete(directoryUri.LocalPath, true);