Merge pull request #135 from IndicoDataSolutions/nate/resolve_unit_tests

nateshim-indico · web-flow · commit b411c129e2e0 · 2024-01-31T12:51:21.000-08:00
[CAT-892] resolve unit tests
diff --git a/IndicoV2.Abstractions/Submissions/Models/SubmissionStatus.cs b/IndicoV2.Abstractions/Submissions/Models/SubmissionStatus.cs
@@ -3,10 +3,10 @@
     public enum SubmissionStatus
     {
         PROCESSING,
-        FAILED,
-        COMPLETE,
-        PENDING_ADMIN_REVIEW,
+        PENDING_AUTO_REVIEW,
         PENDING_REVIEW,
-        PENDING_AUTO_REVIEW
+        PENDING_ADMIN_REVIEW,
+        COMPLETE,
+        FAILED
     }
 }
diff --git a/IndicoV2.IntegrationTests/DataSets/DataSetClientTests.cs b/IndicoV2.IntegrationTests/DataSets/DataSetClientTests.cs
@@ -17,6 +17,8 @@ public class DataSetClientTests
         private DataHelper _dataHelper;
         private IndicoConfigs _indicoConfigs;
         private int _dataSetId;
+        private int _documentDataSetId;
+        private int _csvDataSetId;
 
         [SetUp]
         public async Task SetUp()
@@ -29,12 +31,16 @@ public async Task SetUp()
             var _rawDataSetId = _indicoConfigs.DatasetId;
             if (_rawDataSetId == 0)
             {
-                var dataset = (await _dataHelper.DataSets().GetAny());
+                var dataset = await _dataHelper.DataSets().GetAny();
                 _dataSetId = dataset.Id;
+                _documentDataSetId = _dataSetId;
+                _csvDataSetId = _dataSetId;
             }
             else
             {
                 _dataSetId = _rawDataSetId;
+                _documentDataSetId = _indicoConfigs.DocumentDatasetId;
+                _csvDataSetId = _indicoConfigs.CsvDatasetId;
             }
         }
 
@@ -88,15 +94,14 @@ public async Task ProcessFiles_ShouldStartProcessing()
         {
             // Arrange
             var files = new[] {_dataHelper.Files().GetSampleFilePath()};
-            await _dataSetClient.AddFilesAsync(_dataSetId, files, default);
+            await _dataSetClient.AddFilesAsync(_documentDataSetId, files, default);
             var downloadedFiles =
-                (await _dataSetClient.FileUploadStatusAsync(_dataSetId, default))
+                (await _dataSetClient.FileUploadStatusAsync(_documentDataSetId, default))
                 .Dataset.Files
                 .Where(f => f.Status == FileStatus.Downloaded)
                 .Select(f => f.Id.Value);
-
             // Act
-            var result = await _dataSetClient.ProcessFileAsync(_dataSetId, downloadedFiles, default);
+            var result = await _dataSetClient.ProcessFileAsync(_documentDataSetId, downloadedFiles, default);
 
             // Assert
             result.Should().NotBeNull();
@@ -107,15 +112,14 @@ public async Task ProcessCsv_ShouldStartProcessing()
         {
             // Arrange
             var files = new[] { _dataHelper.Files().GetSampleCsvPath() };
-            await _dataSetClient.AddFilesAsync(_dataSetId, files, default);
+            await _dataSetClient.AddFilesAsync(_csvDataSetId, files, default);
             var downloadedFiles =
-                (await _dataSetClient.FileUploadStatusAsync(_dataSetId, default))
+                (await _dataSetClient.FileUploadStatusAsync(_csvDataSetId, default))
                 .Dataset.Files
                 .Where(f => f.Status == FileStatus.Downloaded)
                 .Select(f => f.Id.Value);
-
             // Act
-            var result = await _dataSetClient.ProcessCsvAsync(_dataSetId, downloadedFiles, default);
+            var result = await _dataSetClient.ProcessCsvAsync(_csvDataSetId, downloadedFiles, default);
 
             // Assert
             result.Should().NotBeNull();
diff --git a/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs b/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs
@@ -77,7 +77,7 @@ public async Task CreateAsync_ShouldCreateSubmission_FromFilePath()
             var filePath = _dataHelper.Files().GetSampleFilePath();
 
             // Act
-            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, new[] { filePath });
+            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, paths: new[] { filePath });
 
             // Assert
             var submissionId = submissionIds.Single();
@@ -91,13 +91,13 @@ public async Task CreateAsync_ShouldCreateSubmission_FromUri()
             var uri = _dataHelper.Uris().GetSampleUri();
 
             // Act
-            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, new[] { uri });
+            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, uris: new[] { uri });
 
             // Assert
             var submissionId = submissionIds.Single();
             submissionId.Should().BeGreaterThan(0);
         }
-        
+
         [Test]
         public async Task GetAsync_ShouldFetchSubmission()
         {
diff --git a/IndicoV2.IntegrationTests/Utils/Configs/IndicoConfigs.cs b/IndicoV2.IntegrationTests/Utils/Configs/IndicoConfigs.cs
@@ -8,6 +8,9 @@ internal class IndicoConfigs
 
         public int DatasetId => ParseEnvVar("INDICO_TEST_DATASET_ID");
 
+        public int DocumentDatasetId => ParseEnvVar("INDICO_TEST_DOCUMENT_DATASET_ID");
+        public int CsvDatasetId => ParseEnvVar("INDICO_TEST_CSV_DATASET_ID");
+
         public int ModelGroupId => ParseEnvVar("INDICO_TEST_MODELGROUP_ID");
 
         public static int ParseEnvVar(string varName)
diff --git a/IndicoV2.IntegrationTests/Utils/DataHelpers/Files/workflow-sample.pdf b/IndicoV2.IntegrationTests/Utils/DataHelpers/Files/workflow-sample.pdf
diff --git a/IndicoV2.StrawberryShake/DataSets/DataSetSsClient.cs b/IndicoV2.StrawberryShake/DataSets/DataSetSsClient.cs
@@ -40,7 +40,7 @@ public Task<IDataSetGetDatasetsResult> ListAsync(CancellationToken cancellationT
 
         public Task<IDataSetGetDatasetsFullResult> ListFullAsync(int? limit, CancellationToken cancellationToken) =>
             ExecuteAsync(() => _services.GetRequiredService<DataSetGetDatasetsFullQuery>().ExecuteAsync(
-                limit,
+                limit == null ? 100 : limit,
                 cancellationToken));
     }
 }
diff --git a/IndicoV2.StrawberryShake/Exceptions/ErrorHandlingWrapper.cs b/IndicoV2.StrawberryShake/Exceptions/ErrorHandlingWrapper.cs
@@ -11,7 +11,6 @@ protected async Task<TResult> ExecuteAsync<TResult>(Func<Task<IOperationResult<T
             where TResult : class
         {
             var result = await executeAsync();
-
             if (result.Errors != null && result.Errors.Any())
             {
                 throw new GraphQlException(result);
diff --git a/IndicoV2.StrawberryShake/Submissions/SubmissionSsClient.cs b/IndicoV2.StrawberryShake/Submissions/SubmissionSsClient.cs
@@ -20,7 +20,7 @@ public async Task<IEnumerable<int>> CreateUri(int workflowId, IEnumerable<Uri> u
             CancellationToken cancellationToken = default, SubmissionResultVersion? resultsFileVersion = null) =>
             (await ExecuteAsync(async () => await _services.GetRequiredService<WorkflowUrlSubmissionMutation>().ExecuteAsync(
                 workflowId,
-                (IReadOnlyList<string>)uris,
+                uris.Select(s => s.ToString()).ToList(),
                 resultsFileVersion,
                 cancellationToken
             )))
diff --git a/IndicoV2/Ocr/OcrClient.cs b/IndicoV2/Ocr/OcrClient.cs
@@ -33,7 +33,7 @@ internal async Task<JArray> Upload(List<string> filePaths)
         public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtractionPreset preset, CancellationToken cancellationToken)
         {
             JArray fileMetadata;
-            var files = new List<object>();
+            var files = new List<FileInput>();
             fileMetadata = await Upload(new List<string> { filePath });
             foreach (JObject uploadMeta in fileMetadata)
             {
@@ -44,10 +44,10 @@ public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtracti
                     { "upload_type", (string)uploadMeta.GetValue("upload_type") }
                 };
 
-                var file = new
+                var file = new FileInput
                 {
-                    filename = (string)uploadMeta.GetValue("name"),
-                    filemeta = meta.ToString()
+                    Filename = (string)uploadMeta.GetValue("name"),
+                    Filemeta = meta.ToString()
                 };
 
                 files.Add(file);
@@ -62,7 +62,7 @@ public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtracti
             return job.DocumentExtraction.JobIds.First();
         }
         public async Task<string> GetExtractionResultAsync(Uri documentUri, CancellationToken cancellationToken) =>
-            (await GetExtractionResultAsync<JToken>(documentUri, cancellationToken)).Value<string>("text");
+            (await GetExtractionResultAsync<JToken>(documentUri, cancellationToken)).ToString();
 
         public async Task<TResult> GetExtractionResultAsync<TResult>(Uri documentUri, CancellationToken cancellationToken)
         {
diff --git a/IndicoV2/Submissions/SubmissionsClient.cs b/IndicoV2/Submissions/SubmissionsClient.cs
@@ -68,7 +68,8 @@ public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<Uri>
             await _strawberryShakeClient.Submissions().CreateUri(workflowId, uris, cancellationToken);
 
         public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<string> paths,
-          CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null) {
+          CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null)
+        {
             var filesToUpload = new List<(string Name, Stream content)>();
             foreach (var path in paths)
             {
@@ -83,12 +84,16 @@ public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<stri
                 }
 
             }
-            return await _strawberryShakeClient.Submissions().Create(workflowId, (IEnumerable<(string Name, string Meta)>)filesToUpload, cancellationToken, (SubmissionResultVersion?)resultsFileVersion);
+            return await CreateAsync(workflowId, filesToUpload: filesToUpload, cancellationToken, resultsFileVersion);
         }
 
         [Obsolete("This is the Legacy version and will be deprecated. Please use ListAsync instead.")]
         public async Task<IEnumerable<ISubmission>> ListAsync(IEnumerable<int> submissionIds, IEnumerable<int> workflowIds, IFilter filters, int limit = 1000,
-            CancellationToken cancellationToken = default) => (IEnumerable<ISubmission>)await ListAsync(submissionIds, workflowIds, filters, null, limit, cancellationToken);
+            CancellationToken cancellationToken = default)
+        {
+            var result = await ListAsync(submissionIds, workflowIds, filters, null, limit, cancellationToken);
+            return result.Data;
+        }
 
 
         public async Task<IHasCursor<IEnumerable<ISubmission>>> ListAsync(IEnumerable<int> submissionIds, IEnumerable<int> workflowIds, IFilter filters, int? after, int limit = 1000, CancellationToken cancellationToken = default)
@@ -116,14 +121,14 @@ public async Task<IHasCursor<IEnumerable<ISubmission>>> ListAsync(IEnumerable<in
         public async Task<ISubmission> GetAsync(int submissionId, CancellationToken cancellationToken = default)
         {
             var result = await _strawberryShakeClient.Submissions().Get(submissionId, cancellationToken);
-            if (!Enum.TryParse(result.Status.ToString().ToUpper(),out Models.SubmissionStatus parsed))
+            if (!Enum.IsDefined(typeof(StrawberryShake.SubmissionStatus), result.Status))
             {
                 throw new NotSupportedException($"Cannot read submission status: {result.Status}");
             }
             return new Submission
             {
                 Id = result.Id ?? 0,
-                Status = parsed,
+                Status = (Models.SubmissionStatus) result.Status,
                 DatasetId = result.DatasetId ?? 0,
                 WorkflowId = result.WorkflowId ?? 0,
                 InputFile = result.InputFile,
@@ -141,7 +146,7 @@ public async Task<string> GenerateSubmissionResultAsync(int submissionId, Cancel
         public async Task<ISubmission> MarkSubmissionAsRetrieved(int submissionId, bool retrieved = true, CancellationToken cancellationToken = default)
         {
             var resultId = await _strawberryShakeClient.Submissions().MarkRetrieved(submissionId, retrieved, cancellationToken);
-            var result = await _strawberryShakeClient.Submissions().List((IReadOnlyList<int?>)new List<int>(submissionId).AsReadOnly(), default, default, default, default, cancellationToken);
+            var result = await _strawberryShakeClient.Submissions().List(new List<int?>(submissionId).AsReadOnly(), default, default, default, default, cancellationToken);
             return new SubmissionSs(result?.Submissions?[0]);
         }
 

Original file line number	Diff line number	Diff line change
`@@ -3,10 +3,10 @@`
`3`	`3`	`public enum SubmissionStatus`
`4`	`4`	`{`
`5`	`5`	`PROCESSING,`
`6`		`- FAILED,`
`7`		`- COMPLETE,`
`8`		`- PENDING_ADMIN_REVIEW,`
	`6`	`+ PENDING_AUTO_REVIEW,`
`9`	`7`	`PENDING_REVIEW,`
`10`		`- PENDING_AUTO_REVIEW`
	`8`	`+ PENDING_ADMIN_REVIEW,`
	`9`	`+ COMPLETE,`
	`10`	`+ FAILED`
`11`	`11`	`}`
`12`	`12`	`}`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ public Task<IDataSetGetDatasetsResult> ListAsync(CancellationToken cancellationT`
`40`	`40`
`41`	`41`	`public Task<IDataSetGetDatasetsFullResult> ListFullAsync(int? limit, CancellationToken cancellationToken) =>`
`42`	`42`	`ExecuteAsync(() => _services.GetRequiredService<DataSetGetDatasetsFullQuery>().ExecuteAsync(`
`43`		`- limit,`
	`43`	`+ limit == null ? 100 : limit,`
`44`	`44`	`cancellationToken));`
`45`	`45`	`}`
`46`	`46`	`}`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,6 @@ protected async Task<TResult> ExecuteAsync<TResult>(Func<Task<IOperationResult<T`
`11`	`11`	`where TResult : class`
`12`	`12`	`{`
`13`	`13`	`var result = await executeAsync();`
`14`		`-`
`15`	`14`	`if (result.Errors != null && result.Errors.Any())`
`16`	`15`	`{`
`17`	`16`	`throw new GraphQlException(result);`
Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ internal async Task<JArray> Upload(List<string> filePaths)`
`33`	`33`	`public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtractionPreset preset, CancellationToken cancellationToken)`
`34`	`34`	`{`
`35`	`35`	`JArray fileMetadata;`
`36`		`- var files = new List<object>();`
	`36`	`+ var files = new List<FileInput>();`
`37`	`37`	`fileMetadata = await Upload(new List<string> { filePath });`
`38`	`38`	`foreach (JObject uploadMeta in fileMetadata)`
`39`	`39`	`{`
`@@ -44,10 +44,10 @@ public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtracti`
`44`	`44`	`{ "upload_type", (string)uploadMeta.GetValue("upload_type") }`
`45`	`45`	`};`
`46`	`46`
`47`		`- var file = new`
	`47`	`+ var file = new FileInput`
`48`	`48`	`{`
`49`		`- filename = (string)uploadMeta.GetValue("name"),`
`50`		`- filemeta = meta.ToString()`
	`49`	`+ Filename = (string)uploadMeta.GetValue("name"),`
	`50`	`+ Filemeta = meta.ToString()`
`51`	`51`	`};`
`52`	`52`
`53`	`53`	`files.Add(file);`
`@@ -62,7 +62,7 @@ public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtracti`
`62`	`62`	`return job.DocumentExtraction.JobIds.First();`
`63`	`63`	`}`
`64`	`64`	`public async Task<string> GetExtractionResultAsync(Uri documentUri, CancellationToken cancellationToken) =>`
`65`		`- (await GetExtractionResultAsync<JToken>(documentUri, cancellationToken)).Value<string>("text");`
	`65`	`+ (await GetExtractionResultAsync<JToken>(documentUri, cancellationToken)).ToString();`
`66`	`66`
`67`	`67`	`public async Task<TResult> GetExtractionResultAsync<TResult>(Uri documentUri, CancellationToken cancellationToken)`
`68`	`68`	`{`
Original file line number	Diff line number	Diff line change
`@@ -68,7 +68,8 @@ public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<Uri>`
`68`	`68`	`await _strawberryShakeClient.Submissions().CreateUri(workflowId, uris, cancellationToken);`
`69`	`69`
`70`	`70`	`public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<string> paths,`
`71`		`- CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null) {`
	`71`	`+ CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null)`
	`72`	`+ {`
`72`	`73`	`var filesToUpload = new List<(string Name, Stream content)>();`
`73`	`74`	`foreach (var path in paths)`
`74`	`75`	`{`
`@@ -83,12 +84,16 @@ public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<stri`
`83`	`84`	`}`
`84`	`85`
`85`	`86`	`}`
`86`		`- return await _strawberryShakeClient.Submissions().Create(workflowId, (IEnumerable<(string Name, string Meta)>)filesToUpload, cancellationToken, (SubmissionResultVersion?)resultsFileVersion);`
	`87`	`+ return await CreateAsync(workflowId, filesToUpload: filesToUpload, cancellationToken, resultsFileVersion);`
`87`	`88`	`}`
`88`	`89`
`89`	`90`	`[Obsolete("This is the Legacy version and will be deprecated. Please use ListAsync instead.")]`
`90`	`91`	`public async Task<IEnumerable<ISubmission>> ListAsync(IEnumerable<int> submissionIds, IEnumerable<int> workflowIds, IFilter filters, int limit = 1000,`
`91`		`- CancellationToken cancellationToken = default) => (IEnumerable<ISubmission>)await ListAsync(submissionIds, workflowIds, filters, null, limit, cancellationToken);`
	`92`	`+ CancellationToken cancellationToken = default)`
	`93`	`+ {`
	`94`	`+ var result = await ListAsync(submissionIds, workflowIds, filters, null, limit, cancellationToken);`
	`95`	`+ return result.Data;`
	`96`	`+ }`
`92`	`97`
`93`	`98`
`94`	`99`	`public async Task<IHasCursor<IEnumerable<ISubmission>>> ListAsync(IEnumerable<int> submissionIds, IEnumerable<int> workflowIds, IFilter filters, int? after, int limit = 1000, CancellationToken cancellationToken = default)`
`@@ -116,14 +121,14 @@ public async Task<IHasCursor<IEnumerable<ISubmission>>> ListAsync(IEnumerable<in`
`116`	`121`	`public async Task<ISubmission> GetAsync(int submissionId, CancellationToken cancellationToken = default)`
`117`	`122`	`{`
`118`	`123`	`var result = await _strawberryShakeClient.Submissions().Get(submissionId, cancellationToken);`
`119`		`- if (!Enum.TryParse(result.Status.ToString().ToUpper(),out Models.SubmissionStatus parsed))`
	`124`	`+ if (!Enum.IsDefined(typeof(StrawberryShake.SubmissionStatus), result.Status))`
`120`	`125`	`{`
`121`	`126`	`throw new NotSupportedException($"Cannot read submission status: {result.Status}");`
`122`	`127`	`}`
`123`	`128`	`return new Submission`
`124`	`129`	`{`
`125`	`130`	`Id = result.Id ?? 0,`
`126`		`- Status = parsed,`
	`131`	`+ Status = (Models.SubmissionStatus) result.Status,`
`127`	`132`	`DatasetId = result.DatasetId ?? 0,`
`128`	`133`	`WorkflowId = result.WorkflowId ?? 0,`
`129`	`134`	`InputFile = result.InputFile,`
`@@ -141,7 +146,7 @@ public async Task<string> GenerateSubmissionResultAsync(int submissionId, Cancel`
`141`	`146`	`public async Task<ISubmission> MarkSubmissionAsRetrieved(int submissionId, bool retrieved = true, CancellationToken cancellationToken = default)`
`142`	`147`	`{`
`143`	`148`	`var resultId = await _strawberryShakeClient.Submissions().MarkRetrieved(submissionId, retrieved, cancellationToken);`
`144`		`- var result = await _strawberryShakeClient.Submissions().List((IReadOnlyList<int?>)new List<int>(submissionId).AsReadOnly(), default, default, default, default, cancellationToken);`
	`149`	`+ var result = await _strawberryShakeClient.Submissions().List(new List<int?>(submissionId).AsReadOnly(), default, default, default, default, cancellationToken);`
`145`	`150`	`return new SubmissionSs(result?.Submissions?[0]);`
`146`	`151`	`}`
`147`	`152`