[ML] File upload: include existing doc count when checking for ingested doc count (elastic#234333)

When checking to see of all ingested docs are now searchable, it should
take into account existing docs as we could be uploading to an existing
index.

Author: jgowdyelastic

x-pack/platform/packages/shared/file-upload/file_upload_manager/doc_count_service.ts

@@ -15,6 +15,7 @@ export class DocCountService {
   private indexName: string | null = null;
   private indexSearchableSubscription: Subscription | null = null;
   private allDocsSearchableSubscription: Subscription | null = null;
+  private initialDocCount: number = 0;
 
   constructor(
     private fileUpload: FileUploadStartApi,
@@ -44,10 +45,11 @@ export class DocCountService {
   }
 
   public startAllDocsSearchableCheck(indexName: string, totalDocCount: number): void {
+    const expectedDocCount = totalDocCount + this.initialDocCount;
     this.indexName = indexName;
     this.allDocsSearchableSubscription = timer(0, POLL_INTERVAL * 1000)
       .pipe(
-        exhaustMap(() => this.isSearchable$(totalDocCount)),
+        exhaustMap(() => this.isSearchable$(expectedDocCount)),
         takeWhile((isSearchable) => !isSearchable, true) // takeUntil we get `true`, including the final one
       )
       .pipe(finalize(() => this.onAllDocsSearchable(indexName)))
@@ -65,4 +67,13 @@ export class DocCountService {
       catchError((err) => throwError(() => err))
     );
   }
+
+  public async loadInitialIndexCount(indexName: string) {
+    const { count } = await this.fileUpload.isIndexSearchable(indexName, 0);
+    this.initialDocCount = count;
+  }
+
+  public resetInitialDocCount() {
+    this.initialDocCount = 0;
+  }
 }

x-pack/platform/packages/shared/file-upload/file_upload_manager/file_manager.ts

@@ -483,13 +483,19 @@ export class FileUploadManager {
     let pipelinesCreated = false;
     let initializeImportResp: InitializeImportResponse | undefined;
 
+    this.docCountService.resetInitialDocCount();
+    const isExistingIndex = this.isExistingIndexUpload();
+    if (isExistingIndex) {
+      await this.docCountService.loadInitialIndexCount(indexName);
+    }
+
     try {
       initializeImportResp = await this.importer.initializeImport(
         indexName,
         this.getSettings().json,
         mappings,
         pipelines,
-        this.isExistingIndexUpload()
+        isExistingIndex
       );
 
       this.docCountService.startIndexSearchableCheck(indexName);

x-pack/platform/plugins/private/file_upload/public/api/index.ts

@@ -47,6 +47,7 @@ export interface GetTimeFieldRangeResponse {
 
 export interface IsIndexSearchableResponse {
   isSearchable: boolean;
+  count: number;
 }
 
 export const FileUploadComponent = GeoUploadWizardAsyncWrapper;

x-pack/platform/plugins/private/file_upload/server/routes.ts

@@ -592,7 +592,7 @@ export function fileUploadRoutes(coreSetup: CoreSetup<StartDeps, unknown>, logge
           const isSearchable = count >= expectedCount;
 
           return response.ok({
-            body: { isSearchable },
+            body: { isSearchable, count },
           });
         } catch (e) {
           return response.customError(wrapError(e));