Merge pull request #496 from KxSystems/KXI-33288

[KXI-33288] Fix UX for huge data and other fixes

Author: Philip-Carneiro-KX

.vscode/launch.json

@@ -9,7 +9,10 @@
       "name": "Run Extension",
       "type": "extensionHost",
       "request": "launch",
-      "args": ["--extensionDevelopmentPath=${workspaceFolder}"],
+      "args": [
+        "--disable-extensions",
+        "--extensionDevelopmentPath=${workspaceFolder}"
+      ],
       "outFiles": ["${workspaceFolder}/out/extension.js*"],
       "preLaunchTask": "npm: watch"
     },

src/classes/localConnection.ts

@@ -195,15 +195,17 @@ export class LocalConnection {
         this.isError = true;
         result = handleQueryResults(err.toString(), QueryResultType.Error);
       }
-      if (res.errored) {
-        this.isError = true;
-        result = handleQueryResults(
-          res.error + (res.backtrace ? "\n" + res.backtrace : ""),
-          QueryResultType.Error,
-        );
-      } else {
-        result = res.result === null ? "" : res.result;
-        base64 = res.base64 || false;
+      if (res) {
+        if (res.errored) {
+          this.isError = true;
+          result = handleQueryResults(
+            res.error + (res.backtrace ? "\n" + res.backtrace : ""),
+            QueryResultType.Error,
+          );
+        } else {
+          result = res.result === null ? "" : res.result;
+          base64 = res.base64 || false;
+        }
       }
     });
 
@@ -215,6 +217,11 @@ export class LocalConnection {
 
     this.updateGlobal();
 
+    if (this.isError) {
+      this.isError = false;
+      return result;
+    }
+
     if (base64) {
       return { base64, result };
     }
@@ -224,10 +231,6 @@ export class LocalConnection {
     }
 
     if (ext.isResultsTabVisible && stringify) {
-      if (this.isError) {
-        this.isError = false;
-        return result;
-      }
       return convertStringToArray(result ? result : "");
     }
 

src/commands/serverCommand.ts

@@ -1226,13 +1226,21 @@ export async function writeQueryResultsToView(
     connVersion,
     isPython,
   );
+  let isSuccess = true;
+
   if (!checkIfIsDatasource(type)) {
+    if (typeof result === "string") {
+      const res = decodeQUTF(result);
+      if (res.startsWith(queryConstants.error)) {
+        isSuccess = false;
+      }
+    }
     addQueryHistory(
       query,
       executorName,
       connLabel,
       isInsights ? ServerType.INSIGHTS : ServerType.KDB,
-      true,
+      isSuccess,
       isPython,
       type === "WORKBOOK",
       undefined,

src/services/resultsPanelProvider.ts

@@ -51,8 +51,6 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
       guid: "text",
       byte: "number",
       short: "number",
-      int: "number",
-      long: "number",
       real: "number",
       float: "number",
       char: "text",
@@ -199,7 +197,9 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
       () => ({}),
     );
 
-    columns.forEach((column) => {
+    const columnsArray = Array.isArray(columns) ? columns : [columns];
+
+    columnsArray.forEach((column) => {
       const { name, values } = column;
       values.forEach((value, index) => {
         rowData[index][name] = decodeQUTF(value);
@@ -212,7 +212,9 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
   updatedExtractColumnDefs(results: StructuredTextResults) {
     const { columns } = results;
 
-    const columnDefs = columns.map((column) => {
+    const columnsArray = Array.isArray(columns) ? columns : [columns];
+
+    const columnDefs = columnsArray.map((column) => {
       const sanitizedKey = this.sanitizeString(column.name);
       const cellDataType = this.kdbToAgGridCellType(column.type);
       const headerName = column.type
@@ -303,24 +305,8 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
     }
 
     const gridOptions: GridOptions = {
-      defaultColDef: {
-        sortable: true,
-        resizable: true,
-        filter: true,
-        flex: 1,
-        minWidth: 100,
-      },
       rowData: rowData,
       columnDefs: columnDefs,
-      domLayout: "autoHeight",
-      pagination: true,
-      paginationPageSize: 100,
-      enableCellTextSelection: true,
-      ensureDomOrder: true,
-      suppressContextMenu: true,
-      suppressDragLeaveHidesColumns: true,
-      tooltipShowDelay: 200,
-      loading: true,
     };
 
     return gridOptions;
@@ -362,17 +348,16 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
     this._results = queryResult;
     let result = "";
     let gridOptions = undefined;
+
     if (!this._view) {
       kdbOutputLog("[Results Tab] No view to update", "ERROR");
       return;
     }
+
+    this._view.webview.postMessage({ command: "loading" });
+
     if (typeof queryResult === "string" || typeof queryResult === "number") {
-      result =
-        queryResult !== ""
-          ? `<p class="results-txt">${queryResult
-              .toString()
-              .replace(/\n/g, "<br/>")}</p>`
-          : "<p>No results to show</p>";
+      result = this.formatResult(queryResult);
     } else if (queryResult) {
       gridOptions = this.convertToGrid(
         queryResult,
@@ -381,16 +366,33 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
         this.isPython,
       );
     }
-    if (gridOptions) {
-      this._view.webview.postMessage({
-        command: "setGridOptions",
-        gridOptions: gridOptions,
-      });
-    } else {
-      this._view.webview.postMessage({
-        command: "setResultsContent",
-        results: result,
-      });
+
+    this.postMessageToWebview(gridOptions, result);
+  }
+
+  private formatResult(queryResult: string | number): string {
+    return queryResult !== ""
+      ? `<p class="results-txt">${queryResult.toString().replace(/\n/g, "<br/>")}</p>`
+      : "<p>No results to show</p>";
+  }
+
+  private postMessageToWebview(
+    gridOptions: GridOptions | undefined,
+    result: string,
+  ) {
+    if (this._view) {
+      if (gridOptions) {
+        this._view.webview.postMessage({
+          command: "setGridDatasource",
+          results: gridOptions.rowData,
+          columnDefs: gridOptions.columnDefs,
+        });
+      } else {
+        this._view.webview.postMessage({
+          command: "setResultsContent",
+          results: result,
+        });
+      }
     }
   }
 
@@ -420,16 +422,33 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
               "ag-grid-community.min.js",
             )}"></script>
           </head>
-          <body>     
+          <body>
             <div id="results" class="results-view-container">
               <div class="content-wrapper"></div>
-            </div>      
+            </div>
+            <div id="overlay" class="overlay">
+            <div class="loading-box">
+              <div class="spinner"></div>
+              <div class="loading-text">Loading data...</div>
+            </div>
+          </div>
             <script type="module" nonce="${nonce}" src="${webviewUri}"></script>
             <div id="grid" style="height: 100%;  width:100%;" class="${agGridTheme}"></div>
-            <script nonce="${nonce}" >          
+            <script nonce="${nonce}" >
               const vscode = acquireVsCodeApi();
+              const gridDiv = document.getElementById('grid');
+              const resultsDiv = document.querySelector('#results .content-wrapper');
+              const overlay = document.getElementById('overlay');
               let gridApi;
 
+              function showOverlay() {
+              overlay.style.display = 'flex';
+            }
+
+            function hideOverlay() {
+              overlay.style.display = 'none';
+            }
+
               function saveColumnWidths() {
                 if (!gridApi) {return null};
                 return gridApi.getColumnState();
@@ -442,30 +461,78 @@ export class KdbResultsViewProvider implements WebviewViewProvider {
 
               window.addEventListener('message', event => {
                 const message = event.data;
-                console.log(event)
-                if (message.command === 'setGridOptions') {
+                showOverlay();
+
+                const handleSetGridDatasource = () => {
                   const columnWidths = saveColumnWidths();
-                  const gridOptions = message.gridOptions;
-                  const gridDiv = document.getElementById('grid');
-                  const resultsDiv = document.querySelector('#results .content-wrapper');
-                  resultsDiv.innerHTML = ''; 
-                  gridDiv.innerHTML = ''; 
-                  const rowData = gridOptions.rowData;
-                  gridOptions.rowData = [];
+                  const gridOptions = {
+                    defaultColDef: {
+                      sortable: true,
+                      resizable: true,
+                      filter: true,
+                      flex: 1,
+                      minWidth: 100,
+                    },
+                    columnDefs: message.columnDefs,
+                    domLayout: "autoHeight",
+                    pagination: true,
+                    enableCellTextSelection: true,
+                    ensureDomOrder: true,
+                    suppressContextMenu: true,
+                    suppressDragLeaveHidesColumns: true,
+                    tooltipShowDelay: 200,
+                    rowBuffer: 0,
+                    rowModelType: "infinite",
+                    cacheBlockSize: 100,
+                    cacheOverflowSize: 2,
+                    maxConcurrentDatasourceRequests: 1,
+                    infiniteInitialRowCount: 10000,
+                    maxBlocksInCache: 10,
+                    datasource: {
+                      rowCount: undefined,
+                      getRows: function(params) {
+                        showOverlay();
+                        const results = message.results;
+                        setTimeout(() => {
+                          const lastRow = results.length;
+                          const rowsThisPage = results.slice(params.startRow, params.endRow);
+                          params.successCallback(rowsThisPage, lastRow);
+                          hideOverlay();
+                        }, 500);
+                      }
+                    }
+                  };
+                  resultsDiv.innerHTML = '';
+                  gridDiv.innerHTML = '';
                   gridApi = agGrid.createGrid(gridDiv, gridOptions);
                   restoreColumnWidths(columnWidths);
-                  setTimeout(() => {
-                    gridApi.setGridOption("rowData", rowData);
-                    gridApi.setGridOption("loading", false);
-                  }, 500);
                   document.getElementById("results").scrollIntoView();
-                } else if (message.command === 'setResultsContent') {
+                };
+
+                const handleSetResultsContent = () => {
                   const resultsContent = message.results;
-                  const resultsDiv = document.querySelector('#results .content-wrapper');
-                  const gridDiv = document.getElementById('grid');
-                  gridDiv.innerHTML = ''; 
-                  resultsDiv.innerHTML = ''; 
+                  gridDiv.innerHTML = '';
+                  resultsDiv.innerHTML = '';
                   resultsDiv.innerHTML = resultsContent;
+                  hideOverlay();
+                };
+
+                const handleLoading = () => {
+                  gridDiv.innerHTML = '';
+                  resultsDiv.innerHTML = '';
+                };
+
+                switch (message.command) {
+                  case 'setGridDatasource':
+                    handleSetGridDatasource();
+                    break;
+                  case 'setResultsContent':
+                    handleSetResultsContent();
+                    break;
+                  default:
+                  case 'loading':
+                    handleLoading();
+                    break;
                 }
               });
               document.addEventListener('contextmenu', (e) => {

src/webview/styles/resultsPanel.css

@@ -49,3 +49,49 @@ body {
   font-size: 12px;
   color: #666666;
 }
+
+.overlay {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background: rgba(0, 0, 0, 0.5);
+  justify-content: center;
+  align-items: center;
+  z-index: 1000;
+  display: none;
+}
+
+.loading-box {
+  background: var(--vscode-editor-background);
+  color: var(--vscode-editor-foreground);
+  padding: 20px;
+  border-radius: 8px;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.spinner {
+  border: 4px solid rgba(0, 0, 0, 0.1);
+  border-left-color: var(--vscode-progressBar-background);
+  border-radius: 50%;
+  width: 40px;
+  height: 40px;
+  animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+  0% {
+    transform: rotate(0deg);
+  }
+  100% {
+    transform: rotate(360deg);
+  }
+}
+
+.loading-text {
+  margin-top: 10px;
+  font-size: 14px;
+}