Use Structured Text for datasources

eslint.config.mjs

@@ -9,7 +9,7 @@ const currentYear = new Date().getFullYear();
 
 export default [
   {
-    ignores: ["**/*.d.ts", "**/*.js", "**/*.mjs", "src/ipc/**"],
+    ignores: ["**/*.d.ts", "**/*.js", "**/*.mjs", "src/ipc/**", "test/fixtures/**"],
   },
   js.configs.recommended,
   ...tseslint.configs.recommended,

src/classes/insightsConnection.ts

@@ -16,7 +16,6 @@ import { jwtDecode } from "jwt-decode";
 import * as url from "url";
 
 import { ext } from "../extensionVariables";
-import { isCompressed, uncompress } from "../ipc/c";
 import {
   InsightsApiConfig,
   InsightsConfig,
@@ -49,7 +48,7 @@ import { retrieveUDAtoCreateReqBody } from "../utils/uda";
 const logger = "insightsConnection";
 
 const customHeadersOctet = {
-  Accept: "application/octet-stream",
+  Accept: "application/struct-text",
   "Content-Type": "application/json",
 };
 const customHeadersJson = {
@@ -483,7 +482,6 @@ export class InsightsConnection {
       if (!options) {
         return undefined;
       }
-      options.responseType = "arraybuffer";
 
       notify("REST", MessageKind.DEBUG, {
         logger,
@@ -497,14 +495,9 @@ export class InsightsConnection {
             MessageKind.DEBUG,
             { logger },
           );
-          if (isCompressed(response.data)) {
-            response.data = uncompress(response.data);
-          }
           return {
             error: "",
-            arrayBuffer: response.data.buffer
-              ? response.data.buffer
-              : response.data,
+            results: response.data.payload,
           };
         })
         .catch((error: any) => {
@@ -514,7 +507,7 @@ export class InsightsConnection {
             { logger, params: error },
           );
           return {
-            error: { buffer: error.response.data },
+            error: error.response.data.header.ai,
             arrayBuffer: undefined,
           };
         });

src/commands/dataSourceCommand.ts

@@ -43,11 +43,13 @@ import {
 import { MessageKind, notify } from "../utils/notifications";
 import {
   addQueryHistory,
+  convertRows,
   getQSQLWrapper,
   handleScratchpadTableRes,
   handleWSError,
   handleWSResults,
 } from "../utils/queryUtils";
+import { updatedExtractRowData } from "../utils/resultsRenderer";
 import { retrieveUDAtoCreateReqBody } from "../utils/uda";
 import { validateScratchpadOutputVariableName } from "../validators/interfaceValidator";
 
@@ -200,7 +202,12 @@ export async function runDataSource(
       }
       if (isNotebook || ext.isResultsTabVisible) {
         if (success) {
-          const resultCount = typeof res === "string" ? "0" : res.rows.length;
+          const resultCount =
+            typeof res === "string"
+              ? "0"
+              : res.rows
+                ? res.rows.length
+                : res.columns?.[0]?.values?.length || 0;
           notify(`Results: ${resultCount} rows`, MessageKind.DEBUG, {
             logger,
           });
@@ -231,8 +238,12 @@ export async function runDataSource(
           res = res.errorMsg ? res.errorMsg : res.error;
         }
 
+        const rowData = res.columns
+          ? convertRows(updatedExtractRowData(res))
+          : res;
+
         await writeQueryResultsToConsole(
-          res,
+          rowData,
           query,
           connLabel,
           executorName,
@@ -312,6 +323,8 @@ export async function runApiDataSource(
 
   if (apiCall?.error) {
     return parseError(apiCall.error);
+  } else if (apiCall?.results) {
+    return apiCall.results;
   } else if (apiCall?.arrayBuffer) {
     const results = handleWSResults(apiCall.arrayBuffer);
     return handleScratchpadTableRes(results);
@@ -423,6 +436,8 @@ export async function runQsqlDataSource(
 
   if (qsqlCall?.error) {
     return parseError(qsqlCall.error);
+  } else if (qsqlCall?.results) {
+    return qsqlCall.results;
   } else if (qsqlCall?.arrayBuffer) {
     const results = handleWSResults(qsqlCall.arrayBuffer, isTableView);
     return handleScratchpadTableRes(results);
@@ -446,6 +461,8 @@ export async function runSqlDataSource(
 
   if (sqlCall?.error) {
     return parseError(sqlCall.error);
+  } else if (sqlCall?.results) {
+    return sqlCall.results;
   } else if (sqlCall?.arrayBuffer) {
     const results = handleWSResults(sqlCall.arrayBuffer, isTableView);
     return handleScratchpadTableRes(results);
@@ -484,6 +501,8 @@ export async function executeUDARequest(
 
   if (udaCall?.error) {
     return parseError(udaCall.error);
+  } else if (udaCall?.results) {
+    return udaCall.results;
   } else if (udaCall?.arrayBuffer) {
     const results = handleWSResults(udaCall.arrayBuffer);
     return handleScratchpadTableRes(results);

src/models/data.ts

@@ -11,6 +11,8 @@
  * specific language governing permissions and limitations under the License.
  */
 
+import { StructuredTextResults } from "./queryResult";
+
 export type GetDataError = string | { buffer: ArrayBuffer };
 
 export type GetDataObjectPayload = {
@@ -22,6 +24,7 @@ export type GetDataObjectPayload = {
     columns: string[];
     rows: any;
   };
+  results?: StructuredTextResults;
   arrayBuffer?: ArrayBuffer;
 };
 

src/utils/resultsRenderer.ts

@@ -28,7 +28,10 @@ export function convertToGrid(
   if (
     (!isInsights && !isPython) ||
     /* TODO: Workaround for Python structuredText bug */
-    (!isPython && connVersion && isBaseVersionGreaterOrEqual(connVersion, 1.12))
+    (!isPython &&
+      connVersion &&
+      isBaseVersionGreaterOrEqual(connVersion, 1.12)) ||
+    results.columns
   ) {
     rowData = updatedExtractRowData(results);
     columnDefs = updatedExtractColumnDefs(results);

test/fixtures/api/getData.ts

@@ -0,0 +1,31 @@
+import { GetDataObjectPayload } from "../../../src/models/data";
+
+export const getDataResponse: GetDataObjectPayload = {
+  results: {
+    count: 2,
+    columns: [
+      {
+        name: "time",
+        type: "timestamps",
+        values: [
+          "2025.01.01D00:00:00.000000000",
+          "2025.01.01D00:00:00.000000000",
+        ],
+        order: [0, 1],
+      },
+      { name: "val", type: "ints", values: ["2", "3"], order: [0, 1] },
+    ],
+  },
+  error: "",
+};
+
+export const getDataIntResponse: GetDataObjectPayload = {
+  results: {
+    count: 2,
+    columns: [
+      { name: "a", type: "ints", values: ["1", "3", "5"], order: [0, 1, 2] },
+      { name: "b", type: "ints", values: ["2", "4", "6"], order: [0, 1, 2] },
+    ],
+  },
+  error: "",
+};

test/suite/services/services.utils.test.ts → test/fixtures/api/getMeta.ts

@@ -1,19 +1,6 @@
-/*
- * Copyright (c) 1998-2025 KX Systems Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
- * License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- */
-
 import { MetaObject } from "../../../src/models/meta";
 
-export const dummyMeta: MetaObject = {
+export const getMetaResponse: MetaObject = {
   header: {
     ac: "0",
     agg: ":127.0.0.1:5070",
@@ -69,3 +56,54 @@ export const dummyMeta: MetaObject = {
     schema: [],
   },
 };
+
+export const getMetaNoAssemblyResponse: MetaObject = {
+  header: {
+    ac: "0",
+    agg: ":127.0.0.1:5070",
+    ai: "",
+    api: ".kxi.getMeta",
+    client: ":127.0.0.1:5050",
+    corr: "CorrHash",
+    http: "json",
+    logCorr: "logCorrHash",
+    protocol: "gw",
+    rc: "0",
+    rcvTS: "2099-05-22T11:06:33.650000000",
+    retryCount: "0",
+    to: "2099-05-22T11:07:33.650000000",
+    userID: "dummyID",
+    userName: "testUser",
+  },
+  payload: {
+    rc: [
+      {
+        api: 3,
+        agg: 1,
+        assembly: 1,
+        schema: 1,
+        rc: "dummy-rc",
+        labels: [{ kxname: "dummy-assembly" }],
+        started: "2023-10-04T17:20:57.659088747",
+      },
+    ],
+    dap: [],
+    api: [],
+    agg: [
+      {
+        aggFn: ".sgagg.aggFnDflt",
+        custom: false,
+        full: true,
+        metadata: {
+          description: "dummy desc.",
+          params: [{ description: "dummy desc." }],
+          return: { description: "dummy desc." },
+          misc: {},
+        },
+        procs: [],
+      },
+    ],
+    assembly: [],
+    schema: [],
+  },
+};

test/fixtures/config/datasource.ts

@@ -0,0 +1,45 @@
+import {
+  DataSourceFiles,
+  DataSourceTypes,
+} from "../../../src/models/dataSource";
+
+type DataSource = DataSourceFiles["dataSource"];
+
+export function createMockDatasource(
+  dataSourceOverrides?: Partial<DataSource>,
+): DataSourceFiles {
+  return {
+    name: "mockDatasource",
+    insightsNode: "mockNode",
+    dataSource: {
+      selectedType: DataSourceTypes.API,
+      api: {
+        selectedApi: "getData",
+        table: "mock_table",
+        startTS: "2023-09-10T09:30",
+        endTS: "2023-09-19T12:30",
+        fill: "",
+        filter: [],
+        groupBy: [],
+        labels: [],
+        slice: [],
+        sortCols: [],
+        temporality: "",
+        agg: [],
+      },
+      qsql: {
+        selectedTarget: "mock_table rdb",
+        query: "mock QSQL query",
+      },
+      sql: {
+        query: "mock SQL query",
+      },
+      uda: {
+        name: "test query",
+        description: "test description",
+        params: [],
+      },
+      ...dataSourceOverrides,
+    },
+  };
+}