Merge pull request #722 from KxSystems/ee-fixes

Implement structured text for python for kdb and IE sp

Author: ecmel

CHANGELOG.md

@@ -6,30 +6,28 @@ All notable changes to the **kdb VS Code extension** are documented in this file
 
 ### Enhancements
 
-- Added quick connections
-- Run on REPL by default
-- SQL Workbook
-- Connect and auto execute query functionality
-- Use own grid instead of ag-grid-community for kdb Results View
-- Nested objects are better supported in kdb Results View
-- Add support for KDB-X modules in language server
-- Moved documentation to its own site
-- Ability to execute selection in quke files
+- Added quick connections for faster access to kdb environments
+- REPL is now the default execution target
+- Introduced SQL Workbook support
+- Added ability to connect and automatically execute queries
+- Replaced ag-grid-community with a custom grid for the kdb Results View
+- Improved support for nested objects in the kdb Results View
+- Moved documentation to a dedicated [documentation site](code.kx.com/vscode)
+- Added ability to execute selection in quke files
 
 ### Fixes
 
-- kdb output channel should support log levels
-- CSV export does not work properly for cells that contain lists or strings
-- Auto resizing of grid columns
-- Specific python query executions not working
+- kdb output channel now correctly supports log levels
+- Fixed CSV export issues for cells containing lists or strings
+- Fixed auto-resizing behavior for grid columns
+- Resolved issues with specific Python query executions
 
 ### Internal Improvements
 
-- Updated telemetry data sent
-- Use structured text for datasources
-- Integrate KDB-X KX_TTY feature in REPL
+- Updated telemetry data collection
+- Implemented the use of structured text for datasources
+- Added use of structured text for Python
 - Added q CI testing
-- Updated query wrappers
 
 # v1.16.1
 

src/classes/insightsConnection.ts

@@ -724,11 +724,7 @@ export class InsightsConnection {
       };
 
       if (this.insightsVersion) {
-        /* TODO: Workaround for Python structuredText bug */
-        if (
-          !isPython &&
-          isBaseVersionGreaterOrEqual(this.insightsVersion, 1.12)
-        ) {
+        if (isBaseVersionGreaterOrEqual(this.insightsVersion, 1.12)) {
           body.returnFormat = isTableView ? "structuredText" : "text";
         } else {
           body.isTableView = isTableView;
@@ -768,8 +764,6 @@ export class InsightsConnection {
           if (!response.data.error) {
             if (isTableView) {
               if (
-                /* TODO: Workaround for Python structuredText bug */
-                !isPython &&
                 this.insightsVersion &&
                 isBaseVersionGreaterOrEqual(this.insightsVersion, 1.12)
               ) {

src/classes/localConnection.ts

@@ -19,7 +19,7 @@ import { commands } from "vscode";
 import { ext } from "../extensionVariables";
 import { QueryResult, QueryResultType } from "../models/queryResult";
 import { ServerObject } from "../models/serverObject";
-import { convertStringToArray, handleQueryResults } from "../utils/execution";
+import { handleQueryResults } from "../utils/execution";
 import { MessageKind, notify } from "../utils/notifications";
 import { queryWrapper } from "../utils/queryUtils";
 
@@ -128,7 +128,12 @@ export class LocalConnection {
         const args: any[] = [];
         const wrapper = queryWrapper(!!isPython);
         if (isPython) {
-          args.push(stringify ? "text" : "serialized", command, "first", 10000);
+          args.push(
+            stringify ? "text" : "structuredText",
+            command,
+            "first",
+            10000,
+          );
         } else {
           args.push(
             context ?? ".",
@@ -148,12 +153,10 @@ export class LocalConnection {
             );
           } else {
             const result = res.result === null ? "" : res.result;
-            if (!stringify && !isPython) {
-              resolve(JSON.parse(result));
-            } else if (ext.isResultsTabVisible && stringify) {
-              resolve(convertStringToArray(result ? result : ""));
-            } else {
+            if (stringify) {
               resolve(result);
+            } else {
+              resolve(JSON.parse(result));
             }
           }
           this.updateGlobal();

src/commands/dataSourceCommand.ts

@@ -557,7 +557,10 @@ export function getPartialDatasourceFile(
     : <DataSourceFiles>{
         dataSource: {
           selectedType: "QSQL",
-          qsql: { query: getQSQLWrapper(query, isPython), selectedTarget },
+          qsql: {
+            query: getQSQLWrapper(query, "serialized", isPython),
+            selectedTarget,
+          },
           source: query,
         },
       };

src/commands/workspaceCommand.ts

@@ -574,7 +574,7 @@ export async function runOnRepl(editor: TextEditor, type?: ExecutionTypes) {
       repl.show();
       return repl.executeQuery(
         isPython(uri)
-          ? getPythonWrapper(text)
+          ? getPythonWrapper(text, "serialized")
           : isSql(uri)
             ? getSQLWrapper(text)
             : text,

src/services/notebookController.ts

@@ -87,7 +87,7 @@ export class KxNotebookController {
         const text = cell.document.getText();
         const result = await repl.executeQuery(
           kind === CellKind.PYTHON
-            ? getPythonWrapper(text)
+            ? getPythonWrapper(text, "serialized")
             : kind === CellKind.SQL
               ? getSQLWrapper(text)
               : text,

src/utils/queryUtils.ts

@@ -281,7 +281,7 @@ export function normalizePyQuery(query: string): string {
 
 export function getPythonWrapper(
   query: string,
-  returnFormat: "serialized" | "text" | "structuredText" = "serialized",
+  returnFormat: "serialized" | "text" | "structuredText",
 ): string {
   const wrapper = normalizeQSQLQuery(queryWrapper(true));
   const args = {
@@ -293,8 +293,14 @@ export function getPythonWrapper(
   return `{[returnFormat;code;sample_fn;sample_size] res:${wrapper}[returnFormat;code;sample_fn;sample_size];$[res\`errored;res\`error;res\`result]}["${args.returnFormat}";"${args.code}";"${args.sample_fn}";${args.sample_size}]`;
 }
 
-export function getQSQLWrapper(query: string, isPython?: boolean): string {
-  return isPython ? getPythonWrapper(query) : normalizeQSQLQuery(query);
+export function getQSQLWrapper(
+  query: string,
+  returnFormat: "serialized" | "text" | "structuredText",
+  isPython?: boolean,
+): string {
+  return isPython
+    ? getPythonWrapper(query, returnFormat)
+    : normalizeQSQLQuery(query);
 }
 
 export function getSQLWrapper(query: string): string {

test/suite/utils/queryUtils.test.ts

@@ -615,13 +615,13 @@ describe("queryUtils", () => {
     let queryWrappeStub: sinon.SinonStub;
 
     it("should not add extra semicolon", () => {
-      const res = queryUtils.getQSQLWrapper("a:1;\na");
+      const res = queryUtils.getQSQLWrapper("a:1;\na", "serialized");
       assert.strictEqual(res, "a:1;\na");
     });
 
     it("should normalize python code using wrapper", () => {
       assert.throws(() => {
-        queryUtils.getQSQLWrapper(``, true);
+        queryUtils.getQSQLWrapper(``, "serialized", true);
         sinon.assert.calledOnce(queryWrappeStub);
       });
     });