diff --git a/resources/evaluate.q b/resources/evaluate.q index 876a9d62..5ef7f579 100644 --- a/resources/evaluate.q +++ b/resources/evaluate.q @@ -1,4 +1,4 @@ -{[ctx; code; returnFormat] +{[ctx; code; sampleFn; returnFormat] if [`histogram in key `.qp; if [not `display2 in key `.qp; .qp.display2: (')[{x[`output][`bytes]}; .qp.display] @@ -108,22 +108,22 @@ prefix: ";[::;"; suffix: $[(not isLastLine) and not ";" ~ last expr; ";]"; "]"]; expr: prefix , expr , suffix; - result: .Q.trp[{[expr] `result`errored`error`backtrace!({$[x ~ (::); (::); x]} value expr; 0b; ""; ())}; + result: .Q.trp[{[expr] `data`error`errorMsg`stacktrace!({$[x ~ (::); (::); x]} value expr; 0b; ""; ())}; expr; - {[suffix; prefix; err; backtrace] + {[suffix; prefix; err; stacktrace] if [err ~ enlist " "; err: "syntax error"]; - userCode: (-1 + last where (.Q.trp ~ first first @) each backtrace) # backtrace; + userCode: (-1 + last where (.Q.trp ~ first first @) each stacktrace) # stacktrace; userCode[;3]: reverse 1 + til count userCode; userCode[-1 + count userCode; 1; 3]: (neg count suffix) _ (count prefix) _ userCode[-1 + count userCode; 1; 3]; userCode[-1 + count userCode; 2]-: count prefix; (!) . flip ( - (`result; ::); - (`errored; 1b); - (`error; err); - (`backtrace; .Q.sbt userCode)) + (`data; ::); + (`error; 1b); + (`errorMsg; err); + (`stacktrace; .Q.sbt userCode)) }[suffix; prefix]]; - if [isLastLine or result`errored; + if [isLastLine or result`error; system "d ", cachedCtx; : result]; index +: 1]; @@ -138,11 +138,27 @@ `compoundChar`compoundSymbol`compoundTimestamp`compoundMonth`compoundDate`compoundDatetime`compoundTimespan`compoundMinute`compoundSecond, `compoundTime`compoundEnum`table`dictionary`lambda`unary`binary`ternary`projection`composition, `$("f'";"f/";"f\\";"f':";"f/:";"f\\:";"dynamicload"); - removeTrailingNewline: {[text] + removeTrailingNewline: {[text] if ["\n" = last text; text: -1 _ text]; text }; + typeOf: {$[0>type x; .axq.i_PRIMCODE neg type x; .axq.i_NONPRIMCODE type x]}; + isAtom: {not type[x] within 0 99h}; + sample: {[sampleFn; sampleSize; data] + sampleSize: min (sampleSize; count data); + fn: $[ sampleFn ~ "random"; + {[sampleSize; data] + $[ type[data] ~ 99h; + [ ii: neg[sampleSize]?count data; + (key[data] ii)!value[data]ii]; + neg[sampleSize]?data] + }; + sampleFn ~ "first"; #; + sampleFn ~ "last"; {neg[x]#y}; + ' "Unrecognized sample function"]; + fn[sampleSize; data] + }; generateColumns:{[removeTrailingNewline; toString; originalType; isAtomic; isKey; data; name] attributes: attr data; types: $[ @@ -167,10 +183,25 @@ ' "This view is not supported for splayed tables"]; generateColumns[originalType; isAtom; isKey] ./: flip (value; key) @\: flip data }[generateColumns]; - toStructuredText:{[generateTableColumns; generateColumns; data; quantity; isAtom; originalType] - if[(type data) ~ 10h; data: enlist data]; + toStructuredText:{[generateTableColumns; generateColumns; sample; data; sampleFn] + DEFAULT_TABULAR_LIMIT: 600000; + TABULAR_LIMIT: DEFAULT_TABULAR_LIMIT^"J"$getenv `TABULAR_LIMIT; + isNumber: {abs[type[x]] within abs[5 9h]}; + itemLimit: TABULAR_LIMIT; + if[not isNumber itemLimit; itemLimit: DEFAULT_TABULAR_LIMIT]; + isEmpty: {0 ~ count x}; + warnings: (); isTable: .Q.qt data; isDict: 99h ~ type data; + truncateSize: $[isTable;ceiling itemLimit%count cols data;isDict;ceiling itemLimit%2;itemLimit]; + if[not isEmpty data;if[(sum count each data) > truncateSize;data: sample["first";truncateSize;data];warnings,: enlist "Results truncated to TABULAR_LIMIT. Console view is faster for large data."];]; + typeOf: {$[0>type x; .axq.i_PRIMCODE neg type x; .axq.i_NONPRIMCODE type x]}; + isAtom: {not type[x] within 0 99h}; + isAtom: isAtom data; + originalType: typeOf data; + quantity: count data; + data: sampleFn data; + if[(type data) ~ 10h; data: enlist data]; columns: $[ isTable and isDict; raze (generateTableColumns[::;0b;1b;key data]; generateTableColumns[::;0b;0b;value data]); @@ -180,29 +211,13 @@ generateTableColumns[originalType;isAtom;0b;data]; enlist generateColumns[originalType;isAtom;0b;data;"values"] ]; - : .j.j `count`columns!(quantity; columns) - }[generateTableColumns; generateColumns]; - typeOf: {$[0>type x; .axq.i_PRIMCODE neg type x; .axq.i_NONPRIMCODE type x]}; - isAtom: {not type[x] within 0 99h}; - sample: {[sampleFn; sampleSize; data] - sampleSize: min (sampleSize; count data); - fn: $[ sampleFn ~ "random"; - {[sampleSize; data] - $[ type[data] ~ 99h; - [ ii: neg[sampleSize]?count data; - (key[data] ii)!value[data]ii]; - neg[sampleSize]?data] - }; - sampleFn ~ "first"; #; - sampleFn ~ "last"; {neg[x]#y}; - ' "Unrecognized sample function"]; - fn[sampleSize; data] - } + : .j.j `count`columns`warnings!(quantity; columns;warnings) + }[generateTableColumns; generateColumns; sample]; result: evalInContext[ctx; splitExpression stripTrailingSemi wrapLines removeMultilineComments code]; - if [result `errored; :result]; + if [result`error; :result]; if [returnFormat ~ "text"; - result[`result]: toString result `result]; + result[`data]: toString result `data]; if [returnFormat ~ "structuredText"; - result[`result]: toStructuredText[result `result;count result`result; isAtom result`result; typeOf result`result]]; + result[`data]: toStructuredText[result`data; sampleFn]]; result - } + } \ No newline at end of file diff --git a/resources/evaluatePy.q b/resources/evaluatePy.q index d06d6d2b..3373c2a2 100644 --- a/resources/evaluatePy.q +++ b/resources/evaluatePy.q @@ -5,143 +5,212 @@ ]]; if[()~key`.pykx; :(!). flip( - (`result;::); - (`errored;1b); - (`error;".pykx is not defined: please load pykx"))]; + (`data;::); + (`error;1b); + (`errorMsg;".pykx is not defined: please load pykx"))]; .pykx.pyexec "def _kx_execution_context(): import traceback import sys - import ast as ast + import ast as pystruct_ast from io import BytesIO import tokenize - def is_expr(code): + def pystruct_is_expr(code): try: - ast.parse(code,mode='eval') + pystruct_ast.parse(code,mode='eval') return True except SyntaxError: return False - def run_line(code): - return eval(code)if is_expr(code)else exec(code,globals(),globals()) + def pystruct_run_line(code): + return eval(code) if pystruct_is_expr(code) else exec(code, globals(), globals()) - def range_to_text(lines,range): - code=lines[range[0][0]:range[1][0]+1] + def pystruct_range_to_text(lines, range): + code = lines[range[0][0]:range[1][0] + 1] + # The end must be removed first, as if the start is removed first, + # indices after it will have shifted code[-1]=code[-1][:range[1][1]] code[0]=code[0][range[0][1]:] return code - def run(code,as_string): - code=code.py().decode('utf-8') - code+='\\n\\n' - starts=[(x.lineno-1,x.col_offset)for x in ast.parse(code).body] - if len(starts)==0: - return str(None).encode('UTF-8') if as_string else None - - lines=str.splitlines(code) - starts.append((len(lines)-1,0)) - ranges=list(zip(starts,starts[1:])) - - [run_line('\\n'.join(range_to_text(lines,range))) for range in ranges[:-1]] - last_line='\\n'.join(range_to_text(lines,ranges[-1])) - result=run_line(last_line) - - return str(result).encode('UTF-8') if as_string else result + def pystruct_run(payload, as_string): + payload = payload.py().decode('utf-8') + payload += '\\n\\n' + # Get the start of each expression/statement + def get_start_pos(node): + if hasattr(node, 'decorator_list') and len(node.decorator_list): + return (min([x.lineno - 1 for x in node.decorator_list]), node.col_offset) + else: + return (node.lineno - 1, node.col_offset) + + # Get the start of each expression/statement + starts = [get_start_pos(x) for x in pystruct_ast.parse(payload).body] + + + if len(starts) == 0: + # encode('UTF-8') is to return a string, to avoid polluting the symbol table. + return str(None).encode('UTF-8') if as_string else None + + # Since ast.parse only returns the start of each expression, + # the ranges must be made by joining consecutive pairs. + # This means the end must be added manually + lines = str.splitlines(payload) + starts.append((len(lines) - 1, 0)) + + # Join consecutive pairs + ranges = list(zip(starts, starts[1:])) + + # Run all expressions, except the last one + [pystruct_run_line('\\n'.join(pystruct_range_to_text(lines, bounds))) for bounds in ranges[:-1]] + + # Run the last expression + last_line = '\\n'.join(pystruct_range_to_text(lines, ranges[-1])) + result = pystruct_run_line(last_line) + + # Preserve the result of python query execution for cases it doesn't need to run again. + global pystruct_last_query + pystruct_last_query = result + + # encode('UTF-8') is to return a string, to avoid polluting the symbol table. + return repr(result).encode('UTF-8') if as_string else result - def run_wrapped(code,returnFormat): + def pystruct_run_wrapped(code, as_string): try: return{ - 'result': run(code,returnFormat), - 'errored':False, - 'error':'' + 'error': False, + 'errorMsg': str(None).encode('UTF-8'), + 'data': pystruct_run(code, as_string) } except Exception as e: - type,error,tb=sys.exc_info() - stacktrace=traceback.extract_tb(tb) - offset=-1*(len(stacktrace)-3) - tb2=traceback.format_exception(type,error,tb,offset) - tb2=''.join(tb2) - tb2=tb2.rstrip() + type, error, tb = sys.exc_info() + stacktrace = traceback.extract_tb(tb) + offset = -1 * (len(stacktrace) - 3) + formatted_tb = traceback.format_exception(type, error, tb, offset) + formatted_tb = ''.join(formatted_tb) + formatted_tb = formatted_tb.rstrip() return{ - 'result':None, - 'errored':True, - 'error':str(e), - 'backtrace':tb2 + 'error': True, + 'errorMsg': str(e.args).encode('UTF-8'), + 'data': None, + 'stacktrace': formatted_tb } - def find_strings(code): - tokens=tokenize.tokenize(BytesIO(code.py()).readline) - strings=filter(lambda x:x.type==tokenize.STRING,tokens) - return[(token.start[0]-1,token.end[0]-1)for token in strings] + def pystruct_find_strings (code): + from io import BytesIO as pystruct_BytesIO + import tokenize as pystruct_tokenize + tokens = pystruct_tokenize.tokenize(pystruct_BytesIO(code.py()).readline) + strings = filter(lambda x: x.type == pystruct_tokenize.STRING, tokens) + return [(token.start[0] - 1, token.end[0] - 1) for token in strings] return{ - 'run_wrapped':run_wrapped, - 'run':run, - 'find_strings':find_strings + 'pystruct_run_wrapped':pystruct_run_wrapped, + 'pystruct_run':pystruct_run, + 'pystruct_find_strings':pystruct_find_strings }"; .pykx.pyexec"_kx_execution_context=_kx_execution_context()"; - .pykx.pyexec "def com_kx_edi_to_structured_text(data, length): + .pykx.pyexec "def pystruct_to_structured_text(data, length): # Importing libraries needed for code execution inside the function import json import numpy as np import pandas as pd + import os from collections.abc import Iterator import pykx as kx + import math + import sys + + # Read TABULAR_LIMIT, set to DEFAULT_TABULAR_LIMIT if no variable found + DEFAULT_TABULAR_LIMIT = 600000 + + try: + raw = kx.q('.struct.TABULAR_LIMIT') + except kx.QError: + # Use default + limit = DEFAULT_TABULAR_LIMIT + else: + # Convert to a Python int + try: + # If it's a PyKX object .py() gives the Python value; if it's already + # a plain Python value won't affect it + val = raw.py() if hasattr(raw, 'py') else raw + limit = int(val) + except (TypeError, ValueError, kx.QError): + limit = DEFAULT_TABULAR_LIMIT + + # create warnings list, append if any warnings + warnings = [] + + # Custom limits depending on data types + # for tables + if (isinstance(data, pd.DataFrame) and data.shape[1] > 0) or isinstance(data, pykx.Table): + truncateSize = math.ceil(limit / data.shape[1]) + # dictionaries have two 'columns', so halve the limit + elif isinstance(data, dict) or isinstance(data, pykx.Dictionary): + truncateSize = limit // 2 + elif isinstance(data, str): + truncateSize = sys.maxsize + else: + truncateSize = limit + + if(length > truncateSize): + data = pystruct_sample(data, 'first', truncateSize) + warnings.append(kx.CharVector('Results truncated to TABULAR_LIMIT. Console view is faster for large data.')) if isinstance(data, list): - columns = [com_kx_edi_generate_columns(False, data, 'values')] + columns = [pystruct_generate_columns(False, data, 'values')] # First we have to find out if the table is keyed or unkeyed. Since keyed tables also return true in dictionary case, we put this one here first elif isinstance(data, pykx.KeyedTable): columns = [] for element in kx.q.key(data).columns: - columns.append(com_kx_edi_generate_columns(True, data.get([element]).values(), str(element))) + columns.append(pystruct_generate_columns(True, data.get([element]).values()[0], str(element))) for element in data.values(): - columns.append(com_kx_edi_generate_columns(False, data.get([element]).values(), str(element))) + columns.append(pystruct_generate_columns(False, data.get([element]).values()[0], str(element))) elif isinstance(data, dict) or isinstance(data, pykx.Dictionary): - columns = [com_kx_edi_generate_columns(True,list(data.keys()), 'keys'),com_kx_edi_generate_columns(False, list(data.values()), 'values')] + columns = [pystruct_generate_columns(True,list(data.keys()), 'keys'),pystruct_generate_columns(False, list(data.values()), 'values')] elif isinstance(data, pd.DataFrame): columns = [] # Adding multi indexes; if more than one index use data.index.names, else use name. # This eliminates the issue of a single name being enlisted if not pd.Index(np.arange(0, len(data))).equals(data.index): if len(data.index.names) > 1: - columns.append(com_kx_edi_generate_columns(True, data.index.to_list(), str(data.index.names))) + columns.append(pystruct_generate_columns(True, data.index.to_list(), str(data.index.names))) else: - columns.append(com_kx_edi_generate_columns(True, data.index.to_list(), str(data.index.name))) + columns.append(pystruct_generate_columns(True, data.index.to_list(), str(data.index.name))) # Edge case to add a empty dataframe while still giving it a column entry if(data.empty): - columns.append(com_kx_edi_generate_columns(False, data, 'value')) + columns.append(pystruct_generate_columns(False, data, 'value')) # Getting the values of a data frame that are not index for element in data: - columns.append(com_kx_edi_generate_columns(False,data[element].to_list(),element)) + columns.append(pystruct_generate_columns(False,data[element].to_list(),element)) # This case is for unkeyed tables elif isinstance(data, pykx.Table): columns = [] for element in data: - columns.append(com_kx_edi_generate_columns(False, data.get([element]).values(), str(element))) + columns.append(pystruct_generate_columns(False, data.get([element]).values()[0], str(element))) else: - columns = [com_kx_edi_generate_columns(False, data, 'value' if length == 1 else 'values')] + columns = [pystruct_generate_columns(False, data, 'value' if length == 1 else 'values')] finalData = { 'count': length, - 'columns': columns + 'columns': columns, + 'warnings': warnings } - # Strings returned from Python should be explicitly cast to strings. - # Otherwise they will be added to the q symbol table for the life of the process - return kx.CharVector(json.dumps(finalData, ensure_ascii=False))"; - .pykx.pyexec "def com_kx_edi_generate_columns(isKey,data,name): - # Importing libraries needed for code execution inside the function + return finalData"; + + .pykx.pyexec "def pystruct_generate_columns(isKey,data,name): + # Importing libraries needed for code execution inside the function import numpy as np import pandas as pd from collections.abc import Iterator + import pykx as kx # Cache the original type of the data t = type(data) @@ -168,8 +237,11 @@ # Check explicitly for iterators, cannot iterate through iterators as it is destructive elif isinstance(data, Iterator): values = [str(data)] - # Check for strings and ranges, stringify whole value instead of iterating - elif t.__name__ == 'str' or t.__name__ == 'range': + # explicit check for strings to wrap return values in quotes + elif t.__name__ == 'str': + values = [repr(data)] + # Check for ranges and stringify whole value instead of iterating + elif t.__name__ == 'range' or t.__name__ == 'CharVector': values = [str(data)] else: try: @@ -177,12 +249,18 @@ except: values = [str(data)] - if len(values) == 1 or isinstance(values, str): + if len(values) == 1 or isinstance(values, str) or ((isinstance(data, pykx.List) and (pykx.q.count(values) == 0).py())): order = [0] else: try: - # Axis is along which way to sort, axis 0 means just cols, but -1(default) is rows and cols, None is a flattened array - order = np.argsort(data, axis=None,kind='stable').tolist() + if isinstance(data, pd.Series): + order = np.argsort(data.fillna(np.inf), kind='stable').tolist() + elif isinstance(data, pd.CategoricalIndex): + # Categorical indexes cannot have a axis when sorting + order = np.argsort(data, kind='stable').tolist() + else: + # Axis is along which way to sort, axis 0 means just cols, but -1(default) is rows and cols, None is a flattened array + order = np.argsort(data, axis=None,kind='stable').tolist() # For 2d lists, or lists of tuples, np.argsort flattens the array before sorting, # which isn't useful for our purposes. @@ -191,12 +269,12 @@ order = [i for (v, i) in sorted((v, i) for (i, v) in enumerate(data))] except Exception as e: - order = 'Sort unsupported: ' + str(e) + order = kx.CharVector('Sort unsupported: ' + str(e)) result = { - 'name': name, - 'type': t.__name__, - 'values': values, + 'name': kx.CharVector(name), + 'type': kx.CharVector(t.__name__), + 'values': [kx.CharVector(x) for x in values], 'order': order } @@ -205,7 +283,7 @@ return result"; - .pykx.pyexec "def com_kx_edi_sample(data, sample_fn, sample_size): + .pykx.pyexec "def pystruct_sample(data, sample_fn, sample_size): from random import sample import pandas as pd @@ -256,8 +334,23 @@ return data"; - .pykx.pyexec "def com_kx_edi_to_structured_text_wrapper(code, sample_fn, sample_size): - result = _kx_execution_context['run'](code, False) + .pykx.pyexec "def pystruct_to_structured_text_wrapper(code, sample_fn, sample_size): + try: + result = _kx_execution_context['pystruct_run'](code, False) + except Exception as e: + type, error, tb = sys.exc_info() + stacktrace = traceback.extract_tb(tb) + offset = -1 * (len(stacktrace) - 3) + formatted_tb = traceback.format_exception(type, error, tb, offset) + formatted_tb = ''.join(formatted_tb) + formatted_tb = formatted_tb.rstrip() + return { + 'error': True, + 'errorMsg': str(e.args).encode('UTF-8'), + 'data': None, + 'stacktrace': str(formatted_tb).encode('UTF-8') + } + # If length is not possible (e.g. functions) then assign length to 1 try: @@ -268,13 +361,13 @@ # Sampling can be a destructive operation, so only do it when necessary # e.g. When sampling a named tuple, the type of the result will be a regular tuple if sample_size < length or (sample_fn == 'random'): - result = com_kx_edi_sample(result, str(sample_fn), int(sample_size)) + result = pystruct_sample(result, str(sample_fn), int(sample_size)) try: return{ - 'result': com_kx_edi_to_structured_text(result, length), - 'errored': False, - 'error':'' + 'data': pystruct_to_structured_text(result, length), + 'error': False, + 'errorMsg':str(None).encode('UTF-8') } except Exception as e: type,error,tb=sys.exc_info() @@ -284,9 +377,9 @@ tb2=''.join(tb2) tb2=tb2.rstrip() return{ - 'result':None, - 'errored':True, - 'error':str(e), + 'data':None, + 'error':True, + 'errorMsg':str(e).encode('UTF-8'), 'backtrace':tb2 }"; @@ -294,7 +387,7 @@ removeExtraIndents:{[code] if[1 ~ count code; code: enlist code]; inStrings:$[(count ss[code;"'''"])or count ss[code;"\"\"\""]; - 1+raze{x+til y-x}./:.pykx.qeval["_kx_execution_context['find_strings']"]code; + 1+raze{x+til y-x}./:.pykx.qeval["_kx_execution_context['pystruct_find_strings']"]code; ()]; lines:"\n" vs code; skippedLines:all each lines in " \t"; @@ -307,30 +400,23 @@ code:removeExtraIndents code; defaultConv:.pykx.util.defaultConv; .pykx.util.defaultConv:"k"; - result: $[asString ~ "text"; .pykx.qeval["_kx_execution_context['run_wrapped']"][code;1b]; - asString ~ "serialized"; .pykx.qeval["_kx_execution_context['run_wrapped']"][code;0b]; - asString ~ "structuredText"; .pykx.qeval["com_kx_edi_to_structured_text_wrapper"][code;sample_fn;sample_size]; - // What should we return as the error case in which no return format is specified? I took this from db.q + result: $[asString ~ "text"; .pykx.qeval["_kx_execution_context['pystruct_run_wrapped']"][code;1b]; + asString ~ "serialized"; .pykx.qeval["_kx_execution_context['pystruct_run_wrapped']"][code;0b]; + asString ~ "structuredText"; .pykx.qeval["pystruct_to_structured_text_wrapper"][code;sample_fn;sample_size]; `error`errorMsg`data!(1b; "Invalid returnFormat specified"; ::) ]; .pykx.util.defaultConv:defaultConv; - result[`error]:string result`error; - if [`backtrace in key result; - result[`backtrace]:string result`backtrace]; - if[result `errored; :result]; - $[result`errored; + result[`data]: $[result[`error]; ::; - returnResult; - $[asString ~ "string"; - result[`result]:{[text] + $[asString ~ "text"; + result[`data]:{[text] maxSize:250000; $[count[text]>maxSize; sublist[maxSize;text],$["\n" in text;"\n..";".."]; text],"\n" - }(),result`result; - result`result]; - // not sure what this line does if anything. why is it in here - $[asString;"";::]]; + }(),result`data; + result`data]]; + if[asString ~ "structuredText"; result[`data] : .j.j result[`data]]; result }; run[1b;returnFormat;code;sample_fn;sample_size] diff --git a/test/q/main.q b/test/q/main.q index 5ec015b0..fcfaa607 100644 --- a/test/q/main.q +++ b/test/q/main.q @@ -1,2 +1,3 @@ @[system"l ",;"pykx.q";{::}] if[getenv[`CI]~"true";system "cd /app/project"]; +\c 10000 10000 \ No newline at end of file diff --git a/test/q/tests/evaluate.quke b/test/q/tests/evaluate.quke index 9ab2569b..ec2aeae4 100644 --- a/test/q/tests/evaluate.quke +++ b/test/q/tests/evaluate.quke @@ -1,8 +1,8 @@ feature EDI before .test.src: "\n" sv read0 `:resources/evaluate.q; - .test.evaluate: {[ctx; code; returnFormat] - 0i (.test.src; ctx; code; returnFormat) + .test.evaluate: {[ctx; code; sampleFn; returnFormat] + 0i (.test.src; ctx; code; sampleFn; returnFormat) }; .test.dims : system "c"; .test.precision: system "P"; @@ -20,122 +20,122 @@ feature EDI should ensure EDI performs display correctly for standard expressions // @TICKET KXI-12260 expect simple expression of 3+3 to return 6: - .qu.compare[.test.evaluate["."; "3+3"; "serialized"]`result; 6] + .qu.compare[.test.evaluate["."; "3+3"; ::; "serialized"]`data; 6] // This test is skipped because coverage is not handling generic null correctly in QBUILD 1.0.17 and greater // @TICKET KXI-12260 xexpect projection null to return as generic null - .qu.compare[.test.evaluate["."; "last value (;;)"; "serialized"]`result; (::)] + .qu.compare[.test.evaluate["."; "last value (;;)"; "serialized"]`data; (::)] expect function definition to work: - .qu.compare[.test.evaluate["."; ".test.fn333: {: 333}"; "serialized"]`result; {: 333}] + .qu.compare[.test.evaluate["."; ".test.fn333: {: 333}"; ::; "serialized"]`data; {: 333}] expect function call to work and return 333: - .qu.compare[.test.evaluate["."; ".test.fn333 []"; "serialized"]`result; 333] + .qu.compare[.test.evaluate["."; ".test.fn333 []"; ::; "serialized"]`data; 333] xexpect each right and each left of similar expressions to be equal: .com_kx_edi.dropIrrelevantRecords[.com_kx_ediNoCoverage.evaluate["1 2 3 +\\: 10"; "."]] ~ .com_kx_edi.dropIrrelevantRecords .com_kx_ediNoCoverage.evaluate["10 +/: 1 2 3"; "."] expect scan to work and return 11 13 16: - .qu.compare[.test.evaluate["."; "10 +\\ 1 2 3"; "serialized"]`result; 11 13 16] + .qu.compare[.test.evaluate["."; "10 +\\ 1 2 3"; ::; "serialized"]`data; 11 13 16] expect over to work and return 16: - .qu.compare[.test.evaluate["."; "10 +/ 1 2 3"; "serialized"]`result; 16] + .qu.compare[.test.evaluate["."; "10 +/ 1 2 3"; ::; "serialized"]`data; 16] expect a list created by EDI to be equal to the list: - .qu.compare[.test.evaluate["."; "1 2 3 4 5 6 7 8"; "serialized"]`result; 1 2 3 4 5 6 7 8] + .qu.compare[.test.evaluate["."; "1 2 3 4 5 6 7 8"; ::; "serialized"]`data; 1 2 3 4 5 6 7 8] expect an expression that creates a list to return the correct list: - .qu.compare[.test.evaluate["."; "`char$97+til 26"; "serialized"]`result; "abcdefghijklmnopqrstuvwxyz"] + .qu.compare[.test.evaluate["."; "`char$97+til 26"; ::; "serialized"]`data; "abcdefghijklmnopqrstuvwxyz"] expect the concat of two lists to be the same: - .qu.compare[.test.evaluate["."; "(til 100),(til 100)"; "serialized"]`result; (til[100], til 100)] + .qu.compare[.test.evaluate["."; "(til 100),(til 100)"; ::; "serialized"]`data; (til[100], til 100)] // @TICKET KXI-9607 should ensure EDI performs display correctly for k expressions expect simple k expressions to work, 3+3=6: - .qu.compare[.test.evaluate["."; "k) 3 + 3"; "serialized"]`result; 6] + .qu.compare[.test.evaluate["."; "k) 3 + 3"; ::; "serialized"]`data; 6] expect more complicated k expressions to work such as over: - .qu.compare[.test.evaluate["."; "k) +/1 2 3 4"; "serialized"]`result; 10] + .qu.compare[.test.evaluate["."; "k) +/1 2 3 4"; ::; "serialized"]`data; 10] expect more complicated k expressions to work such as dyadic function over: - .qu.compare[.test.evaluate["."; "k) {x+2*y}/1 2 3 4"; "serialized"]`result; 19] + .qu.compare[.test.evaluate["."; "k) {x+2*y}/1 2 3 4"; ::; "serialized"]`data; 19] expect more complicated k expressions to work such as monadic each both: - .qu.compare[.test.evaluate["."; "k) (-':)1 2 3 4"; "serialized"]`result; 1 1 1 1] + .qu.compare[.test.evaluate["."; "k) (-':)1 2 3 4"; ::; "serialized"]`data; 1 1 1 1] expect more complicated k expressions to work such as monadic each both: - .qu.compare[.test.evaluate["."; "k) +\\1 2 3 4"; "serialized"]`result; 1 3 6 10] + .qu.compare[.test.evaluate["."; "k) +\\1 2 3 4"; ::; "serialized"]`data; 1 3 6 10] expect a string to be a string in k: - .qu.compare[.test.evaluate["."; "k) \"hello\\\\there\""; "serialized"]`result; "hello\\there"] + .qu.compare[.test.evaluate["."; "k) \"hello\\\\there\""; ::; "serialized"]`data; "hello\\there"] // @TICKET KXI-9607 should ensure EDI performs display correctly for system expressions expect it to return timing information: // this fails intermittently on windows for some reason ... - 10 > .test.evaluate["."; "\\t count \"doug\""; "serialized"]`result + 10 > .test.evaluate["."; "\\t count \"doug\""; ::; "serialized"]`data expect expression in t to set zz correctly: - (.test.zz ~ 4) and (10 > .test.evaluate["."; "\\t .test.zz: count \"doug\""; "serialized"]`result) + (.test.zz ~ 4) and (10 > .test.evaluate["."; "\\t .test.zz: count \"doug\""; ::; "serialized"]`data) expect simple expression in t to work correctly: - (.test.zz ~ 4) and (10 > .test.evaluate["."; "\\t .test.zz: 1 + 1 + 1 + 1"; "serialized"]`result) + (.test.zz ~ 4) and (10 > .test.evaluate["."; "\\t .test.zz: 1 + 1 + 1 + 1"; ::; "serialized"]`data) expect ts to return time and space: - .qu.compare[count .test.evaluate["."; "\\ts til 10000"; "serialized"]`result; 2] + .qu.compare[count .test.evaluate["."; "\\ts til 10000"; ::; "serialized"]`data; 2] expect workspace information to return correctly (should usually work but may fail): - (last system "w") ~ last .test.evaluate["."; "\\w"; "serialized"]`result + (last system "w") ~ last .test.evaluate["."; "\\w"; ::; "serialized"]`data expect ls os command to return data back as a general list equivalent to system call: $["w" ~ first string .z.o; // The /b is needed to show the simple results, // as if just "dir" is used, the results can change between the two calls - (system "dir /b") ~ .test.evaluate["."; "\\dir /b"; "serialized"]`result; - (system "ls") ~ .test.evaluate["."; "\\ls"; "serialized"]`result]; + (system "dir /b") ~ .test.evaluate["."; "\\dir /b"; ::; "serialized"]`data; + (system "ls") ~ .test.evaluate["."; "\\ls"; ::; "serialized"]`data]; expect pwd os command to return the current directory equivalent to system call: - (system "cd") ~ .test.evaluate["."; "\\cd"; "serialized"]`result + (system "cd") ~ .test.evaluate["."; "\\cd"; ::; "serialized"]`data expect a to return a list of tables: - 11h ~ type .test.evaluate["."; "\\a"; "serialized"]`result + 11h ~ type .test.evaluate["."; "\\a"; ::; "serialized"]`data expect c to return the console size: - 6h ~ type .test.evaluate["."; "\\c"; "serialized"]`result + 6h ~ type .test.evaluate["."; "\\c"; ::; "serialized"]`data expect c 100 100 to set the console to 100 100: - .test.evaluate["."; "\\c 100 100"; "serialized"]`result; + .test.evaluate["."; "\\c 100 100"; ::; "serialized"]`data; 100 100i ~ system "c" expect p to provide the port: - system["p"] ~ .test.evaluate["."; "\\p"; "serialized"]`result + system["p"] ~ .test.evaluate["."; "\\p"; ::; "serialized"]`data expect P to provide the precision: - system["P"] ~ .test.evaluate["."; "\\P"; "serialized"]`result + system["P"] ~ .test.evaluate["."; "\\P"; ::; "serialized"]`data expect setting precision to 4 for it to be set to 4: - (system ["P"] ~ 4i) and (::) ~ .test.evaluate["."; "\\P 4"; "serialized"]`result + (system ["P"] ~ 4i) and (::) ~ .test.evaluate["."; "\\P 4"; ::; "serialized"]`data expect setting the precision to 0 sets it to max (same as 16): - (system ["P"] ~ 0i) and (::) ~ .test.evaluate["."; "\\P 0"; "serialized"]`result + (system ["P"] ~ 0i) and (::) ~ .test.evaluate["."; "\\P 0"; ::; "serialized"]`data // @TICKET KXI-9607 should work for non-default contexts expect correct expression result - 6 ~ .test.evaluate[".test"; "3+3"; "serialized"]`result + 6 ~ .test.evaluate[".test"; "3+3"; ::; "serialized"]`data expect function to be compiled in the correct context - `foo ~ first @[;3] value .test.evaluate[".foo"; "fn:{3+3}"; "serialized"]`result + `foo ~ first @[;3] value .test.evaluate[".foo"; "fn:{3+3}"; ::; "serialized"]`data // @TICKET KXI-9607 should accept alternate values to indicate "." expect correct expression result .test.ctx: system "d"; system "d .ignore"; - .test.evaluate["."; "myVal1: `first"; "serialized"]`result; - .test.evaluate["."; "myVal2: `second"; "serialized"]`result; + .test.evaluate["."; "myVal1: `first"; ::; "serialized"]`data; + .test.evaluate["."; "myVal2: `second"; ::; "serialized"]`data; system "d " , string .test.ctx; `first`second ~ get each `..myVal1`..myVal2; after @@ -147,21 +147,145 @@ feature EDI should not remove things that look like comments expect the /test to not be removed // This needs the trim, because it returns "/test " on windows - trim[.test.evaluate["."; "\\echo /test"; "serialized"]`result] ~ enlist "/test" + trim[.test.evaluate["."; "\\echo /test"; ::; "serialized"]`data] ~ enlist "/test" // @TICKET KXI-9607 // @TICKET KXI-16397 should handle windows line endings expect this to be treated as a single expression - .qu.compare[.test.evaluate["."; "(1\r\n\r\n 2 3)"; "serialized"]`result; 1 2 3] + .qu.compare[.test.evaluate["."; "(1\r\n\r\n 2 3)"; ::; "serialized"]`data; 1 2 3] // @TICKET KXI-9607 should maintain attributes regardless of semi-colon (KXAX-16342) expect an attribute with no semi colons - `s ~ .test.evaluate["."; "\n" sv .test.attrLines; "serialized"]`result + `s ~ .test.evaluate["."; "\n" sv .test.attrLines; ::; "serialized"]`data expect an attribute with semi colons - `s ~ .test.evaluate["."; "\n" sv .test.attrLines ,\: ";"; "serialized"]`result + `s ~ .test.evaluate["."; "\n" sv .test.attrLines ,\: ";"; ::; "serialized"]`data expect an attribute with no semi colons with two newlines - `s ~ .test.evaluate["."; "\n\n" sv .test.attrLines; "serialized"]`result + `s ~ .test.evaluate["."; "\n\n" sv .test.attrLines; ::; "serialized"]`data expect an attribute with semi colons with two newlines - `s ~ .test.evaluate["."; "\n\n" sv .test.attrLines ,\: ";"; "serialized"]`result + `s ~ .test.evaluate["."; "\n\n" sv .test.attrLines ,\: ";"; ::; "serialized"]`data + +feature toStructuredText + before + .test.src: "\n" sv read0 `:resources/evaluate.q; + .test.evaluate: {[ctx; code; sampleFn; returnFormat] + 0i (.test.src; ctx; code; sampleFn; returnFormat) + }; + after + setenv[`TABULAR_LIMIT;"600000"]; + should test the structured text functionality + expect the result for atoms to show the type of the underlying atom rather than the column + .qu.compare[.test.evaluate["."; "100f"; ::; "structuredText"]`data; "{\"count\":1,\"columns\":[{\"name\":\"values\",\"type\":\"float\",\"values\":[\"100f\"],\"order\":[0]}],\"warnings\":[]}"] + expect structured text to handle valid mixed lists // q bug causes the reverse (`orange;1) to throw a `type error + .qu.compare[.test.evaluate["."; "(1;`orange)"; ::; "structuredText"]`data; "{\"count\":2,\"columns\":[{\"name\":\"values\",\"type\":\"general\",\"values\":[\"1\",\"`orange\"],\"order\":[0,1]}],\"warnings\":[]}"] + expect structured text to preserve attributes + .qu.compare[.test.evaluate["."; "`s#10 + til 5"; ::; "structuredText"]`data; "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"longs\",\"values\":[\"10\",\"11\",\"12\",\"13\",\"14\"],\"order\":[0,1,2,3,4],\"attributes\":\"s\"}],\"warnings\":[]}"] + expect structured text to handle functions + .test.exampleFunction: {[x;y]x + y}; + .qu.compare[.test.evaluate["."; ".test.exampleFunction"; ::; "structuredText"]`data; "{\"count\":1,\"columns\":[{\"name\":\"values\",\"type\":\"lambda\",\"values\":[\"{[x;y]x + y}\"],\"order\":[0]}],\"warnings\":[]}"] + expect structured text to handle ? as sampling function + sampleFn: 2?; + .qu.compare[.test.evaluate["."; "0 0 0"; sampleFn; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"longs\",\"values\":[\"0\",\"0\"],\"order\":[0,1]}],\"warnings\":[]}"] + expect structured text to handle atoms + .test.exampleAtom: {[x;y]x + y}; + .qu.compare[.test.evaluate["."; ".test.exampleAtom"; ::; "structuredText"]`data; "{\"count\":1,\"columns\":[{\"name\":\"values\",\"type\":\"lambda\",\"values\":[\"{[x;y]x + y}\"],\"order\":[0]}],\"warnings\":[]}"] + expect structured text to handle lists + .test.exampleList: 1.1 452 -32.9; + .qu.compare[.test.evaluate["."; ".test.exampleList"; ::; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"floats\",\"values\":[\"1.1\",\"452f\",\"-32.9\"],\"order\":[2,0,1]}],\"warnings\":[]}"] + expect structured text to handle 2D lists + .test.example2dList: (1.1 1.0 2.0; 452 112 561); + .qu.compare[.test.evaluate["."; ".test.example2dList"; ::; "structuredText"]`data; "{\"count\":2,\"columns\":[{\"name\":\"values\",\"type\":\"general\",\"values\":[\"1.1 1 2\",\"452 112 561\"],\"order\":[1,0]}],\"warnings\":[]}"] + expect structured text to handle dictionaries + .test.exampleDict: `tom`dick`harry!1040 59 27; + .qu.compare[.test.evaluate["."; ".test.exampleDict"; ::; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"key\",\"type\":\"symbols\",\"values\":[\"`tom\",\"`dick\",\"`harry\"],\"order\":[1,2,0],\"isKey\":true},{\"name\":\"values\",\"type\":\"longs\",\"values\":[\"1040\",\"59\",\"27\"],\"order\":[2,1,0]}],\"warnings\":[]}"] + expect structured text to handle unkeyed tables + .test.exampleTable: flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126); + .qu.compare[.test.evaluate["."; ".test.exampleTable"; ::; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"name\",\"type\":\"symbols\",\"values\":[\"`Dent\",\"`Beeblebrox\",\"`Prefect\"],\"order\":[1,0,2]},{\"name\":\"iq\",\"type\":\"longs\",\"values\":[\"98\",\"42\",\"126\"],\"order\":[1,0,2]}],\"warnings\":[]}"] + expect structured text to handle keyed tables + .test.exampleKeyedTable: ([sym: `AAPL`TSLA`GOOGL] price: 100 200 300); + .qu.compare[.test.evaluate["."; ".test.exampleKeyedTable"; ::; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"sym\",\"type\":\"symbols\",\"values\":[\"`AAPL\",\"`TSLA\",\"`GOOGL\"],\"order\":[0,2,1],\"isKey\":true},{\"name\":\"price\",\"type\":\"longs\",\"values\":[\"100\",\"200\",\"300\"],\"order\":[0,1,2]}],\"warnings\":[]}"] + expect structured text to handle strings + .test.exampleString: "ThisIsAExampleString"; + .qu.compare[.test.evaluate["."; ".test.exampleString"; ::; "structuredText"]`data; "{\"count\":20,\"columns\":[{\"name\":\"values\",\"type\":\"chars\",\"values\":[\"\\\"ThisIsAExampleString\\\"\"],\"order\":[0]}],\"warnings\":[]}"] + expect structured text to handle empty lists + .test.exampleEmptyList: (); + .qu.compare[.test.evaluate["."; ".test.exampleEmptyList"; ::; "structuredText"]`data; "{\"count\":0,\"columns\":[{\"name\":\"values\",\"type\":\"general\",\"values\":[],\"order\":[]}],\"warnings\":[]}"] + expect structured text to handle empty tables + .test.exampleEmptyTable: ([] name:(); iq:()); + .qu.compare[.test.evaluate["."; ".test.exampleEmptyTable"; ::; "structuredText"]`data; "{\"count\":0,\"columns\":[{\"name\":\"name\",\"type\":\"general\",\"values\":[],\"order\":[]},{\"name\":\"iq\",\"type\":\"general\",\"values\":[],\"order\":[]}],\"warnings\":[]}"] + expect structured text to handle empty dictionaries + .test.exampleEmptyDict:(`$())!`float$(); + .qu.compare[.test.evaluate["."; ".test.exampleEmptyDict"; ::; "structuredText"]`data; "{\"count\":0,\"columns\":[{\"name\":\"key\",\"type\":\"symbols\",\"values\":[],\"order\":[],\"isKey\":true},{\"name\":\"values\",\"type\":\"floats\",\"values\":[],\"order\":[]}],\"warnings\":[]}"] + expect structured text to handle enums + .test.d:`a`b`c; + .test.y:`a`b`c`b`a`b`c`c`c`c`c`c`c; + .test.e: `.test.d$.test.y; + .qu.compare[.test.evaluate["."; ".test.e"; ::; "structuredText"]`data; "{\"count\":13,\"columns\":[{\"name\":\"values\",\"type\":\"enum\",\"values\":[\"`.test.d$`a\",\"`.test.d$`b\",\"`.test.d$`c\",\"`.test.d$`b\",\"`.test.d$`a\",\"`.test.d$`b\",\"`.test.d$`c\",\"`.test.d$`c\",\"`.test.d$`c\",\"`.test.d$`c\",\"`.test.d$`c\",\"`.test.d$`c\",\"`.test.d$`c\"],\"order\":[0,4,1,3,5,2,6,7,8,9,10,11,12]}],\"warnings\":[]}"] + xexpect structured text to handle foreign key columns + // stubbing out test due to automatically adding attribute sorted s into data. Filed in https://kxl.atlassian.net/browse/KXI-59085 + .test.name:`$("Stock Exchange";"Boersen";"NYSE"); + .test.country:`$("United Kingdom";"Denmark";"United States"); + .test.city:`$("London";"Copenhagen";"New York"); + .test.id:1001 1002 1003; + .test.market:([.test.id].test.name;.test.country;.test.city); + .test.trade:([] + stock: `ibm`APPL; + market: `.test.market$1001 1001; + price: 122.5 320.9; + amount: 500 40i; + time: 09:04:59:000 08:03:58:000); + .qu.compare[.test.evaluate["."; ".test.trade"; ::; "structuredText"]`data; "{\"count\":2,\"columns\":[{\"name\":\"stock\",\"type\":\"symbols\",\"values\":[\"`ibm\",\"`APPL\"],\"order\":[1,0]},{\"name\":\"market\",\"type\":\"enum\",\"values\":[\"`.test.market$1001\",\"`.test.market$1001\"],\"order\":[0,1]},{\"name\":\"price\",\"type\":\"floats\",\"values\":[\"122.5\",\"320.9\"],\"order\":[0,1],\"attributes\":\"s\"},{\"name\":\"amount\",\"type\":\"ints\",\"values\":[\"500i\",\"40i\"],\"order\":[1,0]},{\"name\":\"time\",\"type\":\"times\",\"values\":[\"09:04:59.000\",\"08:03:58.000\"],\"order\":[1,0]}],\"warnings\":[]}"] + expect structured text to handle lists of dictionaries + .test.exampleList:((`tom`dick`harry!1040 59 27);(`Adrian`Ben!10 20);(`Nathaniel`Igor!("a";"string"))); + .qu.compare[.test.evaluate["."; ".test.exampleList"; ::; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"general\",\"values\":[\"tom | 1040\\ndick | 59\\nharry| 27\",\"Adrian| 10\\nBen | 20\",\"Nathaniel| \\\"a\\\"\\nIgor | \\\"string\\\"\"],\"order\":[2,1,0]}],\"warnings\":[]}"] + expect structured text to handle anymap + originalDir: first system "pwd"; + .test.anymap: get`:/tmp/anymap set ((1 2;3 4);`time`price`vol!(09:30:00;1.;100i);([]a:1 2;b:("ab";"cd"))); + system "cd " , originalDir; + .qu.compare[.test.evaluate["."; ".test.anymap"; ::; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"compoundGeneral\",\"values\":[\"1 2\\n3 4\",\"time | 09:30:00\\nprice| 1f\\nvol | 100i\",\"a b \\n------\\n1 \\\"ab\\\"\\n2 \\\"cd\\\"\"],\"order\":[0,2,1]}],\"warnings\":[]}"] + expect structured text to handle truncated results for all other cases + setenv[`TABULAR_LIMIT;"10"]; + .test.exampleDict: `tavon`kira`zane`milo`dax`nilo`sera`jax`luma`reon`vela`tari`senn`korin`bryn`aiko`dran`lira`zev`orin!(1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16;17;18;19;20); + .qu.compare[.test.evaluate["."; ".test.exampleDict"; ::; "structuredText"]`data; "{\"count\":5,\"columns\":[{\"name\":\"key\",\"type\":\"symbols\",\"values\":[\"`tavon\",\"`kira\",\"`zane\",\"`milo\",\"`dax\"],\"order\":[4,1,3,0,2],\"isKey\":true},{\"name\":\"values\",\"type\":\"longs\",\"values\":[\"1\",\"2\",\"3\",\"4\",\"5\"],\"order\":[0,1,2,3,4]}],\"warnings\":[\"Results truncated to TABULAR_LIMIT. Console view is faster for large data.\"]}"] + expect structured text to handle truncated results for tables + setenv[`TABULAR_LIMIT;"10"]; + .test.exampleTable: flip `name`id!(`Dent`Beeblebrox`Prefect`Zan`Bob`Kox`Lind`Ozy`Drax`Rox`Mop`Lomp;98 42 126 11 21 634 72 212 76 12 156 217); + .qu.compare[.test.evaluate["."; ".test.exampleTable"; ::; "structuredText"]`data; "{\"count\":5,\"columns\":[{\"name\":\"name\",\"type\":\"symbols\",\"values\":[\"`Dent\",\"`Beeblebrox\",\"`Prefect\",\"`Zan\",\"`Bob\"],\"order\":[1,4,0,2,3]},{\"name\":\"id\",\"type\":\"longs\",\"values\":[\"98\",\"42\",\"126\",\"11\",\"21\"],\"order\":[3,4,1,0,2]}],\"warnings\":[\"Results truncated to TABULAR_LIMIT. Console view is faster for large data.\"]}"] + expect structured text to handle truncated results for dictionaries + setenv[`TABULAR_LIMIT;"10"]; + .test.exampleDict: `tavon`kira`zane`milo`dax`nilo`sera`jax`luma`reon`vela`tari`senn`korin`bryn`aiko`dran`lira`zev`orin!(1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16;17;18;19;20); + .qu.compare[.test.evaluate["."; ".test.exampleDict"; ::; "structuredText"]`data; "{\"count\":5,\"columns\":[{\"name\":\"key\",\"type\":\"symbols\",\"values\":[\"`tavon\",\"`kira\",\"`zane\",\"`milo\",\"`dax\"],\"order\":[4,1,3,0,2],\"isKey\":true},{\"name\":\"values\",\"type\":\"longs\",\"values\":[\"1\",\"2\",\"3\",\"4\",\"5\"],\"order\":[0,1,2,3,4]}],\"warnings\":[\"Results truncated to TABULAR_LIMIT. Console view is faster for large data.\"]}"] + expect structured text to handle invalid TABULAR_LIMIT + setenv[`TABULAR_LIMIT;"ADRIAN"]; + .test.exampleList: 1.1 452 -32.9; + .qu.compare[.test.evaluate["."; ".test.exampleList"; ::; "structuredText"]`data; "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"floats\",\"values\":[\"1.1\",\"452f\",\"-32.9\"],\"order\":[2,0,1]}],\"warnings\":[]}"] + should return a dictionary in structured text + expect return dictionary to contain proper keys in the form of `data`error`errorMsg`stacktrace + .test.exampleList: 1.1 452 -32.9; + a: .test.evaluate["."; ".test.exampleList"; ::; "structuredText"]; + .qu.compare[key a; `data`error`errorMsg`stacktrace] + expect return dictionary to contain 0b for error + .test.exampleList: 1.1 452 -32.9; + .qu.compare[.test.evaluate["."; ".test.exampleList"; ::; "structuredText"]`error; 0b] + expect return dictionary to contain proper errorMsg + .test.exampleList: 1.1 452 -32.9; + .qu.compare[.test.evaluate["."; ".test.exampleList"; ::; "structuredText"]`errorMsg; ""] + expect return dictionary to contain a empty stacktrace + .test.exampleList: 1.1 452 -32.9; + .qu.compare[.test.evaluate["."; ".test.exampleList"; ::; "structuredText"]`stacktrace; ()] + should return errors in structured text + expect error dictionary to contain keys in the form of `data`error`errorMsg`stacktrace + a: .test.evaluate["."; "something"; ::; "text"]; + .qu.compare[key a; `data`error`errorMsg`stacktrace] + expect error dictionary to contain null for data + .qu.compare[.test.evaluate["."; "something"; ::; "text"]`data; ::] + expect error dictionary to contain a 1b for error + .qu.compare[.test.evaluate["."; "something"; ::; "text"]`error; 1b] + expect error dictionary to contain proper errorMsg + .qu.compare[.test.evaluate["."; "something"; ::; "text"]`errorMsg; "something"] + expect error dictionary to contain stacktrace + a: .j.j .test.evaluate["."; ".test.adr"; ::; "text"]; + b: "{\"data\":null,\"error\":true,\"errorMsg\":\".test.adr\",\"stacktrace\":\" [1] .test.adr\\n ^\\n\"}"; + .qu.compare[a;b] + \ No newline at end of file diff --git a/test/q/tests/evaluatePy.quke b/test/q/tests/evaluatePy.quke index d6cb55a5..110740d2 100644 --- a/test/q/tests/evaluatePy.quke +++ b/test/q/tests/evaluatePy.quke @@ -4,18 +4,17 @@ feature removeExtraIndents .test.evaluatePy: {[isString; code] 0i (.test.src; isString; code; "first"; 10000) }; - should handle unmatched dedents expect both expressions to be moved to column 0 x: " a = 1 + 2\n", "a + 3"; - .qu.compare[.test.evaluatePy["serialized"; x]`result; 6] + .qu.compare[.test.evaluatePy["serialized"; x]`data; 6] should handle blank lines expect the expression to be moved to column 0 x: "\n", " 1 + 2"; - .qu.compare[.test.evaluatePy["serialized"; x]`result; 3] + .qu.compare[.test.evaluatePy["serialized"; x]`data; 3] should handle tabs expect the expression and statement to be moved to column 0 @@ -23,7 +22,7 @@ feature removeExtraIndents "\t\tdef foo(x):\n", "\t\t\treturn x + 1\n", "\tfoo(a)\n"; - .qu.compare[.test.evaluatePy["serialized"; x]`result; 2] + .qu.compare[.test.evaluatePy["serialized"; x]`data; 2] xshould handle multiline strings expect all statements to be moved to column 0 @@ -33,7 +32,7 @@ feature removeExtraIndents " b = \"one\"\n", " c = \"two\"\n", " (a,b,c)"; - .qu.compare[.test.evaluatePy["serialized"; x]`result; (21; `one; `two)]; + .qu.compare[.test.evaluatePy["serialized"; x]`data; (21; `one; `two)]; should handle functions at decreasing indents expect all statements to start in col 0 @@ -48,7 +47,7 @@ feature removeExtraIndents "\n", "foo(bar(quux(2)))"; - .qu.compare[.test.evaluatePy["serialized"; x]`result; 256]; + .qu.compare[.test.evaluatePy["serialized"; x]`data; 256]; should handle functions at decreasing indents expect relative indenting within the function to be preserved @@ -58,12 +57,12 @@ feature removeExtraIndents " \n", " def indented3 (x):\n", " return x * x"; - .test.evaluatePy["serialized"; x]`result; + .test.evaluatePy["serialized"; x]`data; all ( - .test.evaluatePy["serialized"; "indented1(1)"][`errored] ~ 0b; - .test.evaluatePy["serialized"; "indented2(1)"][`errored] ~ 1b; - .test.evaluatePy["serialized"; "indented3(1)"][`errored] ~ 1b + .test.evaluatePy["serialized"; "indented1(1)"][`error] ~ 0b; + .test.evaluatePy["serialized"; "indented2(1)"][`error] ~ 1b; + .test.evaluatePy["serialized"; "indented3(1)"][`error] ~ 1b ) should leave legitimately incorrect indentation @@ -73,8 +72,8 @@ feature removeExtraIndents " else:\n", " print(2)\n"; - .qu.compare[.test.evaluatePy["serialized"; x]`error; - "unindent does not match any outer indentation level (, line 3)"] + .qu.compare[.test.evaluatePy["serialized"; x]`errorMsg; + "('unindent does not match any outer indentation level', ('', 3, 7, ' else:\\n', 3, -1))"] feature evaluatePy 1 @@ -84,13 +83,13 @@ feature evaluatePy 1 .test.evaluatePy: {[isString; code] 0i (.test.src; isString; code; "first"; 10000) }; - .test.wrap: {`result`errored`error!(x; 0b; "")} + .test.wrap: {`error`errorMsg`data!(0b; "None"; x)} should run python code expect an empty input to return "None" .qu.compare[ .test.evaluatePy["text"; ""]; - .test.wrap "None"] + .test.wrap "None\n"] expect an empty input to return :: .qu.compare[ .test.evaluatePy["serialized"; " \n \n "]; @@ -103,19 +102,19 @@ feature evaluatePy 1 expect expressions to return a value .qu.compare[ .test.evaluatePy["text"; "1+2"]; - .test.wrap "3"] + .test.wrap "3\n"] expect statements to return an empty string .qu.compare[ .test.evaluatePy["text"; "def axedi_test_fn(x):\n return 1 + x"]; - .test.wrap "None"] + .test.wrap "None\n"] expect statements to have been executed .qu.compare[ .test.evaluatePy["text"; "axedi_test_fn(10)"]; - .test.wrap "11"] + .test.wrap "11\n"] expect utf-8 to be handled .qu.compare[ .test.evaluatePy["text"; "'你好'"]; - .test.wrap "你好"] + .test.wrap "'你好'\n"] xexpect return values to be limited to 250,000 characters .qu.compare[ .test.evaluatePy["text"; "'", (251000#.Q.a) , "'"]; @@ -130,19 +129,22 @@ feature evaluatePy 1 .test.evaluatePy["text"; "Long_Text()"]; .test.wrap (raze 25000#enlist "\n123456789") , "\n.."] expect invalid statements to error + result: .test.evaluatePy["text"; "1+++"]; + stacktrace: string result[`stacktrace]; + result[`stacktrace]: stacktrace; .qu.compare[ - .test.evaluatePy["text"; "1+++"]; + result; (!) . flip ( - (`result; ::); - (`errored; 1b); - (`error; "invalid syntax (, line 1)"); - (`backtrace; " File \"\", line 1\n 1+++\n ^\nSyntaxError: invalid syntax"))] + (`error; 1b); + (`errorMsg; "('invalid syntax', ('', 1, 5, '1+++\\n', 1, 5))"); + (`data; ::); + (`stacktrace; " File \"\", line 1\n 1+++\n ^\nSyntaxError: invalid syntax"))] should run multiline python code expect code ending in comments to evaluate correctly .qu.compare[ .test.evaluatePy["text";"a=(1+ # An inline comment\n 2 + 3)\na\n# This is a comment"]; - .test.wrap "6"] + .test.wrap "6\n"] xexpect code multiline strings to return the value of the last expression .qu.compare[ .test.evaluatePy["text";"'''This is\na multiline\nstring'''"]; @@ -162,7 +164,7 @@ feature evaluatePy 1 expect an assignment to return None .qu.compare[ .test.evaluatePy["text";"a=1"]; - .test.wrap "None"] + .test.wrap "None\n"] expect a backslash to work for writing multiline expressions .qu.compare[.test.evaluatePy["text";"\n" sv ( "a = 1 \\"; @@ -171,7 +173,7 @@ feature evaluatePy 1 " - 0"; "a * \\"; " 3")]; - .test.wrap "18"] + .test.wrap "18\n"] should display the stack trace for Python errors expect the stack trace to not show the functions before usercode @@ -182,52 +184,52 @@ feature evaluatePy 1 "def bar(x):"; " return x + 'hello'"); - .qu.compare[.test.evaluatePy["text"; "size = len(foo(5))"]; + result: .test.evaluatePy["text"; "size = len(foo(5))"]; + stacktrace: string result[`stacktrace]; + result[`stacktrace]: stacktrace; + .qu.compare[ + result; (!) . flip ( - (`result; ::); - (`errored; 1b); - (`error; "unsupported operand type(s) for +: 'int' and 'str'"); - (`backtrace; "\n" sv ( - "Traceback (most recent call last):"; - " File \"\", line 1, in "; - " File \"\", line 2, in foo"; - " File \"\", line 5, in bar"; - "TypeError: unsupported operand type(s) for +: 'int' and 'str'")))] + (`error; 1b); + (`errorMsg; "(\"unsupported operand type(s) for +: 'int' and 'str'\",)"); + (`data; ::); + (`stacktrace; "Traceback (most recent call last):\n File \"\", line 1, in \n File \"\", line 2, in foo\n File \"\", line 5, in bar\nTypeError: unsupported operand type(s) for +: 'int' and 'str'"))] expect the stack trace to not show the functions before user code - .qu.compare[.test.evaluatePy["serialized"; "1 2 + 3 4"]; + result: .test.evaluatePy["serialized"; "1 2 + 3 4"]; + stacktrace: string result[`stacktrace]; + result[`stacktrace]: stacktrace; + .qu.compare[ + result; (!) . flip ( - (`result; ::); - (`errored; 1b); - (`error; "invalid syntax (, line 1)"); - (`backtrace; "\n" sv ( - " File \"\", line 1"; - " 1 2 + 3 4"; - " ^"; - "SyntaxError: invalid syntax")))] + (`error; 1b); + (`errorMsg; "('invalid syntax', ('', 1, 3, '1 2 + 3 4\\n', 1, 4))"); + (`data; ::); + (`stacktrace; " File \"\", line 1\n 1 2 + 3 4\n ^\nSyntaxError: invalid syntax"))] expect no stack trace for errors outside a function - .qu.compare[.test.evaluatePy["text"; "1 + 'a'"]; + result: .test.evaluatePy["text"; "1 + 'a'"]; + stacktrace: string result[`stacktrace]; + result[`stacktrace]: stacktrace; + .qu.compare[ + result; (!) . flip ( - (`result; ::); - (`errored; 1b); - (`error; "unsupported operand type(s) for +: 'int' and 'str'"); - (`backtrace; "\n" sv ( - "Traceback (most recent call last):"; - " File \"\", line 1, in "; - "TypeError: unsupported operand type(s) for +: 'int' and 'str'")))] + (`error; 1b); + (`errorMsg; "(\"unsupported operand type(s) for +: 'int' and 'str'\",)"); + (`data; ::); + (`stacktrace; "Traceback (most recent call last):\n File \"\", line 1, in \nTypeError: unsupported operand type(s) for +: 'int' and 'str'"))] expect a stack trace for syntax errors - .qu.compare[.test.evaluatePy["serialized"; "2)"]; + result: .test.evaluatePy["serialized"; "2)"]; + stacktrace: string result[`stacktrace]; + result[`stacktrace]: stacktrace; + .qu.compare[ + result; (!) . flip ( - (`result; ::); - (`errored; 1b); - (`error; "unmatched ')' (, line 1)"); - (`backtrace; "\n" sv ( - " File \"\", line 1"; - " 2)"; - " ^"; - "SyntaxError: unmatched ')'")))] + (`error; 1b); + (`errorMsg; "(\"unmatched ')'\", ('', 1, 2, '2)', 1, 2))"); + (`data; ::); + (`stacktrace; " File \"\", line 1\n 2)\n ^\nSyntaxError: unmatched ')'"))] feature evaluatePy 2 // These are the tests specific to the VS Code extension @@ -241,7 +243,7 @@ feature evaluatePy 2 expect sendMe to not define anything in the remote process .test.evaluatePy["text"; "1+2"]; .test.evaluatePy["serialized"; "1+2"]; - .pykx.qeval "all(map(lambda x: not(x in globals()), ['_kx_ast', 'BytesIO', 'tokenize', 'is_expr', 'run_line', 'range_to_text', 'run', 'find_strings']))"; + .pykx.qeval "all(map(lambda x: not(x in globals()), ['_kx_ast', 'BytesIO', 'tokenize', 'is_expr', 'run_line', 'range_to_text', 'run', 'find_strings']))" expect the nested functions to be undefined @[{.pykx.eval x; 0b}; @@ -261,7 +263,7 @@ feature evaluatePy 2 .pykx.i.defaultConv: original; all ( defaultConvAfter ~ "py"; - result[`result] ~ 3 + result[`data] ~ 3 ) should raise an error when .pykx is not installed @@ -276,6 +278,360 @@ feature evaluatePy 2 .qu.compare[result; (!) . flip ( - (`result; ::); - (`errored; 1b); - (`error; ".pykx is not defined: please load pykx"))] + (`data; ::); + (`error; 1b); + (`errorMsg; ".pykx is not defined: please load pykx"))] + + should test the structured text functionality in python + expect pykx version set to 2.5 for code compatibility + .pykx.pyexec "import pykx as kx"; + like[string .pykx.qeval "kx.__version__";"2.5.?"] + +feature python toStructuredText + before + .pykx.pyexec "import pykx as kx"; + .pykx.pyexec "import re"; + .pykx.pyexec "import pandas as pd"; + .pykx.pyexec "import numpy as np"; + .pykx.pyexec "from datetime import datetime"; + .pykx.pyexec "from collections import namedtuple"; + .pykx.pyexec "import traceback"; + .test.src: "\n" sv read0 `:resources/evaluatePy.q; + .test.evaluatePy: {[isString; code] + 0i (.test.src; isString; code; "first"; 10000) + }; + + should test the structured text functionality in python + expect python structured text to handle lists + .qu.compare[ + .test.evaluatePy["structuredText"; "['apple', 'banana', 'cherry', 'date']"]`data; + "{\"count\":4,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"'apple'\",\"'banana'\",\"'cherry'\",\"'date'\"],\"order\":[0,1,2,3]}],\"warnings\":[]}"] + expect python structured text to handle lists of lists + .qu.compare[ + .test.evaluatePy["structuredText"; "[['zapple', 'banana', 'cherry'],['dog', 'elephant', 'fox'],['green', 'blue', 'red']]"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"['zapple', 'banana', 'cherry']\",\"['dog', 'elephant', 'fox']\",\"['green', 'blue', 'red']\"],\"order\":[1,2,0]}],\"warnings\":[]}"] + expect python structured text to handle lists of mixed data types + .qu.compare[ + .test.evaluatePy["structuredText"; "[['zapple', 'banana', 'cherry'],['dog', 'elephant', 'fox'],[4, 2, 'red']]"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"['zapple', 'banana', 'cherry']\",\"['dog', 'elephant', 'fox']\",\"[4, 2, 'red']\"],\"order\":\"Sort unsupported: '<' not supported between instances of 'int' and 'str'\"}],\"warnings\":[]}"] + expect python structured text to handle a 3d list + .qu.compare[ + .test.evaluatePy["structuredText"; "[ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]] + ]"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"[[1, 2, 3], [4, 5, 6], [7, 8, 9]]\",\"[[10, 11, 12], [13, 14, 15], [16, 17, 18]]\",\"[[19, 20, 21], [22, 23, 24], [25, 26, 27]]\"],\"order\":[0,1,2]}],\"warnings\":[]}"] + expect python structured text to handle dictionaries + .qu.compare[ + .test.evaluatePy["structuredText"; "{ + 'make': 'Toyota', + 'model': 'Camry', + 'year': 2020, + 'color': 'Blue', + 'price': '$24,425' + }"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"keys\",\"type\":\"list\",\"values\":[\"'make'\",\"'model'\",\"'year'\",\"'color'\",\"'price'\"],\"order\":[3,0,1,4,2],\"isKey\":true},{\"name\":\"values\",\"type\":\"list\",\"values\":[\"'Toyota'\",\"'Camry'\",\"2020\",\"'Blue'\",\"'$24,425'\"],\"order\":[4,2,3,1,0]}],\"warnings\":[]}"] + expect python structured text to handle premade functions + .qu.compare[ + .test.evaluatePy["structuredText"; "re.compile"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"function\",\"values\":[\"def compile(pattern, flags=0):\\n \\\"Compile a regular expression pattern, returning a Pattern object.\\\"\\n return _compile(pattern, flags)\\n\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle custom functions + .pykx.pyexec ["def example_function(): + return 'I am a function'"]; + (.test.evaluatePy["structuredText"; "example_function"]`data) like "{\"count\":1,\"columns\":[[]{\"name\":\"value\",\"type\":\"function\",\"values\":[[]\"\"[]],\"order\":[[]0]}[]],\"warnings\":[[][]]}" + expect python structured text to handle a string array + .qu.compare[ + .test.evaluatePy["structuredText"; "['I am an example string']"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"'I am an example string'\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle a string + .qu.compare[ + .test.evaluatePy["structuredText"; "'The brown fox'"]`data; + "{\"count\":13,\"columns\":[{\"name\":\"values\",\"type\":\"str\",\"values\":[\"'The brown fox'\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle unicode characters + .qu.compare[ + .test.evaluatePy["structuredText"; "'你好'"]`data; + "{\"count\":2,\"columns\":[{\"name\":\"values\",\"type\":\"str\",\"values\":[\"'你好'\"],\"order\":[0]}],\"warnings\":[]}"] + // The output's order should be 2 0 1 + expect python structured text to handle a list of unicode characters + .qu.compare[ + .test.evaluatePy["structuredText"; "['你','好','a']"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"'你'\",\"'好'\",\"'a'\"],\"order\":[2,0,1]}],\"warnings\":[]}"] + expect python structured text to handle list of dicts + .qu.compare[ + .test.evaluatePy["structuredText"; "[ + {'make': 'Toyota', 'model': 'Camry', 'year': 2020, 'color': 'Red'}, + {'make': 'Honda', 'model': 'Accord', 'year': 2019, 'color': 'Blue'}, + {'make': 'Ford', 'model': 'Fusion', 'year': 2021, 'color': 'Black'}, + {'make': 'Chevrolet', 'model': 'Impala', 'year': 2018, 'color': 'White'}, + {'make': 'Tesla', 'model': 'Model S', 'year': 2022, 'color': 'Silver'} + ]"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"{'make': 'Toyota', 'model': 'Camry', 'year': 2020, 'color': 'Red'}\",\"{'make': 'Honda', 'model': 'Accord', 'year': 2019, 'color': 'Blue'}\",\"{'make': 'Ford', 'model': 'Fusion', 'year': 2021, 'color': 'Black'}\",\"{'make': 'Chevrolet', 'model': 'Impala', 'year': 2018, 'color': 'White'}\",\"{'make': 'Tesla', 'model': 'Model S', 'year': 2022, 'color': 'Silver'}\"],\"order\":\"Sort unsupported: '<' not supported between instances of 'dict' and 'dict'\"}],\"warnings\":[]}"] + expect python structured text to handle a list of booleans + .qu.compare[ + .test.evaluatePy["structuredText"; "[True, False, True, False, True]"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"True\",\"False\",\"True\",\"False\",\"True\"],\"order\":[1,3,0,2,4]}],\"warnings\":[]}"] + expect python structured text to handle booleans + .qu.compare[ + .test.evaluatePy["structuredText"; "True"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"bool\",\"values\":[\"True\"],\"order\":[0]}],\"warnings\":[]}"] + xexpect python structured text to handle sets + //xexpecting this test out as pykx version 2.5.4 is ambiguous in handling set data. Updating to version 3+ will be deterministic + // this same test is tested on framework and includes parsing the actual data structure to determine correctness. + .qu.compare[ + .test.evaluatePy["structuredText"; "{'apple', 'banana', 'cherry'}"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"set\",\"values\":[\"'apple'\",\"'cherry'\",\"'banana'\"],\"order\":[0,2,1]}],\"warnings\":[]}"] + xexpect python structured text to handle lists of sets + //xexpecting this test out as pykx version 2.5.4 is ambiguous in handling set data. Updating to version 3+ will be deterministic + // this same test is tested on framework and includes parsing the actual data structure to determine correctness. + .qu.compare[ + .test.evaluatePy["structuredText"; "[ + {'apple', 'banana', 'cherry'}, + {'dog', 'cat', 'mouse'}, + {'red', 'green', 'blue'}, + {1, 2, 3}, + {'Python', 'Java', 'C++'} + ]"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"{'apple', 'cherry', 'banana'}\",\"{'mouse', 'dog', 'cat'}\",\"{'green', 'red', 'blue'}\",\"{1, 2, 3}\",\"{'Java', 'C++', 'Python'}\"],\"order\":[0,1,2,3,4]}],\"warnings\":[]}"] + expect python structured text to handle byte arrays + // UTF-8 is standard + .qu.compare[ + .test.evaluatePy["structuredText"; "bytearray(b'hello world')"]`data; + "{\"count\":11,\"columns\":[{\"name\":\"values\",\"type\":\"bytearray\",\"values\":[\"104\",\"101\",\"108\",\"108\",\"111\",\"32\",\"119\",\"111\",\"114\",\"108\",\"100\"],\"order\":[5,10,1,0,2,3,9,4,7,8,6]}],\"warnings\":[]}"] + expect python structured text to handle empty pandas dataframes uniquely + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.DataFrame()"]`data; + "{\"count\":0,\"columns\":[{\"name\":\"value\",\"type\":\"DataFrame\",\"values\":[\"Empty DataFrame\\nColumns: []\\nIndex: []\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle pandas dataframe + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.DataFrame({ + 'Name': ['Alice', 'Bob', 'Charlie'], + 'Age': [25, 30, 35], + 'City': ['New York', 'Los Angeles', 'Chicago'] + })"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"Name\",\"type\":\"list\",\"values\":[\"'Alice'\",\"'Bob'\",\"'Charlie'\"],\"order\":[0,1,2]},{\"name\":\"Age\",\"type\":\"list\",\"values\":[\"25\",\"30\",\"35\"],\"order\":[0,1,2]},{\"name\":\"City\",\"type\":\"list\",\"values\":[\"'New York'\",\"'Los Angeles'\",\"'Chicago'\"],\"order\":[2,1,0]}],\"warnings\":[]}"] + expect python structured text to handle custom indexed pandas dataframe + .pykx.pyexec "data = { + 'Name': ['Alice', 'Bob', 'Charlie'], + 'Age': [25, 30, 35], + 'City': ['New York', 'Los Angeles', 'Chicago'] + }"; + .pykx.pyexec "df = pd.DataFrame(data)"; + .pykx.pyexec "df.set_index('Age', inplace=True)"; + .qu.compare[ + .test.evaluatePy["structuredText"; "df"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"Age\",\"type\":\"list\",\"values\":[\"25\",\"30\",\"35\"],\"order\":[0,1,2],\"isKey\":true},{\"name\":\"Name\",\"type\":\"list\",\"values\":[\"'Alice'\",\"'Bob'\",\"'Charlie'\"],\"order\":[0,1,2]},{\"name\":\"City\",\"type\":\"list\",\"values\":[\"'New York'\",\"'Los Angeles'\",\"'Chicago'\"],\"order\":[2,1,0]}],\"warnings\":[]}"] + expect python structured text to handle multi index pandas dataframes + .pykx.pyexec "data = { + 'level_1': ['A', 'C', 'B', 'D'], + 'level_2': [1, 2, 1, 2], + 'value1': [10, 20, 30, 40], + 'value2': [50, 60, 70, 80] + }"; + .pykx.pyexec "df = pd.DataFrame(data)"; + .pykx.pyexec "df.set_index(['level_1', 'level_2'], inplace=True)"; + .qu.compare[ + .test.evaluatePy["structuredText"; "df"]`data; + "{\"count\":4,\"columns\":[{\"name\":\"['level_1', 'level_2']\",\"type\":\"list\",\"values\":[\"('A', 1)\",\"('C', 2)\",\"('B', 1)\",\"('D', 2)\"],\"order\":[0,2,1,3],\"isKey\":true},{\"name\":\"value1\",\"type\":\"list\",\"values\":[\"10\",\"20\",\"30\",\"40\"],\"order\":[0,1,2,3]},{\"name\":\"value2\",\"type\":\"list\",\"values\":[\"50\",\"60\",\"70\",\"80\"],\"order\":[0,1,2,3]}],\"warnings\":[]}"] + expect python structured text to handle numpy array + .qu.compare[ + .test.evaluatePy["structuredText"; "np.array([1, 2, 3, 4, 5])"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"ndarray\",\"values\":[\"1\",\"2\",\"3\",\"4\",\"5\"],\"order\":[0,1,2,3,4]}],\"warnings\":[]}"] + expect python structured text to handle empty lists + .qu.compare[ + .test.evaluatePy["structuredText"; "[]"]`data; + "{\"count\":0,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[],\"order\":[]}],\"warnings\":[]}"] + expect python structured text to handle empty dictionaries + .qu.compare[ + .test.evaluatePy["structuredText"; "{}"]`data; + "{\"count\":0,\"columns\":[{\"name\":\"keys\",\"type\":\"list\",\"values\":[],\"order\":[],\"isKey\":true},{\"name\":\"values\",\"type\":\"list\",\"values\":[],\"order\":[]}],\"warnings\":[]}"] + expect python structured text to handle empty sets + .qu.compare[ + .test.evaluatePy["structuredText"; "set()"]`data; + "{\"count\":0,\"columns\":[{\"name\":\"values\",\"type\":\"set\",\"values\":[],\"order\":[]}],\"warnings\":[]}"] + expect python structured text to handle None Types + .qu.compare[ + .test.evaluatePy["structuredText"; "None"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"NoneType\",\"values\":[\"None\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle empty strings + .qu.compare[ + .test.evaluatePy["structuredText"; "''"]`data; + "{\"count\":0,\"columns\":[{\"name\":\"values\",\"type\":\"str\",\"values\":[\"''\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle built in functions + .qu.compare[ + .test.evaluatePy["structuredText"; "len"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"builtin_function_or_method\",\"values\":[\"\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle ints + .qu.compare[ + .test.evaluatePy["structuredText"; "10"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"int\",\"values\":[\"10\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle frozen sets + .qu.compare[ + .test.evaluatePy["structuredText"; "frozenset({1, 2, 3, 6, 5})"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"frozenset\",\"values\":[\"1\",\"2\",\"3\",\"5\",\"6\"],\"order\":[0,1,2,3,4]}],\"warnings\":[]}"] + expect python structured text to handle ranges + .qu.compare[ + .test.evaluatePy["structuredText"; "range(10)"]`data; + "{\"count\":10,\"columns\":[{\"name\":\"values\",\"type\":\"range\",\"values\":[\"range(0, 10)\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle date times + .qu.compare[ + .test.evaluatePy["structuredText"; "datetime(2000, 1, 1)"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"datetime\",\"values\":[\"2000-01-01 00:00:00\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle Ellipsis + .qu.compare[ + .test.evaluatePy["structuredText"; "..."]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"ellipsis\",\"values\":[\"Ellipsis\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle Named Tuples + .pykx.pyexec "Point = namedtuple('Point', ['x', 'y'])"; + .qu.compare[ + .test.evaluatePy["structuredText"; "Point(1, 2)"]`data; + "{\"count\":2,\"columns\":[{\"name\":\"values\",\"type\":\"Point\",\"values\":[\"1\",\"2\"],\"order\":[0,1]}],\"warnings\":[]}"] + expect python structured text to handle circular references + .pykx.pyexec "a = []"; + .pykx.pyexec "a.append(a)"; + .qu.compare[ + .test.evaluatePy["structuredText"; "a"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"values\",\"type\":\"list\",\"values\":[\"[[...]]\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle iterators + .pykx.pyexec "iterable = [1, 2, 3]"; + .pykx.pyexec "iterator = iter(iterable)"; + (.test.evaluatePy["structuredText"; "iterator"]`data) + like "{\"count\":1,\"columns\":[[]{\"name\":\"value\",\"type\":\"list_iterator\",\"values\":[[]\"\"[]],\"order\":[[]0]}[]],\"warnings\":[[][]]}" + expect python structured text to handle Q charvectors as Python strings + .qu.compare[ + .test.evaluatePy["structuredText"; "kx.q(\"\\\"hello\\\"\")"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"CharVector\",\"values\":[\"hello\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle Q dictionaries + .qu.compare[ + .test.evaluatePy["structuredText"; "kx.q(\"`a`b`c!3 1 2\")"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"keys\",\"type\":\"list\",\"values\":[\"pykx.SymbolAtom(pykx.q('`a'))\",\"pykx.SymbolAtom(pykx.q('`b'))\",\"pykx.SymbolAtom(pykx.q('`c'))\"],\"order\":[0,1,2],\"isKey\":true},{\"name\":\"values\",\"type\":\"list\",\"values\":[\"pykx.LongAtom(pykx.q('3'))\",\"pykx.LongAtom(pykx.q('1'))\",\"pykx.LongAtom(pykx.q('2'))\"],\"order\":[1,2,0]}],\"warnings\":[]}"] + expect python structured text to handle Q tables + .qu.compare[ + .test.evaluatePy["structuredText"; "kx.q(\"([] x: 1 2 3 4 5; y: 6 7 8 9 10; z: `a`b`c`d`e)\")"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"x\",\"type\":\"LongVector\",\"values\":[\"pykx.LongAtom(pykx.q('1'))\",\"pykx.LongAtom(pykx.q('2'))\",\"pykx.LongAtom(pykx.q('3'))\",\"pykx.LongAtom(pykx.q('4'))\",\"pykx.LongAtom(pykx.q('5'))\"],\"order\":[0,1,2,3,4]},{\"name\":\"y\",\"type\":\"LongVector\",\"values\":[\"pykx.LongAtom(pykx.q('6'))\",\"pykx.LongAtom(pykx.q('7'))\",\"pykx.LongAtom(pykx.q('8'))\",\"pykx.LongAtom(pykx.q('9'))\",\"pykx.LongAtom(pykx.q('10'))\"],\"order\":[0,1,2,3,4]},{\"name\":\"z\",\"type\":\"SymbolVector\",\"values\":[\"pykx.SymbolAtom(pykx.q('`a'))\",\"pykx.SymbolAtom(pykx.q('`b'))\",\"pykx.SymbolAtom(pykx.q('`c'))\",\"pykx.SymbolAtom(pykx.q('`d'))\",\"pykx.SymbolAtom(pykx.q('`e'))\"],\"order\":[0,1,2,3,4]}],\"warnings\":[]}"] + expect python structured text to handle Q keyed tables + .qu.compare[ + .test.evaluatePy["structuredText"; "kx.q(\"([x: 1 2 3 4 5] y: 6 7 8 9 10; z: `a`b`c`d`e)\")"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"x\",\"type\":\"LongVector\",\"values\":[\"pykx.LongAtom(pykx.q('1'))\",\"pykx.LongAtom(pykx.q('2'))\",\"pykx.LongAtom(pykx.q('3'))\",\"pykx.LongAtom(pykx.q('4'))\",\"pykx.LongAtom(pykx.q('5'))\"],\"order\":[0,1,2,3,4],\"isKey\":true},{\"name\":\"y\",\"type\":\"LongVector\",\"values\":[\"pykx.LongAtom(pykx.q('6'))\",\"pykx.LongAtom(pykx.q('7'))\",\"pykx.LongAtom(pykx.q('8'))\",\"pykx.LongAtom(pykx.q('9'))\",\"pykx.LongAtom(pykx.q('10'))\"],\"order\":[0,1,2,3,4]},{\"name\":\"z\",\"type\":\"SymbolVector\",\"values\":[\"pykx.SymbolAtom(pykx.q('`a'))\",\"pykx.SymbolAtom(pykx.q('`b'))\",\"pykx.SymbolAtom(pykx.q('`c'))\",\"pykx.SymbolAtom(pykx.q('`d'))\",\"pykx.SymbolAtom(pykx.q('`e'))\"],\"order\":[0,1,2,3,4]}],\"warnings\":[]}"] + expect python structured text to handle empty lists + .qu.compare[ + .test.evaluatePy["structuredText"; "kx.q(\"()\")"]`data; + "{\"count\":0,\"columns\":[{\"name\":\"values\",\"type\":\"List\",\"values\":[],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle pandas series containing None values + // Note: When supplying a pandas dataframes anything that is NaN whether it be None or np.Nan, it will default to 'nan' + // This is the expected outcome of the test + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.Series([2,3,None, None, None,1])"]`data; + "{\"count\":6,\"columns\":[{\"name\":\"values\",\"type\":\"Series\",\"values\":[\"2.0\",\"3.0\",\"nan\",\"nan\",\"nan\",\"1.0\"],\"order\":[5,0,1,2,3,4]}],\"warnings\":[]}"] + expect python structured text to handle pandas series containing numeric values + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.Series([12,14,63,6,98,3555,1,2,3,4,5])"]`data; + "{\"count\":11,\"columns\":[{\"name\":\"values\",\"type\":\"Series\",\"values\":[\"12\",\"14\",\"63\",\"6\",\"98\",\"3555\",\"1\",\"2\",\"3\",\"4\",\"5\"],\"order\":[6,7,8,9,10,3,0,1,2,4,5]}],\"warnings\":[]}"] + expect python structured text to handle pandas series containing np NaN values + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.Series([1.5, np.nan, 3.2, np.nan, 5.0])"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"Series\",\"values\":[\"1.5\",\"nan\",\"3.2\",\"nan\",\"5.0\"],\"order\":[0,2,4,1,3]}],\"warnings\":[]}"] + expect python structured text to handle pandas series containing strings + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.Series(['zapple', 'banana', 'cherry', 'date'])"]`data; + "{\"count\":4,\"columns\":[{\"name\":\"values\",\"type\":\"Series\",\"values\":[\"'zapple'\",\"'banana'\",\"'cherry'\",\"'date'\"],\"order\":[1,2,3,0]}],\"warnings\":[]}"] + expect python structured text to handle pandas series containing booleans + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.Series([True, False, True, True, False])"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"Series\",\"values\":[\"True\",\"False\",\"True\",\"True\",\"False\"],\"order\":[1,4,0,2,3]}],\"warnings\":[]}"] + expect python structured text to handle pandas series containing mixed data types + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.Series([10, 'hello', 3.14, None, True])"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"Series\",\"values\":[\"10\",\"'hello'\",\"3.14\",\"None\",\"True\"],\"order\":\"Sort unsupported: '<' not supported between instances of 'int' and 'str'\"}],\"warnings\":[]}"] + expect python structured text to handle pandas series containing datetimes + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.Series(pd.to_datetime(['2024-03-10', '2023-12-25', '2024-01-01', '2025-07-04', '2022-06-15']))"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"Series\",\"values\":[\"Timestamp('2024-03-10 00:00:00')\",\"Timestamp('2023-12-25 00:00:00')\",\"Timestamp('2024-01-01 00:00:00')\",\"Timestamp('2025-07-04 00:00:00')\",\"Timestamp('2022-06-15 00:00:00')\"],\"order\":[4,1,2,0,3]}],\"warnings\":[]}"] + expect python structured text to handle pandas categorical indexes + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.CategoricalIndex(['spring', 'summer', 'fall', 'winter'])"]`data; + "{\"count\":4,\"columns\":[{\"name\":\"values\",\"type\":\"CategoricalIndex\",\"values\":[\"'spring'\",\"'summer'\",\"'fall'\",\"'winter'\"],\"order\":[2,0,1,3]}],\"warnings\":[]}"] + expect python structured text to handle pandas categorical indexes with duplicates + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.CategoricalIndex(['apple', 'banana', 'cherry', 'apple', 'banana'])"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"CategoricalIndex\",\"values\":[\"'apple'\",\"'banana'\",\"'cherry'\",\"'apple'\",\"'banana'\"],\"order\":[0,3,1,4,2]}],\"warnings\":[]}"] + expect python structured text to handle pandas categorical indexes with custom order + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.CategoricalIndex(['high', 'medium', 'low'], categories=['low', 'medium', 'high'], ordered=True)"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"values\",\"type\":\"CategoricalIndex\",\"values\":[\"'high'\",\"'medium'\",\"'low'\"],\"order\":[2,1,0]}],\"warnings\":[]}"] + expect python structured text to only handle empty pandas dataframes values when TABULAR_LIMIT is 0 + .struct.TABULAR_LIMIT: 0; + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.DataFrame()"]`data; + "{\"count\":0,\"columns\":[{\"name\":\"value\",\"type\":\"DataFrame\",\"values\":[\"Empty DataFrame\\nColumns: []\\nIndex: []\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to handle q tables when the number of cells equals the TABULAR_LIMIT + .struct.TABULAR_LIMIT : 3; + .qu.compare[ + .test.evaluatePy["structuredText"; "kx.q(\"([] x: 1 2 3 4 5; y: 6 7 8 9 10; z: `a`b`c`d`e)\")"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"x\",\"type\":\"LongVector\",\"values\":[\"pykx.LongAtom(pykx.q('1'))\"],\"order\":[0]},{\"name\":\"y\",\"type\":\"LongVector\",\"values\":[\"pykx.LongAtom(pykx.q('6'))\"],\"order\":[0]},{\"name\":\"z\",\"type\":\"SymbolVector\",\"values\":[\"pykx.SymbolAtom(pykx.q('`a'))\"],\"order\":[0]}],\"warnings\":[\"Results truncated to TABULAR_LIMIT. Console view is faster for large data.\"]}"] + expect python structured text to truncate pandas dataframes tables when number of cells is over TABULAR_LIMIT + .struct.TABULAR_LIMIT : 2; + .qu.compare[ + .test.evaluatePy["structuredText"; "pd.DataFrame({ + 'Name': ['Alice', 'Bob', 'Charlie'], + 'Age': [25, 30, 35], + 'City': ['New York', 'Los Angeles', 'Chicago'] + })"]`data; + "{\"count\":3,\"columns\":[{\"name\":\"Name\",\"type\":\"list\",\"values\":[\"'Alice'\"],\"order\":[0]},{\"name\":\"Age\",\"type\":\"list\",\"values\":[\"25\"],\"order\":[0]},{\"name\":\"City\",\"type\":\"list\",\"values\":[\"'New York'\"],\"order\":[0]}],\"warnings\":[\"Results truncated to TABULAR_LIMIT. Console view is faster for large data.\"]}"] + expect python structured text to truncate dictionaries when number of cells is over TABULAR_LIMIT + .struct.TABULAR_LIMIT : 2; + .qu.compare[ + .test.evaluatePy["structuredText"; "{ + 'make': 'Toyota', + 'model': 'Camry', + 'year': 2020, + 'color': 'Blue', + 'price': '$24,425' + }"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"keys\",\"type\":\"list\",\"values\":[\"'make'\"],\"order\":[0],\"isKey\":true},{\"name\":\"values\",\"type\":\"list\",\"values\":[\"'Toyota'\"],\"order\":[0]}],\"warnings\":[\"Results truncated to TABULAR_LIMIT. Console view is faster for large data.\"]}"] + expect python structured text to ignore truncating strings + .struct.TABULAR_LIMIT : 2; + .qu.compare[ + .test.evaluatePy["structuredText"; "'The brown fox'"]`data; + "{\"count\":13,\"columns\":[{\"name\":\"values\",\"type\":\"str\",\"values\":[\"'The brown fox'\"],\"order\":[0]}],\"warnings\":[]}"] + expect python structured text to truncate all other data types when count of data is over TABULAR_LIMIT + .struct.TABULAR_LIMIT : 2; + .qu.compare[ + .test.evaluatePy["structuredText"; "(1,2,3,4,5)"]`data; + "{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"tuple\",\"values\":[\"1\",\"2\"],\"order\":[0,1]}],\"warnings\":[\"Results truncated to TABULAR_LIMIT. Console view is faster for large data.\"]}"] + // There is currently a bug where type error would be returned even if being handled in the codebase. + // Since this is a relatively minor bug as the code appears to work on runtime, we will xexpect until a fix is found + xexpect python structuredText using the default TABULAR_LIMIT if no valid value is found. + .struct.TABULAR_LIMIT: "Adrian"; + .test.util.run[ + (.j.j {.pykx.qeval["pystruct_to_structured_text_wrapper"][x;y;z]}["(1,2,3,4,5)"; "first"; 100];.pykx.qeval "int(pykx.q('\"J\"$.struct.DEFAULT_TABULAR_LIMIT'))"); + ::; + ("{\"count\":5,\"columns\":[{\"name\":\"values\",\"type\":\"tuple\",\"values\":[\"1\",\"2\",\"3\",\"4\",\"5\"],\"order\":[0,1,2,3,4]}],\"warnings\":[]}";600000) + ] + expect python structured text to work with indented code + .qu.compare[ + .test.evaluatePy["structuredText"; " 1 + 2"]`data; + "{\"count\":1,\"columns\":[{\"name\":\"value\",\"type\":\"int\",\"values\":[\"3\"],\"order\":[0]}],\"warnings\":[]}"] + should return a dictionary in structured text + expect return dictionary to contain proper keys in the form of `data`error`errorMsg`stacktrace + a: .test.evaluatePy["structuredText"; "['apple', 'banana', 'cherry', 'date']"]; + .qu.compare[key a; `data`error`errorMsg] + expect return dictionary to contain 0b for error + .qu.compare[.test.evaluatePy["structuredText"; "['apple', 'banana', 'cherry', 'date']"]`error; 0b] + expect return dictionary to contain proper errorMsg + .qu.compare[.test.evaluatePy["structuredText"; "['apple', 'banana', 'cherry', 'date']"]`errorMsg; "None"] + expect return dictionary to contain a empty stacktrace + .qu.compare[.test.evaluatePy["structuredText"; "['apple', 'banana', 'cherry', 'date']"]`stacktrace; ""] + should return errors in structured text + expect error dictionary to contain keys in the form of `data`error`errorMsg`stacktrace + a: .test.evaluatePy["structuredText"; "1 + '2'"]; + .qu.compare[key a; `error`errorMsg`data`stacktrace] + expect error dictionary to contain null for data + .qu.compare[.test.evaluatePy["structuredText"; "1 + '2'"]`data; "null"] + expect error dictionary to contain a 1b for error + .qu.compare[.test.evaluatePy["structuredText"; "1 + '2'"]`error; 1b] + expect error dictionary to contain proper errorMsg + .qu.compare[.test.evaluatePy["structuredText"; "1 + '2'"]`errorMsg; "(\"unsupported operand type(s) for +: 'int' and 'str'\",)"] + expect error dictionary to contain stacktrace + .qu.compare[.test.evaluatePy["structuredText"; "1 + '2'"]`stacktrace; "Traceback (most recent call last):\n File \"\", line 1, in \nTypeError: unsupported operand type(s) for +: 'int' and 'str'"] + \ No newline at end of file