Addition of support for virtual environments (#103)

* venv support (#99)

* virtual env

* removed setting of python home

* cleaned up p.q

* working version for working environments

* update base prefix

* addition of py_SetProgramName

* added travis virtualenv

* added tests for sys commands. Added warning for windows virtual environment users

* fix windows venv warning

Co-authored-by: Dianeod <[email protected]>
Co-authored-by: Diane ODonoghue <[email protected]>

* Venv (#102)

* virtual env

* removed setting of python home

* cleaned up p.q

* working version for working environments

* update base prefix

* addition of py_SetProgramName

* added travis virtualenv

* added tests for sys commands. Added warning for windows virtual environment users

* fix windows venv warning

* Fixed stdOut issue (#101)

* Optional error message conforming to Python return. KXI-1467 (#100)

* Update to p.q logic for handling sys.argv

* reintroduction of old logic

* addition of initial sys argv and embedpy version check

* Addition of functionality to return Python traceback rather than q shorthand

* Memory leak fix and removal of environment variable check C side

* reformat of prr to use modifiable global rather than env variable

* Initial pass at rectifying potential buffer overflow

* update to py.c to initialize traceback import on init

* closer fit to code base style

Co-authored-by: Conor McCarthy <[email protected]>

Co-authored-by: cmccarthy1 <[email protected]>
Co-authored-by: Conor McCarthy <[email protected]>

* Explicit note that venv support limitation is only expected to hold for Windows

Co-authored-by: Dianeod <[email protected]>
Co-authored-by: Diane ODonoghue <[email protected]>
Co-authored-by: Conor McCarthy <[email protected]>

Author: 3 people

.travis.yml

@@ -35,6 +35,22 @@ script:
   else
     echo No kdb+, no tests;
   fi
+
+## virtualenv
+- pip install virtualenv;
+- virtualenv travisEnv;
+- source travisEnv/bin/activate
+- echo $(which python)
+- if [[ "x$QLIC_KC" != "x" ]]; then
+    echo -n $QLIC_KC |base64 --decode > q/kc.lic;
+    pip install --upgrade pip;
+    pip install --upgrade setuptools;
+    pip -q install -r tests/requirements.txt;
+    q test.q -s 4 -q;
+  else
+    echo No kdb+, no tests;
+  fi
+
 compiler:
 - clang
 deploy:

p.q

@@ -7,9 +7,9 @@ if[not .P.loaded:-1h=type@[`.p@;`numpy;`];
  sc:{"'",x,"'.join([__import__('sysconfig').get_config_var(v)for v in",ssr[.j.j y;"\"";"'"],"])"};pr:{"print(",x,");"};
  c:"-c \"",pr["'.'.join([str(getattr(__import__('sys').version_info,x))for x in ['major','minor']])"],"\"2>",$[.z.o like"w*";"nul <nul";"/dev/null"];
  if[(.z.o like"w*")and `3.6>`$first@[system"python3 ",;c;{system"python ",c}];'"embedPy requires python 3.6 or higher on windows"];
- c:"-c \"",pr[$[.z.o like"w*";sc["/python";`BINDIR`VERSION],"+'.dll'";sc["/";`LIBDIR`INSTSONAME]]],pr[$[.z.o like"m*";sc["/";`PYTHONFRAMEWORKPREFIX`INSTSONAME];.z.o like"l*";sc["/libpython";`LIBDIR`LDVERSION],"+'.so'";"''"]],pr["__import__('sys').prefix"],"\"2>",$[.z.o like"w*";"nul <nul";"/dev/null"];
- `L`M`H set'@[system"python3 ",;c;{system"python ",c}];if[count M;if[k~key k:`$":",M;L::M]];
- .p:(`:./p 2:(`init;2))[L;H]]
+ c:"-c \"",pr[$[.z.o like"w*";sc["/python";`BINDIR`VERSION],"+'.dll'";sc["/";`LIBDIR`INSTSONAME]]],pr[$[.z.o like"m*";sc["/";`PYTHONFRAMEWORKPREFIX`INSTSONAME];.z.o like"l*";sc["/libpython";`LIBDIR`LDVERSION],"+'.so'";"''"]],pr["__import__('sys').base_prefix"],pr["__import__('sys').prefix"],pr["__import__('sys').executable"],"\"2>",$[.z.o like"w*";"nul <nul";"/dev/null"];
+ `L`M`H`P`B set'@[system"python3 ",;c;{system"python ",c}];if[count M;if[k~key k:`$":",M;L::M]];.P.env:not H~P;
+ .p:(`:./p 2:(`init;3))[L;H;B]]
 loaded:.P.loaded
 if[not loaded;
  ei:{eo y _ x;n set .p.get[n:`$(2+x)_(y?"(")#y]value y x;};
@@ -112,3 +112,6 @@ if[not loaded;if[not count .p.import[`sys][`:argv]`;.p.import[`sys][:;`:argv;enl
 / Cleanup
 {![`.p;();0b;x]}`getseq`ntolist`runs`wfunc`gethelp`sp`spq`loaded;
 {@[`.p;x;:;.p.import[`builtins]hsym x]}each`tuple`list`dict`isinstance;
+
+/ VirtualEnv warning for windows users
+if[.P.env&.z.o like"w*";-1"Warning: Virtual Environments not supported for embedPy on Windows. Using the 'BASE' version of Python, not the virtual environment Python";]

py.c

@@ -49,11 +49,9 @@ Z K3(call){P a,k,p;I g;P(!pq(x)||!pq(z)&&z->t!=XD,ktrr())g=g1();a=pq(y)?p1(pg(y)
 Z K1(setconv){P(xt<100||xt>111,ktrr())r0(cf);cf=r1(x);R 0;}Z K1(getconv){R r1(cf);}Z K1(isp){R kb(pq(x));}
 Z K1(setpyerr){P(xt!=-KJ,ktrr())P(!errfmt,krr("traceback.format_exception"))pyerr=xj;R 0;}Z K1(getpyerr){R kj(pyerr);}
 
-
-ZV*t;EXP K2(init){
- ZI i=0;I f,g;S l,h;K n,v;P a,b;P(i,0)l=zs(x),h=zs(y);f=pyl(l);free(l);
- P(!f,krr("libpython"))Py_SetPythonHome(Py_DecodeLocale(h,0));free(h);
- Py_InitializeEx(0);if(PyEval_ThreadsInitialized()&&!PyGILState_Check())t0(PyGILState_GetThisThreadState());PyEval_InitThreads();
+ ZV*t;EXP K3(init){ZI i=0;I f,g;S l,h,hh;K n,v;P a,b,pyhome;P(i,0)l=zs(x),h=zs(y),hh=zs(z);f=pyl(l);free(l);
+ P(!f,krr("libpython"))
+ Py_SetPythonHome(Py_DecodeLocale(h,0));Py_SetProgramName(Py_DecodeLocale(hh,0));free(h);free(hh);Py_InitializeEx(0);if(PyEval_ThreadsInitialized()&&!PyGILState_Check())t0(PyGILState_GetThisThreadState());PyEval_InitThreads();
  M=PyModule_GetDict(PyImport_AddModule("__main__"));cf=k(0,"::",0);n=ktn(KS,0);v=ktn(0,0);
  if(a=PyImport_ImportModule("numpy.core.multiarray")){N=PyCapsule_GetPointer(b=PyObject_GetAttrString(a,"_ARRAY_API"),0);if(!N||!pyn(N))N=0;p0(b);p0(a);}PyErr_Clear();
  if(a=PyImport_ImportModule("traceback")){errfmt=PyObject_GetAttrString(a,"format_exception");p0(a);}PyErr_Clear();

tests/sys.t

@@ -0,0 +1,21 @@
+sys:.p.import`sys
+sysconfig:.p.import`sysconfig
+
+qPrefix:sys[`:prefix]`
+qBasePrefix:sys[`:base_prefix]`
+// Have to remove q path to coincide with python
+qPath:":" sv ((sys[`:path]`) except enlist getenv`QHOME)
+qVersion:3#sys[`:version_info]`
+qLibdir:sysconfig[`:get_config_var]["LIBDIR"]`
+
+pyPrefix:raze system"python -c \"import sys; print(sys.prefix)\""
+pyBasePrefix:raze system"python -c \"import sys; print(sys.base_prefix)\""
+pyPath:raze system"python -c \"import sys; print(':'.join(sys.path))\""
+pyVersion:"J"$3#system"python -c \"import sys; [print(getattr(sys.version_info,val)) for val in ('major','minor','micro','releaselevel','serial')]\""
+pyLibdir:raze system"python -c \"import sysconfig; print(sysconfig.get_config_var('LIBDIR'))\""
+
+qPrefix~pyPrefix
+qBasePrefix~pyBasePrefix
+qPath~pyPath
+qVersion~pyVersion
+qLibdir~pyLibdir