Solved issues in c loader and py port, add python port ci, add tests for python to c pointers.

Author: viferga

.github/workflows/release-python.yml

@@ -0,0 +1,29 @@
+name: Release Python Package
+
+on:
+  push:
+    branches: [ master, develop ]
+    paths:
+      - 'source/ports/py_port/setup.py'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_PIPY_USER: ${{ secrets.PYTHON_PIPY_USER }}
+  PYTHON_PIPY_PASSWORD: ${{ secrets.PYTHON_PIPY_PASSWORD }}
+
+jobs:
+  release:
+    name: Release Python Port
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Release the port
+        run: |
+          cd source/ports/py_port
+          bash ./upload.sh

source/loaders/py_loader/source/py_loader_port.c

@@ -819,10 +819,8 @@ static const char py_loader_capsule_reference_id[] = "__metacall_capsule_referen
 
 static void py_loader_port_value_reference_destroy(PyObject *capsule)
 {
-	void *ref = PyCapsule_GetPointer(capsule, py_loader_capsule_reference_id);
-	void *v = PyCapsule_GetContext(capsule);
+	void *v = PyCapsule_GetPointer(capsule, py_loader_capsule_reference_id);
 
-	metacall_value_destroy(ref);
 	metacall_value_destroy(v);
 }
 
@@ -831,7 +829,7 @@ static PyObject *py_loader_port_value_reference(PyObject *self, PyObject *args)
 	static const char format[] = "O:metacall_value_reference";
 	PyObject *obj;
 	loader_impl impl;
-	void *v, *ref;
+	void *v;
 	PyObject *capsule;
 
 	(void)self;
@@ -858,30 +856,15 @@ static PyObject *py_loader_port_value_reference(PyObject *self, PyObject *args)
 		goto error_none;
 	}
 
-	ref = metacall_value_reference(v);
-
-	if (ref == NULL)
-	{
-		PyErr_SetString(PyExc_ValueError, "Failed to create the reference from MetaCall value.");
-		goto error_value;
-	}
-
-	capsule = PyCapsule_New(ref, py_loader_capsule_reference_id, &py_loader_port_value_reference_destroy);
+	capsule = PyCapsule_New(v, py_loader_capsule_reference_id, &py_loader_port_value_reference_destroy);
 
 	if (capsule == NULL)
 	{
-		goto error_ref;
-	}
-
-	if (PyCapsule_SetContext(capsule, v) != 0)
-	{
-		goto error_ref;
+		goto error_value;
 	}
 
 	return capsule;
 
-error_ref:
-	metacall_value_destroy(ref);
 error_value:
 	metacall_value_destroy(v);
 error_none:
@@ -893,7 +876,7 @@ static PyObject *py_loader_port_value_dereference(PyObject *self, PyObject *args
 	static const char format[] = "O:metacall_value_dereference";
 	PyObject *capsule;
 	const char *name = NULL;
-	void *ref, *v;
+	void *v;
 	loader_impl impl;
 	PyObject *result;
 
@@ -922,21 +905,11 @@ static PyObject *py_loader_port_value_dereference(PyObject *self, PyObject *args
 		return py_loader_port_none();
 	}
 
-	/* Get the reference */
-	ref = PyCapsule_GetPointer(capsule, name);
-
-	if (ref == NULL)
-	{
-		return py_loader_port_none();
-	}
-
 	/* Get the value */
-	v = metacall_value_dereference(ref);
+	v = PyCapsule_GetPointer(capsule, name);
 
-	/* Validate the result */
-	if (v != PyCapsule_GetContext(capsule))
+	if (v == NULL)
 	{
-		PyErr_SetString(PyExc_TypeError, "Invalid reference, the PyCapsule context does not match the dereferenced value");
 		return py_loader_port_none();
 	}
 

source/ports/py_port/README.rst

@@ -13,7 +13,7 @@ transparently execute code from Python to any programming language, for
 example, calling JavaScript, NodeJS, Ruby or C# code from Python.
 
 Install
-========
+=======
 
 Install MetaCall binaries first:
 
@@ -28,7 +28,10 @@ Then install MetaCall Python package through MetaCall:
    metacall pip3 install metacall
 
 Example
-========
+=======
+
+Calling Ruby from Python
+------------------------
 
 ``multiply.rb``
 
@@ -44,7 +47,7 @@ Example
 
    from metacall import metacall_load_from_file, metacall
 
-   metacall_load_from_file('rb', [ 'multiply.rb' ]);
+   metacall_load_from_file('rb', [ 'multiply.rb' ])
 
    metacall('multiply', 3, 4); # 12
 
@@ -53,3 +56,152 @@ Running the example:
 .. code:: console
 
    metacall main.py
+
+Using pointers (calling to a C library)
+---------------------------------------
+
+For a simple case, let's imagine that we have a simple C function that
+has an 'in' parameter and we want to pass a pointer to a long, from
+Python side, and then store some value there for reading it later on.
+Let's assume we have a ``loadtest.h`` and ``libloadtest.so`` and a C
+function from this library could be this one:
+
+.. code:: c
+
+   void modify_int_ptr(long *l)
+   {
+       *l = 111;
+   }
+
+Now if we want to call it from Python side, we should do the following:
+
+.. code:: py
+
+   from metacall import metacall_load_from_package, metacall, metacall_value_reference, metacall_value_dereference
+
+   # Load the library (we can configure the search paths for the .so and .lib with metacall_execution_path)
+   # metacall_execution_path('c', '/usr/local/include')
+   # metacall_execution_path('c', '/usr/local/lib')
+   metacall_load_from_package('c', 'loadtest')
+
+   # Create value pointer (int *)
+   int_val = 324444
+   int_val_ref = metacall_value_reference(int_val)
+
+   # Pass the pointer to the function
+   metacall('modify_int_ptr', int_val_ref)
+
+   # Get the value from pointer
+   int_val_deref = metacall_value_dereference(int_val_ref)
+   print(int_val_deref, '==', 111)
+
+For a more complex case, where we have an in/out parameter, for example
+an opaque struct that we want to alloc from C side. First of all, with
+the following header ``loadtest.h``:
+
+.. code:: c
+
+   #ifndef LIB_LOAD_TEST_H
+   #define LIB_LOAD_TEST_H 1
+
+   #if defined(WIN32) || defined(_WIN32)
+       #define EXPORT __declspec(dllexport)
+   #else
+       #define EXPORT __attribute__((visibility("default")))
+   #endif
+
+   #ifdef __cplusplus
+   extern "C" {
+   #endif
+
+   #include <cstdint>
+
+   typedef struct
+   {
+       uint32_t i;
+       double d;
+   } pair;
+
+   typedef struct
+   {
+       uint32_t size;
+       pair *pairs;
+   } pair_list;
+
+   EXPORT int pair_list_init(pair_list **t);
+
+   EXPORT double pair_list_value(pair_list *t, uint32_t id);
+
+   EXPORT void pair_list_destroy(pair_list *t);
+
+   #ifdef __cplusplus
+   }
+   #endif
+
+   #endif /* LIB_LOAD_TEST_H */
+
+With the following implementation ``loadtest.cpp``:
+
+.. code:: c
+
+   #include "loadtest.h"
+
+   int pair_list_init(pair_list **t)
+   {
+       static const uint32_t size = 3;
+
+       *t = new pair_list();
+
+       (*t)->size = size;
+       (*t)->pairs = new pair[(*t)->size];
+
+       for (uint32_t i = 0; i < size; ++i)
+       {
+           (*t)->pairs[i].i = i;
+           (*t)->pairs[i].d = (double)(((double)i) * 1.0);
+       }
+
+       return 0;
+   }
+
+   double pair_list_value(pair_list *t, uint32_t id)
+   {
+       return t->pairs[id].d;
+   }
+
+   void pair_list_destroy(pair_list *t)
+   {
+       delete[] t->pairs;
+       delete t;
+   }
+
+In this case the structs are not opaque, but they can be opaque and it
+will work in the same way. Now, we can call those functions in the
+following manner:
+
+.. code:: py
+
+   from metacall import metacall_load_from_package, metacall, metacall_value_create_ptr, metacall_value_reference, metacall_value_dereference
+
+   metacall_load_from_package('c', 'loadtest')
+
+   # Create a pointer to void* set to NULL
+   list_pair = metacall_value_create_ptr(None)
+
+   # Create a reference to it (void**)
+   list_pair_ref = metacall_value_reference(list_pair)
+
+   # Call the function
+   result = metacall('pair_list_init', list_pair_ref)
+
+   # Get the result updated (struct allocated)
+   list_pair = metacall_value_dereference(list_pair_ref)
+
+   # Pass it to a function
+   result = metacall('pair_list_value', list_pair, 2)
+
+   # Destroy it
+   metacall('pair_list_destroy', list_pair)
+
+   # Here result will be 2.0 because is the third element in the array of pairs inside the struct
+   print(result, '==', 2.0)

source/ports/py_port/VERSION

@@ -0,0 +1 @@
+0.6.0

source/ports/py_port/setup.py

@@ -32,14 +32,18 @@
 with open(os.path.join(current_path, 'README.rst'), encoding='utf-8') as f:
 	long_description = f.read()
 
+# Get the version
+with open(os.path.join(current_path, 'VERSION')) as f:
+	version = f.read()
+
 # Define set up options
 options = {
 	'name': 'metacall',
 
 	# Versions should comply with PEP440.  For a discussion on single-sourcing
 	# the version across setup.py and the project code, see
 	# https://packaging.python.org/en/latest/single_source_version.html
-	'version': '0.5.0',
+	'version': version,
 
 	'description': 'A library for providing inter-language foreign function interface calls',
 	'long_description': long_description,
@@ -82,6 +86,10 @@
 		'Programming Language :: Python :: 3.7',
 		'Programming Language :: Python :: 3.8',
 		'Programming Language :: Python :: 3.9',
+		'Programming Language :: Python :: 3.10',
+		'Programming Language :: Python :: 3.11',
+		'Programming Language :: Python :: 3.12',
+		'Programming Language :: Python :: 3.13',
 	],
 
 	# Keywords
@@ -114,23 +122,23 @@
 }
 
 # Exclude base packages
-exclude_packages =  ['contrib', 'docs', 'test', 'CMakeLists.txt']
+exclude_packages = ['contrib', 'docs', 'test', 'test.py' 'CMakeLists.txt', '.gitignore', 'upload.sh']
 
 # TODO: Review helper
 # # Detect if metacall port is already installed
 # port_installed = False
 
-# Append environment variable or default install path when building manually (TODO: Cross-platform paths)
-sys.path.append(os.environ.get('PORT_LIBRARY_PATH', os.path.join(os.path.sep, 'usr', 'local', 'lib')));
+# # Append environment variable or default install path when building manually (TODO: Cross-platform paths)
+# sys.path.append(os.environ.get('PORT_LIBRARY_PATH', os.path.join(os.path.sep, 'usr', 'local', 'lib')));
 
-# Find is MetaCall is installed as a distributable tarball (TODO: Cross-platform paths)
-rootdir = os.path.join(os.path.sep, 'gnu', 'store')
-regex = re.compile('.*-metacall-.*')
+# # Find is MetaCall is installed as a distributable tarball (TODO: Cross-platform paths)
+# rootdir = os.path.join(os.path.sep, 'gnu', 'store')
+# regex = re.compile('.*-metacall-.*')
 
-for root, dirs, _ in os.walk(rootdir):
-	for folder in dirs:
-		if regex.match(folder) and not folder.endswith('R'):
-			sys.path.append(os.path.join(rootdir, folder, 'lib'))
+# for root, dirs, _ in os.walk(rootdir):
+# 	for folder in dirs:
+# 		if regex.match(folder) and not folder.endswith('R'):
+# 			sys.path.append(os.path.join(rootdir, folder, 'lib'))
 
 # TODO: Review helper
 # # Find if module is installed
@@ -171,7 +179,7 @@
 #	instead of the old one, but we keep the ./helper folder in order to provide future support for
 #	extra commands, although the main idea is to keep the OS dependant install, this can be useful
 #	for updating or doing Python related things. Meanwhile, it will be avoided.
-exclude_packages.append('helper')
+exclude_packages.extend(['helper', 'helper.py'])
 
 # TODO: Review helper
 # if port_installed == True:

source/ports/py_port/run_tests.py renamed to source/ports/py_port/test.py

@@ -1,3 +1,3 @@
-load py ${CMAKE_CURRENT_SOURCE_DIR}/run_tests.py
+load py ${CMAKE_CURRENT_SOURCE_DIR}/test.py
 call main()
 exit