diff --git a/.ci/install.sh b/.ci/install.sh
index 83d5df01cf6..ba32eab04ea 100755
--- a/.ci/install.sh
+++ b/.ci/install.sh
@@ -36,6 +36,9 @@ python3 -m pip install -U pytest
python3 -m pip install -U pytest-cov
python3 -m pip install -U pytest-timeout
python3 -m pip install pyroma
+# optional test dependency, only install if there's a binary package.
+# fails on beta 3.14 and PyPy
+python3 -m pip install --only-binary=:all: pyarrow || true
if [[ $(uname) != CYGWIN* ]]; then
python3 -m pip install numpy
diff --git a/.github/workflows/macos-install.sh b/.github/workflows/macos-install.sh
index 099f4a582b0..94e3d5d085e 100755
--- a/.github/workflows/macos-install.sh
+++ b/.github/workflows/macos-install.sh
@@ -30,6 +30,9 @@ python3 -m pip install -U pytest-cov
python3 -m pip install -U pytest-timeout
python3 -m pip install pyroma
python3 -m pip install numpy
+# optional test dependency, only install if there's a binary package.
+# fails on beta 3.14 and PyPy
+python3 -m pip install --only-binary=:all: pyarrow || true
# libavif
pushd depends && ./install_libavif.sh && popd
diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml
index 0c3f44e96ef..bf8ec2f2cdf 100644
--- a/.github/workflows/test-windows.yml
+++ b/.github/workflows/test-windows.yml
@@ -88,6 +88,10 @@ jobs:
run: |
python3 -m pip install PyQt6
+ - name: Install PyArrow dependency
+ run: |
+ python3 -m pip install --only-binary=:all: pyarrow || true
+
- name: Install dependencies
id: install
run: |
diff --git a/Tests/test_arrow.py b/Tests/test_arrow.py
new file mode 100644
index 00000000000..b86c77b9aa8
--- /dev/null
+++ b/Tests/test_arrow.py
@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import pytest
+
+from PIL import Image
+
+from .helper import hopper
+
+
+@pytest.mark.parametrize(
+ "mode, dest_modes",
+ (
+ ("L", ["I", "F", "LA", "RGB", "RGBA", "RGBX", "CMYK", "YCbCr", "HSV"]),
+ ("I", ["L", "F"]), # Technically I;32 can work for any 4x8bit storage.
+ ("F", ["I", "L", "LA", "RGB", "RGBA", "RGBX", "CMYK", "YCbCr", "HSV"]),
+ ("LA", ["L", "F"]),
+ ("RGB", ["L", "F"]),
+ ("RGBA", ["L", "F"]),
+ ("RGBX", ["L", "F"]),
+ ("CMYK", ["L", "F"]),
+ ("YCbCr", ["L", "F"]),
+ ("HSV", ["L", "F"]),
+ ),
+)
+def test_invalid_array_type(mode: str, dest_modes: list[str]) -> None:
+ img = hopper(mode)
+ for dest_mode in dest_modes:
+ with pytest.raises(ValueError):
+ Image.fromarrow(img, dest_mode, img.size)
+
+
+def test_invalid_array_size() -> None:
+ img = hopper("RGB")
+
+ assert img.size != (10, 10)
+ with pytest.raises(ValueError):
+ Image.fromarrow(img, "RGB", (10, 10))
+
+
+def test_release_schema() -> None:
+ # these should not error out, valgrind should be clean
+ img = hopper("L")
+ schema = img.__arrow_c_schema__()
+ del schema
+
+
+def test_release_array() -> None:
+ # these should not error out, valgrind should be clean
+ img = hopper("L")
+ array, schema = img.__arrow_c_array__()
+ del array
+ del schema
+
+
+def test_readonly() -> None:
+ img = hopper("L")
+ reloaded = Image.fromarrow(img, img.mode, img.size)
+ assert reloaded.readonly == 1
+ reloaded._readonly = 0
+ assert reloaded.readonly == 1
+
+
+def test_multiblock_l_image() -> None:
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in single channel mode
+ size = (4096, 2 * block_size // 4096)
+ img = Image.new("L", size, 128)
+
+ with pytest.raises(ValueError):
+ (schema, arr) = img.__arrow_c_array__()
+
+
+def test_multiblock_rgba_image() -> None:
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in 4 channel mode
+ size = (4096, (block_size // 4096) // 2)
+ img = Image.new("RGBA", size, (128, 127, 126, 125))
+
+ with pytest.raises(ValueError):
+ (schema, arr) = img.__arrow_c_array__()
+
+
+def test_multiblock_l_schema() -> None:
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in single channel mode
+ size = (4096, 2 * block_size // 4096)
+ img = Image.new("L", size, 128)
+
+ with pytest.raises(ValueError):
+ img.__arrow_c_schema__()
+
+
+def test_multiblock_rgba_schema() -> None:
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in 4 channel mode
+ size = (4096, (block_size // 4096) // 2)
+ img = Image.new("RGBA", size, (128, 127, 126, 125))
+
+ with pytest.raises(ValueError):
+ img.__arrow_c_schema__()
+
+
+def test_singleblock_l_image() -> None:
+ Image.core.set_use_block_allocator(1)
+
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in 4 channel mode
+ size = (4096, 2 * (block_size // 4096))
+ img = Image.new("L", size, 128)
+ assert img.im.isblock()
+
+ (schema, arr) = img.__arrow_c_array__()
+ assert schema
+ assert arr
+
+ Image.core.set_use_block_allocator(0)
+
+
+def test_singleblock_rgba_image() -> None:
+ Image.core.set_use_block_allocator(1)
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in 4 channel mode
+ size = (4096, (block_size // 4096) // 2)
+ img = Image.new("RGBA", size, (128, 127, 126, 125))
+ assert img.im.isblock()
+
+ (schema, arr) = img.__arrow_c_array__()
+ assert schema
+ assert arr
+ Image.core.set_use_block_allocator(0)
+
+
+def test_singleblock_l_schema() -> None:
+ Image.core.set_use_block_allocator(1)
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in single channel mode
+ size = (4096, 2 * block_size // 4096)
+ img = Image.new("L", size, 128)
+ assert img.im.isblock()
+
+ schema = img.__arrow_c_schema__()
+ assert schema
+ Image.core.set_use_block_allocator(0)
+
+
+def test_singleblock_rgba_schema() -> None:
+ Image.core.set_use_block_allocator(1)
+ block_size = Image.core.get_block_size()
+
+ # check a 2 block image in 4 channel mode
+ size = (4096, (block_size // 4096) // 2)
+ img = Image.new("RGBA", size, (128, 127, 126, 125))
+ assert img.im.isblock()
+
+ schema = img.__arrow_c_schema__()
+ assert schema
+ Image.core.set_use_block_allocator(0)
diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py
new file mode 100644
index 00000000000..ece9f8f2657
--- /dev/null
+++ b/Tests/test_pyarrow.py
@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from typing import Any # undone
+
+import pytest
+
+from PIL import Image
+
+from .helper import (
+ assert_deep_equal,
+ assert_image_equal,
+ hopper,
+)
+
+pyarrow = pytest.importorskip("pyarrow", reason="PyArrow not installed")
+
+TEST_IMAGE_SIZE = (10, 10)
+
+
+def _test_img_equals_pyarray(
+ img: Image.Image, arr: Any, mask: list[int] | None
+) -> None:
+ assert img.height * img.width == len(arr)
+ px = img.load()
+ assert px is not None
+ for x in range(0, img.size[0], int(img.size[0] / 10)):
+ for y in range(0, img.size[1], int(img.size[1] / 10)):
+ if mask:
+ for ix, elt in enumerate(mask):
+ pixel = px[x, y]
+ assert isinstance(pixel, tuple)
+ assert pixel[ix] == arr[y * img.width + x].as_py()[elt]
+ else:
+ assert_deep_equal(px[x, y], arr[y * img.width + x].as_py())
+
+
+# really hard to get a non-nullable list type
+fl_uint8_4_type = pyarrow.field(
+ "_", pyarrow.list_(pyarrow.field("_", pyarrow.uint8()).with_nullable(False), 4)
+).type
+
+
+@pytest.mark.parametrize(
+ "mode, dtype, mask",
+ (
+ ("L", pyarrow.uint8(), None),
+ ("I", pyarrow.int32(), None),
+ ("F", pyarrow.float32(), None),
+ ("LA", fl_uint8_4_type, [0, 3]),
+ ("RGB", fl_uint8_4_type, [0, 1, 2]),
+ ("RGBA", fl_uint8_4_type, None),
+ ("RGBX", fl_uint8_4_type, None),
+ ("CMYK", fl_uint8_4_type, None),
+ ("YCbCr", fl_uint8_4_type, [0, 1, 2]),
+ ("HSV", fl_uint8_4_type, [0, 1, 2]),
+ ),
+)
+def test_to_array(mode: str, dtype: Any, mask: list[int] | None) -> None:
+ img = hopper(mode)
+
+ # Resize to non-square
+ img = img.crop((3, 0, 124, 127))
+ assert img.size == (121, 127)
+
+ arr = pyarrow.array(img)
+ _test_img_equals_pyarray(img, arr, mask)
+ assert arr.type == dtype
+
+ reloaded = Image.fromarrow(arr, mode, img.size)
+
+ assert reloaded
+
+ assert_image_equal(img, reloaded)
+
+
+def test_lifetime() -> None:
+ # valgrind shouldn't error out here.
+ # arrays should be accessible after the image is deleted.
+
+ img = hopper("L")
+
+ arr_1 = pyarrow.array(img)
+ arr_2 = pyarrow.array(img)
+
+ del img
+
+ assert arr_1.sum().as_py() > 0
+ del arr_1
+
+ assert arr_2.sum().as_py() > 0
+ del arr_2
+
+
+def test_lifetime2() -> None:
+ # valgrind shouldn't error out here.
+ # img should remain after the arrays are collected.
+
+ img = hopper("L")
+
+ arr_1 = pyarrow.array(img)
+ arr_2 = pyarrow.array(img)
+
+ assert arr_1.sum().as_py() > 0
+ del arr_1
+
+ assert arr_2.sum().as_py() > 0
+ del arr_2
+
+ img2 = img.copy()
+ px = img2.load()
+ assert px # make mypy happy
+ assert isinstance(px[0, 0], int)
diff --git a/docs/reference/Image.rst b/docs/reference/Image.rst
index bc3758218db..a3ba8cfd8e1 100644
--- a/docs/reference/Image.rst
+++ b/docs/reference/Image.rst
@@ -79,6 +79,7 @@ Constructing images
.. autofunction:: new
.. autofunction:: fromarray
+.. autofunction:: fromarrow
.. autofunction:: frombytes
.. autofunction:: frombuffer
@@ -370,6 +371,8 @@ Protocols
.. autoclass:: SupportsArrayInterface
:show-inheritance:
+.. autoclass:: SupportsArrowArrayInterface
+ :show-inheritance:
.. autoclass:: SupportsGetData
:show-inheritance:
diff --git a/docs/reference/arrow_support.rst b/docs/reference/arrow_support.rst
new file mode 100644
index 00000000000..4a5c45e8624
--- /dev/null
+++ b/docs/reference/arrow_support.rst
@@ -0,0 +1,88 @@
+.. _arrow-support:
+
+=============
+Arrow Support
+=============
+
+`Arrow `__
+is an in-memory data exchange format that is the spiritual
+successor to the NumPy array interface. It provides for zero-copy
+access to columnar data, which in our case is ``Image`` data.
+
+The goal with Arrow is to provide native zero-copy interoperability
+with any Arrow provider or consumer in the Python ecosystem.
+
+.. warning:: Zero-copy does not mean zero allocation -- the internal
+ memory layout of Pillow images contains an allocation for row
+ pointers, so there is a non-zero, but significantly smaller than a
+ full-copy memory cost to reading an Arrow image.
+
+
+Data Formats
+============
+
+Pillow currently supports exporting Arrow images in all modes
+**except** for ``BGR;15``, ``BGR;16`` and ``BGR;24``. This is due to
+line-length packing in these modes making for non-continuous memory.
+
+For single-band images, the exported array is width*height elements,
+with each pixel corresponding to the appropriate Arrow type.
+
+For multiband images, the exported array is width*height fixed-length
+four-element arrays of uint8. This is memory compatible with the raw
+image storage of four bytes per pixel.
+
+Mode ``1`` images are exported as one uint8 byte/pixel, as this is
+consistent with the internal storage.
+
+Pillow will accept, but not produce, one other format. For any
+multichannel image with 32-bit storage per pixel, Pillow will accept
+an array of width*height int32 elements, which will then be
+interpreted using the mode-specific interpretation of the bytes.
+
+The image mode must match the Arrow band format when reading single
+channel images.
+
+Memory Allocator
+================
+
+Pillow's default memory allocator, the :ref:`block_allocator`,
+allocates up to a 16 MB block for images by default. Larger images
+overflow into additional blocks. Arrow requires a single continuous
+memory allocation, so images allocated in multiple blocks cannot be
+exported in the Arrow format.
+
+To enable the single block allocator::
+
+ from PIL import Image
+ Image.core.set_use_block_allocator(1)
+
+Note that this is a global setting, not a per-image setting.
+
+Unsupported Features
+====================
+
+* Table/dataframe protocol. We support a single array.
+* Null markers, producing or consuming. Null values are inferred from
+ the mode, e.g. RGB images are stored in the first three bytes of
+ each 32-bit pixel, and the last byte is an implied null.
+* Schema negotiation. There is an optional schema for the requested
+ datatype in the Arrow source interface. We ignore that
+ parameter.
+* Array metadata.
+
+Internal Details
+================
+
+Python Arrow C interface:
+https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
+
+The memory that is exported from the Arrow interface is shared -- not
+copied, so the lifetime of the memory allocation is no longer strictly
+tied to the life of the Python object.
+
+The core imaging struct now has a refcount associated with it, and the
+lifetime of the core image struct is now divorced from the Python
+image object. Creating an arrow reference to the image increments the
+refcount, and the imaging struct is only released when the refcount
+reaches zero.
diff --git a/docs/reference/block_allocator.rst b/docs/reference/block_allocator.rst
index 1abe5280fbf..f4d27e24e57 100644
--- a/docs/reference/block_allocator.rst
+++ b/docs/reference/block_allocator.rst
@@ -1,3 +1,6 @@
+
+.. _block_allocator:
+
Block Allocator
===============
diff --git a/docs/reference/internal_design.rst b/docs/reference/internal_design.rst
index 99a18e9ea99..0411779535b 100644
--- a/docs/reference/internal_design.rst
+++ b/docs/reference/internal_design.rst
@@ -9,3 +9,4 @@ Internal Reference
block_allocator
internal_modules
c_extension_debugging
+ arrow_support
diff --git a/pyproject.toml b/pyproject.toml
index 780a938a32c..8564192154d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,6 +54,10 @@ optional-dependencies.fpx = [
optional-dependencies.mic = [
"olefile",
]
+optional-dependencies.test-arrow = [
+ "pyarrow",
+]
+
optional-dependencies.tests = [
"check-manifest",
"coverage>=7.4.2",
@@ -67,6 +71,7 @@ optional-dependencies.tests = [
"pytest-timeout",
"trove-classifiers>=2024.10.12",
]
+
optional-dependencies.typing = [
"typing-extensions; python_version<'3.10'",
]
diff --git a/setup.py b/setup.py
index 9d69b1d6ea6..5ecd6b8160a 100644
--- a/setup.py
+++ b/setup.py
@@ -65,6 +65,7 @@ def get_version() -> str:
_LIB_IMAGING = (
"Access",
"AlphaComposite",
+ "Arrow",
"Resample",
"Reduce",
"Bands",
diff --git a/src/PIL/Image.py b/src/PIL/Image.py
index 60850f4ffaf..233df592c33 100644
--- a/src/PIL/Image.py
+++ b/src/PIL/Image.py
@@ -577,6 +577,14 @@ def size(self) -> tuple[int, int]:
def mode(self) -> str:
return self._mode
+ @property
+ def readonly(self) -> int:
+ return (self._im and self._im.readonly) or self._readonly
+
+ @readonly.setter
+ def readonly(self, readonly: int) -> None:
+ self._readonly = readonly
+
def _new(self, im: core.ImagingCore) -> Image:
new = Image()
new.im = im
@@ -728,6 +736,16 @@ def __array_interface__(self) -> dict[str, str | bytes | int | tuple[int, ...]]:
new["shape"], new["typestr"] = _conv_type_shape(self)
return new
+ def __arrow_c_schema__(self) -> object:
+ self.load()
+ return self.im.__arrow_c_schema__()
+
+ def __arrow_c_array__(
+ self, requested_schema: object | None = None
+ ) -> tuple[object, object]:
+ self.load()
+ return (self.im.__arrow_c_schema__(), self.im.__arrow_c_array__())
+
def __getstate__(self) -> list[Any]:
im_data = self.tobytes() # load image first
return [self.info, self.mode, self.size, self.getpalette(), im_data]
@@ -3201,6 +3219,18 @@ def __array_interface__(self) -> dict[str, Any]:
raise NotImplementedError()
+class SupportsArrowArrayInterface(Protocol):
+ """
+ An object that has an ``__arrow_c_array__`` method corresponding to the arrow c
+ data interface.
+ """
+
+ def __arrow_c_array__(
+ self, requested_schema: "PyCapsule" = None # type: ignore[name-defined] # noqa: F821, UP037
+ ) -> tuple["PyCapsule", "PyCapsule"]: # type: ignore[name-defined] # noqa: F821, UP037
+ raise NotImplementedError()
+
+
def fromarray(obj: SupportsArrayInterface, mode: str | None = None) -> Image:
"""
Creates an image memory from an object exporting the array interface
@@ -3289,6 +3319,56 @@ def fromarray(obj: SupportsArrayInterface, mode: str | None = None) -> Image:
return frombuffer(mode, size, obj, "raw", rawmode, 0, 1)
+def fromarrow(obj: SupportsArrowArrayInterface, mode, size) -> Image:
+ """Creates an image with zero-copy shared memory from an object exporting
+ the arrow_c_array interface protocol::
+
+ from PIL import Image
+ import pyarrow as pa
+ arr = pa.array([0]*(5*5*4), type=pa.uint8())
+ im = Image.fromarrow(arr, 'RGBA', (5, 5))
+
+ If the data representation of the ``obj`` is not compatible with
+ Pillow internal storage, a ValueError is raised.
+
+ Pillow images can also be converted to Arrow objects::
+
+ from PIL import Image
+ import pyarrow as pa
+ im = Image.open('hopper.jpg')
+ arr = pa.array(im)
+
+ As with array support, when converting Pillow images to arrays,
+ only pixel values are transferred. This means that P and PA mode
+ images will lose their palette.
+
+ :param obj: Object with an arrow_c_array interface
+ :param mode: Image mode.
+ :param size: Image size. This must match the storage of the arrow object.
+ :returns: An Image object
+
+ Note that according to the Arrow spec, both the producer and the
+ consumer should consider the exported array to be immutable, as
+ unsynchronized updates will potentially cause inconsistent data.
+
+ See: :ref:`arrow-support` for more detailed information
+
+ .. versionadded:: 11.2.0
+
+ """
+ if not hasattr(obj, "__arrow_c_array__"):
+ msg = "arrow_c_array interface not found"
+ raise ValueError(msg)
+
+ (schema_capsule, array_capsule) = obj.__arrow_c_array__()
+ _im = core.new_arrow(mode, size, schema_capsule, array_capsule)
+ if _im:
+ return Image()._new(_im)
+
+ msg = "new_arrow returned None without an exception"
+ raise ValueError(msg)
+
+
def fromqimage(im: ImageQt.QImage) -> ImageFile.ImageFile:
"""Creates an image instance from a QImage image"""
from . import ImageQt
diff --git a/src/_imaging.c b/src/_imaging.c
index 330a7eef401..72f12214390 100644
--- a/src/_imaging.c
+++ b/src/_imaging.c
@@ -230,6 +230,93 @@ PyImaging_GetBuffer(PyObject *buffer, Py_buffer *view) {
return PyObject_GetBuffer(buffer, view, PyBUF_SIMPLE);
}
+/* -------------------------------------------------------------------- */
+/* Arrow HANDLING */
+/* -------------------------------------------------------------------- */
+
+PyObject *
+ArrowError(int err) {
+ if (err == IMAGING_CODEC_MEMORY) {
+ return ImagingError_MemoryError();
+ }
+ if (err == IMAGING_ARROW_INCOMPATIBLE_MODE) {
+ return ImagingError_ValueError("Incompatible Pillow mode for Arrow array");
+ }
+ if (err == IMAGING_ARROW_MEMORY_LAYOUT) {
+ return ImagingError_ValueError(
+ "Image is in multiple array blocks, use imaging_new_block for zero copy"
+ );
+ }
+ return ImagingError_ValueError("Unknown error");
+}
+
+void
+ReleaseArrowSchemaPyCapsule(PyObject *capsule) {
+ struct ArrowSchema *schema =
+ (struct ArrowSchema *)PyCapsule_GetPointer(capsule, "arrow_schema");
+ if (schema->release != NULL) {
+ schema->release(schema);
+ }
+ free(schema);
+}
+
+PyObject *
+ExportArrowSchemaPyCapsule(ImagingObject *self) {
+ struct ArrowSchema *schema =
+ (struct ArrowSchema *)calloc(1, sizeof(struct ArrowSchema));
+ int err = export_imaging_schema(self->image, schema);
+ if (err == 0) {
+ return PyCapsule_New(schema, "arrow_schema", ReleaseArrowSchemaPyCapsule);
+ }
+ free(schema);
+ return ArrowError(err);
+}
+
+void
+ReleaseArrowArrayPyCapsule(PyObject *capsule) {
+ struct ArrowArray *array =
+ (struct ArrowArray *)PyCapsule_GetPointer(capsule, "arrow_array");
+ if (array->release != NULL) {
+ array->release(array);
+ }
+ free(array);
+}
+
+PyObject *
+ExportArrowArrayPyCapsule(ImagingObject *self) {
+ struct ArrowArray *array =
+ (struct ArrowArray *)calloc(1, sizeof(struct ArrowArray));
+ int err = export_imaging_array(self->image, array);
+ if (err == 0) {
+ return PyCapsule_New(array, "arrow_array", ReleaseArrowArrayPyCapsule);
+ }
+ free(array);
+ return ArrowError(err);
+}
+
+static PyObject *
+_new_arrow(PyObject *self, PyObject *args) {
+ char *mode;
+ int xsize, ysize;
+ PyObject *schema_capsule, *array_capsule;
+ PyObject *ret;
+
+ if (!PyArg_ParseTuple(
+ args, "s(ii)OO", &mode, &xsize, &ysize, &schema_capsule, &array_capsule
+ )) {
+ return NULL;
+ }
+
+ // ImagingBorrowArrow is responsible for retaining the array_capsule
+ ret =
+ PyImagingNew(ImagingNewArrow(mode, xsize, ysize, schema_capsule, array_capsule)
+ );
+ if (!ret) {
+ return ImagingError_ValueError("Invalid Arrow array mode or size mismatch");
+ }
+ return ret;
+}
+
/* -------------------------------------------------------------------- */
/* EXCEPTION REROUTING */
/* -------------------------------------------------------------------- */
@@ -3655,6 +3742,10 @@ static struct PyMethodDef methods[] = {
/* Misc. */
{"save_ppm", (PyCFunction)_save_ppm, METH_VARARGS},
+ /* arrow */
+ {"__arrow_c_schema__", (PyCFunction)ExportArrowSchemaPyCapsule, METH_VARARGS},
+ {"__arrow_c_array__", (PyCFunction)ExportArrowArrayPyCapsule, METH_VARARGS},
+
{NULL, NULL} /* sentinel */
};
@@ -3722,6 +3813,11 @@ _getattr_unsafe_ptrs(ImagingObject *self, void *closure) {
);
}
+static PyObject *
+_getattr_readonly(ImagingObject *self, void *closure) {
+ return PyLong_FromLong(self->image->read_only);
+}
+
static struct PyGetSetDef getsetters[] = {
{"mode", (getter)_getattr_mode},
{"size", (getter)_getattr_size},
@@ -3729,6 +3825,7 @@ static struct PyGetSetDef getsetters[] = {
{"id", (getter)_getattr_id},
{"ptr", (getter)_getattr_ptr},
{"unsafe_ptrs", (getter)_getattr_unsafe_ptrs},
+ {"readonly", (getter)_getattr_readonly},
{NULL}
};
@@ -3983,6 +4080,21 @@ _set_blocks_max(PyObject *self, PyObject *args) {
Py_RETURN_NONE;
}
+static PyObject *
+_set_use_block_allocator(PyObject *self, PyObject *args) {
+ int use_block_allocator;
+ if (!PyArg_ParseTuple(args, "i:set_use_block_allocator", &use_block_allocator)) {
+ return NULL;
+ }
+ ImagingMemorySetBlockAllocator(&ImagingDefaultArena, use_block_allocator);
+ Py_RETURN_NONE;
+}
+
+static PyObject *
+_get_use_block_allocator(PyObject *self, PyObject *args) {
+ return PyLong_FromLong(ImagingDefaultArena.use_block_allocator);
+}
+
static PyObject *
_clear_cache(PyObject *self, PyObject *args) {
int i = 0;
@@ -4104,6 +4216,7 @@ static PyMethodDef functions[] = {
{"fill", (PyCFunction)_fill, METH_VARARGS},
{"new", (PyCFunction)_new, METH_VARARGS},
{"new_block", (PyCFunction)_new_block, METH_VARARGS},
+ {"new_arrow", (PyCFunction)_new_arrow, METH_VARARGS},
{"merge", (PyCFunction)_merge, METH_VARARGS},
/* Functions */
@@ -4190,9 +4303,11 @@ static PyMethodDef functions[] = {
{"get_alignment", (PyCFunction)_get_alignment, METH_VARARGS},
{"get_block_size", (PyCFunction)_get_block_size, METH_VARARGS},
{"get_blocks_max", (PyCFunction)_get_blocks_max, METH_VARARGS},
+ {"get_use_block_allocator", (PyCFunction)_get_use_block_allocator, METH_VARARGS},
{"set_alignment", (PyCFunction)_set_alignment, METH_VARARGS},
{"set_block_size", (PyCFunction)_set_block_size, METH_VARARGS},
{"set_blocks_max", (PyCFunction)_set_blocks_max, METH_VARARGS},
+ {"set_use_block_allocator", (PyCFunction)_set_use_block_allocator, METH_VARARGS},
{"clear_cache", (PyCFunction)_clear_cache, METH_VARARGS},
{NULL, NULL} /* sentinel */
diff --git a/src/libImaging/Arrow.c b/src/libImaging/Arrow.c
new file mode 100644
index 00000000000..33ff2ce779a
--- /dev/null
+++ b/src/libImaging/Arrow.c
@@ -0,0 +1,299 @@
+
+#include "Arrow.h"
+#include "Imaging.h"
+#include
+
+/* struct ArrowSchema* */
+/* _arrow_schema_channel(char* channel, char* format) { */
+
+/* } */
+
+static void
+ReleaseExportedSchema(struct ArrowSchema *array) {
+ // This should not be called on already released array
+ // assert(array->release != NULL);
+
+ if (!array->release) {
+ return;
+ }
+ if (array->format) {
+ free((void *)array->format);
+ array->format = NULL;
+ }
+ if (array->name) {
+ free((void *)array->name);
+ array->name = NULL;
+ }
+ if (array->metadata) {
+ free((void *)array->metadata);
+ array->metadata = NULL;
+ }
+
+ // Release children
+ for (int64_t i = 0; i < array->n_children; ++i) {
+ struct ArrowSchema *child = array->children[i];
+ if (child->release != NULL) {
+ child->release(child);
+ child->release = NULL;
+ }
+ // UNDONE -- should I be releasing the children?
+ }
+
+ // Release dictionary
+ struct ArrowSchema *dict = array->dictionary;
+ if (dict != NULL && dict->release != NULL) {
+ dict->release(dict);
+ dict->release = NULL;
+ }
+
+ // TODO here: release and/or deallocate all data directly owned by
+ // the ArrowArray struct, such as the private_data.
+
+ // Mark array released
+ array->release = NULL;
+}
+
+int
+export_named_type(struct ArrowSchema *schema, char *format, char *name) {
+ char *formatp;
+ char *namep;
+ size_t format_len = strlen(format) + 1;
+ size_t name_len = strlen(name) + 1;
+
+ formatp = calloc(format_len, 1);
+
+ if (!formatp) {
+ return IMAGING_CODEC_MEMORY;
+ }
+
+ namep = calloc(name_len, 1);
+ if (!namep) {
+ free(formatp);
+ return IMAGING_CODEC_MEMORY;
+ }
+
+ strncpy(formatp, format, format_len);
+ strncpy(namep, name, name_len);
+
+ *schema = (struct ArrowSchema){// Type description
+ .format = formatp,
+ .name = namep,
+ .metadata = NULL,
+ .flags = 0,
+ .n_children = 0,
+ .children = NULL,
+ .dictionary = NULL,
+ // Bookkeeping
+ .release = &ReleaseExportedSchema
+ };
+ return 0;
+}
+
+int
+export_imaging_schema(Imaging im, struct ArrowSchema *schema) {
+ int retval = 0;
+
+ if (strcmp(im->arrow_band_format, "") == 0) {
+ return IMAGING_ARROW_INCOMPATIBLE_MODE;
+ }
+
+ /* for now, single block images */
+ if (!(im->blocks_count == 0 || im->blocks_count == 1)) {
+ return IMAGING_ARROW_MEMORY_LAYOUT;
+ }
+
+ if (im->bands == 1) {
+ return export_named_type(schema, im->arrow_band_format, im->band_names[0]);
+ }
+
+ retval = export_named_type(schema, "+w:4", "");
+ if (retval != 0) {
+ return retval;
+ }
+ // if it's not 1 band, it's an int32 at the moment. 4 uint8 bands.
+ schema->n_children = 1;
+ schema->children = calloc(1, sizeof(struct ArrowSchema *));
+ schema->children[0] = (struct ArrowSchema *)calloc(1, sizeof(struct ArrowSchema));
+ retval = export_named_type(schema->children[0], im->arrow_band_format, "pixel");
+ if (retval != 0) {
+ free(schema->children[0]);
+ schema->release(schema);
+ return retval;
+ }
+ return 0;
+}
+
+static void
+release_const_array(struct ArrowArray *array) {
+ Imaging im = (Imaging)array->private_data;
+
+ if (array->n_children == 0) {
+ ImagingDelete(im);
+ }
+
+ // Free the buffers and the buffers array
+ if (array->buffers) {
+ free(array->buffers);
+ array->buffers = NULL;
+ }
+ if (array->children) {
+ // undone -- does arrow release all the children recursively?
+ for (int i = 0; i < array->n_children; i++) {
+ if (array->children[i]->release) {
+ array->children[i]->release(array->children[i]);
+ array->children[i]->release = NULL;
+ free(array->children[i]);
+ }
+ }
+ free(array->children);
+ array->children = NULL;
+ }
+ // Mark released
+ array->release = NULL;
+}
+
+int
+export_single_channel_array(Imaging im, struct ArrowArray *array) {
+ int length = im->xsize * im->ysize;
+
+ /* for now, single block images */
+ if (!(im->blocks_count == 0 || im->blocks_count == 1)) {
+ return IMAGING_ARROW_MEMORY_LAYOUT;
+ }
+
+ if (im->lines_per_block && im->lines_per_block < im->ysize) {
+ length = im->xsize * im->lines_per_block;
+ }
+
+ MUTEX_LOCK(&im->mutex);
+ im->refcount++;
+ MUTEX_UNLOCK(&im->mutex);
+ // Initialize primitive fields
+ *array = (struct ArrowArray){// Data description
+ .length = length,
+ .offset = 0,
+ .null_count = 0,
+ .n_buffers = 2,
+ .n_children = 0,
+ .children = NULL,
+ .dictionary = NULL,
+ // Bookkeeping
+ .release = &release_const_array,
+ .private_data = im
+ };
+
+ // Allocate list of buffers
+ array->buffers = (const void **)malloc(sizeof(void *) * array->n_buffers);
+ // assert(array->buffers != NULL);
+ array->buffers[0] = NULL; // no nulls, null bitmap can be omitted
+
+ if (im->block) {
+ array->buffers[1] = im->block;
+ } else {
+ array->buffers[1] = im->blocks[0].ptr;
+ }
+ return 0;
+}
+
+int
+export_fixed_pixel_array(Imaging im, struct ArrowArray *array) {
+ int length = im->xsize * im->ysize;
+
+ /* for now, single block images */
+ if (!(im->blocks_count == 0 || im->blocks_count == 1)) {
+ return IMAGING_ARROW_MEMORY_LAYOUT;
+ }
+
+ if (im->lines_per_block && im->lines_per_block < im->ysize) {
+ length = im->xsize * im->lines_per_block;
+ }
+
+ MUTEX_LOCK(&im->mutex);
+ im->refcount++;
+ MUTEX_UNLOCK(&im->mutex);
+ // Initialize primitive fields
+ // Fixed length arrays are 1 buffer of validity, and the length in pixels.
+ // Data is in a child array.
+ *array = (struct ArrowArray){// Data description
+ .length = length,
+ .offset = 0,
+ .null_count = 0,
+ .n_buffers = 1,
+ .n_children = 1,
+ .children = NULL,
+ .dictionary = NULL,
+ // Bookkeeping
+ .release = &release_const_array,
+ .private_data = im
+ };
+
+ // Allocate list of buffers
+ array->buffers = (const void **)calloc(1, sizeof(void *) * array->n_buffers);
+ if (!array->buffers) {
+ goto err;
+ }
+ // assert(array->buffers != NULL);
+ array->buffers[0] = NULL; // no nulls, null bitmap can be omitted
+
+ // if it's not 1 band, it's an int32 at the moment. 4 uint8 bands.
+ array->n_children = 1;
+ array->children = calloc(1, sizeof(struct ArrowArray *));
+ if (!array->children) {
+ goto err;
+ }
+ array->children[0] = (struct ArrowArray *)calloc(1, sizeof(struct ArrowArray));
+ if (!array->children[0]) {
+ goto err;
+ }
+
+ MUTEX_LOCK(&im->mutex);
+ im->refcount++;
+ MUTEX_UNLOCK(&im->mutex);
+ *array->children[0] = (struct ArrowArray){// Data description
+ .length = length * 4,
+ .offset = 0,
+ .null_count = 0,
+ .n_buffers = 2,
+ .n_children = 0,
+ .children = NULL,
+ .dictionary = NULL,
+ // Bookkeeping
+ .release = &release_const_array,
+ .private_data = im
+ };
+
+ array->children[0]->buffers =
+ (const void **)calloc(2, sizeof(void *) * array->n_buffers);
+
+ if (im->block) {
+ array->children[0]->buffers[1] = im->block;
+ } else {
+ array->children[0]->buffers[1] = im->blocks[0].ptr;
+ }
+ return 0;
+
+err:
+ if (array->children[0]) {
+ free(array->children[0]);
+ }
+ if (array->children) {
+ free(array->children);
+ }
+ if (array->buffers) {
+ free(array->buffers);
+ }
+ return IMAGING_CODEC_MEMORY;
+}
+
+int
+export_imaging_array(Imaging im, struct ArrowArray *array) {
+ if (strcmp(im->arrow_band_format, "") == 0) {
+ return IMAGING_ARROW_INCOMPATIBLE_MODE;
+ }
+
+ if (im->bands == 1) {
+ return export_single_channel_array(im, array);
+ }
+
+ return export_fixed_pixel_array(im, array);
+}
diff --git a/src/libImaging/Arrow.h b/src/libImaging/Arrow.h
new file mode 100644
index 00000000000..0b285fe8053
--- /dev/null
+++ b/src/libImaging/Arrow.h
@@ -0,0 +1,48 @@
+#include
+#include
+
+// Apache License 2.0.
+// Source apache arrow project
+// https://arrow.apache.org/docs/format/CDataInterface.html
+
+#ifndef ARROW_C_DATA_INTERFACE
+#define ARROW_C_DATA_INTERFACE
+
+#define ARROW_FLAG_DICTIONARY_ORDERED 1
+#define ARROW_FLAG_NULLABLE 2
+#define ARROW_FLAG_MAP_KEYS_SORTED 4
+
+struct ArrowSchema {
+ // Array type description
+ const char *format;
+ const char *name;
+ const char *metadata;
+ int64_t flags;
+ int64_t n_children;
+ struct ArrowSchema **children;
+ struct ArrowSchema *dictionary;
+
+ // Release callback
+ void (*release)(struct ArrowSchema *);
+ // Opaque producer-specific data
+ void *private_data;
+};
+
+struct ArrowArray {
+ // Array data description
+ int64_t length;
+ int64_t null_count;
+ int64_t offset;
+ int64_t n_buffers;
+ int64_t n_children;
+ const void **buffers;
+ struct ArrowArray **children;
+ struct ArrowArray *dictionary;
+
+ // Release callback
+ void (*release)(struct ArrowArray *);
+ // Opaque producer-specific data
+ void *private_data;
+};
+
+#endif // ARROW_C_DATA_INTERFACE
diff --git a/src/libImaging/Imaging.h b/src/libImaging/Imaging.h
index 0fc191d158b..234f9943c5a 100644
--- a/src/libImaging/Imaging.h
+++ b/src/libImaging/Imaging.h
@@ -20,6 +20,8 @@ extern "C" {
#define M_PI 3.1415926535897932384626433832795
#endif
+#include "Arrow.h"
+
/* -------------------------------------------------------------------- */
/*
@@ -104,6 +106,21 @@ struct ImagingMemoryInstance {
/* Virtual methods */
void (*destroy)(Imaging im);
+
+ /* arrow */
+ int refcount; /* Number of arrow arrays that have been allocated */
+ char band_names[4][3]; /* names of bands, max 2 char + null terminator */
+ char arrow_band_format[2]; /* single character + null terminator */
+
+ int read_only; /* flag for read-only. set for arrow borrowed arrays */
+ PyObject *arrow_array_capsule; /* upstream arrow array source */
+
+ int blocks_count; /* Number of blocks that have been allocated */
+ int lines_per_block; /* Number of lines in a block have been allocated */
+
+#ifdef Py_GIL_DISABLED
+ PyMutex mutex;
+#endif
};
#define IMAGING_PIXEL_1(im, x, y) ((im)->image8[(y)][(x)])
@@ -161,6 +178,7 @@ typedef struct ImagingMemoryArena {
int stats_reallocated_blocks; /* Number of blocks which were actually reallocated
after retrieving */
int stats_freed_blocks; /* Number of freed blocks */
+ int use_block_allocator; /* don't use arena, use block allocator */
#ifdef Py_GIL_DISABLED
PyMutex mutex;
#endif
@@ -174,6 +192,8 @@ extern int
ImagingMemorySetBlocksMax(ImagingMemoryArena arena, int blocks_max);
extern void
ImagingMemoryClearCache(ImagingMemoryArena arena, int new_size);
+extern void
+ImagingMemorySetBlockAllocator(ImagingMemoryArena arena, int use_block_allocator);
extern Imaging
ImagingNew(const char *mode, int xsize, int ysize);
@@ -187,6 +207,15 @@ ImagingDelete(Imaging im);
extern Imaging
ImagingNewBlock(const char *mode, int xsize, int ysize);
+extern Imaging
+ImagingNewArrow(
+ const char *mode,
+ int xsize,
+ int ysize,
+ PyObject *schema_capsule,
+ PyObject *array_capsule
+);
+
extern Imaging
ImagingNewPrologue(const char *mode, int xsize, int ysize);
extern Imaging
@@ -700,6 +729,13 @@ _imaging_seek_pyFd(PyObject *fd, Py_ssize_t offset, int whence);
extern Py_ssize_t
_imaging_tell_pyFd(PyObject *fd);
+/* Arrow */
+
+extern int
+export_imaging_array(Imaging im, struct ArrowArray *array);
+extern int
+export_imaging_schema(Imaging im, struct ArrowSchema *schema);
+
/* Errcodes */
#define IMAGING_CODEC_END 1
#define IMAGING_CODEC_OVERRUN -1
@@ -707,6 +743,8 @@ _imaging_tell_pyFd(PyObject *fd);
#define IMAGING_CODEC_UNKNOWN -3
#define IMAGING_CODEC_CONFIG -8
#define IMAGING_CODEC_MEMORY -9
+#define IMAGING_ARROW_INCOMPATIBLE_MODE -10
+#define IMAGING_ARROW_MEMORY_LAYOUT -11
#include "ImagingUtils.h"
extern UINT8 *clip8_lookups;
diff --git a/src/libImaging/Storage.c b/src/libImaging/Storage.c
index 522e9f37557..4fa4ecd1ce4 100644
--- a/src/libImaging/Storage.c
+++ b/src/libImaging/Storage.c
@@ -58,19 +58,22 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
/* Setup image descriptor */
im->xsize = xsize;
im->ysize = ysize;
-
+ im->refcount = 1;
im->type = IMAGING_TYPE_UINT8;
+ strcpy(im->arrow_band_format, "C");
if (strcmp(mode, "1") == 0) {
/* 1-bit images */
im->bands = im->pixelsize = 1;
im->linesize = xsize;
+ strcpy(im->band_names[0], "1");
} else if (strcmp(mode, "P") == 0) {
/* 8-bit palette mapped images */
im->bands = im->pixelsize = 1;
im->linesize = xsize;
im->palette = ImagingPaletteNew("RGB");
+ strcpy(im->band_names[0], "P");
} else if (strcmp(mode, "PA") == 0) {
/* 8-bit palette with alpha */
@@ -78,23 +81,36 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->pixelsize = 4; /* store in image32 memory */
im->linesize = xsize * 4;
im->palette = ImagingPaletteNew("RGB");
+ strcpy(im->band_names[0], "P");
+ strcpy(im->band_names[1], "X");
+ strcpy(im->band_names[2], "X");
+ strcpy(im->band_names[3], "A");
} else if (strcmp(mode, "L") == 0) {
/* 8-bit grayscale (luminance) images */
im->bands = im->pixelsize = 1;
im->linesize = xsize;
+ strcpy(im->band_names[0], "L");
} else if (strcmp(mode, "LA") == 0) {
/* 8-bit grayscale (luminance) with alpha */
im->bands = 2;
im->pixelsize = 4; /* store in image32 memory */
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "L");
+ strcpy(im->band_names[1], "X");
+ strcpy(im->band_names[2], "X");
+ strcpy(im->band_names[3], "A");
} else if (strcmp(mode, "La") == 0) {
/* 8-bit grayscale (luminance) with premultiplied alpha */
im->bands = 2;
im->pixelsize = 4; /* store in image32 memory */
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "L");
+ strcpy(im->band_names[1], "X");
+ strcpy(im->band_names[2], "X");
+ strcpy(im->band_names[3], "a");
} else if (strcmp(mode, "F") == 0) {
/* 32-bit floating point images */
@@ -102,6 +118,8 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->pixelsize = 4;
im->linesize = xsize * 4;
im->type = IMAGING_TYPE_FLOAT32;
+ strcpy(im->arrow_band_format, "f");
+ strcpy(im->band_names[0], "F");
} else if (strcmp(mode, "I") == 0) {
/* 32-bit integer images */
@@ -109,6 +127,8 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->pixelsize = 4;
im->linesize = xsize * 4;
im->type = IMAGING_TYPE_INT32;
+ strcpy(im->arrow_band_format, "i");
+ strcpy(im->band_names[0], "I");
} else if (strcmp(mode, "I;16") == 0 || strcmp(mode, "I;16L") == 0 ||
strcmp(mode, "I;16B") == 0 || strcmp(mode, "I;16N") == 0) {
@@ -118,12 +138,18 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->pixelsize = 2;
im->linesize = xsize * 2;
im->type = IMAGING_TYPE_SPECIAL;
+ strcpy(im->arrow_band_format, "s");
+ strcpy(im->band_names[0], "I");
} else if (strcmp(mode, "RGB") == 0) {
/* 24-bit true colour images */
im->bands = 3;
im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "R");
+ strcpy(im->band_names[1], "G");
+ strcpy(im->band_names[2], "B");
+ strcpy(im->band_names[3], "X");
} else if (strcmp(mode, "BGR;15") == 0) {
/* EXPERIMENTAL */
@@ -132,6 +158,8 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->pixelsize = 2;
im->linesize = (xsize * 2 + 3) & -4;
im->type = IMAGING_TYPE_SPECIAL;
+ /* not allowing arrow due to line length packing */
+ strcpy(im->arrow_band_format, "");
} else if (strcmp(mode, "BGR;16") == 0) {
/* EXPERIMENTAL */
@@ -140,6 +168,8 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->pixelsize = 2;
im->linesize = (xsize * 2 + 3) & -4;
im->type = IMAGING_TYPE_SPECIAL;
+ /* not allowing arrow due to line length packing */
+ strcpy(im->arrow_band_format, "");
} else if (strcmp(mode, "BGR;24") == 0) {
/* EXPERIMENTAL */
@@ -148,32 +178,54 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->pixelsize = 3;
im->linesize = (xsize * 3 + 3) & -4;
im->type = IMAGING_TYPE_SPECIAL;
+ /* not allowing arrow due to line length packing */
+ strcpy(im->arrow_band_format, "");
} else if (strcmp(mode, "RGBX") == 0) {
/* 32-bit true colour images with padding */
im->bands = im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "R");
+ strcpy(im->band_names[1], "G");
+ strcpy(im->band_names[2], "B");
+ strcpy(im->band_names[3], "X");
} else if (strcmp(mode, "RGBA") == 0) {
/* 32-bit true colour images with alpha */
im->bands = im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "R");
+ strcpy(im->band_names[1], "G");
+ strcpy(im->band_names[2], "B");
+ strcpy(im->band_names[3], "A");
} else if (strcmp(mode, "RGBa") == 0) {
/* 32-bit true colour images with premultiplied alpha */
im->bands = im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "R");
+ strcpy(im->band_names[1], "G");
+ strcpy(im->band_names[2], "B");
+ strcpy(im->band_names[3], "a");
} else if (strcmp(mode, "CMYK") == 0) {
/* 32-bit colour separation */
im->bands = im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "C");
+ strcpy(im->band_names[1], "M");
+ strcpy(im->band_names[2], "Y");
+ strcpy(im->band_names[3], "K");
} else if (strcmp(mode, "YCbCr") == 0) {
/* 24-bit video format */
im->bands = 3;
im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "Y");
+ strcpy(im->band_names[1], "Cb");
+ strcpy(im->band_names[2], "Cr");
+ strcpy(im->band_names[3], "X");
} else if (strcmp(mode, "LAB") == 0) {
/* 24-bit color, luminance, + 2 color channels */
@@ -181,6 +233,10 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->bands = 3;
im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "L");
+ strcpy(im->band_names[1], "a");
+ strcpy(im->band_names[2], "b");
+ strcpy(im->band_names[3], "X");
} else if (strcmp(mode, "HSV") == 0) {
/* 24-bit color, luminance, + 2 color channels */
@@ -188,6 +244,10 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
im->bands = 3;
im->pixelsize = 4;
im->linesize = xsize * 4;
+ strcpy(im->band_names[0], "H");
+ strcpy(im->band_names[1], "S");
+ strcpy(im->band_names[2], "V");
+ strcpy(im->band_names[3], "X");
} else {
free(im);
@@ -218,6 +278,7 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size) {
break;
}
+ // UNDONE -- not accurate for arrow
MUTEX_LOCK(&ImagingDefaultArena.mutex);
ImagingDefaultArena.stats_new_count += 1;
MUTEX_UNLOCK(&ImagingDefaultArena.mutex);
@@ -238,8 +299,18 @@ ImagingDelete(Imaging im) {
return;
}
+ MUTEX_LOCK(&im->mutex);
+ im->refcount--;
+
+ if (im->refcount > 0) {
+ MUTEX_UNLOCK(&im->mutex);
+ return;
+ }
+ MUTEX_UNLOCK(&im->mutex);
+
if (im->palette) {
ImagingPaletteDelete(im->palette);
+ im->palette = NULL;
}
if (im->destroy) {
@@ -270,6 +341,7 @@ struct ImagingMemoryArena ImagingDefaultArena = {
0,
0,
0, // Stats
+ 0, // use_block_allocator
#ifdef Py_GIL_DISABLED
{0},
#endif
@@ -302,6 +374,11 @@ ImagingMemorySetBlocksMax(ImagingMemoryArena arena, int blocks_max) {
return 1;
}
+void
+ImagingMemorySetBlockAllocator(ImagingMemoryArena arena, int use_block_allocator) {
+ arena->use_block_allocator = use_block_allocator;
+}
+
void
ImagingMemoryClearCache(ImagingMemoryArena arena, int new_size) {
while (arena->blocks_cached > new_size) {
@@ -396,11 +473,13 @@ ImagingAllocateArray(Imaging im, ImagingMemoryArena arena, int dirty, int block_
if (lines_per_block == 0) {
lines_per_block = 1;
}
+ im->lines_per_block = lines_per_block;
blocks_count = (im->ysize + lines_per_block - 1) / lines_per_block;
// printf("NEW size: %dx%d, ls: %d, lpb: %d, blocks: %d\n",
// im->xsize, im->ysize, aligned_linesize, lines_per_block, blocks_count);
/* One extra pointer is always NULL */
+ im->blocks_count = blocks_count;
im->blocks = calloc(sizeof(*im->blocks), blocks_count + 1);
if (!im->blocks) {
return (Imaging)ImagingError_MemoryError();
@@ -487,6 +566,58 @@ ImagingAllocateBlock(Imaging im) {
return im;
}
+/* Borrowed Arrow Storage Type */
+/* --------------------------- */
+/* Don't allocate the image. */
+
+static void
+ImagingDestroyArrow(Imaging im) {
+ // Rely on the internal Python destructor for the array capsule.
+ if (im->arrow_array_capsule) {
+ Py_DECREF(im->arrow_array_capsule);
+ im->arrow_array_capsule = NULL;
+ }
+}
+
+Imaging
+ImagingBorrowArrow(
+ Imaging im,
+ struct ArrowArray *external_array,
+ int offset_width,
+ PyObject *arrow_capsule
+) {
+ // offset_width is the number of char* for a single offset from arrow
+ Py_ssize_t y, i;
+
+ char *borrowed_buffer = NULL;
+ struct ArrowArray *arr = external_array;
+
+ if (arr->n_children == 1) {
+ arr = arr->children[0];
+ }
+ if (arr->n_buffers == 2) {
+ // buffer 0 is the null list
+ // buffer 1 is the data
+ borrowed_buffer = (char *)arr->buffers[1] + (offset_width * arr->offset);
+ }
+
+ if (!borrowed_buffer) {
+ return (Imaging
+ )ImagingError_ValueError("Arrow Array, exactly 2 buffers required");
+ }
+
+ for (y = i = 0; y < im->ysize; y++) {
+ im->image[y] = borrowed_buffer + i;
+ i += im->linesize;
+ }
+ im->read_only = 1;
+ Py_INCREF(arrow_capsule);
+ im->arrow_array_capsule = arrow_capsule;
+ im->destroy = ImagingDestroyArrow;
+
+ return im;
+}
+
/* --------------------------------------------------------------------
* Create a new, internally allocated, image.
*/
@@ -529,11 +660,17 @@ ImagingNewInternal(const char *mode, int xsize, int ysize, int dirty) {
Imaging
ImagingNew(const char *mode, int xsize, int ysize) {
+ if (ImagingDefaultArena.use_block_allocator) {
+ return ImagingNewBlock(mode, xsize, ysize);
+ }
return ImagingNewInternal(mode, xsize, ysize, 0);
}
Imaging
ImagingNewDirty(const char *mode, int xsize, int ysize) {
+ if (ImagingDefaultArena.use_block_allocator) {
+ return ImagingNewBlock(mode, xsize, ysize);
+ }
return ImagingNewInternal(mode, xsize, ysize, 1);
}
@@ -558,6 +695,66 @@ ImagingNewBlock(const char *mode, int xsize, int ysize) {
return NULL;
}
+Imaging
+ImagingNewArrow(
+ const char *mode,
+ int xsize,
+ int ysize,
+ PyObject *schema_capsule,
+ PyObject *array_capsule
+) {
+ /* A borrowed arrow array */
+ Imaging im;
+ struct ArrowSchema *schema =
+ (struct ArrowSchema *)PyCapsule_GetPointer(schema_capsule, "arrow_schema");
+
+ struct ArrowArray *external_array =
+ (struct ArrowArray *)PyCapsule_GetPointer(array_capsule, "arrow_array");
+
+ if (xsize < 0 || ysize < 0) {
+ return (Imaging)ImagingError_ValueError("bad image size");
+ }
+
+ im = ImagingNewPrologue(mode, xsize, ysize);
+ if (!im) {
+ return NULL;
+ }
+
+ int64_t pixels = (int64_t)xsize * (int64_t)ysize;
+
+ // fmt:off // don't reformat this
+ if (((strcmp(schema->format, "I") == 0 // int32
+ && im->pixelsize == 4 // 4xchar* storage
+ && im->bands >= 2) // INT32 into any INT32 Storage mode
+ || // (()||()) &&
+ (strcmp(schema->format, im->arrow_band_format) == 0 // same mode
+ && im->bands == 1)) // Single band match
+ && pixels == external_array->length) {
+ // one arrow element per, and it matches a pixelsize*char
+ if (ImagingBorrowArrow(im, external_array, im->pixelsize, array_capsule)) {
+ return im;
+ }
+ }
+ if (strcmp(schema->format, "+w:4") == 0 // 4 up array
+ && im->pixelsize == 4 // storage as 32 bpc
+ && schema->n_children > 0 // make sure schema is well formed.
+ && schema->children // make sure schema is well formed
+ && strcmp(schema->children[0]->format, "C") == 0 // Expected format
+ && strcmp(im->arrow_band_format, "C") == 0 // Expected Format
+ && pixels == external_array->length // expected length
+ && external_array->n_children == 1 // array is well formed
+ && external_array->children // array is well formed
+ && 4 * pixels == external_array->children[0]->length) {
+ // 4 up element of char into pixelsize == 4
+ if (ImagingBorrowArrow(im, external_array, 1, array_capsule)) {
+ return im;
+ }
+ }
+ // fmt: on
+ ImagingDelete(im);
+ return NULL;
+}
+
Imaging
ImagingNew2Dirty(const char *mode, Imaging imOut, Imaging imIn) {
/* allocate or validate output image */