Merge branch 'main' into jpivarski/dont-compare-big-endian-in-awkward

Author: jpivarski

.all-contributorsrc

@@ -596,6 +596,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "rossodisera",
+      "name": "Andrea Serafini",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46961664?v=4",
+      "profile": "https://github.com/rossodisera",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -192,6 +192,7 @@ Thanks especially to the gracious help of Uproot contributors (including the [or
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/MatousVozak"><img src="https://avatars.githubusercontent.com/u/33348191?v=4?s=100" width="100px;" alt="MatousVozak"/><br /><sub><b>MatousVozak</b></sub></a><br /><a href="https://github.com/scikit-hep/uproot5/commits?author=MatousVozak" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rossodisera"><img src="https://avatars.githubusercontent.com/u/46961664?v=4?s=100" width="100px;" alt="Andrea Serafini"/><br /><sub><b>Andrea Serafini</b></sub></a><br /><a href="https://github.com/scikit-hep/uproot5/commits?author=rossodisera" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

pyproject.toml

@@ -83,6 +83,7 @@ test = [
 ]
 test-pyodide = [
   "pytest>=6",
+  "selenium<=4.25.0",  # unpin once >4.26.0 is available
   "pytest-pyodide",
   "pytest-timeout",
   "scikit-hep-testdata"

src/uproot/behaviors/TBranch.py

@@ -210,10 +210,16 @@ def iterate(
                             arrays, report = item
                             arrays = library.global_index(arrays, global_offset)
                             report = report.to_global(global_offset)
-                            yield arrays, report
+                            popper = [arrays]
+                            del arrays
+                            del item
+                            yield popper.pop(), report
+
                         else:
-                            arrays = library.global_index(item, global_offset)
-                            yield arrays
+                            popper = [library.global_index(item, global_offset)]
+                            del item
+                            yield popper.pop()
+
                 except uproot.exceptions.KeyInFileError:
                     if allow_missing:
                         continue
@@ -1111,6 +1117,9 @@ def iterate(
                     ak_add_doc,
                 )
 
+                # no longer needed; save memory
+                del output
+
                 next_baskets = {}
                 for branch, basket_num, basket in ranges_or_baskets:
                     basket_entry_start, basket_entry_stop = basket.entry_start_stop
@@ -1119,10 +1128,14 @@ def iterate(
 
                 previous_baskets = next_baskets
 
+                # no longer needed; save memory
+                popper = [out]
+                del out
+
                 if report:
-                    yield out, Report(self, sub_entry_start, sub_entry_stop)
+                    yield popper.pop(), Report(self, sub_entry_start, sub_entry_stop)
                 else:
-                    yield out
+                    yield popper.pop()
 
     def keys(
         self,

src/uproot/const.py

@@ -5,7 +5,7 @@
 """
 from __future__ import annotations
 
-import struct
+from enum import IntEnum
 
 import numpy
 
@@ -118,8 +118,6 @@
 kStreamedMemberWise = numpy.uint16(1 << 14)
 
 ############ RNTuple https://github.com/root-project/root/blob/master/tree/ntuple/v7/doc/specifications.md
-_rntuple_frame_format = struct.Struct("<Q")
-rntuple_env_header = _rntuple_frame_format.pack(0)  # TODO: need to check this
 rntuple_col_num_to_dtype_dict = {
     1: "uint64",
     2: "uint32",
@@ -149,6 +147,8 @@
     26: "int64",  # split + zigzag encoding
     27: "int32",  # split + zigzag encoding
     28: "int16",  # split + zigzag encoding
+    29: "float32trunc",
+    30: "float32quant",
 }
 rntuple_col_num_to_size_dict = {
     1: 64,
@@ -179,6 +179,8 @@
     26: 64,  # split + zigzag encoding
     27: 32,  # split + zigzag encoding
     28: 16,  # split + zigzag encoding
+    29: 32,  # TODO: variable size
+    30: 32,  # TODO: variable size
 }
 
 rntuple_col_type_to_num_dict = {
@@ -212,7 +214,49 @@
     "splitzigzagint16": 28,
 }
 
-rntuple_role_leaf = 0
-rntuple_role_vector = 1
-rntuple_role_struct = 2
-rntuple_role_union = 3
+
+class RNTupleLocatorType(IntEnum):
+    STANDARD = 0x00
+    LARGE = 0x01
+    DAOS = 0x02
+
+
+class RNTupleEnvelopeType(IntEnum):
+    RESERVED = 0x00
+    HEADER = 0x01
+    FOOTER = 0x02
+    PAGELIST = 0x03
+
+
+class RNTupleFieldRole(IntEnum):
+    LEAF = 0x00
+    VECTOR = 0x01
+    STRUCT = 0x02
+    UNION = 0x03
+    UNSPLIT = 0x04
+
+
+class RNTupleFieldFlag(IntEnum):
+    REPETITIVE = 0x01
+    PROJECTED = 0x02
+    CHECKSUM = 0x04
+
+
+class RNTupleColumnFlag(IntEnum):
+    DEFERRED = 0x08
+    RANGE = 0x10
+
+
+class RNTupleExtraTypeIdentifier(IntEnum):
+    ROOT = 0x00
+
+
+class RNTupleUserMetadataType(IntEnum):
+    INT = 0x01
+    BOOL = 0x02
+    DOUBLE = 0x03
+    STRING = 0x04
+
+
+class RNTupleClusterFlag(IntEnum):
+    SHARDED = 0x01

src/uproot/containers.py

@@ -1541,7 +1541,7 @@ def __eq__(self, other):
             return False
 
     def __array__(self, *args, **kwargs):
-        return numpy.asarray(self._vector, *args, **kwargs)
+        return numpy.asarray(self._values, *args, **kwargs)
 
     def tolist(self):
         return [
@@ -1656,7 +1656,7 @@ def __eq__(self, other):
             return False
 
     def __array__(self, *args, **kwargs):
-        return numpy.asarray(self._vector, *args, **kwargs)
+        return numpy.asarray(self._values, *args, **kwargs)
 
     def tolist(self):
         return [

src/uproot/language/python.py

@@ -516,6 +516,10 @@ def getter(name):
                 else:
                     output[name] = output[name][cut]
 
+        # clear dicts to get rid of big arrays.
+        # note: without this these arrays are not properly released from memory!
+        values.clear()
+        scope.clear()
         return output