Assorted cleanups

Author: WillAyd

pandas/core/arrays/list_.py

@@ -74,7 +74,7 @@ class ListDtype(ArrowDtype):
     An ExtensionDtype suitable for storing homogeneous lists of data.
     """
 
-    _is_immutable = True  # TODO(wayd): should we allow mutability?
+    _is_immutable = True
 
     def __init__(self, value_dtype: pa.DataType) -> None:
         super().__init__(pa.large_list(value_dtype))
@@ -100,10 +100,7 @@ def name(self) -> str:  # type: ignore[override]
         """
         A string identifying the data type.
         """
-        # TODO: reshaping tests require the name list to match the large_list
-        # implementation; assumedly there are some astype(str(dtype)) casts
-        # going on. Should fix so this can just be "list[...]" for end user
-        return f"large_list[{self.pyarrow_dtype.value_type!s}]"
+        return f"list[{self.pyarrow_dtype.value_type!s}]"
 
     @property
     def kind(self) -> str:
@@ -124,7 +121,6 @@ def construct_array_type(cls) -> type_t[ListArray]:
         return ListArray
 
     def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None:
-        # TODO(wayd): should we implemented value type support?
         for dtype in dtypes:
             if (
                 isinstance(dtype, ListDtype)
@@ -153,8 +149,7 @@ def __init__(
                 if isinstance(values, (pa.Array, pa.ChunkedArray)):
                     parent_type = values.type
                     if not isinstance(parent_type, (pa.ListType, pa.LargeListType)):
-                        # Ideally could cast here, but I don't think pyarrow implements
-                        # many list casts
+                        # TODO: maybe implement native casts in pyarrow
                         new_values = [
                             [x.as_py()] if x.is_valid else None for x in values
                         ]
@@ -164,12 +159,10 @@ def __init__(
                 else:
                     value_type = pa.array(values).type.value_type
 
-                # Internally always use large_string instead of string
                 if value_type == pa.string():
                     value_type = pa.large_string()
 
             if not isinstance(values, pa.ChunkedArray):
-                # To support NA, we need to create an Array first :-(
                 arr = pa.array(values, type=pa.large_list(value_type), from_pandas=True)
                 self._pa_array = pa.chunked_array(arr, type=pa.large_list(value_type))
             else:
@@ -200,8 +193,6 @@ def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False):
             values = pa.array(scalars, from_pandas=True)
 
         if values.type == "null" and dtype is not None:
-            # TODO: the sequencing here seems wrong; just making the tests pass for now
-            # but this needs a comprehensive review
             pa_type = string_to_pyarrow_type(str(dtype))
             values = pa.array(values, type=pa_type)
 
@@ -232,8 +223,6 @@ def _box_pa(
         return cls._box_pa_array(value, pa_type)
 
     def __getitem__(self, item):
-        # PyArrow does not support NumPy's selection with an equal length
-        # mask, so let's convert those to integral positions if needed
         if isinstance(item, (np.ndarray, ExtensionArray)):
             if is_bool_dtype(item.dtype):
                 mask_len = len(item)
@@ -305,9 +294,6 @@ def _empty(cls, shape: Shape, dtype: ExtensionDtype):
         ExtensionDtype.empty
             ExtensionDtype.empty is the 'official' public version of this API.
         """
-        # Implementer note: while ExtensionDtype.empty is the public way to
-        # call this method, it is still required to implement this `_empty`
-        # method as well (it is called internally in pandas)
         if isinstance(shape, tuple):
             if len(shape) > 1:
                 raise ValueError("ListArray may only be 1-D")
@@ -334,9 +320,9 @@ def __eq__(self, other):
         elif isinstance(other, (pa.ListScalar, pa.LargeListScalar)):
             from pandas.arrays import BooleanArray
 
-            # TODO: pyarrow.compute does not implement broadcasting equality
-            # for an array of lists to a listscalar
-            # TODO: pyarrow doesn't compare missing values as missing???
+            # TODO: pyarrow.compute does not implement equal for lists
+            # https://github.com/apache/arrow/issues/45167
+            # TODO: pyarrow doesn't compare missing values in Python as missing???
             # arr = pa.array([1, 2, None])
             # pc.equal(arr, arr[2]) returns all nulls but
             # arr[2] == arr[2] returns True

pandas/core/internals/construction.py

@@ -13,7 +13,6 @@
 
 import numpy as np
 from numpy import ma
-import pyarrow as pa
 
 from pandas._config import using_string_dtype
 
@@ -462,8 +461,6 @@ def treat_as_nested(data, dtype) -> bool:
         len(data) > 0
         and is_list_like(data[0])
         and getattr(data[0], "ndim", 1) == 1
-        # TODO(wayd): hack so pyarrow list elements don't expand
-        and not isinstance(data[0], pa.ListScalar)
         and not isinstance(dtype, ListDtype)
         and not (isinstance(data, ExtensionArray) and data.ndim == 2)
     )

pandas/core/internals/managers.py

@@ -1976,10 +1976,7 @@ def from_blocks(
 
     @classmethod
     def from_array(
-        cls,
-        array: ArrayLike,
-        index: Index,
-        refs: BlockValuesRefs | None = None,
+        cls, array: ArrayLike, index: Index, refs: BlockValuesRefs | None = None
     ) -> SingleBlockManager:
         """
         Constructor for if we have an array that is not yet a Block.