Skip to content

Commit 4b8629f

Browse files
authored
fix: map Athena varbinary correctly in athena2pyarrow (#3413)
The membership check used `("binary" or "varbinary")`, which evaluates to the string "binary" and breaks varbinary casting while accepting substrings.
1 parent 8a73ecb commit 4b8629f

2 files changed

Lines changed: 30 additions & 1 deletion

File tree

awswrangler/_data_types.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ def athena2pyarrow(dtype: str, df_type: str | None = None) -> pa.DataType: # no
344344
return pa.timestamp(unit="ns")
345345
if dtype == "date":
346346
return pa.date32()
347-
if dtype in ("binary" or "varbinary"):
347+
if dtype in ("binary", "varbinary"):
348348
return pa.binary()
349349
if dtype.startswith("decimal") is True:
350350
precision, scale = dtype.replace("decimal(", "").replace(")", "").split(sep=",")

tests/unit/test_data_types.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import pyarrow as pa
2+
import pytest
3+
4+
from awswrangler._data_types import athena2pandas, athena2pyarrow
5+
from awswrangler.exceptions import UnsupportedType
6+
7+
8+
@pytest.mark.parametrize(
9+
"dtype,expected",
10+
[
11+
("binary", pa.binary()),
12+
("varbinary", pa.binary()),
13+
("BINARY", pa.binary()),
14+
("VARBINARY", pa.binary()),
15+
],
16+
)
17+
def test_athena2pyarrow_binary_types(dtype, expected):
18+
assert athena2pyarrow(dtype) == expected
19+
20+
21+
@pytest.mark.parametrize("dtype", ["i", "n", "ary", "bin"])
22+
def test_athena2pyarrow_rejects_binary_substrings(dtype):
23+
with pytest.raises(UnsupportedType, match=f"Unsupported Athena type: {dtype}"):
24+
athena2pyarrow(dtype)
25+
26+
27+
@pytest.mark.parametrize("dtype", ["binary", "varbinary"])
28+
def test_athena2pandas_binary_types(dtype):
29+
assert athena2pandas(dtype) == "bytes"

0 commit comments

Comments
 (0)