Skip to content

Commit 7dd342f

Browse files
ya7010ya7010
committed
Merge branch 'main' of github.com:yassun7010/turu-py into fix_limit
2 parents d9c4a18 + e265ee3 commit 7dd342f

File tree

12 files changed

+322
-328
lines changed

12 files changed

+322
-328
lines changed

‎docs/adapters/snowflake.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,13 @@
99
pip install "turu[snowflake]"
1010
```
1111

12+
!!! warning
13+
If you have any dependencies on [pyarrow](https://arrow.apache.org/), please install the following stubs in your development environment to avoid errors due to `Unknown` types.
14+
15+
```sh
16+
pip install --dev pyarrow-stubs
17+
```
18+
1219
## Usage
1320
### Basic Usage
1421

‎poetry.lock‎

Lines changed: 87 additions & 87 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎turu-snowflake/poetry.lock‎

Lines changed: 168 additions & 192 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎turu-snowflake/pyproject.toml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,16 @@ classifiers = [
2828
]
2929

3030
[tool.poetry.extras]
31+
pandas = ["pandas"]
32+
pyarrow = ["pyarrow", "pyarrow-stubs"]
3133
pandera = ["pandera"]
3234

3335
[tool.poetry.dependencies]
3436
python = "^3.9"
3537
typing-extensions = "^4.9.0"
3638
snowflake-connector-python = "^3.6.0"
39+
pandas = { version = "*", optional = true }
40+
pyarrow = { version = "*", optional = true }
3741
pandera = { version = "^0.18.0", optional = true }
3842

3943
[tool.poetry.dependencies.turu-core]
@@ -61,9 +65,8 @@ pydantic = "^2.5.2"
6165
pytest-xdist = "^3.5.0"
6266
pytest-cov = "^4.1.0"
6367
pytest-asyncio = "^0.23.2"
64-
pandas = "^2.1.4"
65-
pyarrow = "^14.0.2"
66-
pyarrow-stubs = "^10.0.1.7"
68+
69+
pyarrow-stubs = { version = "^10.0.1.7", optional = true }
6770
numpy = "^1.26.3"
6871

6972
[tool.taskipy.tasks]

‎turu-snowflake/src/turu/snowflake/mock_async_connection.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
from turu.core.mock.exception import TuruCsvHeaderOptionRequiredError
1313
from turu.snowflake.cursor import GenericPandasDataFrame, GenericPyArrowTable
1414
from turu.snowflake.features import (
15+
USE_PANDAS,
16+
USE_PYARROW,
1517
GenericPanderaDataFrameModel,
1618
PandasDataFrame,
1719
PanderaDataFrameModel,
@@ -111,8 +113,10 @@ def inject_response_from_csv( # type: ignore[override]
111113
**options: Unpack[CSVOptions],
112114
) -> Self:
113115
if row_type is not None:
114-
if issubclass(row_type, (PandasDataFrame, PanderaDataFrameModel)):
115-
import pandas
116+
if USE_PANDAS and issubclass(
117+
row_type, (PandasDataFrame, PanderaDataFrameModel)
118+
):
119+
import pandas # type: ignore[import]
116120

117121
if options.get("header", True) is False:
118122
raise TuruCsvHeaderOptionRequiredError(row_type)
@@ -122,8 +126,8 @@ def inject_response_from_csv( # type: ignore[override]
122126
pandas.read_csv(filepath, **options), # type: ignore
123127
)
124128

125-
elif issubclass(row_type, PyArrowTable):
126-
import pyarrow.csv
129+
elif USE_PYARROW and issubclass(row_type, PyArrowTable):
130+
import pyarrow.csv # type: ignore[import]
127131

128132
self.inject_response(
129133
row_type,

‎turu-snowflake/src/turu/snowflake/mock_async_cursor.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ async def execute_map(
102102
...
103103

104104
@override
105-
async def execute_map(
105+
async def execute_map( # type: ignore[override]
106106
self,
107107
row_type: Union[
108108
Type[GenericNewRowType],
@@ -167,7 +167,7 @@ async def executemany_map(
167167
...
168168

169169
@override
170-
async def executemany_map(
170+
async def executemany_map( # type: ignore[override]
171171
self,
172172
row_type: Union[
173173
Type[GenericNewRowType],

‎turu-snowflake/src/turu/snowflake/mock_connection.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
from turu.core.mock.exception import TuruCsvHeaderOptionRequiredError
1212
from turu.snowflake.cursor import GenericPandasDataFrame, GenericPyArrowTable
1313
from turu.snowflake.features import (
14+
USE_PANDAS,
15+
USE_PYARROW,
1416
GenericPanderaDataFrameModel,
1517
PandasDataFrame,
1618
PanderaDataFrameModel,
@@ -111,8 +113,10 @@ def inject_response_from_csv( # type: ignore[override]
111113
**options: Unpack[CSVOptions],
112114
) -> Self:
113115
if row_type is not None:
114-
if issubclass(row_type, (PandasDataFrame, PanderaDataFrameModel)):
115-
import pandas
116+
if USE_PANDAS and issubclass(
117+
row_type, (PandasDataFrame, PanderaDataFrameModel)
118+
):
119+
import pandas # type: ignore[import]
116120

117121
pd_options = {}
118122
if not options.get("header", True):
@@ -123,8 +127,8 @@ def inject_response_from_csv( # type: ignore[override]
123127
pandas.read_csv(filepath, **pd_options), # type: ignore
124128
)
125129

126-
elif issubclass(row_type, PyArrowTable):
127-
import pyarrow.csv
130+
elif USE_PYARROW and issubclass(row_type, PyArrowTable):
131+
import pyarrow.csv # type: ignore[import]
128132

129133
if not options.get("header", True):
130134
raise TuruCsvHeaderOptionRequiredError(row_type)

‎turu-snowflake/src/turu/snowflake/mock_cursor.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ def execute_map(
103103
...
104104

105105
@override
106-
def execute_map(
106+
def execute_map( # type: ignore[override]
107107
self,
108108
row_type: Union[
109109
Type[GenericNewRowType],
@@ -168,7 +168,7 @@ def executemany_map(
168168
...
169169

170170
@override
171-
def executemany_map(
171+
def executemany_map( # type: ignore[override]
172172
self,
173173
row_type: Union[
174174
Type[GenericNewRowType],

‎turu-snowflake/tests/turu/test_snowflake.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ class Row(NamedTuple):
5656

5757
@pytest.mark.skipif(not USE_PANDAS, reason="pandas is not installed")
5858
def test_execute_map_pandas_type(self, connection: turu.snowflake.Connection):
59-
import pandas as pd
59+
import pandas as pd # type: ignore[import]
6060

6161
_cursor: turu.snowflake.Cursor[
6262
Never, pd.DataFrame, Never
@@ -203,7 +203,7 @@ def test_cursor_use_role(self, connection: turu.snowflake.Connection):
203203
reason="pyarrow is not installed",
204204
)
205205
def test_fetch_arrow_all(self, connection: turu.snowflake.Connection):
206-
import pyarrow as pa
206+
import pyarrow as pa # type: ignore[import]
207207

208208
expected: pa.Table = pa.table(
209209
data=[pa.array([1, 2], type=pa.int8())],
@@ -215,8 +215,8 @@ def test_fetch_arrow_all(self, connection: turu.snowflake.Connection):
215215

216216
@pytest.mark.skipif(not USE_PYARROW, reason="pyarrow is not installed")
217217
def test_fetch_arrow_batches(self, connection: turu.snowflake.Connection):
218-
from pandas import DataFrame
219-
from pandas.testing import assert_frame_equal
218+
from pandas import DataFrame # type: ignore[import]
219+
from pandas.testing import assert_frame_equal # type: ignore[import]
220220

221221
with connection.execute("select 1 as ID union all select 2 as ID") as cursor:
222222
for row in cursor.fetch_arrow_batches():
@@ -232,17 +232,17 @@ def test_fetch_pandas_all(self, connection: turu.snowflake.Connection):
232232

233233
@pytest.mark.skipif(not USE_PANDAS, reason="pandas is not installed")
234234
def test_fetch_pandas_batches(self, connection: turu.snowflake.Connection):
235-
from pandas import DataFrame
236-
from pandas.testing import assert_frame_equal
235+
from pandas import DataFrame # type: ignore[import]
236+
from pandas.testing import assert_frame_equal # type: ignore[import]
237237

238238
with connection.execute("select 1 as ID union all select 2 AS ID") as cursor:
239239
for df in cursor.fetch_pandas_batches():
240240
assert_frame_equal(df, DataFrame({"ID": [1, 2]}, dtype="int8"))
241241

242242
@pytest.mark.skipif(not USE_PANDAS, reason="pandas is not installed")
243243
def test_record_pandas_dataframe(self, connection: turu.snowflake.Connection):
244-
import pandas as pd
245-
from pandas.testing import assert_frame_equal
244+
import pandas as pd # type: ignore[import]
245+
from pandas.testing import assert_frame_equal # type: ignore[import]
246246

247247
with tempfile.NamedTemporaryFile() as file:
248248
with record_to_csv(

‎turu-snowflake/tests/turu/test_snowflake_async.py‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ class Row(NamedTuple):
6161
async def test_execute_map_pandas_type(
6262
self, async_connection: turu.snowflake.AsyncConnection
6363
):
64-
import pandas as pd
64+
import pandas as pd # type: ignore[import]
6565

6666
_cursor: turu.snowflake.AsyncCursor[
6767
Never, pd.DataFrame, Never
@@ -301,7 +301,7 @@ async def test_cursor_use_role(
301301
async def test_fetch_arrow_all(
302302
self, async_connection: turu.snowflake.AsyncConnection
303303
):
304-
import pyarrow
304+
import pyarrow # type: ignore[import]
305305

306306
async with await async_connection.execute_map(
307307
pyarrow.Table, "select 1 as ID union all select 2 as ID"
@@ -318,9 +318,9 @@ async def test_fetch_arrow_all(
318318
async def test_fetch_arrow_batches(
319319
self, async_connection: turu.snowflake.AsyncConnection
320320
):
321-
import pyarrow
322-
from pandas import DataFrame
323-
from pandas.testing import assert_frame_equal
321+
import pyarrow # type: ignore[import]
322+
from pandas import DataFrame # type: ignore[import]
323+
from pandas.testing import assert_frame_equal # type: ignore[import]
324324

325325
async with await async_connection.execute_map(
326326
pyarrow.Table, "select 1 as ID union all select 2 as ID"
@@ -339,7 +339,7 @@ async def test_fetch_arrow_batches(
339339
async def test_fetch_pandas_all(
340340
self, async_connection: turu.snowflake.AsyncConnection
341341
):
342-
from pandas import DataFrame
342+
from pandas import DataFrame # type: ignore[import]
343343

344344
async with await async_connection.execute_map(
345345
DataFrame, "select 1 as ID union all select 2 ID"
@@ -354,8 +354,8 @@ async def test_fetch_pandas_all(
354354
async def test_fetch_pandas_batches(
355355
self, async_connection: turu.snowflake.AsyncConnection
356356
):
357-
from pandas import DataFrame
358-
from pandas.testing import assert_frame_equal
357+
from pandas import DataFrame # type: ignore[import]
358+
from pandas.testing import assert_frame_equal # type: ignore[import]
359359

360360
async with await async_connection.execute_map(
361361
DataFrame, "select 1 as ID union all select 2 AS ID"
@@ -368,8 +368,8 @@ async def test_fetch_pandas_batches(
368368
async def test_record_pandas_dataframe(
369369
self, async_connection: turu.snowflake.AsyncConnection
370370
):
371-
import pandas as pd
372-
from pandas.testing import assert_frame_equal
371+
import pandas as pd # type: ignore[import]
372+
from pandas.testing import assert_frame_equal # type: ignore[import]
373373

374374
with tempfile.NamedTemporaryFile() as file:
375375
async with record_to_csv(

0 commit comments

Comments
 (0)