Merge pull request #3 from dlt-hub/feat/replace-decimal-by-float

replace Decimal by float

Author: zilto

app/models.py

@@ -5,7 +5,6 @@
 from typing import Optional, List
 
 from pydantic import BaseModel
-from decimal import Decimal
 from datetime import datetime
 
 
@@ -21,7 +20,7 @@ class Customer(BaseModel):
 class Supply(BaseModel):
     id: str
     name: str
-    cost: Decimal
+    cost: float
     perishable: bool
     sku: str
 
@@ -37,22 +36,22 @@ class Order(BaseModel):
     customer_id: str
     store_id: str
     ordered_at: datetime
-    subtotal: Decimal
-    tax_paid: Decimal
-    order_total: Decimal
+    subtotal: float
+    tax_paid: float
+    order_total: float
     items: Optional[List[Item]]
 
 
 class Product(BaseModel):
     sku: str
     name: str
     type: str
-    price: Decimal
+    price: float
     description: str
 
 
 class Store(BaseModel):
     id: str
     name: str
     opened_at: datetime
-    tax_rate: Decimal
+    tax_rate: float

pyproject.toml

@@ -15,6 +15,6 @@ dependencies = [
 dev = [
     "ruff>=0.11.2",
     "pytest>=8.3.5",
-    "dlt==1.8.1",
+    "dlt==1.17.1",
     "pyarrow>=16.0.0",
 ]

tests/test_dlt_extraction.py

@@ -13,7 +13,6 @@
 from app.main import app
 
 from dlt.sources.rest_api import rest_api_source
-from dlt.common.destination.dataset import SupportsReadableDataset
 
 from tests.utils import EXPECTED_TABLES_COUNTS_JANUARY_2017
 
@@ -31,9 +30,7 @@ def send(self, prepared_request, **kwargs):
         )
 
 
-def row_counts(
-    dataset: SupportsReadableDataset, tables: list[str] = None
-) -> dict[str, int]:
+def row_counts(dataset: dlt.Dataset, tables: list[str] = None) -> dict[str, int]:
     counts = dataset.row_counts(table_names=tables).arrow().to_pydict()
     return {t: c for t, c in zip(counts["table_name"], counts["row_count"])}
 
@@ -135,3 +132,113 @@ def test_live_jaffle_shop():
         "orders": 3303,  # 3303 orders in january 2017
         "orders__items": 5072,
     }
+
+
+def test_expected_columns_schema():
+    """
+    This tests extracts the full dataset with the dlt rest_api source
+    """
+    source = rest_api_source(
+        {
+            "client": {
+                "base_url": "http://localhost:8000/api/v1",
+                "paginator": {
+                    "type": "header_link",
+                },
+                "session": FastAPISession(app),
+            },
+            "resources": [
+                "customers",
+                "products",
+                "stores",
+                "supplies",
+                # orders includes items
+                # we only load orders for january 2017
+                {
+                    "name": "orders",
+                    "endpoint": {
+                        "path": "orders",
+                        "params": {
+                            "page_size": 1000,
+                            "start_date": "2017-01-01",
+                            "end_date": "2017-01-31",
+                        },
+                    },
+                },
+            ],
+        },
+    )
+    pipeline = dlt.pipeline(
+        pipeline_name="rest_api_example",
+        destination=duckdb(credentials="test.db"),
+        dataset_name="rest_api_data",
+        dev_mode=True,
+    )
+    pipeline.run(source)
+
+    expected_table_columns = {
+        "customers": {
+            "id": {"name": "id", "data_type": "text", "nullable": True},
+            "name": {"name": "name", "data_type": "text", "nullable": True},
+        },
+        "products": {
+            "sku": {"name": "sku", "data_type": "text", "nullable": True},
+            "name": {"name": "name", "data_type": "text", "nullable": True},
+            "type": {"name": "type", "data_type": "text", "nullable": True},
+            "price": {"name": "price", "data_type": "double", "nullable": True},
+            "description": {
+                "name": "description",
+                "data_type": "text",
+                "nullable": True,
+            },
+        },
+        "stores": {
+            "id": {"name": "id", "data_type": "text", "nullable": True},
+            "name": {"name": "name", "data_type": "text", "nullable": True},
+            "opened_at": {
+                "name": "opened_at",
+                "data_type": "timestamp",
+                "nullable": True,
+            },
+            "tax_rate": {"name": "tax_rate", "data_type": "double", "nullable": True},
+        },
+        "supplies": {
+            "id": {"name": "id", "data_type": "text", "nullable": True},
+            "name": {"name": "name", "data_type": "text", "nullable": True},
+            "cost": {"name": "cost", "data_type": "double", "nullable": True},
+            "perishable": {"name": "perishable", "data_type": "bool", "nullable": True},
+            "sku": {"name": "sku", "data_type": "text", "nullable": True},
+        },
+        "orders": {
+            "id": {"name": "id", "data_type": "text", "nullable": True},
+            "customer_id": {
+                "name": "customer_id",
+                "data_type": "text",
+                "nullable": True,
+            },
+            "store_id": {"name": "store_id", "data_type": "text", "nullable": True},
+            "ordered_at": {
+                "name": "ordered_at",
+                "data_type": "timestamp",
+                "nullable": True,
+            },
+            "subtotal": {"name": "subtotal", "data_type": "double", "nullable": True},
+            "tax_paid": {"name": "tax_paid", "data_type": "double", "nullable": True},
+            "order_total": {
+                "name": "order_total",
+                "data_type": "double",
+                "nullable": True,
+            },
+        },
+        "orders__items": {
+            "id": {"name": "id", "data_type": "text", "nullable": True},
+            "order_id": {"name": "order_id", "data_type": "text", "nullable": True},
+            "sku": {"name": "sku", "data_type": "text", "nullable": True},
+        },
+    }
+
+    for expected_table_name, expected_columns in expected_table_columns.items():
+        table = pipeline.default_schema.tables[expected_table_name]
+        table_columns = table["columns"]
+        for expected_col_name, expected_col in expected_columns.items():
+            assert table_columns[expected_col_name] == expected_col