issue204/ Validate correct handling of Numeric Datatypes in BigQuery (#210)

* adding bq numeric testing to validate behavior of bignumeric and other cross datatype tests

* tagging

* test exact error and lint

* add changes with working BIGNUMERIC test

* remove unused imports

* adding a timestamp to test

Author: dhercher

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Untagged
 
+## 1.1.7
+
+- Adding tests to validate BIGNUMERIC BQ type behavior
+
 ## 1.1.6
 
 - Minor fix for Teradata client from breaking IBis changes

setup.py

@@ -18,7 +18,7 @@
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "1.1.6"
+version = "1.1.7"
 release_status = "Development Status :: 3 - Alpha"
 
 with open("README.md", "r") as fh:

tests/resources/bigquery_test_tables.sql

@@ -0,0 +1,10 @@
+
+-- BigQuery table with several different numeric datatypes with the same value
+CREATE OR REPLACE TABLE pso_data_validator.test_data_types AS
+SELECT
+    CAST('1234567890123456789012345678901234.00' AS BIGNUMERIC) bignumeric_type,
+    CAST(2 AS INT64) int_type,
+    CAST(2 AS DECIMAL) decimal_type,
+    CAST(2 AS STRING) text_type,
+    CAST('2021-01-01 00:00:00' AS TIMESTAMP) timestamp_type
+

tests/system/data_sources/test_bigquery.py

@@ -96,6 +96,39 @@
     ],
 }
 
+# TODO: The definition for this table is stored in: ./tests/resources/
+CONFIG_NUMERIC_AGG_VALID = {
+    # BigQuery Specific Connection Config
+    consts.CONFIG_SOURCE_CONN: BQ_CONN,
+    consts.CONFIG_TARGET_CONN: BQ_CONN,
+    # Validation Type
+    consts.CONFIG_TYPE: "Column",
+    # Configuration Required Depending on Validator Type
+    consts.CONFIG_SCHEMA_NAME: "pso_data_validator",
+    consts.CONFIG_TABLE_NAME: "test_data_types",
+    consts.CONFIG_AGGREGATES: [
+        {
+            consts.CONFIG_TYPE: "count",
+            consts.CONFIG_SOURCE_COLUMN: None,
+            consts.CONFIG_TARGET_COLUMN: None,
+            consts.CONFIG_FIELD_ALIAS: "count",
+        },
+        {
+            consts.CONFIG_TYPE: "sum",
+            consts.CONFIG_SOURCE_COLUMN: "int_type",
+            consts.CONFIG_TARGET_COLUMN: "decimal_type",
+            consts.CONFIG_FIELD_ALIAS: "compare_int_decimal",
+        },
+        {
+            consts.CONFIG_TYPE: "sum",
+            consts.CONFIG_SOURCE_COLUMN: "bignumeric_type",
+            consts.CONFIG_TARGET_COLUMN: "bignumeric_type",
+            consts.CONFIG_FIELD_ALIAS: "compare_bignumeric",
+        },
+    ],
+    consts.CONFIG_GROUPED_COLUMNS: [],
+}
+
 BQ_CONN_NAME = "bq-integration-test"
 CLI_CONFIG_FILE = "example_test.yaml"
 
@@ -185,6 +218,16 @@ def test_grouped_count_validator():
         assert row["source_agg_value"] == row["target_agg_value"]
 
 
+def test_numeric_types():
+    validator = data_validation.DataValidation(CONFIG_NUMERIC_AGG_VALID, verbose=True)
+    df = validator.execute()
+
+    for validation in df.to_dict(orient="records"):
+        assert float(validation["source_agg_value"]) == float(
+            validation["target_agg_value"]
+        )
+
+
 def test_cli_store_yaml_then_run():
     # Store BQ Connection
     _store_bq_conn()

third_party/ibis/ibis_addon/datatypes.py

@@ -21,8 +21,8 @@
 
 
 # BigQuery BIGNUMERIC support needs to be pushed to Ibis
-bigquery._pandas_helpers.BQ_TO_ARROW_SCALARS["BIGNUMERIC"] = pyarrow.string
-_DTYPE_TO_IBIS_TYPE["BIGNUMERIC"] = dt.string
+bigquery._pandas_helpers.BQ_TO_ARROW_SCALARS["BIGNUMERIC"] = pyarrow.decimal256
+_DTYPE_TO_IBIS_TYPE["BIGNUMERIC"] = dt.float64
 _DTYPE_TO_IBIS_TYPE["NUMERIC"] = dt.float64