[Ingest] Accept strings with numeric values (within range) for numeric columns (#1358)

Before:
dots weren't shown on 2 maps, and we had only nulls in the Clickhouse
table (as we somehow get `"35.5"`, not `35.5` in ingest for geopoints,
and previous logic put all of that to the attributes, leaving columns as
`NULLs`)
<img width="1726" alt="Screenshot 2025-03-09 at 13 44 32"
src="https://github.com/user-attachments/assets/967db4dc-d946-4379-a4db-a46a45927e1d"
/>




After:
<img width="1723" alt="Screenshot 2025-03-08 at 14 39 38"
src="https://github.com/user-attachments/assets/8ca15ee9-9033-46ee-b2b5-5b05891a11d4"
/>
<img width="1728" alt="Screenshot 2025-03-08 at 14 42 08"
src="https://github.com/user-attachments/assets/221cdcf7-a38f-4a30-abcf-cba7454744c7"
/>
<img width="1728" alt="Screenshot 2025-03-08 at 14 42 50"
src="https://github.com/user-attachments/assets/cdd2be7b-e9ee-4aac-9d83-98b90d7b6925"
/>

---------

Co-authored-by: trzysiek <[email protected]>

Author: trzysiek

ci/it/testcases/test_ingest_types.go

@@ -309,7 +309,7 @@ func (a *IngestTypesTestcase) testSupportedTypesInDefaultSetup(ctx context.Conte
 			resp, bytes := a.RequestToQuesma(ctx, t, "GET", "/"+indexName+"/_search", []byte(`
 { "query": { "match_all": {} } }
 `))
-			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			assert.Equal(t, http.StatusOK, resp.StatusCode, "response status code: %d, response body: %s", resp.StatusCode, string(bytes))
 
 			r.querySuccess = true
 

platform/ingest/ingest_validator.go

@@ -8,6 +8,7 @@ import (
 	"github.com/QuesmaOrg/quesma/platform/clickhouse"
 	"github.com/QuesmaOrg/quesma/platform/logger"
 	"github.com/QuesmaOrg/quesma/platform/types"
+	"github.com/QuesmaOrg/quesma/platform/util"
 	"math"
 )
 
@@ -94,6 +95,28 @@ func validateNumericType(columnType string, incomingValueType string, value inte
 	if isIntegerType(columnType) && isIntegerType(incomingValueType) {
 		return validateNumericRange(columnType, value)
 	}
+
+	// numbers incoming as strings
+	if isFloatingPointType(columnType) && incomingValueType == "String" {
+		if valueAsStr, ok := value.(string); ok {
+			return util.IsFloat(valueAsStr)
+		} else {
+			logger.Error().Msgf("Invalid value type for column of type %s: %T, value: %v", columnType, value, value)
+		}
+	}
+	if isIntegerType(columnType) && incomingValueType == "String" {
+		if valueAsStr, ok := value.(string); ok && util.IsInt(valueAsStr) {
+			valueAsInt, err := util.ToInt64(valueAsStr)
+			if err != nil {
+				logger.Error().Msgf("Failed to convert value to int: %v", valueAsStr)
+				return false
+			}
+			return validateNumericRange(columnType, valueAsInt)
+		} else {
+			logger.Error().Msgf("Invalid value type for column of type %s: %T, value: %v", columnType, value, value)
+		}
+	}
+
 	return false
 }
 

platform/ingest/ingest_validator_test.go

@@ -69,6 +69,8 @@ func TestIngestValidation(t *testing.T) {
 		`{"int32_field":15}`,
 		`{"float_field":7.5}`,
 		`{"float_field":15}`,
+		`{"float_field":"15"}`,
+		`{"float_field":"15.55"}`,
 
 		`{"int32_field":2147483647}`,
 		`{"int32_field":2147483648}`,
@@ -126,7 +128,7 @@ func TestIngestValidation(t *testing.T) {
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"int_field":15}`, tableName),
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"int_field":15}`, tableName),
 
-		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"attributes_values":{"int_field":"15"},"attributes_metadata":{"int_field":"v1;String"}}`, tableName),
+		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"int_field":"15"}`, tableName),
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"attributes_values":{"int_field":"1.5"},"attributes_metadata":{"int_field":"v1;String"}}`, tableName),
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"attributes_values":{"string_field":"15"},"attributes_metadata":{"string_field":"v1;Int64"}}`, tableName),
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"attributes_values":{"string_field":"1.5"},"attributes_metadata":{"string_field":"v1;Float64"}}`, tableName),
@@ -138,6 +140,8 @@ func TestIngestValidation(t *testing.T) {
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"int32_field":15}`, tableName),
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"float_field":7.5}`, tableName),
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"float_field":15}`, tableName),
+		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"float_field":"15"}`, tableName),
+		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"float_field":"15.55"}`, tableName),
 
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"int32_field":2147483647}`, tableName),
 		fmt.Sprintf(`INSERT INTO "%s" FORMAT JSONEachRow {"attributes_values":{"int32_field":"2147483648"},"attributes_metadata":{"int32_field":"v1;Int64"}}`, tableName),

platform/util/maths.go

@@ -2,7 +2,10 @@
 // SPDX-License-Identifier: Elastic-2.0
 package util
 
-import "math"
+import (
+	"math"
+	"strconv"
+)
 
 // IsSmaller checks if a is smaller than b (with a small epsilon, due to float inaccuracies)
 func IsSmaller(a, b float64) bool {
@@ -14,3 +17,20 @@ func IsSmaller(a, b float64) bool {
 func IsFloat64AnInt64(f float64) bool {
 	return f == math.Ceil(f) && f <= float64(math.MaxInt64)
 }
+
+// IsFloat checks if a string is a float
+func IsFloat(s string) bool {
+	_, err := strconv.ParseFloat(s, 64)
+	return err == nil
+}
+
+// IsInt checks if a string is an integer (in range of int64)
+func IsInt(s string) bool {
+	_, err := strconv.ParseInt(s, 10, 64)
+	return err == nil
+}
+
+// ToInt64 converts a string to int64
+func ToInt64(s string) (int64, error) {
+	return strconv.ParseInt(s, 10, 64)
+}