bug fix for chunk_size and overlap cause error in dataprep ingestion (#1643)

* bug fix for dataingest url

Signed-off-by: Mustafa <[email protected]>

* add validation function

Signed-off-by: Mustafa <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* validation update

Signed-off-by: Mustafa <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* update validation function

Signed-off-by: Mustafa <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: Mustafa <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: MSCetin37

comps/dataprep/src/utils.py

@@ -667,11 +667,55 @@ def parse_html(input):
     return chucks
 
 
+def validate_and_convert_chunk_params(chunk_size, chunk_overlap):
+    """Validate and convert chunk_size and chunk_overlap to integers if they are strings.
+
+    Ensure chunk_size is a positive integer, chunk_overlap is a non-negative integer,
+    and chunk_overlap is not larger than chunk_size.
+    """
+
+    def validate_param_instance(param, param_name):
+        """Validate that the parameter is an integer or a string that can be converted to an integer.
+
+        Raise a ValueError if the validation fails.
+        """
+        if not isinstance(param, (int, str)):
+            raise ValueError(f"{param_name} must be an integer or a string representing an integer.")
+
+        if isinstance(param, str):
+            try:
+                return int(param)  # Attempt to convert the string to an integer
+            except ValueError:
+                raise ValueError(f"{param_name} must be an integer or a string that can be converted to an integer.")
+        else:
+            return param
+
+    # Validate chunk_size and chunk_overlap, Convert to integers if they are strings
+    chunk_size = validate_param_instance(chunk_size, "chunk_size")
+    chunk_overlap = validate_param_instance(chunk_overlap, "chunk_overlap")
+
+    def validate_param_value(param, param_name, min_value):
+        if param < min_value:
+            raise ValueError(f"{param_name} must be a {min_value} or greater.")
+
+    # Validate chunk_size and chunk_overlap
+    validate_param_value(chunk_size, "chunk_size", 1)
+    validate_param_value(chunk_overlap, "chunk_overlap", 0)
+
+    # Ensure chunk_overlap is not larger than chunk_size
+    if chunk_overlap > chunk_size:
+        raise ValueError("chunk_overlap cannot be larger than chunk_size.")
+
+    return chunk_size, chunk_overlap
+
+
 def load_html_content(links, chunk_size=1500, chunk_overlap=50):
     from langchain.text_splitter import RecursiveCharacterTextSplitter
     from langchain_community.document_loaders import AsyncHtmlLoader
     from langchain_community.document_transformers import Html2TextTransformer
 
+    chunk_size, chunk_overlap = validate_and_convert_chunk_params(chunk_size, chunk_overlap)
+
     loader = AsyncHtmlLoader(links, ignore_load_errors=True, trust_env=True)
     docs = loader.load()
     html2text = Html2TextTransformer()

tests/dataprep/dataprep_utils.sh

@@ -39,7 +39,7 @@ function _invoke_curl() {
     RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
 }
 
-#
+
 function _add_db_params() {
     local db=$1
     if [[ "$db" == "redis" ]]; then
@@ -102,6 +102,14 @@ function ingest_external_link() {
     _invoke_curl $fqdn $port ingest -F 'link_list=["https://www.ces.tech/"]' $extra_args $@
 }
 
+function ingest_external_link_with_chunk_parameters() {
+    local fqdn=$1
+    local port=$2
+    local index_name=$3
+    shift 3
+    _invoke_curl $fqdn $port ingest -F 'link_list=["https://www.ces.tech/"]' -F "chunk_size=1500" -F "chunk_overlap=100" -F "index_name=${index_name}" $@
+}
+
 function delete_all() {
     local fqdn=$1
     local port=$2

tests/dataprep/test_dataprep_redis.sh

@@ -77,6 +77,9 @@ function validate_microservice() {
     ingest_external_link ${ip_address} ${DATAPREP_PORT}
     check_result "dataprep - upload - link" "Data preparation succeeded" dataprep-redis-server ${LOG_PATH}/dataprep_upload_file.log
 
+    ingest_external_link_with_chunk_parameters ${ip_address} ${DATAPREP_PORT} "rag_redis_test_link_params"
+    check_result "dataprep - upload - link" "Data preparation succeeded" dataprep-redis-server ${LOG_PATH}/dataprep_upload_file.log
+
     ingest_txt_with_index_name ${ip_address} ${DATAPREP_PORT} rag_redis_test
     check_result "dataprep - upload with index - txt" "Data preparation succeeded" dataprep-redis-server ${LOG_PATH}/dataprep_upload_file.log