@@ -118,6 +118,33 @@ def __init__(
118118 # get token encoder
119119 self ._token_encoder = tiktoken .encoding_for_model (name )
120120
121+ def _validate_docs (self , docs : List [str ]) -> List [str ]:
122+ """Validate embedding input and return normalized docs list."""
123+ if not isinstance (docs , list ):
124+ raise ValueError (
125+ "OpenAI encoder input must be a list of strings, got "
126+ f"{ type (docs ).__name__ } ."
127+ )
128+ if len (docs ) == 0 :
129+ raise ValueError ("OpenAI encoder input cannot be empty." )
130+
131+ normalized_docs : List [str ] = []
132+ for idx , doc in enumerate (docs ):
133+ if not isinstance (doc , str ):
134+ raise ValueError (
135+ "OpenAI encoder input must contain only strings, got "
136+ f"{ type (doc ).__name__ } at index { idx } ."
137+ )
138+ cleaned = doc .strip ()
139+ if cleaned == "" :
140+ raise ValueError (
141+ "OpenAI encoder input contains an empty string at "
142+ f"index { idx } ."
143+ )
144+ normalized_docs .append (cleaned )
145+
146+ return normalized_docs
147+
121148 def __call__ (self , docs : List [str ], truncate : bool = True ) -> List [List [float ]]:
122149 """Encode a list of text documents into embeddings using OpenAI API.
123150
@@ -130,6 +157,8 @@ def __call__(self, docs: List[str], truncate: bool = True) -> List[List[float]]:
130157 raise ValueError ("OpenAI client is not initialized." )
131158 embeds = None
132159
160+ docs = self ._validate_docs (docs )
161+
133162 if truncate :
134163 # check if any document exceeds token limit and truncate if so
135164 docs = [self ._truncate (doc ) for doc in docs ]
@@ -202,6 +231,8 @@ async def acall(self, docs: List[str], truncate: bool = True) -> List[List[float
202231 raise ValueError ("OpenAI async client is not initialized." )
203232 embeds = None
204233
234+ docs = self ._validate_docs (docs )
235+
205236 if truncate :
206237 # check if any document exceeds token limit and truncate if so
207238 docs = [self ._truncate (doc ) for doc in docs ]
0 commit comments