Skip to content

Commit 34c191a

Browse files
ByteDanceLiuYangByteDanceLiuYang
authored
fix(vikingdb): trust OpenViking schema for volcengine api_key mode (#2981)
1 parent 0279102 commit 34c191a

4 files changed

Lines changed: 249 additions & 5 deletions

File tree

‎openviking/storage/vectordb/collection/vikingdb_collection.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ def upsert_data(self, data_list: List[Dict[str, Any]], ttl: int = 0):
158158
"collection_name": self.collection_name,
159159
"data": data_list,
160160
"ttl": ttl,
161+
"ignore_unknown_fields": True,
161162
}
162163
return self._data_post(path, data)
163164

@@ -167,6 +168,7 @@ def update_data(self, data_list: List[Dict[str, Any]]):
167168
"project": self.project_name,
168169
"collection_name": self.collection_name,
169170
"data": data_list,
171+
"ignore_unknown_fields": True,
170172
}
171173
return self._data_post(path, data)
172174

@@ -176,6 +178,7 @@ def fetch_data(self, primary_keys: List[Any]) -> FetchDataInCollectionResult:
176178
"project": self.project_name,
177179
"collection_name": self.collection_name,
178180
"ids": primary_keys,
181+
"ignore_unknown_fields": True,
179182
}
180183
resp_data = self._data_post(path, data)
181184
return self._parse_fetch_result(resp_data)
@@ -248,6 +251,7 @@ def search_by_vector(
248251
"output_fields": output_fields,
249252
"limit": limit,
250253
"offset": offset,
254+
"ignore_unknown_fields": True,
251255
}
252256
if sparse_vector:
253257
data["sparse_vector"] = sparse_vector
@@ -273,6 +277,7 @@ def search_by_id(
273277
"output_fields": output_fields,
274278
"limit": limit,
275279
"offset": offset,
280+
"ignore_unknown_fields": True,
276281
}
277282
resp_data = self._data_post(path, data)
278283
return self._parse_search_result(resp_data)
@@ -300,6 +305,7 @@ def search_by_multimodal(
300305
"output_fields": output_fields,
301306
"limit": limit,
302307
"offset": offset,
308+
"ignore_unknown_fields": True,
303309
}
304310
resp_data = self._data_post(path, data)
305311
return self._parse_search_result(resp_data)
@@ -321,6 +327,7 @@ def search_by_random(
321327
"output_fields": output_fields,
322328
"limit": limit,
323329
"offset": offset,
330+
"ignore_unknown_fields": True,
324331
}
325332
resp_data = self._data_post(path, data)
326333
return self._parse_search_result(resp_data)
@@ -346,6 +353,7 @@ def search_by_keywords(
346353
"output_fields": output_fields,
347354
"limit": limit,
348355
"offset": offset,
356+
"ignore_unknown_fields": True,
349357
}
350358
data = {k: v for k, v in data.items() if v is not None}
351359
resp_data = self._data_post(path, data)
@@ -372,6 +380,7 @@ def search_by_scalar(
372380
"output_fields": output_fields,
373381
"limit": limit,
374382
"offset": offset,
383+
"ignore_unknown_fields": True,
375384
}
376385
resp_data = self._data_post(path, data)
377386
return self._parse_search_result(resp_data)

‎openviking/storage/vectordb/collection/volcengine_api_key_collection.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,12 +238,18 @@ def update(
238238
def get_meta_data(self):
239239
from openviking.storage.collection_schemas import CollectionSchemas
240240

241+
schema = CollectionSchemas.context_collection(
242+
self.collection_name,
243+
int(self.meta_data.get("VectorDim") or self.meta_data.get("Dimension") or 0),
244+
)
241245
return {
242246
"ProjectName": self.project_name,
243247
"CollectionName": self.collection_name,
244248
"IndexName": self.index_name,
245249
"Description": "data-plane only backend",
246-
"Fields": CollectionSchemas.context_collection.get("Fields", []),
250+
"Fields": schema.get("Fields", []),
251+
"ScalarIndex": schema.get("ScalarIndex", []),
252+
"FullText": schema.get("FullText", []),
247253
}
248254

249255
def close(self):
@@ -297,6 +303,7 @@ def upsert_data(self, data_list: List[Dict[str, Any]], ttl: int = 0):
297303
**self._base_data_payload(),
298304
"data": data_list,
299305
"ttl": ttl,
306+
"ignore_unknown_fields": True,
300307
}
301308
return self._data_post(path, data)
302309

@@ -305,6 +312,7 @@ def update_data(self, data_list: List[Dict[str, Any]]):
305312
data = {
306313
**self._base_data_payload(),
307314
"data": data_list,
315+
"ignore_unknown_fields": True,
308316
}
309317
return self._data_post(path, data)
310318

@@ -313,6 +321,7 @@ def fetch_data(self, primary_keys: List[Any]) -> FetchDataInCollectionResult:
313321
data = {
314322
**self._base_data_payload(),
315323
"ids": primary_keys,
324+
"ignore_unknown_fields": True,
316325
}
317326
resp_data = self._data_post(path, data)
318327
return self._parse_fetch_result(resp_data)
@@ -354,6 +363,7 @@ def search_by_vector(
354363
"output_fields": output_fields,
355364
"limit": limit,
356365
"offset": offset,
366+
"ignore_unknown_fields": True,
357367
}
358368
if sparse_vector:
359369
data["sparse_vector"] = sparse_vector
@@ -379,6 +389,7 @@ def search_by_keywords(
379389
"output_fields": output_fields,
380390
"limit": limit,
381391
"offset": offset,
392+
"ignore_unknown_fields": True,
382393
}
383394
data = {k: v for k, v in data.items() if v is not None}
384395
resp_data = self._data_post(path, data)
@@ -401,6 +412,7 @@ def search_by_id(
401412
"output_fields": output_fields,
402413
"limit": limit,
403414
"offset": offset,
415+
"ignore_unknown_fields": True,
404416
}
405417
resp_data = self._data_post(path, data)
406418
return self._parse_search_result(resp_data)
@@ -426,6 +438,7 @@ def search_by_multimodal(
426438
"output_fields": output_fields,
427439
"limit": limit,
428440
"offset": offset,
441+
"ignore_unknown_fields": True,
429442
}
430443
resp_data = self._data_post(path, data)
431444
return self._parse_search_result(resp_data)
@@ -445,6 +458,7 @@ def search_by_random(
445458
"output_fields": output_fields,
446459
"limit": limit,
447460
"offset": offset,
461+
"ignore_unknown_fields": True,
448462
}
449463
resp_data = self._data_post(path, data)
450464
return self._parse_search_result(resp_data)
@@ -468,6 +482,7 @@ def search_by_scalar(
468482
"output_fields": output_fields,
469483
"limit": limit,
470484
"offset": offset,
485+
"ignore_unknown_fields": True,
471486
}
472487
resp_data = self._data_post(path, data)
473488
return self._parse_search_result(resp_data)

‎openviking/storage/vectordb/collection/volcengine_collection.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ def upsert_data(self, data_list: List[Dict[str, Any]], ttl: int = 0):
392392
"collection_name": self.collection_name,
393393
"data": data_list,
394394
"ttl": ttl,
395+
"ignore_unknown_fields": True,
395396
}
396397
return self._data_post(path, data)
397398

@@ -401,6 +402,7 @@ def update_data(self, data_list: List[Dict[str, Any]]):
401402
"project": self.project_name,
402403
"collection_name": self.collection_name,
403404
"data": data_list,
405+
"ignore_unknown_fields": True,
404406
}
405407
return self._data_post(path, data)
406408

@@ -410,6 +412,7 @@ def fetch_data(self, primary_keys: List[Any]) -> FetchDataInCollectionResult:
410412
"project": self.project_name,
411413
"collection_name": self.collection_name,
412414
"ids": primary_keys,
415+
"ignore_unknown_fields": True,
413416
}
414417
resp_data = self._data_post(path, data)
415418
# print(resp_data)
@@ -483,6 +486,7 @@ def search_by_vector(
483486
"output_fields": output_fields,
484487
"limit": limit,
485488
"offset": offset,
489+
"ignore_unknown_fields": True,
486490
}
487491
if sparse_vector:
488492
data["sparse_vector"] = sparse_vector
@@ -508,6 +512,7 @@ def search_by_id(
508512
"output_fields": output_fields,
509513
"limit": limit,
510514
"offset": offset,
515+
"ignore_unknown_fields": True,
511516
}
512517
resp_data = self._data_post(path, data)
513518
return self._parse_search_result(resp_data)
@@ -535,6 +540,7 @@ def search_by_multimodal(
535540
"output_fields": output_fields,
536541
"limit": limit,
537542
"offset": offset,
543+
"ignore_unknown_fields": True,
538544
}
539545
resp_data = self._data_post(path, data)
540546
return self._parse_search_result(resp_data)
@@ -556,6 +562,7 @@ def search_by_random(
556562
"output_fields": output_fields,
557563
"limit": limit,
558564
"offset": offset,
565+
"ignore_unknown_fields": True,
559566
}
560567
resp_data = self._data_post(path, data)
561568
return self._parse_search_result(resp_data)
@@ -581,6 +588,7 @@ def search_by_keywords(
581588
"output_fields": output_fields,
582589
"limit": limit,
583590
"offset": offset,
591+
"ignore_unknown_fields": True,
584592
}
585593
data = {k: v for k, v in data.items() if v is not None}
586594
resp_data = self._data_post(path, data)
@@ -607,6 +615,7 @@ def search_by_scalar(
607615
"output_fields": output_fields,
608616
"limit": limit,
609617
"offset": offset,
618+
"ignore_unknown_fields": True,
610619
}
611620
resp_data = self._data_post(path, data)
612621
return self._parse_search_result(resp_data)

0 commit comments

Comments
 (0)