[BUG]

[andyuan1000]

[root@localhost e5-base-v2]# docker logs bfcd5ec7df71
External vector store not configured. Using local vector store
Waiting for vector store to start
Marqo did not find an existing vector store. Setting up vector store...
Waiting for vector store to be available...
Waiting for vector store to be available...
Waiting for vector store to be available...
Vector store is available. Vector store setup complete
Starting Marqo throttling
Called Marqo throttling start command
Marqo throttling is now running
/usr/local/lib64/python3.9/site-packages/torchvision/transforms/_functional_video.py:6: UserWarning: The 'torchvision.transforms._functional_video' module is deprecated since 0.12 and will be retional' module instead.
warnings.warn(
/usr/local/lib64/python3.9/site-packages/torchvision/transforms/_transforms_video.py:25: UserWarning: The 'torchvision.transforms._transforms_video' module is deprecated since 0.12 and will be rdule instead.
warnings.warn(
INFO:ModelsForStartup:pre-loading ['hf/e5-base-v2', 'open_clip/ViT-B-32/laion2b_s34b_b79k'] onto devices=['cpu']
INFO:marqo.core.index_management.vespa_application_package:Bootstrapping the vector store to 2.16.1
INFO:marqo.tensor_search.index_meta_cache:Starting index cache refresh thread

###########################################################
###########################################################

STARTING DOWNLOAD OF MARQO ARTEFACTS################

###########################################################
###########################################################

INFO:CUDA device summary:Found devices [{'id': -1, 'name': ['cpu']}]
INFO:SetBestAvailableDevice:Best available device set to: cpu
INFO:marqo.tensor_search.index_meta_cache:Last index cache refresh at 1753789463.0874805
INFO:marqo.tensor_search.on_start_script:Video processing with GPU acceleration is disabled
WARNING:marqo.core.inference.embedding_models.hugging_face_model_properties:Could not infer pooling method from the model intfloat/e5-base-v2. Defaulting to mean pooling.
/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py:1142: FutureWarning: resume_download is deprecated and will be removed in version 1.0.0. Downloads always resume when pownload=True`.
warnings.warn(
ERROR:marqo.s2_inference.s2_inference:Error loading model hf/e5-base-v2 on device cpu with normalization=True.
Error message is Marqo encountered an error loading the Hugging Face model, modelProperties=dimensions=768 type='hf' name='intfloat/e5-base-v2' tokens=512 url=None model_location=None note=None ode=False. Please ensure that the model is a valid Hugging Face model and retry.
Original error message = We couldn't connect to 'https://huggingface.co' to load this file, couldn't find it in the cached files and it looks like intfloat/e5-base-v2 is not the path to a direc
Checkout your internet connection or see how to run the library in offline mode at 'https://huggingface.co/docs/transformers/installation#offline-mode'.
loading for: model_name=hf/e5-base-v2 and properties={'name': 'intfloat/e5-base-v2', 'dimensions': 768, 'tokens': 512, 'type': 'hf', 'model_size': 0.438, 'text_query_prefix': 'query: ', 'text_ch
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 95, in create_connection
raise err
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 85, in create_connection
sock.connect(sa)
OSError: [Errno 101] Network is unreachable

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 403, in _make_request
self._validate_conn(conn)
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1053, in _validate_conn
conn.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 363, in connect
self.sock = conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7fad05687880>: Failed to establish a new connection: [Errno 101] Network is unreachable

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 489, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 798, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /intfloat/e5-base-v2/resolve/main/config.json (Caused by NewConnectionError7880>: Failed to establish a new connection: [Errno 101] Network is unreachable'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1746, in _get_metadata_or_catch_error
metadata = get_hf_file_metadata(
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1666, in get_hf_file_metadata
r = _request_wrapper(
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 364, in _request_wrapper
response = _request_wrapper(
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 387, in _request_wrapper
response = get_session().request(method=method, url=url, **params)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/utils/_http.py", line 93, in send
return super().send(request, *args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 565, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: (MaxRetryError("HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /intfloat/e5-base-v2/resolve/main/config.json (Caused bybject at 0x7fad05687880>: Failed to establish a new connection: [Errno 101] Network is unreachable'))"), '(Request ID: 15005765-dcd7-44b6-998f-316eb029fae7)')

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/transformers/utils/hub.py", line 399, in cached_file
resolved_file = hf_hub_download(
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py", line 101, in inner_f
return f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1232, in hf_hub_download
return _hf_hub_download_to_cache_dir(
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1339, in _hf_hub_download_to_cache_dir
_raise_on_head_call_error(head_call_error, force_download, local_files_only)
File "/usr/local/lib/python3.9/site-packages/huggingface_hub/file_download.py", line 1857, in _raise_on_head_call_error
raise LocalEntryNotFoundError(
huggingface_hub.errors.LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/app/src/marqo/core/inference/embedding_models/hugging_face_model.py", line 125, in _load_from_hugging_face_repo
model = AutoModel.from_pretrained(
File "/usr/local/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py", line 523, in from_pretrained
config, kwargs = AutoConfig.from_pretrained(
File "/usr/local/lib/python3.9/site-packages/transformers/models/auto/configuration_auto.py", line 934, in from_pretrained
config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
File "/usr/local/lib/python3.9/site-packages/transformers/configuration_utils.py", line 632, in get_config_dict
config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
File "/usr/local/lib/python3.9/site-packages/transformers/configuration_utils.py", line 689, in _get_config_dict
resolved_config_file = cached_file(
File "/usr/local/lib/python3.9/site-packages/transformers/utils/hub.py", line 442, in cached_file
raise EnvironmentError(
OSError: We couldn't connect to 'https://huggingface.co' to load this file, couldn't find it in the cached files and it looks like intfloat/e5-base-v2 is not the path to a directory containing a
Checkout your internet connection or see how to run the library in offline mode at 'https://huggingface.co/docs/transformers/installation#offline-mode'.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/app/src/marqo/s2_inference/s2_inference.py", line 306, in _update_available_models
AvailableModelsKey.model: _load_model(
File "/app/src/marqo/s2_inference/s2_inference.py", line 561, in _load_model
model.load()
File "/app/src/marqo/core/inference/embedding_models/abstract_embedding_model.py", line 34, in load
self._load_necessary_components()
File "/app/src/marqo/core/inference/embedding_models/hugging_face_model.py", line 80, in _load_necessary_components
self._model, self._tokenizer = self._load_from_hugging_face_repo()
File "/app/src/marqo/core/inference/embedding_models/hugging_face_model.py", line 132, in _load_from_hugging_face_repo
raise InvalidModelPropertiesError(
marqo.s2_inference.errors.InvalidModelPropertiesError: Marqo encountered an error loading the Hugging Face model, modelProperties=dimensions=768 type='hf' name='intfloat/e5-base-v2' tokens=512 uingMethod.Mean: 'mean'> trust_remote_code=False. Please ensure that the model is a valid Hugging Face model and retry.
Original error message = We couldn't connect to 'https://huggingface.co' to load this file, couldn't find it in the cached files and it looks like intfloat/e5-base-v2 is not the path to a direc
Checkout your internet connection or see how to run the library in offline mode at 'https://huggingface.co/docs/transformers/installation#offline-mode'.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/usr/local/bin/uvicorn", line 8, in
sys.exit(main())
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1161, in call
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1082, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1443, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 788, in invoke
return __callback(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/uvicorn/main.py", line 412, in main
run(
File "/usr/local/lib/python3.9/site-packages/uvicorn/main.py", line 579, in run
server.run()
File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 65, in run
return asyncio.run(self.serve(sockets=sockets))
File "/usr/lib64/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "uvloop/loop.pyx", line 1517, in uvloop.loop.Loop.run_until_complete
File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 69, in serve
await self._serve(sockets)
File "/usr/local/lib/python3.9/site-packages/uvicorn/server.py", line 76, in _serve
config.load()
File "/usr/local/lib/python3.9/site-packages/uvicorn/config.py", line 434, in load
self.loaded_app = import_from_string(self.app)
File "/usr/local/lib/python3.9/site-packages/uvicorn/importer.py", line 19, in import_from_string
module = importlib.import_module(module_str)
File "/usr/lib64/python3.9/importlib/init.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "", line 1030, in _gcd_import
File "", line 1007, in _find_and_load
File "", line 986, in _find_and_load_unlocked
File "", line 680, in _load_unlocked
File "", line 850, in exec_module
File "", line 228, in _call_with_frames_removed
File "/app/src/marqo/tensor_search/api.py", line 74, in
on_start(_config)
File "/app/src/marqo/tensor_search/on_start_script.py", line 50, in on_start
thing_to_start.run()
File "/app/src/marqo/tensor_search/on_start_script.py", line 188, in run
_ = _preload_model(model=model, content=test_string, device=device)
File "/app/src/marqo/tensor_search/on_start_script.py", line 366, in _preload_model
_ = vectorise(
File "/app/src/marqo/s2_inference/s2_inference.py", line 59, in vectorise
_update_available_models(
File "/app/src/marqo/s2_inference/s2_inference.py", line 324, in _update_available_models
raise ModelLoadError(
marqo.s2_inference.errors.ModelLoadError: Unable to load model=hf/e5-base-v2 on device=cpu with normalization=True. If you are trying to load a custom model, please check that model_properties={: 512, 'type': 'hf', 'model_size': 0.438, 'text_query_prefix': 'query: ', 'text_chunk_prefix': 'passage: ', 'notes': ''} is correct and Marqo has access to the weights file.
[root@localhost e5-base-v2]#
Socket error Event: 32 Error: 10053.