[BUG] Error with vllm #93
Description
Python Version
python3 -vv
import _frozen_importlib # frozen
import _imp # builtin
import '_thread' # <class '_frozen_importlib.BuiltinImporter'>
import '_warnings' # <class '_frozen_importlib.BuiltinImporter'>
import '_weakref' # <class '_frozen_importlib.BuiltinImporter'>
import '_io' # <class '_frozen_importlib.BuiltinImporter'>
import 'marshal' # <class '_frozen_importlib.BuiltinImporter'>
import 'posix' # <class '_frozen_importlib.BuiltinImporter'>
import '_frozen_importlib_external' # <class '_frozen_importlib.FrozenImporter'>
# installing zipimport hook
import 'time' # <class '_frozen_importlib.BuiltinImporter'>
import 'zipimport' # <class '_frozen_importlib.FrozenImporter'>
# installed zipimport hook
# /usr/lib/python3.10/encodings/__pycache__/__init__.cpython-310.pyc matches /usr/lib/python3.10/encodings/__init__.py
# code object from '/usr/lib/python3.10/encodings/__pycache__/__init__.cpython-310.pyc'
# trying /usr/lib/python3.10/codecs.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/codecs.abi3.so
# trying /usr/lib/python3.10/codecs.so
# trying /usr/lib/python3.10/codecs.py
# /usr/lib/python3.10/__pycache__/codecs.cpython-310.pyc matches /usr/lib/python3.10/codecs.py
# code object from '/usr/lib/python3.10/__pycache__/codecs.cpython-310.pyc'
import '_codecs' # <class '_frozen_importlib.BuiltinImporter'>
import 'codecs' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4bc3f610>
# trying /usr/lib/python3.10/encodings/aliases.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/encodings/aliases.abi3.so
# trying /usr/lib/python3.10/encodings/aliases.so
# trying /usr/lib/python3.10/encodings/aliases.py
# /usr/lib/python3.10/encodings/__pycache__/aliases.cpython-310.pyc matches /usr/lib/python3.10/encodings/aliases.py
# code object from '/usr/lib/python3.10/encodings/__pycache__/aliases.cpython-310.pyc'
import 'encodings.aliases' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba34bb0>
import 'encodings' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4bc3f3a0>
# trying /usr/lib/python3.10/encodings/utf_8.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/encodings/utf_8.abi3.so
# trying /usr/lib/python3.10/encodings/utf_8.so
# trying /usr/lib/python3.10/encodings/utf_8.py
# /usr/lib/python3.10/encodings/__pycache__/utf_8.cpython-310.pyc matches /usr/lib/python3.10/encodings/utf_8.py
# code object from '/usr/lib/python3.10/encodings/__pycache__/utf_8.cpython-310.pyc'
import 'encodings.utf_8' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4bc3f340>
import '_signal' # <class '_frozen_importlib.BuiltinImporter'>
# trying /usr/lib/python3.10/io.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/io.abi3.so
# trying /usr/lib/python3.10/io.so
# trying /usr/lib/python3.10/io.py
# /usr/lib/python3.10/__pycache__/io.cpython-310.pyc matches /usr/lib/python3.10/io.py
# code object from '/usr/lib/python3.10/__pycache__/io.cpython-310.pyc'
# trying /usr/lib/python3.10/abc.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/abc.abi3.so
# trying /usr/lib/python3.10/abc.so
# trying /usr/lib/python3.10/abc.py
# /usr/lib/python3.10/__pycache__/abc.cpython-310.pyc matches /usr/lib/python3.10/abc.py
# code object from '/usr/lib/python3.10/__pycache__/abc.cpython-310.pyc'
import '_abc' # <class '_frozen_importlib.BuiltinImporter'>
import 'abc' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba34f40>
import 'io' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba34d30>
# trying /usr/lib/python3.10/site.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/site.abi3.so
# trying /usr/lib/python3.10/site.so
# trying /usr/lib/python3.10/site.py
# /usr/lib/python3.10/__pycache__/site.cpython-310.pyc matches /usr/lib/python3.10/site.py
# code object from '/usr/lib/python3.10/__pycache__/site.cpython-310.pyc'
# trying /usr/lib/python3.10/os.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/os.abi3.so
# trying /usr/lib/python3.10/os.so
# trying /usr/lib/python3.10/os.py
# /usr/lib/python3.10/__pycache__/os.cpython-310.pyc matches /usr/lib/python3.10/os.py
# code object from '/usr/lib/python3.10/__pycache__/os.cpython-310.pyc'
# trying /usr/lib/python3.10/stat.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/stat.abi3.so
# trying /usr/lib/python3.10/stat.so
# trying /usr/lib/python3.10/stat.py
# /usr/lib/python3.10/__pycache__/stat.cpython-310.pyc matches /usr/lib/python3.10/stat.py
# code object from '/usr/lib/python3.10/__pycache__/stat.cpython-310.pyc'
import '_stat' # <class '_frozen_importlib.BuiltinImporter'>
import 'stat' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba37160>
# trying /usr/lib/python3.10/_collections_abc.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/_collections_abc.abi3.so
# trying /usr/lib/python3.10/_collections_abc.so
# trying /usr/lib/python3.10/_collections_abc.py
# /usr/lib/python3.10/__pycache__/_collections_abc.cpython-310.pyc matches /usr/lib/python3.10/_collections_abc.py
# code object from '/usr/lib/python3.10/__pycache__/_collections_abc.cpython-310.pyc'
import '_collections_abc' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba37490>
# trying /usr/lib/python3.10/posixpath.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/posixpath.abi3.so
# trying /usr/lib/python3.10/posixpath.so
# trying /usr/lib/python3.10/posixpath.py
# /usr/lib/python3.10/__pycache__/posixpath.cpython-310.pyc matches /usr/lib/python3.10/posixpath.py
# code object from '/usr/lib/python3.10/__pycache__/posixpath.cpython-310.pyc'
# trying /usr/lib/python3.10/genericpath.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/genericpath.abi3.so
# trying /usr/lib/python3.10/genericpath.so
# trying /usr/lib/python3.10/genericpath.py
# /usr/lib/python3.10/__pycache__/genericpath.cpython-310.pyc matches /usr/lib/python3.10/genericpath.py
# code object from '/usr/lib/python3.10/__pycache__/genericpath.cpython-310.pyc'
import 'genericpath' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba75cf0>
import 'posixpath' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba37640>
import 'os' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba35d80>
# trying /usr/lib/python3.10/_sitebuiltins.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/_sitebuiltins.abi3.so
# trying /usr/lib/python3.10/_sitebuiltins.so
# trying /usr/lib/python3.10/_sitebuiltins.py
# /usr/lib/python3.10/__pycache__/_sitebuiltins.cpython-310.pyc matches /usr/lib/python3.10/_sitebuiltins.py
# code object from '/usr/lib/python3.10/__pycache__/_sitebuiltins.cpython-310.pyc'
import '_sitebuiltins' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba36e30>
Processing user site-packages
Adding directory: '/home/barbatus/.local/lib/python3.10/site-packages'
Processing global site-packages
Adding directory: '/usr/local/lib/python3.10/dist-packages'
Adding directory: '/usr/lib/python3/dist-packages'
Processing .pth file: '/usr/lib/python3/dist-packages/zope.interface-5.4.0-nspkg.pth'
# trying /usr/lib/python3.10/types.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/types.abi3.so
# trying /usr/lib/python3.10/types.so
# trying /usr/lib/python3.10/types.py
# /usr/lib/python3.10/__pycache__/types.cpython-310.pyc matches /usr/lib/python3.10/types.py
# code object from '/usr/lib/python3.10/__pycache__/types.cpython-310.pyc'
import 'types' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba77d30>
# /usr/lib/python3.10/importlib/__pycache__/__init__.cpython-310.pyc matches /usr/lib/python3.10/importlib/__init__.py
# code object from '/usr/lib/python3.10/importlib/__pycache__/__init__.cpython-310.pyc'
# trying /usr/lib/python3.10/warnings.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/warnings.abi3.so
# trying /usr/lib/python3.10/warnings.so
# trying /usr/lib/python3.10/warnings.py
# /usr/lib/python3.10/__pycache__/warnings.cpython-310.pyc matches /usr/lib/python3.10/warnings.py
# code object from '/usr/lib/python3.10/__pycache__/warnings.cpython-310.pyc'
import 'warnings' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba77370>
import 'importlib' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba776a0>
# trying /usr/lib/python3.10/importlib/util.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/importlib/util.abi3.so
# trying /usr/lib/python3.10/importlib/util.so
# trying /usr/lib/python3.10/importlib/util.py
# /usr/lib/python3.10/importlib/__pycache__/util.cpython-310.pyc matches /usr/lib/python3.10/importlib/util.py
# code object from '/usr/lib/python3.10/importlib/__pycache__/util.cpython-310.pyc'
# trying /usr/lib/python3.10/importlib/_abc.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/importlib/_abc.abi3.so
# trying /usr/lib/python3.10/importlib/_abc.so
# trying /usr/lib/python3.10/importlib/_abc.py
# /usr/lib/python3.10/importlib/__pycache__/_abc.cpython-310.pyc matches /usr/lib/python3.10/importlib/_abc.py
# code object from '/usr/lib/python3.10/importlib/__pycache__/_abc.cpython-310.pyc'
import 'importlib._abc' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba76260>
# trying /usr/lib/python3.10/contextlib.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/contextlib.abi3.so
# trying /usr/lib/python3.10/contextlib.so
# trying /usr/lib/python3.10/contextlib.py
# /usr/lib/python3.10/__pycache__/contextlib.cpython-310.pyc matches /usr/lib/python3.10/contextlib.py
# code object from '/usr/lib/python3.10/__pycache__/contextlib.cpython-310.pyc'
# /usr/lib/python3.10/collections/__pycache__/__init__.cpython-310.pyc matches /usr/lib/python3.10/collections/__init__.py
# code object from '/usr/lib/python3.10/collections/__pycache__/__init__.cpython-310.pyc'
import 'itertools' # <class '_frozen_importlib.BuiltinImporter'>
# trying /usr/lib/python3.10/keyword.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/keyword.abi3.so
# trying /usr/lib/python3.10/keyword.so
# trying /usr/lib/python3.10/keyword.py
# /usr/lib/python3.10/__pycache__/keyword.cpython-310.pyc matches /usr/lib/python3.10/keyword.py
# code object from '/usr/lib/python3.10/__pycache__/keyword.cpython-310.pyc'
import 'keyword' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa3d30>
# trying /usr/lib/python3.10/operator.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/operator.abi3.so
# trying /usr/lib/python3.10/operator.so
# trying /usr/lib/python3.10/operator.py
# /usr/lib/python3.10/__pycache__/operator.cpython-310.pyc matches /usr/lib/python3.10/operator.py
# code object from '/usr/lib/python3.10/__pycache__/operator.cpython-310.pyc'
import '_operator' # <class '_frozen_importlib.BuiltinImporter'>
import 'operator' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa3dc0>
# trying /usr/lib/python3.10/reprlib.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/reprlib.abi3.so
# trying /usr/lib/python3.10/reprlib.so
# trying /usr/lib/python3.10/reprlib.py
# /usr/lib/python3.10/__pycache__/reprlib.cpython-310.pyc matches /usr/lib/python3.10/reprlib.py
# code object from '/usr/lib/python3.10/__pycache__/reprlib.cpython-310.pyc'
import 'reprlib' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa3fd0>
import '_collections' # <class '_frozen_importlib.BuiltinImporter'>
import 'collections' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa0970>
# trying /usr/lib/python3.10/functools.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/functools.abi3.so
# trying /usr/lib/python3.10/functools.so
# trying /usr/lib/python3.10/functools.py
# /usr/lib/python3.10/__pycache__/functools.cpython-310.pyc matches /usr/lib/python3.10/functools.py
# code object from '/usr/lib/python3.10/__pycache__/functools.cpython-310.pyc'
import '_functools' # <class '_frozen_importlib.BuiltinImporter'>
import 'functools' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4bae4d90>
import 'contextlib' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa1b70>
import 'importlib.util' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba76d70>
# trying /usr/lib/python3.10/importlib/machinery.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/importlib/machinery.abi3.so
# trying /usr/lib/python3.10/importlib/machinery.so
# trying /usr/lib/python3.10/importlib/machinery.py
# /usr/lib/python3.10/importlib/__pycache__/machinery.cpython-310.pyc matches /usr/lib/python3.10/importlib/machinery.py
# code object from '/usr/lib/python3.10/importlib/__pycache__/machinery.cpython-310.pyc'
import 'importlib.machinery' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa1330>
# trying /usr/lib/python3.10/sitecustomize.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/sitecustomize.abi3.so
# trying /usr/lib/python3.10/sitecustomize.so
# trying /usr/lib/python3.10/sitecustomize.py
# /usr/lib/python3.10/__pycache__/sitecustomize.cpython-310.pyc matches /usr/lib/python3.10/sitecustomize.py
# code object from '/usr/lib/python3.10/__pycache__/sitecustomize.cpython-310.pyc'
# trying /usr/lib/python3.10/apport_python_hook.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/apport_python_hook.abi3.so
# trying /usr/lib/python3.10/apport_python_hook.so
# trying /usr/lib/python3.10/apport_python_hook.py
# trying /usr/lib/python3.10/apport_python_hook.pyc
# trying /usr/lib/python3.10/lib-dynload/apport_python_hook.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/lib-dynload/apport_python_hook.abi3.so
# trying /usr/lib/python3.10/lib-dynload/apport_python_hook.so
# trying /usr/lib/python3.10/lib-dynload/apport_python_hook.py
# trying /usr/lib/python3.10/lib-dynload/apport_python_hook.pyc
# trying /home/barbatus/.local/lib/python3.10/site-packages/apport_python_hook.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/.local/lib/python3.10/site-packages/apport_python_hook.abi3.so
# trying /home/barbatus/.local/lib/python3.10/site-packages/apport_python_hook.so
# trying /home/barbatus/.local/lib/python3.10/site-packages/apport_python_hook.py
# trying /home/barbatus/.local/lib/python3.10/site-packages/apport_python_hook.pyc
# trying /usr/local/lib/python3.10/dist-packages/apport_python_hook.cpython-310-x86_64-linux-gnu.so
# trying /usr/local/lib/python3.10/dist-packages/apport_python_hook.abi3.so
# trying /usr/local/lib/python3.10/dist-packages/apport_python_hook.so
# trying /usr/local/lib/python3.10/dist-packages/apport_python_hook.py
# trying /usr/local/lib/python3.10/dist-packages/apport_python_hook.pyc
# trying /usr/lib/python3/dist-packages/apport_python_hook.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3/dist-packages/apport_python_hook.abi3.so
# trying /usr/lib/python3/dist-packages/apport_python_hook.so
# trying /usr/lib/python3/dist-packages/apport_python_hook.py
# /usr/lib/python3/dist-packages/__pycache__/apport_python_hook.cpython-310.pyc matches /usr/lib/python3/dist-packages/apport_python_hook.py
# code object from '/usr/lib/python3/dist-packages/__pycache__/apport_python_hook.cpython-310.pyc'
import 'apport_python_hook' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba77f10>
import 'sitecustomize' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba77220>
# trying /usr/lib/python3.10/usercustomize.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/usercustomize.abi3.so
# trying /usr/lib/python3.10/usercustomize.so
# trying /usr/lib/python3.10/usercustomize.py
# trying /usr/lib/python3.10/usercustomize.pyc
# trying /usr/lib/python3.10/lib-dynload/usercustomize.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/lib-dynload/usercustomize.abi3.so
# trying /usr/lib/python3.10/lib-dynload/usercustomize.so
# trying /usr/lib/python3.10/lib-dynload/usercustomize.py
# trying /usr/lib/python3.10/lib-dynload/usercustomize.pyc
# trying /home/barbatus/.local/lib/python3.10/site-packages/usercustomize.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/.local/lib/python3.10/site-packages/usercustomize.abi3.so
# trying /home/barbatus/.local/lib/python3.10/site-packages/usercustomize.so
# trying /home/barbatus/.local/lib/python3.10/site-packages/usercustomize.py
# trying /home/barbatus/.local/lib/python3.10/site-packages/usercustomize.pyc
# trying /usr/local/lib/python3.10/dist-packages/usercustomize.cpython-310-x86_64-linux-gnu.so
# trying /usr/local/lib/python3.10/dist-packages/usercustomize.abi3.so
# trying /usr/local/lib/python3.10/dist-packages/usercustomize.so
# trying /usr/local/lib/python3.10/dist-packages/usercustomize.py
# trying /usr/local/lib/python3.10/dist-packages/usercustomize.pyc
# trying /usr/lib/python3/dist-packages/usercustomize.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3/dist-packages/usercustomize.abi3.so
# trying /usr/lib/python3/dist-packages/usercustomize.so
# trying /usr/lib/python3/dist-packages/usercustomize.py
# trying /usr/lib/python3/dist-packages/usercustomize.pyc
import 'site' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4ba35720>
Python 3.10.12 (main, Jul 29 2024, 16:56:48) [GCC 11.4.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
# trying /home/barbatus/finetuning/models/7B/readline.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/readline.abi3.so
# trying /home/barbatus/finetuning/models/7B/readline.so
# trying /home/barbatus/finetuning/models/7B/readline.py
# trying /home/barbatus/finetuning/models/7B/readline.pyc
# trying /usr/lib/python3.10/readline.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/readline.abi3.so
# trying /usr/lib/python3.10/readline.so
# trying /usr/lib/python3.10/readline.py
# trying /usr/lib/python3.10/readline.pyc
# trying /usr/lib/python3.10/lib-dynload/readline.cpython-310-x86_64-linux-gnu.so
# extension module 'readline' loaded from '/usr/lib/python3.10/lib-dynload/readline.cpython-310-x86_64-linux-gnu.so'
# extension module 'readline' executed from '/usr/lib/python3.10/lib-dynload/readline.cpython-310-x86_64-linux-gnu.so'
import 'readline' # <_frozen_importlib_external.ExtensionFileLoader object at 0x7f5f4ba771f0>
import 'atexit' # <class '_frozen_importlib.BuiltinImporter'>
# trying /home/barbatus/finetuning/models/7B/rlcompleter.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/rlcompleter.abi3.so
# trying /home/barbatus/finetuning/models/7B/rlcompleter.so
# trying /home/barbatus/finetuning/models/7B/rlcompleter.py
# trying /home/barbatus/finetuning/models/7B/rlcompleter.pyc
# trying /usr/lib/python3.10/rlcompleter.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/rlcompleter.abi3.so
# trying /usr/lib/python3.10/rlcompleter.so
# trying /usr/lib/python3.10/rlcompleter.py
# /usr/lib/python3.10/__pycache__/rlcompleter.cpython-310.pyc matches /usr/lib/python3.10/rlcompleter.py
# code object from '/usr/lib/python3.10/__pycache__/rlcompleter.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/inspect.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/inspect.abi3.so
# trying /home/barbatus/finetuning/models/7B/inspect.so
# trying /home/barbatus/finetuning/models/7B/inspect.py
# trying /home/barbatus/finetuning/models/7B/inspect.pyc
# trying /usr/lib/python3.10/inspect.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/inspect.abi3.so
# trying /usr/lib/python3.10/inspect.so
# trying /usr/lib/python3.10/inspect.py
# /usr/lib/python3.10/__pycache__/inspect.cpython-310.pyc matches /usr/lib/python3.10/inspect.py
# code object from '/usr/lib/python3.10/__pycache__/inspect.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/ast.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/ast.abi3.so
# trying /home/barbatus/finetuning/models/7B/ast.so
# trying /home/barbatus/finetuning/models/7B/ast.py
# trying /home/barbatus/finetuning/models/7B/ast.pyc
# trying /usr/lib/python3.10/ast.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/ast.abi3.so
# trying /usr/lib/python3.10/ast.so
# trying /usr/lib/python3.10/ast.py
# /usr/lib/python3.10/__pycache__/ast.cpython-310.pyc matches /usr/lib/python3.10/ast.py
# code object from '/usr/lib/python3.10/__pycache__/ast.cpython-310.pyc'
import '_ast' # <class '_frozen_importlib.BuiltinImporter'>
# trying /home/barbatus/finetuning/models/7B/enum.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/enum.abi3.so
# trying /home/barbatus/finetuning/models/7B/enum.so
# trying /home/barbatus/finetuning/models/7B/enum.py
# trying /home/barbatus/finetuning/models/7B/enum.pyc
# trying /usr/lib/python3.10/enum.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/enum.abi3.so
# trying /usr/lib/python3.10/enum.so
# trying /usr/lib/python3.10/enum.py
# /usr/lib/python3.10/__pycache__/enum.cpython-310.pyc matches /usr/lib/python3.10/enum.py
# code object from '/usr/lib/python3.10/__pycache__/enum.cpython-310.pyc'
import 'enum' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b9238e0>
import 'ast' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4bae77f0>
# trying /home/barbatus/finetuning/models/7B/dis.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/dis.abi3.so
# trying /home/barbatus/finetuning/models/7B/dis.so
# trying /home/barbatus/finetuning/models/7B/dis.py
# trying /home/barbatus/finetuning/models/7B/dis.pyc
# trying /usr/lib/python3.10/dis.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/dis.abi3.so
# trying /usr/lib/python3.10/dis.so
# trying /usr/lib/python3.10/dis.py
# /usr/lib/python3.10/__pycache__/dis.cpython-310.pyc matches /usr/lib/python3.10/dis.py
# code object from '/usr/lib/python3.10/__pycache__/dis.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/opcode.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/opcode.abi3.so
# trying /home/barbatus/finetuning/models/7B/opcode.so
# trying /home/barbatus/finetuning/models/7B/opcode.py
# trying /home/barbatus/finetuning/models/7B/opcode.pyc
# trying /usr/lib/python3.10/opcode.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/opcode.abi3.so
# trying /usr/lib/python3.10/opcode.so
# trying /usr/lib/python3.10/opcode.py
# /usr/lib/python3.10/__pycache__/opcode.cpython-310.pyc matches /usr/lib/python3.10/opcode.py
# code object from '/usr/lib/python3.10/__pycache__/opcode.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/_opcode.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/_opcode.abi3.so
# trying /home/barbatus/finetuning/models/7B/_opcode.so
# trying /home/barbatus/finetuning/models/7B/_opcode.py
# trying /home/barbatus/finetuning/models/7B/_opcode.pyc
# trying /usr/lib/python3.10/_opcode.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/_opcode.abi3.so
# trying /usr/lib/python3.10/_opcode.so
# trying /usr/lib/python3.10/_opcode.py
# trying /usr/lib/python3.10/_opcode.pyc
# trying /usr/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so
# extension module '_opcode' loaded from '/usr/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so'
# extension module '_opcode' executed from '/usr/lib/python3.10/lib-dynload/_opcode.cpython-310-x86_64-linux-gnu.so'
import '_opcode' # <_frozen_importlib_external.ExtensionFileLoader object at 0x7f5f4b961f00>
import 'opcode' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b961c90>
import 'dis' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4bae7f70>
# trying /usr/lib/python3.10/collections/abc.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/collections/abc.abi3.so
# trying /usr/lib/python3.10/collections/abc.so
# trying /usr/lib/python3.10/collections/abc.py
# /usr/lib/python3.10/collections/__pycache__/abc.cpython-310.pyc matches /usr/lib/python3.10/collections/abc.py
# code object from '/usr/lib/python3.10/collections/__pycache__/abc.cpython-310.pyc'
import 'collections.abc' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b962dd0>
# trying /home/barbatus/finetuning/models/7B/linecache.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/linecache.abi3.so
# trying /home/barbatus/finetuning/models/7B/linecache.so
# trying /home/barbatus/finetuning/models/7B/linecache.py
# trying /home/barbatus/finetuning/models/7B/linecache.pyc
# trying /usr/lib/python3.10/linecache.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/linecache.abi3.so
# trying /usr/lib/python3.10/linecache.so
# trying /usr/lib/python3.10/linecache.py
# /usr/lib/python3.10/__pycache__/linecache.cpython-310.pyc matches /usr/lib/python3.10/linecache.py
# code object from '/usr/lib/python3.10/__pycache__/linecache.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/tokenize.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/tokenize.abi3.so
# trying /home/barbatus/finetuning/models/7B/tokenize.so
# trying /home/barbatus/finetuning/models/7B/tokenize.py
# trying /home/barbatus/finetuning/models/7B/tokenize.pyc
# trying /usr/lib/python3.10/tokenize.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/tokenize.abi3.so
# trying /usr/lib/python3.10/tokenize.so
# trying /usr/lib/python3.10/tokenize.py
# /usr/lib/python3.10/__pycache__/tokenize.cpython-310.pyc matches /usr/lib/python3.10/tokenize.py
# code object from '/usr/lib/python3.10/__pycache__/tokenize.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/re.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/re.abi3.so
# trying /home/barbatus/finetuning/models/7B/re.so
# trying /home/barbatus/finetuning/models/7B/re.py
# trying /home/barbatus/finetuning/models/7B/re.pyc
# trying /usr/lib/python3.10/re.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/re.abi3.so
# trying /usr/lib/python3.10/re.so
# trying /usr/lib/python3.10/re.py
# /usr/lib/python3.10/__pycache__/re.cpython-310.pyc matches /usr/lib/python3.10/re.py
# code object from '/usr/lib/python3.10/__pycache__/re.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/sre_compile.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/sre_compile.abi3.so
# trying /home/barbatus/finetuning/models/7B/sre_compile.so
# trying /home/barbatus/finetuning/models/7B/sre_compile.py
# trying /home/barbatus/finetuning/models/7B/sre_compile.pyc
# trying /usr/lib/python3.10/sre_compile.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/sre_compile.abi3.so
# trying /usr/lib/python3.10/sre_compile.so
# trying /usr/lib/python3.10/sre_compile.py
# /usr/lib/python3.10/__pycache__/sre_compile.cpython-310.pyc matches /usr/lib/python3.10/sre_compile.py
# code object from '/usr/lib/python3.10/__pycache__/sre_compile.cpython-310.pyc'
import '_sre' # <class '_frozen_importlib.BuiltinImporter'>
# trying /home/barbatus/finetuning/models/7B/sre_parse.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/sre_parse.abi3.so
# trying /home/barbatus/finetuning/models/7B/sre_parse.so
# trying /home/barbatus/finetuning/models/7B/sre_parse.py
# trying /home/barbatus/finetuning/models/7B/sre_parse.pyc
# trying /usr/lib/python3.10/sre_parse.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/sre_parse.abi3.so
# trying /usr/lib/python3.10/sre_parse.so
# trying /usr/lib/python3.10/sre_parse.py
# /usr/lib/python3.10/__pycache__/sre_parse.cpython-310.pyc matches /usr/lib/python3.10/sre_parse.py
# code object from '/usr/lib/python3.10/__pycache__/sre_parse.cpython-310.pyc'
# trying /home/barbatus/finetuning/models/7B/sre_constants.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/sre_constants.abi3.so
# trying /home/barbatus/finetuning/models/7B/sre_constants.so
# trying /home/barbatus/finetuning/models/7B/sre_constants.py
# trying /home/barbatus/finetuning/models/7B/sre_constants.pyc
# trying /usr/lib/python3.10/sre_constants.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/sre_constants.abi3.so
# trying /usr/lib/python3.10/sre_constants.so
# trying /usr/lib/python3.10/sre_constants.py
# /usr/lib/python3.10/__pycache__/sre_constants.cpython-310.pyc matches /usr/lib/python3.10/sre_constants.py
# code object from '/usr/lib/python3.10/__pycache__/sre_constants.cpython-310.pyc'
import 'sre_constants' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b9a8730>
import 'sre_parse' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b963ee0>
import 'sre_compile' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b9635e0>
import '_locale' # <class '_frozen_importlib.BuiltinImporter'>
# trying /home/barbatus/finetuning/models/7B/copyreg.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/copyreg.abi3.so
# trying /home/barbatus/finetuning/models/7B/copyreg.so
# trying /home/barbatus/finetuning/models/7B/copyreg.py
# trying /home/barbatus/finetuning/models/7B/copyreg.pyc
# trying /usr/lib/python3.10/copyreg.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/copyreg.abi3.so
# trying /usr/lib/python3.10/copyreg.so
# trying /usr/lib/python3.10/copyreg.py
# /usr/lib/python3.10/__pycache__/copyreg.cpython-310.pyc matches /usr/lib/python3.10/copyreg.py
# code object from '/usr/lib/python3.10/__pycache__/copyreg.cpython-310.pyc'
import 'copyreg' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b9aa200>
import 're' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b961e40>
# trying /home/barbatus/finetuning/models/7B/token.cpython-310-x86_64-linux-gnu.so
# trying /home/barbatus/finetuning/models/7B/token.abi3.so
# trying /home/barbatus/finetuning/models/7B/token.so
# trying /home/barbatus/finetuning/models/7B/token.py
# trying /home/barbatus/finetuning/models/7B/token.pyc
# trying /usr/lib/python3.10/token.cpython-310-x86_64-linux-gnu.so
# trying /usr/lib/python3.10/token.abi3.so
# trying /usr/lib/python3.10/token.so
# trying /usr/lib/python3.10/token.py
# /usr/lib/python3.10/__pycache__/token.cpython-310.pyc matches /usr/lib/python3.10/token.py
# code object from '/usr/lib/python3.10/__pycache__/token.cpython-310.pyc'
import 'token' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b9aa470>
import 'tokenize' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b962a40>
import 'linecache' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4b962c20>
import 'inspect' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa0400>
import 'rlcompleter' # <_frozen_importlib_external.SourceFileLoader object at 0x7f5f4baa01f0>Pip Freeze
absl-py==2.1.0
aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
async-timeout==4.0.3
attrs==24.2.0
Automat==20.2.0
Babel==2.8.0
bcrypt==3.2.0
bitsandbytes==0.43.3
blinker==1.4
cachetools==5.5.0
certifi==2020.6.20
chardet==4.0.0
charset-normalizer==3.3.2
click==8.0.3
cloud-init==24.1.3
cloudpickle==3.0.0
cmake==3.30.2
colorama==0.4.4
command-not-found==0.3
configobj==5.0.6
constantly==15.1.0
cryptography==3.4.8
datasets==2.21.0
dbus-python==1.2.18
dill==0.3.8
diskcache==5.6.3
distro==1.7.0
distro-info==1.1+ubuntu0.2
docker-pycreds==0.4.0
docstring_parser==0.16
exceptiongroup==1.2.2
fastapi==0.112.1
filelock==3.15.4
fire==0.6.0
frozenlist==1.4.1
fsspec==2024.6.1
gitdb==4.0.11
GitPython==3.1.43
grpcio==1.65.5
h11==0.14.0
httpcore==1.0.5
httplib2==0.20.2
httptools==0.6.1
httpx==0.27.0
huggingface-hub==0.24.6
hyperlink==21.0.0
idna==3.3
importlib-metadata==4.6.4
incremental==21.3.0
interegular==0.3.3
jeepney==0.7.1
Jinja2==3.0.3
jiter==0.5.0
jsonpatch==1.32
jsonpointer==2.0
jsonschema==4.23.0
jsonschema-specifications==2023.12.1
keyring==23.5.0
lark==1.2.2
launchpadlib==1.10.16
lazr.restfulclient==0.14.4
lazr.uri==1.0.6
llvmlite==0.43.0
lm-format-enforcer==0.10.3
Markdown==3.7
MarkupSafe==2.1.5
mistral_common==1.3.4
mistral_inference==1.3.1
more-itertools==8.10.0
mpmath==1.3.0
msgpack==1.0.8
multidict==6.0.5
multiprocess==0.70.16
nest-asyncio==1.6.0
netifaces==0.11.0
networkx==3.3
ninja==1.11.1.1
numba==0.60.0
numpy==1.26.4
nvidia-cublas-cu12==12.1.3.1
nvidia-cuda-cupti-cu12==12.1.105
nvidia-cuda-nvrtc-cu12==12.1.105
nvidia-cuda-runtime-cu12==12.1.105
nvidia-cudnn-cu12==9.1.0.70
nvidia-cufft-cu12==11.0.2.54
nvidia-curand-cu12==10.3.2.106
nvidia-cusolver-cu12==11.4.5.107
nvidia-cusparse-cu12==12.1.0.106
nvidia-ml-py==12.535.161
nvidia-nccl-cu12==2.20.5
nvidia-nvjitlink-cu12==12.6.20
nvidia-nvtx-cu12==12.1.105
nvitop==1.3.2
oauthlib==3.2.0
openai==1.42.0
outlines==0.0.46
packaging==24.1
pandas==2.2.2
pexpect==4.8.0
pillow==10.4.0
platformdirs==4.2.2
prometheus-fastapi-instrumentator==7.0.0
prometheus_client==0.20.0
protobuf==5.27.3
psutil==6.0.0
ptyprocess==0.7.0
py-cpuinfo==9.0.0
pyairports==2.1.1
pyarrow==17.0.0
pyasn1==0.4.8
pyasn1-modules==0.2.1
pycountry==24.6.1
pydantic==2.8.2
pydantic_core==2.20.1
PyGObject==3.42.1
PyHamcrest==2.0.2
PyJWT==2.3.0
pyOpenSSL==21.0.0
pyparsing==2.4.7
pyrsistent==0.18.1
pyserial==3.5
python-apt==2.4.0+ubuntu3
python-dateutil==2.9.0.post0
python-debian==0.1.43+ubuntu1.1
python-dotenv==1.0.1
python-magic==0.4.24
pytz==2022.1
PyYAML==5.4.1
pyzmq==26.1.1
ray==2.34.0
referencing==0.35.1
regex==2024.7.24
requests==2.32.3
rpds-py==0.20.0
safetensors==0.4.4
screen-resolution-extra==0.0.0
SecretStorage==3.3.1
sentencepiece==0.2.0
sentry-sdk==2.13.0
service-identity==18.1.0
setproctitle==1.3.3
simple_parsing==0.1.5
six==1.16.0
smmap==5.0.1
sniffio==1.3.1
sos==4.5.6
ssh-import-id==5.11
starlette==0.38.2
sympy==1.13.2
systemd-python==234
tensorboard==2.17.1
tensorboard-data-server==0.7.2
termcolor==2.4.0
tiktoken==0.7.0
tokenizers==0.19.1
torch==2.4.0
torchvision==0.19.0
tqdm==4.66.5
transformers==4.44.1
triton==3.0.0
Twisted==22.1.0
typing_extensions==4.12.2
tzdata==2024.1
ubuntu-drivers-common==0.0.0
ubuntu-pro-client==8001
ufw==0.36.1
unattended-upgrades==0.1
urllib3==2.2.2
uvicorn==0.30.6
uvloop==0.20.0
vllm==0.5.4
vllm-flash-attn==2.6.1
wadllib==1.3.6
wandb==0.17.7
watchfiles==0.23.0
websockets==13.0
Werkzeug==3.0.3
xformers==0.0.27.post2
xkit==0.0.0
xxhash==3.5.0
yarl==1.9.4
zipp==1.0.0
zope.interface==5.4.0Reproduction Steps
- Follow the tutorial / Downlaod mistral-7B-instruct
- vllm serve ~/finetuning/models/7B
INFO 08-22 14:29:18 api_server.py:339] vLLM API server version 0.5.4
INFO 08-22 14:29:18 api_server.py:340] args: Namespace(model_tag='/home/barbatus/finetuning/models/7B', host=None, port=8000, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, model='/home/barbatus/finetuning/models/7B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=False, download_dir=None, load_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=None, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=1, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=False, disable_sliding_window=False, use_v2_block_manager=False, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=None, max_num_seqs=256, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=False, max_context_len_to_capture=None, max_seq_len_to_capture=8192, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', scheduler_delay_factor=0.0, enable_chunked_prefill=None, speculative_model=None, num_speculative_tokens=None, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=None, qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, engine_use_ray=False, disable_log_requests=False, max_log_len=None, dispatch_function=<function serve at 0x7fbd2bdbd510>)
WARNING 08-22 14:29:18 config.py:1454] Casting torch.bfloat16 to torch.float16.
INFO 08-22 14:29:18 llm_engine.py:174] Initializing an LLM engine (v0.5.4) with config: model='/home/barbatus/finetuning/models/7B', speculative_config=None, tokenizer='/home/barbatus/finetuning/models/7B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None), seed=0, served_model_name=/home/barbatus/finetuning/models/7B, use_v2_block_manager=False, enable_prefix_caching=False)
You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message
You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message.
INFO 08-22 14:29:19 model_runner.py:720] Starting to load model /home/barbatus/finetuning/models/7B...
Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00<?, ?it/s]
Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/rpc/server.py", line 217, in run_rpc_server
server = AsyncEngineRPCServer(async_engine_args, usage_context, port)
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/rpc/server.py", line 25, in __init__
self.engine = AsyncLLMEngine.from_engine_args(async_engine_args,
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 471, in from_engine_args
engine = cls(
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 381, in __init__
self.engine = self._init_engine(*args, **kwargs)
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 552, in _init_engine
return engine_class(*args, **kwargs)
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 249, in __init__
self.model_executor = executor_class(
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 47, in __init__
self._init_executor()
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/executor/gpu_executor.py", line 36, in _init_executor
self.driver_worker.load_model()
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/worker/worker.py", line 139, in load_model
self.model_runner.load_model()
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 722, in load_model
self.model = get_model(model_config=self.model_config,
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/model_executor/model_loader/__init__.py", line 21, in get_model
return loader.load_model(model_config=model_config,
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/model_executor/model_loader/loader.py", line 327, in load_model
model.load_weights(
File "/home/barbatus/.local/lib/python3.10/site-packages/vllm/model_executor/models/llama.py", line 513, in load_weights
param = params_dict[name]
KeyError: 'layers.0.attention.wk.weight'
Expected Behavior
To launch the model
Additional Context
I am trying to launch vllm with my trained model, before pluging LoRA layer, I am trying to load the base model but it fails. Can you help?
Thank you
Suggested Solutions
No response