Skip to content

tests: UnicodeEncodeError on windows with python 3.11  #9413

Open
@efiop

Description

@efiop
________________________ ERROR at setup of test_gc_api ________________________
2023-05-05T17:17:38.5255905Z [gw0] win32 -- Python 3.11.3 C:\hostedtoolcache\windows\Python\3.11.3\x64\python.exe
2023-05-05T17:17:38.5256251Z 
2023-05-05T17:17:38.5256446Z self = <LoggerHandler 1 (DEBUG)>
2023-05-05T17:17:38.5258261Z record = <LogRecord: dvc_objects.fs.utils, 10, C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_objects\fs\utils.py, 95, "Removing '%s'">
2023-05-05T17:17:38.5258762Z 
2023-05-05T17:17:38.5258971Z     def emit(self, record):
2023-05-05T17:17:38.5259494Z         """Write to Tqdm's stream so as to not break progress-bars"""
2023-05-05T17:17:38.5259891Z         try:
2023-05-05T17:17:38.5260229Z             if record.exc_info:
2023-05-05T17:17:38.5260608Z                 _, exc, *_ = record.exc_info
2023-05-05T17:17:38.5260996Z                 if hasattr(exc, "__pretty_exc__"):
2023-05-05T17:17:38.5261351Z                     try:
2023-05-05T17:17:38.5261951Z                         self.emit_pretty_exception(exc, verbose=_is_verbose())
2023-05-05T17:17:38.5262397Z                         if not _is_verbose():
2023-05-05T17:17:38.5262753Z                             return
2023-05-05T17:17:38.5263229Z                     # pylint: disable-next=broad-except
2023-05-05T17:17:38.5263625Z                     except Exception:  # noqa: BLE001, S110  # nosec B110
2023-05-05T17:17:38.5264030Z                         pass
2023-05-05T17:17:38.5264346Z     
2023-05-05T17:17:38.5264675Z             msg = self.format(record)
2023-05-05T17:17:38.5265121Z >           Tqdm.write(msg, file=self.stream, end=getattr(self, "terminator", "\n"))
2023-05-05T17:17:38.5265423Z 
2023-05-05T17:17:38.5265627Z D:\a\dvc\dvc\dvc\logger.py:159: 
2023-05-05T17:17:38.5266015Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2023-05-05T17:17:38.5266302Z 
2023-05-05T17:17:38.5266574Z cls = <class 'dvc.progress.Tqdm'>
2023-05-05T17:17:38.5269328Z s = "\\x1b[34m2023-05-05 17:03:30,036\\x1b[39m \\x1b[34mDEBUG\\x1b[39m: Removing 'C:\\\\Users\\\\runneradmin\\\\AppData\\\\Local\\\\Temp\\\\pytest-of-runneradmin\\\\pytest-0\\\\popen-gw0\\\\test_gc_api0\\\\data\\\\sub\\\\\u0442\u0435\u0441\u0442'"
2023-05-05T17:17:38.5270338Z file = <_io.TextIOWrapper name=1 mode='w' encoding='cp1252'>, end = '\n'
2023-05-05T17:17:38.5270744Z nolock = False
2023-05-05T17:17:38.5270967Z 
2023-05-05T17:17:38.5271379Z     @classmethod
2023-05-05T17:17:38.5271779Z     def write(cls, s, file=None, end="\n", nolock=False):
2023-05-05T17:17:38.5272223Z         """Print a message via tqdm (without overlap with bars)."""
2023-05-05T17:17:38.5272669Z         fp = file if file is not None else sys.stdout
2023-05-05T17:17:38.5273363Z         with cls.external_write_mode(file=file, nolock=nolock):
2023-05-05T17:17:38.5273758Z             # Write the message
2023-05-05T17:17:38.5274092Z >           fp.write(s)
2023-05-05T17:17:38.5274314Z 
2023-05-05T17:17:38.5274713Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\tqdm\std.py:610: 
2023-05-05T17:17:38.5275361Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2023-05-05T17:17:38.5275600Z 
2023-05-05T17:17:38.5275882Z self = <encodings.cp1252.IncrementalEncoder object at 0x0000029654D04110>
2023-05-05T17:17:38.5276880Z input = "\\x1b[34m2023-05-05 17:03:30,036\\x1b[39m \\x1b[34mDEBUG\\x1b[39m: Removing 'C:\\\\Users\\\\runneradmin\\\\AppData\\\\Local\\\\Temp\\\\pytest-of-runneradmin\\\\pytest-0\\\\popen-gw0\\\\test_gc_api0\\\\data\\\\sub\\\\\u0442\u0435\u0441\u0442'"
2023-05-05T17:17:38.5277548Z final = False
2023-05-05T17:17:38.5277761Z 
2023-05-05T17:17:38.5277970Z     def encode(self, input, final=False):
2023-05-05T17:17:38.5278428Z >       return codecs.charmap_encode(input,self.errors,encoding_table)[0]
2023-05-05T17:17:38.5279122Z E       UnicodeEncodeError: 'charmap' codec can't encode characters in position 164-167: character maps to <undefined>
2023-05-05T17:17:38.5279480Z 
2023-05-05T17:17:38.5280261Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\encodings\cp1252.py:19: UnicodeEncodeError
2023-05-05T17:17:38.5280608Z 
2023-05-05T17:17:38.5280864Z During handling of the above exception, another exception occurred:
2023-05-05T17:17:38.5281157Z 
2023-05-05T17:17:38.5281622Z tmp_dir = WindowsTmpDir('C:/Users/runneradmin/AppData/Local/Temp/pytest-of-runneradmin/pytest-0/popen-gw0/test_gc_api0')
2023-05-05T17:17:38.5282400Z dvc = Repo: 'C:\Users\runneradmin\AppData\Local\Temp\pytest-of-runneradmin\pytest-0\popen-gw0\test_gc_api0'
2023-05-05T17:17:38.5282777Z 
2023-05-05T17:17:38.5282954Z     @pytest.fixture
2023-05-05T17:17:38.5283315Z     def good_and_bad_cache(tmp_dir, dvc):
2023-05-05T17:17:38.5283697Z         tmp_dir.dvc_gen("foo", "foo")
2023-05-05T17:17:38.5284120Z >       (stage,) = tmp_dir.dvc_gen(
2023-05-05T17:17:38.5284448Z             "data",
2023-05-05T17:17:38.5284902Z             {"sub": {"data_sub": "data_sub", "data": "data", "\u0442\u0435\u0441\u0442": "\u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430"}},
2023-05-05T17:17:38.5285360Z         )
2023-05-05T17:17:38.5285566Z 
2023-05-05T17:17:38.5285772Z D:\a\dvc\dvc\tests\func\test_gc.py:19: 
2023-05-05T17:17:38.5286167Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2023-05-05T17:17:38.5286573Z D:\a\dvc\dvc\dvc\testing\tmp_dir.py:150: in dvc_gen
2023-05-05T17:17:38.5286994Z     return self.dvc_add(paths, commit=commit)
2023-05-05T17:17:38.5287407Z D:\a\dvc\dvc\dvc\testing\tmp_dir.py:171: in dvc_add
2023-05-05T17:17:38.5287801Z     stages = self.dvc.add(filenames)
2023-05-05T17:17:38.5288217Z D:\a\dvc\dvc\dvc\utils\collections.py:160: in inner
2023-05-05T17:17:38.5288624Z     result = func(*ba.args, **ba.kwargs)
2023-05-05T17:17:38.5289024Z D:\a\dvc\dvc\dvc\repo\__init__.py:65: in wrapper
2023-05-05T17:17:38.5289811Z     return f(repo, *args, **kwargs)
2023-05-05T17:17:38.5290323Z D:\a\dvc\dvc\dvc\repo\scm_context.py:151: in run
2023-05-05T17:17:38.5290722Z     return method(repo, *args, **kw)
2023-05-05T17:17:38.5291130Z D:\a\dvc\dvc\dvc\repo\add.py:206: in add
2023-05-05T17:17:38.5291454Z     stage.commit()
2023-05-05T17:17:38.5292131Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\funcy\decorators.py:47: in wrapper
2023-05-05T17:17:38.5292570Z     return deco(call, *dargs, **dkwargs)
2023-05-05T17:17:38.5293247Z D:\a\dvc\dvc\dvc\stage\decorators.py:43: in rwlocked
2023-05-05T17:17:38.5293822Z     return call()
2023-05-05T17:17:38.5294390Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\funcy\decorators.py:68: in __call__
2023-05-05T17:17:38.5294860Z     return self._func(*self._args, **self._kwargs)
2023-05-05T17:17:38.5310415Z D:\a\dvc\dvc\dvc\stage\__init__.py:554: in commit
2023-05-05T17:17:38.5311136Z     out.commit(filter_info=filter_info, **kwargs)
2023-05-05T17:17:38.5314846Z D:\a\dvc\dvc\dvc\output.py:744: in commit
2023-05-05T17:17:38.5315414Z     self._checkout(
2023-05-05T17:17:38.5315865Z D:\a\dvc\dvc\dvc\output.py:705: in _checkout
2023-05-05T17:17:38.5316574Z     return checkout(*args, **kwargs)
2023-05-05T17:17:38.5317435Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_data\hashfile\checkout.py:274: in checkout
2023-05-05T17:17:38.5317976Z     _checkout(
2023-05-05T17:17:38.5318658Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_data\hashfile\checkout.py:221: in _checkout
2023-05-05T17:17:38.5319212Z     _checkout_file(
2023-05-05T17:17:38.5319899Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_data\hashfile\checkout.py:88: in _checkout_file
2023-05-05T17:17:38.5320430Z     _relink(
2023-05-05T17:17:38.5321035Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_data\hashfile\checkout.py:59: in _relink
2023-05-05T17:17:38.5321683Z     _remove(path, fs, in_cache, force=force, prompt=prompt)
2023-05-05T17:17:38.5322394Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_data\hashfile\checkout.py:55: in _remove
2023-05-05T17:17:38.5322915Z     fs.remove(path)
2023-05-05T17:17:38.5323554Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_objects\fs\base.py:470: in rm
2023-05-05T17:17:38.5324132Z     self.fs.rm(path, recursive=recursive, **kwargs)
2023-05-05T17:17:38.5324802Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_objects\fs\local.py:119: in rm
2023-05-05T17:17:38.5325304Z     remove(p)
2023-05-05T17:17:38.5325936Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_objects\fs\utils.py:95: in remove
2023-05-05T17:17:38.5326559Z     logger.debug("Removing '%s'", path)
2023-05-05T17:17:38.5327364Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\logging\__init__.py:1477: in debug
2023-05-05T17:17:38.5327962Z     self._log(DEBUG, msg, args, **kwargs)
2023-05-05T17:17:38.5328541Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\logging\__init__.py:1634: in _log
2023-05-05T17:17:38.5329050Z     self.handle(record)
2023-05-05T17:17:38.5329575Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\logging\__init__.py:1644: in handle
2023-05-05T17:17:38.5330005Z     self.callHandlers(record)
2023-05-05T17:17:38.5330448Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\logging\__init__.py:1706: in callHandlers
2023-05-05T17:17:38.5331006Z     hdlr.handle(record)
2023-05-05T17:17:38.5331407Z C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\logging\__init__.py:978: in handle
2023-05-05T17:17:38.5331751Z     self.emit(record)
2023-05-05T17:17:38.5332065Z D:\a\dvc\dvc\dvc\logger.py:164: in emit
2023-05-05T17:17:38.5332390Z     self.handleError(record)
2023-05-05T17:17:38.5332717Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2023-05-05T17:17:38.5332903Z 
2023-05-05T17:17:38.5333041Z self = <LoggerHandler 1 (DEBUG)>
2023-05-05T17:17:38.5333784Z record = <LogRecord: dvc_objects.fs.utils, 10, C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_objects\fs\utils.py, 95, "Removing '%s'">
2023-05-05T17:17:38.5334146Z 
2023-05-05T17:17:38.5334311Z     def handleError(self, record):  # noqa: N802
2023-05-05T17:17:38.5334644Z         super().handleError(record)
2023-05-05T17:17:38.5334981Z >       raise LoggingException(record)
2023-05-05T17:17:38.5335737Z E       dvc.logger.LoggingException: failed to log <LogRecord: dvc_objects.fs.utils, 10, C:\hostedtoolcache\windows\Python\3.11.3\x64\Lib\site-packages\dvc_objects\fs\utils.py, 95, "Removing '%s'">
2023-05-05T17:17:38.5336380Z 
2023-05-05T17:17:38.5336568Z D:\a\dvc\dvc\dvc\logger.py:139: LoggingException

Metadata

Metadata

Assignees

No one assigned

    Labels

    testingRelated to the tests and the testing infrastructure

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions