Merge branch 'develop' into master

* develop:
  Update CHANGELOG.md
  Add .xz and increase performance of compression module (#875)
  Bump pypa/gh-action-pypi-publish in /.github/workflows (#878)
  Bump actions/checkout from 4 to 5 in the github-actions group (#877)
  Fix release.sh for the final merge back into develop (#872)

Author: ddelange

.github/workflows/python-package.yml

@@ -11,7 +11,7 @@ jobs:
   linters:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
@@ -52,7 +52,7 @@ jobs:
           - {python-version: '3.12', os: windows-2025}
           - {python-version: '3.13', os: windows-2025}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
@@ -97,7 +97,7 @@ jobs:
           # - {python-version: '3.13', os: windows-2025}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
@@ -137,7 +137,7 @@ jobs:
           # - {python-version: '3.13', os: windows-2025}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
@@ -189,7 +189,7 @@ jobs:
           # - {python-version: '3.13', os: windows-2025}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
       pull-requests: write # apexskier/github-release-commenter
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
       with:
         fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
@@ -38,7 +38,7 @@ jobs:
 
     # https://github.com/pypa/gh-action-pypi-publish#trusted-publishing
     - name: Publish package distributions to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.12.4
+      uses: pypa/gh-action-pypi-publish@v1.13.0
 
     - uses: apexskier/[email protected]
       with:

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 7.3.1, 2025-09-08
+
+- Fix release.sh for the final merge back into develop (PR [#872](https://github.com/piskvorky/smart_open/pull/872), [@ddelange](https://github.com/ddelange))
+- Bump actions/checkout from 4 to 5 in the github-actions group (PR [#877](https://github.com/piskvorky/smart_open/pull/877), [@dependabot[bot]](https://github.com/apps/dependabot))
+- Bump pypa/gh-action-pypi-publish from 1.12.4 to 1.13.0 in /.github/workflows (PR [#878](https://github.com/piskvorky/smart_open/pull/878), [@dependabot[bot]](https://github.com/apps/dependabot))
+- Add .xz and increase performance of compression module (PR [#875](https://github.com/piskvorky/smart_open/pull/875), [@ddelange](https://github.com/ddelange))
+
 # 7.3.0.post1, 2025-07-03
 
 - Fix release.sh merge message and final merge (PR [#868](https://github.com/piskvorky/smart_open/pull/868), [@ddelange](https://github.com/ddelange))

README.rst

@@ -245,7 +245,7 @@ By default, ``smart_open`` determines the compression algorithm to use based on
 
 .. code-block:: python
 
-    >>> from smart_open import open, register_compressor
+    >>> from smart_open import open
     >>> with open('tests/test_data/1984.txt.gz') as fin:
     ...     print(fin.read(32))
     It was a bright cold day in Apri
@@ -255,7 +255,7 @@ To disable compression:
 
 .. code-block:: python
 
-    >>> from smart_open import open, register_compressor
+    >>> from smart_open import open
     >>> with open('tests/test_data/1984.txt.gz', 'rb', compression='disable') as fin:
     ...     print(fin.read(32))
     b'\x1f\x8b\x08\x08\x85F\x94\\\x00\x031984.txt\x005\x8f=r\xc3@\x08\x85{\x9d\xe2\x1d@'
@@ -265,7 +265,7 @@ To specify the algorithm explicitly (e.g. for non-standard file extensions):
 
 .. code-block:: python
 
-    >>> from smart_open import open, register_compressor
+    >>> from smart_open import open
     >>> with open('tests/test_data/1984.txt.gzip', compression='.gz') as fin:
     ...     print(fin.read(32))
     It was a bright cold day in Apri
@@ -279,18 +279,15 @@ For example, to open xz-compressed files:
     >>> from smart_open import open, register_compressor
 
     >>> def _handle_xz(file_obj, mode):
-    ...      return lzma.LZMAFile(filename=file_obj, mode=mode, format=lzma.FORMAT_XZ)
+    ...      return lzma.LZMAFile(filename=file_obj, mode=mode)
 
     >>> register_compressor('.xz', _handle_xz)
 
     >>> with open('tests/test_data/1984.txt.xz') as fin:
     ...     print(fin.read(32))
     It was a bright cold day in Apri
 
-``lzma`` is in the standard library in Python 3.3 and greater.
-For 2.7, use `backports.lzma`_.
-
-.. _backports.lzma: https://pypi.org/project/backports.lzma/
+This is just an example: ``lzma`` is in the standard library and is registered by default.
 
 Transport-specific Options
 --------------------------

extending.md

@@ -128,7 +128,7 @@ For example:
 ```python
 def _handle_xz(file_obj, mode):
     import lzma
-    return lzma.LZMAFile(filename=file_obj, mode=mode, format=lzma.FORMAT_XZ)
+    return lzma.LZMAFile(filename=file_obj, mode=mode)
 
 
 register_compressor('.xz', _handle_xz)

help.txt

@@ -326,6 +326,7 @@ FUNCTIONS
 
         * .bz2
         * .gz
+        * .xz
         * .zst
 
         The function depends on the file extension to determine the appropriate codec.
@@ -405,10 +406,10 @@ FUNCTIONS
         Parameters
         ----------
         ext: str
-            The extension.  Must include the leading period, e.g. ``.gz``.
+            The extension.  Must include the leading period, e.g. `.gz`.
         callback: callable
             The callback.  It must accept two position arguments, file_obj and mode.
-            This function will be called when ``smart_open`` is opening a file with
+            This function will be called when `smart_open` is opening a file with
             the specified extension.
 
         Examples
@@ -419,10 +420,12 @@ FUNCTIONS
 
         >>> def _handle_xz(file_obj, mode):
         ...     import lzma
-        ...     return lzma.LZMAFile(filename=file_obj, mode=mode, format=lzma.FORMAT_XZ)
+        ...     return lzma.LZMAFile(filename=file_obj, mode=mode)
         >>>
         >>> register_compressor('.xz', _handle_xz)
 
+        This is just an example: `lzma` is in the standard library and is registered by default.
+
     s3_iter_bucket(bucket_name, prefix='', accept_key=None, key_limit=None, workers=16, retries=3, **session_kwargs)
         Deprecated.  Use smart_open.s3.iter_bucket instead.
 

release/release.sh

@@ -20,6 +20,9 @@ git pull
 # Merge `develop` into `master` and push
 git merge develop --no-ff -m "Merge branch 'develop' into master"
 git push
+# Make sure you're on `develop` and you're up to date locally
+git checkout develop
+git pull
 # Merge `master` back into `develop` and push
 git merge master --no-ff -m "Merge branch 'master' into develop"
 git push

smart_open/compression.py

@@ -5,7 +5,7 @@
 # This code is distributed under the terms and conditions
 # from the MIT License (MIT).
 #
-"""Implements the compression layer of the ``smart_open`` library."""
+"""Implements the compression layer of the `smart_open` library."""
 import io
 import logging
 import os.path
@@ -42,10 +42,10 @@ def register_compressor(ext, callback):
     Parameters
     ----------
     ext: str
-        The extension.  Must include the leading period, e.g. ``.gz``.
+        The extension.  Must include the leading period, e.g. `.gz`.
     callback: callable
         The callback.  It must accept two position arguments, file_obj and mode.
-        This function will be called when ``smart_open`` is opening a file with
+        This function will be called when `smart_open` is opening a file with
         the specified extension.
 
     Examples
@@ -56,10 +56,12 @@ def register_compressor(ext, callback):
 
     >>> def _handle_xz(file_obj, mode):
     ...     import lzma
-    ...     return lzma.LZMAFile(filename=file_obj, mode=mode, format=lzma.FORMAT_XZ)
+    ...     return lzma.LZMAFile(filename=file_obj, mode=mode)
     >>>
     >>> register_compressor('.xz', _handle_xz)
 
+    This is just an example: `lzma` is in the standard library and is registered by default.
+
     """
     if not (ext and ext[0] == '.'):
         raise ValueError('ext must be a string starting with ., not %r' % ext)
@@ -72,7 +74,7 @@ def register_compressor(ext, callback):
 def tweak_close(outer, inner):
     """Ensure that closing the `outer` stream closes the `inner` stream as well.
 
-    Deprecated: smart_open.open().__exit__ now always calls __exit__ on the
+    Deprecated: `smart_open.open().__exit__` now always calls `__exit__` on the
     underlying filestream.
 
     Use this when your compression library's `close` method does not
@@ -94,33 +96,38 @@ def close_both(*args):
     outer.close = close_both
 
 
-def _handle_bz2(file_obj, mode):
-    from bz2 import BZ2File
-    result = BZ2File(file_obj, mode)
+def _maybe_wrap_buffered(file_obj, mode):
+    # https://github.com/piskvorky/smart_open/issues/760#issuecomment-1553971657
+    result = file_obj
+    if "b" in mode and "w" in mode:
+        result = io.BufferedWriter(result)
+    elif "b" in mode and "r" in mode:
+        result = io.BufferedReader(result)
     return result
 
 
+def _handle_bz2(file_obj, mode):
+    import bz2
+    result = bz2.open(filename=file_obj, mode=mode)
+    return _maybe_wrap_buffered(result, mode)
+
+
 def _handle_gzip(file_obj, mode):
     import gzip
-    result = gzip.GzipFile(fileobj=file_obj, mode=mode)
-    return result
+    result = gzip.open(filename=file_obj, mode=mode)
+    return _maybe_wrap_buffered(result, mode)
 
 
 def _handle_zstd(file_obj, mode):
-    import zstandard  # type: ignore
+    import zstandard
     result = zstandard.open(filename=file_obj, mode=mode)
-    # zstandard.open returns an io.TextIOWrapper in text mode, but otherwise
-    # returns a raw stream reader/writer, and we need the `io` wrapper
-    # to make FileLikeProxy work correctly.
-    #
-    # See:
-    #
-    # https://github.com/indygreg/python-zstandard/blob/d7d81e79dbe74feb22fb73405ebfb3e20f4c4653/zstandard/__init__.py#L169-L174
-    if "b" in mode and "w" in mode:
-        result = io.BufferedWriter(result)
-    elif "b" in mode and "r" in mode:
-        result = io.BufferedReader(result)
-    return result
+    return _maybe_wrap_buffered(result, mode)
+
+
+def _handle_xz(file_obj, mode):
+    import lzma
+    result = lzma.open(filename=file_obj, mode=mode)
+    return _maybe_wrap_buffered(result, mode)
 
 
 def compression_wrapper(file_obj, mode, compression=INFER_FROM_EXTENSION, filename=None):
@@ -165,3 +172,4 @@ def compression_wrapper(file_obj, mode, compression=INFER_FROM_EXTENSION, filena
 register_compressor('.bz2', _handle_bz2)
 register_compressor('.gz', _handle_gzip)
 register_compressor('.zst', _handle_zstd)
+register_compressor('.xz', _handle_xz)

tests/test_compression.py

@@ -5,8 +5,10 @@
 # This code is distributed under the terms and conditions
 # from the MIT License (MIT).
 #
+import bz2
 import gzip
 import io
+import lzma
 
 import pytest
 import zstandard as zstd
@@ -37,6 +39,14 @@ def label(thing, name):
         (io.BytesIO(zstd.ZstdCompressor().compress(plain)), 'infer_from_extension', 'file.ZST'),
         (label(io.BytesIO(zstd.ZstdCompressor().compress(plain)), 'file.zst'), 'infer_from_extension', ''),
         (io.BytesIO(zstd.ZstdCompressor().compress(plain)), '.zst', 'file.zst'),
+        (io.BytesIO(lzma.compress(plain)), 'infer_from_extension', 'file.xz'),
+        (io.BytesIO(lzma.compress(plain)), 'infer_from_extension', 'file.XZ'),
+        (label(io.BytesIO(lzma.compress(plain)), 'file.xz'), 'infer_from_extension', ''),
+        (io.BytesIO(lzma.compress(plain)), '.xz', 'file.xz'),
+        (io.BytesIO(bz2.compress(plain)), 'infer_from_extension', 'file.bz2'),
+        (io.BytesIO(bz2.compress(plain)), 'infer_from_extension', 'file.BZ2'),
+        (label(io.BytesIO(bz2.compress(plain)), 'file.bz2'), 'infer_from_extension', ''),
+        (io.BytesIO(bz2.compress(plain)), '.bz2', 'file.bz2'),
     ]
 )
 def test_compression_wrapper_read(fileobj, compression, filename):

tests/test_smart_open.py

@@ -78,28 +78,37 @@ def named_temporary_file(mode='w+b', prefix=None, suffix=None, delete=True):
             logger.error(e)
 
 
-def test_zst_write():
-    with named_temporary_file(suffix=".zst") as tmp:
-        with smart_open.open(tmp.name, "wt") as fout:
-            print("hello world", file=fout)
-            print("this is a test", file=fout)
+def test_compression_extensions():
+    for extension in smart_open.compression.get_supported_extensions():
+        with named_temporary_file(suffix=extension) as tmp:
+            with smart_open.open(tmp.name, "wt") as fout:
+                print("hello world", file=fout)
+                print("this is a test", file=fout)
 
-        with smart_open.open(tmp.name, "rt") as fin:
-            got = list(fin)
+            with smart_open.open(tmp.name, "rt") as fin:
+                got = list(fin)
 
-    assert got == ["hello world\n", "this is a test\n"]
+        assert got == ["hello world\n", "this is a test\n"], f"Error for {extension=}, mode='wt'"
 
+        with named_temporary_file(suffix=extension) as tmp:
+            with smart_open.open(tmp.name, "w") as fout:
+                fout.write("hello world\n")
+                fout.write("this is a test\n")
 
-def test_zst_write_binary():
-    with named_temporary_file(suffix=".zst") as tmp:
-        with smart_open.open(tmp.name, "wb") as fout:
-            fout.write(b"hello world\n")
-            fout.write(b"this is a test\n")
+            with smart_open.open(tmp.name, "r") as fin:
+                got = list(fin)
 
-        with smart_open.open(tmp.name, "rb") as fin:
-            got = list(fin)
+        assert got == ["hello world\n", "this is a test\n"], f"Error for {extension=}, mode='w'"
 
-    assert got == [b"hello world\n", b"this is a test\n"]
+        with named_temporary_file(suffix=extension) as tmp:
+            with smart_open.open(tmp.name, "wb") as fout:
+                fout.write(b"hello world\n")
+                fout.write(b"this is a test\n")
+
+            with smart_open.open(tmp.name, "rb") as fin:
+                got = list(fin)
+
+        assert got == [b"hello world\n", b"this is a test\n"], f"Error for {extension=}, mode='wb'"
 
 
 class ParseUriTest(unittest.TestCase):