Inckude zlib related extra

Author: rogerbinns

apsw/sqlite_extra.json

@@ -35,6 +35,10 @@
         "description": "A virtual table that returns suggested completions for a partial SQL input",
         "type": "extension"
     },
+    "compress": {
+        "description": "SQL compression functions",
+        "type": "extension"
+    },
     "csv": {
         "description": "A virtual table for reading CSV files",
         "type": "extension"
@@ -99,6 +103,10 @@
         "description": "Search a large vocabulary for close matches",
         "type": "extension"
     },
+    "sqlar": {
+        "description": "Command line SQL archive tool",
+        "type": "executable"
+    },
     "sqlite3_dbdump": {
         "description": "Converts the content of a SQLite database into UTF-8 text SQL statements that can be used to exactly recreate the original database",
         "type": "executable"
@@ -211,6 +219,10 @@
         "description": "Virtual table printing diagnostic information for interactive analysis and debugging",
         "type": "extension"
     },
+    "zipfile": {
+        "description": "Read/Write access to simple archives",
+        "type": "extension"
+    },
     "zorder": {
         "description": "Functions for z-order (Morton code) transformations",
         "type": "extension"

apsw/tests/extratest.py

@@ -85,6 +85,9 @@ def testExecutable(self):
                             print(f"  >> Executable {name}")
                         cmd = apsw.sqlite_extra.path(name)
                         match name:
+                            case "sqlar":
+                                self.run_cmd([cmd, pathlib.Path(tmpd) / f"{spicy}.sqlar", dbf])
+
                             case "sqlite3_dbdump" | "sqlite3_dbhash" | "sqlite3_dbtotxt":
                                 self.run_cmd([cmd, dbf], spicy)
 

checksums

@@ -1,14 +1,21 @@
-# This file contains checksums/hashes of SQLite so that we can verify
+# This file contains checksums/hashes of SQLite downloads so that we can verify
 # downloads using fetch command to setup.py have not been
 # tampered with.  See https://rogerbinns.github.io/apsw/install.html#fetch
 # for information on setup.py flags.
 
 # Format is URL length sha256 sha3_256 all on one line.
 # This extra checking mechanism is to give you a little bit
-# more peace of mind because the SQLite releases are not signed in any
+# more peace of mind because the releases are not signed in any
 # way.
 
+# These are used too compile sqlite extra
+
 https://sqlite.org/vec1/zip/vec1-20260306155250-d070184523.zip 75980 6a58eebbe1e4679645c80084fdfc44f2902e19cde95da5af932bb51bfdc7d02d 77785c8f4e0a9672dded6794861a2ca9d5a5a8dda43300b183690cd84149bf35
+https://sqlite.org/sqlar/zip/sqlar-src-20180107193712-4824e73896.zip 1980460 63bc4b3459fd00498f161d87d8b4516a581278da900001af183486458396806c c7b6a21b58e7499b7c3e36f26b7fafe44ddbd85097b37d905d72ccf5ff3e7cb8
+https://www.zlib.net/zlib132.zip 1616754 e8bf55f3017aa181690990cb58a994e77885da140609fc8f94abe9b65d2cae28 6b35b981f73425dc5f142cbee457f078d42240c022c804982849b962f1d1e2c6
+
+# SQLite releases
+
 https://sqlite.org/2026/sqlite-autoconf-3520000.tar.gz 3258980 f6b50b0c103392af32a8be15b2b9d25959de9a00a70c3979128aafeaa5338b3f 45a4911475950ab5fd486afb776102eb29d69e7569b48947f21e3c8501b51822
 https://sqlite.org/2026/sqlite-src-3520000.zip 14346407 652a98ca833ed638809a52bec225a7f37799f71a995778f9ccb68ad03bd1fc11 a2f0a14b6530138b0c8c096f4b5e5c9d3c1b8a5effa565c91f983e1235f9e26a
 

doc/changes.rst

@@ -10,6 +10,17 @@ history <https://devguide.python.org/versions/>`__.
 APSW changes by version
 -----------------------
 
+3.51.3.0
+========
+
+Includes all the changes from the 3.52.0.0 release listed below,
+except :code:`SQLITE_UTF8_ZT` and :code:`sqlite3_carray_bind_v2`.
+
+:doc:`extra` adds extensions and programs that require the `zlib
+compression library <https://www.zlib.net/>`__, notably `zipfile
+<https://sqlite.org/zipfile.html>`__ and `sqlar
+<https://sqlite.org/sqlar.html>`__
+
 3.52.0.0
 ========
 

doc/extra.rst

@@ -8,7 +8,7 @@ documentation as the main library.
 
 Full APSW builds such as those on PyPI include all the ones that
 compile for that platform, without any modifications.  This is for
-convenience and to help promote these great extras.  They add just
+convenience, and to help promote these great extras.  They add just
 over 1MB to the download and 3MB of disk space.
 
 Access is provided via an :ref:`API <extra_api>`, :ref:`command line
@@ -27,17 +27,30 @@ the SQLite team.  It is also included with the extras if possible.
 Dependencies
 ------------
 
-There are no dependencies for the extensions and programs.  That means
-they can be used on other compatible systems.  Some of the programs
-require the SQLite library alongside the program which **must** be
-placed in the same directory as the program if you copy the program
-elsewhere.  (The SQLite library in that directory deliberately has a
-different name to avoid interactions with the standard system SQLite
-library.)
-
-Extensions and programs that require third party libraries (eg
-compression), or TCL are not included, and optional third party
-libraries (eg readline) are not used.
+There are no dependencies for the extensions or tools.
+
+zlib
+++++
+
+Some of the extensions and programs use the `zlib compression library
+<https://www.zlib.net/>`__ (`license
+<https://www.zlib.net/zlib_license.html>`__) which is included
+statically for those extensions and programs.
+
+TCL
++++
+
+Some of the tools require `TCL <https://www.tcl-lang.org/>`__ and are
+omitted.
+
+Readline
+++++++++
+
+The shell can optionally use the readline (or editline) library for
+editing at the shell prompt.  This is disabled to avoid the
+dependency.  You can use the APSW :doc:`shell`, or the extra provided
+one with `rlwrap <https://linux.die.net/man/1/rlwrap>`__ if you need
+significant editing functionality.
 
 Marking
 -------

doc/sqlite_extra.rst-inc

@@ -5,6 +5,10 @@ Programs
 --------
 
 
+sqlar (`doc <https://sqlite.org/sqlar/>`__)
+
+   Command line SQL archive tool
+
 sqlite3_dbdump (`doc <https://sqlite.org/src/file?ci=trunk&name=ext%2fmisc%2fdbdump.c>`__)
 
    Converts the content of a SQLite database into UTF-8 text SQL statements that can be used to exactly recreate the original database
@@ -148,6 +152,12 @@ Extensions
       -
         * VTable: :code:`completion`
 
+    * - compress
+      - `link <https://sqlite.org/src/file?ci=trunk&name=ext%2fmisc%2fcompress.c>`__
+      - SQL compression functions
+      -
+        * Function: :code:`compress` :code:`uncompress`
+
     * - csv
       - `link <https://sqlite.org/csv.html>`__
       - A virtual table for reading CSV files
@@ -305,6 +315,13 @@ Extensions
       -
         * VTable: :code:`vtablog`
 
+    * - zipfile
+      - `link <https://sqlite.org/zipfile.html>`__
+      - Read/Write access to simple archives
+      -
+        * Function: :code:`zipfile` :code:`zipfile_cds`
+        * VTable: :code:`zipfile`
+
     * - zorder
       - `link <https://sqlite.org/src/file?ci=trunk&name=ext%2fmisc%2fzorder.c>`__
       - Functions for z-order (Morton code) transformations

setup.py

@@ -298,18 +298,32 @@ def run(self):
                     modtime = datetime.datetime(*zi.date_time).timestamp()
                     self.extract_entry(zipf.read(zi), zi.filename, modtime)
 
-            write("  Getting the experiment vec1 extension")
-
-            AURL = "https://sqlite.org/vec1/zip/vec1-20260306155250-d070184523.zip"
-
-            data = self.download(AURL, checksum=True)
-
-            with zipfile.ZipFile(data) as zipf:
-                for zi in zipf.infolist():
-                    if zi.is_dir():
-                        continue
-                    modtime = datetime.datetime(*zi.date_time).timestamp()
-                    self.extract_entry(zipf.read(zi), zi.filename, modtime, replace="sqlite3/vec1")
+            for desc, url, replace in (
+                (
+                    "experimental vec1 extension source",
+                    "https://sqlite.org/vec1/zip/vec1-20260306155250-d070184523.zip",
+                    "sqlite3/vec1",
+                ),
+                (
+                    "sqlar tool source",
+                    "https://sqlite.org/sqlar/zip/sqlar-src-20180107193712-4824e73896.zip",
+                    "sqlite3/sqlar",
+                ),
+                (
+                    "zlib source",
+                    "https://www.zlib.net/zlib132.zip",
+                    "sqlite3/zlib",
+                ),
+            ):
+                write(f"  Getting the {desc}")
+                data = self.download(url, checksum=True)
+
+                with zipfile.ZipFile(data) as zipf:
+                    for zi in zipf.infolist():
+                        if zi.is_dir():
+                            continue
+                        modtime = datetime.datetime(*zi.date_time).timestamp()
+                        self.extract_entry(zipf.read(zi), zi.filename, modtime, replace=replace)
 
         ## The amalgamation is a .tar.gz
         if self.sqlite:

tools/checksums.py

@@ -17,7 +17,9 @@
     "3510000",
 )
 
-vec1_urls = ("https://sqlite.org/vec1/zip/vec1-20260306155250-d070184523.zip",)
+other_urls = ("https://sqlite.org/vec1/zip/vec1-20260306155250-d070184523.zip",
+              "https://sqlite.org/sqlar/zip/sqlar-src-20180107193712-4824e73896.zip",
+              "https://www.zlib.net/zlib132.zip",)
 
 fixup_download_url = setup.fixup_download_url
 
@@ -47,13 +49,13 @@ def check(url, data):
     else:
         print(url, d[0], d[1], d[2])
 
-for vec1 in vec1_urls:
+for url in other_urls:
     try:
-        data = urllib.request.urlopen(vec1).read()
+        data = urllib.request.urlopen(url).read()
     except:
-        print(vec1)
+        print(url)
         raise
-    check(vec1, data)
+    check(url, data)
 
 for v in sqlitevers:
     # All platforms amalgamation

tools/vend.py

@@ -32,6 +32,8 @@ class Extra:
     "true if needing to link against sqlite3.c"
     lib_sqlite_stdio: bool = False
     "true if needing sqlite3_stdio"
+    lib_zlib: bool = False
+    "true if needing zlib - we statically link in"
     sources: list[str] = dataclasses.field(default_factory=list[str])
     "list of files making source for this extra relative to sqlite3 directory"
     type: Literal["extension"] | Literal["executable"] = "extension"
@@ -91,6 +93,11 @@ def __post_init__(self):
         description="A virtual table that returns suggested completions for a partial SQL input",
         doc="completion.html",
     ),
+    Extra(
+        name="compress",
+        description="SQL compression functions",
+        lib_zlib=True,
+    ),
     Extra(
         name="csv",
         description="A virtual table for reading CSV files",
@@ -199,6 +206,11 @@ def __post_init__(self):
     ),
     # vtshim: not useful
     # wholenumber: use generate_series
+    Extra(
+        name="zipfile",
+        doc="zipfile.html",
+        description="Read/Write access to simple archives",
+    ),
     # zipfile: requires libz
     Extra(
         name="zorder",
@@ -302,7 +314,7 @@ def __post_init__(self):
         lib_sqlite=True,
         # work around sqlite's mistaken handling of utf8 on windows.  it should be
         # using manifest but instead only has a hacky main thing going on
-        defines = [("main", "main")]
+        defines=[("main", "main")],
     ),
     Extra(
         name="sqlite3_showdb",
@@ -348,6 +360,14 @@ def __post_init__(self):
         sources=["tool/showwal.c"],
         description="Shows low level content of a WAL file",
     ),
+    Extra(
+        name="sqlar",
+        type="executable",
+        sources=["sqlar/sqlar.c"],
+        doc="sqlar/",
+        description="Command line SQL archive tool",
+        lib_sqlite=True,
+    ),
 ]
 
 import os
@@ -436,7 +456,7 @@ def make_windows_resource(manifest_filename: str | None, **fields):
 """
 
 
-def resource_file(build_dir, compiler, extra: Extra):
+def resource_file(build_dir, compiler, extra: Extra) -> str:
     if compiler.compiler_type == "msvc":
         if extra.type == "executable":
             with open(build_dir / "utf8_manifest", "wt") as mf:
@@ -493,18 +513,21 @@ def exc_type(exc: Exception) -> str:
         raise
     return type(exc).__name__
 
+
 @dataclasses.dataclass
 class CompilerImplementation:
     """What is the actual compiler
 
     Sometimes call we know is it is cc so the implementation
     details are determined by compile_check"""
+
     name: Literal["gcc"] | Literal["clang"] | Literal["msvc"] | Literal["unknown"]
     version: str
     "random text"
     misc: set[str]
     "the various other things"
 
+
 def do_build(what: set[str], verbose: bool, fail_fast: bool = False):
     get_version()
     compiler = ccompiler.new_compiler(verbose=True)
@@ -539,7 +562,7 @@ def do_build(what: set[str], verbose: bool, fail_fast: bool = False):
         objs = compiler.compile([compile_check_name], output_dir=str(build_dir))
         compiler.link_executable(objs, "compile_check", output_dir=str(build_dir))
         out_name = f"{build_dir}/compile_check{compiler.exe_extension if compiler.exe_extension else ''}"
-        p =subprocess.run([out_name], capture_output=True, encoding="utf8", text=True)
+        p = subprocess.run([out_name], capture_output=True, encoding="utf8", text=True)
         p.check_returncode()
         info = p.stdout.splitlines()
         ci = CompilerImplementation(info[0], info[1], set(info[2:]))
@@ -609,18 +632,33 @@ def do_build(what: set[str], verbose: bool, fail_fast: bool = False):
     lib_enables = "CARRAY COLUMN_METADATA DBPAGE_VTAB DBSTAT_VTAB FTS4 FTS5 GEOPOLY MATH_FUNCTIONS PERCENTILE PREUPDATE_HOOK RTREE SESSION STAT4".split()
 
     macros = [(f"SQLITE_ENABLE_{enable}", 1) for enable in lib_enables]
+
+    # add in others
+    macros.extend(
+        [
+            ("SQLITE_CONFIG_URI", 1),
+            ("SQLITE_THREADSAFE", 1),
+        ]
+    )
+
     cfg = pathlib.Path("sqlite3") / "sqlite_cfg.h"
     if cfg.exists():
         macros.append(("_HAVE_SQLITE_CONFIG_H", 1))
-    macros.append(("SQLITE_THREADSAFE", 1))
+
+    # we need a subset of zlib source files
+    zlib_sources = [
+        str(pathlib.Path("sqlite3/zlib") / f"{name}.c")
+        for name in "adler32 crc32 deflate inflate inffast inftrees zutil trees compress uncompr".split()
+    ]
+
     if compiler.compiler_type == "msvc":
         macros.append(("SQLITE_API", "__declspec(dllexport)"))
     try:
         lib_resource = resource_file(
             build_dir, compiler, Extra(name="libsqlite3", description="SQLite 3 library", doc="")
         )
         lib_objs = compiler.compile(
-            [str(pathlib.Path("sqlite3") / "sqlite3.c"), lib_resource],
+            [str(pathlib.Path("sqlite3") / "sqlite3.c"), lib_resource] + zlib_sources,
             output_dir=str(build_dir),
             macros=macros,
             extra_preargs=compile_extra_preargs,