fileio related changes

Author: rogerbinns

apsw/tests/extratest.py

@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import tempfile
+import random
 import unittest
 
 import apsw
@@ -25,6 +26,7 @@ def testLoadExtension(self):
         db = apsw.Connection(":memory:")
         for name, extra in self.extras.items():
             if extra["type"] == "extension" and apsw.sqlite_extra.has(name):
+                self.assertEqual("extension", apsw.sqlite_extra.has(name))
                 with self.subTest(name=name):
                     if self.verbose:
                         print(f"  >> Extension {name}")
@@ -52,10 +54,10 @@ def testLoadExtension(self):
                         self.assertGreater(num_diff, 0)
 
     def testExecutable(self):
-        with tempfile.TemporaryDirectory(prefix="apsw-extra-test") as tmpd:
-            # https://news.ycombinator.com/item?id=42647101 test case
+        spicy = "√π⁷≤∞"
 
-            spicy = "√π⁷≤∞"
+        with tempfile.TemporaryDirectory(prefix=f"apsw-extra-{spicy}-test") as tmpd:
+            # https://news.ycombinator.com/item?id=42647101 test case
 
             with open(pathlib.Path(tmpd) / f"{spicy}.sql", "wt", encoding="utf8") as sqlf:
                 print(f"CrEaTe       Table {spicy}(one, two);\ninsert into {spicy} values(7,8)", file=sqlf)
@@ -78,6 +80,7 @@ def testExecutable(self):
 
             for name, extra in self.extras.items():
                 if extra["type"] == "executable" and apsw.sqlite_extra.has(name):
+                    self.assertEqual("executable", apsw.sqlite_extra.has(name))
                     with self.subTest(name=name):
                         if self.verbose:
                             print(f"  >> Executable {name}")
@@ -187,6 +190,51 @@ def testShell(self):
         # deliberate error that shouldn't show sqlite_extra attempt
         self.assertRaises(apsw.ExtensionLoadingError, s.process_command, ".load thisdoesnotexistandshouldgiveanerror")
 
+    def testOther(self):
+        self.assertIsNone(apsw.sqlite_extra.has("kjldhsfk does not exist jhdskjfhdsfdsfsd"))
+        # we don't type check at the moment
+        self.assertIsNone(apsw.sqlite_extra.has(3 + 4j))
+        if apsw.sqlite_extra.has("sha1"):
+            self.assertTrue(os.path.exists(apsw.sqlite_extra.path("sha1")))
+
+    def testFileIO(self):
+        # we add some extra code to make it compile under windows, so
+        # test that works
+        if not apsw.sqlite_extra.has("fileio") or not hasattr(apsw, "enable_load_extension"):
+            return
+
+        spicy = "√π⁷≤∞"
+        with tempfile.TemporaryDirectory(prefix=f"apsw-extra-{spicy}-test") as tmpd:
+            db = apsw.Connection("")
+
+            size = 12345
+
+            blob = db.execute("SELECT randomblob(?)", (size,)).get
+            db.enable_load_extension(True)
+            db.load_extension(apsw.sqlite_extra.path("fileio"))
+
+            # the names came from gemini trying to get contrasting
+            # utf8 and utf16 encoded lengths
+            names = (
+                spicy,
+                "𐐀𐐁𐐂𐐃𐐄𐐅𐐆𐐇𐐈𐐉𐐊𐐋𐐌𐐍𐐎𐐏𐐐𐐑𐐒𐐓𐐔𐐕𐐖𐐗𐐘",
+                "The quick brown fox jumps over the lazy dog 1234567",
+                "😀😁😂🤣😃😄😅😆😉😊😋😎😍😘🥰😗😙😚☺️🙂🤗🤩🤔🤨😐😑😶🙄😏😣😥😮🤐",
+            )
+
+            for name in names:
+                fname = str(pathlib.Path(tmpd) / f"{name}.blob")
+                res = db.execute("SELECT writefile(?, ?)", (fname, blob)).get
+                self.assertEqual(res, size)
+
+                self.assertEqual(blob, db.execute("SELECT readfile(?)", (fname,)).get)
+
+
+            # check listing works
+            for name, data in db.execute("select name, data from fsdir(?)", (tmpd,)):
+                print(f"{name=}")
+                self.assertEqual(blob, data)
+
 
 __all__ = ("Extra",)
 

src/fileio_win32.c

@@ -0,0 +1,77 @@
+/*
+
+The fileio extension calls two functions that are in the SQLite
+library but are not part of the sqlite extension api.  For our
+extra build we link against libsqlite_tool but that isn't useful.
+
+* APSW is using an amalgamation not that sqlite dll
+* APSW doesn't include the qlite_extra_binaries directory in its
+  runtime library path
+* .. and it would result in two SQLites anyway
+* .. which would confuse whose malloc is being used and just make
+  things worse
+
+So we link with this file containing the routines
+*/
+
+#if defined(WIN32) || defined(_WIN32)
+
+#include "sqlite3ext.h"
+
+/* this points the api pointers in fileio.c */
+SQLITE_EXTENSION_INIT3
+
+#include <windows.h>
+
+/* These implementations are semantically the same as in the SQLite
+source but directly use public functions directly */
+
+LPWSTR
+sqlite3_win32_utf8_to_unicode(const char *zText)
+{
+  LPWSTR zWideText = 0;
+  int nChar;
+
+  /* the conversion is two passes - first to figure out the
+    size including null terminator */
+  nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0);
+  if (!nChar)
+    return 0;
+  zWideText = sqlite3_malloc(nChar * sizeof(WCHAR));
+  if (!zWideText)
+    return 0;
+
+  /* second pass - do the conversion */
+  nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar);
+  if (!nChar)
+  {
+    sqlite3_free(zWideText);
+    return 0;
+  }
+  return zWideText;
+}
+
+char *
+sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText)
+{
+  char *zText = 0;
+  int nByte;
+
+  /* same pattern as above - corresponding opposite API */
+  nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0);
+  if (!nByte)
+    return 0;
+  zText = sqlite3_malloc(nByte);
+  if (!zText)
+    return 0;
+
+  nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0);
+  if (!nByte)
+  {
+    sqlite3_free(zText);
+    return 0;
+  }
+  return zText;
+}
+
+#endif

tools/vend.py

@@ -108,6 +108,7 @@ def __post_init__(self):
     Extra(
         name="fileio",
         description="Implements SQL functions readfile() and writefile(), and eponymous virtual type 'fsdir'",
+        sources=["ext/misc/fileio.c", "../src/fileio_win32.c"],
     ),
     # ::TODO:: fossildelta once RBU extension is wrapped
     Extra(