mip: Support optional per-entry native code compatibility tags.

Related to #1140 / micropython#19478 / micropython#19479: those add
URL-templating for single-file natmod installs, but don't help when
a single manifest needs to serve different files per architecture.

Adds an optional third element to hashes/urls entries: a raw
sys.implementation._mpy-shaped integer. Entries without it install
unconditionally (backward compatible); entries with it only install
if _mpy_tag_ok() matches - exact match on version/sub-version/arch,
subset match on arch-flags (a variant that doesn't require an
extension must still install on hardware that supports it).

Signed-off-by: o-murphy <thehelixpg@gmail.com>

Author: o-murphy

micropython/mip/mip/__init__.py

@@ -24,6 +24,29 @@
 
 _ALLOWED_MIP_URL_PREFIXES = const(("http://", "https://", "codeberg:", "github:", "gitlab:"))
 
+# Bits 0-15 of sys.implementation._mpy: version/sub-version/arch (exact
+# match required). Bits 16+: optional arch-flags (e.g. RV32 extensions) -
+# a tag's flags must be a subset of what this device supports, not equal.
+_MPY_VERSION_MASK = const(0xFF)
+_MPY_BASE_MASK = const(0xFFFF)
+
+
+def _mpy_tag_ok(tag, device_mpy=None):
+    if device_mpy is None:
+        device_mpy = getattr(sys.implementation, "_mpy", None)
+    if device_mpy is None:
+        return False
+    # A bytecode-only tag (arch nibble 0) only needs the major version to
+    # match - mirrors mp_raw_code_load()'s own arch == MP_NATIVE_ARCH_NONE
+    # fast path in py/persistentcode.c, which skips the sub-version check
+    # entirely for non-native code (sub-version only encodes native-code ABI
+    # changes and is meaningless for portable bytecode).
+    if (tag >> 10) & 0xF == 0:
+        return (tag & _MPY_VERSION_MASK) == (device_mpy & _MPY_VERSION_MASK)
+    if (tag & _MPY_BASE_MASK) != (device_mpy & _MPY_BASE_MASK):
+        return False
+    return (tag >> 16) & (device_mpy >> 16) == (tag >> 16)
+
 
 # This implements os.makedirs(os.dirname(path))
 def _ensure_path_exists(path):
@@ -112,7 +135,10 @@ def _install_json(package_json_url, index, target, version, mpy):
         package_json = response.json()
     finally:
         response.close()
-    for target_path, short_hash in package_json.get("hashes", ()):
+    for entry in package_json.get("hashes", ()):
+        target_path, short_hash = entry[0], entry[1]
+        if len(entry) > 2 and not _mpy_tag_ok(entry[2]):
+            continue
         fs_target_path = target + "/" + target_path
         if _check_exists(fs_target_path, short_hash):
             print("Exists:", fs_target_path)
@@ -122,7 +148,10 @@ def _install_json(package_json_url, index, target, version, mpy):
                 print("File not found: {} {}".format(target_path, short_hash))
                 return False
     base_url = package_json_url.rpartition("/")[0]
-    for target_path, url in package_json.get("urls", ()):
+    for entry in package_json.get("urls", ()):
+        target_path, url = entry[0], entry[1]
+        if len(entry) > 2 and not _mpy_tag_ok(entry[2]):
+            continue
         fs_target_path = target + "/" + target_path
         is_full_url = any(url.startswith(p) for p in _ALLOWED_MIP_URL_PREFIXES)
         if base_url and not is_full_url:

micropython/mip/test_mip_tag.py

@@ -0,0 +1,31 @@
+from mip import _mpy_tag_ok
+
+# Exact match on version/sub-version/arch (bits 0-15), no arch-flags.
+assert _mpy_tag_ok(0x0A06, 0x0A06) is True
+
+# Different version/sub-version/arch -> reject.
+assert _mpy_tag_ok(0x0A06, 0x0B06) is False
+assert _mpy_tag_ok(0x0A06, 0x0A07) is False
+
+# Arch-flags (bits 16+): tag's required flags must be a subset of the
+# device's, not an exact match.
+assert _mpy_tag_ok(0x0A06 | (0b001 << 16), 0x0A06 | (0b011 << 16)) is True
+assert _mpy_tag_ok(0x0A06 | (0b011 << 16), 0x0A06 | (0b001 << 16)) is False
+assert _mpy_tag_ok(0x0A06, 0x0A06 | (0b111 << 16)) is True  # tag needs nothing
+assert _mpy_tag_ok(0x0A06 | (0b111 << 16), 0x0A06) is False  # device supports nothing
+
+# No _mpy support on the device (e.g. bytecode-only build).
+assert _mpy_tag_ok(0x0A06, None) is False
+
+# Bytecode-only tag (arch nibble 0): only the major version must match --
+# sub-version and arch are ignored, mirroring mp_raw_code_load()'s own
+# fast path for non-native code in py/persistentcode.c.
+_BYTECODE_TAG = 0x0206  # version=6, sub-version=2, arch=NONE(0)
+assert _mpy_tag_ok(_BYTECODE_TAG, 0x0A06) is True  # same major, different sub/arch on device
+assert _mpy_tag_ok(_BYTECODE_TAG, 0x0107) is False  # different major version
+
+# Omitting device_mpy falls back to sys.implementation._mpy; just check it
+# runs without raising, since that value depends on the build running the test.
+_mpy_tag_ok(0x0A06)
+
+print("PASS")

tools/build.py

@@ -99,6 +99,10 @@
 #   "v": 1,     <-- file format version
 #   "hashes": [
 #     ["aioble/server.mpy", "e39dbf64"],
+#     ["aioble/server.mpy", "e39dbf64", 2822],  # optional 3rd element: only
+#                                                # installed if it matches the
+#                                                # device's sys.implementation._mpy
+#                                                # (not emitted by this script)
 #     ...
 #   ],
 #   "urls": [   <-- not used by micropython-lib packages

tools/ci.sh

@@ -50,13 +50,15 @@ function ci_package_tests_setup_lib {
     $CP -r python-stdlib/unittest/unittest "${VIRTUAL_ENV}/lib/"
     $CP -r python-stdlib/unittest-discover/unittest "${VIRTUAL_ENV}/lib/"
     $CP unix-ffi/ffilib/ffilib.py "${VIRTUAL_ENV}/lib/"
+    $CP -r python-ecosys/requests/requests "${VIRTUAL_ENV}/lib/"
     tree "${VIRTUAL_ENV}"
 }
 
 function ci_package_tests_run {
     export MICROPYPATH
     for test in \
         micropython/drivers/storage/sdcard/sdtest.py \
+        micropython/mip/test_mip_tag.py \
         micropython/umqtt.simple/test_umqtt_simple.py \
         micropython/xmltok/test_xmltok.py \
         python-ecosys/requests/test_requests.py \