Merge pull request #109 from nvt/bytecode/v6-header-rev

bytecode: upgrade to v6 with more descriptive header

Author: nvt

core/vm-loader.c

@@ -253,7 +253,7 @@ read_program(const char *name)
 {
   vm_program_t *program;
   vm_loader_handle_t handle;
-  unsigned char buf[VM_HEADER_SIZE];
+  unsigned char buf[VM_HEADER_FIXED_SIZE];
   unsigned i;
   unsigned code_size;
 
@@ -279,30 +279,60 @@ read_program(const char *name)
   program->exec_count = NULL;
 #endif
 
-  if(extract_program_name(&program->name, name) == 0) {
-    VM_DEBUG(VM_DEBUG_LOW, "Failed to extract the program name from \"%s\"",
-             name);
-    goto error;
-  }
-
-  VM_DEBUG(VM_DEBUG_LOW, "Loading program \"%s\"", program->name);
-
-  if(VM_LOADER_READ(handle, buf, 3) != 3) {
+  if(VM_LOADER_READ(handle, buf, VM_HEADER_FIXED_SIZE) !=
+     VM_HEADER_FIXED_SIZE) {
     VM_DEBUG(VM_DEBUG_LOW, "Read error on program header");
     goto error;
   }
 
-  if(buf[0] != VM_FILE_ID1 || buf[1] != VM_FILE_ID2) {
+  if(buf[VM_HEADER_OFFSET_MAGIC1] != VM_FILE_ID1 ||
+     buf[VM_HEADER_OFFSET_MAGIC2] != VM_FILE_ID2) {
     VM_DEBUG(VM_DEBUG_LOW, "%s: invalid program header", name);
     goto error;
   }
 
-  if(buf[2] != VM_BYTECODE_VERSION) {
-    VM_DEBUG(VM_DEBUG_LOW, "%s: unsupported bytecode version %d",
-             name, buf[2]);
-    goto error;
+  {
+    uint16_t version =
+        (uint16_t)buf[VM_HEADER_OFFSET_VERSION]
+      | ((uint16_t)buf[VM_HEADER_OFFSET_VERSION + 1] << 8);
+    if(version != VM_BYTECODE_VERSION) {
+      VM_DEBUG(VM_DEBUG_LOW, "%s: unsupported bytecode version %u",
+               name, (unsigned)version);
+      goto error;
+    }
+  }
+
+  uint32_t total_len =
+      (uint32_t)buf[VM_HEADER_OFFSET_TOTAL_LEN]
+    | ((uint32_t)buf[VM_HEADER_OFFSET_TOTAL_LEN + 1] << 8)
+    | ((uint32_t)buf[VM_HEADER_OFFSET_TOTAL_LEN + 2] << 16)
+    | ((uint32_t)buf[VM_HEADER_OFFSET_TOTAL_LEN + 3] << 24);
+
+  /* Read the variable-length program name. */
+  {
+    unsigned name_len = buf[VM_HEADER_OFFSET_NAME_LEN];
+    if(name_len > 0) {
+      char *header_name = VM_MALLOC(name_len + 1);
+      if(header_name == NULL) {
+        goto error;
+      }
+      if(VM_LOADER_READ(handle, header_name, name_len) !=
+         (vm_loader_offset_t)name_len) {
+        VM_DEBUG(VM_DEBUG_LOW, "Read error on program name");
+        VM_FREE(header_name);
+        goto error;
+      }
+      header_name[name_len] = '\0';
+      program->name = header_name;
+    } else if(extract_program_name(&program->name, name) == 0) {
+      VM_DEBUG(VM_DEBUG_LOW,
+               "Failed to extract the program name from \"%s\"", name);
+      goto error;
+    }
   }
 
+  VM_DEBUG(VM_DEBUG_LOW, "Loading program \"%s\"", program->name);
+
   if(read_table(&program->strings, handle) == 0) {
     VM_DEBUG(VM_DEBUG_LOW, "Failed to read the string table");
     goto error;
@@ -420,6 +450,22 @@ read_program(const char *name)
     goto error;
   }
 
+  /* Truncation / padding check: the position after the last body byte
+     must equal the total length declared in the header. */
+  {
+    vm_loader_offset_t end_offset = VM_LOADER_SEEK_RELATIVE(handle, 0);
+    if(end_offset == (vm_loader_offset_t)-1) {
+      VM_DEBUG(VM_DEBUG_LOW, "loader:seek failed");
+      goto error;
+    }
+    if((uint32_t)end_offset != total_len) {
+      VM_DEBUG(VM_DEBUG_LOW,
+               "%s: total-length mismatch (header says %u, parsed %u)",
+               name, (unsigned)total_len, (unsigned)end_offset);
+      goto error;
+    }
+  }
+
   VM_DEBUG(VM_DEBUG_MEDIUM, "Loading bytecode consisting of %u byte%s",
            code_size, code_size != 1 ? "s" : "");
 

doc/bytecode-format.md

@@ -8,18 +8,25 @@ VeloxVM uses a custom bytecode format designed for efficient storage and executi
 
 ### Magic Number and Header
 
-VeloxVM bytecode files (.vm) begin with a 3-byte header:
+VeloxVM bytecode files (.vm) begin with a 9-byte fixed prefix followed by an N-byte program name:
 
 ```
 Offset  Size  Description
 ------  ----  -----------
 0x00    1     File ID 1: 0x5E (94 decimal)
 0x01    1     File ID 2: 0xB5 (181 decimal)
-0x02    1     Bytecode version (currently 5; see `VM_BYTECODE_VERSION` in `include/vm-bytecode.h`)
+0x02    2     Bytecode version (uint16 LE; currently 6, see `VM_BYTECODE_VERSION` in `include/vm-bytecode.h`)
+0x04    4     Total file length in bytes (uint32 LE) — header + body
+0x08    1     Program name length (N, 0..255)
+0x09    N     Program name (UTF-8, no terminator)
 ```
 
 The magic number `0x5E 0xB5` (or `0xB55E` in little-endian short format) can be used to identify VeloxVM bytecode files with the `file` utility.
 
+The **total length** field is the size of the entire file in bytes including the header. The loader verifies it against the actual file size after parsing the body and rejects mismatches (truncation or padding).
+
+The **program name** is the canonical identity used by logs, profiler output, and the control plane. If N is 0 the loader falls back to deriving the name from the filename (with `.vm` stripped). Encoders typically populate the field from the source file's basename.
+
 ### File Structure
 
 After the header, the file contains three variable-length tables followed by a captures block:

include/vm-bytecode.h

@@ -35,14 +35,25 @@
 
 #include <stdint.h>
 
-/* The header consists of two bytes for the file ID and a byte for the
-   bytecode version. */
-#define VM_HEADER_SIZE 3
+/* v6 header layout (9 fixed bytes + N-byte program name):
+     0x00  2  Magic (VM_FILE_ID1, VM_FILE_ID2)
+     0x02  2  Bytecode version (uint16 LE)
+     0x04  4  Total file length in bytes (uint32 LE)
+     0x08  1  Program name length N (0..255)
+     0x09  N  Program name (UTF-8, no terminator)
+   The body (tables + captures + expression bytecode) follows the name. */
+#define VM_HEADER_FIXED_SIZE 9
+#define VM_HEADER_OFFSET_MAGIC1      0
+#define VM_HEADER_OFFSET_MAGIC2      1
+#define VM_HEADER_OFFSET_VERSION     2
+#define VM_HEADER_OFFSET_TOTAL_LEN   4
+#define VM_HEADER_OFFSET_NAME_LEN    8
+#define VM_HEADER_OFFSET_NAME        9
 
 #define VM_FILE_ID1 94
 #define VM_FILE_ID2 181
 
-#define VM_BYTECODE_VERSION 5
+#define VM_BYTECODE_VERSION 6
 
 #define VM_TOKEN_ATOM 0
 #define VM_TOKEN_FORM 1

languages/python/pyvelox/bytecode.py

@@ -108,7 +108,7 @@ class Bytecode:
     def __init__(self):
         # File header
         self.magic = 0x5EB5  # 0x5E, 0xB5 in little-endian
-        self.version = 5  # VM_BYTECODE_VERSION from include/vm-bytecode.h
+        self.version = 6  # VM_BYTECODE_VERSION from include/vm-bytecode.h
 
         # Symbol table management
         self.symbol_table = SymbolTable()

languages/python/pyvelox/disassembler.py

@@ -71,16 +71,24 @@ def read_bytes(self, n: int) -> bytes:
 
     def disassemble(self) -> str:
         """Disassemble the entire bytecode file."""
-        # Read header
+        # Read v6 header (9 fixed bytes + N-byte program name).
         magic1 = self.read_byte()
         magic2 = self.read_byte()
-        version = self.read_byte()
-
         if magic1 != 0x5E or magic2 != 0xB5:
             raise ValueError(f"Invalid magic bytes: 0x{magic1:02X}{magic2:02X}")
 
+        version = self.read_u16()
+        total_len = (self.read_byte() | (self.read_byte() << 8)
+                     | (self.read_byte() << 16) | (self.read_byte() << 24))
+        name_len = self.read_byte()
+        prog_name = self.read_bytes(name_len).decode('utf-8',
+                                                    errors='replace') \
+            if name_len > 0 else ''
+
         output = [f"; VeloxVM Bytecode Disassembly"]
-        output.append(f"; Magic: 0x{magic1:02X}{magic2:02X}, Version: {version}")
+        output.append(
+            f"; Magic: 0x{magic1:02X}{magic2:02X}, Version: {version}, "
+            f"Total length: {total_len}, Program: {prog_name!r}")
         output.append("")
 
         # Read string table

languages/python/pyvelox/writer.py

@@ -33,6 +33,7 @@
 following the VeloxVM bytecode format specification.
 """
 
+import io
 import struct
 from pathlib import Path
 from typing import Union, List
@@ -41,37 +42,49 @@
 
 def write_bytecode_file(path: Union[str, Path], bc: Bytecode):
     """
-    Write bytecode to a .vm file.
-
-    File format:
-    - Header (3 bytes): 0x5E, 0xB5, version
+    Write bytecode to a .vm file in the v6 format.
+
+    File layout:
+    - Header (9 fixed bytes + N-byte program name):
+        0x00  2  Magic (0x5E 0xB5)
+        0x02  2  Version (uint16 LE)
+        0x04  4  Total file length (uint32 LE)
+        0x08  1  Program name length N
+        0x09  N  Program name (UTF-8, no terminator)
     - String table: count (16-bit) + items (16-bit length + data)
     - Symbol table: count (16-bit) + items (16-bit length + data)
     - Expression table: count (16-bit) + items (16-bit length + data)
-    - Captures section: count (16-bit) + entries. Each entry is
-      (length:uint16, expr_id:uint16, symbol_id:uint16 ...). The entry
-      length is the byte count of the entry's payload, i.e.
-      2 + 2 * len(symbol_ids).
+    - Captures section: count (16-bit) + entries (length:uint16,
+      expr_id:uint16, symbol_id:uint16 ...).
 
     Args:
         path: Output file path
         bc: Bytecode container to write
     """
-    with open(path, 'wb') as f:
-        # Write header (3 bytes)
-        f.write(bytes([0x5E, 0xB5, bc.version]))
-
-        # Write string table
-        _write_table(f, bc.symbol_table.strings, _encode_string_item)
+    path = Path(path)
+    prog_name = path.stem.encode('utf-8')
+    if len(prog_name) > 255:
+        raise ValueError(f"Program name too long ({len(prog_name)} bytes, "
+                         f"max 255): {path.stem}")
+
+    # Build the body in memory so we can fill in the total-length header
+    # field before any bytes hit disk.
+    body = io.BytesIO()
+    _write_table(body, bc.symbol_table.strings, _encode_string_item)
+    _write_table(body, bc.symbol_table.symbols, _encode_string_item)
+    _write_table(body, bc.expressions, _encode_bytes_item)
+    _write_captures_section(body, bc.captures)
+    body_bytes = body.getvalue()
+
+    total_len = 9 + len(prog_name) + len(body_bytes)
 
-        # Write symbol table
-        _write_table(f, bc.symbol_table.symbols, _encode_string_item)
-
-        # Write expression table
-        _write_table(f, bc.expressions, _encode_bytes_item)
-
-        # Write captures section
-        _write_captures_section(f, bc.captures)
+    with open(path, 'wb') as f:
+        f.write(bytes([0x5E, 0xB5]))            # Magic
+        f.write(struct.pack('<H', bc.version))  # Version (uint16 LE)
+        f.write(struct.pack('<I', total_len))   # Total length (uint32 LE)
+        f.write(struct.pack('<B', len(prog_name)))  # Name length
+        f.write(prog_name)                      # Name bytes
+        f.write(body_bytes)
 
 
 def _write_captures_section(f, captures):
@@ -150,12 +163,16 @@ def read_bytecode_file(path: Union[str, Path]) -> Bytecode:
     has the authoritative bytecode loader.
     """
     with open(path, 'rb') as f:
-        # Read header
+        # Read v6 fixed prefix (9 bytes)
         magic_bytes = f.read(2)
         if magic_bytes != bytes([0x5E, 0xB5]):
             raise ValueError(f"Invalid magic number: {magic_bytes.hex()}")
 
-        version = f.read(1)[0]
+        version = struct.unpack('<H', f.read(2))[0]
+        _total_len = struct.unpack('<I', f.read(4))[0]
+        name_len = struct.unpack('<B', f.read(1))[0]
+        if name_len > 0:
+            _ = f.read(name_len)  # consume program name
 
         # Create bytecode container
         bc = Bytecode()

languages/scheme/bytecode.rkt

@@ -79,7 +79,7 @@
 ;; Create new bytecode container
 (define (make-bytecode)
   (bytecode #x5EB5        ; Magic number
-            5             ; Version
+            6             ; Version
             '()           ; strings-rev
             (make-hash)   ; strings-index
             '()           ; symbols-rev
@@ -405,40 +405,59 @@
 ;; ============================================================================
 
 (define (write-bytecode-file filename bc)
+  ;; v6 header: 9 fixed bytes + N-byte program name.
+  ;; The program name is derived from the destination filename's basename.
+  (define prog-name
+    (let* ([p (if (path? filename) filename (string->path filename))]
+           [base (file-name-from-path p)])
+      (path->string (path-replace-extension base #""))))
+  (define name-bytes (string->bytes/utf-8 prog-name))
+  (define name-len (bytes-length name-bytes))
+  (when (> name-len 255)
+    (error 'write-bytecode-file
+           "Program name too long (~a bytes, max 255): ~a"
+           name-len prog-name))
+
+  ;; Build the body in memory so we can compute the total length for the
+  ;; header before flushing to disk.
+  (define body
+    (with-output-to-bytes
+     (lambda ()
+       (write-table (bytecode-strings bc) write-string-entry)
+       (write-table (bytecode-symbols bc) write-symbol-entry)
+       (write-table (bytecode-expressions bc) write-expr-entry)
+       ;; Captures section: count of {expr_id, [symbol_id, ...]} entries.
+       ;; compile-lambda populates bc's captures-list as it analyzes lambda
+       ;; bodies for free variables; here we serialize that out.
+       (let ([entries (bytecode-captures-list bc)])
+         (write-u16 (length entries))
+         (for ([entry entries])
+           (let* ([expr-id (car entry)]
+                  [sym-ids (cdr entry)]
+                  [entry-bytes (+ 2 (* 2 (length sym-ids)))])
+             (write-u16 entry-bytes)
+             (write-u16 expr-id)
+             (for ([sid sym-ids])
+               (write-u16 sid))))))))
+
+  (define total-len (+ 9 name-len (bytes-length body)))
+
   (with-output-to-file filename
     #:exists 'replace
     #:mode 'binary
     (lambda ()
-      ;; Magic number
+      ;; Magic (bytes 0-1, big-endian for `file(1)` compatibility)
       (write-bytes (integer->integer-bytes (bytecode-magic bc) 2 #f #t))
-
-      ;; Version
-      (write-byte (bytecode-version bc))
-
-      ;; String table
-      (write-table (bytecode-strings bc) write-string-entry)
-
-      ;; Symbol table
-      (write-table (bytecode-symbols bc) write-symbol-entry)
-
-      ;; Expression table
-      (write-table (bytecode-expressions bc) write-expr-entry)
-
-      ;; Captures section: count of {expr_id, [symbol_id, ...]} entries.
-      ;; Each entry is uint16-length-prefixed, then uint16 expr_id +
-      ;; uint16 symbol_id per captured free variable. compile-lambda
-      ;; populates bc's captures-list as it analyzes lambda bodies for
-      ;; free variables; here we serialize that out.
-      (let ([entries (bytecode-captures-list bc)])
-        (write-u16 (length entries))
-        (for ([entry entries])
-          (let* ([expr-id (car entry)]
-                 [sym-ids (cdr entry)]
-                 [entry-bytes (+ 2 (* 2 (length sym-ids)))])
-            (write-u16 entry-bytes)
-            (write-u16 expr-id)
-            (for ([sid sym-ids])
-              (write-u16 sid))))))))
+      ;; Version (bytes 2-3, uint16 LE)
+      (write-bytes (integer->integer-bytes (bytecode-version bc) 2 #f #f))
+      ;; Total file length (bytes 4-7, uint32 LE)
+      (write-bytes (integer->integer-bytes total-len 4 #f #f))
+      ;; Program name length (byte 8)
+      (write-byte name-len)
+      ;; Program name (bytes 9..9+N)
+      (write-bytes name-bytes)
+      ;; Body
+      (write-bytes body))))
 
 (define (write-table items write-item)
   (write-u16 (length items))