transcribefile: Metal GPU support via llamafile's runtime loader (#1010)

Author: aittalam

diffusionfile/BUILD.mk

@@ -66,8 +66,7 @@ DIFFUSIONFILE_INCLUDES := \
 # ==============================================================================
 
 DIFFUSIONFILE_CPPFLAGS := \
-	$(DIFFUSIONFILE_INCLUDES) \
-	-DLLAMAFILE_VERSION_STRING=\"$(LLAMAFILE_VERSION_STRING)\"
+	$(DIFFUSIONFILE_INCLUDES)
 
 # ==============================================================================
 # Dependencies - llamafile objects for GPU support

diffusionfile/diffusionfile.cpp

@@ -19,11 +19,7 @@
 #include <stdio.h>
 
 #include "llamafile/llamafile.h"
-
-// LLAMAFILE_VERSION_STRING is defined by BUILD.mk
-#ifndef LLAMAFILE_VERSION_STRING
-#define LLAMAFILE_VERSION_STRING "0.0.0-dev"
-#endif
+#include "llamafile/version.h"
 
 // Forward declaration - defined in stable-diffusion.cpp/examples/cli/main.cpp
 // When compiled with -DDIFFUSIONFILE, main.cpp renames main() to diffusion_cli_main()

llamafile/llamafile.c

@@ -458,14 +458,28 @@ static const char *llamafile_get_home_dir(void) {
     return homedir;
 }
 
+// Basename of the per-app dot directory under $HOME. Defaults to
+// "llamafile"; other products (e.g. transcribefile) override it via
+// llamafile_set_app_name() so their cached artifacts — compiled GPU
+// dylibs, extracted ggml sources — live in their own tree and can
+// diverge from llamafile's without overwriting anything.
+static const char *g_app_name = "llamafile";
+
+void llamafile_set_app_name(const char *name) {
+    if (name && *name)
+        g_app_name = name;
+}
+
 /**
  * Returns path of directory for app-specific files.
- * Path includes version number: ~/.llamafile/v/<major>.<minor>.<patch>/
+ * Path includes version number: ~/.<app>/v/<major>.<minor>.<patch>/
+ * where <app> defaults to "llamafile" (see llamafile_set_app_name).
  * This ensures different versions don't overwrite each other's compiled dylibs.
  */
 void llamafile_get_app_dir(char *path, size_t size) {
-    snprintf(path, size, "%s/.llamafile/v/%d.%d.%d/",
+    snprintf(path, size, "%s/.%s/v/%d.%d.%d/",
              llamafile_get_home_dir(),
+             g_app_name,
              LLAMAFILE_MAJOR,
              LLAMAFILE_MINOR,
              LLAMAFILE_PATCH);

llamafile/llamafile.h

@@ -77,6 +77,7 @@ char *llamafile_get_prompt(void);                         // NOT DEFINED
 // USED: Defined in llamafile.c
 bool llamafile_has(char **, const char *);
 void llamafile_get_app_dir(char *, size_t);
+void llamafile_set_app_name(const char *); // app dir basename, default "llamafile"
 bool llamafile_extract(const char *, const char *);
 int llamafile_is_file_newer_than(const char *, const char *);
 

tests/transcribefile_smoke.sh

@@ -1,19 +1,23 @@
 #!/usr/bin/env bash
 # transcribefile_smoke.sh — regression smoke for the transcribefile APE.
 #
-# Two layers:
+# Three layers:
 #   1. Model-free probes — always run. Catches CLI argv / WAV-loader /
 #      help-text regressions without needing any model artifact.
 #   2. Parakeet end-to-end — gated on TRANSCRIBEFILE_PARAKEET_GGUF. Skipped
 #      (with a warning, not a failure) when the model isn't available, so
 #      `make check` stays green on machines without the model.
+#   3. Metal backend — needs layer 2's model plus macOS on Apple Silicon
+#      with a registered Metal device; skipped elsewhere. Asserts that
+#      --backend metal selects an MTL device and that the Metal and CPU
+#      transcripts match exactly.
 #
 # Usage:
 #   tests/transcribefile_smoke.sh <path-to-transcribefile-binary>
 #
 # Env:
 #   TRANSCRIBEFILE_PARAKEET_GGUF  Path to a parakeet GGUF; if unset or the
-#                                 file is missing, layer 2 is skipped.
+#                                 file is missing, layers 2 and 3 are skipped.
 
 set -euo pipefail
 
@@ -74,3 +78,43 @@ echo "$out" | grep -q 'realtime:' || fail "parakeet output missing 'realtime:' l
 echo "$out" | grep -qi 'country' || fail "parakeet transcription missing 'country'"
 realtime=$(echo "$out" | grep -oE 'realtime:[[:space:]]+[0-9]+x' | head -1 || true)
 pass "parakeet transcribes jfk.wav (${realtime:-realtime: ?})"
+
+echo "[smoke] layer 3 — metal backend"
+
+# Metal is macOS/Apple-Silicon only. Elsewhere (and on macs where the
+# runtime dylib build isn't possible, e.g. no Xcode CLT) the contract is
+# graceful degradation to CPU, so absence of a metal device is a SKIP,
+# not a failure.
+if [ "$(uname -s)" != "Darwin" ] || [ "$(uname -m)" != "arm64" ]; then
+    echo "  SKIP metal: requires macOS on Apple Silicon" >&2
+    exit 0
+fi
+if ! "$APE" --list-devices 2>/dev/null | grep -q 'kind=metal'; then
+    echo "  SKIP metal: no metal device registered" \
+         "(Xcode command-line tools missing?)" >&2
+    exit 0
+fi
+
+# Explicit --backend metal must actually select an MTL device and produce
+# the same transcript TEXT as the CPU backend. Only the text: fields are
+# compared: word timestamps and token probabilities may legitimately
+# differ across backends — different kernels and accumulation orders give
+# slightly different logits, so near-tie decisions (a timestamp on a
+# frame boundary, a probability rounding) can flip, and quantized models
+# amplify this. The decoded text is expected to be stable.
+mtl=$("$APE" --backend metal -q -m "$MODEL" "$SAMPLE" 2>/dev/null) \
+    || fail "metal run exited non-zero"
+echo "$mtl" | grep -qE 'backend:[[:space:]]+MTL' || fail "metal run did not select an MTL device"
+echo "$mtl" | grep -qi 'country' || fail "metal transcription missing 'country'"
+pass "parakeet transcribes jfk.wav on metal"
+
+cpu=$("$APE" --backend cpu -q -m "$MODEL" "$SAMPLE" 2>/dev/null) \
+    || fail "cpu run exited non-zero"
+mtl_text=$(echo "$mtl" | grep '^text:' || true)
+cpu_text=$(echo "$cpu" | grep '^text:' || true)
+[ -n "$mtl_text" ] || fail "metal output has no text: line"
+if [ "$mtl_text" != "$cpu_text" ]; then
+    printf 'metal:\n%s\ncpu:\n%s\n' "$mtl_text" "$cpu_text" >&2
+    fail "metal and cpu transcript text differs"
+fi
+pass "metal and cpu transcripts match"

transcribe.cpp.patches/llamafile-files/BUILD.mk

@@ -45,7 +45,15 @@ TRANSCRIBE_LIB_INCS := \
 # Common Compiler Flags
 # ==============================================================================
 
+# GGML_MULTIPLATFORM turns the GGML_CALL annotations (ported from the
+# llama.cpp patch set) into __ms_abi__ on x86-64, keeping the backend
+# interface structs ABI-identical to the GPU dylibs, which are built
+# with the same define (a no-op on aarch64, where the attribute doesn't
+# exist — hence the global -Wno-attributes). It is already inherited
+# from the global CPPFLAGS_ in build/config.mk; it is repeated here, as
+# in llama.cpp's BUILD.mk, so the cross-DSO ABI contract is explicit.
 TRANSCRIBE_GGML_CPPFLAGS := \
+	-DGGML_MULTIPLATFORM \
 	-DGGML_USE_CPU \
 	-DGGML_CPU_GENERIC \
 	-DGGML_USE_CPU_REPACK \
@@ -63,6 +71,7 @@ TRANSCRIBE_GGML_CPPFLAGS := \
 # zlib downstream (NO_ZLIB_COMPATIBLE_NAMES — call sites use the
 # mz_-prefixed API).
 TRANSCRIBE_LIB_CPPFLAGS := \
+	-DGGML_MULTIPLATFORM \
 	-DTRANSCRIBE_BUILD \
 	-DGGML_SCHED_MAX_COPIES=4 \
 	-DMINIZ_NO_INFLATE_APIS \

transcribe.cpp.patches/patches/ggml_include_ggml-backend.h.patch

@@ -0,0 +1,45 @@
+diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
+--- a/transcribe.cpp/ggml/include/ggml-backend.h
++++ b/transcribe.cpp/ggml/include/ggml-backend.h
+@@ -17,6 +17,16 @@
+ #    define GGML_BACKEND_API extern
+ #endif
+ 
++#ifdef GGML_MULTIPLATFORM
++#    ifdef _MSC_VER
++#        define GGML_CALL  /* ms_abi is the default on MSVC */
++#    else
++#        define GGML_CALL __attribute__((__ms_abi__))
++#    endif
++#else
++#    define GGML_CALL
++#endif
++
+ #ifdef  __cplusplus
+ extern "C" {
+ #endif
+@@ -208,19 +218,19 @@ extern "C" {
+     typedef bool   (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);
+ 
+     // Split buffer type for tensor parallelism (old)
+-    typedef ggml_backend_buffer_type_t   (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
++    typedef ggml_backend_buffer_type_t   (GGML_CALL *ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
+     // Set the number of threads for the backend
+-    typedef void                         (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
++    typedef void                         (GGML_CALL *ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
+     // Get additional buffer types provided by the device (returns a NULL-terminated array)
+-    typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
++    typedef ggml_backend_buffer_type_t * (GGML_CALL *ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
+     // Set the abort callback for the backend
+-    typedef void                         (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
++    typedef void                         (GGML_CALL *ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
+     // Get a list of feature flags supported by the backend (returns a NULL-terminated array)
+     struct ggml_backend_feature {
+         const char * name;
+         const char * value;
+     };
+-    typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
++    typedef struct ggml_backend_feature * (GGML_CALL *ggml_backend_get_features_t)(ggml_backend_reg_t reg);
+ 
+     //
+     // Backend registry

transcribe.cpp.patches/patches/ggml_include_ggml-cpu.h.patch

@@ -0,0 +1,16 @@
+diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h
+--- a/transcribe.cpp/ggml/include/ggml-cpu.h
++++ b/transcribe.cpp/ggml/include/ggml-cpu.h
+@@ -131,9 +131,9 @@ extern "C" {
+     GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void);
+ 
+     GGML_BACKEND_API bool ggml_backend_is_cpu                (ggml_backend_t backend);
+-    GGML_BACKEND_API void ggml_backend_cpu_set_n_threads     (ggml_backend_t backend_cpu, int n_threads);
+-    GGML_BACKEND_API void ggml_backend_cpu_set_threadpool    (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);
+-    GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
++    GGML_BACKEND_API void GGML_CALL ggml_backend_cpu_set_n_threads     (ggml_backend_t backend_cpu, int n_threads);
++    GGML_BACKEND_API void            ggml_backend_cpu_set_threadpool    (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);
++    GGML_BACKEND_API void GGML_CALL ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
+ 
+     GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref);
+