Skip to content

Commit 741032c

Browse files
aittalamclaude
aittalam
and
claude
committed
transcribefile: Metal GPU support via llamafile's runtime loader
Wires llamafile's GPU machinery into transcribefile so parakeet runs on Apple Silicon GPUs: 51x realtime cold / ~120x warm vs 17x CPU on jfk.wav, with transcripts byte-identical to the CPU backend. How it works: - transcribefile links llamafile.o + gpu.a + zip.o + check_cpu.o (the whisperfile TOOL_LLAMAFILE_OBJS pattern) plus $(LLAMAFILE_METAL_SOURCES), so metal.c can extract the bundled (patched) ggml sources from the zip store and compile ggml-metal.dylib at runtime, sharing the cached copy under ~/.llamafile with llamafile itself. - ggml_backend_register resolves from transcribe.cpp.a, so the loaded backend registers into transcribe.cpp's own ggml device registry; the upstream CLI's --backend / --device / --list-devices logic works unchanged. - transcribefile/main.cpp gains load_gpu_backends(): --backend cpu / cpu_accel never touch the GPU machinery; auto / metal try Metal and degrade silently to CPU when unavailable, leaving backend-missing errors to transcribe.cpp's own reporting. - 7 new transcribe.cpp patches port the host-side ggml ABI surface (GGML_CALL calling-convention annotations + the free_struct buffer callback) from the llama.cpp patch set, unmodified except for paths -- both trees vendor ggml 0.15.2. This keeps the interface structs exchanged with the (llama.cpp-built) backend dylibs ABI-identical, which is a no-op for Metal on macOS today but load-bearing on Windows (ms_abi) and for the Vulkan/CUDA follow-ups. - transcribefile_smoke.sh gains a Metal layer: asserts an MTL device is actually selected and that Metal/CPU transcripts match; skips cleanly off Apple Silicon or when no Metal device registers. Vulkan and CUDA are intentionally not wired up yet (follow-up PRs); on this branch --backend vulkan/cuda report "backend requested but not available" via transcribe.cpp as before. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 1281813 commit 741032c

10 files changed

Lines changed: 1024 additions & 4 deletions

‎tests/transcribefile_smoke.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,35 @@ echo "$out" | grep -q 'realtime:' || fail "parakeet output missing 'realtime:' l
7474
echo "$out" | grep -qi 'country' || fail "parakeet transcription missing 'country'"
7575
realtime=$(echo "$out" | grep -oE 'realtime:[[:space:]]+[0-9]+x' | head -1 || true)
7676
pass "parakeet transcribes jfk.wav (${realtime:-realtime: ?})"
77+
78+
echo "[smoke] layer 3 — metal backend"
79+
80+
# Metal is macOS/Apple-Silicon only. Elsewhere (and on macs where the
81+
# runtime dylib build isn't possible, e.g. no Xcode CLT) the contract is
82+
# graceful degradation to CPU, so absence of a metal device is a SKIP,
83+
# not a failure.
84+
if [ "$(uname -s)" != "Darwin" ] || [ "$(uname -m)" != "arm64" ]; then
85+
echo " SKIP metal: requires macOS on Apple Silicon" >&2
86+
exit 0
87+
fi
88+
if ! "$APE" --list-devices 2>/dev/null | grep -q 'kind=metal'; then
89+
echo " SKIP metal: no metal device registered" \
90+
"(Xcode command-line tools missing?)" >&2
91+
exit 0
92+
fi
93+
94+
# Explicit --backend metal must actually select an MTL device and produce
95+
# the same transcript as the CPU backend (drop the backend/realtime lines
96+
# before comparing; they legitimately differ).
97+
mtl=$("$APE" --backend metal -q -m "$MODEL" "$SAMPLE" 2>/dev/null) \
98+
|| fail "metal run exited non-zero"
99+
echo "$mtl" | grep -qE 'backend:[[:space:]]+MTL' || fail "metal run did not select an MTL device"
100+
echo "$mtl" | grep -qi 'country' || fail "metal transcription missing 'country'"
101+
pass "parakeet transcribes jfk.wav on metal"
102+
103+
cpu=$("$APE" --backend cpu -q -m "$MODEL" "$SAMPLE" 2>/dev/null) \
104+
|| fail "cpu run exited non-zero"
105+
mtl_text=$(echo "$mtl" | grep -vE 'backend:|realtime:')
106+
cpu_text=$(echo "$cpu" | grep -vE 'backend:|realtime:')
107+
[ "$mtl_text" = "$cpu_text" ] || fail "metal and cpu transcripts differ"
108+
pass "metal and cpu transcripts match"

‎transcribe.cpp.patches/patches/ggml_include_ggml-backend.h.patch‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
2+
--- a/transcribe.cpp/ggml/include/ggml-backend.h
3+
+++ b/transcribe.cpp/ggml/include/ggml-backend.h
4+
@@ -17,6 +17,16 @@
5+
# define GGML_BACKEND_API extern
6+
#endif
7+
8+
+#ifdef GGML_MULTIPLATFORM
9+
+# ifdef _MSC_VER
10+
+# define GGML_CALL /* ms_abi is the default on MSVC */
11+
+# else
12+
+# define GGML_CALL __attribute__((__ms_abi__))
13+
+# endif
14+
+#else
15+
+# define GGML_CALL
16+
+#endif
17+
+
18+
#ifdef __cplusplus
19+
extern "C" {
20+
#endif
21+
@@ -208,19 +218,19 @@ extern "C" {
22+
typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);
23+
24+
// Split buffer type for tensor parallelism (old)
25+
- typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
26+
+ typedef ggml_backend_buffer_type_t (GGML_CALL *ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
27+
// Set the number of threads for the backend
28+
- typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
29+
+ typedef void (GGML_CALL *ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
30+
// Get additional buffer types provided by the device (returns a NULL-terminated array)
31+
- typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
32+
+ typedef ggml_backend_buffer_type_t * (GGML_CALL *ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
33+
// Set the abort callback for the backend
34+
- typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
35+
+ typedef void (GGML_CALL *ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
36+
// Get a list of feature flags supported by the backend (returns a NULL-terminated array)
37+
struct ggml_backend_feature {
38+
const char * name;
39+
const char * value;
40+
};
41+
- typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
42+
+ typedef struct ggml_backend_feature * (GGML_CALL *ggml_backend_get_features_t)(ggml_backend_reg_t reg);
43+
44+
//
45+
// Backend registry

‎transcribe.cpp.patches/patches/ggml_include_ggml-cpu.h.patch‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h
2+
--- a/transcribe.cpp/ggml/include/ggml-cpu.h
3+
+++ b/transcribe.cpp/ggml/include/ggml-cpu.h
4+
@@ -131,9 +131,9 @@ extern "C" {
5+
GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void);
6+
7+
GGML_BACKEND_API bool ggml_backend_is_cpu (ggml_backend_t backend);
8+
- GGML_BACKEND_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);
9+
- GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);
10+
- GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
11+
+ GGML_BACKEND_API void GGML_CALL ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);
12+
+ GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);
13+
+ GGML_BACKEND_API void GGML_CALL ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
14+
15+
GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref);
16+

0 commit comments

Comments
 (0)