arcaneum/benchmarks/results/cuda-apple-m2-pro-20260804.json at main · cwensel/arcaneum · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
{
  "backend": {
    "device": "cuda",
    "dimensions": 0,
    "model": "jina-code-st",
    "name": "pytorch-cuda-worker",
    "precision": "float32"
  },
  "correctness": {
    "finite": false,
    "max_absolute_error": 0.0,
    "mean_absolute_error": 0.0,
    "minimum_cosine_similarity": 0.0,
    "passed": false,
    "reference_backend": "pytorch-cpu-worker",
    "shape": [
      0,
      0
    ]
  },
  "environment": {
    "architecture": "arm64",
    "cpu": "arm",
    "dependencies": {
      "numpy": "2.2.6",
      "psutil": "7.1.3",
      "sentence-transformers": "5.2.0",
      "torch": "2.8.0",
      "transformers": "4.53.3"
    },
    "logical_cpu_count": 12,
    "memory_bytes": 34359738368,
    "nvidia_smi": "unavailable",
    "os": "Darwin",
    "os_release": "25.6.0",
    "python": "3.12.13"
  },
  "fixture": {
    "length_classes": [
      "long",
      "medium",
      "oversized",
      "short"
    ],
    "manifest_sha256": "da62785afe275cff1b607cebb62f415a9c21089c842a4fe18a86c45e8199e165",
    "manifest_version": "1.0.0",
    "total_texts": 15,
    "unique_texts": 4
  },
  "harness_version": "1.0.0",
  "performance": {
    "cold_start_seconds": 0.0,
    "cuda_peak_allocated_bytes": null,
    "cuda_peak_reserved_bytes": null,
    "latency_seconds": {
      "p50": 0.0,
      "p95": 0.0,
      "samples": []
    },
    "peak_rss_bytes": 40108032,
    "throughput_texts_per_second": 0.0,
    "warm_iterations": 0,
    "warm_total_seconds": 0.0
  },
  "qualification": {
    "containment_evidence": {
      "bounded_oom_retries": "tests/unit/benchmarks/test_cuda_qualification.py",
      "timeout_reaps_worker": "tests/embeddings/test_worker_protocol.py",
      "validated": true
    },
    "decision": "experimental",
    "failure_policy": {
      "promotion_requires_zero_failures": true,
      "qualification_worker_restarts": 0,
      "timeout_reaps_worker": true,
      "worker_oom_retries": 2
    },
    "reason": "WorkerCrashedError: PyTorch CUDA is not available; qualification cannot run",
    "soak_target_batches": null,
    "soak_target_seconds": 10800,
    "soak_target_texts": 100000,
    "speedup_gate": 1.25,
    "token_budget": {
      "estimator": "conservative-utf8-bytes-divided-by-four",
      "max_actual_tokens": 8192,
      "max_batch_size": 8,
      "max_padded_tokens": 8192,
      "max_sequence_tokens": 8192
    }
  },
  "reliability": {
    "attempted_batches": 0,
    "completed_batches": 0,
    "completed_soak_texts": 0,
    "failures": 1,
    "fallbacks": 0,
    "restarts": 0,
    "soak_seconds": 0.0
  },
  "run": {
    "id": "20260804T230445Z-cuda",
    "recorded_at": "2026-08-04T23:04:45.194720+00:00",
    "status": "inconclusive"
  },
  "schema_version": "1.0.0"
}