evals(community): OsaurusAI/Qwen3.8-Flash-Next-JANG_4M on Apple M5 Max

reports/community/apple-m5-max-OsaurusAI-Qwen3.8-Flash-Next-JANG_4M-20260901.json

@@ -0,0 +1,197 @@
+{
+  "domains" : [
+    "agent_loop",
+    "apple_script",
+    "cache_proof",
+    "capability_claims",
+    "computer_use_loop",
+    "default_agent",
+    "http_api",
+    "memory",
+    "micro_perf",
+    "reasoning_channel",
+    "subagent"
+  ],
+  "generatedAt" : "2026-09-02T02:49:37.356Z",
+  "models" : [
+    {
+      "actionCompletionCoverage" : 0.9705882352941176,
+      "agentLoopDiagnostics" : {
+        "exitCounts" : {
+          "cancelled" : 4,
+          "clarifyRequested" : 2,
+          "emptyResponseExhausted" : 2,
+          "finalResponse" : 86,
+          "iterationCapReached" : 7,
+          "overBudget" : 1
+        },
+        "exitOriginCounts" : {
+          "ordinaryModelFinal" : 79,
+          "typedTerminal" : 7,
+          "unrecorded" : 16
+        },
+        "measuredRows" : 102,
+        "recoveryRetriesTotal" : 5,
+        "rowsWithExitTelemetry" : 102
+      },
+      "cacheTotals" : {
+        "diskL2Evictions" : 485,
+        "diskL2Hits" : 687,
+        "diskL2Misses" : 20109,
+        "diskL2Stores" : 2135,
+        "kvPrefixHits" : 0,
+        "kvPrefixMisses" : 0,
+        "pagedEvictions" : 0,
+        "ssmCompanionHits" : 687,
+        "ssmCompanionMisses" : 0,
+        "ssmCompanionReDerives" : 0
+      },
+      "chatModelPassed" : 255,
+      "chatModelScored" : 289,
+      "correctFileDeliveryRate" : 0.8043478260869565,
+      "environment" : {
+        "caseCount" : 19,
+        "catalogHash" : "2e766b2b52f4cf37",
+        "chip" : "Apple M5 Max",
+        "commit" : "1bbd959",
+        "contributor" : "jax-0n-git",
+        "cpuCores" : 18,
+        "harness" : "osaurus",
+        "judge" : "self-judge",
+        "lowPowerMode" : false,
+        "osVersion" : "26.6.2",
+        "powerSource" : "AC",
+        "runModel" : "OsaurusAI/Qwen3.8-Flash-Next-JANG_4M",
+        "thermalState" : "fair",
+        "totalRamMb" : 131072
+      },
+      "flakyCases" : 1,
+      "harness" : "osaurus",
+      "loopFreeRate" : 0.9117647058823529,
+      "meanAgentLoopWallTimeMs" : 66466.26223769842,
+      "meanCpuPercent" : 124.87489364258133,
+      "meanDecodeTokensPerSecond" : 22.18650959264567,
+      "meanFirstActionMs" : 12750.408082297354,
+      "meanFirstStepContextTokens" : 3315.29702970297,
+      "meanFrictionCalls" : 0.5428571428571428,
+      "meanPromptTokensPerTask" : 16634.962732919255,
+      "meanTotalTokensPerTask" : 15633.570621468927,
+      "meanTtftMs" : 1051.6646425679046,
+      "medianModelSteps" : 4,
+      "modelId" : "OsaurusAI/Qwen3.8-Flash-Next-JANG_4M",
+      "mtpTotals" : {
+        "acceptedDraftTokens" : 6516,
+        "arFallbackTokens" : 0,
+        "bonusTokens" : 1748,
+        "rejectedTokens" : 3009,
+        "stepsWithMTP" : 82,
+        "verifyCalls" : 4757
+      },
+      "peakCpuPercent" : 539.7784071747797,
+      "peakPhysFootprintMb" : 113028.62930297852,
+      "perDomain" : {
+        "agent_loop" : {
+          "errored" : 0,
+          "passed" : 84,
+          "scored" : 102,
+          "skipReasons" : {
+            "other: no AppleScript model installed; applescript/mac_query delegation tools withheld — case skipped" : 3
+          },
+          "skipped" : 3
+        },
+        "apple_script" : {
+          "errored" : 0,
+          "passed" : 26,
+          "scored" : 32,
+          "skipReasons" : {
+            "other: liveProof environment not ready — Automation permission is denied for this process (error -1743). Grant Osaurus access to the target app under System Settings → Privacy & Security → Automation, then re-run locally." : 3
+          },
+          "skipped" : 3
+        },
+        "cache_proof" : {
+          "errored" : 0,
+          "passed" : 5,
+          "scored" : 11,
+          "skipped" : 0
+        },
+        "capability_claims" : {
+          "errored" : 0,
+          "passed" : 10,
+          "scored" : 11,
+          "skipped" : 0
+        },
+        "computer_use_loop" : {
+          "errored" : 0,
+          "passed" : 21,
+          "scored" : 23,
+          "skipped" : 0
+        },
+        "default_agent" : {
+          "errored" : 0,
+          "passed" : 49,
+          "scored" : 49,
+          "skipped" : 0
+        },
+        "http_api" : {
+          "errored" : 0,
+          "passed" : 15,
+          "scored" : 16,
+          "skipReasons" : {
+            "other: scenario: multimodal_video · port 52792" : 1
+          },
+          "skipped" : 1
+        },
+        "memory" : {
+          "errored" : 0,
+          "passed" : 8,
+          "scored" : 8,
+          "skipped" : 0
+        },
+        "micro_perf" : {
+          "errorCategories" : {
+            "model_load" : 1
+          },
+          "errored" : 1,
+          "passed" : 5,
+          "scored" : 5,
+          "skipped" : 0
+        },
+        "reasoning_channel" : {
+          "errored" : 0,
+          "passed" : 9,
+          "scored" : 13,
+          "skipped" : 0
+        },
+        "subagent" : {
+          "errorCategories" : {
+            "runtime_other" : 1
+          },
+          "errored" : 1,
+          "passed" : 35,
+          "scored" : 35,
+          "skipReasons" : {
+            "missingDependency: live spawn_model_residency lane unavailable on this host: residency lane unavailable: orchestrator 'mlx-community/Qwen3.5-4B-OptiQ-4bit' is not installed locally" : 8,
+            "other: live image lane unavailable on this host: Image edit is not enabled for this agent." : 1,
+            "other: live image lane unavailable on this host: Image generation is not enabled for this agent." : 1,
+            "other: live spawn_model_residency lane unavailable on this host: residency lane unavailable: target 'mlx-community/Qwen3.5-4B-OptiQ-4bit' is neither installed locally nor routable via a connected provider" : 2,
+            "other: live spawn_model_residency lane unavailable on this host: residency lane unavailable: target 'xai/grok-4.3' is neither installed locally nor routable via a connected provider" : 2
+          },
+          "skipped" : 14
+        }
+      },
+      "startedAt" : "2026-09-01T22:48:27Z",
+      "subsystemPassed" : 12,
+      "subsystemScored" : 16,
+      "topContextContributors" : [
+        "tool:redact_file=491",
+        "tool:detect_pii=353",
+        "tool:file_read=279",
+        "tool:spawn_batch=266",
+        "tool:share_artifact=266",
+        "§pluginCreator=252"
+      ],
+      "totalPassed" : 267,
+      "totalScored" : 305
+    }
+  ]
+}