v2.1: idle memory unload + honest "predictability" reframe

Two changes driven by "no second chance at a first impression".

1) Idle memory unload. The int8 model uses ~1.5GB resident (ORT arena), which
   shouldn't sit on a shared server when idle. The engine now lazy-loads on the
   first request and disposes the session after IdleUnloadSeconds (default 300)
   with no traffic, reloading from disk (~1.5s) on demand. Concurrency-safe:
   a gate serializes load/unload and an active-request counter prevents
   unloading mid-inference. Verified locally: ~1500MB loaded -> ~280MB idle,
   cold reload ~1.5s, warm ~150ms. GET / now reports modelLoaded.

2) Perplexity reframed from an AI verdict to a "predictability" meter. Building
   a 170-text labeled corpus (Wikipedia human EN/ES + diverse generated AI) and
   scoring it exposed that perplexity does NOT separate AI from human: memorized
   human text (Wikipedia) scores LOWER/more-predictable than fresh AI text
   (AUC ~0.1 in the "low=AI" direction). A confident "Likely AI %" verdict would
   false-positive on formulaic/encyclopedic human writing. So the panel now shows
   how predictable/generic the phrasing is (very-predictable | typical | varied),
   calibrated per language, with an honest caveat that it's a signal, not proof —
   the actual AI verdict stays with the explainable rule tells. On-brand for an
   explainable linter. Response: predictability + band replace aiLikelihood/verdict.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: peopleworks

src/SignsOfAI.Perplexity.Api/Config/PerplexityOptions.cs

@@ -1,16 +1,21 @@
 namespace SignsOfAI.Perplexity.Api.Config;
 
-/// <summary>Per-language calibration of the perplexity → AI-likelihood mapping (measured on the chosen model).</summary>
+/// <summary>
+/// Per-language calibration of the perplexity → <b>predictability</b> meter. Perplexity measures how
+/// predictable/generic the phrasing is (NOT AI-vs-human: on a labeled corpus the two overlap badly —
+/// memorized human text like Wikipedia scores <i>lower</i>/more-predictable than fresh AI text). So we
+/// surface predictability honestly, centered per language (Spanish runs higher-perplexity than English).
+/// </summary>
 public sealed class LangBaseline
 {
-    /// <summary>Decision boundary in natural-log perplexity: below ⇒ machine-leaning, above ⇒ human-leaning.</summary>
-    public double BoundaryLogPpl { get; init; }
+    /// <summary>Natural-log perplexity that maps to 50% predictability (the per-language corpus center).</summary>
+    public double Center { get; init; }
 
-    /// <summary>Spread (≈ half the human↔AI gap) — the z-score denominator.</summary>
-    public double Spread { get; init; } = 0.5;
+    /// <summary>Spread (≈ 1 std of log-perplexity in this language) — the logistic denominator.</summary>
+    public double Spread { get; init; } = 0.8;
 
-    /// <summary>Logistic steepness mapping z-score → AI likelihood.</summary>
-    public double Steepness { get; init; } = 1.4;
+    /// <summary>Logistic steepness mapping (center − logPpl)/spread → predictability.</summary>
+    public double Steepness { get; init; } = 1.3;
 }
 
 /// <summary>
@@ -45,21 +50,34 @@ public sealed class PerplexityOptions
     /// <summary>Intra-op thread count for the ONNX session (0 ⇒ ORT default).</summary>
     public int IntraOpThreads { get; init; } = 4;
 
-    // ── Scoring ───────────────────────────────────────────────────────────────
+    /// <summary>
+    /// Free the model from RAM after this many seconds with no requests; the next request lazily
+    /// reloads it from disk (~1-2s). ≤ 0 keeps it resident once loaded. Keeps the server light when idle.
+    /// </summary>
+    public int IdleUnloadSeconds { get; init; } = 300;
+
+    /// <summary>Load the model into RAM at startup. Default false ⇒ lazy-load on first request (idle = ~0 model RAM).</summary>
+    public bool PreloadModel { get; init; }
+
+    // ── Predictability meter ────────────────────────────────────────────────────
     public Dictionary<string, LangBaseline> Baselines { get; init; } = new(StringComparer.OrdinalIgnoreCase);
-    public double AiThreshold { get; init; } = 0.66;
-    public double HumanThreshold { get; init; } = 0.34;
+
+    /// <summary>Predictability at/above which we call the phrasing "very-predictable" (generic).</summary>
+    public double PredictableAbove { get; init; } = 0.60;
+
+    /// <summary>Predictability at/below which we call the phrasing "varied" (surprising).</summary>
+    public double VariedBelow { get; init; } = 0.40;
 
     /// <summary>
-    /// Defaults calibrated locally on <c>model_int8.onnx</c> (2026-07-06): human EN log-ppl ≈ 4.6,
-    /// AI EN ≈ 3.7 (boundary ≈ 4.1); human ES ≈ 4.5, AI ES ≈ 3.8. Refined with more samples over time.
+    /// Defaults calibrated on the instruct model over a 170-text EN/ES corpus (2026-07-07). Centers are
+    /// the per-language corpus means of log-perplexity (EN ≈ 3.95, ES ≈ 4.18); ES runs higher.
     /// </summary>
     public static PerplexityOptions Defaults() => new()
     {
         Baselines =
         {
-            ["en"] = new LangBaseline { BoundaryLogPpl = 4.10, Spread = 0.50, Steepness = 1.4 },
-            ["es"] = new LangBaseline { BoundaryLogPpl = 4.10, Spread = 0.42, Steepness = 1.4 },
+            ["en"] = new LangBaseline { Center = 4.35, Spread = 0.75, Steepness = 1.3 },
+            ["es"] = new LangBaseline { Center = 4.55, Spread = 0.75, Steepness = 1.3 },
         },
     };
 }

src/SignsOfAI.Perplexity.Api/Engine/IPerplexityEngine.cs

@@ -17,9 +17,12 @@ public interface IPerplexityEngine
     /// <summary>Short model identifier surfaced to clients (e.g. "qwen2.5-0.5b").</summary>
     string ModelId { get; }
 
-    /// <summary>True once the model + tokenizer are loaded and ready to serve.</summary>
+    /// <summary>True once the model file is present and the engine can serve (it may lazy-load on demand).</summary>
     bool IsReady { get; }
 
+    /// <summary>True when the model is currently resident in RAM (false while idle-unloaded).</summary>
+    bool IsLoaded { get; }
+
     /// <summary>Runs one forward pass and returns the perplexity of <paramref name="text"/>.</summary>
     Task<PerplexityRaw> ScoreAsync(string text, CancellationToken ct = default);
 }

src/SignsOfAI.Perplexity.Api/Engine/OnnxPerplexityEngine.cs

@@ -7,86 +7,136 @@ namespace SignsOfAI.Perplexity.Api.Engine;
 
 /// <summary>
 /// Computes causal-LM perplexity with a Qwen2.5-0.5B ONNX model via ONNX Runtime (CPU) and the
-/// Hugging Face tokenizer. The model is a merged (with-past) export, so a from-scratch forward pass
-/// supplies empty KV-cache tensors. Loaded once (see <see cref="InitializeAsync"/>) and reused;
+/// Hugging Face tokenizer. To keep the server light when idle, the model is <b>lazily loaded</b> on
+/// the first request and <b>unloaded from RAM after an idle period</b> (see <see cref="TryUnloadIfIdle"/>),
+/// reloading from disk (~1-2s) on the next request. Loads/unloads are serialized by a gate; inference
+/// runs outside the gate and an active-request counter prevents unloading mid-inference.
 /// <see cref="InferenceSession.Run(RunOptions, IReadOnlyDictionary{string, OrtValue}, IReadOnlyCollection{string})"/>
 /// is thread-safe, and each call builds its own inputs, so concurrent scoring is safe.
 /// </summary>
-public sealed class OnnxPerplexityEngine(PerplexityOptions options, IHostEnvironment env, ILogger<OnnxPerplexityEngine> log)
-    : IPerplexityEngine, IDisposable
+public sealed class OnnxPerplexityEngine : IPerplexityEngine, IDisposable
 {
-    private readonly PerplexityOptions _o = options;
+    private readonly PerplexityOptions _o;
+    private readonly ILogger<OnnxPerplexityEngine> _log;
+    private readonly string _dir, _modelPath, _tokPath;
+
+    private readonly SemaphoreSlim _gate = new(1, 1);
     private InferenceSession? _session;
     private Tokenizer? _tokenizer;
-    private volatile bool _ready;
+    private int _active;
+    private DateTime _lastUsedUtc = DateTime.UtcNow;
+    private volatile bool _filesReady;
+    private volatile bool _loaded;
+
+    public OnnxPerplexityEngine(PerplexityOptions options, IHostEnvironment env, ILogger<OnnxPerplexityEngine> log)
+    {
+        _o = options; _log = log;
+        _dir = Path.IsPathRooted(_o.ModelDir) ? _o.ModelDir : Path.Combine(env.ContentRootPath, _o.ModelDir);
+        _modelPath = Path.Combine(_dir, _o.ModelFile);
+        _tokPath = Path.Combine(_dir, _o.TokenizerFile);
+    }
 
     public string ModelId => _o.ModelId;
-    public bool IsReady => _ready;
+    public bool IsReady => _filesReady;
+    public bool IsLoaded => _loaded;
 
-    /// <summary>Resolves + downloads the model/tokenizer if needed, then loads them. Call once at startup.</summary>
-    public async Task InitializeAsync(CancellationToken ct = default)
+    /// <summary>Ensures the model + tokenizer FILES exist (downloading once if needed), without loading
+    /// them into RAM. Call at startup. If <see cref="PerplexityOptions.PreloadModel"/> is set, also warms.</summary>
+    public async Task EnsureFilesAsync(CancellationToken ct = default)
     {
-        var dir = Path.IsPathRooted(_o.ModelDir) ? _o.ModelDir : Path.Combine(env.ContentRootPath, _o.ModelDir);
-        Directory.CreateDirectory(dir);
-        var modelPath = Path.Combine(dir, _o.ModelFile);
-        var tokPath = Path.Combine(dir, _o.TokenizerFile);
-
-        await EnsureFileAsync(modelPath, _o.ModelUrl, ct);
-        await EnsureFileAsync(tokPath, _o.TokenizerUrl, ct);
+        Directory.CreateDirectory(_dir);
+        await EnsureFileAsync(_modelPath, _o.ModelUrl, ct);
+        await EnsureFileAsync(_tokPath, _o.TokenizerUrl, ct);
 
-        if (!File.Exists(modelPath) || !File.Exists(tokPath))
+        _filesReady = File.Exists(_modelPath) && File.Exists(_tokPath);
+        if (!_filesReady)
         {
-            log.LogError("Perplexity model/tokenizer missing at {Dir} and no download URL configured; engine stays not-ready.", dir);
+            _log.LogError("Perplexity model/tokenizer missing at {Dir} and no download URL configured; engine not ready.", _dir);
             return;
         }
+        _log.LogInformation("Perplexity engine ready (files present): {Model}. Idle-unload={Idle}s, preload={Preload}.",
+            _o.ModelFile, _o.IdleUnloadSeconds, _o.PreloadModel);
 
-        var so = new Microsoft.ML.OnnxRuntime.SessionOptions();
-        if (_o.IntraOpThreads > 0) so.IntraOpNumThreads = _o.IntraOpThreads;
+        if (_o.PreloadModel) { var _ = await AcquireAsync(ct); Release(); }
+    }
 
-        var sw = Stopwatch.StartNew();
-        _tokenizer = new Tokenizer(tokPath);
-        _session = new InferenceSession(modelPath, so);
-        sw.Stop();
+    public async Task<PerplexityRaw> ScoreAsync(string text, CancellationToken ct = default)
+    {
+        if (!_filesReady) throw new InvalidOperationException("Perplexity engine is not ready.");
 
-        _ready = true;
-        log.LogInformation("Perplexity engine ready: {Model} loaded in {Ms} ms.", _o.ModelFile, sw.ElapsedMilliseconds);
+        var (session, tokenizer) = await AcquireAsync(ct);
+        try
+        {
+            return await Task.Run(() => Forward(session, tokenizer, text, ct), ct);
+        }
+        finally { Release(); }
     }
 
-    public Task<PerplexityRaw> ScoreAsync(string text, CancellationToken ct = default)
+    // ── Load / unload lifecycle ──────────────────────────────────────────────
+    private async Task<(InferenceSession, Tokenizer)> AcquireAsync(CancellationToken ct)
     {
-        if (!_ready || _session is null || _tokenizer is null)
-            throw new InvalidOperationException("Perplexity engine is not ready.");
-
-        // CPU-bound work; hand off so we don't block the request pipeline thread.
-        return Task.Run(() =>
+        await _gate.WaitAsync(ct);
+        try
         {
-            var ids = Array.ConvertAll(_tokenizer.Encode(text), x => (long)x);
-            if (ids.Length > _o.MaxTokens)
-                ids = ids[.._o.MaxTokens]; // right-truncate for bounded latency
+            if (_session is null) LoadLocked();
+            _active++;
+            return (_session!, _tokenizer!);
+        }
+        finally { _gate.Release(); }
+    }
 
-            if (ids.Length < 2)
-                return new PerplexityRaw(1.0, 0.0, 0, 0);
+    private void Release()
+    {
+        _gate.Wait();
+        try { if (_active > 0) _active--; _lastUsedUtc = DateTime.UtcNow; }
+        finally { _gate.Release(); }
+    }
 
-            var sw = Stopwatch.StartNew();
-            var (ppl, meanLogProb, scored) = Forward(ids, ct);
-            sw.Stop();
-            return new PerplexityRaw(ppl, meanLogProb, scored, sw.ElapsedMilliseconds);
-        }, ct);
+    private void LoadLocked()
+    {
+        var so = new Microsoft.ML.OnnxRuntime.SessionOptions();
+        if (_o.IntraOpThreads > 0) so.IntraOpNumThreads = _o.IntraOpThreads;
+        var sw = Stopwatch.StartNew();
+        _tokenizer = new Tokenizer(_tokPath);
+        _session = new InferenceSession(_modelPath, so);
+        _loaded = true;
+        _log.LogInformation("Perplexity model loaded into RAM in {Ms} ms.", sw.ElapsedMilliseconds);
     }
 
-    private (double ppl, double meanLogProb, int scored) Forward(long[] ids, CancellationToken ct)
+    /// <summary>If the model is loaded, no request is in flight, and it's been idle longer than
+    /// <paramref name="idle"/>, dispose it to free RAM. Non-blocking; returns whether it unloaded.</summary>
+    public bool TryUnloadIfIdle(TimeSpan idle)
     {
+        if (!_gate.Wait(0)) return false;
+        try
+        {
+            if (_session is null || _active != 0 || DateTime.UtcNow - _lastUsedUtc <= idle)
+                return false;
+            _session.Dispose(); _session = null;
+            _tokenizer?.Dispose(); _tokenizer = null;
+            _loaded = false;
+            GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect();
+            return true;
+        }
+        finally { _gate.Release(); }
+    }
+
+    // ── Inference ─────────────────────────────────────────────────────────────
+    private PerplexityRaw Forward(InferenceSession session, Tokenizer tokenizer, string text, CancellationToken ct)
+    {
+        var ids = Array.ConvertAll(tokenizer.Encode(text), x => (long)x);
+        if (ids.Length > _o.MaxTokens) ids = ids[.._o.MaxTokens]; // right-truncate for bounded latency
+        if (ids.Length < 2) return new PerplexityRaw(1.0, 0.0, 0, 0);
+
         int n = ids.Length, vocab = _o.Vocab;
         var toDispose = new List<OrtValue>(2 * _o.NumLayers + 3);
         OrtValue Mk<T>(T[] data, long[] shape) where T : unmanaged
         { var v = OrtValue.CreateTensorValueFromMemory(data, shape); toDispose.Add(v); return v; }
 
+        var sw = Stopwatch.StartNew();
         try
         {
-            var inputs = new Dictionary<string, OrtValue>(2 * _o.NumLayers + 3)
-            {
-                ["input_ids"] = Mk(ids, [1, n]),
-            };
+            var inputs = new Dictionary<string, OrtValue>(2 * _o.NumLayers + 3) { ["input_ids"] = Mk(ids, [1, n]) };
             var attn = new long[n]; Array.Fill(attn, 1L);
             inputs["attention_mask"] = Mk(attn, [1, n]);
             var pos = new long[n]; for (int i = 0; i < n; i++) pos[i] = i;
@@ -100,10 +150,9 @@ OrtValue Mk<T>(T[] data, long[] shape) where T : unmanaged
             }
 
             using var ro = new RunOptions();
-            using var results = _session!.Run(ro, inputs, ["logits"]);
+            using var results = session.Run(ro, inputs, ["logits"]);
             var logits = results[0].GetTensorDataAsSpan<float>(); // [1, n, vocab] row-major
 
-            // Shift: logits at position t predict token ids[t+1]. Score t = 0..n-2.
             double sumNll = 0; int scored = 0;
             for (int t = 0; t < n - 1; t++)
             {
@@ -117,40 +166,54 @@ OrtValue Mk<T>(T[] data, long[] shape) where T : unmanaged
                 sumNll += -(logits[off + (int)ids[t + 1]] - logZ);
                 scored++;
             }
-
+            sw.Stop();
             double meanNll = sumNll / scored;
-            return (Math.Exp(meanNll), -meanNll, scored);
-        }
-        finally
-        {
-            foreach (var d in toDispose) d.Dispose();
+            return new PerplexityRaw(Math.Exp(meanNll), -meanNll, scored, sw.ElapsedMilliseconds);
         }
+        finally { foreach (var d in toDispose) d.Dispose(); }
     }
 
     private async Task EnsureFileAsync(string path, string? url, CancellationToken ct)
     {
         if (File.Exists(path) || string.IsNullOrWhiteSpace(url)) return;
-        log.LogInformation("Downloading {Url} → {Path} …", url, path);
+        _log.LogInformation("Downloading {Url} → {Path} …", url, path);
         using var http = new HttpClient { Timeout = TimeSpan.FromMinutes(20) };
         using var resp = await http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct);
         resp.EnsureSuccessStatusCode();
         var tmp = path + ".part";
         await using (var fs = File.Create(tmp))
             await resp.Content.CopyToAsync(fs, ct);
         File.Move(tmp, path, overwrite: true);
-        log.LogInformation("Downloaded {Path} ({Bytes:N0} bytes).", path, new FileInfo(path).Length);
+        _log.LogInformation("Downloaded {Path} ({Bytes:N0} bytes).", path, new FileInfo(path).Length);
     }
 
-    public void Dispose() => _session?.Dispose();
+    public void Dispose()
+    {
+        _session?.Dispose();
+        _tokenizer?.Dispose();
+        _gate.Dispose();
+    }
 }
 
-/// <summary>Loads the model at startup (downloading if needed) without blocking app boot.</summary>
-public sealed class ModelWarmupService(OnnxPerplexityEngine engine, ILogger<ModelWarmupService> log) : BackgroundService
+/// <summary>At startup: ensures the model file is on disk (download once). Then, if idle-unloading is
+/// enabled, periodically frees the model from RAM after it goes idle.</summary>
+public sealed class ModelLifecycleService(OnnxPerplexityEngine engine, PerplexityOptions options, ILogger<ModelLifecycleService> log)
+    : BackgroundService
 {
     protected override async Task ExecuteAsync(CancellationToken ct)
     {
-        try { await engine.InitializeAsync(ct); }
-        catch (Exception ex) when (ex is not OperationCanceledException)
-        { log.LogError(ex, "Perplexity engine failed to initialize."); }
+        try { await engine.EnsureFilesAsync(ct); }
+        catch (Exception ex) when (ex is not OperationCanceledException) { log.LogError(ex, "Perplexity engine failed to initialize."); }
+
+        if (options.IdleUnloadSeconds <= 0) return; // keep resident once loaded
+
+        var idle = TimeSpan.FromSeconds(options.IdleUnloadSeconds);
+        var interval = TimeSpan.FromSeconds(Math.Clamp(options.IdleUnloadSeconds / 4.0, 15, 120));
+        while (!ct.IsCancellationRequested)
+        {
+            try { await Task.Delay(interval, ct); } catch (OperationCanceledException) { break; }
+            if (engine.TryUnloadIfIdle(idle))
+                log.LogInformation("Perplexity model unloaded after {Idle}s idle — RAM freed.", options.IdleUnloadSeconds);
+        }
     }
 }