Skip to content

Commit 6a18037

Browse files
committed
feat(core): introduce global cache metadata and upgrade bubble manifests to schema 2.0
- Add global cache identity fields to MetadataGatherer: record_hash (byte-level identity), content_hash, wheel_abi_tag, and symlink_targets - Introduce global_cache_key and resolved_bubble_deps for deterministic deduplication across bubbles - Upgrade bubble manifests to schema 2.0: - Embed global cache metadata into .omnipkg_manifest.json - Add resolved_dependencies snapshot per bubble - Implement migration logic for legacy manifests - Stabilize FS watcher operation signaling: - Add start/end operation protocol using .omnipkg_op.lock - Suppress false external-change detection during active installs - Improve daemon coordination to prevent duplicate event emission - Fix dependency resolution and naming consistency: - Resolve canonicalization mismatch (hyphen vs underscore) - Improve metadata builder stability during verification flows New files: • tests/test_verify_bubble_deps.py (193 lines) • tests/test_verify_bubble_deps2.py (127 lines) Modified: • src/omnipkg/core.py (+507/-127 lines) • src/omnipkg/isolation/fs_watcher.py (+293/-16 lines) • src/omnipkg/isolation/worker_daemon.py (+58/-23 lines) • src/omnipkg/package_meta_builder.py (+432/-21 lines) • .gitignore (+1/-0 lines) [gitship-generated]
1 parent f581ac0 commit 6a18037

7 files changed

Lines changed: 1609 additions & 185 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,3 +208,4 @@ src/omnipkg/_vendor/wheels/
208208
*hunk_merger*.json
209209
.hunk_merger_state.json
210210
.gitattributes
211+
.omnipkg_baseline

src/omnipkg/core.py

Lines changed: 507 additions & 127 deletions
Large diffs are not rendered by default.

src/omnipkg/isolation/fs_watcher.py

Lines changed: 292 additions & 15 deletions
Large diffs are not rendered by default.

src/omnipkg/isolation/worker_daemon.py

Lines changed: 58 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3084,6 +3084,10 @@ def start(self, daemonize: bool = True, wait_for_ready: bool = False):
30843084
# 2. A foreground process if daemonize=False.
30853085
# It is NOT executed by the initial parent process that the user runs.
30863086
self._initialize_daemon_process()
3087+
3088+
# 🔥 Start the FS Watcher in the background before blocking on the server loop
3089+
threading.Thread(target=self._start_fs_watcher, name="omnipkg-fs-watcher-boot", daemon=True).start()
3090+
30873091
self._run_socket_server() # This is a blocking call that starts the server loop.
30883092

30893093
def _replenish_idle_pool(self, python_exe: str):
@@ -3640,51 +3644,35 @@ def _handle_client(self, conn: socket.socket):
36403644
sp_path = req.get("site_packages_path")
36413645
installed = req.get("installed", [])
36423646
removed = req.get("removed", [])
3643-
now = time.monotonic()
3644-
GRACE_SECS = 5.0
36453647
forwarded = 0
3646-
suppressed = 0
36473648

36483649
patch_msg = json.dumps({
36493650
"type": "patch_cache",
36503651
"installed": installed,
36513652
"removed": removed,
36523653
}) + "\n"
36533654

3654-
# ── UV workers (existing) ──
3655+
# Forward the patch to all relevant UV workers.
3656+
# The watcher is now the sole authority on suppression; the daemon is just a message bus.
36553657
for py_exe, uv_worker in list(self.uv_workers.items()):
36563658
try:
36573659
worker_sp = _resolve_target_paths(self.cm, py_exe).get("site_packages_path")
36583660
if sp_path is not None and worker_sp != sp_path:
36593661
continue
3660-
lock = self.uv_worker_locks.get(py_exe)
3661-
in_flight = lock is not None and lock.locked()
3662-
just_finished = (now - self._uv_last_install_ts.get(py_exe, 0.0)) < GRACE_SECS
3663-
if in_flight or just_finished:
3664-
suppressed += 1
3665-
continue
36663662
uv_worker.process.stdin.write(patch_msg)
36673663
uv_worker.process.stdin.flush()
36683664
forwarded += 1
3669-
self._uv_last_install_ts[py_exe] = time.monotonic()
36703665
except Exception:
36713666
pass
3672-
3673-
# CLI workers (idle_pools) excluded from patch_cache.
3674-
# They have no uv_ffi — only UV workers need cache coherency.
3675-
if forwarded:
3667+
3668+
if forwarded > 0:
36763669
safe_print(
3677-
f"[FS-WATCHER] External change — patched {forwarded} UV worker(s) "
3670+
f"[DAEMON-PATCH] Forwarded patch from watcher to {forwarded} UV worker(s) "
36783671
f"inst={installed} rem={removed}",
36793672
file=sys.stderr,
36803673
)
3681-
elif suppressed:
3682-
safe_print(
3683-
f"[FS-WATCHER] Suppressed patch (our own install, {suppressed} worker(s))",
3684-
file=sys.stderr,
3685-
)
3686-
res = {"success": True, "forwarded": forwarded, "suppressed": suppressed}
3687-
# Keep old invalidate message type as fallback for any stragglers
3674+
res = {"success": True, "forwarded": forwarded}
3675+
36883676
elif req["type"] == "invalidate_site_packages_cache":
36893677
sp_path = req.get("site_packages_path")
36903678
now = time.monotonic()
@@ -3705,6 +3693,23 @@ def _handle_client(self, conn: socket.socket):
37053693
except Exception:
37063694
pass
37073695
res = {"success": True, "forwarded": forwarded}
3696+
elif req["type"] == "start_omnipkg_op":
3697+
safe_print("[DAEMON] Signal: Omnipkg op START", file=sys.stderr)
3698+
if hasattr(self, '_fs_watcher') and self._fs_watcher:
3699+
self._fs_watcher.start_omnipkg_op()
3700+
res = {"success": True}
3701+
elif req["type"] == "end_omnipkg_op":
3702+
inst = req.get("installed", [])
3703+
rem = req.get("removed", [])
3704+
# 🔥 TRUTH-TRACKING: Log exactly what the daemon received
3705+
safe_print(f"[DAEMON-ROUTER] Received end_op: INST={inst}, REM={rem}", file=sys.stderr)
3706+
3707+
if hasattr(self, '_fs_watcher') and self._fs_watcher:
3708+
self._fs_watcher.end_omnipkg_op(
3709+
installed=inst,
3710+
removed=rem
3711+
)
3712+
res = {"success": True}
37083713
elif req["type"] == "get_site_packages_state":
37093714
state = {}
37103715
for py_exe, worker in list(self.uv_workers.items()):
@@ -5675,6 +5680,21 @@ def status(self):
56755680
def shutdown(self):
56765681
return self._send({"type": "shutdown"})
56775682

5683+
def start_omnipkg_op(self):
5684+
"""Signals the FS watcher that a long Omnipkg install is STARTING."""
5685+
return self._send({"type": "start_omnipkg_op"})
5686+
5687+
def end_omnipkg_op(self, installed: list, removed: list):
5688+
"""
5689+
Signals the FS watcher that a long Omnipkg install has ENDED,
5690+
providing the official changelog from the FFI for validation.
5691+
"""
5692+
return self._send({
5693+
"type": "end_omnipkg_op",
5694+
"installed": installed,
5695+
"removed": removed,
5696+
})
5697+
56785698
def _spawn_daemon(self):
56795699

56805700
daemon_script = os.path.abspath(__file__)
@@ -6767,6 +6787,21 @@ def shutdown(self):
67676787
except Exception:
67686788
pass
67696789

6790+
def start_omnipkg_op(self):
6791+
"""Signals the FS watcher that a long Omnipkg install is STARTING."""
6792+
return self._send({"type": "start_omnipkg_op"})
6793+
6794+
def end_omnipkg_op(self, installed: list, removed: list):
6795+
"""
6796+
Signals the FS watcher that a long Omnipkg install has ENDED,
6797+
providing the official changelog from the FFI.
6798+
"""
6799+
return self._send({
6800+
"type": "end_omnipkg_op",
6801+
"installed": installed,
6802+
"removed": removed,
6803+
})
6804+
67706805

67716806
# ═══════════════════════════════════════════════════════════════
67726807
# 5. CLI FUNCTIONS

0 commit comments

Comments
 (0)