Skip to content

Commit d6fc893

Browse files
committed
feat: Add sterile subprocess verification with dependency bubble awareness
Major improvements to package verification and installation: VERIFICATION STRATEGY (verification_strategy.py): - CRITICAL FIX V2: Use sterile subprocess isolation to prevent ABI conflicts - Pass existing dependency bubbles to verification so keras can find tensorflow - Include setuptools in subprocess path for compatibility - Replace in-process imports with subprocess-based testing - Increase timeout from 30s to 300s for complex verifications - Remove sys.path manipulation in favor of clean subprocess environment CORE BUBBLE MANAGER (core.py): - Add _find_dependency_bubbles() to locate related package bubbles - Pass existing_bubble_paths to verification strategy - Enable verification groups to access their dependencies during testing - Improve verification logging with bubble discovery feedback VERIFICATION GROUPS (verification_groups.py): - Add 'triton' to PyTorch verification group packages - Update test order to verify triton before torch PACKAGE INDEX REGISTRY (package_index_registry.py): - Return ecosystem URLs as EXTRA index instead of primary - Allow pip to find dependencies (like triton) on PyPI while using custom indices WORKER DAEMON (worker_daemon.py): - Increase DaemonClient timeout from 30s to 300s - Remove extraneous blank lines TRANSLATIONS (zh_CN/omnipkg.po): - Update Chinese translations with 2089 line changes - Improve translation coverage from previous state These changes fix critical verification issues where packages like keras couldn't find tensorflow during import testing, and prevent ABI conflicts from polluted Python environments.
1 parent 6aec447 commit d6fc893

6 files changed

Lines changed: 1880 additions & 614 deletions

File tree

src/omnipkg/core.py

Lines changed: 89 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3207,18 +3207,19 @@ def install_and_verify(
32073207
observed_dependencies: Optional[Dict[str, str]] = None,
32083208
):
32093209
"""
3210-
(V7 - SMART STRATEGY) Builds a bubble and verifies using smart group-aware testing.
3211-
3212-
This version uses the verification_strategy module to intelligently test
3213-
packages together when they have interdependencies.
3210+
(V8 - SMART STRATEGY WITH BUBBLE AWARENESS)
3211+
Builds a bubble and verifies using smart group-aware testing.
3212+
3213+
CRITICAL FIX: Now passes already-created bubbles to verification so that
3214+
packages like keras can find their dependencies like tensorflow during testing.
32143215
"""
32153216
if destination_path.exists():
32163217
shutil.rmtree(destination_path, ignore_errors=True)
3217-
3218+
32183219
with tempfile.TemporaryDirectory() as temp_dir:
32193220
staging_path = Path(temp_dir)
32203221
safe_print(f" - 🏗️ Staging install for {package_name}=={version}...")
3221-
3222+
32223223
# 1. Install to staging
32233224
return_code, stdout = self.parent_omnipkg._run_pip_install(
32243225
[f"{package_name}=={version}"],
@@ -3227,66 +3228,134 @@ def install_and_verify(
32273228
index_url=index_url,
32283229
extra_index_url=extra_index_url,
32293230
)
3231+
32303232
if return_code != 0:
32313233
safe_print(" ❌ Pip install failed in staging area.")
32323234
return False
3233-
3234-
# 2. Use SMART verification strategy
3235+
3236+
# 2. Find already-created bubbles for dependencies
3237+
safe_print(" - 🔍 Locating dependency bubbles for verification...")
3238+
existing_bubble_paths = self._find_dependency_bubbles(
3239+
package_name, destination_path.parent
3240+
)
3241+
3242+
if existing_bubble_paths:
3243+
safe_print(f" Found {len(existing_bubble_paths)} dependency bubble(s):")
3244+
for bubble_path in existing_bubble_paths:
3245+
safe_print(f" 📦 {bubble_path.name}")
3246+
3247+
# 3. Use SMART verification strategy WITH bubble awareness
32353248
safe_print(" - 🧪 Running SMART import verification...")
3236-
32373249
try:
3238-
# Import the smart strategy
32393250
from .installation.verification_strategy import (
32403251
verify_bubble_with_smart_strategy,
32413252
)
32423253
except ImportError:
32433254
from omnipkg.installation.verification_strategy import (
32443255
verify_bubble_with_smart_strategy,
32453256
)
3246-
3257+
32473258
# Instantiate gatherer
32483259
try:
32493260
from .package_meta_builder import omnipkgMetadataGatherer
32503261
except ImportError:
32513262
from omnipkg.package_meta_builder import omnipkgMetadataGatherer
3252-
3263+
32533264
gatherer = omnipkgMetadataGatherer(
32543265
config=self.parent_omnipkg.config,
32553266
env_id=self.parent_omnipkg.env_id,
32563267
omnipkg_instance=self.parent_omnipkg,
32573268
target_context_version=python_context_version,
32583269
)
3259-
3260-
# Run smart verification
3270+
3271+
# Run smart verification WITH dependency bubbles
32613272
verification_passed = verify_bubble_with_smart_strategy(
3262-
self.parent_omnipkg, package_name, version, staging_path, gatherer
3273+
self.parent_omnipkg,
3274+
package_name,
3275+
version,
3276+
staging_path,
3277+
gatherer,
3278+
existing_bubble_paths=existing_bubble_paths # NEW!
32633279
)
3264-
3280+
32653281
if not verification_passed:
32663282
safe_print(f" ❌ CRITICAL: Smart verification failed for '{package_name}'.")
3283+
# Don't trigger full rollback - let caller decide
32673284
return False
3268-
3269-
# 3. Analyze and move
3285+
3286+
# 4. Analyze and move
32703287
installed_tree = self._analyze_installed_tree(staging_path)
32713288
safe_print(_(' - 🚚 Moving verified build to bubble: {}').format(destination_path))
3289+
32723290
# Python 3.7 compatible: remove destination if it exists, then copy
32733291
if destination_path.exists():
32743292
shutil.rmtree(destination_path)
32753293
shutil.copytree(staging_path, destination_path)
3276-
3294+
32773295
stats = {
32783296
"total_files": sum(len(info.get("files", [])) for info in installed_tree.values())
32793297
}
3298+
32803299
self._create_bubble_manifest(
32813300
destination_path,
32823301
installed_tree,
32833302
stats,
32843303
python_context_version=python_context_version,
32853304
observed_dependencies=observed_dependencies,
32863305
)
3287-
3306+
32883307
return True
32893308

3309+
def _find_dependency_bubbles(
3310+
self, package_name: str, bubble_dir: Path
3311+
) -> List[Path]:
3312+
"""
3313+
Find already-created bubbles that are in the same verification group.
3314+
3315+
This allows keras to find tensorflow, tensorboard to find tensorflow, etc.
3316+
during verification testing.
3317+
3318+
Args:
3319+
package_name: The package being installed (e.g., "keras")
3320+
bubble_dir: The .omnipkg_versions directory
3321+
3322+
Returns:
3323+
List of Path objects to dependency bubbles
3324+
"""
3325+
try:
3326+
from .installation.verification_groups import find_verification_group
3327+
except ImportError:
3328+
from omnipkg.installation.verification_groups import find_verification_group
3329+
3330+
existing_bubble_paths = []
3331+
3332+
# Find verification group for this package
3333+
canonical_name = package_name.lower().replace("_", "-")
3334+
group_def = find_verification_group(canonical_name)
3335+
3336+
if not group_def:
3337+
# No group = no dependencies to include
3338+
return existing_bubble_paths
3339+
3340+
# Look for bubbles of other packages in the same group
3341+
for dep_pkg in group_def.packages:
3342+
if dep_pkg == canonical_name:
3343+
continue # Skip self
3344+
3345+
# Find ANY version of this dependency
3346+
# Pattern: dep_pkg-* (e.g., tensorflow-*)
3347+
matching_bubbles = list(bubble_dir.glob(f"{dep_pkg}-*"))
3348+
3349+
if matching_bubbles:
3350+
# Take the most recent bubble (sorted by name, highest version last)
3351+
matching_bubbles.sort()
3352+
bubble_path = matching_bubbles[-1]
3353+
3354+
if bubble_path.is_dir():
3355+
existing_bubble_paths.append(bubble_path)
3356+
3357+
return existing_bubble_paths
3358+
32903359
def _granular_verify_and_heal(self, staging_path: Path, package_name: str, python_exe: str):
32913360
"""
32923361
Iterates through every module in the package.

src/omnipkg/installation/package_index_registry.py

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,18 +88,44 @@ def detect_index_url(
8888
"""
8989
Detect the appropriate index URL for a package variant.
9090
91-
Args:
92-
package_name: The package name (e.g., "torch")
93-
version: The version string (e.g., "2.1.3+cu118")
94-
9591
Returns:
96-
Tuple of (index_url, extra_index_url) or (None, None) if standard PyPI
92+
Tuple of (index_url, extra_index_url)
9793
"""
9894
if not version:
9995
return None, None
10096

10197
pkg_lower = package_name.lower()
10298

99+
# Check each ecosystem in the registry
100+
for ecosystem_name, ecosystem_data in self.registry.items():
101+
if ecosystem_name.startswith("_"): continue
102+
if "packages" not in ecosystem_data: continue
103+
if pkg_lower not in [p.lower() for p in ecosystem_data["packages"]]: continue
104+
105+
# Try to match against rules
106+
for rule in ecosystem_data.get("rules", []):
107+
pattern = rule.get("pattern", "")
108+
if not pattern: continue
109+
110+
match = re.search(pattern, version)
111+
if match:
112+
url_template = rule.get("url")
113+
if not url_template: continue
114+
115+
if "{0}" in url_template:
116+
captured = match.group(1) if match.groups() else ""
117+
index_url = url_template.format(captured)
118+
else:
119+
index_url = url_template
120+
121+
# CRITICAL CHANGE: Return as EXTRA index URL, not primary
122+
# This allows pip to find dependencies (like triton) on PyPI
123+
return None, index_url
124+
125+
return None, None
126+
127+
pkg_lower = package_name.lower()
128+
103129
# Check each ecosystem in the registry
104130
for ecosystem_name, ecosystem_data in self.registry.items():
105131
# Skip metadata fields

src/omnipkg/installation/verification_groups.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,12 @@ class VerificationGroup:
5555
# PyTorch Ecosystem
5656
"torch": VerificationGroup(
5757
name="torch",
58-
packages={"torch", "torchvision", "torchaudio", "torchtext"},
58+
# Added 'triton' to packages
59+
packages={"torch", "torchvision", "torchaudio", "torchtext", "triton"},
5960
primary_package="torch",
6061
reason="PyTorch extensions require torch to be imported first.",
61-
test_order=["torch", "torchvision", "torchaudio", "torchtext"],
62+
# Added 'triton' to test order
63+
test_order=["triton", "torch", "torchvision", "torchaudio", "torchtext"],
6264
),
6365
# Jupyter/IPython
6466
"jupyter": VerificationGroup(

0 commit comments

Comments
 (0)