Skip to content

Commit 6088919

Browse files
authored
fix: sibling module dependencies are eagerly loaded even if only for type annotations (aws#5172)
## Problem Every generated `init.py` file has import statements at the top that look like this, for example ``` from .aws_iam import IGrantable as _IGrantable_ef567890 from .aws_kms import IKey as _IKey_abc12345 ... ``` and cross-module imports that could look like this. for example ``` import scope.jsii_calc_base ... ``` These load immediately whenever each module loads, which in turn load their own dependencies. This causes a huge cascade of imports at module load time. A large chunk of these imports only exist to satisfy type annotations however, or runtime type checks, and therefore this creates a need for optimization. ## Solution Cross-module imports are now deferred using a `typing.TYPE_CHECKING` split plus a lazy import proxy. Instead of importing each symbol eagerly, the generator emits: ```python class _LazyImport: def __init__(self, module_name: str) -> None: self._module_name = module_name self._module: typing.Any = None def __getattr__(self, name: str) -> typing.Any: if self._module is None: import importlib self._module = importlib.import_module(self._module_name) return getattr(self._module, name) if typing.TYPE_CHECKING: import jsii_calc.composition as _composition_4f38e801 import scope.jsii_calc_lib as _scope_jsii_calc_lib_c61f082f else: _composition_4f38e801 = _LazyImport("jsii_calc.composition") _scope_jsii_calc_lib_c61f082f = _LazyImport("scope.jsii_calc_lib") ``` mypy and pyright evaluate the if statement, so they see the real module imports and resolve all types normally. At runtime, the `else:` branch executes instead, binding each module alias to a `_LazyImport` proxy. The actual `importlib.import_module()` call is deferred until the first attribute access through the proxy's `__getattr__`. --- By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license]. [Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0
1 parent f3e1928 commit 6088919

4 files changed

Lines changed: 775 additions & 321 deletions

File tree

packages/jsii-pacmak/lib/targets/python.ts

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import {
2525
toTypeName,
2626
PythonImports,
2727
mergePythonImports,
28+
hasImports,
2829
toPackageName,
2930
toPythonFqn,
3031
IntersectionTypesRegistry,
@@ -1777,6 +1778,15 @@ class PythonModule implements PythonType {
17771778
// Before we write anything else, we need to write out our module headers, this
17781779
// is where we handle stuff like imports, any required initialization, etc.
17791780

1781+
// Emit PEP 563 (from __future__ import annotations) if this module has
1782+
// cross-module imports. This makes all annotations strings at definition
1783+
// time, preventing lazy proxy resolution during module load.
1784+
// Must be the first statement in the file (after docstring) per Python rules.
1785+
if (hasImports(this.requiredImports(context))) {
1786+
code.line('from __future__ import annotations');
1787+
code.line();
1788+
}
1789+
17801790
// If multiple packages use the same namespace (in Python, a directory) it
17811791
// depends on how they are laid out on disk if deep imports of multiple packages
17821792
// will succeed. `pip` merges all packages into the same directory, and deep
@@ -2113,6 +2123,67 @@ class PythonModule implements PythonType {
21132123
}
21142124
}
21152125

2126+
/**
2127+
* Emit the `_LazyImport` helper class that defers `importlib.import_module()`
2128+
* until first attribute access via `__getattr__`.
2129+
*
2130+
* The class is emitted once per module that has cross-module imports. It wraps
2131+
* a module name string and caches the imported module after first access.
2132+
* Failed imports are NOT cached, allowing retry on subsequent access.
2133+
*/
2134+
private emitLazyImportClass(code: CodeMaker) {
2135+
code.line();
2136+
code.openBlock('class _LazyImport');
2137+
// __init__
2138+
code.openBlock('def __init__(self, module_name: str) -> None');
2139+
code.line('self._module_name = module_name');
2140+
code.line('self._module: typing.Any = None');
2141+
code.closeBlock();
2142+
// __getattr__
2143+
code.openBlock('def __getattr__(self, name: str) -> typing.Any');
2144+
code.openBlock('if self._module is None');
2145+
code.line('import importlib');
2146+
code.line('self._module = importlib.import_module(self._module_name)');
2147+
code.closeBlock();
2148+
code.line('return getattr(self._module, name)');
2149+
code.closeBlock();
2150+
code.closeBlock();
2151+
}
2152+
2153+
/**
2154+
* Emit `_LazyImport(...)` assignments for all cross-module imports.
2155+
*
2156+
* Only processes entries where the items set contains '' (empty string),
2157+
* indicating a full module import. Parses the sourcePackage string which
2158+
* has the format "module.name as _alias" to extract the module name and alias.
2159+
* Sorts assignments by alias for deterministic output.
2160+
*/
2161+
private emitLazyProxyAssignments(code: CodeMaker, imports: PythonImports) {
2162+
const assignments = Object.entries(imports)
2163+
.filter(([, items]) => items.has('')) // Only full module imports (empty string = import the whole module)
2164+
.map(([sourcePackage]) => {
2165+
// sourcePackage is like "aws_cdk.aws_iam as _aws_cdk_aws_iam_abcd1234"
2166+
const match = sourcePackage.match(/^(.+)\s+as\s+(.+)$/);
2167+
if (match) {
2168+
const [, moduleName, alias] = match;
2169+
return { moduleName, alias };
2170+
}
2171+
// Fallback: no alias
2172+
return {
2173+
moduleName: sourcePackage,
2174+
alias: `_${sourcePackage.replace(/\./g, '_')}`,
2175+
};
2176+
})
2177+
.sort((a, b) => a.alias.localeCompare(b.alias));
2178+
2179+
if (assignments.length > 0) {
2180+
code.line();
2181+
for (const { moduleName, alias } of assignments) {
2182+
code.line(`${alias} = _LazyImport("${moduleName}")`);
2183+
}
2184+
}
2185+
}
2186+
21162187
/**
21172188
* Emit a mapping from submodule FQNs to their Python module paths.
21182189
*
@@ -2142,8 +2213,28 @@ class PythonModule implements PythonType {
21422213
}
21432214

21442215
private emitRequiredImports(code: CodeMaker, context: EmitContext) {
2145-
const requiredImports = this.requiredImports(context);
2146-
const statements = Object.entries(requiredImports)
2216+
const allImports = this.requiredImports(context);
2217+
if (!hasImports(allImports)) {
2218+
return;
2219+
}
2220+
2221+
// Emit _LazyImport class definition
2222+
this.emitLazyImportClass(code);
2223+
2224+
// Emit TYPE_CHECKING block with original import statements for static type checkers,
2225+
// and lazy proxy assignments in the else branch for runtime.
2226+
// This structure ensures pyright/mypy see the real module types while runtime uses lazy proxies.
2227+
code.line();
2228+
code.openBlock('if typing.TYPE_CHECKING');
2229+
this.emitImportStatements(code, allImports);
2230+
code.closeBlock();
2231+
code.openBlock('else');
2232+
this.emitLazyProxyAssignments(code, allImports);
2233+
code.closeBlock();
2234+
}
2235+
2236+
private emitImportStatements(code: CodeMaker, imports: PythonImports) {
2237+
const statements = Object.entries(imports)
21472238
.map(([sourcePackage, items]) => toImportStatements(sourcePackage, items))
21482239
.reduce(
21492240
(acc, elt) => [...acc, ...elt],

packages/jsii-pacmak/lib/targets/python/type-name.ts

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,13 @@ export function mergePythonImports(
152152
return result;
153153
}
154154

155+
/**
156+
* Check if a PythonImports object has any entries.
157+
*/
158+
export function hasImports(imports: PythonImports): boolean {
159+
return Object.keys(imports).length > 0;
160+
}
161+
155162
function isOptionalValue(
156163
type: OptionalValue | TypeReference,
157164
): type is OptionalValue {
@@ -398,20 +405,18 @@ class UserType implements TypeName {
398405
.split('.');
399406
const aliasSuffix = createHash('sha256')
400407
.update(typeSubmodulePythonName)
401-
.update('.')
402-
.update(toImport)
408+
.update('.*')
403409
.digest('hex')
404410
.substring(0, 8);
405-
const alias = `_${toImport}_${aliasSuffix}`;
411+
const moduleAlias = `_${lastComponent(typeSubmodulePythonName)}_${aliasSuffix}`;
406412

407413
return {
408-
pythonType: wrapType([alias, ...nested].join('.')),
414+
pythonType: wrapType(
415+
`${moduleAlias}.${toImport}${nested.length > 0 ? `.${nested.join('.')}` : ''}`,
416+
),
409417
requiredImport: {
410-
sourcePackage: relativeImportPath(
411-
submodulePythonName,
412-
typeSubmodulePythonName,
413-
),
414-
item: `${toImport} as ${alias}`,
418+
sourcePackage: `${typeSubmodulePythonName} as ${moduleAlias}`,
419+
item: '',
415420
},
416421
};
417422

@@ -488,7 +493,7 @@ export function toPythonFqn(fqn: string, rootAssm: Assembly) {
488493
* relativeImportPath('A.B.C', 'A.B') === '..';
489494
* relativeImportPath('A.B', 'A.B.C') === '.C';
490495
*/
491-
function relativeImportPath(fromPkg: string, toPkg: string): string {
496+
export function relativeImportPath(fromPkg: string, toPkg: string): string {
492497
if (toPkg.startsWith(fromPkg)) {
493498
// from A.B to A.B.C === .C
494499
return `.${toPkg.substring(fromPkg.length + 1)}`;

0 commit comments

Comments
 (0)