Coverage for packages / griffelib / src / griffe / _internal / loader.py: 83.42%

1# This module contains all the logic for loading API data from sources or compiled modules. 

2 

3from __future__ import annotations 

4 

5import re 

6import shutil 

7import subprocess 

8import sys 

9import tempfile 

10from contextlib import suppress 

11from datetime import datetime, timezone 

12from functools import cached_property 

13from importlib.util import find_spec 

14from pathlib import Path 

15from typing import TYPE_CHECKING, ClassVar, cast 

16 

17from griffe._internal.agents.inspector import inspect 

18from griffe._internal.agents.visitor import visit 

19from griffe._internal.collections import LinesCollection, ModulesCollection 

20from griffe._internal.enumerations import Kind 

21from griffe._internal.exceptions import ( 

22 AliasResolutionError, 

23 CyclicAliasError, 

24 LoadingError, 

25 UnimportableModuleError, 

26) 

27from griffe._internal.expressions import ExprName 

28from griffe._internal.extensions.base import Extensions, load_extensions 

29from griffe._internal.finder import ModuleFinder, NamespacePackage, Package 

30from griffe._internal.git import GitInfo, _tmp_worktree 

31from griffe._internal.importer import dynamic_import 

32from griffe._internal.logger import logger 

33from griffe._internal.merger import merge_stubs 

34from griffe._internal.models import Alias, Module, Object 

35from griffe._internal.stats import Stats 

36 

37if TYPE_CHECKING: 

38 from collections.abc import Sequence 

39 

40 from griffe._internal.docstrings.parsers import DocstringOptions, DocstringStyle 

41 from griffe._internal.enumerations import Parser 

42 

43 

44class GriffeLoader: 

45 """The Griffe loader, allowing to load data from modules.""" 

46 

47 ignored_modules: ClassVar[set[str]] = {"debugpy", "_pydev"} 

48 """Special modules to ignore when loading. 

49 

50 For example, `debugpy` and `_pydev` are used when debugging with VSCode and should generally never be loaded. 

51 """ 

52 

53 def __init__( 

54 self, 

55 *, 

56 extensions: Extensions | None = None, 

57 search_paths: Sequence[str | Path] | None = None, 

58 docstring_parser: DocstringStyle | Parser | None = None, 

59 docstring_options: DocstringOptions | None = None, 

60 lines_collection: LinesCollection | None = None, 

61 modules_collection: ModulesCollection | None = None, 

62 allow_inspection: bool = True, 

63 force_inspection: bool = False, 

64 store_source: bool = True, 

65 ) -> None: 

66 """Initialize the loader. 

67 

68 Parameters: 

69 extensions: The extensions to use. 

70 search_paths: The paths to search into. 

71 docstring_parser: The docstring parser to use. By default, no parsing is done. 

72 docstring_options: Docstring parsing options. 

73 lines_collection: A collection of source code lines. 

74 modules_collection: A collection of modules. 

75 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

76 store_source: Whether to store code source in the lines collection. 

77 """ 

78 self.extensions: Extensions = extensions or load_extensions() 

79 """Loaded Griffe extensions.""" 

80 self.docstring_parser: DocstringStyle | Parser | None = docstring_parser 

81 """Selected docstring parser.""" 

82 self.docstring_options: DocstringOptions = docstring_options or {} 

83 """Configured parsing options.""" 

84 self.lines_collection: LinesCollection = lines_collection or LinesCollection() 

85 """Collection of source code lines.""" 

86 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection() 

87 """Collection of modules.""" 

88 self.allow_inspection: bool = allow_inspection 

89 """Whether to allow inspecting (importing) modules for which we can't find sources.""" 

90 self.force_inspection: bool = force_inspection 

91 """Whether to force inspecting (importing) modules, even when sources were found.""" 

92 self.store_source: bool = store_source 

93 """Whether to store source code in the lines collection.""" 

94 self._search_paths: Sequence[str | Path] | None = search_paths 

95 self._time_stats: dict = { 

96 "time_spent_visiting": 0, 

97 "time_spent_inspecting": 0, 

98 } 

99 

100 @cached_property 

101 def finder(self) -> ModuleFinder: 

102 """The module source finder.""" 

103 return ModuleFinder(search_paths=self._search_paths) 

104 

105 def load( 

106 self, 

107 objspec: str | Path | None = None, 

108 /, 

109 *, 

110 submodules: bool = True, 

111 try_relative_path: bool = True, 

112 find_stubs_package: bool = False, 

113 ) -> Object | Alias: 

114 """Load an object as a Griffe object, given its Python or file path. 

115 

116 Note that this will load the whole object's package, 

117 and return only the specified object. 

118 The rest of the package can be accessed from the returned object 

119 with regular methods and properties (`parent`, `members`, etc.). 

120 

121 Examples: 

122 >>> loader.load("griffe.Module") 

123 Alias("Module", "griffe._internal.models.Module") 

124 

125 Parameters: 

126 objspec: The Python path of an object, or file path to a module. 

127 submodules: Whether to recurse on the submodules. 

128 This parameter only makes sense when loading a package (top-level module). 

129 try_relative_path: Whether to try finding the module as a relative path. 

130 find_stubs_package: Whether to search for stubs-only package. 

131 If both the package and its stubs are found, they'll be merged together. 

132 If only the stubs are found, they'll be used as the package itself. 

133 

134 Raises: 

135 LoadingError: When loading a module failed for various reasons. 

136 ModuleNotFoundError: When a module was not found and inspection is disallowed. 

137 

138 Returns: 

139 A Griffe object. 

140 """ 

141 obj_path: str 

142 package = None 

143 top_module = None 

144 

145 # We always start by searching paths on the disk, 

146 # even if inspection is forced. 

147 logger.debug("Searching path(s) for %s", objspec) 

148 try: 

149 obj_path, package = self.finder.find_spec( 

150 objspec, # ty:ignore[invalid-argument-type] 

151 try_relative_path=try_relative_path, 

152 find_stubs_package=find_stubs_package, 

153 ) 

154 except ModuleNotFoundError: 

155 # If we couldn't find paths on disk and inspection is disabled, 

156 # re-raise ModuleNotFoundError. 

157 logger.debug("Could not find path for %s on disk", objspec) 

158 if not (self.allow_inspection or self.force_inspection): 

159 raise 

160 

161 # Otherwise we try to dynamically import the top-level module. 

162 obj_path = str(objspec) 

163 top_module_name = obj_path.split(".", 1)[0] 

164 logger.debug("Trying to dynamically import %s", top_module_name) 

165 top_module_object = dynamic_import(top_module_name, self.finder.search_paths) 

166 

167 try: 

168 top_module_path = top_module_object.__path__ 

169 if not top_module_path: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 raise ValueError(f"Module {top_module_name} has no paths set") # noqa: TRY301 

171 except (AttributeError, ValueError): 

172 # If the top-level module has no `__path__`, we inspect it as-is, 

173 # and do not try to recurse into submodules (there shouldn't be any in builtin/compiled modules). 

174 logger.debug("Module %s has no paths set (built-in module?). Inspecting it as-is.", top_module_name) 

175 top_module = self._inspect_module(top_module_name) 

176 self.modules_collection.set_member(top_module.path, top_module) 

177 return self._post_load(top_module, obj_path) 

178 

179 # We found paths, and use them to build our intermediate Package or NamespacePackage struct. 

180 logger.debug("Module %s has paths set: %s", top_module_name, top_module_path) 

181 top_module_path = [Path(path) for path in top_module_path] 

182 if len(top_module_path) > 1: 182 ↛ 183line 182 didn't jump to line 183 because the condition on line 182 was never true

183 package = NamespacePackage(top_module_name, top_module_path) 

184 else: 

185 package = Package(top_module_name, top_module_path[0]) 

186 

187 # We have an intermediate package, and an object path: we're ready to load. 

188 logger.debug("Found %s: loading", objspec) 

189 try: 

190 top_module = self._load_package(package, submodules=submodules) 

191 except LoadingError: 

192 logger.exception("Could not load package %s", package) 

193 raise 

194 

195 return self._post_load(top_module, obj_path) 

196 

197 def _fire_load_events(self, obj: Object) -> None: 

198 # Wrapping in tuple() to avoid "dictionary changed size during iteration" errors. 

199 for member in tuple(obj.members.values()): 

200 if member.is_alias: 

201 self.extensions.call("on_alias", alias=member, loader=self) 

202 continue 

203 self.extensions.call("on_object", obj=member, loader=self) 

204 if member.is_module: 

205 self.extensions.call("on_module", mod=member, loader=self) 

206 elif member.is_class: 

207 self.extensions.call("on_class", cls=member, loader=self) 

208 elif member.is_function: 

209 self.extensions.call("on_function", func=member, loader=self) 

210 elif member.is_attribute: 

211 self.extensions.call("on_attribute", attr=member, loader=self) 

212 elif member.is_type_alias: 212 ↛ 214line 212 didn't jump to line 214 because the condition on line 212 was always true

213 self.extensions.call("on_type_alias", type_alias=member, loader=self) 

214 self._fire_load_events(member) # ty:ignore[invalid-argument-type] 

215 

216 def _post_load(self, module: Module, obj_path: str) -> Object | Alias: 

217 # Pre-emptively expand exports (`__all__` values), 

218 # as well as wildcard imports (without ever loading additional packages). 

219 # This is a best-effort to return the most correct API data 

220 # before firing the load events. 

221 # 

222 # Packages that wildcard imports from external, non-loaded packages 

223 # will still have incomplete data, requiring subsequent calls to 

224 # `load()` and/or `resolve_aliases()`. 

225 self.expand_exports(module) 

226 self.expand_wildcards(module, external=False) 

227 # Populate Git information if possible. 

228 module.git_info = GitInfo.from_package(module) 

229 # Package is loaded, we now retrieve the initially requested object, 

230 # fire load events, and return it. 

231 obj = self.modules_collection.get_member(obj_path) 

232 self.extensions.call("on_package", pkg=module, loader=self) 

233 self.extensions.call("on_module", mod=module, loader=self) 

234 self._fire_load_events(module) 

235 return obj 

236 

237 def resolve_aliases( 

238 self, 

239 *, 

240 implicit: bool = False, 

241 external: bool | None = None, 

242 max_iterations: int | None = None, 

243 ) -> tuple[set[str], int]: 

244 """Resolve aliases. 

245 

246 Parameters: 

247 implicit: When false, only try to resolve an alias if it is explicitly exported. 

248 external: When false, don't try to load unspecified modules to resolve aliases. 

249 max_iterations: Maximum number of iterations on the loader modules collection. 

250 

251 Returns: 

252 The unresolved aliases and the number of iterations done. 

253 """ 

254 if max_iterations is None: 

255 max_iterations = float("inf") # ty:ignore[invalid-assignment] 

256 prev_unresolved: set[str] = set() 

257 unresolved: set[str] = set("0") # Init to enter loop. 

258 iteration = 0 

259 collection = self.modules_collection.members 

260 

261 # Before resolving aliases, we try to expand wildcard imports again 

262 # (this was already done in `_post_load()`), 

263 # this time with the user-configured `external` setting, 

264 # and with potentially more packages loaded in the collection, 

265 # allowing to resolve more aliases. 

266 for wildcards_module in list(collection.values()): 

267 self.expand_wildcards(wildcards_module, external=external) 

268 

269 load_failures: set[str] = set() 

270 while unresolved and unresolved != prev_unresolved and iteration < max_iterations: # ty:ignore[unsupported-operator] 

271 prev_unresolved = unresolved - {"0"} 

272 unresolved = set() 

273 resolved: set[str] = set() 

274 iteration += 1 

275 for module_name in list(collection.keys()): 

276 module = collection[module_name] 

277 next_resolved, next_unresolved = self.resolve_module_aliases( 

278 module, 

279 implicit=implicit, 

280 external=external, 

281 load_failures=load_failures, 

282 ) 

283 resolved |= next_resolved 

284 unresolved |= next_unresolved 

285 logger.debug( 

286 "Iteration %s finished, %s aliases resolved, still %s to go", 

287 iteration, 

288 len(resolved), 

289 len(unresolved), 

290 ) 

291 return unresolved, iteration 

292 

293 def expand_exports(self, module: Module, seen: set | None = None) -> None: 

294 """Expand exports: try to recursively expand all module exports (`__all__` values). 

295 

296 See also: [`Module.exports`][griffe.Module.exports]. 

297 

298 Parameters: 

299 module: The module to recurse on. 

300 seen: Used to avoid infinite recursion. 

301 """ 

302 seen = seen or set() 

303 seen.add(module.path) 

304 if module.exports is None: 

305 return 

306 

307 expanded = [] 

308 for export in module.exports: 

309 # It's a name: we resolve it, get the module it comes from, 

310 # recurse into it, and add its exports to the current ones. 

311 if isinstance(export, ExprName): 

312 module_path = export.canonical_path.rsplit(".", 1)[0] # Remove trailing `.__all__`. 

313 try: 

314 next_module = self.modules_collection.get_member(module_path) 

315 except KeyError: 

316 logger.debug("Cannot expand '%s', try pre-loading corresponding package", export.canonical_path) 

317 continue 

318 if next_module.path not in seen: 

319 self.expand_exports(next_module, seen) 

320 try: 

321 expanded += [export for export in next_module.exports if export not in expanded] 

322 except TypeError: 

323 logger.warning("Unsupported item in %s.__all__: %s (use strings only)", module.path, export) 

324 # It's a string, simply add it to the current exports. 

325 else: 

326 expanded.append(export) 

327 module.exports = expanded 

328 

329 # Make sure to expand exports in all modules. 

330 for submodule in module.modules.values(): 

331 if not submodule.is_alias and submodule.path not in seen: 

332 self.expand_exports(submodule, seen) 

333 

334 def expand_wildcards( 

335 self, 

336 obj: Object, 

337 *, 

338 external: bool | None = None, 

339 seen: set | None = None, 

340 ) -> None: 

341 """Expand wildcards: try to recursively expand all found wildcards. 

342 

343 See also: [`Alias.wildcard`][griffe.Alias.wildcard]. 

344 

345 Parameters: 

346 obj: The object and its members to recurse on. 

347 external: When true, try to load unspecified modules to expand wildcards. 

348 seen: Used to avoid infinite recursion. 

349 """ 

350 expanded = [] 

351 to_remove = [] 

352 seen = seen or set() 

353 seen.add(obj.path) 

354 

355 # First we expand wildcard imports and store the objects in a temporary `expanded` variable, 

356 # while also keeping track of the members representing wildcard import, to remove them later. 

357 for member in obj.members.values(): 

358 # Handle a wildcard. 

359 if member.is_alias and member.wildcard: # ty:ignore[possibly-missing-attribute] 

360 package = member.wildcard.split(".", 1)[0] # ty:ignore[possibly-missing-attribute] 

361 not_loaded = obj.package.path != package and package not in self.modules_collection 

362 

363 # Try loading the (unknown) package containing the wildcard importe module (if allowed to). 

364 if not_loaded: 

365 if external is False or (external is None and package != f"_{obj.package.name}"): 

366 continue 

367 try: 

368 self.load(package, try_relative_path=False) 

369 except (ImportError, LoadingError) as error: 

370 logger.debug("Could not expand wildcard import %s in %s: %s", member.name, obj.path, error) 

371 continue 

372 

373 # Try getting the module from which every public object is imported. 

374 try: 

375 target = self.modules_collection.get_member(member.target_path) # ty:ignore[possibly-missing-attribute] 

376 except KeyError: 

377 logger.debug( 

378 "Could not expand wildcard import %s in %s: %s not found in modules collection", 

379 member.name, 

380 obj.path, 

381 cast("Alias", member).target_path, 

382 ) 

383 continue 

384 

385 # Recurse into this module, expanding wildcards there before collecting everything. 

386 if target.path not in seen: 

387 try: 

388 self.expand_wildcards(target, external=external, seen=seen) 

389 except (AliasResolutionError, CyclicAliasError) as error: 

390 logger.debug("Could not expand wildcard import %s in %s: %s", member.name, obj.path, error) 

391 continue 

392 

393 # Collect every imported object. 

394 expanded.extend(self._expand_wildcard(member)) # ty:ignore[invalid-argument-type] 

395 to_remove.append(member.name) 

396 

397 # Recurse in unseen submodules. 

398 elif not member.is_alias and member.is_module and member.path not in seen: 

399 self.expand_wildcards(member, external=external, seen=seen) # ty:ignore[invalid-argument-type] 

400 

401 # Then we remove the members representing wildcard imports. 

402 for name in to_remove: 

403 obj.del_member(name) 

404 

405 # Finally we process the collected objects. 

406 for new_member, alias_lineno, alias_endlineno in expanded: 

407 overwrite = False 

408 already_present = new_member.name in obj.members 

409 self_alias = ( 

410 new_member.is_alias and cast("Alias", new_member).target_path == f"{obj.path}.{new_member.name}" 

411 ) 

412 

413 # If a member with the same name is already present in the current object, 

414 # we only overwrite it if the alias is imported lower in the module 

415 # (meaning that the alias takes precedence at runtime). 

416 if already_present: 

417 old_member = obj.get_member(new_member.name) 

418 old_lineno = old_member.alias_lineno if old_member.is_alias else old_member.lineno 

419 overwrite = alias_lineno > (old_lineno or 0) 

420 

421 # 1. If the expanded member is an alias with a target path equal to its own path, we stop. 

422 # This situation can arise because of Griffe's mishandling of (abusive) wildcard imports. 

423 # We have yet to check how Python handles this itself, or if there's an algorithm 

424 # that we could follow to untangle back-and-forth wildcard imports. 

425 # 2. If the expanded member was already present and we decided not to overwrite it, we stop. 

426 # 3. Otherwise we proceed further. 

427 if not self_alias and (not already_present or overwrite): 

428 alias = Alias( 

429 new_member.name, 

430 new_member, 

431 lineno=alias_lineno, 

432 endlineno=alias_endlineno, 

433 parent=obj, # ty:ignore[invalid-argument-type] 

434 wildcard_imported=True, 

435 ) 

436 # Special case: we avoid overwriting a submodule with an alias. 

437 # Griffe suffers from this limitation where an object cannot store both 

438 # a submodule and a member of the same name, while this poses (almost) no issue in Python. 

439 # We always give precedence to the submodule. 

440 # See the "avoid member-submodule name shadowing" section in the "Python code" docs page. 

441 if already_present: 

442 prev_member = obj.get_member(new_member.name) 

443 with suppress(AliasResolutionError, CyclicAliasError): 

444 if prev_member.is_module: 

445 continue 

446 

447 # Everything went right (supposedly), we add the alias as a member of the current object. 

448 obj.set_member(new_member.name, alias) 

449 

450 def resolve_module_aliases( 

451 self, 

452 obj: Object | Alias, 

453 *, 

454 implicit: bool = False, 

455 external: bool | None = None, 

456 seen: set[str] | None = None, 

457 load_failures: set[str] | None = None, 

458 ) -> tuple[set[str], set[str]]: 

459 """Follow aliases: try to recursively resolve all found aliases. 

460 

461 Parameters: 

462 obj: The object and its members to recurse on. 

463 implicit: When false, only try to resolve an alias if it is explicitly exported. 

464 external: When false, don't try to load unspecified modules to resolve aliases. 

465 seen: Used to avoid infinite recursion. 

466 load_failures: Set of external packages we failed to load (to prevent retries). 

467 

468 Returns: 

469 Both sets of resolved and unresolved aliases. 

470 """ 

471 resolved = set() 

472 unresolved = set() 

473 if load_failures is None: 473 ↛ 474line 473 didn't jump to line 474 because the condition on line 473 was never true

474 load_failures = set() 

475 seen = seen or set() 

476 seen.add(obj.path) 

477 

478 for member in obj.members.values(): 

479 # Handle aliases. 

480 if member.is_alias: 

481 if member.wildcard or member.resolved: # ty:ignore[possibly-missing-attribute] 

482 continue 

483 if not implicit and not member.is_exported: 

484 continue 

485 

486 # Try resolving the alias. If it fails, check if it is because it comes 

487 # from an external package, and decide if we should load that package 

488 # to allow the alias to be resolved at the next iteration (maybe). 

489 try: 

490 member.resolve_target() # ty:ignore[possibly-missing-attribute] 

491 except AliasResolutionError as error: 

492 target = error.alias.target_path 

493 unresolved.add(member.path) 

494 package = target.split(".", 1)[0] 

495 load_module = ( 

496 (external is True or (external is None and package == f"_{obj.package.name}")) 

497 and package not in load_failures 

498 and obj.package.path != package 

499 and package not in self.modules_collection 

500 ) 

501 if load_module: 

502 logger.debug("Failed to resolve alias %s -> %s", member.path, target) 

503 try: 

504 self.load(package, try_relative_path=False) 

505 except (ImportError, LoadingError) as error: 

506 logger.debug("Could not follow alias %s: %s", member.path, error) 

507 load_failures.add(package) 

508 except CyclicAliasError as error: 

509 logger.debug(str(error)) 

510 else: 

511 logger.debug("Alias %s was resolved to %s", member.path, member.final_target.path) # ty:ignore[possibly-missing-attribute] 

512 resolved.add(member.path) 

513 

514 # Recurse into unseen modules and classes. 

515 elif member.kind in {Kind.MODULE, Kind.CLASS} and member.path not in seen: 

516 sub_resolved, sub_unresolved = self.resolve_module_aliases( 

517 member, 

518 implicit=implicit, 

519 external=external, 

520 seen=seen, 

521 load_failures=load_failures, 

522 ) 

523 resolved |= sub_resolved 

524 unresolved |= sub_unresolved 

525 

526 return resolved, unresolved 

527 

528 def stats(self) -> Stats: 

529 """Compute some statistics. 

530 

531 Returns: 

532 Some statistics. 

533 """ 

534 stats = Stats(self) 

535 stats.time_spent_visiting = self._time_stats["time_spent_visiting"] 

536 stats.time_spent_inspecting = self._time_stats["time_spent_inspecting"] 

537 return stats 

538 

539 def _load_package(self, package: Package | NamespacePackage, *, submodules: bool = True) -> Module: 

540 top_module = self._load_module(package.name, package.path, submodules=submodules) 

541 if isinstance(package, NamespacePackage): 

542 return top_module 

543 if package.stubs: 

544 self.expand_wildcards(top_module) 

545 # If stubs are in the package itself, they have been merged while loading modules, 

546 # so only the top-level init module needs to be merged still. 

547 # If stubs are in another package (a stubs-only package), 

548 # then we need to load the entire stubs package to merge everything. 

549 submodules = submodules and package.stubs.parent != package.path.parent 

550 stubs = self._load_module(package.name, package.stubs, submodules=submodules) 

551 return merge_stubs(top_module, stubs) 

552 return top_module 

553 

554 def _load_module( 

555 self, 

556 module_name: str, 

557 module_path: Path | list[Path], 

558 *, 

559 submodules: bool = True, 

560 parent: Module | None = None, 

561 ) -> Module: 

562 try: 

563 return self._load_module_path(module_name, module_path, submodules=submodules, parent=parent) 

564 except SyntaxError as error: 

565 raise LoadingError(f"Syntax error: {error}") from error 

566 except ImportError as error: 

567 raise LoadingError(f"Import error: {error}") from error 

568 except UnicodeDecodeError as error: 

569 raise LoadingError(f"UnicodeDecodeError when loading {module_path}: {error}") from error 

570 except OSError as error: 

571 raise LoadingError(f"OSError when loading {module_path}: {error}") from error 

572 

573 def _load_module_path( 

574 self, 

575 module_name: str, 

576 module_path: Path | list[Path], 

577 *, 

578 submodules: bool = True, 

579 parent: Module | None = None, 

580 ) -> Module: 

581 logger.debug("Loading path %s", module_path) 

582 if isinstance(module_path, list): 

583 module = self._create_module(module_name, module_path) 

584 elif self.force_inspection: 

585 module = self._inspect_module(module_name, module_path, parent) 

586 elif module_path.suffix in {".py", ".pyi"}: 

587 module = self._visit_module(module_name, module_path, parent) 

588 elif self.allow_inspection: 588 ↛ 591line 588 didn't jump to line 591 because the condition on line 588 was always true

589 module = self._inspect_module(module_name, module_path, parent) 

590 else: 

591 raise LoadingError("Cannot load compiled module without inspection") 

592 if parent is None: 

593 self.modules_collection.set_member(module.path, module) 

594 if submodules: 

595 self._load_submodules(module) 

596 return module 

597 

598 def _load_submodules(self, module: Module) -> None: 

599 for subparts, subpath in self.finder.submodules(module): 

600 self._load_submodule(module, subparts, subpath) 

601 

602 def _load_submodule(self, module: Module, subparts: tuple[str, ...], subpath: Path) -> None: 

603 for subpart in subparts: 

604 if "." in subpart: 

605 logger.debug("Skip %s, dots in filenames are not supported", subpath) 

606 return 

607 try: 

608 parent_module = self._get_or_create_parent_module(module, subparts, subpath) 

609 except UnimportableModuleError as error: 

610 # NOTE: Why don't we load submodules when there's no init module in their folder? 

611 # Usually when a folder with Python files does not have an __init__.py module, 

612 # it's because the Python files are scripts that should never be imported. 

613 # Django has manage.py somewhere for example, in a folder without init module. 

614 # This script isn't part of the Python API, as it's meant to be called on the CLI exclusively 

615 # (at least it was the case a few years ago when I was still using Django). 

616 

617 # The other case when there's no init module is when a package is a native namespace package (PEP 420). 

618 # It does not make sense to have a native namespace package inside of a regular package (having init modules above), 

619 # because the regular package above blocks the namespace feature from happening, so I consider it a user error. 

620 # It's true that users could have a native namespace package inside of a pkg_resources-style namespace package, 

621 # but I've never seen this happen. 

622 

623 # It's also true that Python can actually import the module under the (wrongly declared) native namespace package, 

624 # so the Griffe debug log message is a bit misleading, 

625 # but that's because in that case Python acts like the whole tree is a regular package. 

626 # It works when the namespace package appears in only one search path (`sys.path`), 

627 # but will fail if it appears in multiple search paths: Python will only find the first occurrence. 

628 # It's better to not falsely support this, and to warn users. 

629 logger.debug("%s. Missing __init__ module?", error) 

630 return 

631 submodule_name = subparts[-1] 

632 try: 

633 submodule = self._load_module( 

634 submodule_name, 

635 subpath, 

636 submodules=False, 

637 parent=parent_module, 

638 ) 

639 except LoadingError as error: 

640 logger.debug(str(error)) 

641 else: 

642 if submodule_name in parent_module.members: 

643 member = parent_module.members[submodule_name] 

644 if member.is_alias or not member.is_module: 

645 logger.debug( 

646 "Submodule '%s' is shadowing the member at the same path. " 

647 "We recommend renaming the member or the submodule (for example prefixing it with `_`), " 

648 "see https://mkdocstrings.github.io/griffe/best_practices/#avoid-member-submodule-name-shadowing.", 

649 submodule.path, 

650 ) 

651 parent_module.set_member(submodule_name, submodule) 

652 

653 def _create_module(self, module_name: str, module_path: Path | list[Path]) -> Module: 

654 return Module( 

655 module_name, 

656 filepath=module_path, 

657 lines_collection=self.lines_collection, 

658 modules_collection=self.modules_collection, 

659 ) 

660 

661 def _visit_module(self, module_name: str, module_path: Path, parent: Module | None = None) -> Module: 

662 code = module_path.read_text(encoding="utf-8-sig") 

663 if self.store_source: 

664 self.lines_collection[module_path] = code.splitlines(keepends=False) 

665 start = datetime.now(tz=timezone.utc) 

666 module = visit( 

667 module_name, 

668 filepath=module_path, 

669 code=code, 

670 extensions=self.extensions, 

671 parent=parent, 

672 docstring_parser=self.docstring_parser, 

673 docstring_options=self.docstring_options, 

674 lines_collection=self.lines_collection, 

675 modules_collection=self.modules_collection, 

676 ) 

677 elapsed = datetime.now(tz=timezone.utc) - start 

678 self._time_stats["time_spent_visiting"] += elapsed.microseconds 

679 return module 

680 

681 def _inspect_module(self, module_name: str, filepath: Path | None = None, parent: Module | None = None) -> Module: 

682 for prefix in self.ignored_modules: 

683 if module_name.startswith(prefix): 683 ↛ 684line 683 didn't jump to line 684 because the condition on line 683 was never true

684 raise ImportError(f"Ignored module '{module_name}'") 

685 if self.store_source and filepath and filepath.suffix in {".py", ".pyi"}: 

686 self.lines_collection[filepath] = filepath.read_text(encoding="utf-8-sig").splitlines(keepends=False) 

687 start = datetime.now(tz=timezone.utc) 

688 try: 

689 module = inspect( 

690 module_name, 

691 filepath=filepath, 

692 import_paths=self.finder.search_paths, 

693 extensions=self.extensions, 

694 parent=parent, 

695 docstring_parser=self.docstring_parser, 

696 docstring_options=self.docstring_options, 

697 lines_collection=self.lines_collection, 

698 modules_collection=self.modules_collection, 

699 ) 

700 except SystemExit as error: 

701 raise ImportError(f"Importing '{module_name}' raised a system exit") from error 

702 except Exception as error: 

703 raise ImportError(f"Importing '{module_name}' raised an exception") from error 

704 elapsed = datetime.now(tz=timezone.utc) - start 

705 self._time_stats["time_spent_inspecting"] += elapsed.microseconds 

706 return module 

707 

708 def _get_or_create_parent_module( 

709 self, 

710 module: Module, 

711 subparts: tuple[str, ...], 

712 subpath: Path, 

713 ) -> Module: 

714 parent_parts = subparts[:-1] 

715 if not parent_parts: 

716 return module 

717 parent_module = module 

718 parents = list(subpath.parents) 

719 if subpath.stem == "__init__": 

720 parents.pop(0) 

721 for parent_offset, parent_part in enumerate(parent_parts, 2): 

722 module_filepath = parents[len(subparts) - parent_offset] 

723 try: 

724 parent_module = parent_module.get_member(parent_part) 

725 except KeyError as error: 

726 if parent_module.is_namespace_package or parent_module.is_namespace_subpackage: 726 ↛ 731line 726 didn't jump to line 731 because the condition on line 726 was always true

727 next_parent_module = self._create_module(parent_part, [module_filepath]) 

728 parent_module.set_member(parent_part, next_parent_module) 

729 parent_module = next_parent_module 

730 else: 

731 raise UnimportableModuleError(f"Skip {subpath}, it is not importable") from error 

732 else: 

733 parent_namespace = parent_module.is_namespace_package or parent_module.is_namespace_subpackage 

734 if parent_namespace and module_filepath not in parent_module.filepath: 

735 parent_module.filepath.append(module_filepath) 

736 return parent_module 

737 

738 def _expand_wildcard(self, wildcard_obj: Alias) -> list[tuple[Object | Alias, int | None, int | None]]: 

739 module = self.modules_collection.get_member(wildcard_obj.wildcard) # ty:ignore[invalid-argument-type] 

740 return [ 

741 (imported_member, wildcard_obj.alias_lineno, wildcard_obj.alias_endlineno) 

742 for imported_member in module.members.values() 

743 if imported_member.is_wildcard_exposed 

744 ] 

745 

746 

747def load( 

748 objspec: str | Path | None = None, 

749 /, 

750 *, 

751 submodules: bool = True, 

752 try_relative_path: bool = True, 

753 extensions: Extensions | None = None, 

754 search_paths: Sequence[str | Path] | None = None, 

755 docstring_parser: DocstringStyle | Parser | None = None, 

756 docstring_options: DocstringOptions | None = None, 

757 lines_collection: LinesCollection | None = None, 

758 modules_collection: ModulesCollection | None = None, 

759 allow_inspection: bool = True, 

760 force_inspection: bool = False, 

761 store_source: bool = True, 

762 find_stubs_package: bool = False, 

763 resolve_aliases: bool = False, 

764 resolve_external: bool | None = None, 

765 resolve_implicit: bool = False, 

766) -> Object | Alias: 

767 """Load and return a Griffe object. 

768 

769 In Griffe's context, loading means: 

770 

771 - searching for a package, and finding it on the file system or as a builtin module 

772 (see the [`ModuleFinder`][griffe.ModuleFinder] class for more information) 

773 - extracting information from each of its (sub)modules, by either parsing 

774 the source code (see the [`visit`][griffe.visit] function) 

775 or inspecting the module at runtime (see the [`inspect`][griffe.inspect] function) 

776 

777 The extracted information is stored in a collection of modules, which can be queried later. 

778 Each collected module is a tree of objects, representing the structure of the module. 

779 See the [`Module`][griffe.Module], [`Class`][griffe.Class], 

780 [`Function`][griffe.Function], [`Attribute`][griffe.Attribute], and 

781 [`TypeAlias`][griffe.TypeAlias] classes for more information. 

782 

783 The main class used to load modules is [`GriffeLoader`][griffe.GriffeLoader]. 

784 Convenience functions like this one and [`load_git`][griffe.load_git] are also available. 

785 

786 Example: 

787 ```python 

788 import griffe 

789 

790 module = griffe.load(...) 

791 ``` 

792 

793 This is a shortcut for: 

794 

795 ```python 

796 from griffe import GriffeLoader 

797 

798 loader = GriffeLoader(...) 

799 module = loader.load(...) 

800 ``` 

801 

802 See the documentation for the loader: [`GriffeLoader`][griffe.GriffeLoader]. 

803 

804 Parameters: 

805 objspec: The Python path of an object, or file path to a module. 

806 submodules: Whether to recurse on the submodules. 

807 This parameter only makes sense when loading a package (top-level module). 

808 try_relative_path: Whether to try finding the module as a relative path. 

809 extensions: The extensions to use. 

810 search_paths: The paths to search into. 

811 docstring_parser: The docstring parser to use. By default, no parsing is done. 

812 docstring_options: Docstring parsing options.