Coverage for src/_griffe/loader.py: 91.81%

1# This module contains all the logic for loading API data from sources or compiled modules. 

2 

3from __future__ import annotations 

4 

5import sys 

6from contextlib import suppress 

7from datetime import datetime, timezone 

8from pathlib import Path 

9from typing import TYPE_CHECKING, Any, ClassVar, cast 

10 

11from _griffe.agents.inspector import inspect 

12from _griffe.agents.visitor import visit 

13from _griffe.collections import LinesCollection, ModulesCollection 

14from _griffe.enumerations import Kind 

15from _griffe.exceptions import ( 

16 AliasResolutionError, 

17 CyclicAliasError, 

18 LoadingError, 

19 UnimportableModuleError, 

20) 

21from _griffe.expressions import ExprName 

22from _griffe.extensions.base import Extensions, load_extensions 

23from _griffe.finder import ModuleFinder, NamespacePackage, Package 

24from _griffe.git import tmp_worktree 

25from _griffe.importer import dynamic_import 

26from _griffe.logger import logger 

27from _griffe.merger import merge_stubs 

28from _griffe.models import Alias, Module, Object 

29from _griffe.stats import Stats 

30 

31if TYPE_CHECKING: 

32 from collections.abc import Sequence 

33 

34 from _griffe.enumerations import Parser 

35 

36_builtin_modules: set[str] = set(sys.builtin_module_names) 

37 

38 

39class GriffeLoader: 

40 """The Griffe loader, allowing to load data from modules.""" 

41 

42 ignored_modules: ClassVar[set[str]] = {"debugpy", "_pydev"} 

43 """Special modules to ignore when loading. 

44 

45 For example, `debugpy` and `_pydev` are used when debugging with VSCode and should generally never be loaded. 

46 """ 

47 

48 def __init__( 

49 self, 

50 *, 

51 extensions: Extensions | None = None, 

52 search_paths: Sequence[str | Path] | None = None, 

53 docstring_parser: Parser | None = None, 

54 docstring_options: dict[str, Any] | None = None, 

55 lines_collection: LinesCollection | None = None, 

56 modules_collection: ModulesCollection | None = None, 

57 allow_inspection: bool = True, 

58 force_inspection: bool = False, 

59 store_source: bool = True, 

60 ) -> None: 

61 """Initialize the loader. 

62 

63 Parameters: 

64 extensions: The extensions to use. 

65 search_paths: The paths to search into. 

66 docstring_parser: The docstring parser to use. By default, no parsing is done. 

67 docstring_options: Additional docstring parsing options. 

68 lines_collection: A collection of source code lines. 

69 modules_collection: A collection of modules. 

70 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

71 store_source: Whether to store code source in the lines collection. 

72 """ 

73 self.extensions: Extensions = extensions or load_extensions() 

74 """Loaded Griffe extensions.""" 

75 self.docstring_parser: Parser | None = docstring_parser 

76 """Selected docstring parser.""" 

77 self.docstring_options: dict[str, Any] = docstring_options or {} 

78 """Configured parsing options.""" 

79 self.lines_collection: LinesCollection = lines_collection or LinesCollection() 

80 """Collection of source code lines.""" 

81 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection() 

82 """Collection of modules.""" 

83 self.allow_inspection: bool = allow_inspection 

84 """Whether to allow inspecting (importing) modules for which we can't find sources.""" 

85 self.force_inspection: bool = force_inspection 

86 """Whether to force inspecting (importing) modules, even when sources were found.""" 

87 self.store_source: bool = store_source 

88 """Whether to store source code in the lines collection.""" 

89 self.finder: ModuleFinder = ModuleFinder(search_paths) 

90 """The module source finder.""" 

91 self._time_stats: dict = { 

92 "time_spent_visiting": 0, 

93 "time_spent_inspecting": 0, 

94 } 

95 

96 def load( 

97 self, 

98 objspec: str | Path | None = None, 

99 /, 

100 *, 

101 submodules: bool = True, 

102 try_relative_path: bool = True, 

103 find_stubs_package: bool = False, 

104 ) -> Object | Alias: 

105 """Load an object as a Griffe object, given its Python or file path. 

106 

107 Note that this will load the whole object's package, 

108 and return only the specified object. 

109 The rest of the package can be accessed from the returned object 

110 with regular methods and properties (`parent`, `members`, etc.). 

111 

112 Examples: 

113 >>> loader.load("griffe.Module") 

114 Alias("Module", "_griffe.models.Module") 

115 

116 Parameters: 

117 objspec: The Python path of an object, or file path to a module. 

118 submodules: Whether to recurse on the submodules. 

119 This parameter only makes sense when loading a package (top-level module). 

120 try_relative_path: Whether to try finding the module as a relative path. 

121 find_stubs_package: Whether to search for stubs-only package. 

122 If both the package and its stubs are found, they'll be merged together. 

123 If only the stubs are found, they'll be used as the package itself. 

124 

125 Raises: 

126 LoadingError: When loading a module failed for various reasons. 

127 ModuleNotFoundError: When a module was not found and inspection is disallowed. 

128 

129 Returns: 

130 A Griffe object. 

131 """ 

132 obj_path: str 

133 package = None 

134 top_module = None 

135 

136 # We always start by searching paths on the disk, 

137 # even if inspection is forced. 

138 logger.debug("Searching path(s) for %s", objspec) 

139 try: 

140 obj_path, package = self.finder.find_spec( 

141 objspec, # type: ignore[arg-type] 

142 try_relative_path=try_relative_path, 

143 find_stubs_package=find_stubs_package, 

144 ) 

145 except ModuleNotFoundError: 

146 # If we couldn't find paths on disk and inspection is disabled, 

147 # re-raise ModuleNotFoundError. 

148 logger.debug("Could not find path for %s on disk", objspec) 

149 if not (self.allow_inspection or self.force_inspection): 

150 raise 

151 

152 # Otherwise we try to dynamically import the top-level module. 

153 obj_path = str(objspec) 

154 top_module_name = obj_path.split(".", 1)[0] 

155 logger.debug("Trying to dynamically import %s", top_module_name) 

156 top_module_object = dynamic_import(top_module_name, self.finder.search_paths) 

157 

158 try: 

159 top_module_path = top_module_object.__path__ 

160 if not top_module_path: 160 ↛ 161line 160 didn't jump to line 161 because the condition on line 160 was never true

161 raise ValueError(f"Module {top_module_name} has no paths set") # noqa: TRY301 

162 except (AttributeError, ValueError): 

163 # If the top-level module has no `__path__`, we inspect it as-is, 

164 # and do not try to recurse into submodules (there shouldn't be any in builtin/compiled modules). 

165 logger.debug("Module %s has no paths set (built-in module?). Inspecting it as-is.", top_module_name) 

166 top_module = self._inspect_module(top_module_name) 

167 self.modules_collection.set_member(top_module.path, top_module) 

168 return self._post_load(top_module, obj_path) 

169 

170 # We found paths, and use them to build our intermediate Package or NamespacePackage struct. 

171 logger.debug("Module %s has paths set: %s", top_module_name, top_module_path) 

172 top_module_path = [Path(path) for path in top_module_path] 

173 if len(top_module_path) > 1: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true

174 package = NamespacePackage(top_module_name, top_module_path) 

175 else: 

176 package = Package(top_module_name, top_module_path[0]) 

177 

178 # We have an intermediate package, and an object path: we're ready to load. 

179 logger.debug("Found %s: loading", objspec) 

180 try: 

181 top_module = self._load_package(package, submodules=submodules) 

182 except LoadingError: 

183 logger.exception("Could not load package %s", package) 

184 raise 

185 

186 return self._post_load(top_module, obj_path) 

187 

188 def _post_load(self, module: Module, obj_path: str) -> Object | Alias: 

189 # Pre-emptively expand exports (`__all__` values), 

190 # as well as wildcard imports (without ever loading additional packages). 

191 # This is a best-effort to return the most correct API data 

192 # before firing the `on_package_loaded` event, 

193 # because extensions registering hooks for this event 

194 # might trigger computation of inherited members, 

195 # which are cached, and could be cached too early 

196 # (aliases to base class still being unresolvable, 

197 # preventing to fetch some inherited members). 

198 # 

199 # Another solution to this "too-early-computation-of-inherited-members" 

200 # issue would be to stop caching resolved bases and inherited members, 

201 # but we would have to measure the performance impact of this change, 

202 # as each use of `obj["name"]` would compute resolved bases, MRO 

203 # and inherited members again. 

204 # 

205 # Packages that wildcard imports from external, non-loaded packages 

206 # will still have incomplete data, requiring subsequent calls to 

207 # `load()` and/or `resolve_aliases()`. 

208 self.expand_exports(module) 

209 self.expand_wildcards(module, external=False) 

210 # Package is loaded, we now retrieve the initially requested object and return it. 

211 obj = self.modules_collection.get_member(obj_path) 

212 self.extensions.call("on_package_loaded", pkg=module, loader=self) 

213 return obj 

214 

215 def resolve_aliases( 

216 self, 

217 *, 

218 implicit: bool = False, 

219 external: bool | None = None, 

220 max_iterations: int | None = None, 

221 ) -> tuple[set[str], int]: 

222 """Resolve aliases. 

223 

224 Parameters: 

225 implicit: When false, only try to resolve an alias if it is explicitly exported. 

226 external: When false, don't try to load unspecified modules to resolve aliases. 

227 max_iterations: Maximum number of iterations on the loader modules collection. 

228 

229 Returns: 

230 The unresolved aliases and the number of iterations done. 

231 """ 

232 if max_iterations is None: 

233 max_iterations = float("inf") # type: ignore[assignment] 

234 prev_unresolved: set[str] = set() 

235 unresolved: set[str] = set("0") # init to enter loop 

236 iteration = 0 

237 collection = self.modules_collection.members 

238 

239 # Before resolving aliases, we try to expand wildcard imports again 

240 # (this was already done in `_post_load()`), 

241 # this time with the user-configured `external` setting, 

242 # and with potentially more packages loaded in the collection, 

243 # allowing to resolve more aliases. 

244 for wildcards_module in list(collection.values()): 

245 self.expand_wildcards(wildcards_module, external=external) 

246 

247 load_failures: set[str] = set() 

248 while unresolved and unresolved != prev_unresolved and iteration < max_iterations: # type: ignore[operator] 

249 prev_unresolved = unresolved - {"0"} 

250 unresolved = set() 

251 resolved: set[str] = set() 

252 iteration += 1 

253 for module_name in list(collection.keys()): 

254 module = collection[module_name] 

255 next_resolved, next_unresolved = self.resolve_module_aliases( 

256 module, 

257 implicit=implicit, 

258 external=external, 

259 load_failures=load_failures, 

260 ) 

261 resolved |= next_resolved 

262 unresolved |= next_unresolved 

263 logger.debug( 

264 "Iteration %s finished, {len(resolved)} aliases resolved, still {len(unresolved)} to go", 

265 iteration, 

266 ) 

267 return unresolved, iteration 

268 

269 def expand_exports(self, module: Module, seen: set | None = None) -> None: 

270 """Expand exports: try to recursively expand all module exports (`__all__` values). 

271 

272 Parameters: 

273 module: The module to recurse on. 

274 seen: Used to avoid infinite recursion. 

275 """ 

276 seen = seen or set() 

277 seen.add(module.path) 

278 if module.exports is None: 

279 return 

280 

281 expanded = set() 

282 for export in module.exports: 

283 # It's a name: we resolve it, get the module it comes from, 

284 # recurse into it, and add its exports to the current ones. 

285 if isinstance(export, ExprName): 

286 module_path = export.canonical_path.rsplit(".", 1)[0] # remove trailing .__all__ 

287 try: 

288 next_module = self.modules_collection.get_member(module_path) 

289 except KeyError: 

290 logger.debug("Cannot expand '%s', try pre-loading corresponding package", export.canonical_path) 

291 continue 

292 if next_module.path not in seen: 

293 self.expand_exports(next_module, seen) 

294 try: 

295 expanded |= next_module.exports 

296 except TypeError: 

297 logger.warning("Unsupported item in %s.__all__: %s (use strings only)", module.path, export) 

298 # It's a string, simply add it to the current exports. 

299 else: 

300 expanded.add(export) 

301 module.exports = expanded 

302 

303 # Make sure to expand exports in all modules. 

304 for submodule in module.modules.values(): 

305 if not submodule.is_alias and submodule.path not in seen: 

306 self.expand_exports(submodule, seen) 

307 

308 def expand_wildcards( 

309 self, 

310 obj: Object, 

311 *, 

312 external: bool | None = None, 

313 seen: set | None = None, 

314 ) -> None: 

315 """Expand wildcards: try to recursively expand all found wildcards. 

316 

317 Parameters: 

318 obj: The object and its members to recurse on. 

319 external: When true, try to load unspecified modules to expand wildcards. 

320 seen: Used to avoid infinite recursion. 

321 """ 

322 expanded = [] 

323 to_remove = [] 

324 seen = seen or set() 

325 seen.add(obj.path) 

326 

327 # First we expand wildcard imports and store the objects in a temporary `expanded` variable, 

328 # while also keeping track of the members representing wildcard import, to remove them later. 

329 for member in obj.members.values(): 

330 # Handle a wildcard. 

331 if member.is_alias and member.wildcard: # type: ignore[union-attr] # we know it's an alias 

332 package = member.wildcard.split(".", 1)[0] # type: ignore[union-attr] 

333 not_loaded = obj.package.path != package and package not in self.modules_collection 

334 

335 # Try loading the (unknown) package containing the wildcard importe module (if allowed to). 

336 if not_loaded: 

337 if external is False or (external is None and package != f"_{obj.package.name}"): 

338 continue 

339 try: 

340 self.load(package, try_relative_path=False) 

341 except (ImportError, LoadingError) as error: 

342 logger.debug("Could not expand wildcard import %s in %s: %s", member.name, obj.path, error) 

343 continue 

344 

345 # Try getting the module from which every public object is imported. 

346 try: 

347 target = self.modules_collection.get_member(member.target_path) # type: ignore[union-attr] 

348 except KeyError: 

349 logger.debug( 

350 "Could not expand wildcard import %s in %s: %s not found in modules collection", 

351 member.name, 

352 obj.path, 

353 cast(Alias, member).target_path, 

354 ) 

355 continue 

356 

357 # Recurse into this module, expanding wildcards there before collecting everything. 

358 if target.path not in seen: 

359 try: 

360 self.expand_wildcards(target, external=external, seen=seen) 

361 except (AliasResolutionError, CyclicAliasError) as error: 

362 logger.debug("Could not expand wildcard import %s in %s: %s", member.name, obj.path, error) 

363 continue 

364 

365 # Collect every imported object. 

366 expanded.extend(self._expand_wildcard(member)) # type: ignore[arg-type] 

367 to_remove.append(member.name) 

368 

369 # Recurse in unseen submodules. 

370 elif not member.is_alias and member.is_module and member.path not in seen: 

371 self.expand_wildcards(member, external=external, seen=seen) # type: ignore[arg-type] 

372 

373 # Then we remove the members representing wildcard imports. 

374 for name in to_remove: 

375 obj.del_member(name) 

376 

377 # Finally we process the collected objects. 

378 for new_member, alias_lineno, alias_endlineno in expanded: 

379 overwrite = False 

380 already_present = new_member.name in obj.members 

381 self_alias = new_member.is_alias and cast(Alias, new_member).target_path == f"{obj.path}.{new_member.name}" 

382 

383 # If a member with the same name is already present in the current object, 

384 # we only overwrite it if the alias is imported lower in the module 

385 # (meaning that the alias takes precedence at runtime). 

386 if already_present: 

387 old_member = obj.get_member(new_member.name) 

388 old_lineno = old_member.alias_lineno if old_member.is_alias else old_member.lineno 

389 overwrite = alias_lineno > (old_lineno or 0) # type: ignore[operator] 

390 

391 # 1. If the expanded member is an alias with a target path equal to its own path, we stop. 

392 # This situation can arise because of Griffe's mishandling of (abusive) wildcard imports. 

393 # We have yet to check how Python handles this itself, or if there's an algorithm 

394 # that we could follow to untangle back-and-forth wildcard imports. 

395 # 2. If the expanded member was already present and we decided not to overwrite it, we stop. 

396 # 3. Otherwise we proceed further. 

397 if not self_alias and (not already_present or overwrite): 

398 alias = Alias( 

399 new_member.name, 

400 new_member, 

401 lineno=alias_lineno, 

402 endlineno=alias_endlineno, 

403 parent=obj, # type: ignore[arg-type] 

404 ) 

405 # Special case: we avoid overwriting a submodule with an alias pointing to it. 

406 # Griffe suffers from this design flaw where an object cannot store both 

407 # a submodule and a member of the same name, while this poses (almost) no issue in Python. 

408 # We at least prevent this case where a submodule is overwritten by an imported version of itself. 

409 if already_present: 

410 prev_member = obj.get_member(new_member.name) 

411 with suppress(AliasResolutionError, CyclicAliasError): 

412 if prev_member.is_module: 

413 if prev_member.is_alias: 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true

414 prev_member = prev_member.final_target 

415 if alias.final_target is prev_member: 415 ↛ 417line 415 didn't jump to line 417 because the condition on line 415 was never true

416 # Alias named after the module it targets: skip to avoid cyclic aliases. 

417 continue 

418 

419 # Everything went right (supposedly), we add the alias as a member of the current object. 

420 obj.set_member(new_member.name, alias) 

421 self.extensions.call("on_wildcard_expansion", alias=alias, loader=self) 

422 

423 def resolve_module_aliases( 

424 self, 

425 obj: Object | Alias, 

426 *, 

427 implicit: bool = False, 

428 external: bool | None = None, 

429 seen: set[str] | None = None, 

430 load_failures: set[str] | None = None, 

431 ) -> tuple[set[str], set[str]]: 

432 """Follow aliases: try to recursively resolve all found aliases. 

433 

434 Parameters: 

435 obj: The object and its members to recurse on. 

436 implicit: When false, only try to resolve an alias if it is explicitly exported. 

437 external: When false, don't try to load unspecified modules to resolve aliases. 

438 seen: Used to avoid infinite recursion. 

439 load_failures: Set of external packages we failed to load (to prevent retries). 

440 

441 Returns: 

442 Both sets of resolved and unresolved aliases. 

443 """ 

444 resolved = set() 

445 unresolved = set() 

446 if load_failures is None: 446 ↛ 447line 446 didn't jump to line 447 because the condition on line 446 was never true

447 load_failures = set() 

448 seen = seen or set() 

449 seen.add(obj.path) 

450 

451 for member in obj.members.values(): 

452 # Handle aliases. 

453 if member.is_alias: 

454 if member.wildcard or member.resolved: # type: ignore[union-attr] 

455 continue 

456 if not implicit and not member.is_exported: 

457 continue 

458 

459 # Try resolving the alias. If it fails, check if it is because it comes 

460 # from an external package, and decide if we should load that package 

461 # to allow the alias to be resolved at the next iteration (maybe). 

462 try: 

463 member.resolve_target() # type: ignore[union-attr] 

464 except AliasResolutionError as error: 

465 target = error.alias.target_path 

466 unresolved.add(member.path) 

467 package = target.split(".", 1)[0] 

468 load_module = ( 

469 (external is True or (external is None and package == f"_{obj.package.name}")) 

470 and package not in load_failures 

471 and obj.package.path != package 

472 and package not in self.modules_collection 

473 ) 

474 if load_module: 

475 logger.debug("Failed to resolve alias %s -> %s", member.path, target) 

476 try: 

477 self.load(package, try_relative_path=False) 

478 except (ImportError, LoadingError) as error: 

479 logger.debug("Could not follow alias %s: %s", member.path, error) 

480 load_failures.add(package) 

481 except CyclicAliasError as error: 

482 logger.debug(str(error)) 

483 else: 

484 logger.debug("Alias %s was resolved to %s", member.path, member.final_target.path) # type: ignore[union-attr] 

485 resolved.add(member.path) 

486 

487 # Recurse into unseen modules and classes. 

488 elif member.kind in {Kind.MODULE, Kind.CLASS} and member.path not in seen: 

489 sub_resolved, sub_unresolved = self.resolve_module_aliases( 

490 member, 

491 implicit=implicit, 

492 external=external, 

493 seen=seen, 

494 load_failures=load_failures, 

495 ) 

496 resolved |= sub_resolved 

497 unresolved |= sub_unresolved 

498 

499 return resolved, unresolved 

500 

501 def stats(self) -> Stats: 

502 """Compute some statistics. 

503 

504 Returns: 

505 Some statistics. 

506 """ 

507 stats = Stats(self) 

508 stats.time_spent_visiting = self._time_stats["time_spent_visiting"] 

509 stats.time_spent_inspecting = self._time_stats["time_spent_inspecting"] 

510 return stats 

511 

512 def _load_package(self, package: Package | NamespacePackage, *, submodules: bool = True) -> Module: 

513 top_module = self._load_module(package.name, package.path, submodules=submodules) 

514 self.modules_collection.set_member(top_module.path, top_module) 

515 if isinstance(package, NamespacePackage): 

516 return top_module 

517 if package.stubs: 

518 self.expand_wildcards(top_module) 

519 # If stubs are in the package itself, they have been merged while loading modules, 

520 # so only the top-level init module needs to be merged still. 

521 # If stubs are in another package (a stubs-only package), 

522 # then we need to load the entire stubs package to merge everything. 

523 submodules = submodules and package.stubs.parent != package.path.parent 

524 stubs = self._load_module(package.name, package.stubs, submodules=submodules) 

525 return merge_stubs(top_module, stubs) 

526 return top_module 

527 

528 def _load_module( 

529 self, 

530 module_name: str, 

531 module_path: Path | list[Path], 

532 *, 

533 submodules: bool = True, 

534 parent: Module | None = None, 

535 ) -> Module: 

536 try: 

537 return self._load_module_path(module_name, module_path, submodules=submodules, parent=parent) 

538 except SyntaxError as error: 

539 raise LoadingError(f"Syntax error: {error}") from error 

540 except ImportError as error: 

541 raise LoadingError(f"Import error: {error}") from error 

542 except UnicodeDecodeError as error: 

543 raise LoadingError(f"UnicodeDecodeError when loading {module_path}: {error}") from error 

544 except OSError as error: 

545 raise LoadingError(f"OSError when loading {module_path}: {error}") from error 

546 

547 def _load_module_path( 

548 self, 

549 module_name: str, 

550 module_path: Path | list[Path], 

551 *, 

552 submodules: bool = True, 

553 parent: Module | None = None, 

554 ) -> Module: 

555 logger.debug("Loading path %s", module_path) 

556 if isinstance(module_path, list): 

557 module = self._create_module(module_name, module_path) 

558 elif self.force_inspection: 

559 module = self._inspect_module(module_name, module_path, parent) 

560 elif module_path.suffix in {".py", ".pyi"}: 

561 module = self._visit_module(module_name, module_path, parent) 

562 elif self.allow_inspection: 562 ↛ 565line 562 didn't jump to line 565 because the condition on line 562 was always true

563 module = self._inspect_module(module_name, module_path, parent) 

564 else: 

565 raise LoadingError("Cannot load compiled module without inspection") 

566 if submodules: 

567 self._load_submodules(module) 

568 return module 

569 

570 def _load_submodules(self, module: Module) -> None: 

571 for subparts, subpath in self.finder.submodules(module): 

572 self._load_submodule(module, subparts, subpath) 

573 

574 def _load_submodule(self, module: Module, subparts: tuple[str, ...], subpath: Path) -> None: 

575 for subpart in subparts: 

576 if "." in subpart: 

577 logger.debug("Skip %s, dots in filenames are not supported", subpath) 

578 return 

579 try: 

580 parent_module = self._get_or_create_parent_module(module, subparts, subpath) 

581 except UnimportableModuleError as error: 

582 # NOTE: Why don't we load submodules when there's no init module in their folder? 

583 # Usually when a folder with Python files does not have an __init__.py module, 

584 # it's because the Python files are scripts that should never be imported. 

585 # Django has manage.py somewhere for example, in a folder without init module. 

586 # This script isn't part of the Python API, as it's meant to be called on the CLI exclusively 

587 # (at least it was the case a few years ago when I was still using Django). 

588 

589 # The other case when there's no init module is when a package is a native namespace package (PEP 420). 

590 # It does not make sense to have a native namespace package inside of a regular package (having init modules above), 

591 # because the regular package above blocks the namespace feature from happening, so I consider it a user error. 

592 # It's true that users could have a native namespace package inside of a pkg_resources-style namespace package, 

593 # but I've never seen this happen. 

594 

595 # It's also true that Python can actually import the module under the (wrongly declared) native namespace package, 

596 # so the Griffe debug log message is a bit misleading, 

597 # but that's because in that case Python acts like the whole tree is a regular package. 

598 # It works when the namespace package appears in only one search path (`sys.path`), 

599 # but will fail if it appears in multiple search paths: Python will only find the first occurrence. 

600 # It's better to not falsely support this, and to warn users. 

601 logger.debug("%s. Missing __init__ module?", error) 

602 return 

603 submodule_name = subparts[-1] 

604 try: 

605 submodule = self._load_module( 

606 submodule_name, 

607 subpath, 

608 submodules=False, 

609 parent=parent_module, 

610 ) 

611 except LoadingError as error: 

612 logger.debug(str(error)) 

613 else: 

614 if submodule_name in parent_module.members: 

615 member = parent_module.members[submodule_name] 

616 if member.is_alias or not member.is_module: 

617 logger.debug( 

618 "Submodule '%s' is shadowing the member at the same path. " 

619 "We recommend renaming the member or the submodule (for example prefixing it with `_`), " 

620 "see https://mkdocstrings.github.io/griffe/best_practices/#avoid-member-submodule-name-shadowing.", 

621 submodule.path, 

622 ) 

623 parent_module.set_member(submodule_name, submodule) 

624 

625 def _create_module(self, module_name: str, module_path: Path | list[Path]) -> Module: 

626 return Module( 

627 module_name, 

628 filepath=module_path, 

629 lines_collection=self.lines_collection, 

630 modules_collection=self.modules_collection, 

631 ) 

632 

633 def _visit_module(self, module_name: str, module_path: Path, parent: Module | None = None) -> Module: 

634 code = module_path.read_text(encoding="utf8") 

635 if self.store_source: 

636 self.lines_collection[module_path] = code.splitlines(keepends=False) 

637 start = datetime.now(tz=timezone.utc) 

638 module = visit( 

639 module_name, 

640 filepath=module_path, 

641 code=code, 

642 extensions=self.extensions, 

643 parent=parent, 

644 docstring_parser=self.docstring_parser, 

645 docstring_options=self.docstring_options, 

646 lines_collection=self.lines_collection, 

647 modules_collection=self.modules_collection, 

648 ) 

649 elapsed = datetime.now(tz=timezone.utc) - start 

650 self._time_stats["time_spent_visiting"] += elapsed.microseconds 

651 return module 

652 

653 def _inspect_module(self, module_name: str, filepath: Path | None = None, parent: Module | None = None) -> Module: 

654 for prefix in self.ignored_modules: 

655 if module_name.startswith(prefix): 655 ↛ 656line 655 didn't jump to line 656 because the condition on line 655 was never true

656 raise ImportError(f"Ignored module '{module_name}'") 

657 if self.store_source and filepath and filepath.suffix in {".py", ".pyi"}: 

658 self.lines_collection[filepath] = filepath.read_text(encoding="utf8").splitlines(keepends=False) 

659 start = datetime.now(tz=timezone.utc) 

660 try: 

661 module = inspect( 

662 module_name, 

663 filepath=filepath, 

664 import_paths=self.finder.search_paths, 

665 extensions=self.extensions, 

666 parent=parent, 

667 docstring_parser=self.docstring_parser, 

668 docstring_options=self.docstring_options, 

669 lines_collection=self.lines_collection, 

670 modules_collection=self.modules_collection, 

671 ) 

672 except SystemExit as error: 

673 raise ImportError(f"Importing '{module_name}' raised a system exit") from error 

674 elapsed = datetime.now(tz=timezone.utc) - start 

675 self._time_stats["time_spent_inspecting"] += elapsed.microseconds 

676 return module 

677 

678 def _get_or_create_parent_module( 

679 self, 

680 module: Module, 

681 subparts: tuple[str, ...], 

682 subpath: Path, 

683 ) -> Module: 

684 parent_parts = subparts[:-1] 

685 if not parent_parts: 

686 return module 

687 parent_module = module 

688 parents = list(subpath.parents) 

689 if subpath.stem == "__init__": 

690 parents.pop(0) 

691 for parent_offset, parent_part in enumerate(parent_parts, 2): 

692 module_filepath = parents[len(subparts) - parent_offset] 

693 try: 

694 parent_module = parent_module.get_member(parent_part) 

695 except KeyError as error: 

696 if parent_module.is_namespace_package or parent_module.is_namespace_subpackage: 

697 next_parent_module = self._create_module(parent_part, [module_filepath]) 

698 parent_module.set_member(parent_part, next_parent_module) 

699 parent_module = next_parent_module 

700 else: 

701 raise UnimportableModuleError(f"Skip {subpath}, it is not importable") from error 

702 else: 

703 parent_namespace = parent_module.is_namespace_package or parent_module.is_namespace_subpackage 

704 if parent_namespace and module_filepath not in parent_module.filepath: # type: ignore[operator] 

705 parent_module.filepath.append(module_filepath) # type: ignore[union-attr] 

706 return parent_module 

707 

708 def _expand_wildcard(self, wildcard_obj: Alias) -> list[tuple[Object | Alias, int | None, int | None]]: 

709 module = self.modules_collection.get_member(wildcard_obj.wildcard) # type: ignore[arg-type] # we know it's a wildcard 

710 return [ 

711 (imported_member, wildcard_obj.alias_lineno, wildcard_obj.alias_endlineno) 

712 for imported_member in module.members.values() 

713 if imported_member.is_wildcard_exposed 

714 ] 

715 

716 

717def load( 

718 objspec: str | Path | None = None, 

719 /, 

720 *, 

721 submodules: bool = True, 

722 try_relative_path: bool = True, 

723 extensions: Extensions | None = None, 

724 search_paths: Sequence[str | Path] | None = None, 

725 docstring_parser: Parser | None = None, 

726 docstring_options: dict[str, Any] | None = None, 

727 lines_collection: LinesCollection | None = None, 

728 modules_collection: ModulesCollection | None = None, 

729 allow_inspection: bool = True, 

730 force_inspection: bool = False, 

731 store_source: bool = True, 

732 find_stubs_package: bool = False, 

733 resolve_aliases: bool = False, 

734 resolve_external: bool | None = None, 

735 resolve_implicit: bool = False, 

736) -> Object | Alias: 

737 """Load and return a Griffe object. 

738 

739 In Griffe's context, loading means: 

740 

741 - searching for a package, and finding it on the file system or as a builtin module 

742 (see the [`ModuleFinder`][griffe.ModuleFinder] class for more information) 

743 - extracting information from each of its (sub)modules, by either parsing 

744 the source code (see the [`visit`][griffe.visit] function) 

745 or inspecting the module at runtime (see the [`inspect`][griffe.inspect] function) 

746 

747 The extracted information is stored in a collection of modules, which can be queried later. 

748 Each collected module is a tree of objects, representing the structure of the module. 

749 See the [`Module`][griffe.Module], [`Class`][griffe.Class], 

750 [`Function`][griffe.Function], and [`Attribute`][griffe.Attribute] classes 

751 for more information. 

752 

753 The main class used to load modules is [`GriffeLoader`][griffe.GriffeLoader]. 

754 Convenience functions like this one and [`load_git`][griffe.load_git] are also available. 

755 

756 Example: 

757 ```python 

758 import griffe 

759 

760 module = griffe.load(...) 

761 ``` 

762 

763 This is a shortcut for: 

764 

765 ```python 

766 from griffe import GriffeLoader 

767 

768 loader = GriffeLoader(...) 

769 module = loader.load(...) 

770 ``` 

771 

772 See the documentation for the loader: [`GriffeLoader`][griffe.GriffeLoader]. 

773 

774 Parameters: 

775 objspec: The Python path of an object, or file path to a module. 

776 submodules: Whether to recurse on the submodules. 

777 This parameter only makes sense when loading a package (top-level module). 

778 try_relative_path: Whether to try finding the module as a relative path. 

779 extensions: The extensions to use. 

780 search_paths: The paths to search into. 

781 docstring_parser: The docstring parser to use. By default, no parsing is done. 

782 docstring_options: Additional docstring parsing options. 

783 lines_collection: A collection of source code lines. 

784 modules_collection: A collection of modules. 

785 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

786 force_inspection: Whether to force using dynamic analysis when loading data. 

787 store_source: Whether to store code source in the lines collection. 

788 find_stubs_package: Whether to search for stubs-only package. 

789 If both the package and its stubs are found, they'll be merged together. 

790 If only the stubs are found, they'll be used as the package itself. 

791 resolve_aliases: Whether to resolve aliases. 

792 resolve_external: Whether to try to load unspecified modules to resolve aliases. 

793 Default value (`None`) means to load external modules only if they are the private sibling 

794 or the origin module (for example when `ast` imports from `_ast`). 

795 resolve_implicit: When false, only try to resolve an alias if it is explicitly exported. 

796 

797 Returns: 

798 A Griffe object. 

799 """ 

800 loader = GriffeLoader( 

801 extensions=extensions, 

802 search_paths=search_paths, 

803 docstring_parser=docstring_parser, 

804 docstring_options=docstring_options, 

805 lines_collection=lines_collection, 

806 modules_collection=modules_collection, 

807 allow_inspection=allow_inspection, 

808 force_inspection=force_inspection, 

809 store_source=store_source, 

810 ) 

811 result = loader.load( 

812 objspec, 

813 submodules=submodules, 

814 try_relative_path=try_relative_path, 

815 find_stubs_package=find_stubs_package, 

816 ) 

817 if resolve_aliases: 817 ↛ 818line 817 didn't jump to line 818 because the condition on line 817 was never true

818 loader.resolve_aliases(implicit=resolve_implicit, external=resolve_external) 

819 return result 

820 

821 

822def load_git( 

823 objspec: str | Path | None = None, 

824 /, 

825 *, 

826 ref: str = "HEAD", 

827 repo: str | Path = ".", 

828 submodules: bool = True, 

829 extensions: Extensions | None = None, 

830 search_paths: Sequence[str | Path] | None = None, 

831 docstring_parser: Parser | None = None, 

832 docstring_options: dict[str, Any] | None = None,