Coverage for src/griffe/_internal/loader.py: 82.46%

1# This module contains all the logic for loading API data from sources or compiled modules. 

2 

3from __future__ import annotations 

4 

5import re 

6import shutil 

7import subprocess 

8import sys 

9import tempfile 

10from contextlib import suppress 

11from datetime import datetime, timezone 

12from importlib.util import find_spec 

13from pathlib import Path 

14from typing import TYPE_CHECKING, Any, ClassVar, cast 

15 

16from griffe._internal.agents.inspector import inspect 

17from griffe._internal.agents.visitor import visit 

18from griffe._internal.collections import LinesCollection, ModulesCollection 

19from griffe._internal.enumerations import Kind 

20from griffe._internal.exceptions import ( 

21 AliasResolutionError, 

22 CyclicAliasError, 

23 LoadingError, 

24 UnimportableModuleError, 

25) 

26from griffe._internal.expressions import ExprName 

27from griffe._internal.extensions.base import Extensions, load_extensions 

28from griffe._internal.finder import ModuleFinder, NamespacePackage, Package 

29from griffe._internal.git import tmp_worktree 

30from griffe._internal.importer import dynamic_import 

31from griffe._internal.logger import logger 

32from griffe._internal.merger import merge_stubs 

33from griffe._internal.models import Alias, Module, Object 

34from griffe._internal.stats import Stats 

35 

36if TYPE_CHECKING: 

37 from collections.abc import Sequence 

38 

39 from griffe._internal.docstrings.parsers import DocstringStyle 

40 from griffe._internal.enumerations import Parser 

41 

42 

43class GriffeLoader: 

44 """The Griffe loader, allowing to load data from modules.""" 

45 

46 ignored_modules: ClassVar[set[str]] = {"debugpy", "_pydev"} 

47 """Special modules to ignore when loading. 

48 

49 For example, `debugpy` and `_pydev` are used when debugging with VSCode and should generally never be loaded. 

50 """ 

51 

52 def __init__( 

53 self, 

54 *, 

55 extensions: Extensions | None = None, 

56 search_paths: Sequence[str | Path] | None = None, 

57 docstring_parser: DocstringStyle | Parser | None = None, 

58 docstring_options: dict[str, Any] | None = None, 

59 lines_collection: LinesCollection | None = None, 

60 modules_collection: ModulesCollection | None = None, 

61 allow_inspection: bool = True, 

62 force_inspection: bool = False, 

63 store_source: bool = True, 

64 ) -> None: 

65 """Initialize the loader. 

66 

67 Parameters: 

68 extensions: The extensions to use. 

69 search_paths: The paths to search into. 

70 docstring_parser: The docstring parser to use. By default, no parsing is done. 

71 docstring_options: Additional docstring parsing options. 

72 lines_collection: A collection of source code lines. 

73 modules_collection: A collection of modules. 

74 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

75 store_source: Whether to store code source in the lines collection. 

76 """ 

77 self.extensions: Extensions = extensions or load_extensions() 

78 """Loaded Griffe extensions.""" 

79 self.docstring_parser: DocstringStyle | Parser | None = docstring_parser 

80 """Selected docstring parser.""" 

81 self.docstring_options: dict[str, Any] = docstring_options or {} 

82 """Configured parsing options.""" 

83 self.lines_collection: LinesCollection = lines_collection or LinesCollection() 

84 """Collection of source code lines.""" 

85 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection() 

86 """Collection of modules.""" 

87 self.allow_inspection: bool = allow_inspection 

88 """Whether to allow inspecting (importing) modules for which we can't find sources.""" 

89 self.force_inspection: bool = force_inspection 

90 """Whether to force inspecting (importing) modules, even when sources were found.""" 

91 self.store_source: bool = store_source 

92 """Whether to store source code in the lines collection.""" 

93 self.finder: ModuleFinder = ModuleFinder(search_paths) 

94 """The module source finder.""" 

95 self._time_stats: dict = { 

96 "time_spent_visiting": 0, 

97 "time_spent_inspecting": 0, 

98 } 

99 

100 def load( 

101 self, 

102 objspec: str | Path | None = None, 

103 /, 

104 *, 

105 submodules: bool = True, 

106 try_relative_path: bool = True, 

107 find_stubs_package: bool = False, 

108 ) -> Object | Alias: 

109 """Load an object as a Griffe object, given its Python or file path. 

110 

111 Note that this will load the whole object's package, 

112 and return only the specified object. 

113 The rest of the package can be accessed from the returned object 

114 with regular methods and properties (`parent`, `members`, etc.). 

115 

116 Examples: 

117 >>> loader.load("griffe.Module") 

118 Alias("Module", "griffe._internal.models.Module") 

119 

120 Parameters: 

121 objspec: The Python path of an object, or file path to a module. 

122 submodules: Whether to recurse on the submodules. 

123 This parameter only makes sense when loading a package (top-level module). 

124 try_relative_path: Whether to try finding the module as a relative path. 

125 find_stubs_package: Whether to search for stubs-only package. 

126 If both the package and its stubs are found, they'll be merged together. 

127 If only the stubs are found, they'll be used as the package itself. 

128 

129 Raises: 

130 LoadingError: When loading a module failed for various reasons. 

131 ModuleNotFoundError: When a module was not found and inspection is disallowed. 

132 

133 Returns: 

134 A Griffe object. 

135 """ 

136 obj_path: str 

137 package = None 

138 top_module = None 

139 

140 # We always start by searching paths on the disk, 

141 # even if inspection is forced. 

142 logger.debug("Searching path(s) for %s", objspec) 

143 try: 

144 obj_path, package = self.finder.find_spec( 

145 objspec, # type: ignore[arg-type] 

146 try_relative_path=try_relative_path, 

147 find_stubs_package=find_stubs_package, 

148 ) 

149 except ModuleNotFoundError: 

150 # If we couldn't find paths on disk and inspection is disabled, 

151 # re-raise ModuleNotFoundError. 

152 logger.debug("Could not find path for %s on disk", objspec) 

153 if not (self.allow_inspection or self.force_inspection): 

154 raise 

155 

156 # Otherwise we try to dynamically import the top-level module. 

157 obj_path = str(objspec) 

158 top_module_name = obj_path.split(".", 1)[0] 

159 logger.debug("Trying to dynamically import %s", top_module_name) 

160 top_module_object = dynamic_import(top_module_name, self.finder.search_paths) 

161 

162 try: 

163 top_module_path = top_module_object.__path__ 

164 if not top_module_path: 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true

165 raise ValueError(f"Module {top_module_name} has no paths set") # noqa: TRY301 

166 except (AttributeError, ValueError): 

167 # If the top-level module has no `__path__`, we inspect it as-is, 

168 # and do not try to recurse into submodules (there shouldn't be any in builtin/compiled modules). 

169 logger.debug("Module %s has no paths set (built-in module?). Inspecting it as-is.", top_module_name) 

170 top_module = self._inspect_module(top_module_name) 

171 self.modules_collection.set_member(top_module.path, top_module) 

172 return self._post_load(top_module, obj_path) 

173 

174 # We found paths, and use them to build our intermediate Package or NamespacePackage struct. 

175 logger.debug("Module %s has paths set: %s", top_module_name, top_module_path) 

176 top_module_path = [Path(path) for path in top_module_path] 

177 if len(top_module_path) > 1: 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true

178 package = NamespacePackage(top_module_name, top_module_path) 

179 else: 

180 package = Package(top_module_name, top_module_path[0]) 

181 

182 # We have an intermediate package, and an object path: we're ready to load. 

183 logger.debug("Found %s: loading", objspec) 

184 try: 

185 top_module = self._load_package(package, submodules=submodules) 

186 except LoadingError: 

187 logger.exception("Could not load package %s", package) 

188 raise 

189 

190 return self._post_load(top_module, obj_path) 

191 

192 def _post_load(self, module: Module, obj_path: str) -> Object | Alias: 

193 # Pre-emptively expand exports (`__all__` values), 

194 # as well as wildcard imports (without ever loading additional packages). 

195 # This is a best-effort to return the most correct API data 

196 # before firing the `on_package_loaded` event. 

197 # 

198 # Packages that wildcard imports from external, non-loaded packages 

199 # will still have incomplete data, requiring subsequent calls to 

200 # `load()` and/or `resolve_aliases()`. 

201 self.expand_exports(module) 

202 self.expand_wildcards(module, external=False) 

203 # Package is loaded, we now retrieve the initially requested object and return it. 

204 obj = self.modules_collection.get_member(obj_path) 

205 self.extensions.call("on_package_loaded", pkg=module, loader=self) 

206 return obj 

207 

208 def resolve_aliases( 

209 self, 

210 *, 

211 implicit: bool = False, 

212 external: bool | None = None, 

213 max_iterations: int | None = None, 

214 ) -> tuple[set[str], int]: 

215 """Resolve aliases. 

216 

217 Parameters: 

218 implicit: When false, only try to resolve an alias if it is explicitly exported. 

219 external: When false, don't try to load unspecified modules to resolve aliases. 

220 max_iterations: Maximum number of iterations on the loader modules collection. 

221 

222 Returns: 

223 The unresolved aliases and the number of iterations done. 

224 """ 

225 if max_iterations is None: 

226 max_iterations = float("inf") # type: ignore[assignment] 

227 prev_unresolved: set[str] = set() 

228 unresolved: set[str] = set("0") # Init to enter loop. 

229 iteration = 0 

230 collection = self.modules_collection.members 

231 

232 # Before resolving aliases, we try to expand wildcard imports again 

233 # (this was already done in `_post_load()`), 

234 # this time with the user-configured `external` setting, 

235 # and with potentially more packages loaded in the collection, 

236 # allowing to resolve more aliases. 

237 for wildcards_module in list(collection.values()): 

238 self.expand_wildcards(wildcards_module, external=external) 

239 

240 load_failures: set[str] = set() 

241 while unresolved and unresolved != prev_unresolved and iteration < max_iterations: # type: ignore[operator] 

242 prev_unresolved = unresolved - {"0"} 

243 unresolved = set() 

244 resolved: set[str] = set() 

245 iteration += 1 

246 for module_name in list(collection.keys()): 

247 module = collection[module_name] 

248 next_resolved, next_unresolved = self.resolve_module_aliases( 

249 module, 

250 implicit=implicit, 

251 external=external, 

252 load_failures=load_failures, 

253 ) 

254 resolved |= next_resolved 

255 unresolved |= next_unresolved 

256 logger.debug( 

257 "Iteration %s finished, %s aliases resolved, still %s to go", 

258 iteration, 

259 len(resolved), 

260 len(unresolved), 

261 ) 

262 return unresolved, iteration 

263 

264 def expand_exports(self, module: Module, seen: set | None = None) -> None: 

265 """Expand exports: try to recursively expand all module exports (`__all__` values). 

266 

267 See also: [`Module.exports`][griffe.Module.exports]. 

268 

269 Parameters: 

270 module: The module to recurse on. 

271 seen: Used to avoid infinite recursion. 

272 """ 

273 seen = seen or set() 

274 seen.add(module.path) 

275 if module.exports is None: 

276 return 

277 

278 expanded = [] 

279 for export in module.exports: 

280 # It's a name: we resolve it, get the module it comes from, 

281 # recurse into it, and add its exports to the current ones. 

282 if isinstance(export, ExprName): 

283 module_path = export.canonical_path.rsplit(".", 1)[0] # Remove trailing `.__all__`. 

284 try: 

285 next_module = self.modules_collection.get_member(module_path) 

286 except KeyError: 

287 logger.debug("Cannot expand '%s', try pre-loading corresponding package", export.canonical_path) 

288 continue 

289 if next_module.path not in seen: 

290 self.expand_exports(next_module, seen) 

291 try: 

292 expanded += [export for export in next_module.exports if export not in expanded] 

293 except TypeError: 

294 logger.warning("Unsupported item in %s.__all__: %s (use strings only)", module.path, export) 

295 # It's a string, simply add it to the current exports. 

296 else: 

297 expanded.append(export) 

298 module.exports = expanded 

299 

300 # Make sure to expand exports in all modules. 

301 for submodule in module.modules.values(): 

302 if not submodule.is_alias and submodule.path not in seen: 

303 self.expand_exports(submodule, seen) 

304 

305 def expand_wildcards( 

306 self, 

307 obj: Object, 

308 *, 

309 external: bool | None = None, 

310 seen: set | None = None, 

311 ) -> None: 

312 """Expand wildcards: try to recursively expand all found wildcards. 

313 

314 See also: [`Alias.wildcard`][griffe.Alias.wildcard]. 

315 

316 Parameters: 

317 obj: The object and its members to recurse on. 

318 external: When true, try to load unspecified modules to expand wildcards. 

319 seen: Used to avoid infinite recursion. 

320 """ 

321 expanded = [] 

322 to_remove = [] 

323 seen = seen or set() 

324 seen.add(obj.path) 

325 

326 # First we expand wildcard imports and store the objects in a temporary `expanded` variable, 

327 # while also keeping track of the members representing wildcard import, to remove them later. 

328 for member in obj.members.values(): 

329 # Handle a wildcard. 

330 if member.is_alias and member.wildcard: # type: ignore[union-attr] 

331 package = member.wildcard.split(".", 1)[0] # type: ignore[union-attr] 

332 not_loaded = obj.package.path != package and package not in self.modules_collection 

333 

334 # Try loading the (unknown) package containing the wildcard importe module (if allowed to). 

335 if not_loaded: 

336 if external is False or (external is None and package != f"_{obj.package.name}"): 

337 continue 

338 try: 

339 self.load(package, try_relative_path=False) 

340 except (ImportError, LoadingError) as error: 

341 logger.debug("Could not expand wildcard import %s in %s: %s", member.name, obj.path, error) 

342 continue 

343 

344 # Try getting the module from which every public object is imported. 

345 try: 

346 target = self.modules_collection.get_member(member.target_path) # type: ignore[union-attr] 

347 except KeyError: 

348 logger.debug( 

349 "Could not expand wildcard import %s in %s: %s not found in modules collection", 

350 member.name, 

351 obj.path, 

352 cast("Alias", member).target_path, 

353 ) 

354 continue 

355 

356 # Recurse into this module, expanding wildcards there before collecting everything. 

357 if target.path not in seen: 

358 try: 

359 self.expand_wildcards(target, external=external, seen=seen) 

360 except (AliasResolutionError, CyclicAliasError) as error: 

361 logger.debug("Could not expand wildcard import %s in %s: %s", member.name, obj.path, error) 

362 continue 

363 

364 # Collect every imported object. 

365 expanded.extend(self._expand_wildcard(member)) # type: ignore[arg-type] 

366 to_remove.append(member.name) 

367 

368 # Recurse in unseen submodules. 

369 elif not member.is_alias and member.is_module and member.path not in seen: 

370 self.expand_wildcards(member, external=external, seen=seen) # type: ignore[arg-type] 

371 

372 # Then we remove the members representing wildcard imports. 

373 for name in to_remove: 

374 obj.del_member(name) 

375 

376 # Finally we process the collected objects. 

377 for new_member, alias_lineno, alias_endlineno in expanded: 

378 overwrite = False 

379 already_present = new_member.name in obj.members 

380 self_alias = ( 

381 new_member.is_alias and cast("Alias", new_member).target_path == f"{obj.path}.{new_member.name}" 

382 ) 

383 

384 # If a member with the same name is already present in the current object, 

385 # we only overwrite it if the alias is imported lower in the module 

386 # (meaning that the alias takes precedence at runtime). 

387 if already_present: 

388 old_member = obj.get_member(new_member.name) 

389 old_lineno = old_member.alias_lineno if old_member.is_alias else old_member.lineno 

390 overwrite = alias_lineno > (old_lineno or 0) # type: ignore[operator] 

391 

392 # 1. If the expanded member is an alias with a target path equal to its own path, we stop. 

393 # This situation can arise because of Griffe's mishandling of (abusive) wildcard imports. 

394 # We have yet to check how Python handles this itself, or if there's an algorithm 

395 # that we could follow to untangle back-and-forth wildcard imports. 

396 # 2. If the expanded member was already present and we decided not to overwrite it, we stop. 

397 # 3. Otherwise we proceed further. 

398 if not self_alias and (not already_present or overwrite): 

399 alias = Alias( 

400 new_member.name, 

401 new_member, 

402 lineno=alias_lineno, 

403 endlineno=alias_endlineno, 

404 parent=obj, # type: ignore[arg-type] 

405 ) 

406 # Special case: we avoid overwriting a submodule with an alias. 

407 # Griffe suffers from this limitation where an object cannot store both 

408 # a submodule and a member of the same name, while this poses (almost) no issue in Python. 

409 # We always give precedence to the submodule. 

410 # See the "avoid member-submodule name shadowing" section in the "Python code" docs page. 

411 if already_present: 

412 prev_member = obj.get_member(new_member.name) 

413 with suppress(AliasResolutionError, CyclicAliasError): 

414 if prev_member.is_module: 

415 continue 

416 

417 # Everything went right (supposedly), we add the alias as a member of the current object. 

418 obj.set_member(new_member.name, alias) 

419 self.extensions.call("on_wildcard_expansion", alias=alias, loader=self) 

420 

421 def resolve_module_aliases( 

422 self, 

423 obj: Object | Alias, 

424 *, 

425 implicit: bool = False, 

426 external: bool | None = None, 

427 seen: set[str] | None = None, 

428 load_failures: set[str] | None = None, 

429 ) -> tuple[set[str], set[str]]: 

430 """Follow aliases: try to recursively resolve all found aliases. 

431 

432 Parameters: 

433 obj: The object and its members to recurse on. 

434 implicit: When false, only try to resolve an alias if it is explicitly exported. 

435 external: When false, don't try to load unspecified modules to resolve aliases. 

436 seen: Used to avoid infinite recursion. 

437 load_failures: Set of external packages we failed to load (to prevent retries). 

438 

439 Returns: 

440 Both sets of resolved and unresolved aliases. 

441 """ 

442 resolved = set() 

443 unresolved = set() 

444 if load_failures is None: 444 ↛ 445line 444 didn't jump to line 445 because the condition on line 444 was never true

445 load_failures = set() 

446 seen = seen or set() 

447 seen.add(obj.path) 

448 

449 for member in obj.members.values(): 

450 # Handle aliases. 

451 if member.is_alias: 

452 if member.wildcard or member.resolved: # type: ignore[union-attr] 

453 continue 

454 if not implicit and not member.is_exported: 

455 continue 

456 

457 # Try resolving the alias. If it fails, check if it is because it comes 

458 # from an external package, and decide if we should load that package 

459 # to allow the alias to be resolved at the next iteration (maybe). 

460 try: 

461 member.resolve_target() # type: ignore[union-attr] 

462 except AliasResolutionError as error: 

463 target = error.alias.target_path 

464 unresolved.add(member.path) 

465 package = target.split(".", 1)[0] 

466 load_module = ( 

467 (external is True or (external is None and package == f"_{obj.package.name}")) 

468 and package not in load_failures 

469 and obj.package.path != package 

470 and package not in self.modules_collection 

471 ) 

472 if load_module: 

473 logger.debug("Failed to resolve alias %s -> %s", member.path, target) 

474 try: 

475 self.load(package, try_relative_path=False) 

476 except (ImportError, LoadingError) as error: 

477 logger.debug("Could not follow alias %s: %s", member.path, error) 

478 load_failures.add(package) 

479 except CyclicAliasError as error: 

480 logger.debug(str(error)) 

481 else: 

482 logger.debug("Alias %s was resolved to %s", member.path, member.final_target.path) # type: ignore[union-attr] 

483 resolved.add(member.path) 

484 

485 # Recurse into unseen modules and classes. 

486 elif member.kind in {Kind.MODULE, Kind.CLASS} and member.path not in seen: 

487 sub_resolved, sub_unresolved = self.resolve_module_aliases( 

488 member, 

489 implicit=implicit, 

490 external=external, 

491 seen=seen, 

492 load_failures=load_failures, 

493 ) 

494 resolved |= sub_resolved 

495 unresolved |= sub_unresolved 

496 

497 return resolved, unresolved 

498 

499 def stats(self) -> Stats: 

500 """Compute some statistics. 

501 

502 Returns: 

503 Some statistics. 

504 """ 

505 stats = Stats(self) 

506 stats.time_spent_visiting = self._time_stats["time_spent_visiting"] 

507 stats.time_spent_inspecting = self._time_stats["time_spent_inspecting"] 

508 return stats 

509 

510 def _load_package(self, package: Package | NamespacePackage, *, submodules: bool = True) -> Module: 

511 top_module = self._load_module(package.name, package.path, submodules=submodules) 

512 if isinstance(package, NamespacePackage): 

513 return top_module 

514 if package.stubs: 

515 self.expand_wildcards(top_module) 

516 # If stubs are in the package itself, they have been merged while loading modules, 

517 # so only the top-level init module needs to be merged still. 

518 # If stubs are in another package (a stubs-only package), 

519 # then we need to load the entire stubs package to merge everything. 

520 submodules = submodules and package.stubs.parent != package.path.parent 

521 stubs = self._load_module(package.name, package.stubs, submodules=submodules) 

522 return merge_stubs(top_module, stubs) 

523 return top_module 

524 

525 def _load_module( 

526 self, 

527 module_name: str, 

528 module_path: Path | list[Path], 

529 *, 

530 submodules: bool = True, 

531 parent: Module | None = None, 

532 ) -> Module: 

533 try: 

534 return self._load_module_path(module_name, module_path, submodules=submodules, parent=parent) 

535 except SyntaxError as error: 

536 raise LoadingError(f"Syntax error: {error}") from error 

537 except ImportError as error: 

538 raise LoadingError(f"Import error: {error}") from error 

539 except UnicodeDecodeError as error: 

540 raise LoadingError(f"UnicodeDecodeError when loading {module_path}: {error}") from error 

541 except OSError as error: 

542 raise LoadingError(f"OSError when loading {module_path}: {error}") from error 

543 

544 def _load_module_path( 

545 self, 

546 module_name: str, 

547 module_path: Path | list[Path], 

548 *, 

549 submodules: bool = True, 

550 parent: Module | None = None, 

551 ) -> Module: 

552 logger.debug("Loading path %s", module_path) 

553 if isinstance(module_path, list): 

554 module = self._create_module(module_name, module_path) 

555 elif self.force_inspection: 

556 module = self._inspect_module(module_name, module_path, parent) 

557 elif module_path.suffix in {".py", ".pyi"}: 

558 module = self._visit_module(module_name, module_path, parent) 

559 elif self.allow_inspection: 559 ↛ 562line 559 didn't jump to line 562 because the condition on line 559 was always true

560 module = self._inspect_module(module_name, module_path, parent) 

561 else: 

562 raise LoadingError("Cannot load compiled module without inspection") 

563 if parent is None: 

564 self.modules_collection.set_member(module.path, module) 

565 if submodules: 

566 self._load_submodules(module) 

567 return module 

568 

569 def _load_submodules(self, module: Module) -> None: 

570 for subparts, subpath in self.finder.submodules(module): 

571 self._load_submodule(module, subparts, subpath) 

572 

573 def _load_submodule(self, module: Module, subparts: tuple[str, ...], subpath: Path) -> None: 

574 for subpart in subparts: 

575 if "." in subpart: 

576 logger.debug("Skip %s, dots in filenames are not supported", subpath) 

577 return 

578 try: 

579 parent_module = self._get_or_create_parent_module(module, subparts, subpath) 

580 except UnimportableModuleError as error: 

581 # NOTE: Why don't we load submodules when there's no init module in their folder? 

582 # Usually when a folder with Python files does not have an __init__.py module, 

583 # it's because the Python files are scripts that should never be imported. 

584 # Django has manage.py somewhere for example, in a folder without init module. 

585 # This script isn't part of the Python API, as it's meant to be called on the CLI exclusively 

586 # (at least it was the case a few years ago when I was still using Django). 

587 

588 # The other case when there's no init module is when a package is a native namespace package (PEP 420). 

589 # It does not make sense to have a native namespace package inside of a regular package (having init modules above), 

590 # because the regular package above blocks the namespace feature from happening, so I consider it a user error. 

591 # It's true that users could have a native namespace package inside of a pkg_resources-style namespace package, 

592 # but I've never seen this happen. 

593 

594 # It's also true that Python can actually import the module under the (wrongly declared) native namespace package, 

595 # so the Griffe debug log message is a bit misleading, 

596 # but that's because in that case Python acts like the whole tree is a regular package. 

597 # It works when the namespace package appears in only one search path (`sys.path`), 

598 # but will fail if it appears in multiple search paths: Python will only find the first occurrence. 

599 # It's better to not falsely support this, and to warn users. 

600 logger.debug("%s. Missing __init__ module?", error) 

601 return 

602 submodule_name = subparts[-1] 

603 try: 

604 submodule = self._load_module( 

605 submodule_name, 

606 subpath, 

607 submodules=False, 

608 parent=parent_module, 

609 ) 

610 except LoadingError as error: 

611 logger.debug(str(error)) 

612 else: 

613 if submodule_name in parent_module.members: 

614 member = parent_module.members[submodule_name] 

615 if member.is_alias or not member.is_module: 

616 logger.debug( 

617 "Submodule '%s' is shadowing the member at the same path. " 

618 "We recommend renaming the member or the submodule (for example prefixing it with `_`), " 

619 "see https://mkdocstrings.github.io/griffe/best_practices/#avoid-member-submodule-name-shadowing.", 

620 submodule.path, 

621 ) 

622 parent_module.set_member(submodule_name, submodule) 

623 

624 def _create_module(self, module_name: str, module_path: Path | list[Path]) -> Module: 

625 return Module( 

626 module_name, 

627 filepath=module_path, 

628 lines_collection=self.lines_collection, 

629 modules_collection=self.modules_collection, 

630 ) 

631 

632 def _visit_module(self, module_name: str, module_path: Path, parent: Module | None = None) -> Module: 

633 code = module_path.read_text(encoding="utf-8-sig") 

634 if self.store_source: 

635 self.lines_collection[module_path] = code.splitlines(keepends=False) 

636 start = datetime.now(tz=timezone.utc) 

637 module = visit( 

638 module_name, 

639 filepath=module_path, 

640 code=code, 

641 extensions=self.extensions, 

642 parent=parent, 

643 docstring_parser=self.docstring_parser, 

644 docstring_options=self.docstring_options, 

645 lines_collection=self.lines_collection, 

646 modules_collection=self.modules_collection, 

647 ) 

648 elapsed = datetime.now(tz=timezone.utc) - start 

649 self._time_stats["time_spent_visiting"] += elapsed.microseconds 

650 return module 

651 

652 def _inspect_module(self, module_name: str, filepath: Path | None = None, parent: Module | None = None) -> Module: 

653 for prefix in self.ignored_modules: 

654 if module_name.startswith(prefix): 654 ↛ 655line 654 didn't jump to line 655 because the condition on line 654 was never true

655 raise ImportError(f"Ignored module '{module_name}'") 

656 if self.store_source and filepath and filepath.suffix in {".py", ".pyi"}: 

657 self.lines_collection[filepath] = filepath.read_text(encoding="utf-8-sig").splitlines(keepends=False) 

658 start = datetime.now(tz=timezone.utc) 

659 try: 

660 module = inspect( 

661 module_name, 

662 filepath=filepath, 

663 import_paths=self.finder.search_paths, 

664 extensions=self.extensions, 

665 parent=parent, 

666 docstring_parser=self.docstring_parser, 

667 docstring_options=self.docstring_options, 

668 lines_collection=self.lines_collection, 

669 modules_collection=self.modules_collection, 

670 ) 

671 except SystemExit as error: 

672 raise ImportError(f"Importing '{module_name}' raised a system exit") from error 

673 except Exception as error: 

674 raise ImportError(f"Importing '{module_name}' raised an exception") from error 

675 elapsed = datetime.now(tz=timezone.utc) - start 

676 self._time_stats["time_spent_inspecting"] += elapsed.microseconds 

677 return module 

678 

679 def _get_or_create_parent_module( 

680 self, 

681 module: Module, 

682 subparts: tuple[str, ...], 

683 subpath: Path, 

684 ) -> Module: 

685 parent_parts = subparts[:-1] 

686 if not parent_parts: 

687 return module 

688 parent_module = module 

689 parents = list(subpath.parents) 

690 if subpath.stem == "__init__": 

691 parents.pop(0) 

692 for parent_offset, parent_part in enumerate(parent_parts, 2): 

693 module_filepath = parents[len(subparts) - parent_offset] 

694 try: 

695 parent_module = parent_module.get_member(parent_part) 

696 except KeyError as error: 

697 if parent_module.is_namespace_package or parent_module.is_namespace_subpackage: 697 ↛ 702line 697 didn't jump to line 702 because the condition on line 697 was always true

698 next_parent_module = self._create_module(parent_part, [module_filepath]) 

699 parent_module.set_member(parent_part, next_parent_module) 

700 parent_module = next_parent_module 

701 else: 

702 raise UnimportableModuleError(f"Skip {subpath}, it is not importable") from error 

703 else: 

704 parent_namespace = parent_module.is_namespace_package or parent_module.is_namespace_subpackage 

705 if parent_namespace and module_filepath not in parent_module.filepath: 

706 parent_module.filepath.append(module_filepath) 

707 return parent_module 

708 

709 def _expand_wildcard(self, wildcard_obj: Alias) -> list[tuple[Object | Alias, int | None, int | None]]: 

710 module = self.modules_collection.get_member(wildcard_obj.wildcard) # type: ignore[arg-type] 

711 return [ 

712 (imported_member, wildcard_obj.alias_lineno, wildcard_obj.alias_endlineno) 

713 for imported_member in module.members.values() 

714 if imported_member.is_wildcard_exposed 

715 ] 

716 

717 

718def load( 

719 objspec: str | Path | None = None, 

720 /, 

721 *, 

722 submodules: bool = True, 

723 try_relative_path: bool = True, 

724 extensions: Extensions | None = None, 

725 search_paths: Sequence[str | Path] | None = None, 

726 docstring_parser: DocstringStyle | Parser | None = None, 

727 docstring_options: dict[str, Any] | None = None, 

728 lines_collection: LinesCollection | None = None, 

729 modules_collection: ModulesCollection | None = None, 

730 allow_inspection: bool = True, 

731 force_inspection: bool = False, 

732 store_source: bool = True, 

733 find_stubs_package: bool = False, 

734 resolve_aliases: bool = False, 

735 resolve_external: bool | None = None, 

736 resolve_implicit: bool = False, 

737) -> Object | Alias: 

738 """Load and return a Griffe object. 

739 

740 In Griffe's context, loading means: 

741 

742 - searching for a package, and finding it on the file system or as a builtin module 

743 (see the [`ModuleFinder`][griffe.ModuleFinder] class for more information) 

744 - extracting information from each of its (sub)modules, by either parsing 

745 the source code (see the [`visit`][griffe.visit] function) 

746 or inspecting the module at runtime (see the [`inspect`][griffe.inspect] function) 

747 

748 The extracted information is stored in a collection of modules, which can be queried later. 

749 Each collected module is a tree of objects, representing the structure of the module. 

750 See the [`Module`][griffe.Module], [`Class`][griffe.Class], 

751 [`Function`][griffe.Function], [`Attribute`][griffe.Attribute], and 

752 [`TypeAlias`][griffe.TypeAlias] classes for more information. 

753 

754 The main class used to load modules is [`GriffeLoader`][griffe.GriffeLoader]. 

755 Convenience functions like this one and [`load_git`][griffe.load_git] are also available. 

756 

757 Example: 

758 ```python 

759 import griffe 

760 

761 module = griffe.load(...) 

762 ``` 

763 

764 This is a shortcut for: 

765 

766 ```python 

767 from griffe import GriffeLoader 

768 

769 loader = GriffeLoader(...) 

770 module = loader.load(...) 

771 ``` 

772 

773 See the documentation for the loader: [`GriffeLoader`][griffe.GriffeLoader]. 

774 

775 Parameters: 

776 objspec: The Python path of an object, or file path to a module. 

777 submodules: Whether to recurse on the submodules. 

778 This parameter only makes sense when loading a package (top-level module). 

779 try_relative_path: Whether to try finding the module as a relative path. 

780 extensions: The extensions to use. 

781 search_paths: The paths to search into. 

782 docstring_parser: The docstring parser to use. By default, no parsing is done. 

783 docstring_options: Additional docstring parsing options. 

784 lines_collection: A collection of source code lines. 

785 modules_collection: A collection of modules. 

786 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

787 force_inspection: Whether to force using dynamic analysis when loading data. 

788 store_source: Whether to store code source in the lines collection. 

789 find_stubs_package: Whether to search for stubs-only package. 

790 If both the package and its stubs are found, they'll be merged together. 

791 If only the stubs are found, they'll be used as the package itself. 

792 resolve_aliases: Whether to resolve aliases. 

793 resolve_external: Whether to try to load unspecified modules to resolve aliases. 

794 Default value (`None`) means to load external modules only if they are the private sibling 

795 or the origin module (for example when `ast` imports from `_ast`). 

796 resolve_implicit: When false, only try to resolve an alias if it is explicitly exported. 

797 

798 Returns: 

799 A Griffe object. 

800 """ 

801 loader = GriffeLoader( 

802 extensions=extensions, 

803 search_paths=search_paths, 

804 docstring_parser=docstring_parser, 

805 docstring_options=docstring_options, 

806 lines_collection=lines_collection, 

807 modules_collection=modules_collection, 

808 allow_inspection=allow_inspection, 

809 force_inspection=force_inspection, 

810 store_source=store_source, 

811 ) 

812 result = loader.load( 

813 objspec, 

814 submodules=submodules, 

815 try_relative_path=try_relative_path, 

816 find_stubs_package=find_stubs_package, 

817 ) 

818 if resolve_aliases: 

819 loader.resolve_aliases(implicit=resolve_implicit, external=resolve_external) 

820 return result 

821 

822 

823def load_git( 

824 objspec: str | Path | None = None, 

825 /, 

826 *, 

827 ref: str = "HEAD", 

828 repo: str | Path = ".", 

829 submodules: bool = True,