Coverage for src/_griffe/loader.py: 91.24%

1# This module contains all the logic for loading API data from sources or compiled modules. 

2 

3from __future__ import annotations 

4 

5import sys 

6from contextlib import suppress 

7from datetime import datetime, timezone 

8from pathlib import Path 

9from typing import TYPE_CHECKING, Any, ClassVar, Sequence, cast 

10 

11from _griffe.agents.inspector import inspect 

12from _griffe.agents.visitor import visit 

13from _griffe.collections import LinesCollection, ModulesCollection 

14from _griffe.enumerations import Kind 

15from _griffe.exceptions import ( 

16 AliasResolutionError, 

17 CyclicAliasError, 

18 LoadingError, 

19 NameResolutionError, 

20 UnimportableModuleError, 

21) 

22from _griffe.expressions import ExprName 

23from _griffe.extensions.base import Extensions, load_extensions 

24from _griffe.finder import ModuleFinder, NamespacePackage, Package 

25from _griffe.git import tmp_worktree 

26from _griffe.importer import dynamic_import 

27from _griffe.logger import logger 

28from _griffe.merger import merge_stubs 

29from _griffe.models import Alias, Module, Object 

30from _griffe.stats import Stats 

31 

32if TYPE_CHECKING: 

33 from _griffe.enumerations import Parser 

34 

35_builtin_modules: set[str] = set(sys.builtin_module_names) 

36 

37 

38class GriffeLoader: 

39 """The Griffe loader, allowing to load data from modules.""" 

40 

41 ignored_modules: ClassVar[set[str]] = {"debugpy", "_pydev"} 

42 """Special modules to ignore when loading. 

43 

44 For example, `debugpy` and `_pydev` are used when debugging with VSCode and should generally never be loaded. 

45 """ 

46 

47 def __init__( 

48 self, 

49 *, 

50 extensions: Extensions | None = None, 

51 search_paths: Sequence[str | Path] | None = None, 

52 docstring_parser: Parser | None = None, 

53 docstring_options: dict[str, Any] | None = None, 

54 lines_collection: LinesCollection | None = None, 

55 modules_collection: ModulesCollection | None = None, 

56 allow_inspection: bool = True, 

57 force_inspection: bool = False, 

58 store_source: bool = True, 

59 ) -> None: 

60 """Initialize the loader. 

61 

62 Parameters: 

63 extensions: The extensions to use. 

64 search_paths: The paths to search into. 

65 docstring_parser: The docstring parser to use. By default, no parsing is done. 

66 docstring_options: Additional docstring parsing options. 

67 lines_collection: A collection of source code lines. 

68 modules_collection: A collection of modules. 

69 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

70 store_source: Whether to store code source in the lines collection. 

71 """ 

72 self.extensions: Extensions = extensions or load_extensions() 

73 """Loaded Griffe extensions.""" 

74 self.docstring_parser: Parser | None = docstring_parser 

75 """Selected docstring parser.""" 

76 self.docstring_options: dict[str, Any] = docstring_options or {} 

77 """Configured parsing options.""" 

78 self.lines_collection: LinesCollection = lines_collection or LinesCollection() 

79 """Collection of source code lines.""" 

80 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection() 

81 """Collection of modules.""" 

82 self.allow_inspection: bool = allow_inspection 

83 """Whether to allow inspecting (importing) modules for which we can't find sources.""" 

84 self.force_inspection: bool = force_inspection 

85 """Whether to force inspecting (importing) modules, even when sources were found.""" 

86 self.store_source: bool = store_source 

87 """Whether to store source code in the lines collection.""" 

88 self.finder: ModuleFinder = ModuleFinder(search_paths) 

89 """The module source finder.""" 

90 self._time_stats: dict = { 

91 "time_spent_visiting": 0, 

92 "time_spent_inspecting": 0, 

93 } 

94 

95 def load( 

96 self, 

97 objspec: str | Path | None = None, 

98 /, 

99 *, 

100 submodules: bool = True, 

101 try_relative_path: bool = True, 

102 find_stubs_package: bool = False, 

103 ) -> Object | Alias: 

104 """Load an object as a Griffe object, given its Python or file path. 

105 

106 Note that this will load the whole object's package, 

107 and return only the specified object. 

108 The rest of the package can be accessed from the returned object 

109 with regular methods and properties (`parent`, `members`, etc.). 

110 

111 Examples: 

112 >>> loader.load("griffe.Module") 

113 Alias("Module", "_griffe.models.Module") 

114 

115 Parameters: 

116 objspec: The Python path of an object, or file path to a module. 

117 submodules: Whether to recurse on the submodules. 

118 This parameter only makes sense when loading a package (top-level module). 

119 try_relative_path: Whether to try finding the module as a relative path. 

120 find_stubs_package: Whether to search for stubs-only package. 

121 If both the package and its stubs are found, they'll be merged together. 

122 If only the stubs are found, they'll be used as the package itself. 

123 

124 Raises: 

125 LoadingError: When loading a module failed for various reasons. 

126 ModuleNotFoundError: When a module was not found and inspection is disallowed. 

127 

128 Returns: 

129 A Griffe object. 

130 """ 

131 obj_path: str 

132 package = None 

133 top_module = None 

134 

135 # We always start by searching paths on the disk, 

136 # even if inspection is forced. 

137 logger.debug(f"Searching path(s) for {objspec}") 

138 try: 

139 obj_path, package = self.finder.find_spec( 

140 objspec, # type: ignore[arg-type] 

141 try_relative_path=try_relative_path, 

142 find_stubs_package=find_stubs_package, 

143 ) 

144 except ModuleNotFoundError: 

145 # If we couldn't find paths on disk and inspection is disabled, 

146 # re-raise ModuleNotFoundError. 

147 logger.debug(f"Could not find path for {objspec} on disk") 

148 if not (self.allow_inspection or self.force_inspection): 

149 raise 

150 

151 # Otherwise we try to dynamically import the top-level module. 

152 obj_path = str(objspec) 

153 top_module_name = obj_path.split(".", 1)[0] 

154 logger.debug(f"Trying to dynamically import {top_module_name}") 

155 top_module_object = dynamic_import(top_module_name, self.finder.search_paths) 

156 

157 try: 

158 top_module_path = top_module_object.__path__ 

159 if not top_module_path: 159 ↛ 160line 159 didn't jump to line 160 because the condition on line 159 was never true

160 raise ValueError(f"Module {top_module_name} has no paths set") # noqa: TRY301 

161 except (AttributeError, ValueError): 

162 # If the top-level module has no `__path__`, we inspect it as-is, 

163 # and do not try to recurse into submodules (there shouldn't be any in builtin/compiled modules). 

164 logger.debug(f"Module {top_module_name} has no paths set (built-in module?). Inspecting it as-is.") 

165 top_module = self._inspect_module(top_module_name) 

166 self.modules_collection.set_member(top_module.path, top_module) 

167 obj = self.modules_collection.get_member(obj_path) 

168 self.extensions.call("on_package_loaded", pkg=obj) 

169 return obj 

170 

171 # We found paths, and use them to build our intermediate Package or NamespacePackage struct. 

172 logger.debug(f"Module {top_module_name} has paths set: {top_module_path}") 

173 top_module_path = [Path(path) for path in top_module_path] 

174 if len(top_module_path) > 1: 174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true

175 package = NamespacePackage(top_module_name, top_module_path) 

176 else: 

177 package = Package(top_module_name, top_module_path[0]) 

178 

179 # We have an intermediate package, and an object path: we're ready to load. 

180 logger.debug(f"Found {objspec}: loading") 

181 try: 

182 top_module = self._load_package(package, submodules=submodules) 

183 except LoadingError as error: 

184 logger.exception(str(error)) 

185 raise 

186 

187 # Package is loaded, we now retrieve the initially requested object and return it. 

188 obj = self.modules_collection.get_member(obj_path) 

189 self.extensions.call("on_package_loaded", pkg=top_module) 

190 return obj 

191 

192 def resolve_aliases( 

193 self, 

194 *, 

195 implicit: bool = False, 

196 external: bool | None = None, 

197 max_iterations: int | None = None, 

198 ) -> tuple[set[str], int]: 

199 """Resolve aliases. 

200 

201 Parameters: 

202 implicit: When false, only try to resolve an alias if it is explicitly exported. 

203 external: When false, don't try to load unspecified modules to resolve aliases. 

204 max_iterations: Maximum number of iterations on the loader modules collection. 

205 

206 Returns: 

207 The unresolved aliases and the number of iterations done. 

208 """ 

209 if max_iterations is None: 

210 max_iterations = float("inf") # type: ignore[assignment] 

211 prev_unresolved: set[str] = set() 

212 unresolved: set[str] = set("0") # init to enter loop 

213 iteration = 0 

214 collection = self.modules_collection.members 

215 

216 # We must first expand exports (`__all__` values), 

217 # then expand wildcard imports (`from ... import *`), 

218 # and then only we can start resolving aliases. 

219 for exports_module in list(collection.values()): 

220 self.expand_exports(exports_module) 

221 for wildcards_module in list(collection.values()): 

222 self.expand_wildcards(wildcards_module, external=external) 

223 

224 load_failures: set[str] = set() 

225 while unresolved and unresolved != prev_unresolved and iteration < max_iterations: # type: ignore[operator] 

226 prev_unresolved = unresolved - {"0"} 

227 unresolved = set() 

228 resolved: set[str] = set() 

229 iteration += 1 

230 for module_name in list(collection.keys()): 

231 module = collection[module_name] 

232 next_resolved, next_unresolved = self.resolve_module_aliases( 

233 module, 

234 implicit=implicit, 

235 external=external, 

236 load_failures=load_failures, 

237 ) 

238 resolved |= next_resolved 

239 unresolved |= next_unresolved 

240 logger.debug( 

241 f"Iteration {iteration} finished, {len(resolved)} aliases resolved, still {len(unresolved)} to go", 

242 ) 

243 return unresolved, iteration 

244 

245 def expand_exports(self, module: Module, seen: set | None = None) -> None: 

246 """Expand exports: try to recursively expand all module exports (`__all__` values). 

247 

248 Parameters: 

249 module: The module to recurse on. 

250 seen: Used to avoid infinite recursion. 

251 """ 

252 seen = seen or set() 

253 seen.add(module.path) 

254 if module.exports is None: 

255 return 

256 

257 expanded = set() 

258 for export in module.exports: 

259 # It's a name: we resolve it, get the module it comes from, 

260 # recurse into it, and add its exports to the current ones. 

261 if isinstance(export, ExprName): 

262 module_path = export.canonical_path.rsplit(".", 1)[0] # remove trailing .__all__ 

263 try: 

264 next_module = self.modules_collection.get_member(module_path) 

265 except KeyError: 

266 logger.debug(f"Cannot expand '{export.canonical_path}', try pre-loading corresponding package") 

267 continue 

268 if next_module.path not in seen: 

269 self.expand_exports(next_module, seen) 

270 try: 

271 expanded |= next_module.exports 

272 except TypeError: 

273 logger.warning(f"Unsupported item in {module.path}.__all__: {export} (use strings only)") 

274 # It's a string, simply add it to the current exports. 

275 else: 

276 with suppress(NameResolutionError): 

277 if not module.resolve(export).startswith(module.path): 

278 # NOTE: This won't work for built-in attributes during inspection, 

279 # since their canonical module cannot be determined. 

280 logger.debug( 

281 f"Name `{export}` exported by module `{module.path}` doesn't come from this module or from a submodule.", 

282 ) 

283 expanded.add(export) 

284 module.exports = expanded 

285 

286 # Make sure to expand exports in all modules. 

287 for submodule in module.modules.values(): 

288 if not submodule.is_alias and submodule.path not in seen: 

289 self.expand_exports(submodule, seen) 

290 

291 def expand_wildcards( 

292 self, 

293 obj: Object, 

294 *, 

295 external: bool | None = None, 

296 seen: set | None = None, 

297 ) -> None: 

298 """Expand wildcards: try to recursively expand all found wildcards. 

299 

300 Parameters: 

301 obj: The object and its members to recurse on. 

302 external: When true, try to load unspecified modules to expand wildcards. 

303 seen: Used to avoid infinite recursion. 

304 """ 

305 expanded = [] 

306 to_remove = [] 

307 seen = seen or set() 

308 seen.add(obj.path) 

309 

310 # First we expand wildcard imports and store the objects in a temporary `expanded` variable, 

311 # while also keeping track of the members representing wildcard import, to remove them later. 

312 for member in obj.members.values(): 

313 # Handle a wildcard. 

314 if member.is_alias and member.wildcard: # type: ignore[union-attr] # we know it's an alias 

315 package = member.wildcard.split(".", 1)[0] # type: ignore[union-attr] 

316 not_loaded = obj.package.path != package and package not in self.modules_collection 

317 

318 # Try loading the (unknown) package containing the wildcard importe module (if allowed to). 

319 if not_loaded: 

320 if external is False or (external is None and package != f"_{obj.package.name}"): 

321 continue 

322 try: 

323 self.load(package, try_relative_path=False) 

324 except (ImportError, LoadingError) as error: 

325 logger.debug(f"Could not expand wildcard import {member.name} in {obj.path}: {error}") 

326 continue 

327 

328 # Try getting the module from which every public object is imported. 

329 try: 

330 target = self.modules_collection.get_member(member.target_path) # type: ignore[union-attr] 

331 except KeyError: 

332 logger.debug( 

333 f"Could not expand wildcard import {member.name} in {obj.path}: " 

334 f"{cast(Alias, member).target_path} not found in modules collection", 

335 ) 

336 continue 

337 

338 # Recurse into this module, expanding wildcards there before collecting everything. 

339 if target.path not in seen: 

340 try: 

341 self.expand_wildcards(target, external=external, seen=seen) 

342 except (AliasResolutionError, CyclicAliasError) as error: 

343 logger.debug(f"Could not expand wildcard import {member.name} in {obj.path}: {error}") 

344 continue 

345 

346 # Collect every imported object. 

347 expanded.extend(self._expand_wildcard(member)) # type: ignore[arg-type] 

348 to_remove.append(member.name) 

349 

350 # Recurse in unseen submodules. 

351 elif not member.is_alias and member.is_module and member.path not in seen: 

352 self.expand_wildcards(member, external=external, seen=seen) # type: ignore[arg-type] 

353 

354 # Then we remove the members representing wildcard imports. 

355 for name in to_remove: 

356 obj.del_member(name) 

357 

358 # Finally we process the collected objects. 

359 for new_member, alias_lineno, alias_endlineno in expanded: 

360 overwrite = False 

361 already_present = new_member.name in obj.members 

362 self_alias = new_member.is_alias and cast(Alias, new_member).target_path == f"{obj.path}.{new_member.name}" 

363 

364 # If a member with the same name is already present in the current object, 

365 # we only overwrite it if the alias is imported lower in the module 

366 # (meaning that the alias takes precedence at runtime). 

367 if already_present: 

368 old_member = obj.get_member(new_member.name) 

369 old_lineno = old_member.alias_lineno if old_member.is_alias else old_member.lineno 

370 overwrite = alias_lineno > (old_lineno or 0) # type: ignore[operator] 

371 

372 # 1. If the expanded member is an alias with a target path equal to its own path, we stop. 

373 # This situation can arise because of Griffe's mishandling of (abusive) wildcard imports. 

374 # We have yet to check how Python handles this itself, or if there's an algorithm 

375 # that we could follow to untangle back-and-forth wildcard imports. 

376 # 2. If the expanded member was already present and we decided not to overwrite it, we stop. 

377 # 3. Otherwise we proceed further. 

378 if not self_alias and (not already_present or overwrite): 

379 alias = Alias( 

380 new_member.name, 

381 new_member, 

382 lineno=alias_lineno, 

383 endlineno=alias_endlineno, 

384 parent=obj, # type: ignore[arg-type] 

385 ) 

386 # Special case: we avoid overwriting a submodule with an alias pointing to it. 

387 # Griffe suffers from this design flaw where an object cannot store both 

388 # a submodule and a member of the same name, while this poses (almost) no issue in Python. 

389 # We at least prevent this case where a submodule is overwritten by an imported version of itself. 

390 if already_present: 

391 prev_member = obj.get_member(new_member.name) 

392 with suppress(AliasResolutionError, CyclicAliasError): 392 ↛ 359line 392 didn't jump to line 359 because the continue on line 398 wasn't executed

393 if prev_member.is_module: 

394 if prev_member.is_alias: 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true

395 prev_member = prev_member.final_target 

396 if alias.final_target is prev_member: 396 ↛ 398line 396 didn't jump to line 398 because the condition on line 396 was never true

397 # Alias named after the module it targets: skip to avoid cyclic aliases. 

398 continue 

399 

400 # Everything went right (supposedly), we add the alias as a member of the current object. 

401 obj.set_member(new_member.name, alias) 

402 

403 def resolve_module_aliases( 

404 self, 

405 obj: Object | Alias, 

406 *, 

407 implicit: bool = False, 

408 external: bool | None = None, 

409 seen: set[str] | None = None, 

410 load_failures: set[str] | None = None, 

411 ) -> tuple[set[str], set[str]]: 

412 """Follow aliases: try to recursively resolve all found aliases. 

413 

414 Parameters: 

415 obj: The object and its members to recurse on. 

416 implicit: When false, only try to resolve an alias if it is explicitly exported. 

417 external: When false, don't try to load unspecified modules to resolve aliases. 

418 seen: Used to avoid infinite recursion. 

419 load_failures: Set of external packages we failed to load (to prevent retries). 

420 

421 Returns: 

422 Both sets of resolved and unresolved aliases. 

423 """ 

424 resolved = set() 

425 unresolved = set() 

426 if load_failures is None: 426 ↛ 427line 426 didn't jump to line 427 because the condition on line 426 was never true

427 load_failures = set() 

428 seen = seen or set() 

429 seen.add(obj.path) 

430 

431 for member in obj.members.values(): 

432 # Handle aliases. 

433 if member.is_alias: 

434 if member.wildcard or member.resolved: # type: ignore[union-attr] 

435 continue 

436 if not implicit and not member.is_exported: 

437 continue 

438 

439 # Try resolving the alias. If it fails, check if it is because it comes 

440 # from an external package, and decide if we should load that package 

441 # to allow the alias to be resolved at the next iteration (maybe). 

442 try: 

443 member.resolve_target() # type: ignore[union-attr] 

444 except AliasResolutionError as error: 444 ↛ 462line 444 didn't jump to line 462

445 target = error.alias.target_path 

446 unresolved.add(member.path) 

447 package = target.split(".", 1)[0] 

448 load_module = ( 

449 (external is True or (external is None and package == f"_{obj.package.name}")) 

450 and package not in load_failures 

451 and obj.package.path != package 

452 and package not in self.modules_collection 

453 ) 

454 if load_module: 

455 logger.debug(f"Failed to resolve alias {member.path} -> {target}") 

456 try: 

457 self.load(package, try_relative_path=False) 

458 except (ImportError, LoadingError) as error: 

459 logger.debug(f"Could not follow alias {member.path}: {error}") 

460 load_failures.add(package) 

461 # TODO: Immediately try again? 

462 except CyclicAliasError as error: 

463 logger.debug(str(error)) 

464 else: 

465 logger.debug(f"Alias {member.path} was resolved to {member.final_target.path}") # type: ignore[union-attr] 

466 resolved.add(member.path) 

467 

468 # Recurse into unseen modules and classes. 

469 elif member.kind in {Kind.MODULE, Kind.CLASS} and member.path not in seen: 

470 sub_resolved, sub_unresolved = self.resolve_module_aliases( 

471 member, 

472 implicit=implicit, 

473 external=external, 

474 seen=seen, 

475 load_failures=load_failures, 

476 ) 

477 resolved |= sub_resolved 

478 unresolved |= sub_unresolved 

479 

480 return resolved, unresolved 

481 

482 def stats(self) -> Stats: 

483 """Compute some statistics. 

484 

485 Returns: 

486 Some statistics. 

487 """ 

488 stats = Stats(self) 

489 stats.time_spent_visiting = self._time_stats["time_spent_visiting"] 

490 stats.time_spent_inspecting = self._time_stats["time_spent_inspecting"] 

491 return stats 

492 

493 def _load_package(self, package: Package | NamespacePackage, *, submodules: bool = True) -> Module: 

494 top_module = self._load_module(package.name, package.path, submodules=submodules) 

495 self.modules_collection.set_member(top_module.path, top_module) 

496 if isinstance(package, NamespacePackage): 

497 return top_module 

498 if package.stubs: 

499 self.expand_wildcards(top_module) 

500 # If stubs are in the package itself, they have been merged while loading modules, 

501 # so only the top-level init module needs to be merged still. 

502 # If stubs are in another package (a stubs-only package), 

503 # then we need to load the entire stubs package to merge everything. 

504 submodules = submodules and package.stubs.parent != package.path.parent 

505 stubs = self._load_module(package.name, package.stubs, submodules=submodules) 

506 return merge_stubs(top_module, stubs) 

507 return top_module 

508 

509 def _load_module( 

510 self, 

511 module_name: str, 

512 module_path: Path | list[Path], 

513 *, 

514 submodules: bool = True, 

515 parent: Module | None = None, 

516 ) -> Module: 

517 try: 

518 return self._load_module_path(module_name, module_path, submodules=submodules, parent=parent) 

519 except SyntaxError as error: 519 ↛ 521line 519 didn't jump to line 521

520 raise LoadingError(f"Syntax error: {error}") from error 

521 except ImportError as error: 

522 raise LoadingError(f"Import error: {error}") from error 

523 except UnicodeDecodeError as error: 

524 raise LoadingError(f"UnicodeDecodeError when loading {module_path}: {error}") from error 

525 except OSError as error: 

526 raise LoadingError(f"OSError when loading {module_path}: {error}") from error 

527 

528 def _load_module_path( 

529 self, 

530 module_name: str, 

531 module_path: Path | list[Path], 

532 *, 

533 submodules: bool = True, 

534 parent: Module | None = None, 

535 ) -> Module: 

536 logger.debug(f"Loading path {module_path}") 

537 if isinstance(module_path, list): 

538 module = self._create_module(module_name, module_path) 

539 elif self.force_inspection: 

540 module = self._inspect_module(module_name, module_path, parent) 

541 elif module_path.suffix in {".py", ".pyi"}: 

542 module = self._visit_module(module_name, module_path, parent) 

543 elif self.allow_inspection: 543 ↛ 546line 543 didn't jump to line 546 because the condition on line 543 was always true

544 module = self._inspect_module(module_name, module_path, parent) 

545 else: 

546 raise LoadingError("Cannot load compiled module without inspection") 

547 if submodules: 

548 self._load_submodules(module) 

549 return module 

550 

551 def _load_submodules(self, module: Module) -> None: 

552 for subparts, subpath in self.finder.submodules(module): 

553 self._load_submodule(module, subparts, subpath) 

554 

555 def _load_submodule(self, module: Module, subparts: tuple[str, ...], subpath: Path) -> None: 

556 for subpart in subparts: 

557 if "." in subpart: 

558 logger.debug(f"Skip {subpath}, dots in filenames are not supported") 

559 return 

560 try: 

561 parent_module = self._get_or_create_parent_module(module, subparts, subpath) 

562 except UnimportableModuleError as error: 

563 # NOTE: Why don't we load submodules when there's no init module in their folder? 

564 # Usually when a folder with Python files does not have an __init__.py module, 

565 # it's because the Python files are scripts that should never be imported. 

566 # Django has manage.py somewhere for example, in a folder without init module. 

567 # This script isn't part of the Python API, as it's meant to be called on the CLI exclusively 

568 # (at least it was the case a few years ago when I was still using Django). 

569 

570 # The other case when there's no init module is when a package is a native namespace package (PEP 420). 

571 # It does not make sense to have a native namespace package inside of a regular package (having init modules above), 

572 # because the regular package above blocks the namespace feature from happening, so I consider it a user error. 

573 # It's true that users could have a native namespace package inside of a pkg_resources-style namespace package, 

574 # but I've never seen this happen. 

575 

576 # It's also true that Python can actually import the module under the (wrongly declared) native namespace package, 

577 # so the Griffe debug log message is a bit misleading, 

578 # but that's because in that case Python acts like the whole tree is a regular package. 

579 # It works when the namespace package appears in only one search path (`sys.path`), 

580 # but will fail if it appears in multiple search paths: Python will only find the first occurrence. 

581 # It's better to not falsely support this, and to warn users. 

582 logger.debug(f"{error}. Missing __init__ module?") 

583 return 

584 submodule_name = subparts[-1] 

585 try: 

586 submodule = self._load_module( 

587 submodule_name, 

588 subpath, 

589 submodules=False, 

590 parent=parent_module, 

591 ) 

592 except LoadingError as error: 

593 logger.debug(str(error)) 

594 else: 

595 if submodule_name in parent_module.members: 

596 member = parent_module.members[submodule_name] 

597 if member.is_alias or not member.is_module: 

598 logger.debug( 

599 f"Submodule '{submodule.path}' is shadowing the member at the same path. " 

600 "We recommend renaming the member or the submodule (for example prefixing it with `_`), " 

601 "see https://mkdocstrings.github.io/griffe/best_practices/#avoid-member-submodule-name-shadowing.", 

602 ) 

603 parent_module.set_member(submodule_name, submodule) 

604 

605 def _create_module(self, module_name: str, module_path: Path | list[Path]) -> Module: 

606 return Module( 

607 module_name, 

608 filepath=module_path, 

609 lines_collection=self.lines_collection, 

610 modules_collection=self.modules_collection, 

611 ) 

612 

613 def _visit_module(self, module_name: str, module_path: Path, parent: Module | None = None) -> Module: 

614 code = module_path.read_text(encoding="utf8") 

615 if self.store_source: 

616 self.lines_collection[module_path] = code.splitlines(keepends=False) 

617 start = datetime.now(tz=timezone.utc) 

618 module = visit( 

619 module_name, 

620 filepath=module_path, 

621 code=code, 

622 extensions=self.extensions, 

623 parent=parent, 

624 docstring_parser=self.docstring_parser, 

625 docstring_options=self.docstring_options, 

626 lines_collection=self.lines_collection, 

627 modules_collection=self.modules_collection, 

628 ) 

629 elapsed = datetime.now(tz=timezone.utc) - start 

630 self._time_stats["time_spent_visiting"] += elapsed.microseconds 

631 return module 

632 

633 def _inspect_module(self, module_name: str, filepath: Path | None = None, parent: Module | None = None) -> Module: 

634 for prefix in self.ignored_modules: 

635 if module_name.startswith(prefix): 635 ↛ 636line 635 didn't jump to line 636 because the condition on line 635 was never true

636 raise ImportError(f"Ignored module '{module_name}'") 

637 if self.store_source and filepath and filepath.suffix in {".py", ".pyi"}: 

638 self.lines_collection[filepath] = filepath.read_text(encoding="utf8").splitlines(keepends=False) 

639 start = datetime.now(tz=timezone.utc) 

640 try: 

641 module = inspect( 

642 module_name, 

643 filepath=filepath, 

644 import_paths=self.finder.search_paths, 

645 extensions=self.extensions, 

646 parent=parent, 

647 docstring_parser=self.docstring_parser, 

648 docstring_options=self.docstring_options, 

649 lines_collection=self.lines_collection, 

650 modules_collection=self.modules_collection, 

651 ) 

652 except SystemExit as error: 

653 raise ImportError(f"Importing '{module_name}' raised a system exit") from error 

654 elapsed = datetime.now(tz=timezone.utc) - start 

655 self._time_stats["time_spent_inspecting"] += elapsed.microseconds 

656 return module 

657 

658 def _get_or_create_parent_module( 

659 self, 

660 module: Module, 

661 subparts: tuple[str, ...], 

662 subpath: Path, 

663 ) -> Module: 

664 parent_parts = subparts[:-1] 

665 if not parent_parts: 

666 return module 

667 parent_module = module 

668 parents = list(subpath.parents) 

669 if subpath.stem == "__init__": 

670 parents.pop(0) 

671 for parent_offset, parent_part in enumerate(parent_parts, 2): 

672 module_filepath = parents[len(subparts) - parent_offset] 

673 try: 

674 parent_module = parent_module.get_member(parent_part) 

675 except KeyError as error: 

676 if parent_module.is_namespace_package or parent_module.is_namespace_subpackage: 

677 next_parent_module = self._create_module(parent_part, [module_filepath]) 

678 parent_module.set_member(parent_part, next_parent_module) 

679 parent_module = next_parent_module 

680 else: 

681 raise UnimportableModuleError(f"Skip {subpath}, it is not importable") from error 

682 else: 

683 parent_namespace = parent_module.is_namespace_package or parent_module.is_namespace_subpackage 

684 if parent_namespace and module_filepath not in parent_module.filepath: # type: ignore[operator] 

685 parent_module.filepath.append(module_filepath) # type: ignore[union-attr] 

686 return parent_module 

687 

688 def _expand_wildcard(self, wildcard_obj: Alias) -> list[tuple[Object | Alias, int | None, int | None]]: 

689 module = self.modules_collection.get_member(wildcard_obj.wildcard) # type: ignore[arg-type] # we know it's a wildcard 

690 return [ 

691 (imported_member, wildcard_obj.alias_lineno, wildcard_obj.alias_endlineno) 

692 for imported_member in module.members.values() 

693 if imported_member.is_wildcard_exposed 

694 ] 

695 

696 

697def load( 

698 objspec: str | Path | None = None, 

699 /, 

700 *, 

701 submodules: bool = True, 

702 try_relative_path: bool = True, 

703 extensions: Extensions | None = None, 

704 search_paths: Sequence[str | Path] | None = None, 

705 docstring_parser: Parser | None = None, 

706 docstring_options: dict[str, Any] | None = None, 

707 lines_collection: LinesCollection | None = None, 

708 modules_collection: ModulesCollection | None = None, 

709 allow_inspection: bool = True, 

710 force_inspection: bool = False, 

711 store_source: bool = True, 

712 find_stubs_package: bool = False, 

713 resolve_aliases: bool = False, 

714 resolve_external: bool | None = None, 

715 resolve_implicit: bool = False, 

716) -> Object | Alias: 

717 """Load and return a Griffe object. 

718 

719 In Griffe's context, loading means: 

720 

721 - searching for a package, and finding it on the file system or as a builtin module 

722 (see the [`ModuleFinder`][griffe.ModuleFinder] class for more information) 

723 - extracting information from each of its (sub)modules, by either parsing 

724 the source code (see the [`visit`][griffe.visit] function) 

725 or inspecting the module at runtime (see the [`inspect`][griffe.inspect] function) 

726 

727 The extracted information is stored in a collection of modules, which can be queried later. 

728 Each collected module is a tree of objects, representing the structure of the module. 

729 See the [`Module`][griffe.Module], [`Class`][griffe.Class], 

730 [`Function`][griffe.Function], and [`Attribute`][griffe.Attribute] classes 

731 for more information. 

732 

733 The main class used to load modules is [`GriffeLoader`][griffe.GriffeLoader]. 

734 Convenience functions like this one and [`load_git`][griffe.load_git] are also available. 

735 

736 Example: 

737 ```python 

738 import griffe 

739 

740 module = griffe.load(...) 

741 ``` 

742 

743 This is a shortcut for: 

744 

745 ```python 

746 from griffe import GriffeLoader 

747 

748 loader = GriffeLoader(...) 

749 module = loader.load(...) 

750 ``` 

751 

752 See the documentation for the loader: [`GriffeLoader`][griffe.GriffeLoader]. 

753 

754 Parameters: 

755 objspec: The Python path of an object, or file path to a module. 

756 submodules: Whether to recurse on the submodules. 

757 This parameter only makes sense when loading a package (top-level module). 

758 try_relative_path: Whether to try finding the module as a relative path. 

759 extensions: The extensions to use. 

760 search_paths: The paths to search into. 

761 docstring_parser: The docstring parser to use. By default, no parsing is done. 

762 docstring_options: Additional docstring parsing options. 

763 lines_collection: A collection of source code lines. 

764 modules_collection: A collection of modules. 

765 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

766 force_inspection: Whether to force using dynamic analysis when loading data. 

767 store_source: Whether to store code source in the lines collection. 

768 find_stubs_package: Whether to search for stubs-only package. 

769 If both the package and its stubs are found, they'll be merged together. 

770 If only the stubs are found, they'll be used as the package itself. 

771 resolve_aliases: Whether to resolve aliases. 

772 resolve_external: Whether to try to load unspecified modules to resolve aliases. 

773 Default value (`None`) means to load external modules only if they are the private sibling 

774 or the origin module (for example when `ast` imports from `_ast`). 

775 resolve_implicit: When false, only try to resolve an alias if it is explicitly exported. 

776 

777 Returns: 

778 A Griffe object. 

779 """ 

780 loader = GriffeLoader( 

781 extensions=extensions, 

782 search_paths=search_paths, 

783 docstring_parser=docstring_parser, 

784 docstring_options=docstring_options, 

785 lines_collection=lines_collection, 

786 modules_collection=modules_collection, 

787 allow_inspection=allow_inspection, 

788 force_inspection=force_inspection, 

789 store_source=store_source, 

790 ) 

791 result = loader.load( 

792 objspec, 

793 submodules=submodules, 

794 try_relative_path=try_relative_path, 

795 find_stubs_package=find_stubs_package, 

796 ) 

797 if resolve_aliases: 

798 loader.resolve_aliases(implicit=resolve_implicit, external=resolve_external) 

799 return result 

800 

801 

802def load_git( 

803 objspec: str | Path | None = None, 

804 /, 

805 *, 

806 ref: str = "HEAD", 

807 repo: str | Path = ".", 

808 submodules: bool = True, 

809 extensions: Extensions | None = None, 

810 search_paths: Sequence[str | Path] | None = None, 

811 docstring_parser: Parser | None = None, 

812 docstring_options: dict[str, Any] | None = None, 

813 lines_collection: LinesCollection | None = None, 

814 modules_collection: ModulesCollection | None = None, 

815 allow_inspection: bool = True, 

816 force_inspection: bool = False, 

817 find_stubs_package: bool = False, 

818 resolve_aliases: bool = False, 

819 resolve_external: bool | None = None, 

820 resolve_implicit: bool = False, 

821) -> Object | Alias: 

822 """Load and return a module from a specific Git reference. 

823 

824 This function will create a temporary 

825 [git worktree](https://git-scm.com/docs/git-worktree) at the requested reference 

826 before loading `module` with [`griffe.load`][griffe.load]. 

827 

828 This function requires that the `git` executable is installed. 

829 

830 Examples: 

831 ```python 

832 from griffe import load_git 

833 

834 old_api = load_git("my_module", ref="v0.1.0", repo="path/to/repo") 

835 ``` 

836 

837 Parameters: 

838 objspec: The Python path of an object, or file path to a module. 

839 ref: A Git reference such as a commit, tag or branch. 

840 repo: Path to the repository (i.e. the directory *containing* the `.git` directory) 

841 submodules: Whether to recurse on the submodules. 

842 This parameter only makes sense when loading a package (top-level module). 

843 extensions: The extensions to use. 

844 search_paths: The paths to search into (relative to the repository root). 

845 docstring_parser: The docstring parser to use. By default, no parsing is done. 

846 docstring_options: Additional docstring parsing options. 

847 lines_collection: A collection of source code lines. 

848 modules_collection: A collection of modules. 

849 allow_inspection: Whether to allow inspecting modules when visiting them is not possible. 

850 force_inspection: Whether to force using dynamic analysis when loading data. 

851 find_stubs_package: Whether to search for stubs-only package. 

852 If both the package and its stubs are found, they'll be merged together. 

853 If only the stubs are found, they'll be used as the package itself.