Coverage for src/_griffe/agents/inspector.py: 89.51%

1# This module contains our dynamic analysis agent,

2# capable of inspecting modules and objects in memory, at runtime.

4from __future__ import annotations

6import ast

7from inspect import Parameter as SignatureParameter

8from inspect import Signature, cleandoc, getsourcelines

9from inspect import signature as getsignature

10from typing import TYPE_CHECKING, Any, Sequence

12from _griffe.agents.nodes.runtime import ObjectNode

13from _griffe.collections import LinesCollection, ModulesCollection

14from _griffe.enumerations import ObjectKind, ParameterKind

15from _griffe.expressions import safe_get_annotation

16from _griffe.extensions.base import Extensions, load_extensions

17from _griffe.importer import dynamic_import

18from _griffe.logger import logger

19from _griffe.models import Alias, Attribute, Class, Docstring, Function, Module, Parameter, Parameters

21if TYPE_CHECKING:

22 from pathlib import Path

24 from _griffe.enumerations import Parser

25 from _griffe.expressions import Expr

28_empty = Signature.empty

31def inspect(

32 module_name: str,

33 *,

34 filepath: Path | None = None,

35 import_paths: Sequence[str | Path] | None = None,

36 extensions: Extensions | None = None,

37 parent: Module | None = None,

38 docstring_parser: Parser | None = None,

39 docstring_options: dict[str, Any] | None = None,

40 lines_collection: LinesCollection | None = None,

41 modules_collection: ModulesCollection | None = None,

42) -> Module:

43 """Inspect a module.

45 Sometimes we cannot get the source code of a module or an object,

46 typically built-in modules like `itertools`.

47 The only way to know what they are made of is to actually import them and inspect their contents.

49 Sometimes, even if the source code is available,

50 loading the object is desired because it was created or modified dynamically,

51 and our static agent is not powerful enough to infer all these dynamic modifications.

52 In this case, we load the module using introspection.

54 Griffe therefore provides this function for dynamic analysis.

55 It uses a [`NodeVisitor`][ast.NodeVisitor]-like class, the [`Inspector`][griffe.Inspector],

56 to inspect the module with [`inspect.getmembers()`][inspect.getmembers].

58 The inspection agent works similarly to the regular [`Visitor`][griffe.Visitor] agent,

59 in that it maintains a state with the current object being handled, and recursively handle its members.

61 Important:

62 This function is generally not used directly.

63 In most cases, users can rely on the [`GriffeLoader`][griffe.GriffeLoader]

64 and its accompanying [`load`][griffe.load] shortcut and their respective options

65 to load modules using dynamic analysis.

67 Parameters:

68 module_name: The module name (as when importing [from] it).

69 filepath: The module file path.

70 import_paths: Paths to import the module from.

71 extensions: The extensions to use when inspecting the module.

72 parent: The optional parent of this module.

73 docstring_parser: The docstring parser to use. By default, no parsing is done.

74 docstring_options: Additional docstring parsing options.

75 lines_collection: A collection of source code lines.

76 modules_collection: A collection of modules.

78 Returns:

79 The module, with its members populated.

80 """

81 return Inspector(

82 module_name,

83 filepath,

84 extensions or load_extensions(),

85 parent,

86 docstring_parser=docstring_parser,

87 docstring_options=docstring_options,

88 lines_collection=lines_collection,

89 modules_collection=modules_collection,

90 ).get_module(import_paths)

93class Inspector:

94 """This class is used to instantiate an inspector.

96 Inspectors iterate on objects members to extract data from them.

97 """

99 def __init__(

100 self,

101 module_name: str,

102 filepath: Path | None,

103 extensions: Extensions,

104 parent: Module | None = None,

105 docstring_parser: Parser | None = None,

106 docstring_options: dict[str, Any] | None = None,

107 lines_collection: LinesCollection | None = None,

108 modules_collection: ModulesCollection | None = None,

109 ) -> None:

110 """Initialize the inspector.

111

112 Parameters:

113 module_name: The module name.

114 filepath: The optional filepath.

115 extensions: Extensions to use when inspecting.

116 parent: The module parent.

117 docstring_parser: The docstring parser to use.

118 docstring_options: The docstring parsing options.

119 lines_collection: A collection of source code lines.

120 modules_collection: A collection of modules.

121 """

122 super().__init__()

123

124 self.module_name: str = module_name

125 """The module name."""

126

127 self.filepath: Path | None = filepath

128 """The module file path."""

129

130 self.extensions: Extensions = extensions

131 """The extensions to use when inspecting."""

132

133 self.parent: Module | None = parent

134 """An optional parent for the final module object."""

135

136 self.current: Module | Class = None # type: ignore[assignment]

137 """The current object being inspected."""

138

139 self.docstring_parser: Parser | None = docstring_parser

140 """The docstring parser to use."""

141

142 self.docstring_options: dict[str, Any] = docstring_options or {}

143 """The docstring parsing options."""

144

145 self.lines_collection: LinesCollection = lines_collection or LinesCollection()

146 """A collection of source code lines."""

147

148 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection()

149 """A collection of modules."""

150

151 def _get_docstring(self, node: ObjectNode) -> Docstring | None:

152 try:

153 # Access `__doc__` directly to avoid taking the `__doc__` attribute from a parent class.

154 value = getattr(node.obj, "__doc__", None)

155 except Exception: # noqa: BLE001 # getattr can trigger exceptions

156 return None

157 if value is None:

158 return None

159 try:

160 # We avoid `inspect.getdoc` to avoid getting

161 # the `__doc__` attribute from a parent class,

162 # but we still want to clean the doc.

163 cleaned = cleandoc(value)

164 except AttributeError:

165 # Triggered on method descriptors.

166 return None

167 return Docstring(

168 cleaned,

169 parser=self.docstring_parser,

170 parser_options=self.docstring_options,

171 )

172

173 def _get_linenos(self, node: ObjectNode) -> tuple[int, int] | tuple[None, None]:

174 # Line numbers won't be useful if we don't have the source code.

175 if not self.filepath or self.filepath not in self.lines_collection:

176 return None, None

177 try:

178 lines, lineno = getsourcelines(node.obj)

179 except (OSError, TypeError):

180 return None, None

181 return lineno, lineno + "".join(lines).rstrip().count("\n")

182

183 def get_module(self, import_paths: Sequence[str | Path] | None = None) -> Module:

184 """Build and return the object representing the module attached to this inspector.

185

186 This method triggers a complete inspection of the module members.

187

188 Parameters:

189 import_paths: Paths replacing `sys.path` to import the module.

190

191 Returns:

192 A module instance.

193 """

194 import_path = self.module_name

195 if self.parent is not None:

196 import_path = f"{self.parent.path}.{import_path}"

197

198 # Make sure `import_paths` is a list, in case we want to `insert` into it.

199 import_paths = list(import_paths or ())

200

201 # If the thing we want to import has a filepath,

202 # we make sure to insert the right parent directory

203 # at the front of our list of import paths.

204 # We do this by counting the number of dots `.` in the import path,

205 # corresponding to slashes `/` in the filesystem,

206 # and go up in the file tree the same number of times.

207 if self.filepath:

208 parent_path = self.filepath.parent

209 for _ in range(import_path.count(".")):

210 parent_path = parent_path.parent

211 # Climb up one more time for `__init__` modules.

212 if self.filepath.stem == "__init__":

213 parent_path = parent_path.parent

214 if parent_path not in import_paths:

215 import_paths.insert(0, parent_path)

216

217 value = dynamic_import(import_path, import_paths)

218

219 # We successfully imported the given object,

220 # and we now create the object tree with all the necessary nodes,

221 # from the root of the package to this leaf object.

222 parent_node = None

223 if self.parent is not None:

224 for part in self.parent.path.split("."):

225 parent_node = ObjectNode(None, name=part, parent=parent_node)

226 module_node = ObjectNode(value, self.module_name, parent=parent_node)

227

228 self.inspect(module_node)

229 return self.current.module

230

231 def inspect(self, node: ObjectNode) -> None:

232 """Extend the base inspection with extensions.

233

234 Parameters:

235 node: The node to inspect.

236 """

237 getattr(self, f"inspect_{node.kind}", self.generic_inspect)(node)

238

239 def generic_inspect(self, node: ObjectNode) -> None:

240 """Extend the base generic inspection with extensions.

241

242 Parameters:

243 node: The node to inspect.

244 """

245 for child in node.children:

246 if target_path := child.alias_target_path:

247 # If the child is an actual submodule of the current module,

248 # and has no `__file__` set, we won't find it on the disk so we must inspect it now.

249 # For that we instantiate a new inspector and use it to inspect the submodule,

250 # then assign the submodule as member of the current module.

251 # If the submodule has a `__file__` set, the loader should find it on the disk,

252 # so we skip it here (no member, no alias, just skip it).

253 if child.is_module and target_path == f"{self.current.path}.{child.name}":

254 if not hasattr(child.obj, "__file__"): 254 ↛ 255line 254 didn't jump to line 255 because the condition on line 254 was never true

255 logger.debug(f"Module {target_path} is not discoverable on disk, inspecting right now")

256 inspector = Inspector(

257 child.name,

258 filepath=None,

259 parent=self.current.module,

260 extensions=self.extensions,

261 docstring_parser=self.docstring_parser,

262 docstring_options=self.docstring_options,

263 lines_collection=self.lines_collection,

264 modules_collection=self.modules_collection,

265 )

266 inspector.inspect_module(child)

267 self.current.set_member(child.name, inspector.current.module)

268 # Otherwise, alias the object.

269 else:

270 self.current.set_member(child.name, Alias(child.name, target_path))

271 else:

272 self.inspect(child)

273

274 def inspect_module(self, node: ObjectNode) -> None:

275 """Inspect a module.

276

277 Parameters:

278 node: The node to inspect.

279 """

280 self.extensions.call("on_node", node=node, agent=self)

281 self.extensions.call("on_module_node", node=node, agent=self)

282 self.current = module = Module(

283 name=self.module_name,

284 filepath=self.filepath,

285 parent=self.parent,

286 docstring=self._get_docstring(node),

287 lines_collection=self.lines_collection,

288 modules_collection=self.modules_collection,

289 )

290 self.extensions.call("on_instance", node=node, obj=module, agent=self)

291 self.extensions.call("on_module_instance", node=node, mod=module, agent=self)

292 self.generic_inspect(node)

293 self.extensions.call("on_members", node=node, obj=module, agent=self)

294 self.extensions.call("on_module_members", node=node, mod=module, agent=self)

295

296 def inspect_class(self, node: ObjectNode) -> None:

297 """Inspect a class.

298

299 Parameters:

300 node: The node to inspect.

301 """

302 self.extensions.call("on_node", node=node, agent=self)

303 self.extensions.call("on_class_node", node=node, agent=self)

304

305 bases = []

306 for base in node.obj.__bases__:

307 if base is object:

308 continue

309 bases.append(f"{base.__module__}.{base.__qualname__}")

310

311 lineno, endlineno = self._get_linenos(node)

312 class_ = Class(

313 name=node.name,

314 docstring=self._get_docstring(node),

315 bases=bases,

316 lineno=lineno,

317 endlineno=endlineno,

318 )

319 self.current.set_member(node.name, class_)

320 self.current = class_

321 self.extensions.call("on_instance", node=node, obj=class_, agent=self)

322 self.extensions.call("on_class_instance", node=node, cls=class_, agent=self)

323 self.generic_inspect(node)

324 self.extensions.call("on_members", node=node, obj=class_, agent=self)

325 self.extensions.call("on_class_members", node=node, cls=class_, agent=self)

326 self.current = self.current.parent # type: ignore[assignment]

327

328 def inspect_staticmethod(self, node: ObjectNode) -> None:

329 """Inspect a static method.

330

331 Parameters:

332 node: The node to inspect.

333 """

334 self.handle_function(node, {"staticmethod"})

335

336 def inspect_classmethod(self, node: ObjectNode) -> None:

337 """Inspect a class method.

338

339 Parameters:

340 node: The node to inspect.

341 """

342 self.handle_function(node, {"classmethod"})

343

344 def inspect_method_descriptor(self, node: ObjectNode) -> None:

345 """Inspect a method descriptor.

346

347 Parameters:

348 node: The node to inspect.

349 """

350 self.handle_function(node, {"method descriptor"})

351

352 def inspect_builtin_method(self, node: ObjectNode) -> None:

353 """Inspect a builtin method.

354

355 Parameters:

356 node: The node to inspect.

357 """

358 self.handle_function(node, {"builtin"})

359

360 def inspect_method(self, node: ObjectNode) -> None:

361 """Inspect a method.

362

363 Parameters:

364 node: The node to inspect.

365 """

366 self.handle_function(node)

367

368 def inspect_coroutine(self, node: ObjectNode) -> None:

369 """Inspect a coroutine.

370

371 Parameters:

372 node: The node to inspect.

373 """

374 self.handle_function(node, {"async"})

375

376 def inspect_builtin_function(self, node: ObjectNode) -> None:

377 """Inspect a builtin function.

378

379 Parameters:

380 node: The node to inspect.

381 """

382 self.handle_function(node, {"builtin"})

383

384 def inspect_function(self, node: ObjectNode) -> None:

385 """Inspect a function.

386

387 Parameters:

388 node: The node to inspect.

389 """

390 self.handle_function(node)

391

392 def inspect_cached_property(self, node: ObjectNode) -> None:

393 """Inspect a cached property.

394

395 Parameters:

396 node: The node to inspect.

397 """

398 self.handle_function(node, {"cached", "property"})

399

400 def inspect_property(self, node: ObjectNode) -> None:

401 """Inspect a property.

402

403 Parameters:

404 node: The node to inspect.

405 """

406 self.handle_function(node, {"property"})

407

408 def handle_function(self, node: ObjectNode, labels: set | None = None) -> None:

409 """Handle a function.

410

411 Parameters:

412 node: The node to inspect.

413 labels: Labels to add to the data object.

414 """

415 self.extensions.call("on_node", node=node, agent=self)

416 self.extensions.call("on_function_node", node=node, agent=self)

417

418 try:

419 signature = getsignature(node.obj)

420 except Exception: # noqa: BLE001

421 # so many exceptions can be raised here:

422 # AttributeError, NameError, RuntimeError, ValueError, TokenError, TypeError

423 parameters = None

424 returns = None

425 else:

426 parameters = Parameters(

427 *[_convert_parameter(parameter, parent=self.current) for parameter in signature.parameters.values()],

428 )

429 return_annotation = signature.return_annotation

430 returns = (

431 None

432 if return_annotation is _empty

433 else _convert_object_to_annotation(return_annotation, parent=self.current)

434 )

435

436 lineno, endlineno = self._get_linenos(node)

437

438 obj: Attribute | Function

439 labels = labels or set()

440 if "property" in labels:

441 obj = Attribute(

442 name=node.name,

443 value=None,

444 annotation=returns,

445 docstring=self._get_docstring(node),

446 lineno=lineno,

447 endlineno=endlineno,

448 )

449 else:

450 obj = Function(

451 name=node.name,

452 parameters=parameters,

453 returns=returns,

454 docstring=self._get_docstring(node),

455 lineno=lineno,

456 endlineno=endlineno,

457 )

458 obj.labels |= labels

459 self.current.set_member(node.name, obj)

460 self.extensions.call("on_instance", node=node, obj=obj, agent=self)

461 if obj.is_attribute:

462 self.extensions.call("on_attribute_instance", node=node, attr=obj, agent=self)

463 else:

464 self.extensions.call("on_function_instance", node=node, func=obj, agent=self)

465

466 def inspect_attribute(self, node: ObjectNode) -> None:

467 """Inspect an attribute.

468

469 Parameters:

470 node: The node to inspect.

471 """

472 self.handle_attribute(node)

473

474 def handle_attribute(self, node: ObjectNode, annotation: str | Expr | None = None) -> None:

475 """Handle an attribute.

476

477 Parameters:

478 node: The node to inspect.

479 annotation: A potential annotation.

480 """

481 self.extensions.call("on_node", node=node, agent=self)

482 self.extensions.call("on_attribute_node", node=node, agent=self)

483

484 # TODO: to improve

485 parent = self.current

486 labels: set[str] = set()

487

488 if parent.kind is ObjectKind.MODULE: 488 ↛ 489line 488 didn't jump to line 489 because the condition on line 488 was never true

489 labels.add("module")

490 elif parent.kind is ObjectKind.CLASS: 490 ↛ 491line 490 didn't jump to line 491 because the condition on line 490 was never true

491 labels.add("class")

492 elif parent.kind is ObjectKind.FUNCTION: 492 ↛ 493line 492 didn't jump to line 493 because the condition on line 492 was never true

493 if parent.name != "__init__":

494 return

495 parent = parent.parent

496 labels.add("instance")

497

498 try:

499 value = repr(node.obj)

500 except Exception: # noqa: BLE001

501 value = None

502 try:

503 docstring = self._get_docstring(node)

504 except Exception: # noqa: BLE001

505 docstring = None

506

507 attribute = Attribute(

508 name=node.name,

509 value=value,

510 annotation=annotation,

511 docstring=docstring,

512 )

513 attribute.labels |= labels

514 parent.set_member(node.name, attribute)

515

516 if node.name == "__all__":

517 parent.exports = set(node.obj)

518 self.extensions.call("on_instance", node=node, obj=attribute, agent=self)

519 self.extensions.call("on_attribute_instance", node=node, attr=attribute, agent=self)

520

521

522_kind_map = {

523 SignatureParameter.POSITIONAL_ONLY: ParameterKind.positional_only,

524 SignatureParameter.POSITIONAL_OR_KEYWORD: ParameterKind.positional_or_keyword,

525 SignatureParameter.VAR_POSITIONAL: ParameterKind.var_positional,

526 SignatureParameter.KEYWORD_ONLY: ParameterKind.keyword_only,

527 SignatureParameter.VAR_KEYWORD: ParameterKind.var_keyword,

528}

529

530

531def _convert_parameter(parameter: SignatureParameter, parent: Module | Class) -> Parameter:

532 name = parameter.name

533 annotation = (

534 None if parameter.annotation is _empty else _convert_object_to_annotation(parameter.annotation, parent=parent)

535 )

536 kind = _kind_map[parameter.kind]

537 if parameter.default is _empty:

538 default = None

539 elif hasattr(parameter.default, "__name__"):

540 # avoid repr containing chevrons and memory addresses

541 default = parameter.default.__name__

542 else:

543 default = repr(parameter.default)

544 return Parameter(name, annotation=annotation, kind=kind, default=default)

545

546

547def _convert_object_to_annotation(obj: Any, parent: Module | Class) -> str | Expr | None:

548 # even when *we* import future annotations,

549 # the object from which we get a signature

550 # can come from modules which did *not* import them,

551 # so inspect.signature returns actual Python objects

552 # that we must deal with

553 if not isinstance(obj, str):

554 if hasattr(obj, "__name__"): # noqa: SIM108 554 ↛ 559line 554 didn't jump to line 559 because the condition on line 554 was always true

555 # simple types like int, str, custom classes, etc.

556 obj = obj.__name__

557 else:

558 # other, more complex types: hope for the best

559 obj = repr(obj)

560 try:

561 annotation_node = compile(obj, mode="eval", filename="<>", flags=ast.PyCF_ONLY_AST, optimize=2)

562 except SyntaxError:

563 return obj

564 return safe_get_annotation(annotation_node.body, parent=parent) # type: ignore[attr-defined]