Coverage for src/griffe/_internal/agents/inspector.py: 90.39%
265 statements
« prev ^ index » next coverage.py v7.10.2, created at 2025-08-11 13:44 +0200
1# This module contains our dynamic analysis agent,
2# capable of inspecting modules and objects in memory, at runtime.
4from __future__ import annotations
6import ast
7import functools
8import sys
9import types
10import typing
11from inspect import Parameter as SignatureParameter
12from inspect import Signature, cleandoc, getsourcelines, unwrap
13from inspect import signature as getsignature
14from typing import TYPE_CHECKING, Any
16from griffe._internal.agents.nodes.runtime import ObjectNode
17from griffe._internal.collections import LinesCollection, ModulesCollection
18from griffe._internal.enumerations import Kind, ParameterKind, TypeParameterKind
19from griffe._internal.expressions import Expr, ExprBinOp, ExprSubscript, ExprTuple, safe_get_annotation
20from griffe._internal.extensions.base import Extensions, load_extensions
21from griffe._internal.importer import dynamic_import
22from griffe._internal.logger import logger
23from griffe._internal.models import (
24 Alias,
25 Attribute,
26 Class,
27 Docstring,
28 Function,
29 Module,
30 Parameter,
31 Parameters,
32 TypeAlias,
33 TypeParameter,
34 TypeParameters,
35)
37if TYPE_CHECKING:
38 from collections.abc import Sequence
39 from pathlib import Path
41 from griffe._internal.docstrings.parsers import DocstringStyle
42 from griffe._internal.enumerations import Parser
44_TYPING_MODULES: tuple[types.ModuleType, ...]
45try:
46 import typing_extensions
47except ImportError:
48 _TYPING_MODULES = (typing,)
49else:
50 _TYPING_MODULES = (typing, typing_extensions)
52_empty = Signature.empty
55def inspect(
56 module_name: str,
57 *,
58 filepath: Path | None = None,
59 import_paths: Sequence[str | Path] | None = None,
60 extensions: Extensions | None = None,
61 parent: Module | None = None,
62 docstring_parser: DocstringStyle | Parser | None = None,
63 docstring_options: dict[str, Any] | None = None,
64 lines_collection: LinesCollection | None = None,
65 modules_collection: ModulesCollection | None = None,
66) -> Module:
67 """Inspect a module.
69 Sometimes we cannot get the source code of a module or an object,
70 typically built-in modules like `itertools`.
71 The only way to know what they are made of is to actually import them and inspect their contents.
73 Sometimes, even if the source code is available,
74 loading the object is desired because it was created or modified dynamically,
75 and our static agent is not powerful enough to infer all these dynamic modifications.
76 In this case, we load the module using introspection.
78 Griffe therefore provides this function for dynamic analysis.
79 It uses a [`NodeVisitor`][ast.NodeVisitor]-like class, the [`Inspector`][griffe.Inspector],
80 to inspect the module with [`inspect.getmembers()`][inspect.getmembers].
82 The inspection agent works similarly to the regular [`Visitor`][griffe.Visitor] agent,
83 in that it maintains a state with the current object being handled, and recursively handle its members.
85 Important:
86 This function is generally not used directly.
87 In most cases, users can rely on the [`GriffeLoader`][griffe.GriffeLoader]
88 and its accompanying [`load`][griffe.load] shortcut and their respective options
89 to load modules using dynamic analysis.
91 Parameters:
92 module_name: The module name (as when importing [from] it).
93 filepath: The module file path.
94 import_paths: Paths to import the module from.
95 extensions: The extensions to use when inspecting the module.
96 parent: The optional parent of this module.
97 docstring_parser: The docstring parser to use. By default, no parsing is done.
98 docstring_options: Additional docstring parsing options.
99 lines_collection: A collection of source code lines.
100 modules_collection: A collection of modules.
102 Returns:
103 The module, with its members populated.
104 """
105 return Inspector(
106 module_name,
107 filepath,
108 extensions or load_extensions(),
109 parent,
110 docstring_parser=docstring_parser,
111 docstring_options=docstring_options,
112 lines_collection=lines_collection,
113 modules_collection=modules_collection,
114 ).get_module(import_paths)
117class Inspector:
118 """This class is used to instantiate an inspector.
120 Inspectors iterate on objects members to extract data from them.
121 """
123 def __init__(
124 self,
125 module_name: str,
126 filepath: Path | None,
127 extensions: Extensions,
128 parent: Module | None = None,
129 docstring_parser: DocstringStyle | Parser | None = None,
130 docstring_options: dict[str, Any] | None = None,
131 lines_collection: LinesCollection | None = None,
132 modules_collection: ModulesCollection | None = None,
133 ) -> None:
134 """Initialize the inspector.
136 Parameters:
137 module_name: The module name.
138 filepath: The optional filepath.
139 extensions: Extensions to use when inspecting.
140 parent: The module parent.
141 docstring_parser: The docstring parser to use.
142 docstring_options: The docstring parsing options.
143 lines_collection: A collection of source code lines.
144 modules_collection: A collection of modules.
145 """
146 super().__init__()
148 self.module_name: str = module_name
149 """The module name."""
151 self.filepath: Path | None = filepath
152 """The module file path."""
154 self.extensions: Extensions = extensions
155 """The extensions to use when inspecting."""
157 self.parent: Module | None = parent
158 """An optional parent for the final module object."""
160 self.current: Module | Class = None # type: ignore[assignment]
161 """The current object being inspected."""
163 self.docstring_parser: DocstringStyle | Parser | None = docstring_parser
164 """The docstring parser to use."""
166 self.docstring_options: dict[str, Any] = docstring_options or {}
167 """The docstring parsing options."""
169 self.lines_collection: LinesCollection = lines_collection or LinesCollection()
170 """A collection of source code lines."""
172 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection()
173 """A collection of modules."""
175 def _get_docstring(self, node: ObjectNode) -> Docstring | None:
176 try:
177 # Access `__doc__` directly to avoid taking the `__doc__` attribute from a parent class.
178 value = getattr(node.obj, "__doc__", None)
179 except Exception: # noqa: BLE001
180 return None
181 if value is None:
182 return None
183 try:
184 # We avoid `inspect.getdoc` to avoid getting
185 # the `__doc__` attribute from a parent class,
186 # but we still want to clean the doc.
187 cleaned = cleandoc(value)
188 except AttributeError:
189 # Triggered on method descriptors.
190 return None
191 return Docstring(
192 cleaned,
193 parser=self.docstring_parser,
194 parser_options=self.docstring_options,
195 )
197 def _get_linenos(self, node: ObjectNode) -> tuple[int, int] | tuple[None, None]:
198 # Line numbers won't be useful if we don't have the source code.
199 if not self.filepath or self.filepath not in self.lines_collection:
200 return None, None
201 try:
202 lines, lineno = getsourcelines(node.obj)
203 except (OSError, TypeError):
204 return None, None
205 return lineno, lineno + "".join(lines).rstrip().count("\n")
207 def get_module(self, import_paths: Sequence[str | Path] | None = None) -> Module:
208 """Build and return the object representing the module attached to this inspector.
210 This method triggers a complete inspection of the module members.
212 Parameters:
213 import_paths: Paths replacing `sys.path` to import the module.
215 Returns:
216 A module instance.
217 """
218 import_path = self.module_name
219 if self.parent is not None:
220 import_path = f"{self.parent.path}.{import_path}"
222 # Make sure `import_paths` is a list, in case we want to `insert` into it.
223 import_paths = list(import_paths or ())
225 # If the thing we want to import has a filepath,
226 # we make sure to insert the right parent directory
227 # at the front of our list of import paths.
228 # We do this by counting the number of dots `.` in the import path,
229 # corresponding to slashes `/` in the filesystem,
230 # and go up in the file tree the same number of times.
231 if self.filepath:
232 parent_path = self.filepath.parent
233 for _ in range(import_path.count(".")):
234 parent_path = parent_path.parent
235 # Climb up one more time for `__init__` modules.
236 if self.filepath.stem == "__init__":
237 parent_path = parent_path.parent
238 if parent_path not in import_paths:
239 import_paths.insert(0, parent_path)
241 value = dynamic_import(import_path, import_paths)
243 # We successfully imported the given object,
244 # and we now create the object tree with all the necessary nodes,
245 # from the root of the package to this leaf object.
246 parent_node = None
247 if self.parent is not None:
248 for part in self.parent.path.split("."):
249 parent_node = ObjectNode(None, name=part, parent=parent_node)
250 module_node = ObjectNode(value, self.module_name, parent=parent_node)
252 self.inspect(module_node)
253 return self.current.module
255 def inspect(self, node: ObjectNode) -> None:
256 """Extend the base inspection with extensions.
258 Parameters:
259 node: The node to inspect.
260 """
261 getattr(self, f"inspect_{node.kind}", self.generic_inspect)(node)
263 def generic_inspect(self, node: ObjectNode) -> None:
264 """Extend the base generic inspection with extensions.
266 Parameters:
267 node: The node to inspect.
268 """
269 for child in node.children:
270 if target_path := child.alias_target_path:
271 # If the child is an actual submodule of the current module,
272 # and has no `__file__` set, we won't find it on the disk so we must inspect it now.
273 # For that we instantiate a new inspector and use it to inspect the submodule,
274 # then assign the submodule as member of the current module.
275 # If the submodule has a `__file__` set, the loader should find it on the disk,
276 # so we skip it here (no member, no alias, just skip it).
277 if child.is_module and target_path == f"{self.current.path}.{child.name}":
278 if not hasattr(child.obj, "__file__"): 278 ↛ 279line 278 didn't jump to line 279 because the condition on line 278 was never true
279 logger.debug("Module %s is not discoverable on disk, inspecting right now", target_path)
280 inspector = Inspector(
281 child.name,
282 filepath=None,
283 parent=self.current.module,
284 extensions=self.extensions,
285 docstring_parser=self.docstring_parser,
286 docstring_options=self.docstring_options,
287 lines_collection=self.lines_collection,
288 modules_collection=self.modules_collection,
289 )
290 inspector.inspect_module(child)
291 self.current.set_member(child.name, inspector.current.module)
292 # Otherwise, alias the object.
293 else:
294 alias = Alias(child.name, target_path)
295 self.current.set_member(child.name, alias)
296 self.extensions.call("on_alias", alias=alias, node=node, agent=self)
297 else:
298 self.inspect(child)
300 def inspect_module(self, node: ObjectNode) -> None:
301 """Inspect a module.
303 Parameters:
304 node: The node to inspect.
305 """
306 self.extensions.call("on_node", node=node, agent=self)
307 self.extensions.call("on_module_node", node=node, agent=self)
308 self.current = module = Module(
309 name=self.module_name,
310 filepath=self.filepath,
311 parent=self.parent,
312 docstring=self._get_docstring(node),
313 lines_collection=self.lines_collection,
314 modules_collection=self.modules_collection,
315 )
316 self.extensions.call("on_instance", node=node, obj=module, agent=self)
317 self.extensions.call("on_module_instance", node=node, mod=module, agent=self)
318 self.generic_inspect(node)
319 self.extensions.call("on_members", node=node, obj=module, agent=self)
320 self.extensions.call("on_module_members", node=node, mod=module, agent=self)
322 def inspect_class(self, node: ObjectNode) -> None:
323 """Inspect a class.
325 Parameters:
326 node: The node to inspect.
327 """
328 self.extensions.call("on_node", node=node, agent=self)
329 self.extensions.call("on_class_node", node=node, agent=self)
331 bases = []
332 for base in node.obj.__bases__:
333 if base is object:
334 continue
335 bases.append(f"{base.__module__}.{base.__qualname__}")
337 lineno, endlineno = self._get_linenos(node)
338 class_ = Class(
339 name=node.name,
340 docstring=self._get_docstring(node),
341 bases=bases,
342 type_parameters=TypeParameters(*_convert_type_parameters(node.obj, parent=self.current, member=node.name)),
343 lineno=lineno,
344 endlineno=endlineno,
345 )
346 self.current.set_member(node.name, class_)
347 self.current = class_
348 self.extensions.call("on_instance", node=node, obj=class_, agent=self)
349 self.extensions.call("on_class_instance", node=node, cls=class_, agent=self)
350 self.generic_inspect(node)
351 self.extensions.call("on_members", node=node, obj=class_, agent=self)
352 self.extensions.call("on_class_members", node=node, cls=class_, agent=self)
353 self.current = self.current.parent # type: ignore[assignment]
355 def inspect_staticmethod(self, node: ObjectNode) -> None:
356 """Inspect a static method.
358 Parameters:
359 node: The node to inspect.
360 """
361 self.handle_function(node, {"staticmethod"})
363 def inspect_classmethod(self, node: ObjectNode) -> None:
364 """Inspect a class method.
366 Parameters:
367 node: The node to inspect.
368 """
369 self.handle_function(node, {"classmethod"})
371 def inspect_method_descriptor(self, node: ObjectNode) -> None:
372 """Inspect a method descriptor.
374 Parameters:
375 node: The node to inspect.
376 """
377 self.handle_function(node, {"method descriptor"})
379 def inspect_builtin_method(self, node: ObjectNode) -> None:
380 """Inspect a builtin method.
382 Parameters:
383 node: The node to inspect.
384 """
385 self.handle_function(node, {"builtin"})
387 def inspect_method(self, node: ObjectNode) -> None:
388 """Inspect a method.
390 Parameters:
391 node: The node to inspect.
392 """
393 self.handle_function(node)
395 def inspect_coroutine(self, node: ObjectNode) -> None:
396 """Inspect a coroutine.
398 Parameters:
399 node: The node to inspect.
400 """
401 self.handle_function(node, {"async"})
403 def inspect_builtin_function(self, node: ObjectNode) -> None:
404 """Inspect a builtin function.
406 Parameters:
407 node: The node to inspect.
408 """
409 self.handle_function(node, {"builtin"})
411 def inspect_function(self, node: ObjectNode) -> None:
412 """Inspect a function.
414 Parameters:
415 node: The node to inspect.
416 """
417 self.handle_function(node)
419 def inspect_cached_property(self, node: ObjectNode) -> None:
420 """Inspect a cached property.
422 Parameters:
423 node: The node to inspect.
424 """
425 self.handle_function(node, {"cached", "property"})
427 def inspect_property(self, node: ObjectNode) -> None:
428 """Inspect a property.
430 Parameters:
431 node: The node to inspect.
432 """
433 self.handle_function(node, {"property"})
435 def inspect_getset_descriptor(self, node: ObjectNode) -> None:
436 """Inspect a get/set descriptor.
438 Parameters:
439 node: The node to inspect.
440 """
441 self.handle_function(node, {"property"})
443 def handle_function(self, node: ObjectNode, labels: set | None = None) -> None:
444 """Handle a function.
446 Parameters:
447 node: The node to inspect.
448 labels: Labels to add to the data object.
449 """
450 self.extensions.call("on_node", node=node, agent=self)
451 self.extensions.call("on_function_node", node=node, agent=self)
453 try:
454 signature = getsignature(node.obj)
455 except Exception: # noqa: BLE001
456 # So many exceptions can be raised here:
457 # AttributeError, NameError, RuntimeError, ValueError, TokenError, TypeError...
458 parameters = None
459 returns = None
460 else:
461 parameters = Parameters(
462 *[
463 _convert_parameter(parameter, parent=self.current, member=node.name)
464 for parameter in signature.parameters.values()
465 ],
466 )
467 return_annotation = signature.return_annotation
468 returns = (
469 None
470 if return_annotation is _empty
471 else _convert_object_to_annotation(return_annotation, parent=self.current, member=node.name)
472 )
474 lineno, endlineno = self._get_linenos(node)
476 obj: Attribute | Function
477 labels = labels or set()
478 if "property" in labels:
479 obj = Attribute(
480 name=node.name,
481 value=None,
482 annotation=returns,
483 docstring=self._get_docstring(node),
484 lineno=lineno,
485 endlineno=endlineno,
486 )
487 else:
488 obj = Function(
489 name=node.name,
490 parameters=parameters,
491 returns=returns,
492 type_parameters=TypeParameters(
493 *_convert_type_parameters(node.obj, parent=self.current, member=node.name),
494 ),
495 docstring=self._get_docstring(node),
496 lineno=lineno,
497 endlineno=endlineno,
498 )
499 obj.labels |= labels
500 self.current.set_member(node.name, obj)
501 self.extensions.call("on_instance", node=node, obj=obj, agent=self)
502 if obj.is_attribute:
503 self.extensions.call("on_attribute_instance", node=node, attr=obj, agent=self)
504 else:
505 self.extensions.call("on_function_instance", node=node, func=obj, agent=self)
507 def inspect_type_alias(self, node: ObjectNode) -> None:
508 """Inspect a type alias.
510 Parameters:
511 node: The node to inspect.
512 """
513 self.extensions.call("on_node", node=node, agent=self)
514 self.extensions.call("on_type_alias_node", node=node, agent=self)
516 lineno, endlineno = self._get_linenos(node)
518 type_alias = TypeAlias(
519 name=node.name,
520 value=_convert_type_to_annotation(node.obj.__value__, parent=self.current, member=node.name),
521 lineno=lineno,
522 endlineno=endlineno,
523 type_parameters=TypeParameters(*_convert_type_parameters(node.obj, parent=self.current, member=node.name)),
524 docstring=self._get_docstring(node),
525 parent=self.current,
526 )
527 self.current.set_member(node.name, type_alias)
528 self.extensions.call("on_instance", node=node, obj=type_alias, agent=self)
529 self.extensions.call("on_type_alias_instance", node=node, type_alias=type_alias, agent=self)
531 def inspect_attribute(self, node: ObjectNode) -> None:
532 """Inspect an attribute.
534 Parameters:
535 node: The node to inspect.
536 """
537 self.handle_attribute(node)
539 def handle_attribute(self, node: ObjectNode, annotation: str | Expr | None = None) -> None:
540 """Handle an attribute.
542 Parameters:
543 node: The node to inspect.
544 annotation: A potential annotation.
545 """
546 self.extensions.call("on_node", node=node, agent=self)
547 self.extensions.call("on_attribute_node", node=node, agent=self)
549 # TODO: To improve.
550 parent = self.current
551 labels: set[str] = set()
553 if parent.kind is Kind.MODULE:
554 labels.add("module-attribute")
555 elif parent.kind is Kind.CLASS: 555 ↛ 557line 555 didn't jump to line 557 because the condition on line 555 was always true
556 labels.add("class-attribute")
557 elif parent.kind is Kind.FUNCTION:
558 if parent.name != "__init__":
559 return
560 parent = parent.parent # type: ignore[assignment]
561 labels.add("instance-attribute")
563 try:
564 value = repr(node.obj)
565 except Exception: # noqa: BLE001
566 value = None
567 try:
568 docstring = self._get_docstring(node)
569 except Exception: # noqa: BLE001
570 docstring = None
572 attribute = Attribute(
573 name=node.name,
574 value=value,
575 annotation=annotation,
576 docstring=docstring,
577 )
578 attribute.labels |= labels
579 parent.set_member(node.name, attribute)
581 if node.name == "__all__":
582 parent.exports = list(node.obj)
583 self.extensions.call("on_instance", node=node, obj=attribute, agent=self)
584 self.extensions.call("on_attribute_instance", node=node, attr=attribute, agent=self)
587_parameter_kind_map = {
588 SignatureParameter.POSITIONAL_ONLY: ParameterKind.positional_only,
589 SignatureParameter.POSITIONAL_OR_KEYWORD: ParameterKind.positional_or_keyword,
590 SignatureParameter.VAR_POSITIONAL: ParameterKind.var_positional,
591 SignatureParameter.KEYWORD_ONLY: ParameterKind.keyword_only,
592 SignatureParameter.VAR_KEYWORD: ParameterKind.var_keyword,
593}
596def _convert_parameter(
597 parameter: SignatureParameter,
598 *,
599 parent: Module | Class,
600 member: str | None = None,
601) -> Parameter:
602 name = parameter.name
603 annotation = (
604 None
605 if parameter.annotation is _empty
606 else _convert_object_to_annotation(parameter.annotation, parent=parent, member=member)
607 )
608 kind = _parameter_kind_map[parameter.kind]
609 if parameter.default is _empty:
610 default = None
611 elif hasattr(parameter.default, "__name__"):
612 # Avoid `repr` containing chevrons and memory addresses.
613 default = parameter.default.__name__
614 else:
615 default = repr(parameter.default)
616 return Parameter(name, annotation=annotation, kind=kind, default=default)
619def _convert_object_to_annotation(obj: Any, *, parent: Module | Class, member: str | None = None) -> str | Expr | None:
620 # Even when *we* import future annotations,
621 # the object from which we get a signature
622 # can come from modules which did *not* import them,
623 # so `inspect.signature` returns actual Python objects
624 # that we must deal with.
625 if isinstance(obj, str):
626 annotation = obj
627 else:
628 # Always give precedence to the object's representation...
629 obj_repr = repr(obj)
630 if hasattr(obj, "__name__"): # noqa: SIM108
631 # ...unless it contains chevrons (which likely means it's a class),
632 # in which case we use the object's name.
633 annotation = obj.__name__ if "<" in obj_repr else obj_repr
634 else:
635 annotation = obj_repr
636 try:
637 annotation_node = compile(annotation, mode="eval", filename="<>", flags=ast.PyCF_ONLY_AST, optimize=2)
638 except SyntaxError:
639 return obj
640 return safe_get_annotation(annotation_node.body, parent, member=member) # type: ignore[attr-defined]
643_type_parameter_kind_map = {
644 getattr(module, attr): value
645 for attr, value in {
646 "TypeVar": TypeParameterKind.type_var,
647 "TypeVarTuple": TypeParameterKind.type_var_tuple,
648 "ParamSpec": TypeParameterKind.param_spec,
649 }.items()
650 for module in _TYPING_MODULES
651 if hasattr(module, attr)
652}
655def _convert_type_parameters(
656 obj: Any,
657 *,
658 parent: Module | Class,
659 member: str | None = None,
660) -> list[TypeParameter]:
661 obj = unwrap(obj)
663 if not hasattr(obj, "__type_params__"):
664 return []
666 type_parameters = []
667 for type_parameter in obj.__type_params__:
668 bound = getattr(type_parameter, "__bound__", None)
669 if bound is not None:
670 bound = _convert_type_to_annotation(bound, parent=parent, member=member)
671 constraints: list[str | Expr] = [
672 _convert_type_to_annotation(constraint, parent=parent, member=member) # type: ignore[misc]
673 for constraint in getattr(type_parameter, "__constraints__", ())
674 ]
676 if getattr(type_parameter, "has_default", lambda: False)():
677 default = _convert_type_to_annotation(
678 type_parameter.__default__,
679 parent=parent,
680 member=member,
681 )
682 else:
683 default = None
685 type_parameters.append(
686 TypeParameter(
687 type_parameter.__name__,
688 kind=_type_parameter_kind_map[type(type_parameter)],
689 bound=bound,
690 constraints=constraints or None,
691 default=default,
692 ),
693 )
695 return type_parameters
698def _convert_type_to_annotation(obj: Any, *, parent: Module | Class, member: str | None = None) -> str | Expr | None:
699 origin = typing.get_origin(obj)
701 if origin is None:
702 return _convert_object_to_annotation(obj, parent=parent, member=member)
704 args: Sequence[str | Expr | None] = [
705 _convert_type_to_annotation(arg, parent=parent, member=member) for arg in typing.get_args(obj)
706 ]
708 # YORE: EOL 3.9: Replace block with lines 2-3.
709 if sys.version_info >= (3, 10): 709 ↛ 713line 709 didn't jump to line 713 because the condition on line 709 was always true
710 if origin is types.UnionType: 710 ↛ 711line 710 didn't jump to line 711 because the condition on line 710 was never true
711 return functools.reduce(lambda left, right: ExprBinOp(left, "|", right), args) # type: ignore[arg-type]
713 origin = _convert_type_to_annotation(origin, parent=parent, member=member)
714 if origin is None: 714 ↛ 715line 714 didn't jump to line 715 because the condition on line 714 was never true
715 return None
717 return ExprSubscript(origin, ExprTuple(args, implicit=True)) # type: ignore[arg-type]