Coverage for packages / griffelib / src / griffe / _internal / agents / inspector.py: 89.97%
263 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-02-11 11:48 +0100
1# This module contains our dynamic analysis agent,
2# capable of inspecting modules and objects in memory, at runtime.
4from __future__ import annotations
6import ast
7import functools
8import types
9import typing
10from inspect import Parameter as SignatureParameter
11from inspect import Signature, cleandoc, getsourcelines, unwrap
12from inspect import signature as getsignature
13from typing import TYPE_CHECKING, Any
15from griffe._internal.agents.nodes.runtime import ObjectNode
16from griffe._internal.collections import LinesCollection, ModulesCollection
17from griffe._internal.enumerations import Kind, ParameterKind, TypeParameterKind
18from griffe._internal.expressions import Expr, ExprBinOp, ExprSubscript, ExprTuple, safe_get_annotation
19from griffe._internal.extensions.base import Extensions, load_extensions
20from griffe._internal.importer import dynamic_import
21from griffe._internal.logger import logger
22from griffe._internal.models import (
23 Alias,
24 Attribute,
25 Class,
26 Docstring,
27 Function,
28 Module,
29 Parameter,
30 Parameters,
31 TypeAlias,
32 TypeParameter,
33 TypeParameters,
34)
36if TYPE_CHECKING:
37 from collections.abc import Sequence
38 from pathlib import Path
40 from griffe._internal.docstrings.parsers import DocstringOptions, DocstringStyle
41 from griffe._internal.enumerations import Parser
43_TYPING_MODULES: tuple[types.ModuleType, ...]
44try:
45 import typing_extensions
46except ImportError:
47 _TYPING_MODULES = (typing,)
48else:
49 _TYPING_MODULES = (typing, typing_extensions)
51_empty = Signature.empty
54def inspect(
55 module_name: str,
56 *,
57 filepath: Path | None = None,
58 import_paths: Sequence[str | Path] | None = None,
59 extensions: Extensions | None = None,
60 parent: Module | None = None,
61 docstring_parser: DocstringStyle | Parser | None = None,
62 docstring_options: DocstringOptions | None = None,
63 lines_collection: LinesCollection | None = None,
64 modules_collection: ModulesCollection | None = None,
65) -> Module:
66 """Inspect a module.
68 Sometimes we cannot get the source code of a module or an object,
69 typically built-in modules like `itertools`.
70 The only way to know what they are made of is to actually import them and inspect their contents.
72 Sometimes, even if the source code is available,
73 loading the object is desired because it was created or modified dynamically,
74 and our static agent is not powerful enough to infer all these dynamic modifications.
75 In this case, we load the module using introspection.
77 Griffe therefore provides this function for dynamic analysis.
78 It uses a [`NodeVisitor`][ast.NodeVisitor]-like class, the [`Inspector`][griffe.Inspector],
79 to inspect the module with [`inspect.getmembers()`][inspect.getmembers].
81 The inspection agent works similarly to the regular [`Visitor`][griffe.Visitor] agent,
82 in that it maintains a state with the current object being handled, and recursively handle its members.
84 Important:
85 This function is generally not used directly.
86 In most cases, users can rely on the [`GriffeLoader`][griffe.GriffeLoader]
87 and its accompanying [`load`][griffe.load] shortcut and their respective options
88 to load modules using dynamic analysis.
90 Parameters:
91 module_name: The module name (as when importing [from] it).
92 filepath: The module file path.
93 import_paths: Paths to import the module from.
94 extensions: The extensions to use when inspecting the module.
95 parent: The optional parent of this module.
96 docstring_parser: The docstring parser to use. By default, no parsing is done.
97 docstring_options: Docstring parsing options.
98 lines_collection: A collection of source code lines.
99 modules_collection: A collection of modules.
101 Returns:
102 The module, with its members populated.
103 """
104 return Inspector(
105 module_name,
106 filepath,
107 extensions or load_extensions(),
108 parent,
109 docstring_parser=docstring_parser,
110 docstring_options=docstring_options,
111 lines_collection=lines_collection,
112 modules_collection=modules_collection,
113 ).get_module(import_paths)
116class Inspector:
117 """This class is used to instantiate an inspector.
119 Inspectors iterate on objects members to extract data from them.
120 """
122 def __init__(
123 self,
124 module_name: str,
125 filepath: Path | None,
126 extensions: Extensions,
127 parent: Module | None = None,
128 docstring_parser: DocstringStyle | Parser | None = None,
129 docstring_options: DocstringOptions | None = None,
130 lines_collection: LinesCollection | None = None,
131 modules_collection: ModulesCollection | None = None,
132 ) -> None:
133 """Initialize the inspector.
135 Parameters:
136 module_name: The module name.
137 filepath: The optional filepath.
138 extensions: Extensions to use when inspecting.
139 parent: The module parent.
140 docstring_parser: The docstring parser to use.
141 docstring_options: Docstring parsing options.
142 lines_collection: A collection of source code lines.
143 modules_collection: A collection of modules.
144 """
145 super().__init__()
147 self.module_name: str = module_name
148 """The module name."""
150 self.filepath: Path | None = filepath
151 """The module file path."""
153 self.extensions: Extensions = extensions
154 """The extensions to use when inspecting."""
156 self.parent: Module | None = parent
157 """An optional parent for the final module object."""
159 self.current: Module | Class = None
160 """The current object being inspected."""
162 self.docstring_parser: DocstringStyle | Parser | None = docstring_parser
163 """The docstring parser to use."""
165 self.docstring_options: DocstringOptions = docstring_options or {}
166 """The docstring parsing options."""
168 self.lines_collection: LinesCollection = lines_collection or LinesCollection()
169 """A collection of source code lines."""
171 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection()
172 """A collection of modules."""
174 def _get_docstring(self, node: ObjectNode) -> Docstring | None:
175 try:
176 # Access `__doc__` directly to avoid taking the `__doc__` attribute from a parent class.
177 value = getattr(node.obj, "__doc__", None)
178 except Exception: # noqa: BLE001
179 return None
180 if value is None:
181 return None
182 try:
183 # We avoid `inspect.getdoc` to avoid getting
184 # the `__doc__` attribute from a parent class,
185 # but we still want to clean the doc.
186 cleaned = cleandoc(value)
187 except AttributeError:
188 # Triggered on method descriptors.
189 return None
190 return Docstring(
191 cleaned,
192 parser=self.docstring_parser,
193 parser_options=self.docstring_options,
194 )
196 def _get_linenos(self, node: ObjectNode) -> tuple[int, int] | tuple[None, None]:
197 # Line numbers won't be useful if we don't have the source code.
198 if not self.filepath or self.filepath not in self.lines_collection:
199 return None, None
200 try:
201 lines, lineno = getsourcelines(node.obj)
202 except (OSError, TypeError):
203 return None, None
204 return lineno, lineno + "".join(lines).rstrip().count("\n")
206 def get_module(self, import_paths: Sequence[str | Path] | None = None) -> Module:
207 """Build and return the object representing the module attached to this inspector.
209 This method triggers a complete inspection of the module members.
211 Parameters:
212 import_paths: Paths replacing `sys.path` to import the module.
214 Returns:
215 A module instance.
216 """
217 import_path = self.module_name
218 if self.parent is not None:
219 import_path = f"{self.parent.path}.{import_path}"
221 # Make sure `import_paths` is a list, in case we want to `insert` into it.
222 import_paths = list(import_paths or ())
224 # If the thing we want to import has a filepath,
225 # we make sure to insert the right parent directory
226 # at the front of our list of import paths.
227 # We do this by counting the number of dots `.` in the import path,
228 # corresponding to slashes `/` in the filesystem,
229 # and go up in the file tree the same number of times.
230 if self.filepath:
231 parent_path = self.filepath.parent
232 for _ in range(import_path.count(".")):
233 parent_path = parent_path.parent
234 # Climb up one more time for `__init__` modules.
235 if self.filepath.stem == "__init__":
236 parent_path = parent_path.parent
237 if parent_path not in import_paths:
238 import_paths.insert(0, parent_path)
240 value = dynamic_import(import_path, import_paths)
242 # We successfully imported the given object,
243 # and we now create the object tree with all the necessary nodes,
244 # from the root of the package to this leaf object.
245 parent_node = None
246 if self.parent is not None:
247 for part in self.parent.path.split("."):
248 parent_node = ObjectNode(None, name=part, parent=parent_node)
249 module_node = ObjectNode(value, self.module_name, parent=parent_node)
251 self.inspect(module_node)
252 return self.current.module
254 def inspect(self, node: ObjectNode) -> None:
255 """Extend the base inspection with extensions.
257 Parameters:
258 node: The node to inspect.
259 """
260 getattr(self, f"inspect_{node.kind}", self.generic_inspect)(node)
262 def generic_inspect(self, node: ObjectNode) -> None:
263 """Extend the base generic inspection with extensions.
265 Parameters:
266 node: The node to inspect.
267 """
268 for child in node.children:
269 if target_path := child.alias_target_path:
270 # If the child is an actual submodule of the current module,
271 # and has no `__file__` set, we won't find it on the disk so we must inspect it now.
272 # For that we instantiate a new inspector and use it to inspect the submodule,
273 # then assign the submodule as member of the current module.
274 # If the submodule has a `__file__` set, the loader should find it on the disk,
275 # so we skip it here (no member, no alias, just skip it).
276 if child.is_module and target_path == f"{self.current.path}.{child.name}":
277 if not hasattr(child.obj, "__file__"): 277 ↛ 278line 277 didn't jump to line 278 because the condition on line 277 was never true
278 logger.debug("Module %s is not discoverable on disk, inspecting right now", target_path)
279 inspector = Inspector(
280 child.name,
281 filepath=None,
282 parent=self.current.module,
283 extensions=self.extensions,
284 docstring_parser=self.docstring_parser,
285 docstring_options=self.docstring_options,
286 lines_collection=self.lines_collection,
287 modules_collection=self.modules_collection,
288 )
289 inspector.inspect_module(child)
290 self.current.set_member(child.name, inspector.current.module)
291 # Otherwise, alias the object.
292 else:
293 alias = Alias(child.name, target_path, analysis="dynamic")
294 self.current.set_member(child.name, alias)
295 self.extensions.call("on_alias_instance", alias=alias, node=node, agent=self)
296 else:
297 self.inspect(child)
299 def inspect_module(self, node: ObjectNode) -> None:
300 """Inspect a module.
302 Parameters:
303 node: The node to inspect.
304 """
305 self.extensions.call("on_node", node=node, agent=self)
306 self.extensions.call("on_module_node", node=node, agent=self)
307 self.current = module = Module(
308 name=self.module_name,
309 filepath=self.filepath,
310 parent=self.parent,
311 docstring=self._get_docstring(node),
312 lines_collection=self.lines_collection,
313 modules_collection=self.modules_collection,
314 analysis="dynamic",
315 )
316 self.extensions.call("on_instance", node=node, obj=module, agent=self)
317 self.extensions.call("on_module_instance", node=node, mod=module, agent=self)
318 self.generic_inspect(node)
319 self.extensions.call("on_members", node=node, obj=module, agent=self)
320 self.extensions.call("on_module_members", node=node, mod=module, agent=self)
322 def inspect_class(self, node: ObjectNode) -> None:
323 """Inspect a class.
325 Parameters:
326 node: The node to inspect.
327 """
328 self.extensions.call("on_node", node=node, agent=self)
329 self.extensions.call("on_class_node", node=node, agent=self)
331 bases = []
332 for base in node.obj.__bases__:
333 if base is object:
334 continue
335 bases.append(f"{base.__module__}.{base.__qualname__}")
337 lineno, endlineno = self._get_linenos(node)
338 class_ = Class(
339 name=node.name,
340 docstring=self._get_docstring(node),
341 bases=bases,
342 type_parameters=TypeParameters(*_convert_type_parameters(node.obj, parent=self.current, member=node.name)),
343 lineno=lineno,
344 endlineno=endlineno,
345 analysis="dynamic",
346 )
347 self.current.set_member(node.name, class_)
348 self.current = class_
349 self.extensions.call("on_instance", node=node, obj=class_, agent=self)
350 self.extensions.call("on_class_instance", node=node, cls=class_, agent=self)
351 self.generic_inspect(node)
352 self.extensions.call("on_members", node=node, obj=class_, agent=self)
353 self.extensions.call("on_class_members", node=node, cls=class_, agent=self)
354 self.current = self.current.parent # ty:ignore[invalid-assignment]
356 def inspect_staticmethod(self, node: ObjectNode) -> None:
357 """Inspect a static method.
359 Parameters:
360 node: The node to inspect.
361 """
362 self.handle_function(node, {"staticmethod"})
364 def inspect_classmethod(self, node: ObjectNode) -> None:
365 """Inspect a class method.
367 Parameters:
368 node: The node to inspect.
369 """
370 self.handle_function(node, {"classmethod"})
372 def inspect_method_descriptor(self, node: ObjectNode) -> None:
373 """Inspect a method descriptor.
375 Parameters:
376 node: The node to inspect.
377 """
378 self.handle_function(node, {"method descriptor"})
380 def inspect_builtin_method(self, node: ObjectNode) -> None:
381 """Inspect a builtin method.
383 Parameters:
384 node: The node to inspect.
385 """
386 self.handle_function(node, {"builtin"})
388 def inspect_method(self, node: ObjectNode) -> None:
389 """Inspect a method.
391 Parameters:
392 node: The node to inspect.
393 """
394 self.handle_function(node)
396 def inspect_coroutine(self, node: ObjectNode) -> None:
397 """Inspect a coroutine.
399 Parameters:
400 node: The node to inspect.
401 """
402 self.handle_function(node, {"async"})
404 def inspect_builtin_function(self, node: ObjectNode) -> None:
405 """Inspect a builtin function.
407 Parameters:
408 node: The node to inspect.
409 """
410 self.handle_function(node, {"builtin"})
412 def inspect_function(self, node: ObjectNode) -> None:
413 """Inspect a function.
415 Parameters:
416 node: The node to inspect.
417 """
418 self.handle_function(node)
420 def inspect_cached_property(self, node: ObjectNode) -> None:
421 """Inspect a cached property.
423 Parameters:
424 node: The node to inspect.
425 """
426 self.handle_function(node, {"cached", "property"})
428 def inspect_property(self, node: ObjectNode) -> None:
429 """Inspect a property.
431 Parameters:
432 node: The node to inspect.
433 """
434 self.handle_function(node, {"property"})
436 def inspect_getset_descriptor(self, node: ObjectNode) -> None:
437 """Inspect a get/set descriptor.
439 Parameters:
440 node: The node to inspect.
441 """
442 self.handle_function(node, {"property"})
444 def handle_function(self, node: ObjectNode, labels: set | None = None) -> None:
445 """Handle a function.
447 Parameters:
448 node: The node to inspect.
449 labels: Labels to add to the data object.
450 """
451 self.extensions.call("on_node", node=node, agent=self)
452 self.extensions.call("on_function_node", node=node, agent=self)
454 try:
455 signature = getsignature(node.obj)
456 except Exception: # noqa: BLE001
457 # So many exceptions can be raised here:
458 # AttributeError, NameError, RuntimeError, ValueError, TokenError, TypeError...
459 parameters = None
460 returns = None
461 else:
462 parameters = Parameters(
463 *[
464 _convert_parameter(parameter, parent=self.current, member=node.name)
465 for parameter in signature.parameters.values()
466 ],
467 )
468 return_annotation = signature.return_annotation
469 returns = (
470 None
471 if return_annotation is _empty
472 else _convert_object_to_annotation(return_annotation, parent=self.current, member=node.name)
473 )
475 lineno, endlineno = self._get_linenos(node)
477 obj: Attribute | Function
478 labels = labels or set()
479 if "property" in labels:
480 obj = Attribute(
481 name=node.name,
482 value=None,
483 annotation=returns,
484 docstring=self._get_docstring(node),
485 lineno=lineno,
486 endlineno=endlineno,
487 analysis="dynamic",
488 )
489 else:
490 obj = Function(
491 name=node.name,
492 parameters=parameters,
493 returns=returns,
494 type_parameters=TypeParameters(
495 *_convert_type_parameters(node.obj, parent=self.current, member=node.name),
496 ),
497 docstring=self._get_docstring(node),
498 lineno=lineno,
499 endlineno=endlineno,
500 analysis="dynamic",
501 )
502 obj.labels |= labels
503 self.current.set_member(node.name, obj)
504 self.extensions.call("on_instance", node=node, obj=obj, agent=self)
505 if obj.is_attribute:
506 self.extensions.call("on_attribute_instance", node=node, attr=obj, agent=self)
507 else:
508 self.extensions.call("on_function_instance", node=node, func=obj, agent=self)
510 def inspect_type_alias(self, node: ObjectNode) -> None:
511 """Inspect a type alias.
513 Parameters:
514 node: The node to inspect.
515 """
516 self.extensions.call("on_node", node=node, agent=self)
517 self.extensions.call("on_type_alias_node", node=node, agent=self)
519 lineno, endlineno = self._get_linenos(node)
521 type_alias = TypeAlias(
522 name=node.name,
523 value=_convert_type_to_annotation(node.obj.__value__, parent=self.current, member=node.name),
524 lineno=lineno,
525 endlineno=endlineno,
526 type_parameters=TypeParameters(*_convert_type_parameters(node.obj, parent=self.current, member=node.name)),
527 docstring=self._get_docstring(node),
528 parent=self.current,
529 analysis="dynamic",
530 )
531 self.current.set_member(node.name, type_alias)
532 self.extensions.call("on_instance", node=node, obj=type_alias, agent=self)
533 self.extensions.call("on_type_alias_instance", node=node, type_alias=type_alias, agent=self)
535 def inspect_attribute(self, node: ObjectNode) -> None:
536 """Inspect an attribute.
538 Parameters:
539 node: The node to inspect.
540 """
541 self.handle_attribute(node)
543 def handle_attribute(self, node: ObjectNode, annotation: str | Expr | None = None) -> None:
544 """Handle an attribute.
546 Parameters:
547 node: The node to inspect.
548 annotation: A potential annotation.
549 """
550 self.extensions.call("on_node", node=node, agent=self)
551 self.extensions.call("on_attribute_node", node=node, agent=self)
553 # TODO: To improve.
554 parent = self.current
555 labels: set[str] = set()
557 if parent.kind is Kind.MODULE:
558 labels.add("module-attribute")
559 elif parent.kind is Kind.CLASS: 559 ↛ 561line 559 didn't jump to line 561 because the condition on line 559 was always true
560 labels.add("class-attribute")
561 elif parent.kind is Kind.FUNCTION:
562 if parent.name != "__init__":
563 return
564 parent = parent.parent
565 labels.add("instance-attribute")
567 try:
568 value = repr(node.obj)
569 except Exception: # noqa: BLE001
570 value = None
571 try:
572 docstring = self._get_docstring(node)
573 except Exception: # noqa: BLE001
574 docstring = None
576 attribute = Attribute(
577 name=node.name,
578 value=value,
579 annotation=annotation,
580 docstring=docstring,
581 analysis="dynamic",
582 )
583 attribute.labels |= labels
584 parent.set_member(node.name, attribute) # ty:ignore[possibly-missing-attribute]
586 if node.name == "__all__":
587 parent.exports = list(node.obj) # ty:ignore[invalid-assignment]
588 self.extensions.call("on_instance", node=node, obj=attribute, agent=self)
589 self.extensions.call("on_attribute_instance", node=node, attr=attribute, agent=self)
592_parameter_kind_map = {
593 SignatureParameter.POSITIONAL_ONLY: ParameterKind.positional_only,
594 SignatureParameter.POSITIONAL_OR_KEYWORD: ParameterKind.positional_or_keyword,
595 SignatureParameter.VAR_POSITIONAL: ParameterKind.var_positional,
596 SignatureParameter.KEYWORD_ONLY: ParameterKind.keyword_only,
597 SignatureParameter.VAR_KEYWORD: ParameterKind.var_keyword,
598}
601def _convert_parameter(
602 parameter: SignatureParameter,
603 *,
604 parent: Module | Class,
605 member: str | None = None,
606) -> Parameter:
607 name = parameter.name
608 annotation = (
609 None
610 if parameter.annotation is _empty
611 else _convert_object_to_annotation(parameter.annotation, parent=parent, member=member)
612 )
613 kind = _parameter_kind_map[parameter.kind]
614 if parameter.default is _empty:
615 default = None
616 elif hasattr(parameter.default, "__name__"):
617 # Avoid `repr` containing chevrons and memory addresses.
618 default = parameter.default.__name__
619 else:
620 default = repr(parameter.default)
621 return Parameter(name, annotation=annotation, kind=kind, default=default)
624def _convert_object_to_annotation(obj: Any, *, parent: Module | Class, member: str | None = None) -> str | Expr | None:
625 # Even when *we* import future annotations,
626 # the object from which we get a signature
627 # can come from modules which did *not* import them,
628 # so `inspect.signature` returns actual Python objects
629 # that we must deal with.
630 if isinstance(obj, str): 630 ↛ 631line 630 didn't jump to line 631 because the condition on line 630 was never true
631 annotation = obj
632 else:
633 # Always give precedence to the object's representation...
634 obj_repr = repr(obj)
635 if hasattr(obj, "__name__"): # noqa: SIM108
636 # ...unless it contains chevrons (which likely means it's a class),
637 # in which case we use the object's name.
638 annotation = obj.__name__ if "<" in obj_repr else obj_repr
639 else:
640 annotation = obj_repr
641 try:
642 annotation_node = compile(annotation, mode="eval", filename="<>", flags=ast.PyCF_ONLY_AST, optimize=2)
643 except SyntaxError:
644 return obj
645 return safe_get_annotation(annotation_node.body, parent, member=member) # ty:ignore[unresolved-attribute]
648_type_parameter_kind_map = {
649 getattr(module, attr): value
650 for attr, value in {
651 "TypeVar": TypeParameterKind.type_var,
652 "TypeVarTuple": TypeParameterKind.type_var_tuple,
653 "ParamSpec": TypeParameterKind.param_spec,
654 }.items()
655 for module in _TYPING_MODULES
656 if hasattr(module, attr)
657}
660def _convert_type_parameters(
661 obj: Any,
662 *,
663 parent: Module | Class,
664 member: str | None = None,
665) -> list[TypeParameter]:
666 obj = unwrap(obj)
668 if not hasattr(obj, "__type_params__"):
669 return []
671 type_parameters = []
672 for type_parameter in obj.__type_params__:
673 bound = getattr(type_parameter, "__bound__", None)
674 if bound is not None:
675 bound = _convert_type_to_annotation(bound, parent=parent, member=member)
676 constraints: list[str | Expr] = [
677 _convert_type_to_annotation(constraint, parent=parent, member=member)
678 for constraint in getattr(type_parameter, "__constraints__", ())
679 ] # ty:ignore[invalid-assignment]
681 if getattr(type_parameter, "has_default", lambda: False)():
682 default = _convert_type_to_annotation(
683 type_parameter.__default__,
684 parent=parent,
685 member=member,
686 )
687 else:
688 default = None
690 type_parameters.append(
691 TypeParameter(
692 type_parameter.__name__,
693 kind=_type_parameter_kind_map[type(type_parameter)],
694 bound=bound,
695 constraints=constraints or None,
696 default=default,
697 ),
698 )
700 return type_parameters
703def _convert_type_to_annotation(obj: Any, *, parent: Module | Class, member: str | None = None) -> str | Expr | None:
704 origin = typing.get_origin(obj)
706 if origin is None:
707 return _convert_object_to_annotation(obj, parent=parent, member=member)
709 args: Sequence[str | Expr | None] = [
710 _convert_type_to_annotation(arg, parent=parent, member=member) for arg in typing.get_args(obj)
711 ]
713 if origin is types.UnionType: 713 ↛ 714line 713 didn't jump to line 714 because the condition on line 713 was never true
714 return functools.reduce(lambda left, right: ExprBinOp(left, "|", right), args)
716 origin = _convert_type_to_annotation(origin, parent=parent, member=member)
717 if origin is None: 717 ↛ 718line 717 didn't jump to line 718 because the condition on line 717 was never true
718 return None
720 return ExprSubscript(origin, ExprTuple(args, implicit=True)) # ty:ignore[invalid-argument-type]