Coverage for src/_griffe/agents/inspector.py: 89.89%

1# This module contains our dynamic analysis agent,

2# capable of inspecting modules and objects in memory, at runtime.

4from __future__ import annotations

6import ast

7from inspect import Parameter as SignatureParameter

8from inspect import Signature, cleandoc, getsourcelines

9from inspect import signature as getsignature

10from typing import TYPE_CHECKING, Any

12from _griffe.agents.nodes.runtime import ObjectNode

13from _griffe.collections import LinesCollection, ModulesCollection

14from _griffe.enumerations import Kind, ParameterKind

15from _griffe.expressions import safe_get_annotation

16from _griffe.extensions.base import Extensions, load_extensions

17from _griffe.importer import dynamic_import

18from _griffe.logger import logger

19from _griffe.models import Alias, Attribute, Class, Docstring, Function, Module, Parameter, Parameters

21if TYPE_CHECKING:

22 from collections.abc import Sequence

23 from pathlib import Path

25 from _griffe.enumerations import Parser

26 from _griffe.expressions import Expr

29_empty = Signature.empty

32def inspect(

33 module_name: str,

34 *,

35 filepath: Path | None = None,

36 import_paths: Sequence[str | Path] | None = None,

37 extensions: Extensions | None = None,

38 parent: Module | None = None,

39 docstring_parser: Parser | None = None,

40 docstring_options: dict[str, Any] | None = None,

41 lines_collection: LinesCollection | None = None,

42 modules_collection: ModulesCollection | None = None,

43) -> Module:

44 """Inspect a module.

46 Sometimes we cannot get the source code of a module or an object,

47 typically built-in modules like `itertools`.

48 The only way to know what they are made of is to actually import them and inspect their contents.

50 Sometimes, even if the source code is available,

51 loading the object is desired because it was created or modified dynamically,

52 and our static agent is not powerful enough to infer all these dynamic modifications.

53 In this case, we load the module using introspection.

55 Griffe therefore provides this function for dynamic analysis.

56 It uses a [`NodeVisitor`][ast.NodeVisitor]-like class, the [`Inspector`][griffe.Inspector],

57 to inspect the module with [`inspect.getmembers()`][inspect.getmembers].

59 The inspection agent works similarly to the regular [`Visitor`][griffe.Visitor] agent,

60 in that it maintains a state with the current object being handled, and recursively handle its members.

62 Important:

63 This function is generally not used directly.

64 In most cases, users can rely on the [`GriffeLoader`][griffe.GriffeLoader]

65 and its accompanying [`load`][griffe.load] shortcut and their respective options

66 to load modules using dynamic analysis.

68 Parameters:

69 module_name: The module name (as when importing [from] it).

70 filepath: The module file path.

71 import_paths: Paths to import the module from.

72 extensions: The extensions to use when inspecting the module.

73 parent: The optional parent of this module.

74 docstring_parser: The docstring parser to use. By default, no parsing is done.

75 docstring_options: Additional docstring parsing options.

76 lines_collection: A collection of source code lines.

77 modules_collection: A collection of modules.

79 Returns:

80 The module, with its members populated.

81 """

82 return Inspector(

83 module_name,

84 filepath,

85 extensions or load_extensions(),

86 parent,

87 docstring_parser=docstring_parser,

88 docstring_options=docstring_options,

89 lines_collection=lines_collection,

90 modules_collection=modules_collection,

91 ).get_module(import_paths)

94class Inspector:

95 """This class is used to instantiate an inspector.

97 Inspectors iterate on objects members to extract data from them.

98 """

100 def __init__(

101 self,

102 module_name: str,

103 filepath: Path | None,

104 extensions: Extensions,

105 parent: Module | None = None,

106 docstring_parser: Parser | None = None,

107 docstring_options: dict[str, Any] | None = None,

108 lines_collection: LinesCollection | None = None,

109 modules_collection: ModulesCollection | None = None,

110 ) -> None:

111 """Initialize the inspector.

112

113 Parameters:

114 module_name: The module name.

115 filepath: The optional filepath.

116 extensions: Extensions to use when inspecting.

117 parent: The module parent.

118 docstring_parser: The docstring parser to use.

119 docstring_options: The docstring parsing options.

120 lines_collection: A collection of source code lines.

121 modules_collection: A collection of modules.

122 """

123 super().__init__()

124

125 self.module_name: str = module_name

126 """The module name."""

127

128 self.filepath: Path | None = filepath

129 """The module file path."""

130

131 self.extensions: Extensions = extensions

132 """The extensions to use when inspecting."""

133

134 self.parent: Module | None = parent

135 """An optional parent for the final module object."""

136

137 self.current: Module | Class = None # type: ignore[assignment]

138 """The current object being inspected."""

139

140 self.docstring_parser: Parser | None = docstring_parser

141 """The docstring parser to use."""

142

143 self.docstring_options: dict[str, Any] = docstring_options or {}

144 """The docstring parsing options."""

145

146 self.lines_collection: LinesCollection = lines_collection or LinesCollection()

147 """A collection of source code lines."""

148

149 self.modules_collection: ModulesCollection = modules_collection or ModulesCollection()

150 """A collection of modules."""

151

152 def _get_docstring(self, node: ObjectNode) -> Docstring | None:

153 try:

154 # Access `__doc__` directly to avoid taking the `__doc__` attribute from a parent class.

155 value = getattr(node.obj, "__doc__", None)

156 except Exception: # noqa: BLE001 # getattr can trigger exceptions

157 return None

158 if value is None:

159 return None

160 try:

161 # We avoid `inspect.getdoc` to avoid getting

162 # the `__doc__` attribute from a parent class,

163 # but we still want to clean the doc.

164 cleaned = cleandoc(value)

165 except AttributeError:

166 # Triggered on method descriptors.

167 return None

168 return Docstring(

169 cleaned,

170 parser=self.docstring_parser,

171 parser_options=self.docstring_options,

172 )

173

174 def _get_linenos(self, node: ObjectNode) -> tuple[int, int] | tuple[None, None]:

175 # Line numbers won't be useful if we don't have the source code.

176 if not self.filepath or self.filepath not in self.lines_collection:

177 return None, None

178 try:

179 lines, lineno = getsourcelines(node.obj)

180 except (OSError, TypeError):

181 return None, None

182 return lineno, lineno + "".join(lines).rstrip().count("\n")

183

184 def get_module(self, import_paths: Sequence[str | Path] | None = None) -> Module:

185 """Build and return the object representing the module attached to this inspector.

186

187 This method triggers a complete inspection of the module members.

188

189 Parameters:

190 import_paths: Paths replacing `sys.path` to import the module.

191

192 Returns:

193 A module instance.

194 """

195 import_path = self.module_name

196 if self.parent is not None:

197 import_path = f"{self.parent.path}.{import_path}"

198

199 # Make sure `import_paths` is a list, in case we want to `insert` into it.

200 import_paths = list(import_paths or ())

201

202 # If the thing we want to import has a filepath,

203 # we make sure to insert the right parent directory

204 # at the front of our list of import paths.

205 # We do this by counting the number of dots `.` in the import path,

206 # corresponding to slashes `/` in the filesystem,

207 # and go up in the file tree the same number of times.

208 if self.filepath:

209 parent_path = self.filepath.parent

210 for _ in range(import_path.count(".")):

211 parent_path = parent_path.parent

212 # Climb up one more time for `__init__` modules.

213 if self.filepath.stem == "__init__":

214 parent_path = parent_path.parent

215 if parent_path not in import_paths:

216 import_paths.insert(0, parent_path)

217

218 value = dynamic_import(import_path, import_paths)

219

220 # We successfully imported the given object,

221 # and we now create the object tree with all the necessary nodes,

222 # from the root of the package to this leaf object.

223 parent_node = None

224 if self.parent is not None:

225 for part in self.parent.path.split("."):

226 parent_node = ObjectNode(None, name=part, parent=parent_node)

227 module_node = ObjectNode(value, self.module_name, parent=parent_node)

228

229 self.inspect(module_node)

230 return self.current.module

231

232 def inspect(self, node: ObjectNode) -> None:

233 """Extend the base inspection with extensions.

234

235 Parameters:

236 node: The node to inspect.

237 """

238 getattr(self, f"inspect_{node.kind}", self.generic_inspect)(node)

239

240 def generic_inspect(self, node: ObjectNode) -> None:

241 """Extend the base generic inspection with extensions.

242

243 Parameters:

244 node: The node to inspect.

245 """

246 for child in node.children:

247 if target_path := child.alias_target_path:

248 # If the child is an actual submodule of the current module,

249 # and has no `__file__` set, we won't find it on the disk so we must inspect it now.

250 # For that we instantiate a new inspector and use it to inspect the submodule,

251 # then assign the submodule as member of the current module.

252 # If the submodule has a `__file__` set, the loader should find it on the disk,

253 # so we skip it here (no member, no alias, just skip it).

254 if child.is_module and target_path == f"{self.current.path}.{child.name}":

255 if not hasattr(child.obj, "__file__"): 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true

256 logger.debug("Module %s is not discoverable on disk, inspecting right now", target_path)

257 inspector = Inspector(

258 child.name,

259 filepath=None,

260 parent=self.current.module,

261 extensions=self.extensions,

262 docstring_parser=self.docstring_parser,

263 docstring_options=self.docstring_options,

264 lines_collection=self.lines_collection,

265 modules_collection=self.modules_collection,

266 )

267 inspector.inspect_module(child)

268 self.current.set_member(child.name, inspector.current.module)

269 # Otherwise, alias the object.

270 else:

271 alias = Alias(child.name, target_path)

272 self.current.set_member(child.name, alias)

273 self.extensions.call("on_alias", alias=alias, node=node, agent=self)

274 else:

275 self.inspect(child)

276

277 def inspect_module(self, node: ObjectNode) -> None:

278 """Inspect a module.

279

280 Parameters:

281 node: The node to inspect.

282 """

283 self.extensions.call("on_node", node=node, agent=self)

284 self.extensions.call("on_module_node", node=node, agent=self)

285 self.current = module = Module(

286 name=self.module_name,

287 filepath=self.filepath,

288 parent=self.parent,

289 docstring=self._get_docstring(node),

290 lines_collection=self.lines_collection,

291 modules_collection=self.modules_collection,

292 )

293 self.extensions.call("on_instance", node=node, obj=module, agent=self)

294 self.extensions.call("on_module_instance", node=node, mod=module, agent=self)

295 self.generic_inspect(node)

296 self.extensions.call("on_members", node=node, obj=module, agent=self)

297 self.extensions.call("on_module_members", node=node, mod=module, agent=self)

298

299 def inspect_class(self, node: ObjectNode) -> None:

300 """Inspect a class.

301

302 Parameters:

303 node: The node to inspect.

304 """

305 self.extensions.call("on_node", node=node, agent=self)

306 self.extensions.call("on_class_node", node=node, agent=self)

307

308 bases = []

309 for base in node.obj.__bases__:

310 if base is object:

311 continue

312 bases.append(f"{base.__module__}.{base.__qualname__}")

313

314 lineno, endlineno = self._get_linenos(node)

315 class_ = Class(

316 name=node.name,

317 docstring=self._get_docstring(node),

318 bases=bases,

319 lineno=lineno,

320 endlineno=endlineno,

321 )

322 self.current.set_member(node.name, class_)

323 self.current = class_

324 self.extensions.call("on_instance", node=node, obj=class_, agent=self)

325 self.extensions.call("on_class_instance", node=node, cls=class_, agent=self)

326 self.generic_inspect(node)

327 self.extensions.call("on_members", node=node, obj=class_, agent=self)

328 self.extensions.call("on_class_members", node=node, cls=class_, agent=self)

329 self.current = self.current.parent # type: ignore[assignment]

330

331 def inspect_staticmethod(self, node: ObjectNode) -> None:

332 """Inspect a static method.

333

334 Parameters:

335 node: The node to inspect.

336 """

337 self.handle_function(node, {"staticmethod"})

338

339 def inspect_classmethod(self, node: ObjectNode) -> None:

340 """Inspect a class method.

341

342 Parameters:

343 node: The node to inspect.

344 """

345 self.handle_function(node, {"classmethod"})

346

347 def inspect_method_descriptor(self, node: ObjectNode) -> None:

348 """Inspect a method descriptor.

349

350 Parameters:

351 node: The node to inspect.

352 """

353 self.handle_function(node, {"method descriptor"})

354

355 def inspect_builtin_method(self, node: ObjectNode) -> None:

356 """Inspect a builtin method.

357

358 Parameters:

359 node: The node to inspect.

360 """

361 self.handle_function(node, {"builtin"})

362

363 def inspect_method(self, node: ObjectNode) -> None:

364 """Inspect a method.

365

366 Parameters:

367 node: The node to inspect.

368 """

369 self.handle_function(node)

370

371 def inspect_coroutine(self, node: ObjectNode) -> None:

372 """Inspect a coroutine.

373

374 Parameters:

375 node: The node to inspect.

376 """

377 self.handle_function(node, {"async"})

378

379 def inspect_builtin_function(self, node: ObjectNode) -> None:

380 """Inspect a builtin function.

381

382 Parameters:

383 node: The node to inspect.

384 """

385 self.handle_function(node, {"builtin"})

386

387 def inspect_function(self, node: ObjectNode) -> None:

388 """Inspect a function.

389

390 Parameters:

391 node: The node to inspect.

392 """

393 self.handle_function(node)

394

395 def inspect_cached_property(self, node: ObjectNode) -> None:

396 """Inspect a cached property.

397

398 Parameters:

399 node: The node to inspect.

400 """

401 self.handle_function(node, {"cached", "property"})

402

403 def inspect_property(self, node: ObjectNode) -> None:

404 """Inspect a property.

405

406 Parameters:

407 node: The node to inspect.

408 """

409 self.handle_function(node, {"property"})

410

411 def handle_function(self, node: ObjectNode, labels: set | None = None) -> None:

412 """Handle a function.

413

414 Parameters:

415 node: The node to inspect.

416 labels: Labels to add to the data object.

417 """

418 self.extensions.call("on_node", node=node, agent=self)

419 self.extensions.call("on_function_node", node=node, agent=self)

420

421 try:

422 signature = getsignature(node.obj)

423 except Exception: # noqa: BLE001

424 # so many exceptions can be raised here:

425 # AttributeError, NameError, RuntimeError, ValueError, TokenError, TypeError

426 parameters = None

427 returns = None

428 else:

429 parameters = Parameters(

430 *[_convert_parameter(parameter, parent=self.current) for parameter in signature.parameters.values()],

431 )

432 return_annotation = signature.return_annotation

433 returns = (

434 None

435 if return_annotation is _empty

436 else _convert_object_to_annotation(return_annotation, parent=self.current)

437 )

438

439 lineno, endlineno = self._get_linenos(node)

440

441 obj: Attribute | Function

442 labels = labels or set()

443 if "property" in labels:

444 obj = Attribute(

445 name=node.name,

446 value=None,

447 annotation=returns,

448 docstring=self._get_docstring(node),

449 lineno=lineno,

450 endlineno=endlineno,

451 )

452 else:

453 obj = Function(

454 name=node.name,

455 parameters=parameters,

456 returns=returns,

457 docstring=self._get_docstring(node),

458 lineno=lineno,

459 endlineno=endlineno,

460 )

461 obj.labels |= labels

462 self.current.set_member(node.name, obj)

463 self.extensions.call("on_instance", node=node, obj=obj, agent=self)

464 if obj.is_attribute:

465 self.extensions.call("on_attribute_instance", node=node, attr=obj, agent=self)

466 else:

467 self.extensions.call("on_function_instance", node=node, func=obj, agent=self)

468

469 def inspect_attribute(self, node: ObjectNode) -> None:

470 """Inspect an attribute.

471

472 Parameters:

473 node: The node to inspect.

474 """

475 self.handle_attribute(node)

476

477 def handle_attribute(self, node: ObjectNode, annotation: str | Expr | None = None) -> None:

478 """Handle an attribute.

479

480 Parameters:

481 node: The node to inspect.

482 annotation: A potential annotation.

483 """

484 self.extensions.call("on_node", node=node, agent=self)

485 self.extensions.call("on_attribute_node", node=node, agent=self)

486

487 # TODO: To improve.

488 parent = self.current

489 labels: set[str] = set()

490

491 if parent.kind is Kind.MODULE:

492 labels.add("module")

493 elif parent.kind is Kind.CLASS: 493 ↛ 495line 493 didn't jump to line 495 because the condition on line 493 was always true

494 labels.add("class")

495 elif parent.kind is Kind.FUNCTION:

496 if parent.name != "__init__":

497 return

498 parent = parent.parent # type: ignore[assignment]

499 labels.add("instance")

500

501 try:

502 value = repr(node.obj)

503 except Exception: # noqa: BLE001

504 value = None

505 try:

506 docstring = self._get_docstring(node)

507 except Exception: # noqa: BLE001

508 docstring = None

509

510 attribute = Attribute(

511 name=node.name,

512 value=value,

513 annotation=annotation,

514 docstring=docstring,

515 )

516 attribute.labels |= labels

517 parent.set_member(node.name, attribute)

518

519 if node.name == "__all__":

520 parent.exports = set(node.obj)

521 self.extensions.call("on_instance", node=node, obj=attribute, agent=self)

522 self.extensions.call("on_attribute_instance", node=node, attr=attribute, agent=self)

523

524

525_kind_map = {

526 SignatureParameter.POSITIONAL_ONLY: ParameterKind.positional_only,

527 SignatureParameter.POSITIONAL_OR_KEYWORD: ParameterKind.positional_or_keyword,

528 SignatureParameter.VAR_POSITIONAL: ParameterKind.var_positional,

529 SignatureParameter.KEYWORD_ONLY: ParameterKind.keyword_only,

530 SignatureParameter.VAR_KEYWORD: ParameterKind.var_keyword,

531}

532

533

534def _convert_parameter(parameter: SignatureParameter, parent: Module | Class) -> Parameter:

535 name = parameter.name

536 annotation = (

537 None if parameter.annotation is _empty else _convert_object_to_annotation(parameter.annotation, parent=parent)

538 )

539 kind = _kind_map[parameter.kind]

540 if parameter.default is _empty:

541 default = None

542 elif hasattr(parameter.default, "__name__"):

543 # avoid repr containing chevrons and memory addresses

544 default = parameter.default.__name__

545 else:

546 default = repr(parameter.default)

547 return Parameter(name, annotation=annotation, kind=kind, default=default)

548

549

550def _convert_object_to_annotation(obj: Any, parent: Module | Class) -> str | Expr | None:

551 # even when *we* import future annotations,

552 # the object from which we get a signature

553 # can come from modules which did *not* import them,

554 # so inspect.signature returns actual Python objects

555 # that we must deal with

556 if not isinstance(obj, str):

557 if hasattr(obj, "__name__"): # noqa: SIM108 557 ↛ 562line 557 didn't jump to line 562 because the condition on line 557 was always true

558 # simple types like int, str, custom classes, etc.

559 obj = obj.__name__

560 else:

561 # other, more complex types: hope for the best

562 obj = repr(obj)

563 try:

564 annotation_node = compile(obj, mode="eval", filename="<>", flags=ast.PyCF_ONLY_AST, optimize=2)

565 except SyntaxError:

566 return obj

567 return safe_get_annotation(annotation_node.body, parent=parent) # type: ignore[attr-defined]