Coverage for src/mkdocstrings_handlers/python/handler.py: 76.54%
126 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-03-09 18:38 +0100
1"""This module implements a handler for the Python language.
3It collects data with [`pytkdocs`](https://github.com/pawamoy/pytkdocs).
4"""
6import json
7import os
8import posixpath
9import sys
10import traceback
11from collections.abc import Iterator, Mapping, MutableMapping
12from copy import deepcopy
13from pathlib import Path
14from subprocess import PIPE, Popen
15from typing import Any, BinaryIO, ClassVar, Optional
17from mkdocs.config.defaults import MkDocsConfig
18from mkdocs.exceptions import PluginError
19from mkdocstrings import BaseHandler, CollectionError, CollectorItem, Inventory, get_logger
21from mkdocstrings_handlers.python.rendering import (
22 do_brief_xref,
23 rebuild_category_lists,
24 sort_key_alphabetical,
25 sort_key_source,
26 sort_object,
27)
29# TODO: add a deprecation warning once the new handler handles 95% of use-cases
31logger = get_logger(__name__)
34class PythonHandler(BaseHandler):
35 """The Python handler class."""
37 name: ClassVar[str] = "python"
38 """The handler name."""
39 domain: ClassVar[str] = "py" # to match Sphinx's default domain
40 """The domain of the handler."""
41 enable_inventory: ClassVar[bool] = True
42 """Whether the handler supports inventory files."""
44 fallback_theme: ClassVar[str] = "material"
45 """The fallback theme to use when the user-selected theme is not supported."""
46 fallback_config: ClassVar[dict] = {"docstring_style": "markdown", "filters": ["!.*"]}
47 """The configuration used when falling back to re-collecting an object to get its anchor.
49 This configuration is used in [`Handlers.get_anchors`][mkdocstrings.Handlers.get_anchors].
51 When trying to fix (optional) cross-references, the autorefs plugin will try to collect
52 an object with every configured handler until one succeeds. It will then try to get
53 an anchor for it. It's because objects can have multiple identifiers (aliases),
54 for example their definition path and multiple import paths in Python.
56 When re-collecting the object, we have no use for its members, or for its docstring being parsed.
57 This is why the fallback configuration filters every member out, and uses the Markdown style,
58 which we know will not generate any warnings.
59 """
61 default_config: ClassVar[dict] = {
62 "filters": ["!^_[^_]"],
63 "show_root_heading": False,
64 "show_root_toc_entry": True,
65 "show_root_full_path": True,
66 "show_root_members_full_path": False,
67 "show_object_full_path": False,
68 "show_category_heading": False,
69 "show_if_no_docstring": False,
70 "show_signature": True,
71 "show_signature_annotations": False,
72 "show_source": True,
73 "show_bases": True,
74 "group_by_category": True,
75 "heading_level": 2,
76 "members_order": "alphabetical",
77 }
78 """
79 **Headings options:**
81 - `heading_level` (`int`): The initial heading level to use. Default: `2`.
82 - `show_root_heading` (`bool`): Show the heading of the object at the root of the documentation tree
83 (i.e. the object referenced by the identifier after `:::`). Default: `False`.
84 - `show_root_toc_entry` (`bool`): If the root heading is not shown, at least add a ToC entry for it. Default: `True`.
85 - `show_root_full_path` (`bool`): Show the full Python path for the root object heading. Default: `True`.
86 - `show_root_members_full_path` (`bool`): Show the full Python path of the root members. Default: `False`.
87 - `show_object_full_path` (`bool`): Show the full Python path of every object. Default: `False`.
88 - `show_category_heading` (`bool`): When grouped by categories, show a heading for each category. Default: `False`.
90 **Members options:**
92 - `members` (`list[str] | False | None`): An explicit list of members to render. Default: `None`.
93 - `members_order` (`str`): The members ordering to use. Options: `alphabetical` - order by the members names,
94 `source` - order members as they appear in the source file. Default: `"alphabetical"`.
95 - `filters` (`list[str] | None`): A list of filters applied to filter objects based on their name.
96 A filter starting with `!` will exclude matching objects instead of including them.
97 The `members` option takes precedence over `filters` (filters will still be applied recursively
98 to lower members in the hierarchy). Default: `["!^_[^_]"]`.
99 - `group_by_category` (`bool`): Group the object's children by categories: attributes, classes, functions, and modules. Default: `True`.
101 **Docstrings options:**
103 - `docstring_style` (`str`): The docstring style to use: `google`, `numpy`, `restructured-text`, or `None`. Default: `"google"`.
104 - `docstring_options` (`dict`): The options for the docstring parser. See parsers under [`pytkdocs.parsers.docstrings`][].
105 - `show_if_no_docstring` (`bool`): Show the object heading even if it has no docstring or children with docstrings. Default: `False`.
107 **Signatures/annotations options:**
109 - `show_signature` (`bool`): Show methods and functions signatures. Default: `True`.
110 - `show_signature_annotations` (`bool`): Show the type annotations in methods and functions signatures. Default: `False`.
112 **Additional options:**
114 - `show_bases` (`bool`): Show the base classes of a class. Default: `True`.
115 - `show_source` (`bool`): Show the source code of this object. Default: `True`.
116 """
118 def __init__(self, config: dict[str, Any], base_dir: Path, **kwargs: Any) -> None:
119 """Initialize the handler.
121 When instantiating a Python handler, we open a `pytkdocs` subprocess in the background with `subprocess.Popen`.
122 It will allow us to feed input to and read output from this subprocess, keeping it alive during
123 the whole documentation generation. Spawning a new Python subprocess for each "autodoc" instruction would be
124 too resource intensive, and would slow down `mkdocstrings` a lot.
126 Parameters:
127 config: The handler configuration.
128 base_dir: The base directory of the project.
129 **kwargs: Arguments passed to the parent constructor.
130 """
131 super().__init__(**kwargs)
133 self.base_dir = base_dir
134 self.config = config
135 self.global_options = config.get("options", {})
137 logger.debug("Opening 'pytkdocs' subprocess")
138 env = os.environ.copy()
139 env["PYTHONUNBUFFERED"] = "1"
141 paths = config.get("paths") or []
142 if not paths and self.base_dir: 142 ↛ 144line 142 didn't jump to line 144 because the condition on line 142 was always true
143 paths.append(self.base_dir)
144 search_paths = []
145 for path in paths:
146 if not os.path.isabs(path) and self.base_dir: 146 ↛ 148line 146 didn't jump to line 148 because the condition on line 146 was always true
147 path = os.path.abspath(os.path.join(self.base_dir, path)) # noqa: PLW2901
148 if path not in search_paths: 148 ↛ 145line 148 didn't jump to line 145 because the condition on line 148 was always true
149 search_paths.append(path)
150 self._paths = search_paths
152 commands = []
154 if search_paths: 154 ↛ 157line 154 didn't jump to line 157 because the condition on line 154 was always true
155 commands.extend([f"sys.path.insert(0, {path!r})" for path in reversed(search_paths)])
157 if setup_commands := config.get("setup_commands"): 157 ↛ 160line 157 didn't jump to line 160 because the condition on line 157 was never true
158 # prevent the Python interpreter or the setup commands
159 # from writing to stdout as it would break pytkdocs output
160 commands.extend(
161 [
162 "from io import StringIO",
163 "sys.stdout = StringIO()", # redirect stdout to memory buffer
164 *setup_commands,
165 "sys.stdout.flush()",
166 "sys.stdout = sys.__stdout__", # restore stdout
167 ],
168 )
170 if commands: 170 ↛ 179line 170 didn't jump to line 179 because the condition on line 170 was always true
171 final_commands = [
172 "import sys",
173 *commands,
174 "from pytkdocs.cli import main as pytkdocs",
175 "pytkdocs(['--line-by-line'])",
176 ]
177 cmd = [sys.executable, "-c", "; ".join(final_commands)]
178 else:
179 cmd = [sys.executable, "-m", "pytkdocs", "--line-by-line"]
181 self.process = Popen( # noqa: S603
182 cmd,
183 universal_newlines=True,
184 stdout=PIPE,
185 stdin=PIPE,
186 bufsize=-1,
187 env=env,
188 )
190 def get_inventory_urls(self) -> list[tuple[str, dict[str, Any]]]:
191 """Return the URLs of the inventory files to download."""
192 return [
193 (inv.pop("url"), inv) if isinstance(inv, dict) else (inv, {})
194 for inv in deepcopy(self.config.get("import", []))
195 ]
197 @classmethod
198 def load_inventory(
199 cls,
200 in_file: BinaryIO,
201 url: str,
202 base_url: Optional[str] = None,
203 **kwargs: Any, # noqa: ARG003
204 ) -> Iterator[tuple[str, str]]:
205 """Yield items and their URLs from an inventory file streamed from `in_file`.
207 This implements mkdocstrings' `load_inventory` "protocol" (see plugin.py).
209 Arguments:
210 in_file: The binary file-like object to read the inventory from.
211 url: The URL that this file is being streamed from (used to guess `base_url`).
212 base_url: The URL that this inventory's sub-paths are relative to.
213 **kwargs: Ignore additional arguments passed from the config.
215 Yields:
216 Tuples of (item identifier, item URL).
217 """
218 if base_url is None:
219 base_url = posixpath.dirname(url)
221 for item in Inventory.parse_sphinx(in_file, domain_filter=("py",)).values():
222 yield item.name, posixpath.join(base_url, item.uri)
224 def get_options(self, local_options: Mapping[str, Any]) -> MutableMapping[str, Any]:
225 """Return the options to use to collect an object.
227 We merge the global options with the options specific to the object being collected.
229 Arguments:
230 local_options: The selection options.
232 Returns:
233 The options to use to collect an object.
234 """
235 return {**self.default_config, **self.global_options, **local_options}
237 def collect(self, identifier: str, options: MutableMapping[str, Any]) -> CollectorItem:
238 """Collect the documentation tree given an identifier and selection options.
240 In this method, we feed one line of JSON to the standard input of the subprocess that was opened
241 during instantiation of the collector. Then we read one line of JSON on its standard output.
243 We load back the JSON text into a Python dictionary.
244 If there is a decoding error, we log it as error and raise a CollectionError.
246 If the dictionary contains an `error` key, we log it as error (with the optional `traceback` value),
247 and raise a CollectionError.
249 If the dictionary values for keys `loading_errors` and `parsing_errors` are not empty,
250 we log them as warnings.
252 Then we pick up the only object within the `objects` list (there's always only one, because we collect
253 them one by one), rebuild it's categories lists
254 (see [`rebuild_category_lists()`][mkdocstrings_handlers.python.rendering.rebuild_category_lists]),
255 and return it.
257 Arguments:
258 identifier: The dotted-path of a Python object available in the Python path.
259 options: Selection options, used to alter the data collection done by `pytkdocs`.
261 Raises:
262 CollectionError: When there was a problem collecting the object documentation.
264 Returns:
265 The collected object-tree.
266 """
267 pytkdocs_options = {}
268 for option in ("filters", "members", "docstring_style", "docstring_options"):
269 if option in options:
270 pytkdocs_options[option] = options[option]
272 logger.debug("Preparing input")
273 json_input = json.dumps({"objects": [{"path": identifier, **pytkdocs_options}]})
275 logger.debug("Writing to process' stdin")
276 self.process.stdin.write(json_input + "\n") # type: ignore[union-attr]
277 self.process.stdin.flush() # type: ignore[union-attr]
279 logger.debug("Reading process' stdout")
280 stdout = self.process.stdout.readline() # type: ignore[union-attr]
282 logger.debug("Loading JSON output as Python object")
283 try:
284 result = json.loads(stdout)
285 except json.decoder.JSONDecodeError as exception:
286 error = "\n".join(("Error while loading JSON:", stdout, traceback.format_exc()))
287 raise CollectionError(error) from exception
289 if "error" in result:
290 error = result["error"]
291 if "traceback" in result:
292 error += f"\n{result['traceback']}"
293 raise CollectionError(error)
295 for loading_error in result["loading_errors"]: 295 ↛ 296line 295 didn't jump to line 296 because the loop on line 295 never started
296 logger.warning(loading_error)
298 for errors in result["parsing_errors"].values(): 298 ↛ 299line 298 didn't jump to line 299 because the loop on line 298 never started
299 for parsing_error in errors:
300 logger.warning(parsing_error)
302 # We always collect only one object at a time
303 result = result["objects"][0]
305 logger.debug("Rebuilding categories and children lists")
306 rebuild_category_lists(result)
308 return result
310 def teardown(self) -> None:
311 """Terminate the opened subprocess, set it to `None`."""
312 logger.debug("Tearing process down")
313 self.process.terminate()
315 def render(self, data: CollectorItem, options: MutableMapping[str, Any]) -> str:
316 """Render the collected data into HTML."""
317 template = self.env.get_template(f"{data['category']}.html")
319 # Heading level is a "state" variable, that will change at each step
320 # of the rendering recursion. Therefore, it's easier to use it as a plain value
321 # than as an item in a dictionary.
322 heading_level = options["heading_level"]
324 members_order = options["members_order"]
325 if members_order == "alphabetical": 325 ↛ 327line 325 didn't jump to line 327 because the condition on line 325 was always true
326 sort_function = sort_key_alphabetical
327 elif members_order == "source":
328 sort_function = sort_key_source
329 else:
330 raise PluginError(f"Unknown members_order '{members_order}', choose between 'alphabetical' and 'source'.")
332 sort_object(data, sort_function=sort_function)
334 return template.render(
335 **{"config": options, data["category"]: data, "heading_level": heading_level, "root": True},
336 )
338 def get_aliases(self, identifier: str) -> tuple[str, ...]:
339 """Return the aliases of an identifier."""
340 try:
341 data = self.collect(identifier, self.fallback_config)
342 return (data["path"],)
343 except (CollectionError, KeyError):
344 return ()
346 def update_env(self, config: dict) -> None: # noqa: ARG002,D102
347 self.env.trim_blocks = True
348 self.env.lstrip_blocks = True
349 self.env.keep_trailing_newline = False
350 self.env.filters["brief_xref"] = do_brief_xref
353def get_handler(
354 handler_config: MutableMapping[str, Any],
355 tool_config: MkDocsConfig,
356 **kwargs: Any,
357) -> PythonHandler:
358 """Simply return an instance of `PythonHandler`.
360 Arguments:
361 handler_config: The handler configuration.
362 tool_config: The tool (SSG) configuration.
364 Returns:
365 An instance of `PythonHandler`.
366 """
367 base_dir = Path(tool_config.config_file_path or "./mkdocs.yml").parent
368 return PythonHandler(config=dict(handler_config), base_dir=base_dir, **kwargs)