Coverage for src/mkdocstrings_handlers/python/handler.py: 67.08%

1"""This module implements a handler for the Python language. 

2 

3It collects data with [`pytkdocs`](https://github.com/pawamoy/pytkdocs). 

4""" 

5 

6from __future__ import annotations 

7 

8import json 

9import os 

10import posixpath 

11import sys 

12import traceback 

13from collections import ChainMap 

14from subprocess import PIPE, Popen # noqa: S404 

15from typing import Any, BinaryIO, Iterator, List, Optional, Sequence, Tuple 

16 

17from markdown import Markdown 

18 

19from mkdocstrings.extension import PluginError 

20from mkdocstrings.handlers.base import BaseHandler, CollectionError, CollectorItem 

21from mkdocstrings.inventory import Inventory 

22from mkdocstrings.loggers import get_logger 

23from mkdocstrings_handlers.python.rendering import ( 

24 do_brief_xref, 

25 rebuild_category_lists, 

26 sort_key_alphabetical, 

27 sort_key_source, 

28 sort_object, 

29) 

30 

31# TODO: add a deprecation warning once the new handler handles 95% of use-cases 

32 

33logger = get_logger(__name__) 

34 

35 

36class PythonHandler(BaseHandler): 

37 """The Python handler class. 

38 

39 Attributes: 

40 domain: The cross-documentation domain/language for this handler. 

41 enable_inventory: Whether this handler is interested in enabling the creation 

42 of the `objects.inv` Sphinx inventory file. 

43 """ 

44 

45 domain: str = "py" # to match Sphinx's default domain 

46 enable_inventory: bool = True 

47 

48 fallback_theme = "material" 

49 fallback_config = {"docstring_style": "markdown", "filters": ["!.*"]} 

50 """The configuration used when falling back to re-collecting an object to get its anchor. 

51 

52 This configuration is used in [`Handlers.get_anchors`][mkdocstrings.handlers.base.Handlers.get_anchors]. 

53 

54 When trying to fix (optional) cross-references, the autorefs plugin will try to collect 

55 an object with every configured handler until one succeeds. It will then try to get 

56 an anchor for it. It's because objects can have multiple identifiers (aliases), 

57 for example their definition path and multiple import paths in Python. 

58 

59 When re-collecting the object, we have no use for its members, or for its docstring being parsed. 

60 This is why the fallback configuration filters every member out, and uses the Markdown style, 

61 which we know will not generate any warnings. 

62 """ 

63 

64 default_config: dict = { 

65 "filters": ["!^_[^_]"], 

66 "show_root_heading": False, 

67 "show_root_toc_entry": True, 

68 "show_root_full_path": True, 

69 "show_root_members_full_path": False, 

70 "show_object_full_path": False, 

71 "show_category_heading": False, 

72 "show_if_no_docstring": False, 

73 "show_signature": True, 

74 "show_signature_annotations": False, 

75 "show_source": True, 

76 "show_bases": True, 

77 "group_by_category": True, 

78 "heading_level": 2, 

79 "members_order": "alphabetical", 

80 } 

81 """ 

82 **Headings options:** 

83 

84 - `heading_level` (`int`): The initial heading level to use. Default: `2`. 

85 - `show_root_heading` (`bool`): Show the heading of the object at the root of the documentation tree 

86 (i.e. the object referenced by the identifier after `:::`). Default: `False`. 

87 - `show_root_toc_entry` (`bool`): If the root heading is not shown, at least add a ToC entry for it. Default: `True`. 

88 - `show_root_full_path` (`bool`): Show the full Python path for the root object heading. Default: `True`. 

89 - `show_root_members_full_path` (`bool`): Show the full Python path of the root members. Default: `False`. 

90 - `show_object_full_path` (`bool`): Show the full Python path of every object. Default: `False`. 

91 - `show_category_heading` (`bool`): When grouped by categories, show a heading for each category. Default: `False`. 

92 

93 **Members options:** 

94 

95 - `members` (`list[str] | False | None`): An explicit list of members to render. Default: `None`. 

96 - `members_order` (`str`): The members ordering to use. Options: `alphabetical` - order by the members names, 

97 `source` - order members as they appear in the source file. Default: `"alphabetical"`. 

98 - `filters` (`list[str] | None`): A list of filters applied to filter objects based on their name. 

99 A filter starting with `!` will exclude matching objects instead of including them. 

100 The `members` option takes precedence over `filters` (filters will still be applied recursively 

101 to lower members in the hierarchy). Default: `["!^_[^_]"]`. 

102 - `group_by_category` (`bool`): Group the object's children by categories: attributes, classes, functions, and modules. Default: `True`. 

103 

104 **Docstrings options:** 

105 

106 - `docstring_style` (`str`): The docstring style to use: `google`, `numpy`, `sphinx`, or `None`. Default: `"google"`. 

107 - `docstring_options` (`dict`): The options for the docstring parser. See parsers under [`pytkdocs.parsers.docstrings`][]. 

108 - `show_if_no_docstring` (`bool`): Show the object heading even if it has no docstring or children with docstrings. Default: `False`. 

109 

110 **Signatures/annotations options:** 

111 

112 - `show_signature` (`bool`): Show methods and functions signatures. Default: `True`. 

113 - `show_signature_annotations` (`bool`): Show the type annotations in methods and functions signatures. Default: `False`. 

114 

115 **Additional options:** 

116 

117 - `show_bases` (`bool`): Show the base classes of a class. Default: `True`. 

118 - `show_source` (`bool`): Show the source code of this object. Default: `True`. 

119 """ # noqa: E501 

120 

121 def __init__( # noqa: WPS231 

122 self, 

123 *args, 

124 setup_commands: Optional[List[str]] = None, 

125 config_file_path: str | None = None, 

126 paths: list[str] | None = None, 

127 **kwargs, 

128 ) -> None: 

129 """Initialize the handler. 

130 

131 When instantiating a Python handler, we open a `pytkdocs` subprocess in the background with `subprocess.Popen`. 

132 It will allow us to feed input to and read output from this subprocess, keeping it alive during 

133 the whole documentation generation. Spawning a new Python subprocess for each "autodoc" instruction would be 

134 too resource intensive, and would slow down `mkdocstrings` a lot. 

135 

136 Parameters: 

137 *args: Handler name, theme and custom templates. 

138 setup_commands: A list of python commands as strings to be executed in the subprocess before `pytkdocs`. 

139 config_file_path: The MkDocs configuration file path. 

140 paths: A list of paths to use as search paths. 

141 **kwargs: Same thing, but with keyword arguments. 

142 """ 

143 logger.debug("Opening 'pytkdocs' subprocess") 

144 env = os.environ.copy() 

145 env["PYTHONUNBUFFERED"] = "1" 

146 

147 self._config_file_path = config_file_path 

148 paths = paths or [] 

149 if not paths and config_file_path: 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true

150 paths.append(os.path.dirname(config_file_path)) 

151 search_paths = [] 

152 for path in paths: 152 ↛ 153line 152 didn't jump to line 153, because the loop on line 152 never started

153 if not os.path.isabs(path): 

154 if config_file_path: 

155 path = os.path.abspath(os.path.join(os.path.dirname(config_file_path), path)) 

156 if path not in search_paths: 

157 search_paths.append(path) 

158 self._paths = search_paths 

159 

160 commands = [] 

161 

162 if search_paths: 162 ↛ 163line 162 didn't jump to line 163, because the condition on line 162 was never true

163 commands.extend([f"sys.path.insert(0, {path!r})" for path in reversed(search_paths)]) # noqa: WPS441 

164 

165 if setup_commands: 165 ↛ 168line 165 didn't jump to line 168, because the condition on line 165 was never true

166 # prevent the Python interpreter or the setup commands 

167 # from writing to stdout as it would break pytkdocs output 

168 commands.extend( 

169 [ 

170 "from io import StringIO", 

171 "sys.stdout = StringIO()", # redirect stdout to memory buffer 

172 *setup_commands, 

173 "sys.stdout.flush()", 

174 "sys.stdout = sys.__stdout__", # restore stdout 

175 ] 

176 ) 

177 

178 if commands: 178 ↛ 179line 178 didn't jump to line 179

179 final_commands = [ 

180 "import sys", 

181 *commands, 

182 "from pytkdocs.cli import main as pytkdocs", 

183 "pytkdocs(['--line-by-line'])", 

184 ] 

185 cmd = [sys.executable, "-c", "; ".join(final_commands)] 

186 else: 

187 cmd = [sys.executable, "-m", "pytkdocs", "--line-by-line"] 

188 

189 self.process = Popen( # noqa: S603,S607 (we trust the input, and we don't want to use the absolute path) 

190 cmd, 

191 universal_newlines=True, 

192 stdout=PIPE, 

193 stdin=PIPE, 

194 bufsize=-1, 

195 env=env, 

196 ) 

197 super().__init__(*args, **kwargs) 

198 

199 @classmethod 

200 def load_inventory( 

201 cls, in_file: BinaryIO, url: str, base_url: Optional[str] = None, **kwargs 

202 ) -> Iterator[Tuple[str, str]]: 

203 """Yield items and their URLs from an inventory file streamed from `in_file`. 

204 

205 This implements mkdocstrings' `load_inventory` "protocol" (see plugin.py). 

206 

207 Arguments: 

208 in_file: The binary file-like object to read the inventory from. 

209 url: The URL that this file is being streamed from (used to guess `base_url`). 

210 base_url: The URL that this inventory's sub-paths are relative to. 

211 **kwargs: Ignore additional arguments passed from the config. 

212 

213 Yields: 

214 Tuples of (item identifier, item URL). 

215 """ 

216 if base_url is None: 

217 base_url = posixpath.dirname(url) 

218 

219 for item in Inventory.parse_sphinx(in_file, domain_filter=("py",)).values(): # noqa: WPS526 

220 yield item.name, posixpath.join(base_url, item.uri) 

221 

222 def collect(self, identifier: str, config: dict) -> CollectorItem: # noqa: WPS231 

223 """Collect the documentation tree given an identifier and selection options. 

224 

225 In this method, we feed one line of JSON to the standard input of the subprocess that was opened 

226 during instantiation of the collector. Then we read one line of JSON on its standard output. 

227 

228 We load back the JSON text into a Python dictionary. 

229 If there is a decoding error, we log it as error and raise a CollectionError. 

230 

231 If the dictionary contains an `error` key, we log it as error (with the optional `traceback` value), 

232 and raise a CollectionError. 

233 

234 If the dictionary values for keys `loading_errors` and `parsing_errors` are not empty, 

235 we log them as warnings. 

236 

237 Then we pick up the only object within the `objects` list (there's always only one, because we collect 

238 them one by one), rebuild it's categories lists 

239 (see [`rebuild_category_lists()`][mkdocstrings_handlers.python.rendering.rebuild_category_lists]), 

240 and return it. 

241 

242 Arguments: 

243 identifier: The dotted-path of a Python object available in the Python path. 

244 config: Selection options, used to alter the data collection done by `pytkdocs`. 

245 

246 Raises: 

247 CollectionError: When there was a problem collecting the object documentation. 

248 

249 Returns: 

250 The collected object-tree. 

251 """ 

252 final_config = {} 

253 for option in ("filters", "members"): 

254 if option in config: 254 ↛ 255line 254 didn't jump to line 255, because the condition on line 254 was never true

255 final_config[option] = config[option] 

256 elif option in self.default_config: 

257 final_config[option] = self.default_config[option] 

258 

259 logger.debug("Preparing input") 

260 json_input = json.dumps({"objects": [{"path": identifier, **final_config}]}) 

261 

262 logger.debug("Writing to process' stdin") 

263 self.process.stdin.write(json_input + "\n") # type: ignore 

264 self.process.stdin.flush() # type: ignore 

265 

266 logger.debug("Reading process' stdout") 

267 stdout = self.process.stdout.readline() # type: ignore 

268 

269 logger.debug("Loading JSON output as Python object") 

270 try: 

271 result = json.loads(stdout) 

272 except json.decoder.JSONDecodeError as exception: 

273 error = "\n".join(("Error while loading JSON:", stdout, traceback.format_exc())) 

274 raise CollectionError(error) from exception 

275 

276 if "error" in result: 

277 error = result["error"] 

278 if "traceback" in result: 

279 error += f"\n{result['traceback']}" 

280 raise CollectionError(error) 

281 

282 for loading_error in result["loading_errors"]: 282 ↛ 283line 282 didn't jump to line 283, because the loop on line 282 never started

283 logger.warning(loading_error) 

284 

285 for errors in result["parsing_errors"].values(): 285 ↛ 286line 285 didn't jump to line 286, because the loop on line 285 never started

286 for parsing_error in errors: 

287 logger.warning(parsing_error) 

288 

289 # We always collect only one object at a time 

290 result = result["objects"][0] 

291 

292 logger.debug("Rebuilding categories and children lists") 

293 rebuild_category_lists(result) 

294 

295 return result 

296 

297 def teardown(self) -> None: 

298 """Terminate the opened subprocess, set it to `None`.""" 

299 logger.debug("Tearing process down") 

300 self.process.terminate() 

301 

302 def render(self, data: CollectorItem, config: dict) -> str: # noqa: D102 (ignore missing docstring) 

303 final_config = ChainMap(config, self.default_config) 

304 

305 template = self.env.get_template(f"{data['category']}.html") 

306 

307 # Heading level is a "state" variable, that will change at each step 

308 # of the rendering recursion. Therefore, it's easier to use it as a plain value 

309 # than as an item in a dictionary. 

310 heading_level = final_config["heading_level"] 

311 members_order = final_config["members_order"] 

312 

313 if members_order == "alphabetical": 313 ↛ 315line 313 didn't jump to line 315, because the condition on line 313 was never false

314 sort_function = sort_key_alphabetical 

315 elif members_order == "source": 

316 sort_function = sort_key_source 

317 else: 

318 raise PluginError(f"Unknown members_order '{members_order}', choose between 'alphabetical' and 'source'.") 

319 

320 sort_object(data, sort_function=sort_function) 

321 

322 return template.render( 

323 **{"config": final_config, data["category"]: data, "heading_level": heading_level, "root": True}, 

324 ) 

325 

326 def get_anchors(self, data: CollectorItem) -> Sequence[str]: # noqa: D102 (ignore missing docstring) 

327 try: 

328 return (data["path"],) 

329 except KeyError: 

330 return () 

331 

332 def update_env(self, md: Markdown, config: dict) -> None: # noqa: D102 (ignore missing docstring) 

333 super().update_env(md, config) 

334 self.env.trim_blocks = True 

335 self.env.lstrip_blocks = True 

336 self.env.keep_trailing_newline = False 

337 self.env.filters["brief_xref"] = do_brief_xref 

338 

339 

340def get_handler( 

341 theme: str, # noqa: W0613 (unused argument config) 

342 custom_templates: Optional[str] = None, 

343 setup_commands: Optional[List[str]] = None, 

344 config_file_path: str | None = None, 

345 paths: list[str] | None = None, 

346 **config: Any, 

347) -> PythonHandler: 

348 """Simply return an instance of `PythonHandler`. 

349 

350 Arguments: 

351 theme: The theme to use when rendering contents. 

352 custom_templates: Directory containing custom templates. 

353 setup_commands: A list of commands as strings to be executed in the subprocess before `pytkdocs`. 

354 config_file_path: The MkDocs configuration file path. 

355 paths: A list of paths to use as search paths. 

356 config: Configuration passed to the handler. 

357 

358 Returns: 

359 An instance of `PythonHandler`. 

360 """ 

361 return PythonHandler( 

362 handler="python", 

363 theme=theme, 

364 custom_templates=custom_templates, 

365 setup_commands=setup_commands, 

366 config_file_path=config_file_path, 

367 paths=paths, 

368 )