Coverage for src/pytkdocs/parsers/docstrings/google.py: 92.35%

1"""This module defines functions and classes to parse docstrings into structured data.""" 

2import inspect 

3import re 

4from typing import Any, List, Optional, Pattern, Tuple 

5 

6from pytkdocs.parsers.docstrings.base import AnnotatedObject, Attribute, Parameter, Parser, Section, empty 

7 

8SECTIONS_TITLES = { 

9 "args:": Section.Type.PARAMETERS, 

10 "arguments:": Section.Type.PARAMETERS, 

11 "params:": Section.Type.PARAMETERS, 

12 "parameters:": Section.Type.PARAMETERS, 

13 "keyword args:": Section.Type.KEYWORD_ARGS, 

14 "keyword arguments:": Section.Type.KEYWORD_ARGS, 

15 "raise:": Section.Type.EXCEPTIONS, 

16 "raises:": Section.Type.EXCEPTIONS, 

17 "except:": Section.Type.EXCEPTIONS, 

18 "exceptions:": Section.Type.EXCEPTIONS, 

19 "return:": Section.Type.RETURN, 

20 "returns:": Section.Type.RETURN, 

21 "yield:": Section.Type.YIELD, 

22 "yields:": Section.Type.YIELD, 

23 "example:": Section.Type.EXAMPLES, 

24 "examples:": Section.Type.EXAMPLES, 

25 "attribute:": Section.Type.ATTRIBUTES, 

26 "attributes:": Section.Type.ATTRIBUTES, 

27} 

28 

29RE_GOOGLE_STYLE_ADMONITION: Pattern = re.compile(r"^(?P<indent>\s*)(?P<type>[\w-]+):((?:\s+)(?P<title>.+))?$") 

30"""Regular expressions to match lines starting admonitions, of the form `TYPE: [TITLE]`.""" 

31RE_DOCTEST_BLANKLINE: Pattern = re.compile(r"^\s*<BLANKLINE>\s*$") 

32"""Regular expression to match lines of the form `<BLANKLINE>`.""" 

33RE_DOCTEST_FLAGS: Pattern = re.compile(r"(\s*#\s*doctest:.+)$") 

34"""Regular expression to match lines containing doctest flags of the form `# doctest: +FLAG`.""" 

35 

36 

37class Google(Parser): 

38 """A Google-style docstrings parser.""" 

39 

40 def __init__(self, replace_admonitions: bool = True, trim_doctest_flags: bool = True) -> None: 

41 """ 

42 Initialize the object. 

43 

44 Arguments: 

45 replace_admonitions: Whether to replace admonitions by their Markdown equivalent. 

46 trim_doctest_flags: Whether to remove doctest flags. 

47 """ 

48 super().__init__() 

49 self.replace_admonitions = replace_admonitions 

50 self.trim_doctest_flags = trim_doctest_flags 

51 self.section_reader = { 

52 Section.Type.PARAMETERS: self.read_parameters_section, 

53 Section.Type.KEYWORD_ARGS: self.read_keyword_arguments_section, 

54 Section.Type.EXCEPTIONS: self.read_exceptions_section, 

55 Section.Type.EXAMPLES: self.read_examples_section, 

56 Section.Type.ATTRIBUTES: self.read_attributes_section, 

57 Section.Type.RETURN: self.read_return_section, 

58 Section.Type.YIELD: self.read_yield_section, 

59 } 

60 

61 def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102 

62 if "signature" not in self.context: 

63 self.context["signature"] = getattr(self.context["obj"], "signature", None) 

64 if "annotation" not in self.context: 64 ↛ 66line 64 didn't jump to line 66, because the condition on line 64 was never false

65 self.context["annotation"] = getattr(self.context["obj"], "type", empty) 

66 if "attributes" not in self.context: 

67 self.context["attributes"] = {} 

68 

69 sections = [] 

70 current_section = [] 

71 

72 in_code_block = False 

73 

74 lines = docstring.split("\n") 

75 i = 0 

76 

77 while i < len(lines): 

78 line_lower = lines[i].lower() 

79 

80 if in_code_block: 

81 if line_lower.lstrip(" ").startswith("```"): 

82 in_code_block = False 

83 current_section.append(lines[i]) 

84 

85 elif line_lower in SECTIONS_TITLES: 

86 if current_section: 

87 if any(current_section): 87 ↛ 89line 87 didn't jump to line 89, because the condition on line 87 was never false

88 sections.append(Section(Section.Type.MARKDOWN, "\n".join(current_section))) 

89 current_section = [] 

90 section_reader = self.section_reader[SECTIONS_TITLES[line_lower]] 

91 section, i = section_reader(lines, i + 1) 

92 if section: 

93 sections.append(section) 

94 

95 elif line_lower.lstrip(" ").startswith("```"): 

96 in_code_block = True 

97 current_section.append(lines[i]) 

98 

99 else: 

100 if self.replace_admonitions and not in_code_block and i + 1 < len(lines): 

101 match = RE_GOOGLE_STYLE_ADMONITION.match(lines[i]) 

102 if match: 

103 groups = match.groupdict() 

104 indent = groups["indent"] 

105 if lines[i + 1].startswith(indent + " " * 4): 

106 lines[i] = f"{indent}!!! {groups['type'].lower()}" 

107 if groups["title"]: 

108 lines[i] += f' "{groups["title"]}"' 

109 current_section.append(lines[i]) 

110 

111 i += 1 

112 

113 if current_section: 

114 sections.append(Section(Section.Type.MARKDOWN, "\n".join(current_section))) 

115 

116 return sections 

117 

118 def read_block_items(self, lines: List[str], start_index: int) -> Tuple[List[str], int]: 

119 """ 

120 Parse an indented block as a list of items. 

121 

122 The first indentation level is used as a reference to determine if the next lines are new items 

123 or continuation lines. 

124 

125 Arguments: 

126 lines: The block lines. 

127 start_index: The line number to start at. 

128 

129 Returns: 

130 A tuple containing the list of concatenated lines and the index at which to continue parsing. 

131 """ 

132 if start_index >= len(lines): 132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true

133 return [], start_index 

134 

135 i = start_index 

136 items: List[str] = [] 

137 

138 # skip first empty lines 

139 while is_empty_line(lines[i]): 

140 i += 1 

141 

142 # get initial indent 

143 indent = len(lines[i]) - len(lines[i].lstrip()) 

144 

145 if indent == 0: 

146 # first non-empty line was not indented, abort 

147 return [], i - 1 

148 

149 # start processing first item 

150 current_item = [lines[i][indent:]] 

151 i += 1 

152 

153 # loop on next lines 

154 while i < len(lines): 

155 line = lines[i] 

156 

157 if line.startswith(indent * 2 * " "): 

158 # continuation line 

159 current_item.append(line[indent * 2 :]) 

160 

161 elif line.startswith((indent + 1) * " "): 

162 # indent between initial and continuation: append but add error 

163 cont_indent = len(line) - len(line.lstrip()) 

164 current_item.append(line[cont_indent:]) 

165 self.error( 

166 f"Confusing indentation for continuation line {i+1} in docstring, " 

167 f"should be {indent} * 2 = {indent*2} spaces, not {cont_indent}" 

168 ) 

169 

170 elif line.startswith(indent * " "): 

171 # indent equal to initial one: new item 

172 items.append("\n".join(current_item)) 

173 current_item = [line[indent:]] 

174 

175 elif is_empty_line(line): 

176 # empty line: preserve it in the current item 

177 current_item.append("") 

178 

179 else: 

180 # indent lower than initial one: end of section 

181 break 

182 

183 i += 1 

184 

185 if current_item: 185 ↛ 188line 185 didn't jump to line 188, because the condition on line 185 was never false

186 items.append("\n".join(current_item).rstrip("\n")) 

187 

188 return items, i - 1 

189 

190 def read_block(self, lines: List[str], start_index: int) -> Tuple[str, int]: 

191 """ 

192 Parse an indented block. 

193 

194 Arguments: 

195 lines: The block lines. 

196 start_index: The line number to start at. 

197 

198 Returns: 

199 A tuple containing the list of lines and the index at which to continue parsing. 

200 """ 

201 if start_index >= len(lines): 201 ↛ 202line 201 didn't jump to line 202, because the condition on line 201 was never true

202 return "", start_index 

203 

204 i = start_index 

205 block: List[str] = [] 

206 

207 # skip first empty lines 

208 while is_empty_line(lines[i]): 

209 i += 1 

210 

211 # get initial indent 

212 indent = len(lines[i]) - len(lines[i].lstrip()) 

213 

214 if indent == 0: 

215 # first non-empty line was not indented, abort 

216 return "", i - 1 

217 

218 # start processing first item 

219 block.append(lines[i].lstrip()) 

220 i += 1 

221 

222 # loop on next lines 

223 while i < len(lines) and (lines[i].startswith(indent * " ") or is_empty_line(lines[i])): 

224 block.append(lines[i][indent:]) 

225 i += 1 

226 

227 return "\n".join(block).rstrip("\n"), i - 1 

228 

229 def _parse_parameters_section(self, lines: List[str], start_index: int) -> Tuple[List[Parameter], int]: 

230 """ 

231 Parse a "parameters" or "keyword args" section. 

232 

233 Arguments: 

234 lines: The parameters block lines. 

235 start_index: The line number to start at. 

236 

237 Returns: 

238 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

239 """ 

240 parameters = [] 

241 type_: Any 

242 block, i = self.read_block_items(lines, start_index) 

243 

244 for param_line in block: 

245 

246 # Check that there is an annotation in the docstring 

247 try: 

248 name_with_type, description = param_line.split(":", 1) 

249 except ValueError: 

250 self.error(f"Failed to get 'name: description' pair from '{param_line}'") 

251 continue 

252 

253 # Setting defaults 

254 default = empty 

255 annotation = empty 

256 kind = None 

257 # Can only get description from docstring - keep if no type was given 

258 description = description.lstrip() 

259 

260 # If we have managed to find a type in the docstring use this 

261 if " " in name_with_type: 

262 name, type_ = name_with_type.split(" ", 1) 

263 annotation = type_.strip("()") 

264 if annotation.endswith(", optional"): # type: ignore 

265 annotation = annotation[:-10] # type: ignore 

266 # Otherwise try to use the signature as `annotation` would still be empty 

267 else: 

268 name = name_with_type 

269 

270 # Check in the signature to get extra details 

271 try: 

272 signature_param = self.context["signature"].parameters[name.lstrip("*")] 

273 except (AttributeError, KeyError): 

274 if annotation is empty: 

275 self.error(f"No type annotation for parameter '{name}'") 

276 else: 

277 if annotation is empty: 

278 annotation = signature_param.annotation 

279 # If signature_param.X are empty it doesnt matter as defaults are empty anyway 

280 default = signature_param.default 

281 kind = signature_param.kind 

282 

283 parameters.append( 

284 Parameter(name=name, annotation=annotation, description=description, default=default, kind=kind) 

285 ) 

286 

287 return parameters, i 

288 

289 def read_parameters_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]: 

290 """ 

291 Parse a "parameters" section. 

292 

293 Arguments: 

294 lines: The parameters block lines. 

295 start_index: The line number to start at. 

296 

297 Returns: 

298 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

299 """ 

300 parameters, i = self._parse_parameters_section(lines, start_index) 

301 

302 if parameters: 

303 return Section(Section.Type.PARAMETERS, parameters), i 

304 

305 self.error(f"Empty parameters section at line {start_index}") 

306 return None, i 

307 

308 def read_keyword_arguments_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]: 

309 """ 

310 Parse a "keyword arguments" section. 

311 

312 Arguments: 

313 lines: The parameters block lines. 

314 start_index: The line number to start at. 

315 

316 Returns: 

317 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

318 """ 

319 parameters, i = self._parse_parameters_section(lines, start_index) 

320 for parameter in parameters: 

321 parameter.kind = inspect.Parameter.KEYWORD_ONLY 

322 

323 if parameters: 

324 return Section(Section.Type.KEYWORD_ARGS, parameters), i 

325 

326 self.error(f"Empty keyword arguments section at line {start_index}") 

327 return None, i 

328 

329 def read_attributes_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]: 

330 """ 

331 Parse an "attributes" section. 

332 

333 Arguments: 

334 lines: The parameters block lines. 

335 start_index: The line number to start at. 

336 

337 Returns: 

338 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

339 """ 

340 attributes = [] 

341 block, i = self.read_block_items(lines, start_index) 

342 

343 for attr_line in block: 

344 try: 

345 name_with_type, description = attr_line.split(":", 1) 

346 except ValueError: 

347 self.error(f"Failed to get 'name: description' pair from '{attr_line}'") 

348 continue 

349 

350 description = description.lstrip() 

351 

352 if " " in name_with_type: 

353 name, annotation = name_with_type.split(" ", 1) 

354 annotation = annotation.strip("()") 

355 if annotation.endswith(", optional"): 355 ↛ 356line 355 didn't jump to line 356, because the condition on line 355 was never true

356 annotation = annotation[:-10] 

357 else: 

358 name = name_with_type 

359 annotation = self.context["attributes"].get(name, {}).get("annotation", empty) 

360 

361 attributes.append(Attribute(name=name, annotation=annotation, description=description)) 

362 

363 if attributes: 363 ↛ 366line 363 didn't jump to line 366, because the condition on line 363 was never false

364 return Section(Section.Type.ATTRIBUTES, attributes), i 

365 

366 self.error(f"Empty attributes section at line {start_index}") 

367 return None, i 

368 

369 def read_exceptions_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]: 

370 """ 

371 Parse an "exceptions" section. 

372 

373 Arguments: 

374 lines: The exceptions block lines. 

375 start_index: The line number to start at. 

376 

377 Returns: 

378 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

379 """ 

380 exceptions = [] 

381 block, i = self.read_block_items(lines, start_index) 

382 

383 for exception_line in block: 

384 try: 

385 annotation, description = exception_line.split(": ", 1) 

386 except ValueError: 

387 self.error(f"Failed to get 'exception: description' pair from '{exception_line}'") 

388 else: 

389 exceptions.append(AnnotatedObject(annotation, description.lstrip(" "))) 

390 

391 if exceptions: 

392 return Section(Section.Type.EXCEPTIONS, exceptions), i 

393 

394 self.error(f"Empty exceptions section at line {start_index}") 

395 return None, i 

396 

397 def read_return_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]: 

398 """ 

399 Parse an "returns" section. 

400 

401 Arguments: 

402 lines: The return block lines. 

403 start_index: The line number to start at. 

404 

405 Returns: 

406 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

407 """ 

408 text, i = self.read_block(lines, start_index) 

409 

410 # Early exit if there is no text in the return section 

411 if not text: 

412 self.error(f"Empty return section at line {start_index}") 

413 return None, i 

414 

415 # First try to get the annotation and description from the docstring 

416 try: 

417 type_, text = text.split(":", 1) 

418 except ValueError: 

419 description = text 

420 annotation = self.context["annotation"] 

421 # If there was no annotation in the docstring then move to signature 

422 if annotation is empty and self.context["signature"]: 

423 annotation = self.context["signature"].return_annotation 

424 else: 

425 annotation = type_.lstrip() 

426 description = text.lstrip() 

427 

428 # There was no type in the docstring and no annotation 

429 if annotation is empty: 

430 self.error("No return type/annotation in docstring/signature") 

431 

432 return Section(Section.Type.RETURN, AnnotatedObject(annotation, description)), i 

433 

434 def read_yield_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]: 

435 """ 

436 Parse a "yields" section. 

437 

438 Arguments: 

439 lines: The return block lines. 

440 start_index: The line number to start at. 

441 

442 Returns: 

443 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

444 """ 

445 text, i = self.read_block(lines, start_index) 

446 

447 # Early exit if there is no text in the yield section 

448 if not text: 448 ↛ 449line 448 didn't jump to line 449, because the condition on line 448 was never true

449 self.error(f"Empty yield section at line {start_index}") 

450 return None, i 

451 

452 # First try to get the annotation and description from the docstring 

453 try: 

454 type_, text = text.split(":", 1) 

455 except ValueError: 

456 description = text 

457 annotation = self.context["annotation"] 

458 # If there was no annotation in the docstring then move to signature 

459 if annotation is empty and self.context["signature"]: 459 ↛ 466line 459 didn't jump to line 466, because the condition on line 459 was never false

460 annotation = self.context["signature"].return_annotation 

461 else: 

462 annotation = type_.lstrip() 

463 description = text.lstrip() 

464 

465 # There was no type in the docstring and no annotation 

466 if annotation is empty: 466 ↛ 467line 466 didn't jump to line 467, because the condition on line 466 was never true

467 self.error("No yield type/annotation in docstring/signature") 

468 

469 return Section(Section.Type.YIELD, AnnotatedObject(annotation, description)), i 

470 

471 def read_examples_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]: 

472 """ 

473 Parse an "examples" section. 

474 

475 Arguments: 

476 lines: The examples block lines. 

477 start_index: The line number to start at. 

478 

479 Returns: 

480 A tuple containing a `Section` (or `None`) and the index at which to continue parsing. 

481 """ 

482 text, i = self.read_block(lines, start_index) 

483 

484 sub_sections = [] 

485 in_code_example = False 

486 in_code_block = False 

487 current_text: List[str] = [] 

488 current_example: List[str] = [] 

489 

490 for line in text.split("\n"): 

491 if is_empty_line(line): 

492 if in_code_example: 

493 if current_example: 493 ↛ 496line 493 didn't jump to line 496, because the condition on line 493 was never false

494 sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example))) 

495 current_example = [] 

496 in_code_example = False 

497 else: 

498 current_text.append(line) 

499 

500 elif in_code_example: 

501 if self.trim_doctest_flags: 

502 line = RE_DOCTEST_FLAGS.sub("", line) 

503 line = RE_DOCTEST_BLANKLINE.sub("", line) 

504 current_example.append(line) 

505 

506 elif line.startswith("```"): 

507 in_code_block = not in_code_block 

508 current_text.append(line) 

509 

510 elif in_code_block: 

511 current_text.append(line) 

512 

513 elif line.startswith(">>>"): 

514 if current_text: 

515 sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text))) 

516 current_text = [] 

517 in_code_example = True 

518 

519 if self.trim_doctest_flags: 

520 line = RE_DOCTEST_FLAGS.sub("", line) 

521 current_example.append(line) 

522 

523 else: 

524 current_text.append(line) 

525 

526 if current_text: 526 ↛ 527line 526 didn't jump to line 527, because the condition on line 526 was never true

527 sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text))) 

528 elif current_example: 528 ↛ 531line 528 didn't jump to line 531, because the condition on line 528 was never false

529 sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example))) 

530 

531 if sub_sections: 531 ↛ 534line 531 didn't jump to line 534, because the condition on line 531 was never false

532 return Section(Section.Type.EXAMPLES, sub_sections), i 

533 

534 self.error(f"Empty examples section at line {start_index}") 

535 return None, i 

536 

537 

538def is_empty_line(line) -> bool: 

539 """ 

540 Tell if a line is empty. 

541 

542 Arguments: 

543 line: The line to check. 

544 

545 Returns: 

546 True if the line is empty or composed of blanks only, False otherwise. 

547 """ 

548 return not line.strip()