parsers
The docstrings parsers' package.
attributes
¤
Module containing functions to parse attributes in the source code.
docstrings
special
¤
The parsers' package.
base
¤
The base module for docstring parsing.
AnnotatedObject
¤
A helper class to store information about an annotated object.
Source code in pytkdocs/parsers/docstrings/base.py
class AnnotatedObject:
"""A helper class to store information about an annotated object."""
def __init__(self, annotation: Any, description: str) -> None:
"""
Initialize the object.
Arguments:
annotation: The object's annotation.
description: The object's description.
"""
self.annotation = annotation
self.description = description
__init__(self, annotation, description)
special
¤
Initialize the object.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
annotation |
Any |
The object's annotation. |
required |
description |
str |
The object's description. |
required |
Source code in pytkdocs/parsers/docstrings/base.py
def __init__(self, annotation: Any, description: str) -> None:
"""
Initialize the object.
Arguments:
annotation: The object's annotation.
description: The object's description.
"""
self.annotation = annotation
self.description = description
Attribute (AnnotatedObject)
¤
A helper class to store information about a documented attribute.
Source code in pytkdocs/parsers/docstrings/base.py
class Attribute(AnnotatedObject):
"""A helper class to store information about a documented attribute."""
def __init__(self, name: str, annotation: Any, description: str) -> None:
"""
Initialize the object.
Arguments:
name: The attribute's name.
annotation: The object's annotation.
description: The object's description.
"""
super().__init__(annotation, description)
self.name = name
__init__(self, name, annotation, description)
special
¤
Initialize the object.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
name |
str |
The attribute's name. |
required |
annotation |
Any |
The object's annotation. |
required |
description |
str |
The object's description. |
required |
Source code in pytkdocs/parsers/docstrings/base.py
def __init__(self, name: str, annotation: Any, description: str) -> None:
"""
Initialize the object.
Arguments:
name: The attribute's name.
annotation: The object's annotation.
description: The object's description.
"""
super().__init__(annotation, description)
self.name = name
Parameter (AnnotatedObject)
¤
A helper class to store information about a signature parameter.
Source code in pytkdocs/parsers/docstrings/base.py
class Parameter(AnnotatedObject):
"""A helper class to store information about a signature parameter."""
def __init__(self, name: str, annotation: Any, description: str, kind: Any, default: Any = empty) -> None:
"""
Initialize the object.
Arguments:
name: The parameter's name.
annotation: The parameter's annotation.
description: The parameter's description.
kind: The parameter's kind (positional only, keyword only, etc.).
default: The parameter's default value.
"""
super().__init__(annotation, description)
self.name = name
self.kind = kind
self.default = default
def __str__(self):
return self.name
def __repr__(self):
return f"<Parameter({self.name}, {self.annotation}, {self.description}, {self.kind}, {self.default})>"
@property
def is_optional(self):
"""Tell if this parameter is optional."""
return self.default is not empty
@property
def is_required(self):
"""Tell if this parameter is required."""
return not self.is_optional
@property
def is_args(self):
"""Tell if this parameter is positional."""
return self.kind is inspect.Parameter.VAR_POSITIONAL
@property
def is_kwargs(self):
"""Tell if this parameter is a keyword."""
return self.kind is inspect.Parameter.VAR_KEYWORD
@property
def default_string(self):
"""Return the default value as a string."""
if self.is_kwargs:
return "{}"
if self.is_args:
return "()"
if self.is_required:
return ""
return repr(self.default)
default_string
property
readonly
¤
Return the default value as a string.
is_args
property
readonly
¤
Tell if this parameter is positional.
is_kwargs
property
readonly
¤
Tell if this parameter is a keyword.
is_optional
property
readonly
¤
Tell if this parameter is optional.
is_required
property
readonly
¤
Tell if this parameter is required.
__init__(self, name, annotation, description, kind, default)
special
¤
Initialize the object.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
name |
str |
The parameter's name. |
required |
annotation |
Any |
The parameter's annotation. |
required |
description |
str |
The parameter's description. |
required |
kind |
Any |
The parameter's kind (positional only, keyword only, etc.). |
required |
default |
Any |
The parameter's default value. |
required |
Source code in pytkdocs/parsers/docstrings/base.py
def __init__(self, name: str, annotation: Any, description: str, kind: Any, default: Any = empty) -> None:
"""
Initialize the object.
Arguments:
name: The parameter's name.
annotation: The parameter's annotation.
description: The parameter's description.
kind: The parameter's kind (positional only, keyword only, etc.).
default: The parameter's default value.
"""
super().__init__(annotation, description)
self.name = name
self.kind = kind
self.default = default
Parser
¤
A class to parse docstrings.
It is instantiated with an object's path, docstring, signature and return type.
The parse
method then returns structured data,
in the form of a list of Section
s.
It also return the list of errors that occurred during parsing.
Source code in pytkdocs/parsers/docstrings/base.py
class Parser(metaclass=ABCMeta):
"""
A class to parse docstrings.
It is instantiated with an object's path, docstring, signature and return type.
The `parse` method then returns structured data,
in the form of a list of [`Section`][pytkdocs.parsers.docstrings.base.Section]s.
It also return the list of errors that occurred during parsing.
"""
def __init__(self) -> None:
"""Initialize the object."""
self.context: dict = {}
self.errors: List[str] = []
def parse(self, docstring: str, context: Optional[dict] = None) -> Tuple[List[Section], List[str]]:
"""
Parse a docstring and return a list of sections and parsing errors.
Arguments:
docstring: The docstring to parse.
context: Some context helping to parse the docstring.
Returns:
A tuple containing the list of sections and the parsing errors.
"""
self.context = context or {}
self.errors = []
sections = self.parse_sections(docstring)
errors = self.errors
return sections, errors
def error(self, message) -> None:
"""
Record a parsing error.
Arguments:
message: A message described the error.
"""
if self.context["obj"]:
message = f"{self.context['obj'].path}: {message}"
self.errors.append(message)
@abstractmethod
def parse_sections(self, docstring: str) -> List[Section]:
"""
Parse a docstring as a list of sections.
Arguments:
docstring: The docstring to parse.
Returns:
A list of [`Section`][pytkdocs.parsers.docstrings.base.Section]s.
"""
raise NotImplementedError
__init__(self)
special
¤
Initialize the object.
Source code in pytkdocs/parsers/docstrings/base.py
def __init__(self) -> None:
"""Initialize the object."""
self.context: dict = {}
self.errors: List[str] = []
error(self, message)
¤
Record a parsing error.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
message |
A message described the error. |
required |
Source code in pytkdocs/parsers/docstrings/base.py
def error(self, message) -> None:
"""
Record a parsing error.
Arguments:
message: A message described the error.
"""
if self.context["obj"]:
message = f"{self.context['obj'].path}: {message}"
self.errors.append(message)
parse(self, docstring, context=None)
¤
Parse a docstring and return a list of sections and parsing errors.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The docstring to parse. |
required |
context |
Optional[dict] |
Some context helping to parse the docstring. |
None |
Returns:
Type | Description |
---|---|
Tuple[List[pytkdocs.parsers.docstrings.base.Section], List[str]] |
A tuple containing the list of sections and the parsing errors. |
Source code in pytkdocs/parsers/docstrings/base.py
def parse(self, docstring: str, context: Optional[dict] = None) -> Tuple[List[Section], List[str]]:
"""
Parse a docstring and return a list of sections and parsing errors.
Arguments:
docstring: The docstring to parse.
context: Some context helping to parse the docstring.
Returns:
A tuple containing the list of sections and the parsing errors.
"""
self.context = context or {}
self.errors = []
sections = self.parse_sections(docstring)
errors = self.errors
return sections, errors
parse_sections(self, docstring)
¤
Parse a docstring as a list of sections.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The docstring to parse. |
required |
Returns:
Type | Description |
---|---|
List[pytkdocs.parsers.docstrings.base.Section] |
A list of |
Source code in pytkdocs/parsers/docstrings/base.py
@abstractmethod
def parse_sections(self, docstring: str) -> List[Section]:
"""
Parse a docstring as a list of sections.
Arguments:
docstring: The docstring to parse.
Returns:
A list of [`Section`][pytkdocs.parsers.docstrings.base.Section]s.
"""
raise NotImplementedError
Section
¤
A helper class to store a docstring section.
Source code in pytkdocs/parsers/docstrings/base.py
class Section:
"""A helper class to store a docstring section."""
class Type:
"""The possible section types."""
MARKDOWN = "markdown"
PARAMETERS = "parameters"
EXCEPTIONS = "exceptions"
RETURN = "return"
YIELD = "yield"
EXAMPLES = "examples"
ATTRIBUTES = "attributes"
KEYWORD_ARGS = "keyword_args"
def __init__(self, section_type: str, value: Any) -> None:
"""
Initialize the object.
Arguments:
section_type: The type of the section, from the [`Type`][pytkdocs.parsers.docstrings.base.Section.Type] enum.
value: The section value.
"""
self.type = section_type
self.value = value
def __str__(self):
return self.type
def __repr__(self):
return f"<Section(type={self.type!r})>"
Type
¤
The possible section types.
Source code in pytkdocs/parsers/docstrings/base.py
class Type:
"""The possible section types."""
MARKDOWN = "markdown"
PARAMETERS = "parameters"
EXCEPTIONS = "exceptions"
RETURN = "return"
YIELD = "yield"
EXAMPLES = "examples"
ATTRIBUTES = "attributes"
KEYWORD_ARGS = "keyword_args"
__init__(self, section_type, value)
special
¤
Initialize the object.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
section_type |
str |
The type of the section, from the |
required |
value |
Any |
The section value. |
required |
Source code in pytkdocs/parsers/docstrings/base.py
def __init__(self, section_type: str, value: Any) -> None:
"""
Initialize the object.
Arguments:
section_type: The type of the section, from the [`Type`][pytkdocs.parsers.docstrings.base.Section.Type] enum.
value: The section value.
"""
self.type = section_type
self.value = value
google
¤
This module defines functions and classes to parse docstrings into structured data.
RE_DOCTEST_BLANKLINE: Pattern
¤
Regular expression to match lines of the form <BLANKLINE>
.
RE_DOCTEST_FLAGS: Pattern
¤
Regular expression to match lines containing doctest flags of the form # doctest: +FLAG
.
RE_GOOGLE_STYLE_ADMONITION: Pattern
¤
Regular expressions to match lines starting admonitions, of the form TYPE: [TITLE]
.
Google (Parser)
¤
A Google-style docstrings parser.
Source code in pytkdocs/parsers/docstrings/google.py
class Google(Parser):
"""A Google-style docstrings parser."""
def __init__(self, replace_admonitions: bool = True, trim_doctest_flags: bool = True) -> None:
"""
Initialize the object.
Arguments:
replace_admonitions: Whether to replace admonitions by their Markdown equivalent.
trim_doctest_flags: Whether to remove doctest flags.
"""
super().__init__()
self.replace_admonitions = replace_admonitions
self.trim_doctest_flags = trim_doctest_flags
self.section_reader = {
Section.Type.PARAMETERS: self.read_parameters_section,
Section.Type.KEYWORD_ARGS: self.read_keyword_arguments_section,
Section.Type.EXCEPTIONS: self.read_exceptions_section,
Section.Type.EXAMPLES: self.read_examples_section,
Section.Type.ATTRIBUTES: self.read_attributes_section,
Section.Type.RETURN: self.read_return_section,
Section.Type.YIELD: self.read_yield_section,
}
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
if "signature" not in self.context:
self.context["signature"] = getattr(self.context["obj"], "signature", None)
if "annotation" not in self.context:
self.context["annotation"] = getattr(self.context["obj"], "type", empty)
if "attributes" not in self.context:
self.context["attributes"] = {}
sections = []
current_section = []
in_code_block = False
lines = docstring.split("\n")
i = 0
while i < len(lines):
line_lower = lines[i].lower()
if in_code_block:
if line_lower.lstrip(" ").startswith("```"):
in_code_block = False
current_section.append(lines[i])
elif line_lower in SECTIONS_TITLES:
if current_section:
if any(current_section):
sections.append(Section(Section.Type.MARKDOWN, "\n".join(current_section)))
current_section = []
section_reader = self.section_reader[SECTIONS_TITLES[line_lower]]
section, i = section_reader(lines, i + 1)
if section:
sections.append(section)
elif line_lower.lstrip(" ").startswith("```"):
in_code_block = True
current_section.append(lines[i])
else:
if self.replace_admonitions and not in_code_block and i + 1 < len(lines):
match = RE_GOOGLE_STYLE_ADMONITION.match(lines[i])
if match:
groups = match.groupdict()
indent = groups["indent"]
if lines[i + 1].startswith(indent + " " * 4):
lines[i] = f"{indent}!!! {groups['type'].lower()}"
if groups["title"]:
lines[i] += f' "{groups["title"]}"'
current_section.append(lines[i])
i += 1
if current_section:
sections.append(Section(Section.Type.MARKDOWN, "\n".join(current_section)))
return sections
def read_block_items(self, lines: List[str], start_index: int) -> Tuple[List[str], int]:
"""
Parse an indented block as a list of items.
The first indentation level is used as a reference to determine if the next lines are new items
or continuation lines.
Arguments:
lines: The block lines.
start_index: The line number to start at.
Returns:
A tuple containing the list of concatenated lines and the index at which to continue parsing.
"""
if start_index >= len(lines):
return [], start_index
i = start_index
items: List[str] = []
# skip first empty lines
while is_empty_line(lines[i]):
i += 1
# get initial indent
indent = len(lines[i]) - len(lines[i].lstrip())
if indent == 0:
# first non-empty line was not indented, abort
return [], i - 1
# start processing first item
current_item = [lines[i][indent:]]
i += 1
# loop on next lines
while i < len(lines):
line = lines[i]
if line.startswith(indent * 2 * " "):
# continuation line
current_item.append(line[indent * 2 :])
elif line.startswith((indent + 1) * " "):
# indent between initial and continuation: append but add error
cont_indent = len(line) - len(line.lstrip())
current_item.append(line[cont_indent:])
self.error(
f"Confusing indentation for continuation line {i+1} in docstring, "
f"should be {indent} * 2 = {indent*2} spaces, not {cont_indent}"
)
elif line.startswith(indent * " "):
# indent equal to initial one: new item
items.append("\n".join(current_item))
current_item = [line[indent:]]
elif is_empty_line(line):
# empty line: preserve it in the current item
current_item.append("")
else:
# indent lower than initial one: end of section
break
i += 1
if current_item:
items.append("\n".join(current_item).rstrip("\n"))
return items, i - 1
def read_block(self, lines: List[str], start_index: int) -> Tuple[str, int]:
"""
Parse an indented block.
Arguments:
lines: The block lines.
start_index: The line number to start at.
Returns:
A tuple containing the list of lines and the index at which to continue parsing.
"""
if start_index >= len(lines):
return "", start_index
i = start_index
block: List[str] = []
# skip first empty lines
while is_empty_line(lines[i]):
i += 1
# get initial indent
indent = len(lines[i]) - len(lines[i].lstrip())
if indent == 0:
# first non-empty line was not indented, abort
return "", i - 1
# start processing first item
block.append(lines[i].lstrip())
i += 1
# loop on next lines
while i < len(lines) and (lines[i].startswith(indent * " ") or is_empty_line(lines[i])):
block.append(lines[i][indent:])
i += 1
return "\n".join(block).rstrip("\n"), i - 1
def _parse_parameters_section(self, lines: List[str], start_index: int) -> Tuple[List[Parameter], int]:
"""
Parse a "parameters" or "keyword args" section.
Arguments:
lines: The parameters block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
parameters = []
type_: Any
block, i = self.read_block_items(lines, start_index)
for param_line in block:
# Check that there is an annotation in the docstring
try:
name_with_type, description = param_line.split(":", 1)
except ValueError:
self.error(f"Failed to get 'name: description' pair from '{param_line}'")
continue
# Setting defaults
default = empty
annotation = empty
kind = None
# Can only get description from docstring - keep if no type was given
description = description.lstrip()
# If we have managed to find a type in the docstring use this
if " " in name_with_type:
name, type_ = name_with_type.split(" ", 1)
annotation = type_.strip("()")
if annotation.endswith(", optional"): # type: ignore
annotation = annotation[:-10] # type: ignore
# Otherwise try to use the signature as `annotation` would still be empty
else:
name = name_with_type
# Check in the signature to get extra details
try:
signature_param = self.context["signature"].parameters[name.lstrip("*")]
except (AttributeError, KeyError):
if annotation is empty:
self.error(f"No type annotation for parameter '{name}'")
else:
if annotation is empty:
annotation = signature_param.annotation
# If signature_param.X are empty it doesnt matter as defaults are empty anyway
default = signature_param.default
kind = signature_param.kind
parameters.append(
Parameter(name=name, annotation=annotation, description=description, default=default, kind=kind)
)
return parameters, i
def read_parameters_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse a "parameters" section.
Arguments:
lines: The parameters block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
parameters, i = self._parse_parameters_section(lines, start_index)
if parameters:
return Section(Section.Type.PARAMETERS, parameters), i
self.error(f"Empty parameters section at line {start_index}")
return None, i
def read_keyword_arguments_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse a "keyword arguments" section.
Arguments:
lines: The parameters block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
parameters, i = self._parse_parameters_section(lines, start_index)
for parameter in parameters:
parameter.kind = inspect.Parameter.KEYWORD_ONLY
if parameters:
return Section(Section.Type.KEYWORD_ARGS, parameters), i
self.error(f"Empty keyword arguments section at line {start_index}")
return None, i
def read_attributes_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "attributes" section.
Arguments:
lines: The parameters block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
attributes = []
block, i = self.read_block_items(lines, start_index)
for attr_line in block:
try:
name_with_type, description = attr_line.split(":", 1)
except ValueError:
self.error(f"Failed to get 'name: description' pair from '{attr_line}'")
continue
description = description.lstrip()
if " " in name_with_type:
name, annotation = name_with_type.split(" ", 1)
annotation = annotation.strip("()")
if annotation.endswith(", optional"):
annotation = annotation[:-10]
else:
name = name_with_type
annotation = self.context["attributes"].get(name, {}).get("annotation", empty)
attributes.append(Attribute(name=name, annotation=annotation, description=description))
if attributes:
return Section(Section.Type.ATTRIBUTES, attributes), i
self.error(f"Empty attributes section at line {start_index}")
return None, i
def read_exceptions_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "exceptions" section.
Arguments:
lines: The exceptions block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
exceptions = []
block, i = self.read_block_items(lines, start_index)
for exception_line in block:
try:
annotation, description = exception_line.split(": ", 1)
except ValueError:
self.error(f"Failed to get 'exception: description' pair from '{exception_line}'")
else:
exceptions.append(AnnotatedObject(annotation, description.lstrip(" ")))
if exceptions:
return Section(Section.Type.EXCEPTIONS, exceptions), i
self.error(f"Empty exceptions section at line {start_index}")
return None, i
def read_return_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "returns" section.
Arguments:
lines: The return block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
text, i = self.read_block(lines, start_index)
# Early exit if there is no text in the return section
if not text:
self.error(f"Empty return section at line {start_index}")
return None, i
# First try to get the annotation and description from the docstring
try:
type_, text = text.split(":", 1)
except ValueError:
description = text
annotation = self.context["annotation"]
# If there was no annotation in the docstring then move to signature
if annotation is empty and self.context["signature"]:
annotation = self.context["signature"].return_annotation
else:
annotation = type_.lstrip()
description = text.lstrip()
# There was no type in the docstring and no annotation
if annotation is empty:
self.error("No return type/annotation in docstring/signature")
return Section(Section.Type.RETURN, AnnotatedObject(annotation, description)), i
def read_yield_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse a "yields" section.
Arguments:
lines: The return block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
text, i = self.read_block(lines, start_index)
# Early exit if there is no text in the yield section
if not text:
self.error(f"Empty yield section at line {start_index}")
return None, i
# First try to get the annotation and description from the docstring
try:
type_, text = text.split(":", 1)
except ValueError:
description = text
annotation = self.context["annotation"]
# If there was no annotation in the docstring then move to signature
if annotation is empty and self.context["signature"]:
annotation = self.context["signature"].return_annotation
else:
annotation = type_.lstrip()
description = text.lstrip()
# There was no type in the docstring and no annotation
if annotation is empty:
self.error("No yield type/annotation in docstring/signature")
return Section(Section.Type.YIELD, AnnotatedObject(annotation, description)), i
def read_examples_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "examples" section.
Arguments:
lines: The examples block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
text, i = self.read_block(lines, start_index)
sub_sections = []
in_code_example = False
in_code_block = False
current_text: List[str] = []
current_example: List[str] = []
for line in text.split("\n"):
if is_empty_line(line):
if in_code_example:
if current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
current_example = []
in_code_example = False
else:
current_text.append(line)
elif in_code_example:
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
line = RE_DOCTEST_BLANKLINE.sub("", line)
current_example.append(line)
elif line.startswith("```"):
in_code_block = not in_code_block
current_text.append(line)
elif in_code_block:
current_text.append(line)
elif line.startswith(">>>"):
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
current_text = []
in_code_example = True
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
current_example.append(line)
else:
current_text.append(line)
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
elif current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
if sub_sections:
return Section(Section.Type.EXAMPLES, sub_sections), i
self.error(f"Empty examples section at line {start_index}")
return None, i
__init__(self, replace_admonitions=True, trim_doctest_flags=True)
special
¤
Initialize the object.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
replace_admonitions |
bool |
Whether to replace admonitions by their Markdown equivalent. |
True |
trim_doctest_flags |
bool |
Whether to remove doctest flags. |
True |
Source code in pytkdocs/parsers/docstrings/google.py
def __init__(self, replace_admonitions: bool = True, trim_doctest_flags: bool = True) -> None:
"""
Initialize the object.
Arguments:
replace_admonitions: Whether to replace admonitions by their Markdown equivalent.
trim_doctest_flags: Whether to remove doctest flags.
"""
super().__init__()
self.replace_admonitions = replace_admonitions
self.trim_doctest_flags = trim_doctest_flags
self.section_reader = {
Section.Type.PARAMETERS: self.read_parameters_section,
Section.Type.KEYWORD_ARGS: self.read_keyword_arguments_section,
Section.Type.EXCEPTIONS: self.read_exceptions_section,
Section.Type.EXAMPLES: self.read_examples_section,
Section.Type.ATTRIBUTES: self.read_attributes_section,
Section.Type.RETURN: self.read_return_section,
Section.Type.YIELD: self.read_yield_section,
}
parse_sections(self, docstring)
¤
Parse a docstring as a list of sections.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The docstring to parse. |
required |
Returns:
Type | Description |
---|---|
List[pytkdocs.parsers.docstrings.base.Section] |
A list of |
Source code in pytkdocs/parsers/docstrings/google.py
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
if "signature" not in self.context:
self.context["signature"] = getattr(self.context["obj"], "signature", None)
if "annotation" not in self.context:
self.context["annotation"] = getattr(self.context["obj"], "type", empty)
if "attributes" not in self.context:
self.context["attributes"] = {}
sections = []
current_section = []
in_code_block = False
lines = docstring.split("\n")
i = 0
while i < len(lines):
line_lower = lines[i].lower()
if in_code_block:
if line_lower.lstrip(" ").startswith("```"):
in_code_block = False
current_section.append(lines[i])
elif line_lower in SECTIONS_TITLES:
if current_section:
if any(current_section):
sections.append(Section(Section.Type.MARKDOWN, "\n".join(current_section)))
current_section = []
section_reader = self.section_reader[SECTIONS_TITLES[line_lower]]
section, i = section_reader(lines, i + 1)
if section:
sections.append(section)
elif line_lower.lstrip(" ").startswith("```"):
in_code_block = True
current_section.append(lines[i])
else:
if self.replace_admonitions and not in_code_block and i + 1 < len(lines):
match = RE_GOOGLE_STYLE_ADMONITION.match(lines[i])
if match:
groups = match.groupdict()
indent = groups["indent"]
if lines[i + 1].startswith(indent + " " * 4):
lines[i] = f"{indent}!!! {groups['type'].lower()}"
if groups["title"]:
lines[i] += f' "{groups["title"]}"'
current_section.append(lines[i])
i += 1
if current_section:
sections.append(Section(Section.Type.MARKDOWN, "\n".join(current_section)))
return sections
read_attributes_section(self, lines, start_index)
¤
Parse an "attributes" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The parameters block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[Optional[pytkdocs.parsers.docstrings.base.Section], int] |
A tuple containing a |
Source code in pytkdocs/parsers/docstrings/google.py
def read_attributes_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "attributes" section.
Arguments:
lines: The parameters block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
attributes = []
block, i = self.read_block_items(lines, start_index)
for attr_line in block:
try:
name_with_type, description = attr_line.split(":", 1)
except ValueError:
self.error(f"Failed to get 'name: description' pair from '{attr_line}'")
continue
description = description.lstrip()
if " " in name_with_type:
name, annotation = name_with_type.split(" ", 1)
annotation = annotation.strip("()")
if annotation.endswith(", optional"):
annotation = annotation[:-10]
else:
name = name_with_type
annotation = self.context["attributes"].get(name, {}).get("annotation", empty)
attributes.append(Attribute(name=name, annotation=annotation, description=description))
if attributes:
return Section(Section.Type.ATTRIBUTES, attributes), i
self.error(f"Empty attributes section at line {start_index}")
return None, i
read_block(self, lines, start_index)
¤
Parse an indented block.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[str, int] |
A tuple containing the list of lines and the index at which to continue parsing. |
Source code in pytkdocs/parsers/docstrings/google.py
def read_block(self, lines: List[str], start_index: int) -> Tuple[str, int]:
"""
Parse an indented block.
Arguments:
lines: The block lines.
start_index: The line number to start at.
Returns:
A tuple containing the list of lines and the index at which to continue parsing.
"""
if start_index >= len(lines):
return "", start_index
i = start_index
block: List[str] = []
# skip first empty lines
while is_empty_line(lines[i]):
i += 1
# get initial indent
indent = len(lines[i]) - len(lines[i].lstrip())
if indent == 0:
# first non-empty line was not indented, abort
return "", i - 1
# start processing first item
block.append(lines[i].lstrip())
i += 1
# loop on next lines
while i < len(lines) and (lines[i].startswith(indent * " ") or is_empty_line(lines[i])):
block.append(lines[i][indent:])
i += 1
return "\n".join(block).rstrip("\n"), i - 1
read_block_items(self, lines, start_index)
¤
Parse an indented block as a list of items.
The first indentation level is used as a reference to determine if the next lines are new items or continuation lines.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[List[str], int] |
A tuple containing the list of concatenated lines and the index at which to continue parsing. |
Source code in pytkdocs/parsers/docstrings/google.py
def read_block_items(self, lines: List[str], start_index: int) -> Tuple[List[str], int]:
"""
Parse an indented block as a list of items.
The first indentation level is used as a reference to determine if the next lines are new items
or continuation lines.
Arguments:
lines: The block lines.
start_index: The line number to start at.
Returns:
A tuple containing the list of concatenated lines and the index at which to continue parsing.
"""
if start_index >= len(lines):
return [], start_index
i = start_index
items: List[str] = []
# skip first empty lines
while is_empty_line(lines[i]):
i += 1
# get initial indent
indent = len(lines[i]) - len(lines[i].lstrip())
if indent == 0:
# first non-empty line was not indented, abort
return [], i - 1
# start processing first item
current_item = [lines[i][indent:]]
i += 1
# loop on next lines
while i < len(lines):
line = lines[i]
if line.startswith(indent * 2 * " "):
# continuation line
current_item.append(line[indent * 2 :])
elif line.startswith((indent + 1) * " "):
# indent between initial and continuation: append but add error
cont_indent = len(line) - len(line.lstrip())
current_item.append(line[cont_indent:])
self.error(
f"Confusing indentation for continuation line {i+1} in docstring, "
f"should be {indent} * 2 = {indent*2} spaces, not {cont_indent}"
)
elif line.startswith(indent * " "):
# indent equal to initial one: new item
items.append("\n".join(current_item))
current_item = [line[indent:]]
elif is_empty_line(line):
# empty line: preserve it in the current item
current_item.append("")
else:
# indent lower than initial one: end of section
break
i += 1
if current_item:
items.append("\n".join(current_item).rstrip("\n"))
return items, i - 1
read_examples_section(self, lines, start_index)
¤
Parse an "examples" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The examples block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[Optional[pytkdocs.parsers.docstrings.base.Section], int] |
A tuple containing a |
Source code in pytkdocs/parsers/docstrings/google.py
def read_examples_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "examples" section.
Arguments:
lines: The examples block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
text, i = self.read_block(lines, start_index)
sub_sections = []
in_code_example = False
in_code_block = False
current_text: List[str] = []
current_example: List[str] = []
for line in text.split("\n"):
if is_empty_line(line):
if in_code_example:
if current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
current_example = []
in_code_example = False
else:
current_text.append(line)
elif in_code_example:
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
line = RE_DOCTEST_BLANKLINE.sub("", line)
current_example.append(line)
elif line.startswith("```"):
in_code_block = not in_code_block
current_text.append(line)
elif in_code_block:
current_text.append(line)
elif line.startswith(">>>"):
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
current_text = []
in_code_example = True
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
current_example.append(line)
else:
current_text.append(line)
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
elif current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
if sub_sections:
return Section(Section.Type.EXAMPLES, sub_sections), i
self.error(f"Empty examples section at line {start_index}")
return None, i
read_exceptions_section(self, lines, start_index)
¤
Parse an "exceptions" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The exceptions block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[Optional[pytkdocs.parsers.docstrings.base.Section], int] |
A tuple containing a |
Source code in pytkdocs/parsers/docstrings/google.py
def read_exceptions_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "exceptions" section.
Arguments:
lines: The exceptions block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
exceptions = []
block, i = self.read_block_items(lines, start_index)
for exception_line in block:
try:
annotation, description = exception_line.split(": ", 1)
except ValueError:
self.error(f"Failed to get 'exception: description' pair from '{exception_line}'")
else:
exceptions.append(AnnotatedObject(annotation, description.lstrip(" ")))
if exceptions:
return Section(Section.Type.EXCEPTIONS, exceptions), i
self.error(f"Empty exceptions section at line {start_index}")
return None, i
read_keyword_arguments_section(self, lines, start_index)
¤
Parse a "keyword arguments" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The parameters block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[Optional[pytkdocs.parsers.docstrings.base.Section], int] |
A tuple containing a |
Source code in pytkdocs/parsers/docstrings/google.py
def read_keyword_arguments_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse a "keyword arguments" section.
Arguments:
lines: The parameters block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
parameters, i = self._parse_parameters_section(lines, start_index)
for parameter in parameters:
parameter.kind = inspect.Parameter.KEYWORD_ONLY
if parameters:
return Section(Section.Type.KEYWORD_ARGS, parameters), i
self.error(f"Empty keyword arguments section at line {start_index}")
return None, i
read_parameters_section(self, lines, start_index)
¤
Parse a "parameters" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The parameters block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[Optional[pytkdocs.parsers.docstrings.base.Section], int] |
A tuple containing a |
Source code in pytkdocs/parsers/docstrings/google.py
def read_parameters_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse a "parameters" section.
Arguments:
lines: The parameters block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
parameters, i = self._parse_parameters_section(lines, start_index)
if parameters:
return Section(Section.Type.PARAMETERS, parameters), i
self.error(f"Empty parameters section at line {start_index}")
return None, i
read_return_section(self, lines, start_index)
¤
Parse an "returns" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The return block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[Optional[pytkdocs.parsers.docstrings.base.Section], int] |
A tuple containing a |
Source code in pytkdocs/parsers/docstrings/google.py
def read_return_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse an "returns" section.
Arguments:
lines: The return block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
text, i = self.read_block(lines, start_index)
# Early exit if there is no text in the return section
if not text:
self.error(f"Empty return section at line {start_index}")
return None, i
# First try to get the annotation and description from the docstring
try:
type_, text = text.split(":", 1)
except ValueError:
description = text
annotation = self.context["annotation"]
# If there was no annotation in the docstring then move to signature
if annotation is empty and self.context["signature"]:
annotation = self.context["signature"].return_annotation
else:
annotation = type_.lstrip()
description = text.lstrip()
# There was no type in the docstring and no annotation
if annotation is empty:
self.error("No return type/annotation in docstring/signature")
return Section(Section.Type.RETURN, AnnotatedObject(annotation, description)), i
read_yield_section(self, lines, start_index)
¤
Parse a "yields" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
lines |
List[str] |
The return block lines. |
required |
start_index |
int |
The line number to start at. |
required |
Returns:
Type | Description |
---|---|
Tuple[Optional[pytkdocs.parsers.docstrings.base.Section], int] |
A tuple containing a |
Source code in pytkdocs/parsers/docstrings/google.py
def read_yield_section(self, lines: List[str], start_index: int) -> Tuple[Optional[Section], int]:
"""
Parse a "yields" section.
Arguments:
lines: The return block lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
text, i = self.read_block(lines, start_index)
# Early exit if there is no text in the yield section
if not text:
self.error(f"Empty yield section at line {start_index}")
return None, i
# First try to get the annotation and description from the docstring
try:
type_, text = text.split(":", 1)
except ValueError:
description = text
annotation = self.context["annotation"]
# If there was no annotation in the docstring then move to signature
if annotation is empty and self.context["signature"]:
annotation = self.context["signature"].return_annotation
else:
annotation = type_.lstrip()
description = text.lstrip()
# There was no type in the docstring and no annotation
if annotation is empty:
self.error("No yield type/annotation in docstring/signature")
return Section(Section.Type.YIELD, AnnotatedObject(annotation, description)), i
is_empty_line(line)
¤
Tell if a line is empty.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
line |
The line to check. |
required |
Returns:
Type | Description |
---|---|
bool |
True if the line is empty or composed of blanks only, False otherwise. |
Source code in pytkdocs/parsers/docstrings/google.py
def is_empty_line(line) -> bool:
"""
Tell if a line is empty.
Arguments:
line: The line to check.
Returns:
True if the line is empty or composed of blanks only, False otherwise.
"""
return not line.strip()
markdown
¤
This module defines functions and classes to parse docstrings into structured data.
Markdown (Parser)
¤
A Markdown docstrings parser.
Source code in pytkdocs/parsers/docstrings/markdown.py
class Markdown(Parser):
"""A Markdown docstrings parser."""
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
return [Section(Section.Type.MARKDOWN, docstring)]
parse_sections(self, docstring)
¤
Parse a docstring as a list of sections.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The docstring to parse. |
required |
Returns:
Type | Description |
---|---|
List[pytkdocs.parsers.docstrings.base.Section] |
A list of |
Source code in pytkdocs/parsers/docstrings/markdown.py
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
return [Section(Section.Type.MARKDOWN, docstring)]
numpy
¤
This module defines functions and classes to parse docstrings into structured data.
RE_DOCTEST_BLANKLINE: Pattern
¤
Regular expression to match lines of the form <BLANKLINE>
.
RE_DOCTEST_FLAGS: Pattern
¤
Regular expression to match lines containing doctest flags of the form # doctest: +FLAG
.
Numpy (Parser)
¤
A Numpy-style docstrings parser.
Source code in pytkdocs/parsers/docstrings/numpy.py
class Numpy(Parser):
"""A Numpy-style docstrings parser."""
def __init__(self, trim_doctest_flags: bool = True) -> None:
"""
Initialize the objects.
Arguments:
trim_doctest_flags: Whether to remove doctest flags.
"""
super().__init__()
self.trim_doctest_flags = trim_doctest_flags
self.section_reader = {
Section.Type.PARAMETERS: self.read_parameters_section,
Section.Type.EXCEPTIONS: self.read_exceptions_section,
Section.Type.EXAMPLES: self.read_examples_section,
Section.Type.ATTRIBUTES: self.read_attributes_section,
Section.Type.RETURN: self.read_return_section,
}
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
if "signature" not in self.context:
self.context["signature"] = getattr(self.context["obj"], "signature", None)
if "annotation" not in self.context:
self.context["annotation"] = getattr(self.context["obj"], "type", empty)
if "attributes" not in self.context:
self.context["attributes"] = {}
docstring_obj = parse(docstring)
description_all = (
none_str_cast(docstring_obj.short_description) + "\n\n" + none_str_cast(docstring_obj.long_description)
).strip()
sections = [Section(Section.Type.MARKDOWN, description_all)] if description_all else []
sections_other = [
reader(docstring_obj) # type: ignore
if sec == Section.Type.RETURN
else reader(docstring, docstring_obj) # type: ignore
for (sec, reader) in self.section_reader.items()
]
sections.extend([sec for sec in sections_other if sec])
return sections
def read_parameters_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse a "parameters" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
parameters = []
docstring_params = [p for p in docstring_obj.params if p.args[0] == "param"]
for param in docstring_params:
name = param.arg_name
kind = None
type_name = param.type_name
default = param.default or empty
try:
signature_param = self.context["signature"].parameters[name.lstrip("*")]
except (AttributeError, KeyError):
self.error(f"No type annotation for parameter '{name}'")
else:
if signature_param.annotation is not empty:
type_name = signature_param.annotation
if signature_param.default is not empty:
default = signature_param.default
kind = signature_param.kind
description = param.description or ""
if not description:
self.error(f"No description for parameter '{name}'")
parameters.append(
Parameter(
name=param.arg_name,
annotation=type_name,
description=description,
default=default,
kind=kind,
)
)
if parameters:
return Section(Section.Type.PARAMETERS, parameters)
if re.search("Parameters\n", docstring):
self.error("Empty parameter section")
return None
def read_attributes_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse an "attributes" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
attributes = []
docstring_attributes = [p for p in docstring_obj.params if p.args[0] == "attribute"]
for attr in docstring_attributes:
description = attr.description or ""
if not description:
self.error(f"No description for attribute '{attr.arg_name}'")
attributes.append(
Attribute(
name=attr.arg_name,
annotation=attr.type_name,
description=attr.description,
)
)
if attributes:
return Section(Section.Type.ATTRIBUTES, attributes)
if re.search("Attributes\n", docstring):
self.error("Empty attributes section")
return None
def read_exceptions_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse an "exceptions" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
exceptions = []
except_obj = docstring_obj.raises
for exception in except_obj:
description = exception.description or ""
if not description:
self.error(f"No description for exception '{exception.type_name}'")
exceptions.append(AnnotatedObject(exception.type_name, description))
if exceptions:
return Section(Section.Type.EXCEPTIONS, exceptions)
if re.search("Raises\n", docstring):
self.error("Empty exceptions section")
return None
def read_return_section(
self,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse a "returns" section.
Arguments:
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
if docstring_obj.returns:
return_obj = docstring_obj.returns
if return_obj.description:
description = return_obj.description
else:
self.error("Empty return description")
description = ""
if self.context["signature"]:
annotation = self.context["signature"].return_annotation
else:
annotation = self.context["annotation"]
if annotation is empty and return_obj.type_name:
annotation = return_obj.type_name
if not annotation:
self.error("No return type annotation")
annotation = ""
if annotation or description:
return Section(Section.Type.RETURN, AnnotatedObject(annotation, description))
return None
def read_examples_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse an "examples" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
text = next(
(
meta.description
for meta in docstring_obj.meta
if isinstance(meta, DocstringMeta) and meta.args[0] == "examples"
),
"",
)
sub_sections = []
in_code_example = False
in_code_block = False
current_text: List[str] = []
current_example: List[str] = []
if text:
for line in text.split("\n"):
if is_empty_line(line):
if in_code_example:
if current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
current_example = []
in_code_example = False
else:
current_text.append(line)
elif in_code_example:
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
line = RE_DOCTEST_BLANKLINE.sub("", line)
current_example.append(line)
elif line.startswith("```"):
in_code_block = not in_code_block
current_text.append(line)
elif in_code_block:
current_text.append(line)
elif line.startswith(">>>"):
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
current_text = []
in_code_example = True
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
current_example.append(line)
else:
current_text.append(line)
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
elif current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
if sub_sections:
return Section(Section.Type.EXAMPLES, sub_sections)
if re.search("Examples\n", docstring):
self.error("Empty examples section")
return None
__init__(self, trim_doctest_flags=True)
special
¤
Initialize the objects.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
trim_doctest_flags |
bool |
Whether to remove doctest flags. |
True |
Source code in pytkdocs/parsers/docstrings/numpy.py
def __init__(self, trim_doctest_flags: bool = True) -> None:
"""
Initialize the objects.
Arguments:
trim_doctest_flags: Whether to remove doctest flags.
"""
super().__init__()
self.trim_doctest_flags = trim_doctest_flags
self.section_reader = {
Section.Type.PARAMETERS: self.read_parameters_section,
Section.Type.EXCEPTIONS: self.read_exceptions_section,
Section.Type.EXAMPLES: self.read_examples_section,
Section.Type.ATTRIBUTES: self.read_attributes_section,
Section.Type.RETURN: self.read_return_section,
}
parse_sections(self, docstring)
¤
Parse a docstring as a list of sections.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The docstring to parse. |
required |
Returns:
Type | Description |
---|---|
List[pytkdocs.parsers.docstrings.base.Section] |
A list of |
Source code in pytkdocs/parsers/docstrings/numpy.py
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
if "signature" not in self.context:
self.context["signature"] = getattr(self.context["obj"], "signature", None)
if "annotation" not in self.context:
self.context["annotation"] = getattr(self.context["obj"], "type", empty)
if "attributes" not in self.context:
self.context["attributes"] = {}
docstring_obj = parse(docstring)
description_all = (
none_str_cast(docstring_obj.short_description) + "\n\n" + none_str_cast(docstring_obj.long_description)
).strip()
sections = [Section(Section.Type.MARKDOWN, description_all)] if description_all else []
sections_other = [
reader(docstring_obj) # type: ignore
if sec == Section.Type.RETURN
else reader(docstring, docstring_obj) # type: ignore
for (sec, reader) in self.section_reader.items()
]
sections.extend([sec for sec in sections_other if sec])
return sections
read_attributes_section(self, docstring, docstring_obj)
¤
Parse an "attributes" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The raw docstring. |
required |
docstring_obj |
Docstring |
Docstring object parsed by docstring_parser. |
required |
Returns:
Type | Description |
---|---|
Optional[pytkdocs.parsers.docstrings.base.Section] |
A |
Source code in pytkdocs/parsers/docstrings/numpy.py
def read_attributes_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse an "attributes" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
attributes = []
docstring_attributes = [p for p in docstring_obj.params if p.args[0] == "attribute"]
for attr in docstring_attributes:
description = attr.description or ""
if not description:
self.error(f"No description for attribute '{attr.arg_name}'")
attributes.append(
Attribute(
name=attr.arg_name,
annotation=attr.type_name,
description=attr.description,
)
)
if attributes:
return Section(Section.Type.ATTRIBUTES, attributes)
if re.search("Attributes\n", docstring):
self.error("Empty attributes section")
return None
read_examples_section(self, docstring, docstring_obj)
¤
Parse an "examples" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The raw docstring. |
required |
docstring_obj |
Docstring |
Docstring object parsed by docstring_parser. |
required |
Returns:
Type | Description |
---|---|
Optional[pytkdocs.parsers.docstrings.base.Section] |
A |
Source code in pytkdocs/parsers/docstrings/numpy.py
def read_examples_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse an "examples" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
text = next(
(
meta.description
for meta in docstring_obj.meta
if isinstance(meta, DocstringMeta) and meta.args[0] == "examples"
),
"",
)
sub_sections = []
in_code_example = False
in_code_block = False
current_text: List[str] = []
current_example: List[str] = []
if text:
for line in text.split("\n"):
if is_empty_line(line):
if in_code_example:
if current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
current_example = []
in_code_example = False
else:
current_text.append(line)
elif in_code_example:
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
line = RE_DOCTEST_BLANKLINE.sub("", line)
current_example.append(line)
elif line.startswith("```"):
in_code_block = not in_code_block
current_text.append(line)
elif in_code_block:
current_text.append(line)
elif line.startswith(">>>"):
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
current_text = []
in_code_example = True
if self.trim_doctest_flags:
line = RE_DOCTEST_FLAGS.sub("", line)
current_example.append(line)
else:
current_text.append(line)
if current_text:
sub_sections.append((Section.Type.MARKDOWN, "\n".join(current_text)))
elif current_example:
sub_sections.append((Section.Type.EXAMPLES, "\n".join(current_example)))
if sub_sections:
return Section(Section.Type.EXAMPLES, sub_sections)
if re.search("Examples\n", docstring):
self.error("Empty examples section")
return None
read_exceptions_section(self, docstring, docstring_obj)
¤
Parse an "exceptions" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The raw docstring. |
required |
docstring_obj |
Docstring |
Docstring object parsed by docstring_parser. |
required |
Returns:
Type | Description |
---|---|
Optional[pytkdocs.parsers.docstrings.base.Section] |
A |
Source code in pytkdocs/parsers/docstrings/numpy.py
def read_exceptions_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse an "exceptions" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
exceptions = []
except_obj = docstring_obj.raises
for exception in except_obj:
description = exception.description or ""
if not description:
self.error(f"No description for exception '{exception.type_name}'")
exceptions.append(AnnotatedObject(exception.type_name, description))
if exceptions:
return Section(Section.Type.EXCEPTIONS, exceptions)
if re.search("Raises\n", docstring):
self.error("Empty exceptions section")
return None
read_parameters_section(self, docstring, docstring_obj)
¤
Parse a "parameters" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The raw docstring. |
required |
docstring_obj |
Docstring |
Docstring object parsed by docstring_parser. |
required |
Returns:
Type | Description |
---|---|
Optional[pytkdocs.parsers.docstrings.base.Section] |
A |
Source code in pytkdocs/parsers/docstrings/numpy.py
def read_parameters_section(
self,
docstring: str,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse a "parameters" section.
Arguments:
docstring: The raw docstring.
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
parameters = []
docstring_params = [p for p in docstring_obj.params if p.args[0] == "param"]
for param in docstring_params:
name = param.arg_name
kind = None
type_name = param.type_name
default = param.default or empty
try:
signature_param = self.context["signature"].parameters[name.lstrip("*")]
except (AttributeError, KeyError):
self.error(f"No type annotation for parameter '{name}'")
else:
if signature_param.annotation is not empty:
type_name = signature_param.annotation
if signature_param.default is not empty:
default = signature_param.default
kind = signature_param.kind
description = param.description or ""
if not description:
self.error(f"No description for parameter '{name}'")
parameters.append(
Parameter(
name=param.arg_name,
annotation=type_name,
description=description,
default=default,
kind=kind,
)
)
if parameters:
return Section(Section.Type.PARAMETERS, parameters)
if re.search("Parameters\n", docstring):
self.error("Empty parameter section")
return None
read_return_section(self, docstring_obj)
¤
Parse a "returns" section.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring_obj |
Docstring |
Docstring object parsed by docstring_parser. |
required |
Returns:
Type | Description |
---|---|
Optional[pytkdocs.parsers.docstrings.base.Section] |
A |
Source code in pytkdocs/parsers/docstrings/numpy.py
def read_return_section(
self,
docstring_obj: Docstring,
) -> Optional[Section]:
"""
Parse a "returns" section.
Arguments:
docstring_obj: Docstring object parsed by docstring_parser.
Returns:
A `Section` object (or `None` if section is empty).
"""
if docstring_obj.returns:
return_obj = docstring_obj.returns
if return_obj.description:
description = return_obj.description
else:
self.error("Empty return description")
description = ""
if self.context["signature"]:
annotation = self.context["signature"].return_annotation
else:
annotation = self.context["annotation"]
if annotation is empty and return_obj.type_name:
annotation = return_obj.type_name
if not annotation:
self.error("No return type annotation")
annotation = ""
if annotation or description:
return Section(Section.Type.RETURN, AnnotatedObject(annotation, description))
return None
is_empty_line(line)
¤
Tell if a line is empty.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
line |
str |
The line to check. |
required |
Returns:
Type | Description |
---|---|
bool |
True if the line is empty or composed of blanks only, False otherwise. |
Source code in pytkdocs/parsers/docstrings/numpy.py
def is_empty_line(line: str) -> bool:
"""
Tell if a line is empty.
Arguments:
line: The line to check.
Returns:
True if the line is empty or composed of blanks only, False otherwise.
"""
return not line.strip()
restructured_text
¤
This module defines functions and classes to parse docstrings into structured data.
AttributesDict (dict)
¤
Attribute details.
Source code in pytkdocs/parsers/docstrings/restructured_text.py
class AttributesDict(TypedDict):
"""Attribute details."""
docstring: str
annotation: Type # TODO: Not positive this is correct
FieldType
dataclass
¤
Maps directive names to parser functions.
Source code in pytkdocs/parsers/docstrings/restructured_text.py
@dataclass(frozen=True)
class FieldType:
"""Maps directive names to parser functions."""
names: FrozenSet[str]
reader: Callable[[List[str], int], int]
def matches(self, line: str) -> bool:
"""
Check if a line matches the field type.
Args:
line: Line to check against
Returns:
True if the line matches the field type, False otherwise.
"""
return any(line.startswith(f":{name}") for name in self.names)
matches(self, line)
¤
Check if a line matches the field type.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
line |
str |
Line to check against |
required |
Returns:
Type | Description |
---|---|
bool |
True if the line matches the field type, False otherwise. |
Source code in pytkdocs/parsers/docstrings/restructured_text.py
def matches(self, line: str) -> bool:
"""
Check if a line matches the field type.
Args:
line: Line to check against
Returns:
True if the line matches the field type, False otherwise.
"""
return any(line.startswith(f":{name}") for name in self.names)
ParseContext
¤
Typed replacement for context dictionary.
Source code in pytkdocs/parsers/docstrings/restructured_text.py
class ParseContext:
"""Typed replacement for context dictionary."""
obj: Any # I think this might be pytkdos.Object & subclasses
attributes: DefaultDict[str, AttributesDict]
signature: Optional[Signature]
# Not sure real type yet. Maybe Optional[Union[Literal[Signature.empty],str,Type]]
annotation: Any
# This might be be better as the obj & optional attributes
def __init__(self, context: Dict):
"""
Initialize the object.
Args:
context: Context of parsing operation.
"""
self.obj = context["obj"]
self.attributes = defaultdict(cast(Callable[[], AttributesDict], dict))
attributes = context.get("attributes")
if attributes is not None:
self.attributes.update(attributes)
self.signature = getattr(self.obj, "signature", None)
self.annotation = getattr(self.obj, "type", empty)
__init__(self, context)
special
¤
Initialize the object.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
context |
Dict |
Context of parsing operation. |
required |
Source code in pytkdocs/parsers/docstrings/restructured_text.py
def __init__(self, context: Dict):
"""
Initialize the object.
Args:
context: Context of parsing operation.
"""
self.obj = context["obj"]
self.attributes = defaultdict(cast(Callable[[], AttributesDict], dict))
attributes = context.get("attributes")
if attributes is not None:
self.attributes.update(attributes)
self.signature = getattr(self.obj, "signature", None)
self.annotation = getattr(self.obj, "type", empty)
ParsedDirective
dataclass
¤
Directive information that has been parsed from a docstring.
Source code in pytkdocs/parsers/docstrings/restructured_text.py
@dataclass
class ParsedDirective:
"""Directive information that has been parsed from a docstring."""
line: str
next_index: int
directive_parts: List[str]
value: str
invalid: bool = False
ParsedValues
dataclass
¤
Values parsed from the docstring to be used to produce sections.
Source code in pytkdocs/parsers/docstrings/restructured_text.py
@dataclass
class ParsedValues:
"""Values parsed from the docstring to be used to produce sections."""
description: List[str] = field(default_factory=list)
parameters: Dict[str, Parameter] = field(default_factory=dict)
param_types: Dict[str, str] = field(default_factory=dict)
attributes: Dict[str, Attribute] = field(default_factory=dict)
attribute_types: Dict[str, str] = field(default_factory=dict)
exceptions: List[AnnotatedObject] = field(default_factory=list)
return_value: Optional[AnnotatedObject] = None
return_type: Optional[str] = None
RestructuredText (Parser)
¤
A reStructuredText docstrings parser.
Source code in pytkdocs/parsers/docstrings/restructured_text.py
class RestructuredText(Parser):
"""A reStructuredText docstrings parser."""
def __init__(self) -> None:
"""Initialize the object."""
super().__init__()
self._typed_context = ParseContext({"obj": None})
self._parsed_values: ParsedValues = ParsedValues()
# Ordering is significant so that directives like ":vartype" are checked before ":var"
self.field_types = [
FieldType(PARAM_TYPE_NAMES, self._read_parameter_type),
FieldType(PARAM_NAMES, self._read_parameter),
FieldType(ATTRIBUTE_TYPE_NAMES, self._read_attribute_type),
FieldType(ATTRIBUTE_NAMES, self._read_attribute),
FieldType(EXCEPTION_NAMES, self._read_exception),
FieldType(RETURN_NAMES, self._read_return),
FieldType(RETURN_TYPE_NAMES, self._read_return_type),
]
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
self._typed_context = ParseContext(self.context)
self._parsed_values = ParsedValues()
lines = docstring.split("\n")
curr_line_index = 0
while curr_line_index < len(lines):
line = lines[curr_line_index]
for field_type in self.field_types:
if field_type.matches(line):
# https://github.com/python/mypy/issues/5485
curr_line_index = field_type.reader(lines, curr_line_index) # type: ignore
break
else:
self._parsed_values.description.append(line)
curr_line_index += 1
return self._parsed_values_to_sections()
def _read_parameter(self, lines: List[str], start_index: int) -> int:
"""
Parse a parameter value.
Arguments:
lines: The docstring lines.
start_index: The line number to start at.
Returns:
Index at which to continue parsing.
"""
parsed_directive = self._parse_directive(lines, start_index)
if parsed_directive.invalid:
return parsed_directive.next_index
directive_type = None
if len(parsed_directive.directive_parts) == 2:
# no type info
name = parsed_directive.directive_parts[1]
elif len(parsed_directive.directive_parts) == 3:
directive_type = parsed_directive.directive_parts[1]
name = parsed_directive.directive_parts[2]
else:
self.error(f"Failed to parse field directive from '{parsed_directive.line}'")
return parsed_directive.next_index
if name in self._parsed_values.parameters:
self.errors.append(f"Duplicate parameter entry for '{name}'")
return parsed_directive.next_index
annotation = self._determine_param_annotation(name, directive_type)
default, kind = self._determine_param_details(name)
self._parsed_values.parameters[name] = Parameter(
name=name,
annotation=annotation,
description=parsed_directive.value,
default=default,
kind=kind,
)
return parsed_directive.next_index
def _determine_param_details(self, name: str) -> Tuple[Any, Any]:
default = empty
kind = empty
if self._typed_context.signature is not None:
param_signature = self._typed_context.signature.parameters.get(name.lstrip("*"))
# an error for param_signature being none is already reported by _determine_param_annotation()
if param_signature is not None:
if param_signature.default is not empty:
default = param_signature.default
kind = param_signature.kind # type: ignore[assignment]
return default, kind
def _determine_param_annotation(self, name: str, directive_type: Optional[str]) -> Any:
# Annotation precedence:
# - signature annotation
# - in-line directive type
# - "type" directive type
# - empty
annotation = empty
parsed_param_type = self._parsed_values.param_types.get(name)
if parsed_param_type is not None:
annotation = parsed_param_type # type: ignore[assignment]
if directive_type is not None:
annotation = directive_type # type: ignore[assignment]
if directive_type is not None and parsed_param_type is not None:
self.error(f"Duplicate parameter information for '{name}'")
if self._typed_context.signature is not None:
try:
param_signature = self._typed_context.signature.parameters[name.lstrip("*")]
except KeyError:
self.error(f"No matching parameter for '{name}'")
else:
if param_signature.annotation is not empty:
annotation = param_signature.annotation
return annotation
def _read_parameter_type(self, lines: List[str], start_index: int) -> int:
"""
Parse a parameter type.
Arguments:
lines: The docstring lines.
start_index: The line number to start at.
Returns:
Index at which to continue parsing.
"""
parsed_directive = self._parse_directive(lines, start_index)
if parsed_directive.invalid:
return parsed_directive.next_index
param_type = _consolidate_descriptive_type(parsed_directive.value.strip())
if len(parsed_directive.directive_parts) == 2:
param_name = parsed_directive.directive_parts[1]
else:
self.error(f"Failed to get parameter name from '{parsed_directive.line}'")
return parsed_directive.next_index
self._parsed_values.param_types[param_name] = param_type
param = self._parsed_values.parameters.get(param_name)
if param is not None:
if param.annotation is empty:
param.annotation = param_type
else:
self.error(f"Duplicate parameter information for '{param_name}'")
return parsed_directive.next_index
def _read_attribute(self, lines: List[str], start_index: int) -> int:
"""
Parse an attribute value.
Arguments:
lines: The docstring lines.
start_index: The line number to start at.
Returns:
Index at which to continue parsing.
"""
parsed_directive = self._parse_directive(lines, start_index)
if parsed_directive.invalid:
return parsed_directive.next_index
if len(parsed_directive.directive_parts) == 2:
name = parsed_directive.directive_parts[1]
else:
self.error(f"Failed to parse field directive from '{parsed_directive.line}'")
return parsed_directive.next_index
annotation = empty
# Annotation precedence:
# - external context type TODO: spend time understanding where this comes from
# - "vartype" directive type
# - empty
parsed_attribute_type = self._parsed_values.attribute_types.get(name)
if parsed_attribute_type is not None:
annotation = parsed_attribute_type # type: ignore[assignment]
context_attribute_annotation = self._typed_context.attributes[name].get("annotation")
if context_attribute_annotation is not None:
annotation = context_attribute_annotation
if name in self._parsed_values.attributes:
self.errors.append(f"Duplicate attribute entry for '{name}'")
else:
self._parsed_values.attributes[name] = Attribute(
name=name,
annotation=annotation,
description=parsed_directive.value,
)
return parsed_directive.next_index
def _read_attribute_type(self, lines: List[str], start_index: int) -> int:
"""
Parse a parameter type.
Arguments:
lines: The docstring lines.
start_index: The line number to start at.
Returns:
Index at which to continue parsing.
"""
parsed_directive = self._parse_directive(lines, start_index)
if parsed_directive.invalid:
return parsed_directive.next_index
attribute_type = _consolidate_descriptive_type(parsed_directive.value.strip())
if len(parsed_directive.directive_parts) == 2:
attribute_name = parsed_directive.directive_parts[1]
else:
self.error(f"Failed to get attribute name from '{parsed_directive.line}'")
return parsed_directive.next_index
self._parsed_values.attribute_types[attribute_name] = attribute_type
attribute = self._parsed_values.attributes.get(attribute_name)
if attribute is not None:
if attribute.annotation is empty:
attribute.annotation = attribute_type
else:
self.error(f"Duplicate attribute information for '{attribute_name}'")
return parsed_directive.next_index
def _read_exception(self, lines: List[str], start_index: int) -> int:
"""
Parse an exceptions value.
Arguments:
lines: The docstring lines.
start_index: The line number to start at.
Returns:
A tuple containing a `Section` (or `None`) and the index at which to continue parsing.
"""
parsed_directive = self._parse_directive(lines, start_index)
if parsed_directive.invalid:
return parsed_directive.next_index
if len(parsed_directive.directive_parts) == 2:
ex_type = parsed_directive.directive_parts[1]
self._parsed_values.exceptions.append(AnnotatedObject(ex_type, parsed_directive.value))
else:
self.error(f"Failed to parse exception directive from '{parsed_directive.line}'")
return parsed_directive.next_index
def _read_return(self, lines: List[str], start_index: int) -> int:
"""
Parse an return value.
Arguments:
lines: The docstring lines.
start_index: The line number to start at.
Returns:
Index at which to continue parsing.
"""
parsed_directive = self._parse_directive(lines, start_index)
if parsed_directive.invalid:
return parsed_directive.next_index
annotation = empty
# Annotation precedence:
# - signature annotation
# - "rtype" directive type
# - external context type TODO: spend time understanding where this comes from
# - empty
if self._typed_context.signature is not None and self._typed_context.signature.return_annotation is not empty:
annotation = self._typed_context.signature.return_annotation
elif self._parsed_values.return_type is not None:
annotation = self._parsed_values.return_type # type: ignore[assignment]
else:
annotation = self._typed_context.annotation
self._parsed_values.return_value = AnnotatedObject(annotation, parsed_directive.value)
return parsed_directive.next_index
def _read_return_type(self, lines: List[str], start_index: int) -> int:
"""
Parse an return type value.
Arguments:
lines: The docstring lines.
start_index: The line number to start at.
Returns:
Index at which to continue parsing.
"""
parsed_directive = self._parse_directive(lines, start_index)
if parsed_directive.invalid:
return parsed_directive.next_index
return_type = _consolidate_descriptive_type(parsed_directive.value.strip())
self._parsed_values.return_type = return_type
return_value = self._parsed_values.return_value
if return_value is not None:
if return_value.annotation is empty:
return_value.annotation = return_type
else:
self.error("Duplicate type information for return")
return parsed_directive.next_index
def _parsed_values_to_sections(self) -> List[Section]:
markdown_text = "\n".join(_strip_blank_lines(self._parsed_values.description))
result = [Section(Section.Type.MARKDOWN, markdown_text)]
if self._parsed_values.parameters:
param_values = list(self._parsed_values.parameters.values())
result.append(Section(Section.Type.PARAMETERS, param_values))
if self._parsed_values.attributes:
attribute_values = list(self._parsed_values.attributes.values())
result.append(Section(Section.Type.ATTRIBUTES, attribute_values))
if self._parsed_values.return_value is not None:
result.append(Section(Section.Type.RETURN, self._parsed_values.return_value))
if self._parsed_values.exceptions:
result.append(Section(Section.Type.EXCEPTIONS, self._parsed_values.exceptions))
return result
def _parse_directive(self, lines: List[str], start_index: int) -> ParsedDirective:
line, next_index = _consolidate_continuation_lines(lines, start_index)
try:
_, directive, value = line.split(":", 2)
except ValueError:
self.error(f"Failed to get ':directive: value' pair from '{line}'")
return ParsedDirective(line, next_index, [], "", invalid=True)
value = value.strip()
return ParsedDirective(line, next_index, directive.split(" "), value)
parse_sections(self, docstring)
¤
Parse a docstring as a list of sections.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
docstring |
str |
The docstring to parse. |
required |
Returns:
Type | Description |
---|---|
List[pytkdocs.parsers.docstrings.base.Section] |
A list of |
Source code in pytkdocs/parsers/docstrings/restructured_text.py
def parse_sections(self, docstring: str) -> List[Section]: # noqa: D102
self._typed_context = ParseContext(self.context)
self._parsed_values = ParsedValues()
lines = docstring.split("\n")
curr_line_index = 0
while curr_line_index < len(lines):
line = lines[curr_line_index]
for field_type in self.field_types:
if field_type.matches(line):
# https://github.com/python/mypy/issues/5485
curr_line_index = field_type.reader(lines, curr_line_index) # type: ignore
break
else:
self._parsed_values.description.append(line)
curr_line_index += 1
return self._parsed_values_to_sections()