Current section

Files

Jump to
snakebridge lib snakebridge docs rst_parser.ex
Raw

lib/snakebridge/docs/rst_parser.ex

defmodule SnakeBridge.Docs.RstParser do
@moduledoc """
Parses Python docstrings in various formats (Google, NumPy, Sphinx, Epytext).
This module detects the docstring format and extracts structured information
including parameters, return values, exceptions, and examples.
## Supported Formats
- **Google style**: Uses `Args:`, `Returns:`, `Raises:` sections
- **NumPy style**: Uses underlined section headers (`Parameters\n----------`)
- **Sphinx/reST style**: Uses `:param:`, `:type:`, `:returns:` directives
- **Epytext style**: Uses `@param`, `@type`, `@return` tags
"""
@type parsed_doc :: %{
short_description: String.t() | nil,
long_description: String.t() | nil,
params: [param()],
returns: returns() | nil,
raises: [raises()],
examples: [String.t()],
notes: String.t() | nil,
style: atom()
}
@type param :: %{
name: String.t(),
type_name: String.t() | nil,
description: String.t() | nil,
optional: boolean(),
default: String.t() | nil
}
@type returns :: %{
type_name: String.t() | nil,
description: String.t() | nil
}
@type raises :: %{
type_name: String.t(),
description: String.t() | nil
}
@doc """
Parses a Python docstring and returns structured data.
"""
@spec parse(String.t() | nil) :: parsed_doc()
def parse(nil), do: empty_result()
def parse(""), do: empty_result()
def parse(docstring) when is_binary(docstring) do
style = detect_style(docstring)
lines = String.split(docstring, "\n")
{short_desc, rest} = extract_short_description(lines)
{long_desc, sections} = extract_long_description(rest)
%{
short_description: short_desc,
long_description: long_desc,
params: extract_params(sections, style),
returns: extract_returns(sections, style),
raises: extract_raises(sections, style),
examples: extract_examples(sections, style),
notes: extract_notes(sections, style),
style: style
}
end
@doc """
Detects the docstring style based on content patterns.
"""
@spec detect_style(String.t() | nil) :: atom()
def detect_style(nil), do: :unknown
def detect_style(""), do: :unknown
def detect_style(docstring) do
cond do
epytext_style?(docstring) -> :epytext
sphinx_style?(docstring) -> :sphinx
numpy_style?(docstring) -> :numpy
google_style?(docstring) -> :google
true -> :unknown
end
end
defp epytext_style?(docstring) do
docstring =~ ~r/@param\s/ or docstring =~ ~r/@type\s/
end
defp sphinx_style?(docstring) do
docstring =~ ~r/:param\s+\w+:/ or docstring =~ ~r/:returns:/
end
defp numpy_style?(docstring) do
docstring =~ ~r/Parameters\n-+/ or docstring =~ ~r/Returns\n-+/
end
defp google_style?(docstring) do
docstring =~ ~r/\n\s*Args:\s*\n/ or
docstring =~ ~r/\n\s*Arguments:\s*\n/ or
docstring =~ ~r/\n\s*Returns:\s*\n/ or
docstring =~ ~r/\n\s*Raises:\s*\n/ or
docstring =~ ~r/\n\s*Example:\s*\n/ or
docstring =~ ~r/\n\s*Examples:\s*\n/
end
defp empty_result do
%{
short_description: nil,
long_description: nil,
params: [],
returns: nil,
raises: [],
examples: [],
notes: nil,
style: :unknown
}
end
defp extract_short_description([]), do: {nil, []}
defp extract_short_description(lines) do
# Skip leading empty lines
lines = Enum.drop_while(lines, &(String.trim(&1) == ""))
case lines do
[] ->
{nil, []}
[first | rest] ->
short = String.trim(first)
if short == "" do
{nil, rest}
else
{short, rest}
end
end
end
defp extract_long_description(lines) do
# Skip empty lines after short description
lines = Enum.drop_while(lines, &(String.trim(&1) == ""))
# Find where sections start
section_start = find_section_start(lines)
case section_start do
nil ->
# No sections, all is long description
long_desc =
lines
|> Enum.join("\n")
|> String.trim()
{(long_desc == "" && nil) || long_desc, []}
index ->
{desc_lines, section_lines} = Enum.split(lines, index)
long_desc =
desc_lines
|> Enum.join("\n")
|> String.trim()
{(long_desc == "" && nil) || long_desc, section_lines}
end
end
defp find_section_start(lines) do
Enum.find_index(lines, fn line ->
trimmed = String.trim(line)
# Google style sections
# NumPy style sections (check next line for dashes)
# Sphinx style
# Epytext style
trimmed in [
"Args:",
"Arguments:",
"Returns:",
"Yields:",
"Raises:",
"Example:",
"Examples:",
"Note:",
"Notes:",
"Warning:",
"Warnings:"
] or
trimmed in [
"Parameters",
"Returns",
"Yields",
"Raises",
"Examples",
"Notes",
"Warnings",
"See Also",
"References"
] or
String.starts_with?(trimmed, ":param ") or
String.starts_with?(trimmed, ":returns:") or
String.starts_with?(trimmed, "@param ") or
String.starts_with?(trimmed, "@return")
end)
end
defp extract_params(sections, :google), do: extract_google_params(sections)
defp extract_params(sections, :numpy), do: extract_numpy_params(sections)
defp extract_params(sections, :sphinx), do: extract_sphinx_params(sections)
defp extract_params(sections, :epytext), do: extract_epytext_params(sections)
defp extract_params(_sections, _style), do: []
defp extract_google_params(lines) do
lines
|> extract_section(["Args:", "Arguments:"])
|> parse_google_items()
|> Enum.map(&parse_google_param/1)
end
defp extract_numpy_params(lines) do
lines
|> extract_numpy_section("Parameters")
|> parse_numpy_items()
|> Enum.map(&parse_numpy_param/1)
end
defp extract_sphinx_params(lines) do
lines
|> Enum.filter(&String.starts_with?(String.trim(&1), ":param "))
|> Enum.map(&parse_sphinx_param/1)
end
defp extract_epytext_params(lines) do
lines
|> Enum.filter(&String.starts_with?(String.trim(&1), "@param "))
|> Enum.map(&parse_epytext_param/1)
end
defp parse_google_param({name_type, description}) do
{name, type_name, optional, default} = parse_param_name_type(name_type)
%{
name: name,
type_name: type_name,
description: description,
optional: optional,
default: default
}
end
defp parse_numpy_param({name_type, description}) do
[name_part | type_parts] = String.split(name_type, " : ", parts: 2)
name = String.trim(name_part)
type_info = if type_parts != [], do: hd(type_parts), else: nil
{type_name, optional} = parse_numpy_type(type_info)
%{
name: name,
type_name: type_name,
description: description,
optional: optional,
default: nil
}
end
defp parse_sphinx_param(line) do
case Regex.run(~r/:param\s+(\w+):\s*(.*)/, String.trim(line)) do
[_, name, desc] ->
%{name: name, type_name: nil, description: desc, optional: false, default: nil}
_ ->
%{name: "", type_name: nil, description: "", optional: false, default: nil}
end
end
defp parse_epytext_param(line) do
case Regex.run(~r/@param\s+(\w+):\s*(.*)/, String.trim(line)) do
[_, name, desc] ->
%{name: name, type_name: nil, description: desc, optional: false, default: nil}
_ ->
%{name: "", type_name: nil, description: "", optional: false, default: nil}
end
end
defp parse_param_name_type(name_type) do
# Pattern: "name (type, optional): description" or "name (type): description"
case Regex.run(~r/^(\w+)\s*\(([^)]+)\)/, name_type) do
[_, name, type_info] ->
optional = String.contains?(type_info, "optional")
type_name = type_info |> String.replace(~r/,?\s*optional/, "") |> String.trim()
default =
case Regex.run(~r/Defaults? to [`']?([^`']+)[`']?/, name_type) do
[_, val] -> val
_ -> nil
end
{name, type_name, optional, default}
_ ->
# Just name, no type
name = name_type |> String.split() |> List.first() || ""
{name, nil, false, nil}
end
end
defp parse_numpy_type(nil), do: {nil, false}
defp parse_numpy_type(type_info) do
optional = String.contains?(type_info, "optional")
type_name = type_info |> String.replace(~r/,?\s*optional/, "") |> String.trim()
{type_name, optional}
end
defp extract_returns(sections, :google), do: extract_google_returns(sections)
defp extract_returns(sections, :numpy), do: extract_numpy_returns(sections)
defp extract_returns(sections, :sphinx), do: extract_sphinx_returns(sections)
defp extract_returns(sections, :epytext), do: extract_epytext_returns(sections)
defp extract_returns(_sections, _style), do: nil
defp extract_google_returns(lines) do
case extract_section(lines, ["Returns:", "Yields:"]) |> parse_google_items() do
[] ->
nil
[{type_desc, description} | _] ->
%{type_name: String.trim(type_desc), description: description}
end
end
defp extract_numpy_returns(lines) do
case extract_numpy_section(lines, "Returns") |> parse_numpy_items() do
[] ->
nil
[{type_name, description} | _] ->
%{type_name: type_name, description: description}
end
end
defp extract_sphinx_returns(lines) do
case Enum.find(lines, &String.contains?(&1, ":returns:")) do
nil ->
nil
line ->
case Regex.run(~r/:returns:\s*(.*)/, line) do
[_, desc] -> %{type_name: nil, description: desc}
_ -> nil
end
end
end
defp extract_epytext_returns(lines) do
case Enum.find(lines, &String.contains?(&1, "@return")) do
nil ->
nil
line ->
case Regex.run(~r/@return\s*:?\s*(.*)/, line) do
[_, desc] -> %{type_name: nil, description: desc}
_ -> nil
end
end
end
defp extract_raises(sections, :google), do: extract_google_raises(sections)
defp extract_raises(sections, :numpy), do: extract_numpy_raises(sections)
defp extract_raises(sections, :sphinx), do: extract_sphinx_raises(sections)
defp extract_raises(sections, :epytext), do: extract_epytext_raises(sections)
defp extract_raises(_sections, _style), do: []
defp extract_google_raises(lines) do
lines
|> extract_section(["Raises:"])
|> parse_google_items()
|> Enum.map(fn {type_name, description} ->
%{type_name: String.trim(type_name), description: description}
end)
end
defp extract_numpy_raises(lines) do
lines
|> extract_numpy_section("Raises")
|> parse_numpy_items()
|> Enum.map(fn {type_name, description} ->
%{type_name: type_name, description: description}
end)
end
defp extract_sphinx_raises(lines) do
lines
|> Enum.filter(&String.contains?(&1, ":raises"))
|> Enum.map(fn line ->
case Regex.run(~r/:raises?\s+(\w+):\s*(.*)/, line) do
[_, type, desc] -> %{type_name: type, description: desc}
_ -> %{type_name: "Error", description: ""}
end
end)
end
defp extract_epytext_raises(lines) do
lines
|> Enum.filter(&String.contains?(&1, "@raise"))
|> Enum.map(fn line ->
case Regex.run(~r/@raise\s+(\w+):\s*(.*)/, line) do
[_, type, desc] -> %{type_name: type, description: desc}
_ -> %{type_name: "Error", description: ""}
end
end)
end
defp extract_examples(sections, style) do
section_content =
case style do
:google -> extract_section(sections, ["Example:", "Examples:"])
:numpy -> extract_numpy_section(sections, "Examples")
_ -> []
end
section_content
|> Enum.join("\n")
|> String.trim()
|> case do
"" -> []
content -> [content]
end
end
defp extract_notes(sections, style) do
section_content =
case style do
:google -> extract_section(sections, ["Note:", "Notes:"])
:numpy -> extract_numpy_section(sections, "Notes")
_ -> []
end
section_content
|> Enum.join("\n")
|> String.trim()
|> case do
"" -> nil
content -> content
end
end
defp extract_section(lines, headers) do
start_idx =
Enum.find_index(lines, fn line ->
String.trim(line) in headers
end)
case start_idx do
nil -> []
idx -> extract_section_content(lines, idx)
end
end
defp extract_section_content(lines, idx) do
lines
|> Enum.drop(idx + 1)
|> Enum.take_while(&line_in_section?/1)
end
defp line_in_section?(line) do
trimmed = String.trim(line)
cond do
trimmed == "" -> true
section_header?(trimmed) -> false
indented_line?(line) -> true
true -> false
end
end
defp indented_line?(line) do
String.starts_with?(line, " ") or String.starts_with?(line, "\t")
end
defp extract_numpy_section(lines, header) do
start_idx =
Enum.find_index(lines, fn line ->
String.trim(line) == header
end)
case start_idx do
nil -> []
idx -> extract_numpy_section_content(lines, idx)
end
end
defp extract_numpy_section_content(lines, idx) do
lines
|> Enum.drop(idx + 2)
|> Enum.with_index(idx + 2)
|> Enum.take_while(fn {line, line_idx} ->
line_in_numpy_section?(lines, line, line_idx)
end)
|> Enum.map(fn {line, _idx} -> line end)
end
defp line_in_numpy_section?(lines, line, line_idx) do
trimmed = String.trim(line)
cond do
trimmed == "" -> true
numpy_section_header?(lines, line_idx) -> false
true -> true
end
end
defp section_header?(line) do
line in [
"Args:",
"Arguments:",
"Returns:",
"Yields:",
"Raises:",
"Example:",
"Examples:",
"Note:",
"Notes:",
"Warning:",
"Warnings:"
]
end
defp numpy_section_header?(lines, idx) when is_integer(idx) and idx >= 0 do
case Enum.at(lines, idx + 1) do
nil -> false
next_line -> String.trim(next_line) =~ ~r/^-+$/
end
end
defp numpy_section_header?(_lines, _idx), do: false
defp parse_google_items(lines) do
lines
|> Enum.chunk_while(nil, &google_chunk_reducer/2, &chunk_finalizer/1)
|> Enum.reject(&is_nil/1)
|> Enum.map(&split_google_item/1)
end
defp google_chunk_reducer(line, acc) do
trimmed = String.trim(line)
cond do
trimmed == "" ->
emit_chunk_on_empty(acc)
google_item_header?(trimmed) ->
emit_and_start_new_chunk(acc, trimmed)
acc != nil ->
{:cont, acc <> " " <> trimmed}
true ->
{:cont, nil}
end
end
defp google_item_header?(trimmed) do
String.match?(trimmed, ~r/^\w+(\s*\([^)]*\))?:/)
end
defp split_google_item(item) do
case String.split(item, ":", parts: 2) do
[name_type, desc] -> {String.trim(name_type), String.trim(desc)}
[name_type] -> {String.trim(name_type), ""}
end
end
defp parse_numpy_items(lines) do
lines
|> Enum.chunk_while(nil, &numpy_chunk_reducer/2, &chunk_finalizer/1)
|> Enum.reject(&is_nil/1)
|> Enum.map(&split_numpy_item/1)
end
defp numpy_chunk_reducer(line, acc) do
trimmed = String.trim(line)
cond do
trimmed == "" ->
emit_chunk_on_empty(acc)
numpy_item_header?(line) ->
emit_and_start_new_chunk(acc, line)
acc != nil ->
{:cont, acc <> " " <> trimmed}
true ->
{:cont, nil}
end
end
defp numpy_item_header?(line) do
String.contains?(line, " : ") or not String.starts_with?(line, " ")
end
defp split_numpy_item(item) do
case String.split(item, "\n", parts: 2) do
[first, rest] ->
{String.trim(first), String.trim(rest)}
[first] ->
split_numpy_single_line(first)
end
end
defp split_numpy_single_line(line) do
case String.split(line, " : ", parts: 2) do
[name, desc] -> {String.trim(name), String.trim(desc)}
[name] -> {String.trim(name), ""}
end
end
# Shared chunk helpers for both Google and NumPy parsers
defp emit_chunk_on_empty(nil), do: {:cont, nil}
defp emit_chunk_on_empty(acc), do: {:cont, acc, nil}
defp emit_and_start_new_chunk(nil, new_value), do: {:cont, new_value}
defp emit_and_start_new_chunk(acc, new_value), do: {:cont, acc, new_value}
defp chunk_finalizer(nil), do: {:cont, []}
defp chunk_finalizer(acc), do: {:cont, acc, []}
end