Current section

Files

Jump to
cucumber lib gherkin parser.ex
Raw

lib/gherkin/parser.ex

defmodule Gherkin.NimbleParser do
@moduledoc """
NimbleParsec-based Gherkin parser.
This module provides a compiled parser for Gherkin feature files using NimbleParsec.
It produces the same output structs as the original parser for backward compatibility.
## Architecture
The parser is built bottom-up in levels:
- Level 1: Primitives (whitespace, newlines, rest_of_line)
- Level 2: Keywords (Given, When, Then, Feature:, etc.)
- Level 3: Elements (tags, datatables, docstrings)
- Level 4: Steps (keyword + text + optional attachments)
- Level 5: Scenarios (Background, Scenario, ScenarioOutline, Examples)
- Level 6: Feature (top-level parser)
"""
import NimbleParsec
alias Gherkin.{Background, Examples, Feature, Scenario, ScenarioOutline, Step}
# ============================================================
# LEVEL 1: PRIMITIVES
# ============================================================
# Horizontal whitespace (spaces and tabs only)
horizontal_ws = ascii_string([?\s, ?\t], min: 1)
# Optional horizontal whitespace
optional_ws = ignore(optional(horizontal_ws))
# Newline handling (Windows \r\n and Unix \n)
newline = choice([string("\r\n"), string("\n")])
# End of line (newline or end of string)
eol = choice([ignore(newline), eos()])
# Rest of line (everything until newline), trimmed
rest_of_line =
utf8_string([not: ?\n, not: ?\r], min: 0)
|> map({String, :trim, []})
# Blank line (optional whitespace followed by newline)
blank_line =
optional_ws
|> concat(newline)
|> ignore()
# Zero or more blank lines
blank_lines = repeat(blank_line)
# ============================================================
# LEVEL 2: KEYWORDS
# ============================================================
# Step keywords (case-sensitive per Gherkin spec)
step_keyword =
choice([
string("Given"),
string("When"),
string("Then"),
string("And"),
string("But"),
string("*")
])
|> label("step keyword (Given, When, Then, And, But, or *)")
# Section keywords
feature_keyword =
string("Feature:")
|> label("Feature:")
background_keyword =
string("Background:")
|> label("Background:")
scenario_keyword =
string("Scenario:")
|> label("Scenario:")
scenario_outline_keyword =
choice([string("Scenario Outline:"), string("Scenario Template:")])
|> label("Scenario Outline:")
examples_keyword =
choice([string("Examples:"), string("Scenarios:")])
|> label("Examples:")
# ============================================================
# LEVEL 3: ELEMENTS
# ============================================================
# --- Tags ---
# Single tag: @tag_name
tag =
ignore(string("@"))
|> utf8_string([?a..?z, ?A..?Z, ?0..?9, ?_, ?-], min: 1)
|> label("tag name")
# Tag line: @tag1 @tag2 @tag3
tag_line =
optional_ws
|> concat(tag)
|> repeat(
ignore(horizontal_ws)
|> concat(tag)
)
|> concat(eol)
|> wrap()
|> label("tag line")
# Multiple tag lines (before features, scenarios, examples)
tags =
repeat(tag_line)
|> reduce({__MODULE__, :flatten_tags, []})
# --- Data Tables ---
# Table cell: | followed by content (not including the next |)
# A row like "| a | b |" has cells at each | delimiter
# The last | before newline creates an empty cell which we filter out
table_cell =
ignore(string("|"))
|> ignore(optional(horizontal_ws))
|> utf8_string([not: ?|, not: ?\n, not: ?\r], min: 0)
|> map({String, :trim, []})
# Table row: | cell1 | cell2 | cell3 |
# Note: times(table_cell, min: 1) will consume the trailing | as an empty cell
table_row =
optional_ws
|> times(table_cell, min: 1)
|> concat(eol)
|> reduce({__MODULE__, :filter_table_cells, []})
|> label("table row")
# Full data table
datatable =
times(table_row, min: 1)
|> tag(:datatable)
|> label("data table")
# --- DocStrings ---
docstring_delimiter = string(~s("""))
# Content line (anything that's not the closing delimiter)
docstring_content_line =
lookahead_not(
optional_ws
|> concat(docstring_delimiter)
)
|> concat(utf8_string([not: ?\n, not: ?\r], min: 0))
|> ignore(newline)
# Full docstring
docstring =
optional_ws
|> ignore(docstring_delimiter)
|> concat(eol)
|> repeat(docstring_content_line)
|> concat(optional_ws)
|> ignore(docstring_delimiter)
|> concat(eol)
|> reduce({__MODULE__, :join_docstring, []})
|> unwrap_and_tag(:docstring)
|> label("docstring")
# ============================================================
# LEVEL 4: STEPS
# ============================================================
# Step text (everything after keyword until end of line)
step_text =
ignore(horizontal_ws)
|> concat(rest_of_line)
# Section markers that stop step parsing
section_marker =
optional_ws
|> choice([
scenario_keyword,
scenario_outline_keyword,
examples_keyword,
# Tag line indicates new section
string("@")
])
# Complete step with optional docstring/datatable attachment
step =
lookahead_not(section_marker)
|> concat(optional_ws)
|> line(concat(step_keyword, step_text))
|> concat(eol)
|> optional(choice([docstring, datatable]))
|> reduce({__MODULE__, :build_step, []})
|> label("step")
# Multiple steps (also skip blank lines between steps)
steps =
repeat(
choice([
step,
lookahead_not(section_marker) |> concat(blank_line)
])
)
|> reduce({__MODULE__, :filter_steps, []})
# ============================================================
# LEVEL 5: SCENARIOS
# ============================================================
# --- Background ---
# Background section name (optional text after Background:)
background_name =
ignore(background_keyword)
|> concat(rest_of_line)
|> concat(eol)
# Background parser
background =
optional_ws
|> concat(background_name)
|> ignore(blank_lines)
|> concat(steps |> tag(:steps))
|> reduce({__MODULE__, :build_background, []})
|> unwrap_and_tag(:background)
|> label("background section")
# --- Examples ---
examples_name =
ignore(examples_keyword)
|> concat(rest_of_line)
# Lookahead to verify Examples: keyword follows tags
examples_lookahead =
lookahead(
repeat(tag_line)
|> concat(optional_ws)
|> concat(examples_keyword)
)
# Examples block (tags + name + table)
examples_block =
ignore(blank_lines)
|> concat(examples_lookahead)
|> concat(tags |> tag(:tags))
|> concat(optional_ws)
|> line(examples_name)
|> concat(eol)
|> ignore(blank_lines)
|> concat(times(table_row, min: 1) |> tag(:table))
|> reduce({__MODULE__, :build_examples, []})
|> label("examples block")
# --- Scenario ---
scenario_name =
ignore(scenario_keyword)
|> concat(rest_of_line)
# Regular scenario parser
scenario =
tags
|> tag(:tags)
|> concat(optional_ws)
|> line(scenario_name)
|> concat(eol)
|> ignore(blank_lines)
|> concat(steps |> tag(:steps))
|> reduce({__MODULE__, :build_scenario, []})
|> label("scenario")
# --- Scenario Outline ---
outline_name =
ignore(scenario_outline_keyword)
|> concat(rest_of_line)
# Scenario outline with examples
scenario_outline =
tags
|> tag(:tags)
|> concat(optional_ws)
|> line(outline_name)
|> concat(eol)
|> ignore(blank_lines)
|> concat(steps |> tag(:steps))
|> concat(times(examples_block, min: 1) |> tag(:examples))
|> reduce({__MODULE__, :build_scenario_outline, []})
|> label("scenario outline")
# Scenario or scenario outline
scenario_definition =
ignore(blank_lines)
|> choice([
scenario_outline,
scenario
])
# ============================================================
# LEVEL 6: FEATURE
# ============================================================
# Feature name
feature_name =
ignore(feature_keyword)
|> ignore(optional(horizontal_ws))
|> concat(rest_of_line)
|> concat(eol)
# Feature description (lines between feature name and first scenario/background)
# A description line is any line that's not a section start or tag
description_line =
lookahead_not(
optional_ws
|> choice([
background_keyword,
scenario_keyword,
scenario_outline_keyword,
string("@")
])
)
|> concat(rest_of_line)
|> concat(eol)
|> ignore()
# Full feature file parser
feature =
ignore(blank_lines)
|> concat(tags |> tag(:tags))
|> concat(optional_ws)
|> concat(feature_name |> unwrap_and_tag(:name))
|> ignore(repeat(choice([blank_line, description_line])))
|> optional(background)
|> concat(repeat(scenario_definition) |> tag(:scenarios))
|> reduce({__MODULE__, :build_feature, []})
|> label("feature")
# ============================================================
# PUBLIC API
# ============================================================
defparsec(:parse_feature, feature)
@doc """
Parses a Gherkin feature file string into structured data.
Returns a `%Gherkin.Feature{}` struct on success.
Raises `Gherkin.ParseError` on failure.
"""
def parse(gherkin_string) do
# Strip leading newline that heredoc strings add
normalized = String.trim_leading(gherkin_string, "\n")
case parse_feature(normalized) do
{:ok, [feature], "", _context, _line, _offset} ->
feature
{:ok, [feature], _rest, _context, _line, _offset} ->
# Partial parse - return what we have
feature
{:error, message, rest, _context, {line, col}, _offset} ->
raise Gherkin.ParseError,
message: message,
line: line,
column: col,
rest: String.slice(rest, 0, 50)
end
end
# ============================================================
# BUILD HELPERS (post_traverse callbacks)
# ============================================================
@doc false
def join_docstring(lines) do
# Find minimum indentation of non-empty lines
min_indent =
lines
|> Enum.reject(&(&1 == ""))
|> Enum.map(&count_leading_spaces/1)
|> Enum.min(fn -> 0 end)
# Strip baseline indentation from each line
lines
|> Enum.map_join("\n", &strip_indent(&1, min_indent))
|> String.trim_trailing()
end
defp count_leading_spaces(line) do
line
|> String.graphemes()
|> Enum.take_while(&(&1 == " " or &1 == "\t"))
|> length()
end
defp strip_indent(line, amount) when amount > 0 do
String.slice(line, amount, String.length(line))
end
defp strip_indent(line, _amount), do: line
@doc false
def filter_steps(items) do
Enum.filter(items, &is_struct(&1, Step))
end
@doc false
def filter_table_cells(cells) do
# Remove trailing empty cell created by the final |
cells
|> Enum.reject(&(&1 == ""))
end
@doc false
def flatten_tags(tag_lines) do
# tag_lines is a list of lists (each tag_line is wrapped)
# Flatten to a single list of tag strings
tag_lines
|> List.flatten()
|> Enum.filter(&is_binary/1)
end
@doc false
def build_step(args) do
# args structure: [{[keyword, text], {line, _offset}}, optional_attachment]
{step_data, attachment} = extract_step_parts(args)
{[keyword, text], {line_num, _offset}} = step_data
{docstring, datatable} =
case attachment do
{:docstring, ds} -> {ds, nil}
{:datatable, dt} -> {nil, dt}
nil -> {nil, nil}
end
%Step{
keyword: keyword,
text: text,
docstring: docstring,
datatable: datatable,
line: line_num - 1
}
end
defp extract_step_parts([{_data, {_line, _offset}} = step_data]) do
{step_data, nil}
end
defp extract_step_parts([{_data, {_line, _offset}} = step_data, attachment]) do
{step_data, attachment}
end
@doc false
def build_background(args) do
steps = extract_tagged(args, :steps)
%Background{
steps: steps
}
end
@doc false
def build_examples(args) do
tags = extract_tagged(args, :tags)
table = extract_tagged(args, :table)
{name, line_num} = extract_examples_name(args)
[header | body] = table
%Examples{
name: name,
tags: tags,
table_header: header,
table_body: body,
line: if(line_num, do: line_num - 1, else: nil)
}
end
defp extract_examples_name(args), do: extract_name_from_args(args, [:tags, :table])
defp extract_scenario_name(args), do: extract_name_from_args(args, [:tags, :steps, :examples])
defp extract_name_from_args([{tag, _} | rest], skip_tags) when is_atom(tag) do
if tag in skip_tags,
do: extract_name_from_args(rest, skip_tags),
else: {"", nil}
end
defp extract_name_from_args([{[name], {line, _}} | _], _) when is_binary(name),
do: {String.trim(name), line}
defp extract_name_from_args([{name, {line, _}} | _], _) when is_binary(name),
do: {String.trim(name), line}
defp extract_name_from_args([[name] | _], _) when is_binary(name),
do: {String.trim(name), nil}
defp extract_name_from_args([name | _], _) when is_binary(name),
do: {String.trim(name), nil}
defp extract_name_from_args(_, _), do: {"", nil}
@doc false
def build_scenario(args) do
tags = extract_tagged(args, :tags)
steps = extract_tagged(args, :steps)
{name, line_num} = extract_scenario_name(args)
%Scenario{
name: name,
steps: steps,
tags: tags,
line: if(line_num, do: line_num - 1, else: nil)
}
end
@doc false
def build_scenario_outline(args) do
tags = extract_tagged(args, :tags)
steps = extract_tagged(args, :steps)
examples = extract_tagged(args, :examples)
{name, line_num} = extract_scenario_name(args)
%ScenarioOutline{
name: name,
steps: steps,
tags: tags,
examples: examples,
line: if(line_num, do: line_num - 1, else: nil)
}
end
@doc false
def build_feature(args) do
tags = extract_tagged(args, :tags)
name = extract_tagged_single(args, :name) || ""
background = extract_tagged_single(args, :background)
scenarios = extract_tagged(args, :scenarios)
%Feature{
name: name,
description: "",
tags: tags,
background: background,
scenarios: scenarios
}
end
# Extract tagged values from args list (returns list)
# For :table, we need to preserve row structure
defp extract_tagged(args, tag) do
case Enum.find(args, fn
{^tag, _} -> true
_ -> false
end) do
{^tag, values} when tag == :table ->
# Table rows: unwrap one level but preserve row structure
case values do
[rows] when is_list(rows) and is_list(hd(rows)) -> rows
rows -> rows
end
{^tag, values} ->
List.flatten([values])
nil ->
[]
end
end
# Extract single tagged value (returns value or nil)
defp extract_tagged_single(args, tag) do
case Enum.find(args, fn
{^tag, _} -> true
_ -> false
end) do
{^tag, value} when is_list(value) -> List.first(value)
{^tag, value} -> value
nil -> nil
end
end
end
defmodule Gherkin.ParseError do
@moduledoc """
Exception raised when parsing a Gherkin file fails.
Contains detailed information about the parse error including
line number, column, and context.
"""
defexception [:message, :line, :column, :rest]
@impl true
def message(%{message: msg, line: line, column: col, rest: rest}) do
snippet = if rest && rest != "", do: ~s(\nNear: "#{rest}..."), else: ""
"""
Gherkin parse error at line #{line}, column #{col}:
Expected #{msg}#{snippet}
"""
end
end