Packages
floki
0.35.4
0.38.4
0.38.3
0.38.2
0.38.1
0.38.0
0.37.1
0.37.0
0.36.3
0.36.2
0.36.1
0.36.0
0.35.4
0.35.3
0.35.2
0.35.1
0.35.0
0.34.3
0.34.2
0.34.1
0.34.0
0.33.1
0.33.0
0.32.1
0.32.0
0.31.0
0.30.1
0.30.0
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.0
0.23.1
0.23.0
0.22.0
0.21.0
0.20.4
0.20.3
0.20.2
0.20.1
0.20.0
0.19.3
0.19.2
0.19.1
0.19.0
0.18.1
0.18.0
0.17.2
0.17.1
0.17.0
0.16.0
0.15.0
0.14.0
0.13.2
0.13.1
0.13.0
0.12.1
0.12.0
0.11.0
0.10.1
0.10.0
0.9.0
0.8.1
0.8.0
0.7.2
0.7.1
0.7.0
0.6.1
0.6.0
0.5.0
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.1
0.1.0
0.0.5
0.0.4
0.0.3
0.0.2
0.0.1
Floki is a simple HTML parser that enables search for nodes using CSS selectors.
Current section
86 Versions
Jump to
Current section
86 Versions
Compare versions
7
files changed
+208
additions
-107
deletions
| @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. | |
| 7 7 | |
| 8 8 | ## [Unreleased][unreleased] |
| 9 9 | |
| 10 | + ## [0.35.4] - 2024-02-19 |
| 11 | + |
| 12 | + Besides the fix described below, this release also contains more performance |
| 13 | + improvements, thanks to [@ypconstante](https://github.com/ypconstante). |
| 14 | + |
| 15 | + ### Fixed |
| 16 | + |
| 17 | + - Fix order of results for `Floki.find/2`. This was a regression from the |
| 18 | + previous version - thanks [@ypconstante](https://github.com/ypconstante). |
| 19 | + |
| 10 20 | ## [0.35.3] - 2024-01-25 |
| 11 21 | |
| 12 22 | This release has great performance improvements, thanks to the PRs |
| @@ -736,7 +746,8 @@ of the parent element inside HTML. | |
| 736 746 | |
| 737 747 | - Elixir version requirement from "~> 1.0.0" to ">= 1.0.0". |
| 738 748 | |
| 739 | - [unreleased]: https://github.com/philss/floki/compare/v0.35.3...HEAD |
| 749 | + [unreleased]: https://github.com/philss/floki/compare/v0.35.4...HEAD |
| 750 | + [0.35.4]: https://github.com/philss/floki/compare/v0.35.3...v0.35.4 |
| 740 751 | [0.35.3]: https://github.com/philss/floki/compare/v0.35.2...v0.35.3 |
| 741 752 | [0.35.2]: https://github.com/philss/floki/compare/v0.35.1...v0.35.2 |
| 742 753 | [0.35.1]: https://github.com/philss/floki/compare/v0.35.0...v0.35.1 |
| @@ -3,33 +3,33 @@ | |
| 3 3 | {<<"GitHub">>,<<"https://github.com/philss/floki">>}, |
| 4 4 | {<<"Sponsor">>,<<"https://github.com/sponsors/philss">>}]}. |
| 5 5 | {<<"name">>,<<"floki">>}. |
| 6 | - {<<"version">>,<<"0.35.3">>}. |
| 6 | + {<<"version">>,<<"0.35.4">>}. |
| 7 7 | {<<"description">>, |
| 8 8 | <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}. |
| 9 9 | {<<"elixir">>,<<"~> 1.12">>}. |
| 10 10 | {<<"app">>,<<"floki">>}. |
| 11 11 | {<<"licenses">>,[<<"MIT">>]}. |
| 12 12 | {<<"files">>, |
| 13 | - [<<"lib/floki">>,<<"lib/floki/deep_text.ex">>,<<"lib/floki/entities">>, |
| 14 | - <<"lib/floki/entities/codepoints.ex">>,<<"lib/floki/filter_out.ex">>, |
| 15 | - <<"lib/floki/flat_text.ex">>,<<"lib/floki/html">>, |
| 16 | - <<"lib/floki/html/numeric_charref.ex">>,<<"lib/floki/html/tokenizer.ex">>, |
| 17 | - <<"lib/floki/html_parser">>,<<"lib/floki/html_parser/fast_html.ex">>, |
| 13 | + [<<"lib/floki">>,<<"lib/floki/html_parser">>, |
| 18 14 | <<"lib/floki/html_parser/html5ever.ex">>, |
| 19 | - <<"lib/floki/html_parser/mochiweb.ex">>,<<"lib/floki/html_tree">>, |
| 20 | - <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_tree/html_node.ex">>, |
| 21 | - <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/text.ex">>, |
| 22 | - <<"lib/floki/parse_error.ex">>,<<"lib/floki/selector">>, |
| 15 | + <<"lib/floki/html_parser/fast_html.ex">>, |
| 16 | + <<"lib/floki/html_parser/mochiweb.ex">>,<<"lib/floki/traversal.ex">>, |
| 17 | + <<"lib/floki/text_extractor.ex">>,<<"lib/floki/raw_html.ex">>, |
| 18 | + <<"lib/floki/selector.ex">>,<<"lib/floki/selector">>, |
| 23 19 | <<"lib/floki/selector/combinator.ex">>, |
| 24 | - <<"lib/floki/selector/functional.ex">>, |
| 25 | - <<"lib/floki/selector/pseudo_class.ex">>, |
| 26 | - <<"lib/floki/selector/tokenizer.ex">>, |
| 27 20 | <<"lib/floki/selector/attribute_selector.ex">>, |
| 28 | - <<"lib/floki/selector/parser.ex">>,<<"lib/floki/text_extractor.ex">>, |
| 29 | - <<"lib/floki/traversal.ex">>,<<"lib/floki/html_parser.ex">>, |
| 30 | - <<"lib/floki/entities.ex">>,<<"lib/floki/finder.ex">>, |
| 31 | - <<"lib/floki/html_tree.ex">>,<<"lib/floki/raw_html.ex">>, |
| 32 | - <<"lib/floki/selector.ex">>,<<"lib/floki.ex">>, |
| 21 | + <<"lib/floki/selector/tokenizer.ex">>,<<"lib/floki/selector/parser.ex">>, |
| 22 | + <<"lib/floki/selector/functional.ex">>, |
| 23 | + <<"lib/floki/selector/pseudo_class.ex">>,<<"lib/floki/html_tree">>, |
| 24 | + <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/comment.ex">>, |
| 25 | + <<"lib/floki/html_tree/html_node.ex">>,<<"lib/floki/html_tree/text.ex">>, |
| 26 | + <<"lib/floki/finder.ex">>,<<"lib/floki/filter_out.ex">>, |
| 27 | + <<"lib/floki/html_tree.ex">>,<<"lib/floki/html">>, |
| 28 | + <<"lib/floki/html/numeric_charref.ex">>,<<"lib/floki/html/tokenizer.ex">>, |
| 29 | + <<"lib/floki/html_parser.ex">>,<<"lib/floki/deep_text.ex">>, |
| 30 | + <<"lib/floki/entities.ex">>,<<"lib/floki/flat_text.ex">>, |
| 31 | + <<"lib/floki/parse_error.ex">>,<<"lib/floki/entities">>, |
| 32 | + <<"lib/floki/entities/codepoints.ex">>,<<"lib/floki.ex">>, |
| 33 33 | <<"src/floki_selector_lexer.xrl">>,<<"src/floki_mochi_html.erl">>, |
| 34 34 | <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>, |
| 35 35 | <<"CONTRIBUTING.md">>,<<"CHANGELOG.md">>]}. |
| @@ -280,17 +280,13 @@ defmodule Floki do | |
| 280 280 | ) |
| 281 281 | |
| 282 282 | with {:ok, document} <- Floki.parse_document(html) do |
| 283 | - {tree, results} = Finder.find(document, selector) |
| 284 | - |
| 285 | - Enum.map(results, fn html_node -> HTMLTree.to_tuple(tree, html_node) end) |
| 283 | + Finder.find(document, selector) |
| 286 284 | end |
| 287 285 | end |
| 288 286 | |
| 289 287 | def find(html_tree_as_tuple, selector) |
| 290 288 | when is_list(html_tree_as_tuple) or is_html_node(html_tree_as_tuple) do |
| 291 | - {tree, results} = Finder.find(html_tree_as_tuple, selector) |
| 292 | - |
| 293 | - Enum.map(results, fn html_node -> HTMLTree.to_tuple(tree, html_node) end) |
| 289 | + Finder.find(html_tree_as_tuple, selector) |
| 294 290 | end |
| 295 291 | |
| 296 292 | @doc """ |
| @@ -385,7 +381,8 @@ defmodule Floki do | |
| 385 381 | ({String.t(), html_attributes()} -> {String.t(), html_attributes()} | :delete) |
| 386 382 | ) :: html_tree() |
| 387 383 | def find_and_update(html_tree, selector, fun) do |
| 388 | - {tree, results} = Finder.find(html_tree, selector) |
| 384 | + tree = HTMLTree.build(html_tree) |
| 385 | + results = Finder.find(tree, selector) |
| 389 386 | |
| 390 387 | operations_with_nodes = |
| 391 388 | Enum.map(results, fn |
| @@ -609,19 +606,21 @@ defmodule Floki do | |
| 609 606 | """ |
| 610 607 | |
| 611 608 | @spec children(html_node(), Keyword.t()) :: html_tree() | nil |
| 609 | + def children(html_node, opts \\ [include_text: true]) |
| 612 610 | |
| 613 | - def children(html_node, opts \\ [include_text: true]) do |
| 614 | - case html_node do |
| 615 | - {_, _, subtree} -> |
| 616 | - filter_children(subtree, opts[:include_text]) |
| 617 | - |
| 618 | - _ -> |
| 619 | - nil |
| 620 | - end |
| 611 | + def children({_, _, subtree}, include_text: false) do |
| 612 | + Enum.filter(subtree, &is_tuple/1) |
| 621 613 | end |
| 622 614 | |
| 623 | - defp filter_children(children, false), do: Enum.filter(children, &is_tuple(&1)) |
| 624 | - defp filter_children(children, _), do: children |
| 615 | + def children({_, _, subtree}, include_text: _) do |
| 616 | + subtree |
| 617 | + end |
| 618 | + |
| 619 | + def children({_, _, _} = html_node, opts) do |
| 620 | + children(html_node, include_text: opts[:include_text]) |
| 621 | + end |
| 622 | + |
| 623 | + def children(_html_node, _opts), do: nil |
| 625 624 | |
| 626 625 | @doc """ |
| 627 626 | Returns a list with attribute values for a given selector. |
| @@ -6,14 +6,16 @@ defmodule Floki.Finder do | |
| 6 6 | |
| 7 7 | alias Floki.{HTMLTree, Selector} |
| 8 8 | alias HTMLTree.HTMLNode |
| 9 | + import Floki, only: [is_html_node: 1] |
| 9 10 | |
| 10 11 | # Find elements inside a HTML tree. |
| 11 12 | # Second argument can be either a selector string, a selector struct or a list of selector structs. |
| 12 13 | |
| 13 | - @spec find(Floki.html_tree(), Floki.css_selector()) :: {HTMLTree.t(), [HTMLTree.HTMLNode.t()]} |
| 14 | + @spec find(HTMLTree.t(), Floki.css_selector()) :: [HTMLTree.HTMLNode.t()] |
| 15 | + @spec find(Floki.html_tree() | Floki.html_node(), Floki.css_selector()) :: [Floki.html_node()] |
| 14 16 | |
| 15 | - def find([], _), do: {%HTMLTree{}, []} |
| 16 | - def find(html_as_string, _) when is_binary(html_as_string), do: {%HTMLTree{}, []} |
| 17 | + def find([], _), do: [] |
| 18 | + def find(html_as_string, _) when is_binary(html_as_string), do: [] |
| 17 19 | |
| 18 20 | def find(html_tree, selector_as_string) when is_binary(selector_as_string) do |
| 19 21 | selectors = Selector.Parser.parse(selector_as_string) |
| @@ -24,40 +26,57 @@ defmodule Floki.Finder do | |
| 24 26 | find(html_tree, [selector]) |
| 25 27 | end |
| 26 28 | |
| 27 | - def find(html_tree, selectors) when is_list(selectors) do |
| 28 | - tree = HTMLTree.build(html_tree) |
| 29 | + def find(html_tree_as_tuple, selectors) |
| 30 | + when (is_list(html_tree_as_tuple) or is_html_node(html_tree_as_tuple)) and |
| 31 | + is_list(selectors) do |
| 32 | + if traverse_html_tuples?(selectors) do |
| 33 | + html_tree_as_tuple = List.wrap(html_tree_as_tuple) |
| 34 | + stack = Enum.map(selectors, fn s -> {s, html_tree_as_tuple} end) |
| 29 35 | |
| 36 | + results = traverse_html_tuples(stack, []) |
| 37 | + Enum.reverse(results) |
| 38 | + else |
| 39 | + tree = HTMLTree.build(html_tree_as_tuple) |
| 40 | + results = find(tree, selectors) |
| 41 | + Enum.map(results, fn html_node -> HTMLTree.to_tuple(tree, html_node) end) |
| 42 | + end |
| 43 | + end |
| 44 | + |
| 45 | + def find(%HTMLTree{} = tree, selectors) when is_list(selectors) do |
| 30 46 | node_ids = Enum.reverse(tree.node_ids) |
| 31 47 | stack = Enum.map(selectors, fn s -> {s, node_ids} end) |
| 32 48 | |
| 33 | - results = |
| 34 | - traverse_with(:cont, tree, [], stack) |
| 35 | - |> Enum.reverse() |
| 49 | + traverse_html_tree(stack, tree, []) |
| 50 | + |> Enum.sort_by(& &1.node_id) |
| 36 51 | |> Enum.uniq() |
| 37 | - |
| 38 | - {tree, results} |
| 39 52 | end |
| 40 53 | |
| 54 | + # some selectors can be applied with the raw html tree tuples instead of |
| 55 | + # using an intermediate HTMLTree: |
| 56 | + # - single selector |
| 57 | + # - no composite selector |
| 58 | + # - no pseudo classes |
| 59 | + defp traverse_html_tuples?([selector]), do: traverse_html_tuples?(selector) |
| 60 | + defp traverse_html_tuples?(selectors) when is_list(selectors), do: false |
| 61 | + defp traverse_html_tuples?(%Selector{pseudo_classes: [_ | _]}), do: false |
| 62 | + defp traverse_html_tuples?(%Selector{combinator: nil}), do: true |
| 63 | + |
| 64 | + defp traverse_html_tuples?(%Selector{combinator: combinator}), |
| 65 | + do: traverse_html_tuples?(combinator) |
| 66 | + |
| 67 | + defp traverse_html_tuples?(%Selector.Combinator{match_type: :descendant, selector: selector}), |
| 68 | + do: traverse_html_tuples?(selector) |
| 69 | + |
| 70 | + defp traverse_html_tuples?(_), do: false |
| 71 | + |
| 41 72 | # The stack serves as accumulator when there is another combinator to traverse. |
| 42 73 | # So the scope of one combinator is the stack (or acc) or the parent one. |
| 43 | - defp traverse_with(:cont, _, acc, []) do |
| 74 | + defp traverse_html_tree( |
| 75 | + [{%Selector{combinator: nil} = selector, [node_id | selector_rest]} | stack], |
| 76 | + tree, |
| 44 77 | acc |
| 45 | - end |
| 46 | - |
| 47 | - defp traverse_with(:cont, tree, acc, [next | rest]) do |
| 48 | - traverse_with(next, tree, acc, rest) |
| 49 | - end |
| 50 | - |
| 51 | - defp traverse_with({selector, [node_id]}, tree, acc, stack) do |
| 52 | - traverse_with({selector, node_id}, tree, acc, stack) |
| 53 | - end |
| 54 | - |
| 55 | - defp traverse_with({selector, [next | rest]}, tree, acc, stack) do |
| 56 | - stack = [{selector, rest} | stack] |
| 57 | - traverse_with({selector, next}, tree, acc, stack) |
| 58 | - end |
| 59 | - |
| 60 | - defp traverse_with({%Selector{combinator: nil} = selector, node_id}, tree, acc, stack) do |
| 78 | + ) do |
| 79 | + stack = [{selector, selector_rest} | stack] |
| 61 80 | html_node = get_node(node_id, tree) |
| 62 81 | |
| 63 82 | acc = |
| @@ -67,10 +86,15 @@ defmodule Floki.Finder do | |
| 67 86 | acc |
| 68 87 | end |
| 69 88 | |
| 70 | - traverse_with(:cont, tree, acc, stack) |
| 89 | + traverse_html_tree(stack, tree, acc) |
| 71 90 | end |
| 72 91 | |
| 73 | - defp traverse_with({%Selector{combinator: combinator} = selector, node_id}, tree, acc, stack) do |
| 92 | + defp traverse_html_tree( |
| 93 | + [{%Selector{combinator: combinator} = selector, [node_id | selector_rest]} | stack], |
| 94 | + tree, |
| 95 | + acc |
| 96 | + ) do |
| 97 | + stack = [{selector, selector_rest} | stack] |
| 74 98 | html_node = get_node(node_id, tree) |
| 75 99 | |
| 76 100 | stack = |
| @@ -81,7 +105,86 @@ defmodule Floki.Finder do | |
| 81 105 | stack |
| 82 106 | end |
| 83 107 | |
| 84 | - traverse_with(:cont, tree, acc, stack) |
| 108 | + traverse_html_tree(stack, tree, acc) |
| 109 | + end |
| 110 | + |
| 111 | + defp traverse_html_tree([{_selector, []} | rest], tree, acc) do |
| 112 | + traverse_html_tree(rest, tree, acc) |
| 113 | + end |
| 114 | + |
| 115 | + defp traverse_html_tree([], _, acc) do |
| 116 | + acc |
| 117 | + end |
| 118 | + |
| 119 | + defp traverse_html_tuples( |
| 120 | + [ |
| 121 | + { |
| 122 | + %Selector{combinator: nil} = selector, |
| 123 | + [{_type, _attributes, children} = html_tuple | selector_rest] |
| 124 | + } |
| 125 | + | stack |
| 126 | + ], |
| 127 | + acc |
| 128 | + ) do |
| 129 | + stack = [{selector, children}, {selector, selector_rest} | stack] |
| 130 | + |
| 131 | + acc = |
| 132 | + if Selector.match?(html_tuple, selector, nil) do |
| 133 | + [html_tuple | acc] |
| 134 | + else |
| 135 | + acc |
| 136 | + end |
| 137 | + |
| 138 | + traverse_html_tuples(stack, acc) |
| 139 | + end |
| 140 | + |
| 141 | + defp traverse_html_tuples( |
| 142 | + [ |
| 143 | + { |
| 144 | + %Selector{ |
| 145 | + combinator: %Selector.Combinator{ |
| 146 | + match_type: :descendant, |
| 147 | + selector: combinator_selector |
| 148 | + } |
| 149 | + } = selector, |
| 150 | + [{_type, _attributes, children} = html_tuple | selector_rest] |
| 151 | + } |
| 152 | + | stack |
| 153 | + ], |
| 154 | + acc |
| 155 | + ) do |
| 156 | + stack = [{selector, selector_rest} | stack] |
| 157 | + |
| 158 | + stack = |
| 159 | + if Selector.match?(html_tuple, selector, nil) do |
| 160 | + [{combinator_selector, children} | stack] |
| 161 | + else |
| 162 | + [{selector, children} | stack] |
| 163 | + end |
| 164 | + |
| 165 | + traverse_html_tuples(stack, acc) |
| 166 | + end |
| 167 | + |
| 168 | + defp traverse_html_tuples( |
| 169 | + [ |
| 170 | + { |
| 171 | + selector, |
| 172 | + [_ | selector_rest] |
| 173 | + } |
| 174 | + | stack |
| 175 | + ], |
| 176 | + acc |
| 177 | + ) do |
| 178 | + stack = [{selector, selector_rest} | stack] |
| 179 | + traverse_html_tuples(stack, acc) |
| 180 | + end |
| 181 | + |
| 182 | + defp traverse_html_tuples([{_selector, []} | rest], acc) do |
| 183 | + traverse_html_tuples(rest, acc) |
| 184 | + end |
| 185 | + |
| 186 | + defp traverse_html_tuples([], acc) do |
| 187 | + acc |
| 85 188 | end |
| 86 189 | |
| 87 190 | defp get_selector_nodes(%Selector.Combinator{match_type: :child}, html_node, _tree) do |
| @@ -90,8 +193,8 @@ defmodule Floki.Finder do | |
| 90 193 | |
| 91 194 | defp get_selector_nodes(%Selector.Combinator{match_type: :sibling}, html_node, tree) do |
| 92 195 | case get_siblings(html_node, tree) do |
| 93 | - [sibling_id | _] -> sibling_id |
| 94 | - _ -> nil |
| 196 | + [sibling_id | _] -> [sibling_id] |
| 197 | + _ -> [] |
| 95 198 | end |
| 96 199 | end |
| @@ -32,12 +32,13 @@ defmodule Floki.RawHTML do | |
| 32 32 | end |
| 33 33 | |
| 34 34 | @encoder &Floki.Entities.encode/1 |
| 35 | + @no_encoder &Function.identity/1 |
| 35 36 | |
| 36 37 | def raw_html(html_tree, options) do |
| 37 38 | encoder = |
| 38 39 | case Keyword.fetch(options, :encode) do |
| 39 40 | {:ok, true} -> @encoder |
| 40 | - {:ok, false} -> &Function.identity/1 |
| 41 | + {:ok, false} -> @no_encoder |
| 41 42 | :error -> default_encoder() |
| 42 43 | end |
| 43 44 | |
| @@ -127,7 +128,7 @@ defmodule Floki.RawHTML do | |
| 127 128 | end |
| 128 129 | |
| 129 130 | defp tag_attrs(attr_list, encoder) do |
| 130 | - map_intersperse(attr_list, ?\s, &build_attrs(&1, encoder)) |
| 131 | + Enum.map_intersperse(attr_list, ?\s, &build_attrs(&1, encoder)) |
| 131 132 | end |
| 132 133 | |
| 133 134 | defp tag_with_attrs(type, [], children, padding, _encoder, self_closing_tags), |
| @@ -173,8 +174,8 @@ defmodule Floki.RawHTML do | |
| 173 174 | defp tag_for(type, attrs, children, encoder, padding, self_closing_tags) do |
| 174 175 | encoder = |
| 175 176 | case type do |
| 176 | - "script" -> & &1 |
| 177 | - "style" -> & &1 |
| 177 | + "script" -> @no_encoder |
| 178 | + "style" -> @no_encoder |
| 178 179 | _ -> encoder |
| 179 180 | end |
| 180 181 | |
| @@ -196,26 +197,11 @@ defmodule Floki.RawHTML do | |
| 196 197 | if Application.get_env(:floki, :encode_raw_html, true) do |
| 197 198 | @encoder |
| 198 199 | else |
| 199 | - & &1 |
| 200 | + @no_encoder |
| 200 201 | end |
| 201 202 | end |
| 202 203 | |
| 203 204 | # helpers |
| 204 | - |
| 205 | - # TODO: Use Enum.map_intersperse/3 when we require Elixir v1.10+ |
| 206 | - |
| 207 | - defp map_intersperse([], _, _), |
| 208 | - do: [] |
| 209 | - |
| 210 | - defp map_intersperse([last], _, mapper), |
| 211 | - do: [mapper.(last)] |
| 212 | - |
| 213 | - defp map_intersperse([head | rest], separator, mapper), |
| 214 | - do: [mapper.(head), separator | map_intersperse(rest, separator, mapper)] |
| 215 | - |
| 216 | - defp map_intersperse(%{} = attrs, separator, mapper), |
| 217 | - do: map_intersperse(Map.to_list(attrs), separator, mapper) |
| 218 | - |
| 219 205 | defp leftpad(:noop), do: "" |
| 220 206 | defp leftpad(%{pad: pad}), do: pad |
Loading more files…