Packages
floki
0.29.0
0.38.4
0.38.3
0.38.2
0.38.1
0.38.0
0.37.1
0.37.0
0.36.3
0.36.2
0.36.1
0.36.0
0.35.4
0.35.3
0.35.2
0.35.1
0.35.0
0.34.3
0.34.2
0.34.1
0.34.0
0.33.1
0.33.0
0.32.1
0.32.0
0.31.0
0.30.1
0.30.0
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.0
0.23.1
0.23.0
0.22.0
0.21.0
0.20.4
0.20.3
0.20.2
0.20.1
0.20.0
0.19.3
0.19.2
0.19.1
0.19.0
0.18.1
0.18.0
0.17.2
0.17.1
0.17.0
0.16.0
0.15.0
0.14.0
0.13.2
0.13.1
0.13.0
0.12.1
0.12.0
0.11.0
0.10.1
0.10.0
0.9.0
0.8.1
0.8.0
0.7.2
0.7.1
0.7.0
0.6.1
0.6.0
0.5.0
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.1
0.1.0
0.0.5
0.0.4
0.0.3
0.0.2
0.0.1
Floki is a simple HTML parser that enables search for nodes using CSS selectors.
Current section
86 Versions
Jump to
Current section
86 Versions
Compare versions
15
files changed
+201
additions
-135
deletions
| @@ -60,7 +60,7 @@ Add Floki to your `mix.exs`: | |
| 60 60 | ```elixir |
| 61 61 | defp deps do |
| 62 62 | [ |
| 63 | - {:floki, "~> 0.28.0"} |
| 63 | + {:floki, "~> 0.29.0"} |
| 64 64 | ] |
| 65 65 | end |
| 66 66 | ``` |
| @@ -114,7 +114,7 @@ After Rust is set up, you need to add `html5ever` NIF to your dependency list: | |
| 114 114 | ```elixir |
| 115 115 | defp deps do |
| 116 116 | [ |
| 117 | - {:floki, "~> 0.28.0"}, |
| 117 | + {:floki, "~> 0.29.0"}, |
| 118 118 | {:html5ever, "~> 0.8.0"} |
| 119 119 | ] |
| 120 120 | end |
| @@ -142,7 +142,7 @@ First, add `fast_html` to your dependencies: | |
| 142 142 | ```elixir |
| 143 143 | defp deps do |
| 144 144 | [ |
| 145 | - {:floki, "~> 0.28.0"}, |
| 145 | + {:floki, "~> 0.29.0"}, |
| 146 146 | {:fast_html, "~> 2.0"} |
| 147 147 | ] |
| 148 148 | end |
| @@ -4,25 +4,25 @@ | |
| 4 4 | <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}. |
| 5 5 | {<<"elixir">>,<<"~> 1.7">>}. |
| 6 6 | {<<"files">>, |
| 7 | - [<<"lib">>,<<"lib/floki">>,<<"lib/floki/deep_text.ex">>, |
| 7 | + [<<"lib">>,<<"lib/floki">>,<<"lib/floki/html_parser.ex">>, |
| 8 | + <<"lib/floki/html_parser">>,<<"lib/floki/html_parser/fast_html.ex">>, |
| 9 | + <<"lib/floki/html_parser/mochiweb.ex">>, |
| 10 | + <<"lib/floki/html_parser/html5ever.ex">>,<<"lib/floki/selector">>, |
| 11 | + <<"lib/floki/selector/functional.ex">>,<<"lib/floki/selector/parser.ex">>, |
| 12 | + <<"lib/floki/selector/attribute_selector.ex">>, |
| 13 | + <<"lib/floki/selector/pseudo_class.ex">>, |
| 14 | + <<"lib/floki/selector/tokenizer.ex">>, |
| 15 | + <<"lib/floki/selector/combinator.ex">>,<<"lib/floki/deep_text.ex">>, |
| 16 | + <<"lib/floki/filter_out.ex">>,<<"lib/floki/flat_text.ex">>, |
| 17 | + <<"lib/floki/finder.ex">>,<<"lib/floki/traversal.ex">>, |
| 8 18 | <<"lib/floki/raw_html.ex">>,<<"lib/floki/parse_error.ex">>, |
| 9 19 | <<"lib/floki/html_tree">>,<<"lib/floki/html_tree/id_seeder.ex">>, |
| 10 | - <<"lib/floki/html_tree/text.ex">>,<<"lib/floki/html_tree/html_node.ex">>, |
| 11 | - <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_parser">>, |
| 12 | - <<"lib/floki/html_parser/mochiweb.ex">>, |
| 13 | - <<"lib/floki/html_parser/fast_html.ex">>, |
| 14 | - <<"lib/floki/html_parser/html5ever.ex">>,<<"lib/floki/flat_text.ex">>, |
| 15 | - <<"lib/floki/html_tree.ex">>,<<"lib/floki/selector.ex">>, |
| 16 | - <<"lib/floki/finder.ex">>,<<"lib/floki/traversal.ex">>, |
| 17 | - <<"lib/floki/filter_out.ex">>,<<"lib/floki/selector">>, |
| 18 | - <<"lib/floki/selector/pseudo_class.ex">>, |
| 19 | - <<"lib/floki/selector/attribute_selector.ex">>, |
| 20 | - <<"lib/floki/selector/parser.ex">>,<<"lib/floki/selector/combinator.ex">>, |
| 21 | - <<"lib/floki/selector/functional.ex">>, |
| 22 | - <<"lib/floki/selector/tokenizer.ex">>,<<"lib/floki/html_parser.ex">>, |
| 23 | - <<"lib/floki.ex">>,<<"src/floki_selector_lexer.xrl">>, |
| 24 | - <<"src/floki_mochi_html.erl">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>, |
| 25 | - <<"CODE_OF_CONDUCT.md">>,<<"CONTRIBUTING.md">>]}. |
| 20 | + <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_tree/text.ex">>, |
| 21 | + <<"lib/floki/html_tree/html_node.ex">>,<<"lib/floki/selector.ex">>, |
| 22 | + <<"lib/floki/html_tree.ex">>,<<"lib/floki.ex">>, |
| 23 | + <<"src/floki_selector_lexer.xrl">>,<<"src/floki_mochi_html.erl">>, |
| 24 | + <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>, |
| 25 | + <<"CONTRIBUTING.md">>]}. |
| 26 26 | {<<"licenses">>,[<<"MIT">>]}. |
| 27 27 | {<<"links">>,[{<<"GitHub">>,<<"https://github.com/philss/floki">>}]}. |
| 28 28 | {<<"name">>,<<"floki">>}. |
| @@ -32,4 +32,4 @@ | |
| 32 32 | {<<"optional">>,false}, |
| 33 33 | {<<"repository">>,<<"hexpm">>}, |
| 34 34 | {<<"requirement">>,<<"~> 0.5.0">>}]]}. |
| 35 | - {<<"version">>,<<"0.28.0">>}. |
| 35 | + {<<"version">>,<<"0.29.0">>}. |
| @@ -69,6 +69,8 @@ defmodule Floki do | |
| 69 69 | @type html_node :: html_comment() | html_doctype() | html_tag() | html_declaration() |
| 70 70 | @type html_tree :: [html_node()] |
| 71 71 | |
| 72 | + @type css_selector :: String.t() | Floki.Selector.t() | [Floki.Selector.t()] |
| 73 | + |
| 72 74 | @doc """ |
| 73 75 | Parses a HTML Document from a String. |
| 74 76 | |
| @@ -204,7 +206,7 @@ defmodule Floki do | |
| 204 206 | |
| 205 207 | """ |
| 206 208 | |
| 207 | - @spec find(binary | html_tree, binary) :: html_tree |
| 209 | + @spec find(binary() | html_tree(), css_selector()) :: html_tree |
| 208 210 | |
| 209 211 | def find(html, selector) when is_binary(html) do |
| 210 212 | IO.warn( |
| @@ -226,7 +228,7 @@ defmodule Floki do | |
| 226 228 | |
| 227 229 | @doc """ |
| 228 230 | Changes the attribute values of the elements matched by `selector` |
| 229 | - with the function `mutation` and returns the whole element tree |
| 231 | + with the function `mutation` and returns the whole element tree. |
| 230 232 | |
| 231 233 | ## Examples |
| 232 234 | |
| @@ -237,7 +239,7 @@ defmodule Floki do | |
| 237 239 | [{"div", [{"id", "b"}, {"class", "name"}], []}] |
| 238 240 | |
| 239 241 | """ |
| 240 | - @spec attr(binary | html_tree, binary, binary, (binary -> binary)) :: html_tree |
| 242 | + @spec attr(binary | html_tree, css_selector(), binary, (binary -> binary)) :: html_tree |
| 241 243 | |
| 242 244 | def attr(html_elem_tuple, selector, attribute_name, mutation) when is_tuple(html_elem_tuple) do |
| 243 245 | attr([html_elem_tuple], selector, attribute_name, mutation) |
| @@ -254,76 +256,89 @@ defmodule Floki do | |
| 254 256 | end |
| 255 257 | |
| 256 258 | def attr(html_tree_list, selector, attribute_name, mutation) when is_list(html_tree_list) do |
| 257 | - {tree, results} = Finder.find(html_tree_list, selector) |
| 258 | - mutate_attrs(html_tree_list, tree, results, attribute_name, mutation) |
| 259 | - end |
| 260 | - |
| 261 | - defp add_nodes_to_tree(tree, [html_node]) do |
| 262 | - nodes = Map.put(tree.nodes, html_node.node_id, html_node) |
| 263 | - Map.put(tree, :nodes, nodes) |
| 264 | - end |
| 265 | - |
| 266 | - defp add_nodes_to_tree(tree, [html_node | tail]) do |
| 267 | - nodes = Map.put(tree.nodes, html_node.node_id, html_node) |
| 268 | - |
| 269 | - tree |
| 270 | - |> Map.put(:nodes, nodes) |
| 271 | - |> add_nodes_to_tree(tail) |
| 272 | - end |
| 273 | - |
| 274 | - defp mutate_attrs(html_tree_list, _, [], _, _), do: html_tree_list |
| 275 | - |
| 276 | - defp mutate_attrs(_, tree, results, attribute_name, mutation_fn) do |
| 277 | - mutated_nodes = |
| 278 | - Enum.map( |
| 279 | - results, |
| 280 | - fn result -> |
| 281 | - mutated_attributes = |
| 282 | - if Enum.any?(result.attributes, &match?({^attribute_name, _}, &1)) do |
| 283 | - Enum.map( |
| 284 | - result.attributes, |
| 285 | - fn attribute -> |
| 286 | - with {^attribute_name, attribute_value} <- attribute do |
| 287 | - {attribute_name, mutation_fn.(attribute_value)} |
| 288 | - end |
| 259 | + find_and_update(html_tree_list, selector, fn |
| 260 | + {tag, attrs} -> |
| 261 | + modified_attrs = |
| 262 | + if Enum.any?(attrs, &match?({^attribute_name, _}, &1)) do |
| 263 | + Enum.map( |
| 264 | + attrs, |
| 265 | + fn attribute -> |
| 266 | + with {^attribute_name, attribute_value} <- attribute do |
| 267 | + {attribute_name, mutation.(attribute_value)} |
| 289 268 | end |
| 290 | - ) |
| 291 | - else |
| 292 | - [{attribute_name, mutation_fn.(nil)} | result.attributes] |
| 293 | - end |
| 269 | + end |
| 270 | + ) |
| 271 | + else |
| 272 | + [{attribute_name, mutation.(nil)} | attrs] |
| 273 | + end |
| 294 274 | |
| 295 | - Map.put(result, :attributes, mutated_attributes) |
| 296 | - end |
| 297 | - ) |
| 275 | + {tag, modified_attrs} |
| 298 276 | |
| 299 | - tree = add_nodes_to_tree(tree, mutated_nodes) |
| 300 | - |
| 301 | - tree.root_nodes_ids |
| 302 | - |> Enum.reverse() |
| 303 | - |> Enum.map(fn id -> Map.get(tree.nodes, id) end) |
| 304 | - |> Enum.map(fn html_node -> HTMLTree.to_tuple(tree, html_node) end) |
| 277 | + other -> |
| 278 | + other |
| 279 | + end) |
| 305 280 | end |
| 306 281 | |
| 307 | - @doc """ |
| 308 | - It receives a HTML tree structure as tuple and maps |
| 309 | - through all nodes with a given function that receives |
| 310 | - a tuple with {name, attributes}. |
| 311 | - |
| 312 | - It returns that structure transformed by the function. |
| 313 | - |
| 314 | - ## Examples |
| 315 | - |
| 316 | - iex> html = {"div", [{"class", "foo"}], ["text"]} |
| 317 | - iex> Floki.map(html, fn({name, attrs}) -> {name, [{"data-name", "bar"} | attrs]} end) |
| 318 | - {"div", [{"data-name", "bar"}, {"class", "foo"}], ["text"]} |
| 319 | - |
| 282 | + @deprecated """ |
| 283 | + Use `find_and_update/3` or `Enum.map/2` instead. |
| 320 284 | """ |
| 285 | + def map(_html_tree_or_list, _fun) |
| 286 | + |
| 321 287 | def map(html_tree_list, fun) when is_list(html_tree_list) do |
| 322 288 | Enum.map(html_tree_list, &Finder.map(&1, fun)) |
| 323 289 | end |
| 324 290 | |
| 325 291 | def map(html_tree, fun), do: Finder.map(html_tree, fun) |
| 326 292 | |
| 293 | + @doc """ |
| 294 | + Searchs for elements inside the HTML tree and update those that matches the selector. |
| 295 | + |
| 296 | + It will return the updated HTML tree. |
| 297 | + |
| 298 | + This function works in a way similar to `traverse_and_update`, but instead of updating |
| 299 | + the children nodes, it will only updates the `tag` and `attributes` of the matching nodes. |
| 300 | + |
| 301 | + If `fun` returns `:delete`, the HTML node will be removed from the tree. |
| 302 | + |
| 303 | + ## Examples |
| 304 | + |
| 305 | + iex> Floki.find_and_update([{"a", [{"href", "http://elixir-lang.com"}], ["Elixir"]}], "a", fn |
| 306 | + iex> {"a", [{"href", href}]} -> |
| 307 | + iex> {"a", [{"href", String.replace(href, "http://", "https://")}]} |
| 308 | + iex> other -> |
| 309 | + iex> other |
| 310 | + iex> end) |
| 311 | + [{"a", [{"href", "https://elixir-lang.com"}], ["Elixir"]}] |
| 312 | + """ |
| 313 | + |
| 314 | + @spec find_and_update( |
| 315 | + html_tree(), |
| 316 | + css_selector(), |
| 317 | + ({String.t(), [html_attribute()]} -> {String.t(), [html_attribute()]} | :delete) |
| 318 | + ) :: html_tree() |
| 319 | + def find_and_update(html_tree, selector, fun) do |
| 320 | + {tree, results} = Finder.find(html_tree, selector) |
| 321 | + |
| 322 | + operations_with_nodes = |
| 323 | + Enum.map(results, fn |
| 324 | + html_node = %Floki.HTMLTree.HTMLNode{} -> |
| 325 | + case fun.({html_node.type, html_node.attributes}) do |
| 326 | + {updated_tag, updated_attrs} -> |
| 327 | + {:update, %{html_node | type: updated_tag, attributes: updated_attrs}} |
| 328 | + |
| 329 | + :delete -> |
| 330 | + {:delete, html_node} |
| 331 | + end |
| 332 | + |
| 333 | + other -> |
| 334 | + {:no_op, other} |
| 335 | + end) |
| 336 | + |
| 337 | + tree |
| 338 | + |> HTMLTree.patch_nodes(operations_with_nodes) |
| 339 | + |> HTMLTree.to_tuple_list() |
| 340 | + end |
| 341 | + |
| 327 342 | @doc """ |
| 328 343 | Traverses and updates a HTML tree structure. |
| 329 344 | |
| @@ -626,8 +641,8 @@ defmodule Floki do | |
| 626 641 | |
| 627 642 | """ |
| 628 643 | |
| 629 | - @spec filter_out(binary() | html_tree() | html_tag(), FilterOut.selector()) :: |
| 630 | - html_tree() | html_tag() |
| 644 | + @spec filter_out(html_node() | html_tree() | binary(), FilterOut.selector()) :: |
| 645 | + html_node() | html_tree() |
| 631 646 | |
| 632 647 | def filter_out(html, selector) when is_binary(html) do |
| 633 648 | IO.warn( |
| @@ -1,51 +1,29 @@ | |
| 1 1 | defmodule Floki.FilterOut do |
| 2 2 | @moduledoc false |
| 3 3 | |
| 4 | - alias Floki.{HTMLTree, Finder} |
| 5 | - |
| 6 4 | # Helper functions for filtering out a specific element from the tree. |
| 7 5 | |
| 8 | - @type html_tree :: tuple | list |
| 9 | - @type selector :: :comment | :text | Finder.selector() |
| 6 | + @type selector :: :comment | :text | Floki.css_selector() |
| 10 7 | |
| 11 | - @spec filter_out(html_tree, selector) :: tuple | list |
| 12 | - |
| 13 | - def filter_out(html_tree, type) when type in [:text, :comment] do |
| 14 | - mapper(html_tree, type) |
| 8 | + def filter_out(html_tree_or_node, type) when type in [:text, :comment] do |
| 9 | + mapper(html_tree_or_node, type) |
| 15 10 | end |
| 16 11 | |
| 17 | - def filter_out(html_tree, selector) do |
| 18 | - case Finder.find(html_tree, selector) do |
| 19 | - {:empty_tree, _} -> |
| 20 | - html_tree |
| 21 | - |
| 22 | - {tree, results} -> |
| 23 | - new_tree = |
| 24 | - Enum.reduce(results, tree, fn html_node, tree -> |
| 25 | - HTMLTree.delete_node(tree, html_node) |
| 26 | - end) |
| 27 | - |
| 28 | - html_as_tuples = |
| 29 | - new_tree.root_nodes_ids |
| 30 | - |> Enum.reverse() |
| 31 | - |> Enum.map(fn node_id -> |
| 32 | - root = Map.get(new_tree.nodes, node_id) |
| 33 | - |
| 34 | - HTMLTree.to_tuple(new_tree, root) |
| 35 | - end) |
| 36 | - |
| 37 | - case html_tree do |
| 38 | - tree when is_tuple(tree) -> |
| 39 | - first_tuple(html_as_tuples) |
| 40 | - |
| 41 | - trees when is_list(trees) -> |
| 42 | - html_as_tuples |
| 43 | - end |
| 44 | - end |
| 12 | + def filter_out(html_tree, selector) when is_list(html_tree) do |
| 13 | + Floki.find_and_update(html_tree, selector, fn |
| 14 | + {_tag, _attrs} -> :delete |
| 15 | + other -> other |
| 16 | + end) |
| 45 17 | end |
| 46 18 | |
| 47 | - defp first_tuple([]), do: [] |
| 48 | - defp first_tuple([head | _rest]), do: head |
| 19 | + def filter_out(html_node, selector) do |
| 20 | + [html_node] |
| 21 | + |> Floki.find_and_update(selector, fn |
| 22 | + {_tag, _attrs} -> :delete |
| 23 | + other -> other |
| 24 | + end) |
| 25 | + |> List.first() || [] |
| 26 | + end |
| 49 27 | |
| 50 28 | defp filter({nodetext, _, _}, selector) when nodetext === selector, do: false |
| 51 29 | defp filter({nodetext, _}, selector) when nodetext === selector, do: false |
| @@ -9,16 +9,13 @@ defmodule Floki.Finder do | |
| 9 9 | alias Floki.{HTMLTree, Selector} |
| 10 10 | alias HTMLTree.HTMLNode |
| 11 11 | |
| 12 | - @type html_tree :: tuple | list |
| 13 | - @type selector :: binary() | %Selector{} | [%Selector{}] |
| 14 | - |
| 15 12 | # Find elements inside a HTML tree. |
| 16 13 | # Second argument can be either a selector string, a selector struct or a list of selector structs. |
| 17 14 | |
| 18 | - @spec find(html_tree, selector) :: html_tree |
| 15 | + @spec find(Floki.html_tree(), Floki.css_selector()) :: {HTMLTree.t(), [HTMLTree.HTMLNode.t()]} |
| 19 16 | |
| 20 | - def find([], _), do: {:empty_tree, []} |
| 21 | - def find(html_as_string, _) when is_binary(html_as_string), do: {:empty_tree, []} |
| 17 | + def find([], _), do: {%HTMLTree{}, []} |
| 18 | + def find(html_as_string, _) when is_binary(html_as_string), do: {%HTMLTree{}, []} |
| 22 19 | |
| 23 20 | def find(html_tree, selector_as_string) when is_binary(selector_as_string) do |
| 24 21 | selectors = get_selectors(selector_as_string) |
| @@ -33,7 +30,8 @@ defmodule Floki.Finder do | |
| 33 30 | find_selectors(html_tree, [selector]) |
| 34 31 | end |
| 35 32 | |
| 36 | - @spec map(html_tree, function) :: html_tree |
| 33 | + @spec map(Floki.html_tree() | Floki.html_node(), function()) :: |
| 34 | + Floki.html_tree() | Floki.html_node() |
| 37 35 | |
| 38 36 | def map({name, attrs, rest}, fun) do |
| 39 37 | {new_name, new_attrs} = fun.({name, attrs}) |
Loading more files…