Packages
floki
0.28.0
0.38.4
0.38.3
0.38.2
0.38.1
0.38.0
0.37.1
0.37.0
0.36.3
0.36.2
0.36.1
0.36.0
0.35.4
0.35.3
0.35.2
0.35.1
0.35.0
0.34.3
0.34.2
0.34.1
0.34.0
0.33.1
0.33.0
0.32.1
0.32.0
0.31.0
0.30.1
0.30.0
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.0
0.23.1
0.23.0
0.22.0
0.21.0
0.20.4
0.20.3
0.20.2
0.20.1
0.20.0
0.19.3
0.19.2
0.19.1
0.19.0
0.18.1
0.18.0
0.17.2
0.17.1
0.17.0
0.16.0
0.15.0
0.14.0
0.13.2
0.13.1
0.13.0
0.12.1
0.12.0
0.11.0
0.10.1
0.10.0
0.9.0
0.8.1
0.8.0
0.7.2
0.7.1
0.7.0
0.6.1
0.6.0
0.5.0
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.1
0.1.0
0.0.5
0.0.4
0.0.3
0.0.2
0.0.1
Floki is a simple HTML parser that enables search for nodes using CSS selectors.
Current section
86 Versions
Jump to
Current section
86 Versions
Compare versions
7
files changed
+95
additions
-60
deletions
| @@ -60,7 +60,7 @@ Add Floki to your `mix.exs`: | |
| 60 60 | ```elixir |
| 61 61 | defp deps do |
| 62 62 | [ |
| 63 | - {:floki, "~> 0.27.0"} |
| 63 | + {:floki, "~> 0.28.0"} |
| 64 64 | ] |
| 65 65 | end |
| 66 66 | ``` |
| @@ -89,7 +89,7 @@ concerns: | |
| 89 89 | - Performance - It can be [up to 20 times slower than the alternatives](https://hexdocs.pm/fast_html/readme.html#benchmarks) on big HTML |
| 90 90 | documents. |
| 91 91 | - Correctness - in some cases `mochiweb_html` will produce different results |
| 92 | - from what is specified in [HTML5 specification](https://html.spec.whatwg.org/)](https://html.spec.whatwg.org/). |
| 92 | + from what is specified in [HTML5 specification](https://html.spec.whatwg.org/). |
| 93 93 | For example, a correct parser would parse `<title> <b> bold </b> text </title>` |
| 94 94 | as `{"title", [], [" <b> bold </b> text "]}` since content inside `<title>` is |
| 95 95 | to be [treated as plaintext](https://html.spec.whatwg.org/#the-title-element). |
| @@ -97,13 +97,12 @@ concerns: | |
| 97 97 | |
| 98 98 | Floki supports the following alternative parsers: |
| 99 99 | |
| 100 | - - `fast_html` - A wrapper for lexborisov's [myhtml](https://github.com/lexborisov/myhtml/). A pure C HTML parser. |
| 100 | + - `fast_html` - A wrapper for [lexbor](https://github.com/lexbor/lexbor). A pure C HTML parser. |
| 101 101 | - `html5ever` - A wrapper for [html5ever](https://github.com/servo/html5ever) written in Rust, developed as a part of the Servo project. |
| 102 102 | |
| 103 103 | `fast_html` is generally faster, according to the |
| 104 104 | [benchmarks](https://hexdocs.pm/fast_html/readme.html#benchmarks) conducted by |
| 105 | - its developers. Though `html5ever` does have an advantage on really small |
| 106 | - (~4kb) fragments due to it being implemented as a NIF. |
| 105 | + its developers. |
| 107 106 | |
| 108 107 | #### Using `html5ever` as the HTML parser |
| 109 108 | |
| @@ -115,8 +114,8 @@ After Rust is set up, you need to add `html5ever` NIF to your dependency list: | |
| 115 114 | ```elixir |
| 116 115 | defp deps do |
| 117 116 | [ |
| 118 | - {:floki, "~> 0.27.0"}, |
| 119 | - {:html5ever, "~> 0.7.0"} |
| 117 | + {:floki, "~> 0.28.0"}, |
| 118 | + {:html5ever, "~> 0.8.0"} |
| 120 119 | ] |
| 121 120 | end |
| 122 121 | ``` |
| @@ -135,20 +134,16 @@ For more info, check the article [Rustler - Safe Erlang and Elixir NIFs in Rust] | |
| 135 134 | |
| 136 135 | #### Using `fast_html` as the HTML parser |
| 137 136 | |
| 138 | - A C compiler and GNU\Make needs to be installed on the system in order to |
| 139 | - compile myhtml. It's likely that your machine has them already. |
| 140 | - |
| 141 | - Note that you also need to have `epmd` started/available to start due to `fast_html` relying on a |
| 142 | - C-Node worker, usually it will be started automatically, but some distributions |
| 143 | - (i.e Gentoo Linux) enforce only being able to start it as a service. |
| 137 | + A C compiler, GNU\Make and CMake need to be installed on the system in order to |
| 138 | + compile lexbor. |
| 144 139 | |
| 145 140 | First, add `fast_html` to your dependencies: |
| 146 141 | |
| 147 142 | ```elixir |
| 148 143 | defp deps do |
| 149 144 | [ |
| 150 | - {:floki, "~> 0.27.0"}, |
| 151 | - {:fast_html, "~> 1.0"} |
| 145 | + {:floki, "~> 0.28.0"}, |
| 146 | + {:fast_html, "~> 2.0"} |
| 152 147 | ] |
| 153 148 | end |
| 154 149 | ``` |
| @@ -245,6 +240,7 @@ Here you find all the [CSS selectors](https://www.w3.org/TR/selectors/#selectors | |
| 245 240 | | E:nth-last-of-type(n) | an E element, the n-th child of its type among its siblings, counting from bottom to up | |
| 246 241 | | E:first-of-type | an E element, first child of its type among its siblings | |
| 247 242 | | E:last-of-type | an E element, last child of its type among its siblings | |
| 243 | + | E:checked | An E element (checkbox, radio, or option) that is checked | |
| 248 244 | | E.warning | an E element whose class is "warning" | |
| 249 245 | | E#myid | an E element with ID equal to "myid" | |
| 250 246 | | E:not(s) | an E element that does not match simple selector s | |
| @@ -2,27 +2,27 @@ | |
| 2 2 | {<<"build_tools">>,[<<"mix">>]}. |
| 3 3 | {<<"description">>, |
| 4 4 | <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}. |
| 5 | - {<<"elixir">>,<<"~> 1.6">>}. |
| 5 | + {<<"elixir">>,<<"~> 1.7">>}. |
| 6 6 | {<<"files">>, |
| 7 | - [<<"lib">>,<<"lib/floki">>,<<"lib/floki/html_parser.ex">>, |
| 8 | - <<"lib/floki/html_parser">>,<<"lib/floki/html_parser/fast_html.ex">>, |
| 9 | - <<"lib/floki/html_parser/mochiweb.ex">>, |
| 10 | - <<"lib/floki/html_parser/html5ever.ex">>,<<"lib/floki/selector">>, |
| 11 | - <<"lib/floki/selector/functional.ex">>,<<"lib/floki/selector/parser.ex">>, |
| 12 | - <<"lib/floki/selector/attribute_selector.ex">>, |
| 13 | - <<"lib/floki/selector/pseudo_class.ex">>, |
| 14 | - <<"lib/floki/selector/tokenizer.ex">>, |
| 15 | - <<"lib/floki/selector/combinator.ex">>,<<"lib/floki/deep_text.ex">>, |
| 16 | - <<"lib/floki/filter_out.ex">>,<<"lib/floki/flat_text.ex">>, |
| 17 | - <<"lib/floki/finder.ex">>,<<"lib/floki/traversal.ex">>, |
| 7 | + [<<"lib">>,<<"lib/floki">>,<<"lib/floki/deep_text.ex">>, |
| 18 8 | <<"lib/floki/raw_html.ex">>,<<"lib/floki/parse_error.ex">>, |
| 19 9 | <<"lib/floki/html_tree">>,<<"lib/floki/html_tree/id_seeder.ex">>, |
| 20 | - <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_tree/text.ex">>, |
| 21 | - <<"lib/floki/html_tree/html_node.ex">>,<<"lib/floki/selector.ex">>, |
| 22 | - <<"lib/floki/html_tree.ex">>,<<"lib/floki.ex">>, |
| 23 | - <<"src/floki_selector_lexer.xrl">>,<<"src/floki_mochi_html.erl">>, |
| 24 | - <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>, |
| 25 | - <<"CONTRIBUTING.md">>]}. |
| 10 | + <<"lib/floki/html_tree/text.ex">>,<<"lib/floki/html_tree/html_node.ex">>, |
| 11 | + <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_parser">>, |
| 12 | + <<"lib/floki/html_parser/mochiweb.ex">>, |
| 13 | + <<"lib/floki/html_parser/fast_html.ex">>, |
| 14 | + <<"lib/floki/html_parser/html5ever.ex">>,<<"lib/floki/flat_text.ex">>, |
| 15 | + <<"lib/floki/html_tree.ex">>,<<"lib/floki/selector.ex">>, |
| 16 | + <<"lib/floki/finder.ex">>,<<"lib/floki/traversal.ex">>, |
| 17 | + <<"lib/floki/filter_out.ex">>,<<"lib/floki/selector">>, |
| 18 | + <<"lib/floki/selector/pseudo_class.ex">>, |
| 19 | + <<"lib/floki/selector/attribute_selector.ex">>, |
| 20 | + <<"lib/floki/selector/parser.ex">>,<<"lib/floki/selector/combinator.ex">>, |
| 21 | + <<"lib/floki/selector/functional.ex">>, |
| 22 | + <<"lib/floki/selector/tokenizer.ex">>,<<"lib/floki/html_parser.ex">>, |
| 23 | + <<"lib/floki.ex">>,<<"src/floki_selector_lexer.xrl">>, |
| 24 | + <<"src/floki_mochi_html.erl">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>, |
| 25 | + <<"CODE_OF_CONDUCT.md">>,<<"CONTRIBUTING.md">>]}. |
| 26 26 | {<<"licenses">>,[<<"MIT">>]}. |
| 27 27 | {<<"links">>,[{<<"GitHub">>,<<"https://github.com/philss/floki">>}]}. |
| 28 28 | {<<"name">>,<<"floki">>}. |
| @@ -32,4 +32,4 @@ | |
| 32 32 | {<<"optional">>,false}, |
| 33 33 | {<<"repository">>,<<"hexpm">>}, |
| 34 34 | {<<"requirement">>,<<"~> 0.5.0">>}]]}. |
| 35 | - {<<"version">>,<<"0.27.0">>}. |
| 35 | + {<<"version">>,<<"0.28.0">>}. |
| @@ -331,23 +331,31 @@ defmodule Floki do | |
| 331 331 | given `fun` on all nodes. The tree is traversed in a post-walk fashion, where |
| 332 332 | the children are traversed before the parent. |
| 333 333 | |
| 334 | - The function `fun` receives a tuple with `{name, attributes, children}`, and |
| 335 | - should either return a similar tuple or `nil` to delete the current node. |
| 334 | + When the function `fun` encounters HTML tag, it receives a tuple with |
| 335 | + `{name, attributes, children}`, and should either return a similar tuple or |
| 336 | + `nil` to delete the current node. |
| 337 | + |
| 338 | + The function `fun` can also encounter HTML doctype, comment or declaration and |
| 339 | + will receive, and should return, different tuple for these types. See the |
| 340 | + documentation for `t:html_comment/0`, `t:html_doctype/0` and |
| 341 | + `t:html_declaration/0` for details. |
| 336 342 | |
| 337 343 | ## Examples |
| 338 344 | |
| 339 | - iex> html = {"div", [], ["hello"]} |
| 340 | - iex> Floki.traverse_and_update(html, fn {"div", attrs, children} -> |
| 341 | - ...> {"p", attrs, children} |
| 345 | + iex> html = [{"div", [], ["hello"]}] |
| 346 | + iex> Floki.traverse_and_update(html, fn |
| 347 | + ...> {"div", attrs, children} -> {"p", attrs, children} |
| 348 | + ...> other -> other |
| 342 349 | ...> end) |
| 343 | - {"p", [], ["hello"]} |
| 350 | + [{"p", [], ["hello"]}] |
| 344 351 | |
| 345 | - iex> html = {"div", [], [{"span", [], ["hello"]}]} |
| 352 | + iex> html = [{"div", [], [{:comment, "I am comment"}, {"span", [], ["hello"]}]}] |
| 346 353 | iex> Floki.traverse_and_update(html, fn |
| 347 354 | ...> {"span", _attrs, _children} -> nil |
| 348 | - ...> tag -> tag |
| 355 | + ...> {:comment, text} -> {"span", [], text} |
| 356 | + ...> other -> other |
| 349 357 | ...> end) |
| 350 | - {"div", [], []} |
| 358 | + [{"div", [], [{"span", [], "I am comment"}]}] |
| 351 359 | """ |
| 352 360 | |
| 353 361 | @spec traverse_and_update(html_tree(), (html_node() -> html_node() | nil)) :: html_tree() |
| @@ -362,19 +370,27 @@ defmodule Floki do | |
| 362 370 | traversed in a post-walk fashion, where the children are traversed before |
| 363 371 | the parent. |
| 364 372 | |
| 365 | - The function `fun` receives a tuple with `{name, attributes, children}` and |
| 366 | - an accumulator, and should return a 2-tuple like `{new_node, new_acc}`, where |
| 367 | - `new_node` is either a similar tuple or `nil` to delete the current node, and |
| 368 | - `new_acc` is an updated value for the accumulator. |
| 373 | + When the function `fun` encounters HTML tag, it receives a tuple with |
| 374 | + `{name, attributes, children}` and an accumulator. It and should return a |
| 375 | + 2-tuple like `{new_node, new_acc}`, where `new_node` is either a similar tuple |
| 376 | + or `nil` to delete the current node, and `new_acc` is an updated value for the |
| 377 | + accumulator. |
| 378 | + |
| 379 | + The function `fun` can also encounter HTML doctype, comment or declaration and |
| 380 | + will receive, and should return, different tuple for these types. See the |
| 381 | + documentation for `t:html_comment/0`, `t:html_doctype/0` and |
| 382 | + `t:html_declaration/0` for details. |
| 369 383 | |
| 370 384 | ## Examples |
| 371 385 | |
| 372 | - iex> html = [{"div", [], ["hello"]}, {"div", [], ["world"]}] |
| 373 | - iex> Floki.traverse_and_update(html, 0, fn {"div", attrs, children}, acc -> |
| 374 | - ...> {{"p", [{"data-count", to_string(acc)} | attrs], children}, acc + 1} |
| 386 | + iex> html = [{"div", [], [{:comment, "I am a comment"}, "hello"]}, {"div", [], ["world"]}] |
| 387 | + iex> Floki.traverse_and_update(html, 0, fn |
| 388 | + ...> {"div", attrs, children}, acc -> |
| 389 | + ...> {{"p", [{"data-count", to_string(acc)} | attrs], children}, acc + 1} |
| 390 | + ...> other, acc -> {other, acc} |
| 375 391 | ...> end) |
| 376 392 | {[ |
| 377 | - {"p", [{"data-count", "0"}], ["hello"]}, |
| 393 | + {"p", [{"data-count", "0"}], [{:comment, "I am a comment"}, "hello"]}, |
| 378 394 | {"p", [{"data-count", "1"}], ["world"]} |
| 379 395 | ], 2} |
| 380 396 | |
| @@ -457,7 +473,7 @@ defmodule Floki do | |
| 457 473 | end |
| 458 474 | |
| 459 475 | @doc """ |
| 460 | - Returns the direct child nodes of a HTML tree. |
| 476 | + Returns the direct child nodes of a HTML node. |
| 461 477 | |
| 462 478 | By default, it will also include all texts. You can disable |
| 463 479 | this behaviour by using the option `include_text` to `false`. |
| @@ -477,10 +493,10 @@ defmodule Floki do | |
| 477 493 | |
| 478 494 | """ |
| 479 495 | |
| 480 | - @spec children(html_tree, Keyword.t()) :: html_tree | nil |
| 496 | + @spec children(html_node(), Keyword.t()) :: html_tree() | nil |
| 481 497 | |
| 482 | - def children(html, opts \\ [include_text: true]) do |
| 483 | - case html do |
| 498 | + def children(html_node, opts \\ [include_text: true]) do |
| 499 | + case html_node do |
| 484 500 | {_, _, subtree} -> |
| 485 501 | filter_children(subtree, opts[:include_text]) |
| 486 502 | |
| @@ -599,7 +615,7 @@ defmodule Floki do | |
| 599 615 | iex> Floki.filter_out({"div", [], [{"script", [], ["hello"]}, " world"]}, "script") |
| 600 616 | {"div", [], [" world"]} |
| 601 617 | |
| 602 | - iex> Floki.filter_out([{"body", [], [{"script", [], []},{"div", [], []}]}], "script") |
| 618 | + iex> Floki.filter_out([{"body", [], [{"script", [], []}, {"div", [], []}]}], "script") |
| 603 619 | [{"body", [], [{"div", [], []}]}] |
| 604 620 | |
| 605 621 | iex> Floki.filter_out({"div", [], [{:comment, "comment"}, " text"]}, :comment) |
| @@ -610,7 +626,8 @@ defmodule Floki do | |
| 610 626 | |
| 611 627 | """ |
| 612 628 | |
| 613 | - @spec filter_out(binary | html_tree, FilterOut.selector()) :: list |
| 629 | + @spec filter_out(binary() | html_tree() | html_tag(), FilterOut.selector()) :: |
| 630 | + html_tree() | html_tag() |
| 614 631 | |
| 615 632 | def filter_out(html, selector) when is_binary(html) do |
| 616 633 | IO.warn( |
| @@ -10,7 +10,7 @@ defmodule Floki.Finder do | |
| 10 10 | alias HTMLTree.HTMLNode |
| 11 11 | |
| 12 12 | @type html_tree :: tuple | list |
| 13 | - @type selector :: binary | %Selector{} | [%Selector{}] |
| 13 | + @type selector :: binary() | %Selector{} | [%Selector{}] |
| 14 14 | |
| 15 15 | # Find elements inside a HTML tree. |
| 16 16 | # Second argument can be either a selector string, a selector struct or a list of selector structs. |
| @@ -187,6 +187,10 @@ defmodule Floki.Selector do | |
| 187 187 | Enum.all?(pseudo_class.value, &(!Selector.match?(html_node, &1, tree))) |
| 188 188 | end |
| 189 189 | |
| 190 | + defp pseudo_class_match?(html_node, %{name: "checked"}, _tree) do |
| 191 | + PseudoClass.match_checked?(html_node) |
| 192 | + end |
| 193 | + |
| 190 194 | defp pseudo_class_match?(html_node, pseudo_class = %{name: "fl-contains"}, tree) do |
| 191 195 | PseudoClass.match_contains?(tree, html_node, pseudo_class) |
| 192 196 | end |
Loading more files…