Packages
floki
0.16.0
0.38.4
0.38.3
0.38.2
0.38.1
0.38.0
0.37.1
0.37.0
0.36.3
0.36.2
0.36.1
0.36.0
0.35.4
0.35.3
0.35.2
0.35.1
0.35.0
0.34.3
0.34.2
0.34.1
0.34.0
0.33.1
0.33.0
0.32.1
0.32.0
0.31.0
0.30.1
0.30.0
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.0
0.23.1
0.23.0
0.22.0
0.21.0
0.20.4
0.20.3
0.20.2
0.20.1
0.20.0
0.19.3
0.19.2
0.19.1
0.19.0
0.18.1
0.18.0
0.17.2
0.17.1
0.17.0
0.16.0
0.15.0
0.14.0
0.13.2
0.13.1
0.13.0
0.12.1
0.12.0
0.11.0
0.10.1
0.10.0
0.9.0
0.8.1
0.8.0
0.7.2
0.7.1
0.7.0
0.6.1
0.6.0
0.5.0
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.1
0.1.0
0.0.5
0.0.4
0.0.3
0.0.2
0.0.1
Floki is a simple HTML parser that enables search for nodes using CSS selectors.
Current section
86 Versions
Jump to
Current section
86 Versions
Compare versions
8
files changed
+117
additions
-76
deletions
| @@ -4,12 +4,12 @@ | |
| 4 4 | [](https://hex.pm/packages/floki) |
| 5 5 | [](https://hex.pm/packages/floki) |
| 6 6 | [](https://beta.hexfaktor.org/github/philss/floki) |
| 7 | - [](http://inch-ci.org/github/philss/floki) |
| 7 | + [](https://inch-ci.org/github/philss/floki) |
| 8 8 | [](https://ebertapp.io/github/philss/floki) |
| 9 9 | |
| 10 10 | Floki is a simple HTML parser that enables search for nodes using CSS selectors. |
| 11 11 | |
| 12 | - [Check the documentation](http://hexdocs.pm/floki). |
| 12 | + [Check the documentation](https://hexdocs.pm/floki). |
| 13 13 | |
| 14 14 | ## Usage |
| 15 15 | |
| @@ -22,7 +22,7 @@ Take this HTML as an example: | |
| 22 22 | <section id="content"> |
| 23 23 | <p class="headline">Floki</p> |
| 24 24 | <span class="headline">Enables search using CSS selectors</span> |
| 25 | - <a href="http://github.com/philss/floki">Github page</a> |
| 25 | + <a href="https://github.com/philss/floki">Github page</a> |
| 26 26 | <span data-model="user">philss</span> |
| 27 27 | </section> |
| 28 28 | <a href="https://hex.pm/packages/floki">Hex package</a> |
| @@ -33,14 +33,6 @@ Take this HTML as an example: | |
| 33 33 | Here are some queries that you can perform (with return examples): |
| 34 34 | |
| 35 35 | ```elixir |
| 36 | - Floki.find(html, "#content") |
| 37 | - # => [{"section", [{"id", "content"}], |
| 38 | - # => [{"p", [{"class", "headline"}], ["Floki"]}, |
| 39 | - # => {"span", [{"class", "headline"}], ["Enables search using CSS selectors"]}, |
| 40 | - # => {"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}, |
| 41 | - # => {"span", [{"data-model", "user"}], ["philss"]}]}] |
| 42 | - |
| 43 | - |
| 44 36 | Floki.find(html, "p.headline") |
| 45 37 | # => [{"p", [{"class", "headline"}], ["Floki"]}] |
| 46 38 | |
| @@ -50,17 +42,12 @@ Floki.find(html, "p.headline") | |
| 50 42 | # => <p class="headline">Floki</p> |
| 51 43 | |
| 52 44 | |
| 53 | - Floki.find(html, "a") |
| 54 | - # => [{"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}, |
| 55 | - # => {"a", [{"href", "https://hex.pm/packages/floki"}], ["Hex package"]}] |
| 56 | - |
| 57 | - |
| 58 45 | Floki.find(html, "a[href^=https]") |
| 59 46 | # => [{"a", [{"href", "https://hex.pm/packages/floki"}], ["Hex package"]}] |
| 60 47 | |
| 61 48 | |
| 62 49 | Floki.find(html, "#content a") |
| 63 | - # => [{"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}] |
| 50 | + # => [{"a", [{"href", "https://github.com/philss/floki"}], ["Github page"]}] |
| 64 51 | |
| 65 52 | |
| 66 53 | Floki.find(html, "[data-model=user]") |
| @@ -69,7 +56,7 @@ Floki.find(html, "[data-model=user]") | |
| 69 56 | |
| 70 57 | Floki.find(html, ".headline:nth-child(1), a") |
| 71 58 | # => [{"p", [{"class", "headline"}], ["Floki"]}, |
| 72 | - # => {"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}, |
| 59 | + # => {"a", [{"href", "https://github.com/philss/floki"}], ["Github page"]}, |
| 73 60 | # => {"a", [{"href", "https://hex.pm/packages/floki"}], ["Hex package"]}] |
| 74 61 | ``` |
| 75 62 | |
| @@ -81,8 +68,7 @@ Example of node: | |
| 81 68 | |
| 82 69 | {"p", [{"class", "headline"}], ["Floki"]} |
| 83 70 | |
| 84 | - So even if the only child node is the element text, it is represented |
| 85 | - inside a list. |
| 71 | + So even if the only child node is the element text, it is represented inside a list. |
| 86 72 | |
| 87 73 | You can write a simple HTML crawler with Floki and [HTTPoison](https://github.com/edgurgel/httpoison): |
| 88 74 | |
| @@ -103,7 +89,7 @@ Add Floki to your `mix.exs`: | |
| 103 89 | ```elixir |
| 104 90 | defp deps do |
| 105 91 | [ |
| 106 | - {:floki, "~> 0.14.0"} |
| 92 | + {:floki, "~> 0.16.0"} |
| 107 93 | ] |
| 108 94 | end |
| 109 95 | ``` |
| @@ -125,9 +111,8 @@ You can configure Floki to use [html5ever](https://github.com/servo/html5ever) a | |
| 125 111 | This is recommended if you need [better performance](https://gist.github.com/philss/70b4b0294f29501c3c7e0f60338cc8bd) |
| 126 112 | and a more accurate parser. However `html5ever` is being under active development and **may be unstable**. |
| 127 113 | |
| 128 | - Since it's written in Rust, we need to install Rust and compile the project. Luckily we have have the |
| 129 | - [html5ever Elixir NIF](https://github.com/hansihe/html5ever_elixir) that makes the integration very easy. |
| 130 | - For more info, check the article [Rustler - Safe Erlang and Elixir NIFs in Rust](http://hansihe.com/2017/02/05/rustler-safe-erlang-elixir-nifs-in-rust.html). |
| 114 | + Since it's written in Rust, we need to install Rust and compile the project. |
| 115 | + Luckily we have have the [html5ever Elixir NIF](https://github.com/hansihe/html5ever_elixir) that makes the integration very easy. |
| 131 116 | |
| 132 117 | You still need to install Rust in your system. To do that, please |
| 133 118 | [follow the instruction](https://www.rust-lang.org/en-US/install.html) presented in the official page. |
| @@ -139,7 +124,7 @@ After setup Rust, you need to add `html5ever` NIF to your dependency list: | |
| 139 124 | ```elixir |
| 140 125 | defp deps do |
| 141 126 | [ |
| 142 | - {:floki, "~> 0.14.0"}, |
| 127 | + {:floki, "~> 0.16.0"}, |
| 143 128 | {:html5ever, "~> 0.3.0"} |
| 144 129 | ] |
| 145 130 | end |
| @@ -157,6 +142,8 @@ config :floki, :html_parser, Floki.HTMLParser.Html5ever | |
| 157 142 | |
| 158 143 | After that you are able to use `html5ever` as your HTML parser with Floki. |
| 159 144 | |
| 145 | + For more info, check the article [Rustler - Safe Erlang and Elixir NIFs in Rust](http://hansihe.com/2017/02/05/rustler-safe-erlang-elixir-nifs-in-rust.html). |
| 146 | + |
| 160 147 | ## More about Floki API |
| 161 148 | |
| 162 149 | To parse a HTML document, try: |
| @@ -192,7 +179,7 @@ Floki.find(html, ".example") | |
| 192 179 | To fetch some attribute from elements, try: |
| 193 180 | |
| 194 181 | ```elixir |
| 195 | - Floki.attribute(html, ".example", "class") # href or src are good possibilities to fetch links |
| 182 | + Floki.attribute(html, ".example", "class") |
| 196 183 | # => ["example"] |
| 197 184 | ``` |
| @@ -27,4 +27,4 @@ | |
| 27 27 | {<<"name">>,<<"mochiweb">>}, |
| 28 28 | {<<"optional">>,false}, |
| 29 29 | {<<"requirement">>,<<"~> 2.15">>}]]}. |
| 30 | - {<<"version">>,<<"0.15.0">>}. |
| 30 | + {<<"version">>,<<"0.16.0">>}. |
| @@ -9,7 +9,6 @@ defmodule Floki.Finder do | |
| 9 9 | alias Floki.{Combinator, Selector, SelectorParser, SelectorTokenizer} |
| 10 10 | alias Floki.HTMLTree |
| 11 11 | alias Floki.HTMLTree.HTMLNode |
| 12 | - alias Floki.Selector.PseudoClass |
| 13 12 | |
| 14 13 | @type html_tree :: tuple | list |
| 15 14 | @type selector :: binary | %Selector{} | [%Selector{}] |
| @@ -86,27 +85,8 @@ defmodule Floki.Finder do | |
| 86 85 | end |
| 87 86 | end |
| 88 87 | |
| 89 | - defp selector_match?(_tree, html_node, selector = %Selector{pseudo_class: nil}) do |
| 90 | - Selector.match?(html_node, selector) |
| 91 | - end |
| 92 88 | defp selector_match?(tree, html_node, selector) do |
| 93 | - Selector.match?(html_node, selector) && pseudo_class_match?(tree, html_node, selector) |
| 94 | - end |
| 95 | - |
| 96 | - defp pseudo_class_match?(tree, html_node, selector) do |
| 97 | - pseudo_class = selector.pseudo_class |
| 98 | - |
| 99 | - case pseudo_class.name do |
| 100 | - "nth-child" -> |
| 101 | - PseudoClass.match_nth_child?(tree, html_node, pseudo_class) |
| 102 | - "first-child" -> |
| 103 | - PseudoClass.match_nth_child?(tree, html_node, %PseudoClass{name: "nth-child", value: 1}) |
| 104 | - "not" -> |
| 105 | - !Selector.match?(html_node, pseudo_class.value) |
| 106 | - unknown_pseudo_class -> |
| 107 | - Logger.warn("Pseudo-class #{inspect unknown_pseudo_class} is not implemented. Ignoring.") |
| 108 | - false |
| 109 | - end |
| 89 | + Selector.match?(html_node, selector, tree) |
| 110 90 | end |
| 111 91 | |
| 112 92 | # The stack serves as accumulator when there is another combinator to traverse. |
| @@ -1,9 +1,10 @@ | |
| 1 1 | defmodule Floki.Selector do |
| 2 | - @moduledoc """ |
| 3 | - Represents a CSS selector. It also have functions to match nodes with a given selector. |
| 4 | - """ |
| 2 | + require Logger |
| 3 | + @moduledoc false |
| 4 | + # Represents a CSS selector. It also have functions to match nodes with a given selector. |
| 5 5 | |
| 6 6 | alias Floki.{Selector, AttributeSelector} |
| 7 | + alias Floki.Selector.PseudoClass |
| 7 8 | alias Floki.HTMLTree.{HTMLNode, Text, Comment} |
| 8 9 | |
| 9 10 | defstruct id: nil, |
| @@ -14,31 +15,35 @@ defmodule Floki.Selector do | |
| 14 15 | pseudo_class: nil, |
| 15 16 | combinator: nil |
| 16 17 | |
| 17 | - @doc """ |
| 18 | - Returns if a given node matches with a given selector. |
| 19 | - """ |
| 20 | - def match?(_node, %Selector{id: nil, type: nil, classes: [], attributes: [], namespace: nil}) do |
| 18 | + @doc false |
| 19 | + |
| 20 | + # Returns if a given node matches with a given selector. |
| 21 | + def match?(_node, %Selector{id: nil, |
| 22 | + type: nil, |
| 23 | + classes: [], |
| 24 | + attributes: [], |
| 25 | + namespace: nil, |
| 26 | + pseudo_class: nil, |
| 27 | + combinator: nil}, _tree) do |
| 21 28 | false |
| 22 29 | end |
| 23 | - def match?(nil, _selector), do: false |
| 24 | - def match?({:comment, _comment}, _selector), do: false |
| 25 | - def match?({:pi, _xml, _xml_attrs}, _selector), do: false |
| 26 | - def match?(%Text{}, _), do: false |
| 27 | - def match?(%Comment{}, _), do: false |
| 28 | - def match?(%HTMLNode{type: type, attributes: attributes}, selector) do |
| 29 | - Selector.match?({type, attributes, []}, selector) |
| 30 | - end |
| 31 | - def match?(html_node, selector) do |
| 30 | + def match?(nil, _selector, _tree), do: false |
| 31 | + def match?({:comment, _comment}, _selector, _tree), do: false |
| 32 | + def match?({:pi, _xml, _xml_attrs}, _selector, _tree), do: false |
| 33 | + def match?(%Text{}, _selector, _tree), do: false |
| 34 | + def match?(%Comment{}, _selector, _tree), do: false |
| 35 | + def match?(html_node, selector, tree) do |
| 32 36 | id_match?(html_node, selector.id) |
| 33 37 | && namespace_match?(html_node, selector.namespace) |
| 34 38 | && type_match?(html_node, selector.type) |
| 35 39 | && classes_matches?(html_node, selector.classes) |
| 36 40 | && attributes_matches?(html_node, selector.attributes) |
| 41 | + && pseudo_class_match?(html_node, selector.pseudo_class, tree) |
| 37 42 | end |
| 38 43 | |
| 39 44 | defp id_match?(_node, nil), do: true |
| 40 | - defp id_match?({_, [], _}, _), do: false |
| 41 | - defp id_match?({_, attributes, _}, id) do |
| 45 | + defp id_match?(%HTMLNode{attributes: []}, _), do: false |
| 46 | + defp id_match?(%HTMLNode{attributes: attributes}, id) do |
| 42 47 | Enum.any? attributes, fn(attribute) -> |
| 43 48 | case attribute do |
| 44 49 | {"id", ^id} -> true |
| @@ -49,7 +54,8 @@ defmodule Floki.Selector do | |
| 49 54 | |
| 50 55 | defp namespace_match?(_node, nil), do: true |
| 51 56 | defp namespace_match?(_node, "*"), do: true |
| 52 | - defp namespace_match?({type_maybe_with_namespace, _, _}, namespace) do |
| 57 | + defp namespace_match?(%HTMLNode{type: :pi}, _type), do: false |
| 58 | + defp namespace_match?(%HTMLNode{type: type_maybe_with_namespace}, namespace) do |
| 53 59 | case String.split(type_maybe_with_namespace, ":") do |
| 54 60 | [ns, _type] -> |
| 55 61 | ns == namespace |
| @@ -59,7 +65,8 @@ defmodule Floki.Selector do | |
| 59 65 | |
| 60 66 | defp type_match?(_node, nil), do: true |
| 61 67 | defp type_match?(_node, "*"), do: true |
| 62 | - defp type_match?({type_maybe_with_namespace, _, _}, type) do |
| 68 | + defp type_match?(%HTMLNode{type: :pi}, _type), do: false |
| 69 | + defp type_match?(%HTMLNode{type: type_maybe_with_namespace}, type) do |
| 63 70 | case String.split(type_maybe_with_namespace, ":") do |
| 64 71 | [_ns, tp] -> |
| 65 72 | tp == type |
| @@ -70,8 +77,8 @@ defmodule Floki.Selector do | |
| 70 77 | defp type_match?(_, _), do: false |
| 71 78 | |
| 72 79 | defp classes_matches?(_node, []), do: true |
| 73 | - defp classes_matches?({_, [], _}, _), do: false |
| 74 | - defp classes_matches?({_, attributes, _}, classes) do |
| 80 | + defp classes_matches?(%HTMLNode{attributes: []}, _), do: false |
| 81 | + defp classes_matches?(%HTMLNode{attributes: attributes}, classes) do |
| 75 82 | Enum.all? classes, fn(class) -> |
| 76 83 | selector = %AttributeSelector{match_type: :includes, |
| 77 84 | attribute: "class", |
| @@ -82,10 +89,27 @@ defmodule Floki.Selector do | |
| 82 89 | end |
| 83 90 | |
| 84 91 | defp attributes_matches?(_node, []), do: true |
| 85 | - defp attributes_matches?({_, [], _}, _), do: false |
| 86 | - defp attributes_matches?({_, attributes, _}, attributes_selectors) do |
| 92 | + defp attributes_matches?(%HTMLNode{attributes: []}, _), do: false |
| 93 | + defp attributes_matches?(%HTMLNode{attributes: attributes}, attributes_selectors) do |
| 87 94 | Enum.all? attributes_selectors, fn(attribute_selector) -> |
| 88 95 | AttributeSelector.match?(attributes, attribute_selector) |
| 89 96 | end |
| 90 97 | end |
| 98 | + |
| 99 | + defp pseudo_class_match?(_html_node, nil, _tree), do: true |
| 100 | + defp pseudo_class_match?(html_node, pseudo_class, tree) do |
| 101 | + case pseudo_class.name do |
| 102 | + "nth-child" -> |
| 103 | + PseudoClass.match_nth_child?(tree, html_node, pseudo_class) |
| 104 | + "first-child" -> |
| 105 | + PseudoClass.match_nth_child?(tree, html_node, %PseudoClass{name: "nth-child", value: 1}) |
| 106 | + "not" -> |
| 107 | + !Selector.match?(html_node, pseudo_class.value, tree) |
| 108 | + "fl-contains" -> |
| 109 | + PseudoClass.match_contains?(tree, html_node, pseudo_class) |
| 110 | + unknown_pseudo_class -> |
| 111 | + Logger.warn("Pseudo-class #{inspect unknown_pseudo_class} is not implemented. Ignoring.") |
| 112 | + false |
| 113 | + end |
| 114 | + end |
| 91 115 | end |
| @@ -6,7 +6,7 @@ defmodule Floki.Selector.PseudoClass do | |
| 6 6 | # Represents a pseudo-class selector |
| 7 7 | defstruct name: "", value: nil |
| 8 8 | |
| 9 | - alias Floki.HTMLTree.HTMLNode |
| 9 | + alias Floki.HTMLTree.{HTMLNode, Text} |
| 10 10 | |
| 11 11 | def match_nth_child?(_, %HTMLNode{parent_node_id: nil}, _), do: false |
| 12 12 | def match_nth_child?(tree, html_node, %__MODULE__{value: position}) when is_integer(position) do |
| @@ -25,6 +25,17 @@ defmodule Floki.Selector.PseudoClass do | |
| 25 25 | false |
| 26 26 | end |
| 27 27 | |
| 28 | + def match_contains?(tree, html_node, %__MODULE__{value: value}) do |
| 29 | + res = Enum.find(html_node.children_nodes_ids, fn(id) -> |
| 30 | + case Map.get(tree.nodes, id) do |
| 31 | + %Text{content: content} -> content =~ value |
| 32 | + _ -> false |
| 33 | + end |
| 34 | + end) |
| 35 | + |
| 36 | + res != nil |
| 37 | + end |
| 38 | + |
| 28 39 | defp node_position(tree, html_node) do |
| 29 40 | parent_node = Map.get(tree.nodes, html_node.parent_node_id) |
| 30 41 | children_nodes_ids = parent_node.children_nodes_ids |
Loading more files…