Packages

floki

0.18.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
1 Dependency
Dependants
Files
Activity
Hide whitespace

Compare versions

17 files changed
+531 additions
-340 deletions
  @@ -3,7 +3,6 @@
3 3 [![Build status](https://travis-ci.org/philss/floki.svg?branch=master)](https://travis-ci.org/philss/floki)
4 4 [![Floki version](https://img.shields.io/hexpm/v/floki.svg)](https://hex.pm/packages/floki)
5 5 [![Hex.pm](https://img.shields.io/hexpm/dt/floki.svg)](https://hex.pm/packages/floki)
6 - [![Deps Status](https://beta.hexfaktor.org/badge/all/github/philss/floki.svg)](https://beta.hexfaktor.org/github/philss/floki)
7 6 [![Inline docs](https://inch-ci.org/github/philss/floki.svg?branch=master)](https://inch-ci.org/github/philss/floki)
8 7 [![Ebert](https://ebertapp.io/github/philss/floki.svg)](https://ebertapp.io/github/philss/floki)
9 8
  @@ -40,24 +39,6 @@ Floki.find(html, "p.headline")
40 39 Floki.find(html, "p.headline")
41 40 |> Floki.raw_html
42 41 # => <p class="headline">Floki</p>
43 -
44 -
45 - Floki.find(html, "a[href^=https]")
46 - # => [{"a", [{"href", "https://hex.pm/packages/floki"}], ["Hex package"]}]
47 -
48 -
49 - Floki.find(html, "#content a")
50 - # => [{"a", [{"href", "https://github.com/philss/floki"}], ["Github page"]}]
51 -
52 -
53 - Floki.find(html, "[data-model=user]")
54 - # => [{"span", [{"data-model", "user"}], ["philss"]}]
55 -
56 -
57 - Floki.find(html, ".headline:nth-child(1), a")
58 - # => [{"p", [{"class", "headline"}], ["Floki"]},
59 - # => {"a", [{"href", "https://github.com/philss/floki"}], ["Github page"]},
60 - # => {"a", [{"href", "https://hex.pm/packages/floki"}], ["Hex package"]}]
61 42 ```
62 43
63 44 Each HTML node is represented by a tuple like:
  @@ -89,7 +70,7 @@ Add Floki to your `mix.exs`:
89 70 ```elixir
90 71 defp deps do
91 72 [
92 - {:floki, "~> 0.17.0"}
73 + {:floki, "~> 0.18.0"}
93 74 ]
94 75 end
95 76 ```
  @@ -124,8 +105,8 @@ After setup Rust, you need to add `html5ever` NIF to your dependency list:
124 105 ```elixir
125 106 defp deps do
126 107 [
127 - {:floki, "~> 0.17.0"},
128 - {:html5ever, "~> 0.3.0"}
108 + {:floki, "~> 0.18.0"},
109 + {:html5ever, "~> 0.5.0"}
129 110 ]
130 111 end
131 112 ```
  @@ -200,6 +181,37 @@ Floki.find(html, ".headline")
200 181 # => "Floki"
201 182 ```
202 183
184 + ## Supported selectors
185 +
186 + Here you find all the [CSS selectors](https://www.w3.org/TR/selectors/#selectors) supported in the current version:
187 +
188 + | Pattern | Description |
189 + |-----------------|------------------------------|
190 + | * | any element |
191 + | E | an element of type `E` |
192 + | E[foo] | an `E` element with a "foo" attribute |
193 + | E[foo="bar"] | an E element whose "foo" attribute value is exactly equal to "bar" |
194 + | E[foo~="bar"] | an E element whose "foo" attribute value is a list of whitespace-separated values, one of which is exactly equal to "bar" |
195 + | E[foo^="bar"] | an E element whose "foo" attribute value begins exactly with the string "bar" |
196 + | E[foo$="bar"] | an E element whose "foo" attribute value ends exactly with the string "bar" |
197 + | E[foo*="bar"] | an E element whose "foo" attribute value contains the substring "bar" |
198 + | E[foo\|="en"] | an E element whose "foo" attribute has a hyphen-separated list of values beginning (from the left) with "en" |
199 + | E:nth-child(n) | an E element, the n-th child of its parent |
200 + | E:first-child | an E element, first child of its parent |
201 + | E.warning | an E element whose class is "warning" |
202 + | E#myid | an E element with ID equal to "myid" |
203 + | E:not(s) | an E element that does not match simple selector s |
204 + | E F | an F element descendant of an E element |
205 + | E > F | an F element child of an E element |
206 + | E + F | an F element immediately preceded by an E element |
207 + | E ~ F | an F element preceded by an E element |
208 +
209 + There are also some selectors based on non-standard specifications. They are:
210 +
211 + | Pattern | Description |
212 + |----------------------|-----------------------------------------------------|
213 + | E:fl-contains('foo') | an E element that contains "foo" inside a text node |
214 +
203 215 ## License
204 216
205 217 Floki is under MIT license. Check the `LICENSE` file for more details.
  @@ -2,20 +2,20 @@
2 2 {<<"build_tools">>,[<<"mix">>]}.
3 3 {<<"description">>,
4 4 <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}.
5 - {<<"elixir">>,<<">= 1.2.0">>}.
5 + {<<"elixir">>,<<">= 1.3.0">>}.
6 6 {<<"files">>,
7 - [<<"lib/floki.ex">>,<<"lib/floki/attribute_selector.ex">>,
8 - <<"lib/floki/combinator.ex">>,<<"lib/floki/deep_text.ex">>,
7 + [<<"lib/floki.ex">>,<<"lib/floki/deep_text.ex">>,
9 8 <<"lib/floki/filter_out.ex">>,<<"lib/floki/finder.ex">>,
10 9 <<"lib/floki/flat_text.ex">>,<<"lib/floki/html_parser.ex">>,
11 10 <<"lib/floki/html_parser/html5ever.ex">>,
12 11 <<"lib/floki/html_parser/mochiweb.ex">>,<<"lib/floki/html_tree.ex">>,
13 12 <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_tree/html_node.ex">>,
14 13 <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/text.ex">>,
15 - <<"lib/floki/selector.ex">>,<<"lib/floki/selector/pseudo_class.ex">>,
16 - <<"lib/floki/selector_parser.ex">>,<<"lib/floki/selector_tokenizer.ex">>,
17 - <<"src/floki_selector_lexer.xrl">>,<<"mix.exs">>,<<"README.md">>,
18 - <<"LICENSE">>]}.
14 + <<"lib/floki/selector.ex">>,<<"lib/floki/selector/attribute_selector.ex">>,
15 + <<"lib/floki/selector/combinator.ex">>,<<"lib/floki/selector/parser.ex">>,
16 + <<"lib/floki/selector/pseudo_class.ex">>,
17 + <<"lib/floki/selector/tokenizer.ex">>,<<"src/floki_selector_lexer.xrl">>,
18 + <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>]}.
19 19 {<<"licenses">>,[<<"MIT">>]}.
20 20 {<<"links">>,
21 21 [{<<"Docs">>,<<"https://hexdocs.pm/floki">>},
  @@ -27,4 +27,4 @@
27 27 {<<"name">>,<<"mochiweb">>},
28 28 {<<"optional">>,false},
29 29 {<<"requirement">>,<<"~> 2.15">>}]]}.
30 - {<<"version">>,<<"0.17.2">>}.
30 + {<<"version">>,<<"0.18.0">>}.
  @@ -93,6 +93,9 @@ defmodule Floki do
93 93 defp raw_html(tuple, html) when is_tuple(tuple), do: raw_html([tuple], html)
94 94 defp raw_html([string|tail], html) when is_binary(string), do: raw_html(tail, html <> string)
95 95 defp raw_html([{:comment, comment}|tail], html), do: raw_html(tail, html <> "<!--#{comment}-->")
96 + defp raw_html([{:pi, "xml", attrs} |tail], html) do
97 + raw_html(tail, html <> "<?xml " <> tag_attrs(attrs) <> "?>")
98 + end
96 99 defp raw_html([{type, attrs, children}|tail], html) do
97 100 raw_html(tail, html <> tag_for(type, tag_attrs(attrs), children))
98 101 end
  @@ -100,13 +103,13 @@ defmodule Floki do
100 103 defp tag_attrs(attr_list) do
101 104 attr_list
102 105 |> Enum.reduce("", &build_attrs/2)
103 - |> String.strip
106 + |> String.trim
104 107 end
105 108
106 109 defp build_attrs({attr, value}, attrs), do: ~s(#{attrs} #{attr}="#{value}")
107 110 defp build_attrs(attr, attrs), do: "#{attrs} #{attr}"
108 111
109 - defp tag_for(type, attrs, _children) when type in @self_closing_tags do
112 + defp tag_for(type, attrs, []) when type in @self_closing_tags do
110 113 case attrs do
111 114 "" -> "<#{type}/>"
112 115 _ -> "<#{type} #{attrs}/>"
  @@ -153,14 +156,86 @@ defmodule Floki do
153 156 Enum.map(results, fn(html_node) -> HTMLTree.to_tuple(tree, html_node) end)
154 157 end
155 158
156 - def transform(html_tree_list, transformation) when is_list(html_tree_list) do
157 - Enum.map(html_tree_list, fn(html_tree) ->
158 - Finder.apply_transformation(html_tree, transformation)
159 + @doc """
160 + Changes the attribute values of the elements matched by `selector`
161 + with the function `mutation` and returns the whole element tree
162 +
163 + ## Examples
164 + iex> Floki.attr("<div id='a'></div>", "#a", "id", fn(id) -> String.replace(id, "a", "b") end)
165 + [{"div", [{"id", "b"}], []}]
166 +
167 + iex> Floki.attr("<div class='class_name'></div>", "div", "id", fn _ -> "b" end)
168 + [{"div", [{"id", "b"}, {"class", "class_name"}], []}]
169 +
170 + """
171 + @spec attr(binary | html_tree, binary, binary, (binary -> binary)) :: html_tree
172 +
173 + def attr(html_elem_tuple, selector, attribute_name, mutation) when is_tuple(html_elem_tuple) do
174 + attr([html_elem_tuple], selector, attribute_name, mutation)
175 + end
176 + def attr(html_str, selector, attribute_name, mutation) when is_binary(html_str) do
177 + attr(parse(html_str), selector, attribute_name, mutation)
178 + end
179 + def attr(html_tree_list, selector, attribute_name, mutation) when is_list(html_tree_list) do
180 + {tree, results} = Finder.find(html_tree_list, selector)
181 + mutate_attrs(html_tree_list, tree, results, attribute_name, mutation)
182 + end
183 +
184 + defp add_nodes_to_tree(tree, [html_node]) do
185 + nodes = Map.put(tree.nodes, html_node.node_id, html_node)
186 + Map.put(tree, :nodes, nodes)
187 + end
188 + defp add_nodes_to_tree(tree, [html_node | tail]) do
189 + nodes = Map.put(tree.nodes, html_node.node_id, html_node)
190 +
191 + tree
192 + |> Map.put(:nodes, nodes)
193 + |> add_nodes_to_tree(tail)
194 + end
195 +
196 + defp mutate_attrs(html_tree_list, _, [], _, _), do: html_tree_list
197 + defp mutate_attrs(_, tree, results, attribute_name, mutation_fn) do
198 + mutated_nodes = Enum.map(results, fn(result) ->
199 + mutated_attributes =
200 + if Enum.any?(result.attributes, & match?({^attribute_name, _}, &1)) do
201 + Enum.map(result.attributes, fn(attribute) ->
202 + with {^attribute_name, attribute_value} <- attribute do
203 + {attribute_name, mutation_fn.(attribute_value)}
204 + end
205 + end)
206 + else
207 + [{attribute_name, mutation_fn.(nil)} | result.attributes]
208 + end
209 +
210 + Map.put(result, :attributes, mutated_attributes)
159 211 end)
212 +
213 + tree = add_nodes_to_tree(tree, mutated_nodes)
214 +
215 + tree.nodes
216 + |> Map.values
217 + |> Enum.filter(fn(actual_node) -> is_nil(actual_node.parent_node_id) end)
218 + |> Enum.map(fn(html_node) -> HTMLTree.to_tuple(tree, html_node) end)
160 219 end
161 - def transform(html_tree, transformation) do
162 - Finder.apply_transformation(html_tree, transformation)
220 +
221 + @doc """
222 + It receives a HTML tree structure as tuple and maps
223 + through all nodes with a given function that receives
224 + a tuple with {name, attributes}.
225 +
226 + It returns that structure transformed by the function.
227 +
228 + ## Examples
229 +
230 + iex> html = {"div", [{"class", "foo"}], ["text"]}
231 + iex> Floki.map(html, fn({name, attrs}) -> {name, [{"data-name", "bar"} | attrs]} end)
232 + {"div", [{"data-name", "bar"}, {"class", "foo"}], ["text"]}
233 +
234 + """
235 + def map(html_tree_list, fun) when is_list(html_tree_list) do
236 + Enum.map(html_tree_list, &(Finder.map(&1, fun)))
163 237 end
238 + def map(html_tree, fun), do: Finder.map(html_tree, fun)
164 239
165 240 @doc """
166 241 Returns the text nodes from a HTML tree.
  @@ -1,51 +0,0 @@
1 - defmodule Floki.AttributeSelector do
2 - @moduledoc false
3 -
4 - # It is very similar to the `Selector` module, but is specialized in attributes
5 - # and attribute selectors.
6 -
7 - alias Floki.AttributeSelector
8 -
9 - defstruct match_type: nil, attribute: nil, value: nil
10 -
11 - # Returns if attributes of a node matches with a given attribute selector.
12 - def match?(attributes, s = %AttributeSelector{match_type: nil, value: nil}) do
13 - attribute_present?(s.attribute, attributes)
14 - end
15 - def match?(attributes, s = %AttributeSelector{match_type: :equal}) do
16 - get_value(s.attribute, attributes) == s.value
17 - end
18 - def match?(attributes, s = %AttributeSelector{match_type: :includes}) do
19 - value = get_value(s.attribute, attributes)
20 -
21 - whitespace_values = String.split(value, " ")
22 -
23 - Enum.any?(whitespace_values, fn(v) -> v == s.value end)
24 - end
25 - def match?(attributes, s = %AttributeSelector{match_type: :dash_match}) do
26 - value = get_value(s.attribute, attributes)
27 -
28 - value == s.value || String.starts_with?(value, "#{s.value}-")
29 - end
30 - def match?(attributes, s = %AttributeSelector{match_type: :prefix_match}) do
31 - s.attribute |> get_value(attributes) |> String.starts_with?(s.value)
32 - end
33 - def match?(attributes, s = %AttributeSelector{match_type: :sufix_match}) do
34 - s.attribute |> get_value(attributes) |> String.ends_with?(s.value)
35 - end
36 - def match?(attributes, s = %AttributeSelector{match_type: :substring_match}) do
37 - s.attribute |> get_value(attributes) |> String.contains?(s.value)
38 - end
39 -
40 - defp get_value(attr_name, attributes) do
41 - {_attr_name, value} = Enum.find(attributes, {attr_name, ""}, fn({k, _v}) ->
42 - k == attr_name
43 - end)
44 -
45 - value
46 - end
47 -
48 - defp attribute_present?(name, attributes) do
49 - Enum.any?(attributes, fn({k, _v}) -> k == name end)
50 - end
51 - end
  @@ -1,18 +0,0 @@
1 - defmodule Floki.Combinator do
2 - @moduledoc false
3 -
4 - # Represents the conjunction of a combinator with its selector.
5 - #
6 - # Combinators can have the following match types:
7 - #
8 - # - descendant;
9 - # e.g.: "a b"
10 - # - child;
11 - # e.g.: "a > b"
12 - # - adjacent sibling;
13 - # e.g.: "a + b"
14 - # - general sibling;
15 - # e.g.: "a ~ b"
16 -
17 - defstruct match_type: nil, selector: nil
18 - end
Loading more files…