Packages

floki

0.19.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
1 Dependency
Dependants
Files
Activity
Hide whitespace

Compare versions

17 files changed
+550 additions
-190 deletions
  @@ -70,7 +70,7 @@ Add Floki to your `mix.exs`:
70 70 ```elixir
71 71 defp deps do
72 72 [
73 - {:floki, "~> 0.18.0"}
73 + {:floki, "~> 0.19.0"}
74 74 ]
75 75 end
76 76 ```
  @@ -105,7 +105,7 @@ After setup Rust, you need to add `html5ever` NIF to your dependency list:
105 105 ```elixir
106 106 defp deps do
107 107 [
108 - {:floki, "~> 0.18.0"},
108 + {:floki, "~> 0.19.0"},
109 109 {:html5ever, "~> 0.5.0"}
110 110 ]
111 111 end
  @@ -198,6 +198,10 @@ Here you find all the [CSS selectors](https://www.w3.org/TR/selectors/#selectors
198 198 | E[foo\|="en"] | an E element whose "foo" attribute has a hyphen-separated list of values beginning (from the left) with "en" |
199 199 | E:nth-child(n) | an E element, the n-th child of its parent |
200 200 | E:first-child | an E element, first child of its parent |
201 + | E:last-child | an E element, last child of its parent |
202 + | E:nth-of-type(n) | an E element, the n-th child of its type among its siblings |
203 + | E:first-of-type | an E element, first child of its type among its siblings |
204 + | E:last-of-type | an E element, last child of its type among its siblings |
201 205 | E.warning | an E element whose class is "warning" |
202 206 | E#myid | an E element with ID equal to "myid" |
203 207 | E:not(s) | an E element that does not match simple selector s |
  @@ -12,14 +12,13 @@
12 12 <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_tree/html_node.ex">>,
13 13 <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/text.ex">>,
14 14 <<"lib/floki/selector.ex">>,<<"lib/floki/selector/attribute_selector.ex">>,
15 - <<"lib/floki/selector/combinator.ex">>,<<"lib/floki/selector/parser.ex">>,
15 + <<"lib/floki/selector/combinator.ex">>,
16 + <<"lib/floki/selector/functional.ex">>,<<"lib/floki/selector/parser.ex">>,
16 17 <<"lib/floki/selector/pseudo_class.ex">>,
17 18 <<"lib/floki/selector/tokenizer.ex">>,<<"src/floki_selector_lexer.xrl">>,
18 19 <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>]}.
19 20 {<<"licenses">>,[<<"MIT">>]}.
20 - {<<"links">>,
21 - [{<<"Docs">>,<<"https://hexdocs.pm/floki">>},
22 - {<<"GitHub">>,<<"https://github.com/philss/floki">>}]}.
21 + {<<"links">>,[{<<"GitHub">>,<<"https://github.com/philss/floki">>}]}.
23 22 {<<"maintainers">>,[<<"Philip Sampaio Silva">>]}.
24 23 {<<"name">>,<<"floki">>}.
25 24 {<<"requirements">>,
  @@ -27,4 +26,4 @@
27 26 {<<"name">>,<<"mochiweb">>},
28 27 {<<"optional">>,false},
29 28 {<<"requirement">>,<<"~> 2.15">>}]]}.
30 - {<<"version">>,<<"0.18.1">>}.
29 + {<<"version">>,<<"0.19.0">>}.
  @@ -72,7 +72,24 @@ defmodule Floki do
72 72 HTMLParser.parse(html)
73 73 end
74 74
75 - @self_closing_tags ["area", "base", "br", "col", "command", "embed", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"]
75 + @self_closing_tags [
76 + "area",
77 + "base",
78 + "br",
79 + "col",
80 + "command",
81 + "embed",
82 + "hr",
83 + "img",
84 + "input",
85 + "keygen",
86 + "link",
87 + "meta",
88 + "param",
89 + "source",
90 + "track",
91 + "wbr"
92 + ]
76 93
77 94 @doc """
78 95 Converts HTML tree to raw HTML.
  @@ -86,24 +103,29 @@ defmodule Floki do
86 103
87 104 """
88 105
89 - @spec raw_html(html_tree) :: binary
106 + @spec raw_html(html_tree | binary) :: binary
90 107
91 108 def raw_html(html_tree), do: raw_html(html_tree, "")
92 109 defp raw_html([], html), do: html
110 + defp raw_html(string, _html) when is_binary(string), do: string
93 111 defp raw_html(tuple, html) when is_tuple(tuple), do: raw_html([tuple], html)
94 - defp raw_html([string|tail], html) when is_binary(string), do: raw_html(tail, html <> string)
95 - defp raw_html([{:comment, comment}|tail], html), do: raw_html(tail, html <> "<!--#{comment}-->")
96 - defp raw_html([{:pi, "xml", attrs} |tail], html) do
112 + defp raw_html([string | tail], html) when is_binary(string), do: raw_html(tail, html <> string)
113 +
114 + defp raw_html([{:comment, comment} | tail], html),
115 + do: raw_html(tail, html <> "<!--#{comment}-->")
116 +
117 + defp raw_html([{:pi, "xml", attrs} | tail], html) do
97 118 raw_html(tail, html <> "<?xml " <> tag_attrs(attrs) <> "?>")
98 119 end
99 - defp raw_html([{type, attrs, children}|tail], html) do
120 +
121 + defp raw_html([{type, attrs, children} | tail], html) do
100 122 raw_html(tail, html <> tag_for(type, tag_attrs(attrs), children))
101 123 end
102 124
103 125 defp tag_attrs(attr_list) do
104 126 attr_list
105 127 |> Enum.reduce("", &build_attrs/2)
106 - |> String.trim
128 + |> String.trim()
107 129 end
108 130
109 131 defp build_attrs({attr, value}, attrs), do: ~s(#{attrs} #{attr}="#{value}")
  @@ -115,6 +137,7 @@ defmodule Floki do
115 137 _ -> "<#{type} #{attrs}/>"
116 138 end
117 139 end
140 +
118 141 defp tag_for(type, attrs, children) do
119 142 case attrs do
120 143 "" -> "<#{type}>#{raw_html(children)}</#{type}>"
  @@ -148,12 +171,13 @@ defmodule Floki do
148 171
149 172 {tree, results} = Finder.find(html_as_tuple, selector)
150 173
151 - Enum.map(results, fn(html_node) -> HTMLTree.to_tuple(tree, html_node) end)
174 + Enum.map(results, fn html_node -> HTMLTree.to_tuple(tree, html_node) end)
152 175 end
176 +
153 177 def find(html_tree_as_tuple, selector) do
154 178 {tree, results} = Finder.find(html_tree_as_tuple, selector)
155 179
156 - Enum.map(results, fn(html_node) -> HTMLTree.to_tuple(tree, html_node) end)
180 + Enum.map(results, fn html_node -> HTMLTree.to_tuple(tree, html_node) end)
157 181 end
158 182
159 183 @doc """
  @@ -174,9 +198,11 @@ defmodule Floki do
174 198 def attr(html_elem_tuple, selector, attribute_name, mutation) when is_tuple(html_elem_tuple) do
175 199 attr([html_elem_tuple], selector, attribute_name, mutation)
176 200 end
201 +
177 202 def attr(html_str, selector, attribute_name, mutation) when is_binary(html_str) do
178 203 attr(parse(html_str), selector, attribute_name, mutation)
179 204 end
205 +
180 206 def attr(html_tree_list, selector, attribute_name, mutation) when is_list(html_tree_list) do
181 207 {tree, results} = Finder.find(html_tree_list, selector)
182 208 mutate_attrs(html_tree_list, tree, results, attribute_name, mutation)
  @@ -186,6 +212,7 @@ defmodule Floki do
186 212 nodes = Map.put(tree.nodes, html_node.node_id, html_node)
187 213 Map.put(tree, :nodes, nodes)
188 214 end
215 +
189 216 defp add_nodes_to_tree(tree, [html_node | tail]) do
190 217 nodes = Map.put(tree.nodes, html_node.node_id, html_node)
191 218
  @@ -195,28 +222,30 @@ defmodule Floki do
195 222 end
196 223
197 224 defp mutate_attrs(html_tree_list, _, [], _, _), do: html_tree_list
198 - defp mutate_attrs(_, tree, results, attribute_name, mutation_fn) do
199 - mutated_nodes = Enum.map(results, fn(result) ->
200 - mutated_attributes =
201 - if Enum.any?(result.attributes, & match?({^attribute_name, _}, &1)) do
202 - Enum.map(result.attributes, fn(attribute) ->
203 - with {^attribute_name, attribute_value} <- attribute do
204 - {attribute_name, mutation_fn.(attribute_value)}
205 - end
206 - end)
207 - else
208 - [{attribute_name, mutation_fn.(nil)} | result.attributes]
209 - end
210 225
211 - Map.put(result, :attributes, mutated_attributes)
212 - end)
226 + defp mutate_attrs(_, tree, results, attribute_name, mutation_fn) do
227 + mutated_nodes =
228 + Enum.map(results, fn result ->
229 + mutated_attributes =
230 + if Enum.any?(result.attributes, &match?({^attribute_name, _}, &1)) do
231 + Enum.map(result.attributes, fn attribute ->
232 + with {^attribute_name, attribute_value} <- attribute do
233 + {attribute_name, mutation_fn.(attribute_value)}
234 + end
235 + end)
236 + else
237 + [{attribute_name, mutation_fn.(nil)} | result.attributes]
238 + end
239 +
240 + Map.put(result, :attributes, mutated_attributes)
241 + end)
213 242
214 243 tree = add_nodes_to_tree(tree, mutated_nodes)
215 244
216 245 tree.nodes
217 - |> Map.values
218 - |> Enum.filter(fn(actual_node) -> is_nil(actual_node.parent_node_id) end)
219 - |> Enum.map(fn(html_node) -> HTMLTree.to_tuple(tree, html_node) end)
246 + |> Map.values()
247 + |> Enum.filter(fn actual_node -> is_nil(actual_node.parent_node_id) end)
248 + |> Enum.map(fn html_node -> HTMLTree.to_tuple(tree, html_node) end)
220 249 end
221 250
222 251 @doc """
  @@ -234,8 +263,9 @@ defmodule Floki do
234 263
235 264 """
236 265 def map(html_tree_list, fun) when is_list(html_tree_list) do
237 - Enum.map(html_tree_list, &(Finder.map(&1, fun)))
266 + Enum.map(html_tree_list, &Finder.map(&1, fun))
238 267 end
268 +
239 269 def map(html_tree, fun), do: Finder.map(html_tree, fun)
240 270
241 271 @doc """
  @@ -296,7 +326,6 @@ defmodule Floki do
296 326 nil -> search_strategy.get(cleaned_html_tree)
297 327 sep -> search_strategy.get(cleaned_html_tree, sep)
298 328 end
299 -
300 329 end
301 330
302 331 @doc """
  @@ -340,6 +369,7 @@ defmodule Floki do
340 369 |> parse
341 370 |> attribute_values(attribute_name)
342 371 end
372 +
343 373 def attribute(elements, attribute_name) do
344 374 attribute_values(elements, attribute_name)
345 375 end
  @@ -347,23 +377,26 @@ defmodule Floki do
347 377 defp attribute_values(element, attr_name) when is_tuple(element) do
348 378 attribute_values([element], attr_name)
349 379 end
380 +
350 381 defp attribute_values(elements, attr_name) do
351 - values = Enum.reduce elements, [], fn({_, attributes, _}, acc) ->
352 - case attribute_match?(attributes, attr_name) do
353 - {_attr_name, value} ->
354 - [value|acc]
355 - _ ->
356 - acc
357 - end
358 - end
382 + values =
383 + Enum.reduce(elements, [], fn {_, attributes, _}, acc ->
384 + case attribute_match?(attributes, attr_name) do
385 + {_attr_name, value} ->
386 + [value | acc]
387 +
388 + _ ->
389 + acc
390 + end
391 + end)
359 392
360 393 Enum.reverse(values)
361 394 end
362 395
363 396 defp attribute_match?(attributes, attribute_name) do
364 - Enum.find attributes, fn({attr_name, _}) ->
397 + Enum.find(attributes, fn {attr_name, _} ->
365 398 attr_name == attribute_name
366 - end
399 + end)
367 400 end
368 401
369 402 @doc """
  @@ -389,6 +422,7 @@ defmodule Floki do
389 422 |> parse
390 423 |> FilterOut.filter_out(selector)
391 424 end
425 +
392 426 def filter_out(elements, selector) do
393 427 FilterOut.filter_out(elements, selector)
394 428 end
  @@ -14,13 +14,16 @@ defmodule Floki.DeepText do
14 14
15 15 defp get_text(text, "", _sep) when is_binary(text), do: text
16 16 defp get_text(text, acc, sep) when is_binary(text), do: Enum.join([acc, text], sep)
17 +
17 18 defp get_text(nodes, acc, sep) when is_list(nodes) do
18 - Enum.reduce nodes, acc, fn(child, istr) ->
19 + Enum.reduce(nodes, acc, fn child, istr ->
19 20 get_text(child, istr, sep)
20 - end
21 + end)
21 22 end
23 +
22 24 defp get_text({:comment, _}, acc, _), do: acc
23 25 defp get_text({"br", _, _}, acc, _), do: acc <> "\n"
26 +
24 27 defp get_text({_, _, nodes}, acc, sep) do
25 28 get_text(nodes, acc, sep)
26 29 end
  @@ -13,20 +13,24 @@ defmodule Floki.FilterOut do
13 13 def filter_out(html_tree, :comment) do
14 14 mapper(html_tree, :comment)
15 15 end
16 +
16 17 def filter_out(html_tree, selector) do
17 18 case Finder.find(html_tree, selector) do
18 19 {:empty_tree, _} ->
19 20 html_tree
21 +
20 22 {tree, results} ->
21 - new_tree = Enum.reduce(results, tree, fn(html_node, tree) ->
22 - HTMLTree.delete_node(tree, html_node)
23 - end)
23 + new_tree =
24 + Enum.reduce(results, tree, fn html_node, tree ->
25 + HTMLTree.delete_node(tree, html_node)
26 + end)
24 27
25 - html_as_tuples = Enum.map(new_tree.root_nodes_ids, fn(node_id) ->
26 - root = Map.get(new_tree.nodes, node_id)
28 + html_as_tuples =
29 + Enum.map(new_tree.root_nodes_ids, fn node_id ->
30 + root = Map.get(new_tree.nodes, node_id)
27 31
28 - HTMLTree.to_tuple(new_tree, root)
29 - end)
32 + HTMLTree.to_tuple(new_tree, root)
33 + end)
30 34
31 35 case html_tree do
32 36 tree when is_tuple(tree) ->
  @@ -34,6 +38,7 @@ defmodule Floki.FilterOut do
34 38 [] -> []
35 39 [head | _] -> head
36 40 end
41 +
37 42 trees when is_list(trees) ->
38 43 html_as_tuples
39 44 end
  @@ -48,10 +53,12 @@ defmodule Floki.FilterOut do
48 53 nodes
49 54 |> Stream.filter(&filter(&1, selector))
50 55 |> Stream.map(&mapper(&1, selector))
51 - |> Enum.to_list
56 + |> Enum.to_list()
52 57 end
58 +
53 59 defp mapper({nodetext, x, y}, selector) do
54 60 {nodetext, x, mapper(y, selector)}
55 61 end
62 +
56 63 defp mapper(nodetext, _), do: nodetext
57 64 end
Loading more files…