Packages

floki

0.21.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
2 Dependencies
Dependants
Files
Activity
Hide whitespace

Compare versions

9 files changed
+99 additions
-82 deletions
  @@ -4,7 +4,7 @@
4 4 [![Inline docs](https://inch-ci.org/github/philss/floki.svg?branch=master)](https://inch-ci.org/github/philss/floki)
5 5 [![Ebert](https://ebertapp.io/github/philss/floki.svg)](https://ebertapp.io/github/philss/floki)
6 6
7 - <img src="priv/images/floki-logo-with-type.svg" width="500" alt="Floki logo">
7 + <img src="assets/images/floki-logo-with-type.svg" width="500" alt="Floki logo">
8 8
9 9 **Floki is a simple HTML parser that enables search for nodes using CSS selectors**.
10 10
  @@ -70,7 +70,7 @@ Add Floki to your `mix.exs`:
70 70 ```elixir
71 71 defp deps do
72 72 [
73 - {:floki, "~> 0.20.0"}
73 + {:floki, "~> 0.21.0"}
74 74 ]
75 75 end
76 76 ```
  @@ -86,14 +86,14 @@ If you get this [kind of error](https://github.com/philss/floki/issues/35),
86 86 you need to install the `erlang-dev` and `erlang-parsetools` packages in order get the `leex` module.
87 87 The packages names may be different depending on your OS.
88 88
89 - ### Optional - Using http5ever as the HTML parser
89 + ### Optional - Using html5ever as the HTML parser
90 90
91 91 You can configure Floki to use [html5ever](https://github.com/servo/html5ever) as your HTML parser.
92 92 This is recommended if you need [better performance](https://gist.github.com/philss/70b4b0294f29501c3c7e0f60338cc8bd)
93 93 and a more accurate parser. However `html5ever` is being under active development and **may be unstable**.
94 94
95 95 Since it's written in Rust, we need to install Rust and compile the project.
96 - Luckily we have have the [html5ever Elixir NIF](https://github.com/hansihe/html5ever_elixir) that makes the integration very easy.
96 + Luckily we have the [html5ever Elixir NIF](https://github.com/hansihe/html5ever_elixir) that makes the integration very easy.
97 97
98 98 You still need to install Rust in your system. To do that, please
99 99 [follow the instruction](https://www.rust-lang.org/en-US/install.html) presented in the official page.
  @@ -105,8 +105,8 @@ After setup Rust, you need to add `html5ever` NIF to your dependency list:
105 105 ```elixir
106 106 defp deps do
107 107 [
108 - {:floki, "~> 0.20.0"},
109 - {:html5ever, "~> 0.6.1"}
108 + {:floki, "~> 0.21.0"},
109 + {:html5ever, "~> 0.7.0"}
110 110 ]
111 111 end
112 112 ```
  @@ -2,37 +2,36 @@
2 2 {<<"build_tools">>,[<<"mix">>]}.
3 3 {<<"description">>,
4 4 <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}.
5 - {<<"elixir">>,<<">= 1.3.0">>}.
5 + {<<"elixir">>,<<">= 1.4.0">>}.
6 6 {<<"files">>,
7 - [<<"lib">>,<<"lib/floki">>,<<"lib/floki.ex">>,<<"lib/floki/.parser.ex.swp">>,
8 - <<"lib/floki/deep_text.ex">>,<<"lib/floki/filter_out.ex">>,
9 - <<"lib/floki/finder.ex">>,<<"lib/floki/flat_text.ex">>,
10 - <<"lib/floki/html_parser">>,<<"lib/floki/html_parser.ex">>,
11 - <<"lib/floki/html_parser/html5ever.ex">>,
12 - <<"lib/floki/html_parser/mochiweb.ex">>,<<"lib/floki/html_tree">>,
13 - <<"lib/floki/html_tree.ex">>,<<"lib/floki/html_tree/comment.ex">>,
14 - <<"lib/floki/html_tree/html_node.ex">>,
15 - <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/text.ex">>,
16 - <<"lib/floki/raw_html.ex">>,<<"lib/floki/selector">>,
17 - <<"lib/floki/selector.ex">>,<<"lib/floki/selector/attribute_selector.ex">>,
18 - <<"lib/floki/selector/combinator.ex">>,
7 + [<<"lib">>,<<"lib/floki">>,<<"lib/floki/html_parser.ex">>,
8 + <<"lib/floki/html_parser">>,<<"lib/floki/html_parser/mochiweb.ex">>,
9 + <<"lib/floki/html_parser/html5ever.ex">>,<<"lib/floki/selector">>,
19 10 <<"lib/floki/selector/functional.ex">>,<<"lib/floki/selector/parser.ex">>,
11 + <<"lib/floki/selector/attribute_selector.ex">>,
20 12 <<"lib/floki/selector/pseudo_class.ex">>,
21 - <<"lib/floki/selector/tokenizer.ex">>,<<"lib/floki/xpath_selector">>,
13 + <<"lib/floki/selector/tokenizer.ex">>,
14 + <<"lib/floki/selector/combinator.ex">>,<<"lib/floki/deep_text.ex">>,
15 + <<"lib/floki/filter_out.ex">>,<<"lib/floki/flat_text.ex">>,
16 + <<"lib/floki/finder.ex">>,<<"lib/floki/raw_html.ex">>,
17 + <<"lib/floki/html_tree">>,<<"lib/floki/html_tree/id_seeder.ex">>,
18 + <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_tree/text.ex">>,
19 + <<"lib/floki/html_tree/html_node.ex">>,<<"lib/floki/selector.ex">>,
20 + <<"lib/floki/html_tree.ex">>,<<"lib/floki.ex">>,
22 21 <<"src/floki_selector_lexer.xrl">>,<<"mix.exs">>,<<"README.md">>,
23 22 <<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>,<<"CONTRIBUTING.md">>]}.
24 23 {<<"licenses">>,[<<"MIT">>]}.
25 24 {<<"links">>,[{<<"GitHub">>,<<"https://github.com/philss/floki">>}]}.
26 25 {<<"name">>,<<"floki">>}.
27 26 {<<"requirements">>,
28 - [[{<<"app">>,<<"html_entities">>},
29 - {<<"name">>,<<"html_entities">>},
30 - {<<"optional">>,false},
31 - {<<"repository">>,<<"hexpm">>},
32 - {<<"requirement">>,<<"~> 0.4.0">>}],
33 - [{<<"app">>,<<"mochiweb">>},
27 + [[{<<"app">>,<<"mochiweb">>},
34 28 {<<"name">>,<<"mochiweb">>},
35 29 {<<"optional">>,false},
36 30 {<<"repository">>,<<"hexpm">>},
37 - {<<"requirement">>,<<"~> 2.15">>}]]}.
38 - {<<"version">>,<<"0.20.4">>}.
31 + {<<"requirement">>,<<"~> 2.15">>}],
32 + [{<<"app">>,<<"html_entities">>},
33 + {<<"name">>,<<"html_entities">>},
34 + {<<"optional">>,false},
35 + {<<"repository">>,<<"hexpm">>},
36 + {<<"requirement">>,<<"~> 0.4.0">>}]]}.
37 + {<<"version">>,<<"0.21.0">>}.
  @@ -23,12 +23,12 @@ defmodule Floki do
23 23
24 24 Examples of queries that you can perform:
25 25
26 - * Floki.find(html, "#content")
27 - * Floki.find(html, ".headline")
28 - * Floki.find(html, "a")
29 - * Floki.find(html, "[data-model=user]")
30 - * Floki.find(html, "#content a")
31 - * Floki.find(html, ".headline, a")
26 + * `Floki.find(html, "#content")`
27 + * `Floki.find(html, ".headline")`
28 + * `Floki.find(html, "a")`
29 + * `Floki.find(html, "[data-model=user]")`
30 + * `Floki.find(html, "#content a")`
31 + * `Floki.find(html, ".headline, a")`
32 32
33 33 Each HTML node is represented by a tuple like:
34 34
  @@ -83,7 +83,6 @@ defmodule Floki do
83 83 You can also control the encoding behaviour at the application level via
84 84 `config :floki, :encode_raw_html, true | false`
85 85
86 -
87 86 ## Examples
88 87
89 88 iex> Floki.parse(~s(<div class="wrapper">my content</div>)) |> Floki.raw_html
  @@ -180,20 +179,26 @@ defmodule Floki do
180 179
181 180 defp mutate_attrs(_, tree, results, attribute_name, mutation_fn) do
182 181 mutated_nodes =
183 - Enum.map(results, fn result ->
184 - mutated_attributes =
185 - if Enum.any?(result.attributes, &match?({^attribute_name, _}, &1)) do
186 - Enum.map(result.attributes, fn attribute ->
187 - with {^attribute_name, attribute_value} <- attribute do
188 - {attribute_name, mutation_fn.(attribute_value)}
189 - end
190 - end)
191 - else
192 - [{attribute_name, mutation_fn.(nil)} | result.attributes]
193 - end
182 + Enum.map(
183 + results,
184 + fn result ->
185 + mutated_attributes =
186 + if Enum.any?(result.attributes, &match?({^attribute_name, _}, &1)) do
187 + Enum.map(
188 + result.attributes,
189 + fn attribute ->
190 + with {^attribute_name, attribute_value} <- attribute do
191 + {attribute_name, mutation_fn.(attribute_value)}
192 + end
193 + end
194 + )
195 + else
196 + [{attribute_name, mutation_fn.(nil)} | result.attributes]
197 + end
194 198
195 - Map.put(result, :attributes, mutated_attributes)
196 - end)
199 + Map.put(result, :attributes, mutated_attributes)
200 + end
201 + )
197 202
198 203 tree = add_nodes_to_tree(tree, mutated_nodes)
199 204
  @@ -253,23 +258,21 @@ defmodule Floki do
253 258 iex> Floki.text([{"p", [], ["1"]},{"p", [], ["2"]}])
254 259 "12"
255 260
261 + iex> Floki.text("<div><style>hello</style> world</div>")
262 + "hello world"
263 +
264 + iex> Floki.text("<div><style>hello</style> world</div>", style: false)
265 + " world"
256 266 """
257 267
258 268 @spec text(html_tree | binary) :: binary
259 269
260 - def text(html, opts \\ [deep: true, js: false, sep: ""]) do
261 - html_tree =
262 - if is_binary(html) do
263 - parse(html)
264 - else
265 - html
266 - end
267 -
270 + def text(html, opts \\ [deep: true, js: false, style: true, sep: ""]) do
268 271 cleaned_html_tree =
269 - case opts[:js] do
270 - true -> html_tree
271 - _ -> filter_out(html_tree, "script")
272 - end
272 + html
273 + |> parse_it()
274 + |> clean_html_tree(:js, opts[:js])
275 + |> clean_html_tree(:style, opts[:style])
273 276
274 277 search_strategy =
275 278 case opts[:deep] do
  @@ -318,7 +321,6 @@ defmodule Floki do
318 321 """
319 322
320 323 @spec attribute(binary | html_tree, binary) :: list
321 -
322 324 def attribute(html_tree, attribute_name) when is_binary(html_tree) do
323 325 html_tree
324 326 |> parse
  @@ -335,25 +337,41 @@ defmodule Floki do
335 337
336 338 defp attribute_values(elements, attr_name) do
337 339 values =
338 - Enum.reduce(elements, [], fn {_, attributes, _}, acc ->
339 - case attribute_match?(attributes, attr_name) do
340 - {_attr_name, value} ->
341 - [value | acc]
340 + Enum.reduce(
341 + elements,
342 + [],
343 + fn {_, attributes, _}, acc ->
344 + case attribute_match?(attributes, attr_name) do
345 + {_attr_name, value} ->
346 + [value | acc]
342 347
343 - _ ->
344 - acc
348 + _ ->
349 + acc
350 + end
345 351 end
346 - end)
352 + )
347 353
348 354 Enum.reverse(values)
349 355 end
350 356
351 357 defp attribute_match?(attributes, attribute_name) do
352 - Enum.find(attributes, fn {attr_name, _} ->
353 - attr_name == attribute_name
354 - end)
358 + Enum.find(
359 + attributes,
360 + fn {attr_name, _} ->
361 + attr_name == attribute_name
362 + end
363 + )
355 364 end
356 365
366 + defp parse_it(html) when is_binary(html), do: parse(html)
367 + defp parse_it(html), do: html
368 +
369 + defp clean_html_tree(html_tree, :js, true), do: html_tree
370 + defp clean_html_tree(html_tree, :js, _), do: filter_out(html_tree, "script")
371 +
372 + defp clean_html_tree(html_tree, :style, true), do: html_tree
373 + defp clean_html_tree(html_tree, :style, _), do: filter_out(html_tree, "style")
374 +
357 375 @doc """
358 376 Returns the nodes from a HTML tree that don't match the filter selector.
359 377
  @@ -370,7 +388,7 @@ defmodule Floki do
370 388
371 389 """
372 390
373 - @spec filter_out(binary | html_tree, binary) :: list
391 + @spec filter_out(binary | html_tree, FilterOut.selector()) :: list
374 392
375 393 def filter_out(html_tree, selector) when is_binary(html_tree) do
376 394 html_tree
  @@ -6,7 +6,7 @@ defmodule Floki.FilterOut do
6 6 # Helper functions for filtering out a specific element from the tree.
7 7
8 8 @type html_tree :: tuple | list
9 - @type selector :: binary
9 + @type selector :: :comment | Finder.selector()
10 10
11 11 @spec filter_out(html_tree, selector) :: tuple | list
Loading more files…