Current section

86 Versions

Jump to

Compare versions

15 files changed
+518 additions
-487 deletions
  @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
7 7
8 8 ## [Unreleased][unreleased]
9 9
10 + ## [0.35.0] - 2023-10-13
11 +
12 + ### Added
13 +
14 + - Add support for parsing attributes as maps.
15 +
16 + This makes `parse_document/2` and `parse_fragment/2` accept the option
17 + `:attributes_as_maps` to change the behaviour and return attributes as
18 + maps instead of lists of tuples.
19 + The only parser that does not support it yet is the `fast_html`.
20 +
21 + ### Changed
22 +
23 + - Drop support for Elixir v1.11.
24 +
25 + - Change the log level of parsing logger calls from "info" to "debug".
26 + This will help to reduce the amount of noise in production apps.
27 +
10 28 ## [0.34.3] - 2023-06-02
11 29
12 30 ### Added
  @@ -672,7 +690,8 @@ of the parent element inside HTML.
672 690
673 691 - Elixir version requirement from "~> 1.0.0" to ">= 1.0.0".
674 692
675 - [unreleased]: https://github.com/philss/floki/compare/v0.34.3...HEAD
693 + [unreleased]: https://github.com/philss/floki/compare/v0.35.0...HEAD
694 + [0.35.0]: https://github.com/philss/floki/compare/v0.34.3...v0.35.0
676 695 [0.34.3]: https://github.com/philss/floki/compare/v0.34.2...v0.34.3
677 696 [0.34.2]: https://github.com/philss/floki/compare/v0.34.1...v0.34.2
678 697 [0.34.1]: https://github.com/philss/floki/compare/v0.34.0...v0.34.1
  @@ -61,7 +61,7 @@ Add Floki to your `mix.exs`:
61 61 ```elixir
62 62 defp deps do
63 63 [
64 - {:floki, "~> 0.34.0"}
64 + {:floki, "~> 0.35.0"}
65 65 ]
66 66 end
67 67 ```
  @@ -118,8 +118,8 @@ you don't need to install anything to compile it thanks to [RustlerPrecompiled](
118 118 ```elixir
119 119 defp deps do
120 120 [
121 - {:floki, "~> 0.34.0"},
122 - {:html5ever, "~> 0.14.0"}
121 + {:floki, "~> 0.35.0"},
122 + {:html5ever, "~> 0.15.0"}
123 123 ]
124 124 end
125 125 ```
  @@ -146,7 +146,7 @@ First, add `fast_html` to your dependencies:
146 146 ```elixir
147 147 defp deps do
148 148 [
149 - {:floki, "~> 0.34.0"},
149 + {:floki, "~> 0.35.0"},
150 150 {:fast_html, "~> 2.0"}
151 151 ]
152 152 end
  @@ -262,6 +262,14 @@ There are also some selectors based on non-standard specifications. They are:
262 262 | E:fl-contains('foo') | an E element that contains "foo" inside a text node |
263 263 | E:fl-icontains('foo') | an E element that contains "foo" inside a text node (case insensitive) |
264 264
265 + ## Suppressing log messages
266 +
267 + Floki may log debug messages related to problems in the parsing of selectors, or parsing of the HTML tree.
268 + It also may log some "info" messages related to deprecated APIs. If you want to suppress these log messages,
269 + please consider setting the `:compile_time_purge_matching` option for `:logger` in your compile time configuration.
270 +
271 + See https://hexdocs.pm/logger/Logger.html#module-compile-configuration for details.
272 +
265 273 ## Special thanks
266 274
267 275 * [@arasatasaygin](https://github.com/arasatasaygin) for Floki's logo from the [Open Logos project](http://openlogos.org/).
  @@ -1,38 +1,36 @@
1 + {<<"app">>,<<"floki">>}.
2 + {<<"build_tools">>,[<<"mix">>]}.
3 + {<<"description">>,
4 + <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}.
5 + {<<"elixir">>,<<"~> 1.12">>}.
6 + {<<"files">>,
7 + [<<"lib/floki">>,<<"lib/floki/html_tree.ex">>,<<"lib/floki/entities.ex">>,
8 + <<"lib/floki/parse_error.ex">>,<<"lib/floki/deep_text.ex">>,
9 + <<"lib/floki/finder.ex">>,<<"lib/floki/flat_text.ex">>,
10 + <<"lib/floki/selector">>,<<"lib/floki/selector/attribute_selector.ex">>,
11 + <<"lib/floki/selector/pseudo_class.ex">>,<<"lib/floki/selector/parser.ex">>,
12 + <<"lib/floki/selector/functional.ex">>,
13 + <<"lib/floki/selector/combinator.ex">>,
14 + <<"lib/floki/selector/tokenizer.ex">>,<<"lib/floki/raw_html.ex">>,
15 + <<"lib/floki/filter_out.ex">>,<<"lib/floki/html">>,
16 + <<"lib/floki/html/numeric_charref.ex">>,<<"lib/floki/html/tokenizer.ex">>,
17 + <<"lib/floki/html_tree">>,<<"lib/floki/html_tree/comment.ex">>,
18 + <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/text.ex">>,
19 + <<"lib/floki/html_tree/html_node.ex">>,<<"lib/floki/html_parser.ex">>,
20 + <<"lib/floki/entities">>,<<"lib/floki/entities/codepoints.ex">>,
21 + <<"lib/floki/text_extractor.ex">>,<<"lib/floki/selector.ex">>,
22 + <<"lib/floki/traversal.ex">>,<<"lib/floki/html_parser">>,
23 + <<"lib/floki/html_parser/html5ever.ex">>,
24 + <<"lib/floki/html_parser/fast_html.ex">>,
25 + <<"lib/floki/html_parser/mochiweb.ex">>,<<"lib/floki.ex">>,
26 + <<"src/floki_selector_lexer.xrl">>,<<"src/floki_mochi_html.erl">>,
27 + <<"src/floki.gleam">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,
28 + <<"CODE_OF_CONDUCT.md">>,<<"CONTRIBUTING.md">>,<<"CHANGELOG.md">>]}.
29 + {<<"licenses">>,[<<"MIT">>]}.
1 30 {<<"links">>,
2 31 [{<<"Changelog">>,<<"https://hexdocs.pm/floki/changelog.html">>},
3 32 {<<"GitHub">>,<<"https://github.com/philss/floki">>},
4 33 {<<"Sponsor">>,<<"https://github.com/sponsors/philss">>}]}.
5 34 {<<"name">>,<<"floki">>}.
6 - {<<"version">>,<<"0.34.3">>}.
7 - {<<"description">>,
8 - <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}.
9 - {<<"elixir">>,<<"~> 1.11">>}.
10 - {<<"app">>,<<"floki">>}.
11 - {<<"licenses">>,[<<"MIT">>]}.
12 - {<<"files">>,
13 - [<<"lib/floki">>,<<"lib/floki/filter_out.ex">>,<<"lib/floki/finder.ex">>,
14 - <<"lib/floki/html">>,<<"lib/floki/html/numeric_charref.ex">>,
15 - <<"lib/floki/html/tokenizer.ex">>,<<"lib/floki/html_parser">>,
16 - <<"lib/floki/html_parser/fast_html.ex">>,
17 - <<"lib/floki/html_parser/html5ever.ex">>,
18 - <<"lib/floki/html_parser/mochiweb.ex">>,<<"lib/floki/html_tree.ex">>,
19 - <<"lib/floki/html_tree">>,<<"lib/floki/html_tree/comment.ex">>,
20 - <<"lib/floki/html_tree/html_node.ex">>,
21 - <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/text.ex">>,
22 - <<"lib/floki/parse_error.ex">>,<<"lib/floki/selector">>,
23 - <<"lib/floki/selector/functional.ex">>,
24 - <<"lib/floki/selector/tokenizer.ex">>,
25 - <<"lib/floki/selector/attribute_selector.ex">>,
26 - <<"lib/floki/selector/combinator.ex">>,
27 - <<"lib/floki/selector/pseudo_class.ex">>,<<"lib/floki/selector/parser.ex">>,
28 - <<"lib/floki/entities">>,<<"lib/floki/entities/codepoints.ex">>,
29 - <<"lib/floki/raw_html.ex">>,<<"lib/floki/selector.ex">>,
30 - <<"lib/floki/entities.ex">>,<<"lib/floki/html_parser.ex">>,
31 - <<"lib/floki/traversal.ex">>,<<"lib/floki/text_extractor.ex">>,
32 - <<"lib/floki/flat_text.ex">>,<<"lib/floki/deep_text.ex">>,
33 - <<"lib/floki.ex">>,<<"src/floki_selector_lexer.xrl">>,
34 - <<"src/floki_mochi_html.erl">>,<<"src/floki.gleam">>,<<"mix.exs">>,
35 - <<"README.md">>,<<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>,
36 - <<"CONTRIBUTING.md">>,<<"CHANGELOG.md">>]}.
37 35 {<<"requirements">>,[]}.
38 - {<<"build_tools">>,[<<"mix">>]}.
36 + {<<"version">>,<<"0.35.0">>}.
  @@ -63,12 +63,14 @@ defmodule Floki do
63 63 inside a list.
64 64 """
65 65
66 - @type html_declaration :: {:pi, String.t(), [html_attribute()]}
66 + @type html_attribute :: {String.t(), String.t()}
67 + @type html_attributes :: [html_attribute()] | html_attributes_map()
68 + @type html_attributes_map :: %{String.t() => String.t()}
69 + @type html_declaration :: {:pi, String.t(), html_attributes()}
67 70 @type html_comment :: {:comment, String.t()}
68 71 @type html_doctype :: {:doctype, String.t(), String.t(), String.t()}
69 - @type html_attribute :: {String.t(), String.t()}
70 72 @type html_text :: String.t()
71 - @type html_tag :: {String.t(), [html_attribute()], [html_node()]}
73 + @type html_tag :: {String.t(), html_attributes(), [html_node()]}
72 74 @type html_node ::
73 75 html_tag() | html_comment() | html_doctype() | html_declaration() | html_text()
74 76 @type html_tree :: [html_node()]
  @@ -96,11 +98,24 @@ defmodule Floki do
96 98 end
97 99
98 100 @doc """
99 - Parses a HTML Document from a string.
101 + Parses an HTML document from a string.
100 102
101 - It will use the available parser from application env or the one from the
102 - `:html_parser` option.
103 - Check https://github.com/philss/floki#alternative-html-parsers for more details.
103 + This is the main function to get a tree from an HTML string.
104 +
105 + ## Options
106 +
107 + * `:attributes_as_maps` - Change the behaviour of the parser to return the attributes
108 + as maps, instead of a list of `{"key", "value"}`. Default to `false`.
109 +
110 + * `:html_parser` - The module of the backend that is responsible for parsing
111 + the HTML string. By default it is set to the built-in parser, and the module
112 + name is equal to `Floki.HTMLParser.Mochiweb`, or from the value of the
113 + application env of the same name.
114 +
115 + See https://github.com/philss/floki#alternative-html-parsers for more details.
116 +
117 + * `:parser_args` - A list of options to the parser. This can be used to pass options
118 + that are specific for a given parser. Defaults to an empty list.
104 119
105 120 ## Examples
106 121
  @@ -110,6 +125,13 @@ defmodule Floki do
110 125 iex> Floki.parse_document("<html><head></head><body>hello</body></html>", html_parser: Floki.HTMLParser.Mochiweb)
111 126 {:ok, [{"html", [], [{"head", [], []}, {"body", [], ["hello"]}]}]}
112 127
128 + iex> Floki.parse_document(
129 + ...> "<html><head></head><body class=main>hello</body></html>",
130 + ...> attributes_as_maps: true,
131 + ...> html_parser: Floki.HTMLParser.Mochiweb
132 + ...>)
133 + {:ok, [{"html", %{}, [{"head", %{}, []}, {"body", %{"class" => "main"}, ["hello"]}]}]}
134 +
113 135 """
114 136
115 137 @spec parse_document(binary(), Keyword.t()) :: {:ok, html_tree()} | {:error, String.t()}
  @@ -139,12 +161,26 @@ defmodule Floki do
139 161 end
140 162
141 163 @doc """
142 - Parses a HTML fragment from a string.
164 + Parses an HTML fragment from a string.
143 165
144 - It will use the available parser from application env or the one from the
145 - `:html_parser` option.
166 + This is mostly for parsing sections of an HTML document.
167 +
168 + ## Options
169 +
170 + * `:attributes_as_maps` - Change the behaviour of the parser to return the attributes
171 + as maps, instead of a list of `{"key", "value"}`. Remember that maps are no longer
172 + ordered since OTP 26. Default to `false`.
173 +
174 + * `:html_parser` - The module of the backend that is responsible for parsing
175 + the HTML string. By default it is set to the built-in parser, and the module
176 + name is equal to `Floki.HTMLParser.Mochiweb`, or from the value of the
177 + application env of the same name.
178 +
179 + See https://github.com/philss/floki#alternative-html-parsers for more details.
180 +
181 + * `:parser_args` - A list of options to the parser. This can be used to pass options
182 + that are specific for a given parser. Defaults to an empty list.
146 183
147 - Check https://github.com/philss/floki#alternative-html-parsers for more details.
148 184 """
149 185
150 186 @spec parse_fragment(binary(), Keyword.t()) :: {:ok, html_tree()} | {:error, String.t()}
  @@ -169,26 +205,28 @@ defmodule Floki do
169 205
170 206 @doc """
171 207 Converts HTML tree to raw HTML.
208 +
172 209 Note that the resultant HTML may be different from the original one.
173 210 Spaces after tags and doctypes are ignored.
174 211
175 212 ## Options
176 213
177 - - `:encode`: accepts `true` or `false`. Will encode html special characters
178 - to html entities.
179 - You can also control the encoding behaviour at the application level via
180 - `config :floki, :encode_raw_html, true | false`
214 + * `:encode` - A boolean option to control if special HTML characters
215 + should be encoded as HTML entities. Defaults to `true`.
181 216
182 - - `:pretty`: accepts `true` or `false`. Will format the output, ignoring
183 - breaklines and spaces from the input and putting new ones in order to pretty format
184 - the html.
217 + You can also control the encoding behaviour at the application level via
218 + `config :floki, :encode_raw_html, false`
219 +
220 + * `:pretty` - Controls if the output should be formatted, ignoring
221 + breaklines and spaces from the input and putting new ones in order
222 + to pretty format the html. Defaults to `false`.
185 223
186 224 ## Examples
187 225
188 226 iex> Floki.raw_html({"div", [{"class", "wrapper"}], ["my content"]})
189 227 ~s(<div class="wrapper">my content</div>)
190 228
191 - iex> Floki.raw_html({"div", [{"class", "wrapper"}], ["10 > 5"]}, encode: true)
229 + iex> Floki.raw_html({"div", [{"class", "wrapper"}], ["10 > 5"]})
192 230 ~s(<div class="wrapper">10 &gt; 5</div>)
193 231
194 232 iex> Floki.raw_html({"div", [{"class", "wrapper"}], ["10 > 5"]}, encode: false)
  @@ -209,7 +247,7 @@ defmodule Floki do
209 247 defdelegate raw_html(html_tree, options \\ []), to: Floki.RawHTML
210 248
211 249 @doc """
212 - Find elements inside a HTML tree or string.
250 + Find elements inside an HTML tree or string.
213 251
214 252 ## Examples
215 253
  @@ -339,7 +377,7 @@ defmodule Floki do
339 377 @spec find_and_update(
340 378 html_tree(),
341 379 css_selector(),
342 - ({String.t(), [html_attribute()]} -> {String.t(), [html_attribute()]} | :delete)
380 + ({String.t(), html_attributes()} -> {String.t(), html_attributes()} | :delete)
343 381 ) :: html_tree()
344 382 def find_and_update(html_tree, selector, fun) do
345 383 {tree, results} = Finder.find(html_tree, selector)
  @@ -471,6 +509,26 @@ defmodule Floki do
471 509 You can include content of script tags with the option `js` assigned to true.
472 510 You can specify a separator between nodes content.
473 511
512 + ## Options
513 +
514 + * `:deep` - A boolean option to control how deep the search for
515 + text is going to be. If `false`, only the level of the HTML node
516 + or the first level of the HTML document is going to be considered.
517 + Defaults to `true`.
518 +
519 + * `:js` - A boolean option to control if the contents of script tags
520 + should be considered as text. Defaults to `false`.
521 +
522 + * `:sep` - A separator string that is added between text nodes.
523 + Defaults to `""`.
524 +
525 + * `:include_inputs` - A boolean to control if `<input>` or `<textarea>`
526 + values should be included in the resultant string.
527 + Defaults to `false`.
528 +
529 + * `:html_parser` - The module of the backend that is responsible for parsing
530 + the HTML string. By default it is set to `Floki.HTMLParser.Mochiweb`.
531 +
474 532 ## Examples
475 533
476 534 iex> Floki.text({"div", [], [{"span", [], ["hello"]}, " world"]})
  @@ -505,25 +563,23 @@ defmodule Floki do
505 563
506 564 """
507 565
508 - @spec text(html_tree | html_node | binary) :: binary
566 + @spec text(html_tree | html_node | binary, Keyword.t()) :: binary
567 +
568 + def text(html, opts \\ []) do
569 + defaults = [deep: true, js: false, style: true, sep: "", include_inputs: false]
570 +
571 + # We can use `Keyword.validate!` when require Elixir 1.13
572 + opts = Keyword.merge(defaults, opts)
509 573
510 - def text(html, opts \\ [deep: true, js: false, style: true, sep: "", include_inputs: false]) do
511 574 cleaned_html_tree =
512 575 html
513 - |> parse_it()
576 + |> maybe_parse_it()
514 577 |> clean_html_tree(:js, opts[:js])
515 578 |> clean_html_tree(:style, opts[:style])
516 579
517 - search_strategy =
518 - case opts[:deep] do
519 - false -> Floki.FlatText
520 - _ -> Floki.DeepText
521 - end
580 + search_strategy = if opts[:deep], do: Floki.DeepText, else: Floki.FlatText
522 581
523 - case opts[:sep] do
524 - nil -> search_strategy.get(cleaned_html_tree, "", opts[:include_inputs])
525 - sep -> search_strategy.get(cleaned_html_tree, sep, opts[:include_inputs])
526 - end
582 + search_strategy.get(cleaned_html_tree, opts[:sep], opts[:include_inputs])
527 583 end
528 584
529 585 @doc """
  @@ -648,7 +704,7 @@ defmodule Floki do
648 704 )
649 705 end
650 706
651 - defp parse_it(html) when is_binary(html) do
707 + defp maybe_parse_it(html) when is_binary(html) do
652 708 Logger.info(
653 709 "deprecation: parse the HTML with parse_document or parse_fragment before using text/2"
654 710 )
  @@ -657,7 +713,7 @@ defmodule Floki do
657 713 document
658 714 end
659 715
660 - defp parse_it(html), do: html
716 + defp maybe_parse_it(html), do: html
661 717
662 718 defp clean_html_tree(html_tree, :js, true), do: html_tree
663 719 defp clean_html_tree(html_tree, :js, _), do: filter_out(html_tree, "script")
  @@ -1,6 +1,4 @@
1 1 defmodule Floki.Finder do
2 - require Logger
3 -
4 2 @moduledoc false
5 3
6 4 # The finder engine traverse the HTML tree searching for nodes matching
Loading more files…