Packages
floki
0.34.3
0.38.4
0.38.3
0.38.2
0.38.1
0.38.0
0.37.1
0.37.0
0.36.3
0.36.2
0.36.1
0.36.0
0.35.4
0.35.3
0.35.2
0.35.1
0.35.0
0.34.3
0.34.2
0.34.1
0.34.0
0.33.1
0.33.0
0.32.1
0.32.0
0.31.0
0.30.1
0.30.0
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.0
0.23.1
0.23.0
0.22.0
0.21.0
0.20.4
0.20.3
0.20.2
0.20.1
0.20.0
0.19.3
0.19.2
0.19.1
0.19.0
0.18.1
0.18.0
0.17.2
0.17.1
0.17.0
0.16.0
0.15.0
0.14.0
0.13.2
0.13.1
0.13.0
0.12.1
0.12.0
0.11.0
0.10.1
0.10.0
0.9.0
0.8.1
0.8.0
0.7.2
0.7.1
0.7.0
0.6.1
0.6.0
0.5.0
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.1
0.1.0
0.0.5
0.0.4
0.0.3
0.0.2
0.0.1
Floki is a simple HTML parser that enables search for nodes using CSS selectors.
Current section
86 Versions
Jump to
Current section
86 Versions
Compare versions
9
files changed
+142
additions
-55
deletions
| @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. | |
| 7 7 | |
| 8 8 | ## [Unreleased][unreleased] |
| 9 9 | |
| 10 | + ## [0.34.3] - 2023-06-02 |
| 11 | + |
| 12 | + ### Added |
| 13 | + |
| 14 | + - Add boolean option `:include_inputs` to `Floki.text/2` that changes the result |
| 15 | + of this function to include the values of inputs. So if there is any input with |
| 16 | + a "value" attribute, we now include that value if this option is set to `true`. |
| 17 | + Thanks [@viniciusmuller](https://github.com/viniciusmuller). |
| 18 | + |
| 19 | + ### Fixed |
| 20 | + |
| 21 | + - Fix find of elements by classes that contain colons. This is useful for when |
| 22 | + people are trying to find elements that contain Tailwind classes. |
| 23 | + Thanks [@viniciusmuller](https://github.com/viniciusmuller). |
| 24 | + |
| 25 | + - Fix some typespecs that were using types from private modules. This is a fix |
| 26 | + to the documentation. |
| 27 | + |
| 10 28 | ## [0.34.2] - 2023-02-24 |
| 11 29 | |
| 12 30 | ### Added |
| @@ -654,7 +672,8 @@ of the parent element inside HTML. | |
| 654 672 | |
| 655 673 | - Elixir version requirement from "~> 1.0.0" to ">= 1.0.0". |
| 656 674 | |
| 657 | - [unreleased]: https://github.com/philss/floki/compare/v0.34.2...HEAD |
| 675 | + [unreleased]: https://github.com/philss/floki/compare/v0.34.3...HEAD |
| 676 | + [0.34.3]: https://github.com/philss/floki/compare/v0.34.2...v0.34.3 |
| 658 677 | [0.34.2]: https://github.com/philss/floki/compare/v0.34.1...v0.34.2 |
| 659 678 | [0.34.1]: https://github.com/philss/floki/compare/v0.34.0...v0.34.1 |
| 660 679 | [0.34.0]: https://github.com/philss/floki/compare/v0.33.1...v0.34.0 |
| @@ -1,13 +1,19 @@ | |
| 1 | - {<<"app">>,<<"floki">>}. |
| 2 | - {<<"build_tools">>,[<<"mix">>]}. |
| 1 | + {<<"links">>, |
| 2 | + [{<<"Changelog">>,<<"https://hexdocs.pm/floki/changelog.html">>}, |
| 3 | + {<<"GitHub">>,<<"https://github.com/philss/floki">>}, |
| 4 | + {<<"Sponsor">>,<<"https://github.com/sponsors/philss">>}]}. |
| 5 | + {<<"name">>,<<"floki">>}. |
| 6 | + {<<"version">>,<<"0.34.3">>}. |
| 3 7 | {<<"description">>, |
| 4 8 | <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}. |
| 5 9 | {<<"elixir">>,<<"~> 1.11">>}. |
| 10 | + {<<"app">>,<<"floki">>}. |
| 11 | + {<<"licenses">>,[<<"MIT">>]}. |
| 6 12 | {<<"files">>, |
| 7 | - [<<"lib/floki">>,<<"lib/floki/deep_text.ex">>,<<"lib/floki/filter_out.ex">>, |
| 8 | - <<"lib/floki/finder.ex">>,<<"lib/floki/flat_text.ex">>,<<"lib/floki/html">>, |
| 9 | - <<"lib/floki/html/numeric_charref.ex">>,<<"lib/floki/html/tokenizer.ex">>, |
| 10 | - <<"lib/floki/html_parser">>,<<"lib/floki/html_parser/fast_html.ex">>, |
| 13 | + [<<"lib/floki">>,<<"lib/floki/filter_out.ex">>,<<"lib/floki/finder.ex">>, |
| 14 | + <<"lib/floki/html">>,<<"lib/floki/html/numeric_charref.ex">>, |
| 15 | + <<"lib/floki/html/tokenizer.ex">>,<<"lib/floki/html_parser">>, |
| 16 | + <<"lib/floki/html_parser/fast_html.ex">>, |
| 11 17 | <<"lib/floki/html_parser/html5ever.ex">>, |
| 12 18 | <<"lib/floki/html_parser/mochiweb.ex">>,<<"lib/floki/html_tree.ex">>, |
| 13 19 | <<"lib/floki/html_tree">>,<<"lib/floki/html_tree/comment.ex">>, |
| @@ -22,15 +28,11 @@ | |
| 22 28 | <<"lib/floki/entities">>,<<"lib/floki/entities/codepoints.ex">>, |
| 23 29 | <<"lib/floki/raw_html.ex">>,<<"lib/floki/selector.ex">>, |
| 24 30 | <<"lib/floki/entities.ex">>,<<"lib/floki/html_parser.ex">>, |
| 25 | - <<"lib/floki/traversal.ex">>,<<"lib/floki.ex">>, |
| 26 | - <<"src/floki_selector_lexer.xrl">>,<<"src/floki_mochi_html.erl">>, |
| 27 | - <<"src/floki.gleam">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>, |
| 28 | - <<"CODE_OF_CONDUCT.md">>,<<"CONTRIBUTING.md">>,<<"CHANGELOG.md">>]}. |
| 29 | - {<<"licenses">>,[<<"MIT">>]}. |
| 30 | - {<<"links">>, |
| 31 | - [{<<"Changelog">>,<<"https://hexdocs.pm/floki/changelog.html">>}, |
| 32 | - {<<"GitHub">>,<<"https://github.com/philss/floki">>}, |
| 33 | - {<<"Sponsor">>,<<"https://github.com/sponsors/philss">>}]}. |
| 34 | - {<<"name">>,<<"floki">>}. |
| 31 | + <<"lib/floki/traversal.ex">>,<<"lib/floki/text_extractor.ex">>, |
| 32 | + <<"lib/floki/flat_text.ex">>,<<"lib/floki/deep_text.ex">>, |
| 33 | + <<"lib/floki.ex">>,<<"src/floki_selector_lexer.xrl">>, |
| 34 | + <<"src/floki_mochi_html.erl">>,<<"src/floki.gleam">>,<<"mix.exs">>, |
| 35 | + <<"README.md">>,<<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>, |
| 36 | + <<"CONTRIBUTING.md">>,<<"CHANGELOG.md">>]}. |
| 35 37 | {<<"requirements">>,[]}. |
| 36 | - {<<"version">>,<<"0.34.2">>}. |
| 38 | + {<<"build_tools">>,[<<"mix">>]}. |
| @@ -73,7 +73,7 @@ defmodule Floki do | |
| 73 73 | html_tag() | html_comment() | html_doctype() | html_declaration() | html_text() |
| 74 74 | @type html_tree :: [html_node()] |
| 75 75 | |
| 76 | - @type css_selector :: String.t() | Floki.Selector.t() | [Floki.Selector.t()] |
| 76 | + @type css_selector :: String.t() | %Floki.Selector{} | [%Floki.Selector{}] |
| 77 77 | |
| 78 78 | @doc """ |
| 79 79 | Parses a HTML Document from a String. |
| @@ -482,6 +482,9 @@ defmodule Floki do | |
| 482 482 | iex> Floki.text({"div", [], [{"script", [], ["hello"]}, " world"]}) |
| 483 483 | " world" |
| 484 484 | |
| 485 | + iex> Floki.text([{"input", [{"type", "date"}, {"value", "2017-06-01"}], []}], include_inputs: true) |
| 486 | + "2017-06-01" |
| 487 | + |
| 485 488 | iex> Floki.text({"div", [], [{"script", [], ["hello"]}, " world"]}, js: true) |
| 486 489 | "hello world" |
| 487 490 | |
| @@ -504,7 +507,7 @@ defmodule Floki do | |
| 504 507 | |
| 505 508 | @spec text(html_tree | html_node | binary) :: binary |
| 506 509 | |
| 507 | - def text(html, opts \\ [deep: true, js: false, style: true, sep: ""]) do |
| 510 | + def text(html, opts \\ [deep: true, js: false, style: true, sep: "", include_inputs: false]) do |
| 508 511 | cleaned_html_tree = |
| 509 512 | html |
| 510 513 | |> parse_it() |
| @@ -518,8 +521,8 @@ defmodule Floki do | |
| 518 521 | end |
| 519 522 | |
| 520 523 | case opts[:sep] do |
| 521 | - nil -> search_strategy.get(cleaned_html_tree) |
| 522 | - sep -> search_strategy.get(cleaned_html_tree, sep) |
| 524 | + nil -> search_strategy.get(cleaned_html_tree, "", opts[:include_inputs]) |
| 525 | + sep -> search_strategy.get(cleaned_html_tree, sep, opts[:include_inputs]) |
| 523 526 | end |
| 524 527 | end |
| 525 528 | |
| @@ -681,7 +684,7 @@ defmodule Floki do | |
| 681 684 | |
| 682 685 | """ |
| 683 686 | |
| 684 | - @spec filter_out(html_node() | html_tree() | binary(), FilterOut.selector()) :: |
| 687 | + @spec filter_out(html_node() | html_tree() | binary(), :comment | :text | css_selector()) :: |
| 685 688 | html_node() | html_tree() |
| 686 689 | |
| 687 690 | def filter_out(html, selector) when is_binary(html) do |
| @@ -6,25 +6,37 @@ defmodule Floki.DeepText do | |
| 6 6 | |
| 7 7 | @type html_tree :: tuple | list |
| 8 8 | |
| 9 | - @spec get(html_tree, binary) :: binary |
| 9 | + @spec get(html_tree, binary, boolean) :: binary |
| 10 10 | |
| 11 | - def get(html_tree, sep \\ "") do |
| 12 | - get_text(html_tree, "", sep) |
| 11 | + def get(html_tree, sep \\ "", include_inputs? \\ false) |
| 12 | + |
| 13 | + def get(html_tree, sep, include_inputs?) do |
| 14 | + html_tree |
| 15 | + |> get_text([], sep, include_inputs?) |
| 16 | + |> IO.iodata_to_binary() |
| 13 17 | end |
| 14 18 | |
| 15 | - defp get_text(text, "", _sep) when is_binary(text), do: text |
| 16 | - defp get_text(text, acc, sep) when is_binary(text), do: Enum.join([acc, text], sep) |
| 19 | + defp get_text(text, [], _sep, _) when is_binary(text), do: text |
| 20 | + defp get_text(text, acc, sep, _) when is_binary(text), do: [acc, sep, text] |
| 17 21 | |
| 18 | - defp get_text(nodes, acc, sep) when is_list(nodes) do |
| 22 | + defp get_text(nodes, acc, sep, include_inputs?) when is_list(nodes) do |
| 19 23 | Enum.reduce(nodes, acc, fn child, istr -> |
| 20 | - get_text(child, istr, sep) |
| 24 | + get_text(child, istr, sep, include_inputs?) |
| 21 25 | end) |
| 22 26 | end |
| 23 27 | |
| 24 | - defp get_text({:comment, _}, acc, _), do: acc |
| 25 | - defp get_text({"br", _, _}, acc, _), do: acc <> "\n" |
| 28 | + defp get_text({:comment, _}, acc, _, _), do: acc |
| 29 | + defp get_text({"br", _, _}, acc, _, _), do: [acc, "\n"] |
| 26 30 | |
| 27 | - defp get_text({_, _, nodes}, acc, sep) do |
| 28 | - get_text(nodes, acc, sep) |
| 31 | + defp get_text({"input", attrs, _}, acc, _, true) do |
| 32 | + [acc, Floki.TextExtractor.extract_input_value(attrs)] |
| 33 | + end |
| 34 | + |
| 35 | + defp get_text({"textarea", attrs, _}, acc, _, true) do |
| 36 | + [acc, Floki.TextExtractor.extract_input_value(attrs)] |
| 37 | + end |
| 38 | + |
| 39 | + defp get_text({_, _, nodes}, acc, sep, include_inputs?) do |
| 40 | + get_text(nodes, acc, sep, include_inputs?) |
| 29 41 | end |
| 30 42 | end |
| @@ -11,31 +11,40 @@ defmodule Floki.FlatText do | |
| 11 11 | |
| 12 12 | @type html_tree :: tuple | list |
| 13 13 | |
| 14 | - @spec get(html_tree, binary) :: binary |
| 14 | + @spec get(html_tree, binary, boolean) :: binary |
| 15 15 | |
| 16 | - def get(html_nodes, sep \\ "") |
| 16 | + def get(html_nodes, sep \\ "", include_inputs? \\ false) |
| 17 17 | |
| 18 | - def get(html_nodes, sep) when is_list(html_nodes) do |
| 19 | - Enum.reduce(html_nodes, "", fn html_node, acc -> |
| 20 | - text_from_node(html_node, acc, sep) |
| 18 | + def get(html_nodes, sep, include_inputs?) when is_list(html_nodes) do |
| 19 | + html_nodes |
| 20 | + |> Enum.reduce([], fn html_node, acc -> |
| 21 | + text_from_node(html_node, acc, 0, sep, include_inputs?) |
| 21 22 | end) |
| 23 | + |> IO.iodata_to_binary() |
| 22 24 | end |
| 23 25 | |
| 24 | - def get(html_node, sep) do |
| 25 | - text_from_node(html_node, "", sep) |
| 26 | + def get(html_node, sep, include_inputs?) do |
| 27 | + html_node |
| 28 | + |> text_from_node([], 0, sep, include_inputs?) |
| 29 | + |> IO.iodata_to_binary() |
| 26 30 | end |
| 27 31 | |
| 28 | - defp text_from_node({_tag, _attrs, html_nodes}, acc, sep) do |
| 32 | + defp text_from_node({"input", attrs, []}, acc, _, _, true) do |
| 33 | + [acc, Floki.TextExtractor.extract_input_value(attrs)] |
| 34 | + end |
| 35 | + |
| 36 | + defp text_from_node({"textarea", attrs, []}, acc, _, _, true) do |
| 37 | + [acc, Floki.TextExtractor.extract_input_value(attrs)] |
| 38 | + end |
| 39 | + |
| 40 | + defp text_from_node({_tag, _attrs, html_nodes}, acc, depth, sep, include_inputs?) |
| 41 | + when depth < 1 do |
| 29 42 | Enum.reduce(html_nodes, acc, fn html_node, acc -> |
| 30 | - capture_text(html_node, acc, sep) |
| 43 | + text_from_node(html_node, acc, depth + 1, sep, include_inputs?) |
| 31 44 | end) |
| 32 45 | end |
| 33 46 | |
| 34 | - defp text_from_node(text, "", _sep) when is_binary(text), do: text |
| 35 | - defp text_from_node(text, acc, sep) when is_binary(text), do: Enum.join([acc, text], sep) |
| 36 | - defp text_from_node(_, acc, _), do: acc |
| 37 | - |
| 38 | - defp capture_text(text, "", _sep) when is_binary(text), do: text |
| 39 | - defp capture_text(text, acc, sep) when is_binary(text), do: Enum.join([acc, text], sep) |
| 40 | - defp capture_text(_html_node, acc, _), do: acc |
| 47 | + defp text_from_node(text, [], _, _sep, _) when is_binary(text), do: text |
| 48 | + defp text_from_node(text, acc, _, sep, _) when is_binary(text), do: [acc, sep, text] |
| 49 | + defp text_from_node(_, acc, _, _, _), do: acc |
| 41 50 | end |
Loading more files…