Packages
floki
0.2.0
0.38.4
0.38.3
0.38.2
0.38.1
0.38.0
0.37.1
0.37.0
0.36.3
0.36.2
0.36.1
0.36.0
0.35.4
0.35.3
0.35.2
0.35.1
0.35.0
0.34.3
0.34.2
0.34.1
0.34.0
0.33.1
0.33.0
0.32.1
0.32.0
0.31.0
0.30.1
0.30.0
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.0
0.23.1
0.23.0
0.22.0
0.21.0
0.20.4
0.20.3
0.20.2
0.20.1
0.20.0
0.19.3
0.19.2
0.19.1
0.19.0
0.18.1
0.18.0
0.17.2
0.17.1
0.17.0
0.16.0
0.15.0
0.14.0
0.13.2
0.13.1
0.13.0
0.12.1
0.12.0
0.11.0
0.10.1
0.10.0
0.9.0
0.8.1
0.8.0
0.7.2
0.7.1
0.7.0
0.6.1
0.6.0
0.5.0
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.1
0.1.0
0.0.5
0.0.4
0.0.3
0.0.2
0.0.1
Floki is a simple HTML parser that enables search for nodes using CSS selectors.
Current section
86 Versions
Jump to
Current section
86 Versions
Compare versions
6
files changed
+132
additions
-41
deletions
| @@ -1,10 +1,6 @@ | |
| 1 | - Floki |
| 2 | - ===== |
| 1 | + # Floki [](https://travis-ci.org/philss/floki) [](https://hex.pm/packages/floki) [](http://inch-ci.org/github/philss/floki) |
| 3 2 | |
| 4 | - [](https://travis-ci.org/philss/floki) |
| 5 | - [](https://hex.pm/packages/floki) |
| 6 | - |
| 7 | - This is a simple HTML parser that enables search using CSS like selectors. |
| 3 | + Floki is a simple HTML parser that enables search using CSS like selectors. |
| 8 4 | |
| 9 5 | You can search elements by class, tag name and id. |
| 10 6 | |
| @@ -18,22 +14,37 @@ Assuming that you have the following HTML: | |
| 18 14 | <!doctype html> |
| 19 15 | <html> |
| 20 16 | <body> |
| 21 | - <section id="content"> |
| 22 | - <p class="headline">Floki</p> |
| 23 | - <a href="http://github.com/philss/floki">Github page</a> |
| 24 | - </section> |
| 17 | + <section id="content"> |
| 18 | + <p class="headline">Floki</p> |
| 19 | + <a href="http://github.com/philss/floki">Github page</a> |
| 20 | + </section> |
| 21 | + <a href="https://hex.pm/packages/floki">Hex package</a> |
| 25 22 | </body> |
| 26 23 | </html> |
| 27 24 | ``` |
| 28 25 | |
| 29 | - You can perform the following queries: |
| 26 | + Here are some of the queries that you can perform (with return examples): |
| 30 27 | |
| 31 28 | ```elixir |
| 32 | - Floki.find(html, "#content") # returns the section with all children |
| 29 | + Floki.find(html, "#content") |
| 30 | + # => {"section", [{"id", "content"}], |
| 31 | + # => [{"p", [{"class", "headline"}], ["Floki"]}, |
| 32 | + # => {"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}]} |
| 33 | + |
| 33 34 | Floki.find(html, ".headline") # returns a list with the `p` element |
| 34 | - Floki.find(html, "a") # returns a list with the `a` element |
| 35 | - Floki.find(html, "#content a") # returns all links inside content section |
| 36 | - Floki.find(html, ".headline, a") # returns the .headline elements and links |
| 35 | + # => [{"p", [{"class", "headline"}], ["Floki"]}] |
| 36 | + |
| 37 | + Floki.find(html, "a") |
| 38 | + # => [{"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}, |
| 39 | + # => {"a", [{"href", "https://hex.pm/packages/floki"}], ["Hex package"]}] |
| 40 | + |
| 41 | + Floki.find(html, "#content a") |
| 42 | + # => [{"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}] |
| 43 | + |
| 44 | + Floki.find(html, ".headline, a") |
| 45 | + # => [{"p", [{"class", "headline"}], ["Floki"]}, |
| 46 | + # => {"a", [{"href", "http://github.com/philss/floki"}], ["Github page"]}, |
| 47 | + # => {"a", [{"href", "https://hex.pm/packages/floki"}], ["Hex package"]}] |
| 37 48 | ``` |
| 38 49 | |
| 39 50 | Each HTML node is represented by a tuple like: |
| @@ -4,7 +4,9 @@ | |
| 4 4 | {<<"description">>, |
| 5 5 | <<"A HTML parser and searcher.\n\nYou can search inside HTML documents using CSS like selectors.\n">>}. |
| 6 6 | {<<"elixir">>,<<">= 1.0.0">>}. |
| 7 | - {<<"files">>,[<<"lib/floki.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>]}. |
| 7 | + {<<"files">>, |
| 8 | + [<<"lib/floki.ex">>,<<"lib/floki/deep_text.ex">>, |
| 9 | + <<"lib/floki/flat_text.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>]}. |
| 8 10 | {<<"licenses">>,[<<"MIT">>]}. |
| 9 11 | {<<"links">>, |
| 10 12 | [{<<"Docs">>,<<"http://hexdocs.pm/floki">>}, |
| @@ -15,4 +17,4 @@ | |
| 15 17 | [{<<"app">>,<<"mochiweb">>}, |
| 16 18 | {<<"optional">>,nil}, |
| 17 19 | {<<"requirement">>,<<"~> 2.12.2">>}]}]}. |
| 18 | - {<<"version">>,<<"0.1.1">>}. |
| 20 | + {<<"version">>,<<"0.2.0">>}. |
| @@ -64,10 +64,7 @@ defmodule Floki do | |
| 64 64 | |
| 65 65 | @spec parse(binary) :: html_tree |
| 66 66 | |
| 67 | - def parse(html) do |
| 68 | - :mochiweb_html.parse(html) |
| 69 | - end |
| 70 | - |
| 67 | + def parse(html), do: :mochiweb_html.parse(html) |
| 71 68 | |
| 72 69 | @doc """ |
| 73 70 | Finds elements inside a HTML tree or string. |
| @@ -159,46 +156,54 @@ defmodule Floki do | |
| 159 156 | |
| 160 157 | ## Examples |
| 161 158 | |
| 162 | - iex> Floki.attribute("<a href='https://google.com'>Google</a>", "href") |
| 159 | + iex> Floki.attribute("<a href=https://google.com>Google</a>", "href") |
| 163 160 | ["https://google.com"] |
| 164 161 | |
| 165 162 | """ |
| 166 163 | |
| 167 164 | @spec attribute(binary | html_tree, binary) :: list |
| 168 165 | |
| 166 | + def attribute(html_tree, attribute_name) when is_binary(html_tree) do |
| 167 | + html_tree |
| 168 | + |> parse |
| 169 | + |> attribute(attribute_name) |
| 170 | + end |
| 169 171 | def attribute(elements, attribute_name) do |
| 170 172 | elements |
| 171 173 | |> attribute_values(attribute_name) |
| 172 174 | end |
| 173 175 | |
| 174 176 | @doc """ |
| 175 | - Returns the text nodes from a html tree. |
| 177 | + Returns the text nodes from a HTML tree. |
| 178 | + By default, it will perform a deep search through the HTML tree. |
| 179 | + You can disable deep search with the option `deep` assigned to false. |
| 176 180 | |
| 177 181 | ## Examples |
| 178 182 | |
| 179 | - iex> Floki.text("<div><span>something else</span>hello world</div>") |
| 183 | + iex> Floki.text("<div><span>hello</span> world</div>") |
| 180 184 | "hello world" |
| 181 185 | |
| 186 | + iex> Floki.text("<div><span>hello</span> world</div>", deep: false) |
| 187 | + " world" |
| 188 | + |
| 182 189 | """ |
| 183 190 | |
| 184 191 | @spec text(html_tree | binary) :: binary |
| 185 192 | |
| 186 | - def text(html) when is_binary(html), do: parse(html) |> text |
| 187 | - def text(element) when is_tuple(element), do: _text(element, "") |
| 188 | - def text(elements) do |
| 189 | - Enum.reduce elements, "", fn(element, str) -> |
| 190 | - _text(element, str) |
| 191 | - end |
| 192 | - end |
| 193 | - |
| 194 | - defp _text({_, _, children}, acc) do |
| 195 | - Enum.reduce children, acc, fn(child, istr) -> |
| 196 | - if is_binary(child) do |
| 197 | - (istr <> "\s" <> child) |> String.strip |
| 198 | - else |
| 199 | - istr |
| 193 | + def text(html, opts \\ [deep: true]) do |
| 194 | + html_tree = |
| 195 | + case is_binary(html) do |
| 196 | + true -> parse(html) |
| 197 | + false -> html |
| 200 198 | end |
| 201 | - end |
| 199 | + |
| 200 | + search_strategy = |
| 201 | + case opts[:deep] do |
| 202 | + true -> Floki.DeepText |
| 203 | + false -> Floki.FlatText |
| 204 | + end |
| 205 | + |
| 206 | + search_strategy.get(html_tree) |
| 202 207 | end |
| 203 208 | |
| 204 209 | defp attribute_match?(attributes, attribute_name) do |
| @@ -0,0 +1,34 @@ | |
| 1 | + defmodule Floki.DeepText do |
| 2 | + @moduledoc """ |
| 3 | + DeepText is a strategy to get text nodes from a HTML tree using a deep search |
| 4 | + algorithm. It will get all string nodes and concat them. |
| 5 | + """ |
| 6 | + |
| 7 | + @type html_tree :: tuple | list |
| 8 | + |
| 9 | + @spec get(html_tree) :: binary |
| 10 | + |
| 11 | + @doc """ |
| 12 | + Get text nodes from a deep tree of HTML nodes. |
| 13 | + |
| 14 | + |
| 15 | + ## Examples |
| 16 | + |
| 17 | + iex> Floki.DeepText.get([{"a", [], ["The mean of life is...", {"strong", [], ["something else"]}] }]) |
| 18 | + "The mean of life is...something else" |
| 19 | + |
| 20 | + """ |
| 21 | + def get(html_tree) do |
| 22 | + get_text(html_tree, "") |
| 23 | + end |
| 24 | + |
| 25 | + defp get_text(text, acc) when is_binary(text), do: acc <> text |
| 26 | + defp get_text(nodes, acc) when is_list(nodes) do |
| 27 | + Enum.reduce nodes, acc, fn(child, istr) -> |
| 28 | + get_text(child, istr) |
| 29 | + end |
| 30 | + end |
| 31 | + defp get_text({_, _, nodes}, acc) do |
| 32 | + get_text(nodes, acc) |
| 33 | + end |
| 34 | + end |
| @@ -0,0 +1,38 @@ | |
| 1 | + defmodule Floki.FlatText do |
| 2 | + @moduledoc """ |
| 3 | + FlatText is a strategy to get text nodes from a HTML tree without search deep |
| 4 | + in the tree. It only gets the text nodes from the first level of nodes. |
| 5 | + """ |
| 6 | + |
| 7 | + @type html_tree :: tuple | list |
| 8 | + |
| 9 | + @spec get(html_tree) :: binary |
| 10 | + |
| 11 | + @doc """ |
| 12 | + Get text nodes from first level of HTML nodes. |
| 13 | + |
| 14 | + |
| 15 | + ## Examples |
| 16 | + |
| 17 | + iex> Floki.FlatText.get([{"a", [], ["The mean of life is...", {"strong", [], ["something else"]}] }]) |
| 18 | + "The mean of life is..." |
| 19 | + |
| 20 | + """ |
| 21 | + def get(html_nodes) when is_list(html_nodes) do |
| 22 | + Enum.reduce(html_nodes, "", fn(html_node, acc) -> |
| 23 | + text_from_node(html_node, acc) |
| 24 | + end) |
| 25 | + end |
| 26 | + def get(html_node) do |
| 27 | + text_from_node(html_node, "") |
| 28 | + end |
| 29 | + |
| 30 | + defp text_from_node({ _tag, _attrs, html_nodes}, acc) do |
| 31 | + Enum.reduce(html_nodes, acc, &capture_text/2) |
| 32 | + end |
| 33 | + defp text_from_node(text, acc) when is_binary(text), do: acc <> text |
| 34 | + defp text_from_node(_, acc), do: acc |
| 35 | + |
| 36 | + defp capture_text(text, acc) when is_binary(text), do: acc <> text |
| 37 | + defp capture_text(_html_node, acc), do: acc |
| 38 | + end |
Loading more files…