Packages

floki

0.1.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
0 Dependencies
Dependants
Files
Activity
Hide whitespace

Compare versions

4 files changed
+110 additions
-74 deletions
  @@ -1,9 +1,10 @@
1 1 Floki
2 2 =====
3 3
4 - [![Build Status](https://travis-ci.org/philss/floki.svg?branch=master)](https://travis-ci.org/philss/floki)
4 + [![Build status](https://travis-ci.org/philss/floki.svg?branch=master)](https://travis-ci.org/philss/floki)
5 + [![Floki version](https://img.shields.io/hexpm/v/floki.svg)](https://hex.pm/packages/floki)
5 6
6 - This is a simple HTML parser that enables searching using CSS like selectors.
7 + This is a simple HTML parser that enables search using CSS like selectors.
7 8
8 9 You can search elements by class, tag name and id.
9 10
  @@ -27,9 +28,13 @@ Assuming that you have the following HTML:
27 28
28 29 You can perform the following queries:
29 30
30 - * Floki.find(html, "#content") : returns the section with all children;
31 - * Floki.find(html, ".headline") : returns a list with the `p` element;
32 - * Floki.find(html, "a") : returns a list with the `a` element.
31 + ```elixir
32 + Floki.find(html, "#content") # returns the section with all children
33 + Floki.find(html, ".headline") # returns a list with the `p` element
34 + Floki.find(html, "a") # returns a list with the `a` element
35 + Floki.find(html, "#content a") # returns all links inside content section
36 + Floki.find(html, ".headline, a") # returns the .headline elements and links
37 + ```
33 38
34 39 Each HTML node is represented by a tuple like:
35 40
  @@ -46,10 +51,9 @@ You can write a simple HTML crawler (with support of [HTTPoison](https://github.
46 51
47 52 ```elixir
48 53 html
49 - |> Floki.find(".pages")
50 - |> Floki.find("a")
51 - |> Floki.attribute("href")
52 - |> Enum.map(fn(url) -> HTTPoison.get!(url) end)
54 + |> Floki.find(".pages a")
55 + |> Floki.attribute("href")
56 + |> Enum.map(fn(url) -> HTTPoison.get!(url) end)
53 57 ```
54 58
55 59 It is simple as that!
  @@ -89,7 +93,7 @@ You can also get attributes from elements that you already have:
89 93
90 94 ```elixir
91 95 Floki.find(html, ".example")
92 - |> Floki.attribute("class")
96 + |> Floki.attribute("class")
93 97 # => ["example"]
94 98 ```
95 99
  @@ -97,7 +101,7 @@ If you want to get the text from an element, try:
97 101
98 102 ```elixir
99 103 Floki.find(html, ".headline")
100 - |> Floki.text
104 + |> Floki.text
101 105
102 106 # => "Floki"
103 107 ```
  @@ -2,7 +2,7 @@
2 2 {<<"build_tools">>,[<<"mix">>]}.
3 3 {<<"contributors">>,[<<"Philip Sampaio Silva">>]}.
4 4 {<<"description">>,
5 - <<"A HTML parser and seeker.\n\nYou can search inside HTML documents using CSS like selectors.\n">>}.
5 + <<"A HTML parser and searcher.\n\nYou can search inside HTML documents using CSS like selectors.\n">>}.
6 6 {<<"elixir">>,<<">= 1.0.0">>}.
7 7 {<<"files">>,
8 8 [<<"lib/floki.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,
  @@ -10,8 +10,8 @@
10 10 <<"src/mochiweb_charref.erl">>,<<"src/mochiweb_html.erl">>]}.
11 11 {<<"licenses">>,[<<"MIT">>]}.
12 12 {<<"links">>,
13 - #{<<"Docs">> => <<"http://hexdocs.pm/floki">>,
14 - <<"GitHub">> => <<"https://github.com/philss/floki">>}}.
13 + [{<<"Docs">>,<<"http://hexdocs.pm/floki">>},
14 + {<<"GitHub">>,<<"https://github.com/philss/floki">>}]}.
15 15 {<<"name">>,<<"floki">>}.
16 - {<<"requirements">>,#{}}.
17 - {<<"version">>,<<"0.0.5">>}.
16 + {<<"requirements">>,[]}.
17 + {<<"version">>,<<"0.1.0">>}.
  @@ -24,7 +24,9 @@ defmodule Floki do
24 24
25 25 * Floki.find(html, "#content") : returns the section with all children;
26 26 * Floki.find(html, ".headline") : returns a list with the `p` element;
27 - * Floki.find(html, "a") : returns a list with the `a` element.
27 + * Floki.find(html, "a") : returns a list with the `a` element;
28 + * Floki.find(html, "#content a") # returns all links inside content section;
29 + * Floki.find(html, ".headline, a") # returns the .headline elements and links.
28 30
29 31 Each HTML node is represented by a tuple like:
30 32
  @@ -40,10 +42,10 @@ defmodule Floki do
40 42 You can write a simple HTML crawler (with support of [HTTPoison](https://github.com/edgurgel/httpoison)) with a few lines of code:
41 43
42 44 html
43 - |> Floki.find(".pages")
44 - |> Floki.find("a")
45 - |> Floki.attribute("href")
46 - |> Enum.map(fn(url) -> HTTPoison.get!(url) end)
45 + |> Floki.find(".pages")
46 + |> Floki.find("a")
47 + |> Floki.attribute("href")
48 + |> Enum.map(fn(url) -> HTTPoison.get!(url) end)
47 49
48 50 It is simple as that!
49 51 """
  @@ -56,14 +58,17 @@ defmodule Floki do
56 58 ## Examples
57 59
58 60 iex> Floki.parse("<div class=js-action>hello world</div>")
59 - {"div", [{"class", "js-action" }], ["hello world"]}
61 + {"div", [{"class", "js-action"}], ["hello world"]}
60 62
61 63 """
64 +
62 65 @spec parse(binary) :: html_tree
66 +
63 67 def parse(html) do
64 68 :mochiweb_html.parse(html)
65 69 end
66 70
71 +
67 72 @doc """
68 73 Finds elements inside a HTML tree or string.
69 74 You can search by class, tag name or id.
  @@ -71,12 +76,12 @@ defmodule Floki do
71 76 It is possible to compose searches:
72 77
73 78 Floki.find(html_string, ".class")
74 - |> Floki.find(".another-class-inside-small-scope")
79 + |> Floki.find(".another-class-inside-small-scope")
75 80
76 81 ## Examples
77 82
78 83 iex> Floki.find("<p><span class=hint>hello</span></p>", ".hint")
79 - [{"span", [{ "class", "hint" }], ["hello"]}]
84 + [{"span", [{"class", "hint"}], ["hello"]}]
80 85
81 86 iex> "<body><div id=important><div>Content</div></div></body>" |> Floki.find("#important")
82 87 {"div", [{"id", "important"}], [{"div", [], ["Content"]}]}
  @@ -89,23 +94,46 @@ defmodule Floki do
89 94 @spec find(binary | html_tree, binary) :: html_tree
90 95
91 96 def find(html, selector) when is_binary(html) do
92 - parse(html)
93 - |> find(selector)
97 + html_tree = parse(html)
98 +
99 + find(html_tree, selector)
94 100 end
95 101
96 - def find(html_tree, "." <> class) do
97 - find_by_selector(class, html_tree, &class_matcher/3, [])
98 - |> Enum.reverse
99 - end
102 + def find(html_tree, selector) do
103 + cond do
104 + String.contains?(selector, ",") ->
105 + selectors = String.split(selector, ",")
100 106
101 - def find(html_tree, "#" <> id) do
102 - find_by_selector(id, html_tree, &id_matcher/3, [])
103 - |> List.first
104 - end
107 + Enum.reduce selectors, [], fn(selector, acc) ->
108 + selector = String.strip(selector)
105 109
106 - def find(html_tree, tag_name) do
107 - find_by_selector(tag_name, html_tree, &tag_matcher/3, [])
108 - |> Enum.reverse
110 + nodes = find(html_tree, selector)
111 +
112 + unless is_list(nodes), do: nodes = [nodes]
113 +
114 + Enum.concat(acc, nodes)
115 + end
116 + String.contains?(selector, "\s") ->
117 + descendent_selector = String.split(selector)
118 +
119 + Enum.reduce descendent_selector, html_tree, fn(selector, tree) ->
120 + find(tree, selector)
121 + end
122 + String.starts_with?(selector, ".") ->
123 + "." <> class = selector
124 + {:ok, nodes} = find_by_selector(class, html_tree, &class_matcher/3, {:ok, []})
125 +
126 + Enum.reverse(nodes)
127 + String.starts_with?(selector, "#") ->
128 + "#" <> id = selector
129 + {_status, nodes} = find_by_selector(id, html_tree, &id_matcher/3, {:ok, []})
130 +
131 + List.first(nodes)
132 + true ->
133 + {:ok, nodes} = find_by_selector(selector, html_tree, &tag_matcher/3, {:ok, []})
134 +
135 + Enum.reverse(nodes)
136 + end
109 137 end
110 138
111 139 @doc """
  @@ -122,8 +150,8 @@ defmodule Floki do
122 150
123 151 def attribute(html, selector, attribute_name) do
124 152 html
125 - |> find(selector)
126 - |> get_attribute_values(attribute_name)
153 + |> find(selector)
154 + |> attribute_values(attribute_name)
127 155 end
128 156
129 157 @doc """
  @@ -140,10 +168,9 @@ defmodule Floki do
140 168
141 169 def attribute(elements, attribute_name) do
142 170 elements
143 - |> get_attribute_values(attribute_name)
171 + |> attribute_values(attribute_name)
144 172 end
145 173
146 -
147 174 @doc """
148 175 Returns the text nodes from a html tree.
149 176
  @@ -157,14 +184,14 @@ defmodule Floki do
157 184 @spec text(html_tree | binary) :: binary
158 185
159 186 def text(html) when is_binary(html), do: parse(html) |> text
160 - def text(element) when is_tuple(element), do: text(element, "")
187 + def text(element) when is_tuple(element), do: _text(element, "")
161 188 def text(elements) do
162 189 Enum.reduce elements, "", fn(element, str) ->
163 - text(element, str)
190 + _text(element, str)
164 191 end
165 192 end
166 193
167 - defp text({_, _, children}, acc) do
194 + defp _text({_, _, children}, acc) do
168 195 Enum.reduce children, acc, fn(child, istr) ->
169 196 if is_binary(child) do
170 197 (istr <> "\s" <> child) |> String.strip
  @@ -174,77 +201,82 @@ defmodule Floki do
174 201 end
175 202 end
176 203
177 -
178 204 defp attribute_match?(attributes, attribute_name) do
179 - Enum.find(attributes, fn({ attr_name, _ }) ->
205 + Enum.find attributes, fn({attr_name, _}) ->
180 206 attr_name == attribute_name
181 - end)
207 + end
182 208 end
183 209
184 210 defp attribute_match?(attributes, attribute_name, selector_value) do
185 - Enum.find(attributes, fn(attribute) ->
186 - { attr_name, attr_value } = attribute
211 + Enum.find attributes, fn(attribute) ->
212 + {attr_name, attr_value} = attribute
187 213
188 214 attr_name == attribute_name && value_match?(attr_value, selector_value)
189 - end)
215 + end
190 216 end
191 217
192 218 defp find_by_selector(_selector, {}, _, acc), do: acc
193 219 defp find_by_selector(_selector, [], _, acc), do: acc
220 + defp find_by_selector(_selector, _, _, {:done, nodes}), do: {:done, nodes}
194 221 defp find_by_selector(_selector, tree, _, acc) when is_binary(tree), do: acc
195 222 defp find_by_selector(selector, [h|t], matcher, acc) do
196 223 acc = find_by_selector(selector, h, matcher, acc)
197 224 find_by_selector(selector, t, matcher, acc)
198 225 end
199 226 # Ignores comments
200 - defp find_by_selector(_selector, { :comment, _comment }, _, acc), do: acc
201 - defp find_by_selector(selector, node, matcher, acc) when is_tuple(node) do
202 - { _, _, child_node } = node
227 + defp find_by_selector(_selector, {:comment, _comment}, _, acc), do: acc
228 + defp find_by_selector(selector, node, matcher, acc) do
229 + {_, _, child_node} = node
203 230
204 231 acc = matcher.(selector, node, acc)
205 232
206 233 find_by_selector(selector, child_node, matcher, acc)
207 234 end
208 235
209 -
210 - defp get_attribute_values(element, attr_name) when is_tuple(element) do
211 - get_attribute_values([element], attr_name)
236 + defp attribute_values(element, attr_name) when is_tuple(element) do
237 + attribute_values([element], attr_name)
212 238 end
213 - defp get_attribute_values(elements, attr_name) do
214 - Enum.map(elements, fn(el) ->
215 - { _name, attributes, _childs } = el
239 + defp attribute_values(elements, attr_name) do
240 + values = Enum.reduce elements, [], fn({_, attributes, _}, acc) ->
241 + case attribute_match?(attributes, attr_name) do
242 + {_attr_name, value} ->
243 + [value|acc]
244 + _ ->
245 + acc
246 + end
247 + end
216 248
217 - attribute_match?(attributes, attr_name)
218 - end)
219 - |> Enum.reject(fn(x) -> is_nil(x) end)
220 - |> Enum.map(fn({_attr_name, value}) -> value end)
249 + Enum.reverse values
221 250 end
222 251
223 252 defp class_matcher(class_name, node, acc) do
224 - { _, attributes, _ } = node
253 + {_, attributes, _} = node
254 + {:ok, acc_nodes} = acc
225 255
226 256 if attribute_match?(attributes, "class", class_name) do
227 - acc = [node|acc]
257 + acc = {:ok, [node|acc_nodes]}
228 258 end
229 259
230 260 acc
231 261 end
232 262
233 263 defp tag_matcher(tag_name, node, acc) do
234 - { tag, _, _ } = node
264 + {tag, _, _} = node
265 + {:ok, acc_nodes} = acc
235 266
236 267 if tag == tag_name do
237 - acc = [node|acc]
268 + acc = {:ok, [node|acc_nodes]}
238 269 end
239 270
240 271 acc
241 272 end
242 273
243 274 defp id_matcher(id, node, acc) do
244 - { _, attributes, _ } = node
275 + {_, attributes, _} = node
276 + {:ok, acc_nodes} = acc
245 277
246 278 if attribute_match?(attributes, "id", id) do
247 - acc = [node|acc]
279 + acc = {:done, [node|acc_nodes]}
248 280 end
249 281
250 282 acc
  @@ -252,7 +284,7 @@ defmodule Floki do
252 284
253 285 defp value_match?(attribute_value, selector_value) do
254 286 attribute_value
255 - |> String.split
256 - |> Enum.any?(fn(x) -> x == selector_value end)
287 + |> String.split
288 + |> Enum.any?(fn(x) -> x == selector_value end)
257 289 end
258 290 end
  @@ -3,7 +3,7 @@ defmodule Floki.Mixfile do
3 3
4 4 def project do
5 5 [app: :floki,
6 - version: "0.0.5",
6 + version: "0.1.0",
7 7 elixir: ">= 1.0.0",
8 8 package: package,
9 9 description: description,
  @@ -15,7 +15,7 @@ defmodule Floki.Mixfile do
15 15 #
16 16 # Type `mix help compile.app` for more information
17 17 def application do
18 - [applications: [:logger]]
18 + []
19 19 end
20 20
21 21 # Dependencies can be Hex packages:
  @@ -34,7 +34,7 @@ defmodule Floki.Mixfile do
34 34
35 35 defp description do
36 36 """
37 - A HTML parser and seeker.
37 + A HTML parser and searcher.
38 38
39 39 You can search inside HTML documents using CSS like selectors.
40 40 """