Packages

floki

0.1.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
0 Dependencies
Dependants
Files
Activity
Show whitespace

Compare versions

4 files changed
+90 additions
-54 deletions
  @@ -1,9 +1,10 @@
1 1 Floki
2 2 =====
3 3
4 - [![Build Status](https://travis-ci.org/philss/floki.svg?branch=master)](https://travis-ci.org/philss/floki)
4 + [![Build status](https://travis-ci.org/philss/floki.svg?branch=master)](https://travis-ci.org/philss/floki)
5 + [![Floki version](https://img.shields.io/hexpm/v/floki.svg)](https://hex.pm/packages/floki)
5 6
6 - This is a simple HTML parser that enables searching using CSS like selectors.
7 + This is a simple HTML parser that enables search using CSS like selectors.
7 8
8 9 You can search elements by class, tag name and id.
9 10
  @@ -27,9 +28,13 @@ Assuming that you have the following HTML:
27 28
28 29 You can perform the following queries:
29 30
30 - * Floki.find(html, "#content") : returns the section with all children;
31 - * Floki.find(html, ".headline") : returns a list with the `p` element;
32 - * Floki.find(html, "a") : returns a list with the `a` element.
31 + ```elixir
32 + Floki.find(html, "#content") # returns the section with all children
33 + Floki.find(html, ".headline") # returns a list with the `p` element
34 + Floki.find(html, "a") # returns a list with the `a` element
35 + Floki.find(html, "#content a") # returns all links inside content section
36 + Floki.find(html, ".headline, a") # returns the .headline elements and links
37 + ```
33 38
34 39 Each HTML node is represented by a tuple like:
35 40
  @@ -46,8 +51,7 @@ You can write a simple HTML crawler (with support of [HTTPoison](https://github.
46 51
47 52 ```elixir
48 53 html
49 - |> Floki.find(".pages")
50 - |> Floki.find("a")
54 + |> Floki.find(".pages a")
51 55 |> Floki.attribute("href")
52 56 |> Enum.map(fn(url) -> HTTPoison.get!(url) end)
53 57 ```
  @@ -2,7 +2,7 @@
2 2 {<<"build_tools">>,[<<"mix">>]}.
3 3 {<<"contributors">>,[<<"Philip Sampaio Silva">>]}.
4 4 {<<"description">>,
5 - <<"A HTML parser and seeker.\n\nYou can search inside HTML documents using CSS like selectors.\n">>}.
5 + <<"A HTML parser and searcher.\n\nYou can search inside HTML documents using CSS like selectors.\n">>}.
6 6 {<<"elixir">>,<<">= 1.0.0">>}.
7 7 {<<"files">>,
8 8 [<<"lib/floki.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,
  @@ -10,8 +10,8 @@
10 10 <<"src/mochiweb_charref.erl">>,<<"src/mochiweb_html.erl">>]}.
11 11 {<<"licenses">>,[<<"MIT">>]}.
12 12 {<<"links">>,
13 - #{<<"Docs">> => <<"http://hexdocs.pm/floki">>,
14 - <<"GitHub">> => <<"https://github.com/philss/floki">>}}.
13 + [{<<"Docs">>,<<"http://hexdocs.pm/floki">>},
14 + {<<"GitHub">>,<<"https://github.com/philss/floki">>}]}.
15 15 {<<"name">>,<<"floki">>}.
16 - {<<"requirements">>,#{}}.
17 - {<<"version">>,<<"0.0.5">>}.
16 + {<<"requirements">>,[]}.
17 + {<<"version">>,<<"0.1.0">>}.
  @@ -24,7 +24,9 @@ defmodule Floki do
24 24
25 25 * Floki.find(html, "#content") : returns the section with all children;
26 26 * Floki.find(html, ".headline") : returns a list with the `p` element;
27 - * Floki.find(html, "a") : returns a list with the `a` element.
27 + * Floki.find(html, "a") : returns a list with the `a` element;
28 + * Floki.find(html, "#content a") # returns all links inside content section;
29 + * Floki.find(html, ".headline, a") # returns the .headline elements and links.
28 30
29 31 Each HTML node is represented by a tuple like:
30 32
  @@ -59,11 +61,14 @@ defmodule Floki do
59 61 {"div", [{"class", "js-action"}], ["hello world"]}
60 62
61 63 """
64 +
62 65 @spec parse(binary) :: html_tree
66 +
63 67 def parse(html) do
64 68 :mochiweb_html.parse(html)
65 69 end
66 70
71 +
67 72 @doc """
68 73 Finds elements inside a HTML tree or string.
69 74 You can search by class, tag name or id.
  @@ -89,23 +94,46 @@ defmodule Floki do
89 94 @spec find(binary | html_tree, binary) :: html_tree
90 95
91 96 def find(html, selector) when is_binary(html) do
92 - parse(html)
93 - |> find(selector)
97 + html_tree = parse(html)
98 +
99 + find(html_tree, selector)
94 100 end
95 101
96 - def find(html_tree, "." <> class) do
97 - find_by_selector(class, html_tree, &class_matcher/3, [])
98 - |> Enum.reverse
99 - end
102 + def find(html_tree, selector) do
103 + cond do
104 + String.contains?(selector, ",") ->
105 + selectors = String.split(selector, ",")
100 106
101 - def find(html_tree, "#" <> id) do
102 - find_by_selector(id, html_tree, &id_matcher/3, [])
103 - |> List.first
104 - end
107 + Enum.reduce selectors, [], fn(selector, acc) ->
108 + selector = String.strip(selector)
105 109
106 - def find(html_tree, tag_name) do
107 - find_by_selector(tag_name, html_tree, &tag_matcher/3, [])
108 - |> Enum.reverse
110 + nodes = find(html_tree, selector)
111 +
112 + unless is_list(nodes), do: nodes = [nodes]
113 +
114 + Enum.concat(acc, nodes)
115 + end
116 + String.contains?(selector, "\s") ->
117 + descendent_selector = String.split(selector)
118 +
119 + Enum.reduce descendent_selector, html_tree, fn(selector, tree) ->
120 + find(tree, selector)
121 + end
122 + String.starts_with?(selector, ".") ->
123 + "." <> class = selector
124 + {:ok, nodes} = find_by_selector(class, html_tree, &class_matcher/3, {:ok, []})
125 +
126 + Enum.reverse(nodes)
127 + String.starts_with?(selector, "#") ->
128 + "#" <> id = selector
129 + {_status, nodes} = find_by_selector(id, html_tree, &id_matcher/3, {:ok, []})
130 +
131 + List.first(nodes)
132 + true ->
133 + {:ok, nodes} = find_by_selector(selector, html_tree, &tag_matcher/3, {:ok, []})
134 +
135 + Enum.reverse(nodes)
136 + end
109 137 end
110 138
111 139 @doc """
  @@ -123,7 +151,7 @@ defmodule Floki do
123 151 def attribute(html, selector, attribute_name) do
124 152 html
125 153 |> find(selector)
126 - |> get_attribute_values(attribute_name)
154 + |> attribute_values(attribute_name)
127 155 end
128 156
129 157 @doc """
  @@ -140,10 +168,9 @@ defmodule Floki do
140 168
141 169 def attribute(elements, attribute_name) do
142 170 elements
143 - |> get_attribute_values(attribute_name)
171 + |> attribute_values(attribute_name)
144 172 end
145 173
146 -
147 174 @doc """
148 175 Returns the text nodes from a html tree.
149 176
  @@ -157,14 +184,14 @@ defmodule Floki do
157 184 @spec text(html_tree | binary) :: binary
158 185
159 186 def text(html) when is_binary(html), do: parse(html) |> text
160 - def text(element) when is_tuple(element), do: text(element, "")
187 + def text(element) when is_tuple(element), do: _text(element, "")
161 188 def text(elements) do
162 189 Enum.reduce elements, "", fn(element, str) ->
163 - text(element, str)
190 + _text(element, str)
164 191 end
165 192 end
166 193
167 - defp text({_, _, children}, acc) do
194 + defp _text({_, _, children}, acc) do
168 195 Enum.reduce children, acc, fn(child, istr) ->
169 196 if is_binary(child) do
170 197 (istr <> "\s" <> child) |> String.strip
  @@ -174,23 +201,23 @@ defmodule Floki do
174 201 end
175 202 end
176 203
177 -
178 204 defp attribute_match?(attributes, attribute_name) do
179 - Enum.find(attributes, fn({ attr_name, _ }) ->
205 + Enum.find attributes, fn({attr_name, _}) ->
180 206 attr_name == attribute_name
181 - end)
207 + end
182 208 end
183 209
184 210 defp attribute_match?(attributes, attribute_name, selector_value) do
185 - Enum.find(attributes, fn(attribute) ->
211 + Enum.find attributes, fn(attribute) ->
186 212 {attr_name, attr_value} = attribute
187 213
188 214 attr_name == attribute_name && value_match?(attr_value, selector_value)
189 - end)
215 + end
190 216 end
191 217
192 218 defp find_by_selector(_selector, {}, _, acc), do: acc
193 219 defp find_by_selector(_selector, [], _, acc), do: acc
220 + defp find_by_selector(_selector, _, _, {:done, nodes}), do: {:done, nodes}
194 221 defp find_by_selector(_selector, tree, _, acc) when is_binary(tree), do: acc
195 222 defp find_by_selector(selector, [h|t], matcher, acc) do
196 223 acc = find_by_selector(selector, h, matcher, acc)
  @@ -198,7 +225,7 @@ defmodule Floki do
198 225 end
199 226 # Ignores comments
200 227 defp find_by_selector(_selector, {:comment, _comment}, _, acc), do: acc
201 - defp find_by_selector(selector, node, matcher, acc) when is_tuple(node) do
228 + defp find_by_selector(selector, node, matcher, acc) do
202 229 {_, _, child_node} = node
203 230
204 231 acc = matcher.(selector, node, acc)
  @@ -206,25 +233,28 @@ defmodule Floki do
206 233 find_by_selector(selector, child_node, matcher, acc)
207 234 end
208 235
209 -
210 - defp get_attribute_values(element, attr_name) when is_tuple(element) do
211 - get_attribute_values([element], attr_name)
236 + defp attribute_values(element, attr_name) when is_tuple(element) do
237 + attribute_values([element], attr_name)
238 + end
239 + defp attribute_values(elements, attr_name) do
240 + values = Enum.reduce elements, [], fn({_, attributes, _}, acc) ->
241 + case attribute_match?(attributes, attr_name) do
242 + {_attr_name, value} ->
243 + [value|acc]
244 + _ ->
245 + acc
246 + end
212 247 end
213 - defp get_attribute_values(elements, attr_name) do
214 - Enum.map(elements, fn(el) ->
215 - { _name, attributes, _childs } = el
216 248
217 - attribute_match?(attributes, attr_name)
218 - end)
219 - |> Enum.reject(fn(x) -> is_nil(x) end)
220 - |> Enum.map(fn({_attr_name, value}) -> value end)
249 + Enum.reverse values
221 250 end
222 251
223 252 defp class_matcher(class_name, node, acc) do
224 253 {_, attributes, _} = node
254 + {:ok, acc_nodes} = acc
225 255
226 256 if attribute_match?(attributes, "class", class_name) do
227 - acc = [node|acc]
257 + acc = {:ok, [node|acc_nodes]}
228 258 end
229 259
230 260 acc
  @@ -232,9 +262,10 @@ defmodule Floki do
232 262
233 263 defp tag_matcher(tag_name, node, acc) do
234 264 {tag, _, _} = node
265 + {:ok, acc_nodes} = acc
235 266
236 267 if tag == tag_name do
237 - acc = [node|acc]
268 + acc = {:ok, [node|acc_nodes]}
238 269 end
239 270
240 271 acc
  @@ -242,9 +273,10 @@ defmodule Floki do
242 273
243 274 defp id_matcher(id, node, acc) do
244 275 {_, attributes, _} = node
276 + {:ok, acc_nodes} = acc
245 277
246 278 if attribute_match?(attributes, "id", id) do
247 - acc = [node|acc]
279 + acc = {:done, [node|acc_nodes]}
248 280 end
249 281
250 282 acc
  @@ -3,7 +3,7 @@ defmodule Floki.Mixfile do
3 3
4 4 def project do
5 5 [app: :floki,
6 - version: "0.0.5",
6 + version: "0.1.0",
7 7 elixir: ">= 1.0.0",
8 8 package: package,
9 9 description: description,
  @@ -15,7 +15,7 @@ defmodule Floki.Mixfile do
15 15 #
16 16 # Type `mix help compile.app` for more information
17 17 def application do
18 - [applications: [:logger]]
18 + []
19 19 end
20 20
21 21 # Dependencies can be Hex packages:
  @@ -34,7 +34,7 @@ defmodule Floki.Mixfile do
34 34
35 35 defp description do
36 36 """
37 - A HTML parser and seeker.
37 + A HTML parser and searcher.
38 38
39 39 You can search inside HTML documents using CSS like selectors.
40 40 """