Packages

floki

0.0.3
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
0 Dependencies
Dependants
Files
Activity
Hide whitespace

Compare versions

8 files changed
+2977 additions
-25 deletions
  @@ -1,37 +1,55 @@
1 - Floki
1 + Floki - search inside HTML documents
2 2 =====
3 3
4 + [![Build Status](https://travis-ci.org/philss/floki.svg?branch=master)](https://travis-ci.org/philss/floki)
5 +
4 6 Floki is useful to search inside HTML documents using query selectors (like jQuery).
5 7 Under the hood, it uses the [Mochiweb](https://github.com/mochi/mochiweb) HTML parser.
6 8
7 - This version works with simple CSS class selectors (without nesting or group),
8 - like `.class-name`.
9 + This version works with simple CSS selectors (without nesting or group).
10 + List of selectors:
11 +
12 + * class selectors - Ex.: `.class-name`
13 + * id selectors - Ex.: `#element-id`
14 + * tag selectors - Ex.: `img`
9 15
10 16 ## API
11 17
12 18 To parse a HTML document, try:
13 19
14 20 ```elixir
21 + html = """
22 + <html>
23 + <body>
24 + <div class="example"></div>
25 + </body>
26 + </html>
27 + """
28 +
15 29 Floki.parse(html)
30 + # => {"html", [], [{"body", [], [{"div", [{"class", "example"}], []}]}]}
16 31 ```
17 32
18 - To find elements with the class `js-link`, try:
33 + To find elements with the class `example`, try:
19 34
20 35 ```elixir
21 - Floki.find(html, ".js-link")
36 + Floki.find(html, ".example")
37 + # => [{"div", [{"class", "example"}], []}]
22 38 ```
23 39
24 40 To fetch some attribute from elements, try:
25 41
26 42 ```elixir
27 - Floki.attribute(html, ".js-link", "href")
43 + Floki.attribute(html, ".example", "class") # href or src are good possibilities to fetch links
44 + # => ["example"]
28 45 ```
29 46
30 47 You can also get attributes from elements that you already have:
31 48
32 49 ```elixir
33 - Floki.find(html, ".js-link")
34 - |> Floki.attribute("href")
50 + Floki.find(html, ".example")
51 + |> Floki.attribute("class")
52 + # => ["example"]
35 53 ```
36 54
37 55 ## License
  @@ -6,9 +6,10 @@
6 6 {<<"elixir">>,<<"~> 1.0.0">>}.
7 7 {<<"files">>,
8 8 [<<"lib/floki.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,
9 - <<"src/mochiweb_html.erl">>]}.
9 + <<"src/mochinum.erl">>,<<"src/mochiutf8.erl">>,
10 + <<"src/mochiweb_charref.erl">>,<<"src/mochiweb_html.erl">>]}.
10 11 {<<"licenses">>,[<<"MIT">>]}.
11 12 {<<"links">>,#{<<"GitHub">> => <<"https://github.com/philss/floki">>}}.
12 13 {<<"name">>,<<"floki">>}.
13 14 {<<"requirements">>,#{}}.
14 - {<<"version">>,<<"0.0.2">>}.
15 + {<<"version">>,<<"0.0.3">>}.
  @@ -9,7 +9,17 @@ defmodule Floki do
9 9 end
10 10
11 11 def find(html_tree, "." <> class) do
12 - find_by_class(class, html_tree, [])
12 + find_by_selector(class, html_tree, &class_matcher/3, [])
13 + |> Enum.reverse
14 + end
15 +
16 + def find(html_tree, "#" <> id) do
17 + find_by_selector(id, html_tree, &id_matcher/3, [])
18 + |> List.first
19 + end
20 +
21 + def find(html_tree, tag_name) do
22 + find_by_selector(tag_name, html_tree, &tag_matcher/3, [])
13 23 |> Enum.reverse
14 24 end
15 25
  @@ -36,20 +46,20 @@ defmodule Floki do
36 46 end)
37 47 end
38 48
39 - defp find_by_class(_class, {}, acc), do: acc
40 - defp find_by_class(_class, [], acc), do: acc
41 - defp find_by_class(_class, tree, acc) when is_binary(tree), do: acc
42 - defp find_by_class(class, [h|t], acc) do
43 - acc = find_by_class(class, h, acc)
44 - find_by_class(class, t, acc)
49 + defp find_by_selector(_selector, {}, _, acc), do: acc
50 + defp find_by_selector(_selector, [], _, acc), do: acc
51 + defp find_by_selector(_selector, tree, _, acc) when is_binary(tree), do: acc
52 + defp find_by_selector(selector, [h|t], matcher, acc) do
53 + acc = find_by_selector(selector, h, matcher, acc)
54 + find_by_selector(selector, t, matcher, acc)
45 55 end
46 - defp find_by_class(_class, { :comment, _comment }, acc), do: acc
47 - defp find_by_class(class, { name, attributes, child_node }, acc) do
48 - if class_match?(attributes, class) do
49 - acc = [{name, attributes, child_node}|acc]
50 - end
56 + defp find_by_selector(_selector, { :comment, _comment }, _, acc), do: acc
57 + defp find_by_selector(selector, node, matcher, acc) when is_tuple(node) do
58 + { _, _, child_node } = node
51 59
52 - find_by_class(class, child_node, acc)
60 + acc = matcher.(selector, node, acc)
61 +
62 + find_by_selector(selector, child_node, matcher, acc)
53 63 end
54 64
55 65 defp get_attribute_values(elements, attr_name) do
  @@ -61,4 +71,34 @@ defmodule Floki do
61 71 |> Enum.reject(fn(x) -> is_nil(x) end)
62 72 |> Enum.map(fn({_attr_name, value}) -> value end)
63 73 end
74 +
75 + defp class_matcher(selector, node, acc) do
76 + { _, attributes, _ } = node
77 +
78 + if class_match?(attributes, selector) do
79 + acc = [node|acc]
80 + end
81 +
82 + acc
83 + end
84 +
85 + defp tag_matcher(tag_name, node, acc) do
86 + { tag, _, _ } = node
87 +
88 + if tag == tag_name do
89 + acc = [node|acc]
90 + end
91 +
92 + acc
93 + end
94 +
95 + defp id_matcher(id, node, acc) do
96 + { _, attributes, _ } = node
97 +
98 + if attribute_match?(attributes, "id", id) do
99 + acc = [node|acc]
100 + end
101 +
102 + acc
103 + end
64 104 end
  @@ -3,7 +3,7 @@ defmodule Floki.Mixfile do
3 3
4 4 def project do
5 5 [app: :floki,
6 - version: "0.0.2",
6 + version: "0.0.3",
7 7 elixir: "~> 1.0.0",
8 8 package: package,
9 9 description: description,
  @@ -0,0 +1,367 @@
1 + %% @copyright 2007 Mochi Media, Inc.
2 + %% @author Bob Ippolito <bob@mochimedia.com>
3 +
4 + %% This module was extracted from `https://github.com/mochi/mochiweb/blob/master/src/mochinum.erl`
5 + %% and is under the MIT license as described below.
6 +
7 + %% This is the MIT license.
8 + %%
9 + %% Copyright (c) 2007 Mochi Media, Inc.
10 + %%
11 + %% Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
12 + %%
13 + %% The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
14 + %%
15 + %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
16 +
17 + %% @doc Useful numeric algorithms for floats that cover some deficiencies
18 + %% in the math module. More interesting is digits/1, which implements
19 + %% the algorithm from:
20 + %% http://www.cs.indiana.edu/~burger/fp/index.html
21 + %% See also "Printing Floating-Point Numbers Quickly and Accurately"
22 + %% in Proceedings of the SIGPLAN '96 Conference on Programming Language
23 + %% Design and Implementation.
24 +
25 + -module(mochinum).
26 + -author("Bob Ippolito <bob@mochimedia.com>").
27 + -export([digits/1, frexp/1, int_pow/2, int_ceil/1]).
28 +
29 + %% IEEE 754 Float exponent bias
30 + -define(FLOAT_BIAS, 1022).
31 + -define(MIN_EXP, -1074).
32 + -define(BIG_POW, 4503599627370496).
33 +
34 + %% External API
35 +
36 + %% @spec digits(number()) -> string()
37 + %% @doc Returns a string that accurately represents the given integer or float
38 + %% using a conservative amount of digits. Great for generating
39 + %% human-readable output, or compact ASCII serializations for floats.
40 + digits(N) when is_integer(N) ->
41 + integer_to_list(N);
42 + digits(0.0) ->
43 + "0.0";
44 + digits(Float) ->
45 + {Frac1, Exp1} = frexp_int(Float),
46 + [Place0 | Digits0] = digits1(Float, Exp1, Frac1),
47 + {Place, Digits} = transform_digits(Place0, Digits0),
48 + R = insert_decimal(Place, Digits),
49 + case Float < 0 of
50 + true ->
51 + [$- | R];
52 + _ ->
53 + R
54 + end.
55 +
56 + %% @spec frexp(F::float()) -> {Frac::float(), Exp::float()}
57 + %% @doc Return the fractional and exponent part of an IEEE 754 double,
58 + %% equivalent to the libc function of the same name.
59 + %% F = Frac * pow(2, Exp).
60 + frexp(F) ->
61 + frexp1(unpack(F)).
62 +
63 + %% @spec int_pow(X::integer(), N::integer()) -> Y::integer()
64 + %% @doc Moderately efficient way to exponentiate integers.
65 + %% int_pow(10, 2) = 100.
66 + int_pow(_X, 0) ->
67 + 1;
68 + int_pow(X, N) when N > 0 ->
69 + int_pow(X, N, 1).
70 +
71 + %% @spec int_ceil(F::float()) -> integer()
72 + %% @doc Return the ceiling of F as an integer. The ceiling is defined as
73 + %% F when F == trunc(F);
74 + %% trunc(F) when F &lt; 0;
75 + %% trunc(F) + 1 when F &gt; 0.
76 + int_ceil(X) ->
77 + T = trunc(X),
78 + case (X - T) of
79 + Pos when Pos > 0 -> T + 1;
80 + _ -> T
81 + end.
82 +
83 +
84 + %% Internal API
85 +
86 + int_pow(X, N, R) when N < 2 ->
87 + R * X;
88 + int_pow(X, N, R) ->
89 + int_pow(X * X, N bsr 1, case N band 1 of 1 -> R * X; 0 -> R end).
90 +
91 + insert_decimal(0, S) ->
92 + "0." ++ S;
93 + insert_decimal(Place, S) when Place > 0 ->
94 + L = length(S),
95 + case Place - L of
96 + 0 ->
97 + S ++ ".0";
98 + N when N < 0 ->
99 + {S0, S1} = lists:split(L + N, S),
100 + S0 ++ "." ++ S1;
101 + N when N < 6 ->
102 + %% More places than digits
103 + S ++ lists:duplicate(N, $0) ++ ".0";
104 + _ ->
105 + insert_decimal_exp(Place, S)
106 + end;
107 + insert_decimal(Place, S) when Place > -6 ->
108 + "0." ++ lists:duplicate(abs(Place), $0) ++ S;
109 + insert_decimal(Place, S) ->
110 + insert_decimal_exp(Place, S).
111 +
112 + insert_decimal_exp(Place, S) ->
113 + [C | S0] = S,
114 + S1 = case S0 of
115 + [] ->
116 + "0";
117 + _ ->
118 + S0
119 + end,
120 + Exp = case Place < 0 of
121 + true ->
122 + "e-";
123 + false ->
124 + "e+"
125 + end,
126 + [C] ++ "." ++ S1 ++ Exp ++ integer_to_list(abs(Place - 1)).
127 +
128 +
129 + digits1(Float, Exp, Frac) ->
130 + Round = ((Frac band 1) =:= 0),
131 + case Exp >= 0 of
132 + true ->
133 + BExp = 1 bsl Exp,
134 + case (Frac =/= ?BIG_POW) of
135 + true ->
136 + scale((Frac * BExp * 2), 2, BExp, BExp,
137 + Round, Round, Float);
138 + false ->
139 + scale((Frac * BExp * 4), 4, (BExp * 2), BExp,
140 + Round, Round, Float)
141 + end;
142 + false ->
143 + case (Exp =:= ?MIN_EXP) orelse (Frac =/= ?BIG_POW) of
144 + true ->
145 + scale((Frac * 2), 1 bsl (1 - Exp), 1, 1,
146 + Round, Round, Float);
147 + false ->
148 + scale((Frac * 4), 1 bsl (2 - Exp), 2, 1,
149 + Round, Round, Float)
150 + end
151 + end.
152 +
153 + scale(R, S, MPlus, MMinus, LowOk, HighOk, Float) ->
154 + Est = int_ceil(math:log10(abs(Float)) - 1.0e-10),
155 + %% Note that the scheme implementation uses a 326 element look-up table
156 + %% for int_pow(10, N) where we do not.
157 + case Est >= 0 of
158 + true ->
159 + fixup(R, S * int_pow(10, Est), MPlus, MMinus, Est,
160 + LowOk, HighOk);
161 + false ->
162 + Scale = int_pow(10, -Est),
163 + fixup(R * Scale, S, MPlus * Scale, MMinus * Scale, Est,
164 + LowOk, HighOk)
165 + end.
166 +
167 + fixup(R, S, MPlus, MMinus, K, LowOk, HighOk) ->
168 + TooLow = case HighOk of
169 + true ->
170 + (R + MPlus) >= S;
171 + false ->
172 + (R + MPlus) > S
173 + end,
174 + case TooLow of
175 + true ->
176 + [(K + 1) | generate(R, S, MPlus, MMinus, LowOk, HighOk)];
177 + false ->
178 + [K | generate(R * 10, S, MPlus * 10, MMinus * 10, LowOk, HighOk)]
179 + end.
180 +
181 + generate(R0, S, MPlus, MMinus, LowOk, HighOk) ->
182 + D = R0 div S,
183 + R = R0 rem S,
184 + TC1 = case LowOk of
185 + true ->
186 + R =< MMinus;
187 + false ->
188 + R < MMinus
189 + end,
190 + TC2 = case HighOk of
191 + true ->
192 + (R + MPlus) >= S;
193 + false ->
194 + (R + MPlus) > S
195 + end,
196 + case TC1 of
197 + false ->
198 + case TC2 of
199 + false ->
200 + [D | generate(R * 10, S, MPlus * 10, MMinus * 10,
201 + LowOk, HighOk)];
202 + true ->
203 + [D + 1]
204 + end;
205 + true ->
206 + case TC2 of
207 + false ->
208 + [D];
209 + true ->
210 + case R * 2 < S of
211 + true ->
212 + [D];
213 + false ->
214 + [D + 1]
215 + end
216 + end
217 + end.
218 +
219 + unpack(Float) ->
220 + <<Sign:1, Exp:11, Frac:52>> = <<Float:64/float>>,
221 + {Sign, Exp, Frac}.
222 +
223 + frexp1({_Sign, 0, 0}) ->
224 + {0.0, 0};
225 + frexp1({Sign, 0, Frac}) ->
226 + Exp = log2floor(Frac),
227 + <<Frac1:64/float>> = <<Sign:1, ?FLOAT_BIAS:11, (Frac-1):52>>,
228 + {Frac1, -(?FLOAT_BIAS) - 52 + Exp};
229 + frexp1({Sign, Exp, Frac}) ->
230 + <<Frac1:64/float>> = <<Sign:1, ?FLOAT_BIAS:11, Frac:52>>,
231 + {Frac1, Exp - ?FLOAT_BIAS}.
232 +
233 + log2floor(Int) ->
234 + log2floor(Int, 0).
235 +
236 + log2floor(0, N) ->
237 + N;
238 + log2floor(Int, N) ->
239 + log2floor(Int bsr 1, 1 + N).
240 +
241 +
242 + transform_digits(Place, [0 | Rest]) ->
243 + transform_digits(Place, Rest);
244 + transform_digits(Place, Digits) ->
245 + {Place, [$0 + D || D <- Digits]}.
246 +
247 +
248 + frexp_int(F) ->
249 + case unpack(F) of
250 + {_Sign, 0, Frac} ->
251 + {Frac, ?MIN_EXP};
252 + {_Sign, Exp, Frac} ->
253 + {Frac + (1 bsl 52), Exp - 53 - ?FLOAT_BIAS}
254 + end.
255 +
256 + %%
257 + %% Tests
258 + %%
259 + -ifdef(TEST).
260 + -include_lib("eunit/include/eunit.hrl").
261 +
262 + int_ceil_test() ->
263 + ?assertEqual(1, int_ceil(0.0001)),
264 + ?assertEqual(0, int_ceil(0.0)),
265 + ?assertEqual(1, int_ceil(0.99)),
266 + ?assertEqual(1, int_ceil(1.0)),
267 + ?assertEqual(-1, int_ceil(-1.5)),
268 + ?assertEqual(-2, int_ceil(-2.0)),
269 + ok.
270 +
271 + int_pow_test() ->
272 + ?assertEqual(1, int_pow(1, 1)),
273 + ?assertEqual(1, int_pow(1, 0)),
274 + ?assertEqual(1, int_pow(10, 0)),
275 + ?assertEqual(10, int_pow(10, 1)),
276 + ?assertEqual(100, int_pow(10, 2)),
277 + ?assertEqual(1000, int_pow(10, 3)),
278 + ok.
279 +
280 + digits_test() ->
281 + ?assertEqual("0",
282 + digits(0)),
283 + ?assertEqual("0.0",
284 + digits(0.0)),
285 + ?assertEqual("1.0",
286 + digits(1.0)),
287 + ?assertEqual("-1.0",
288 + digits(-1.0)),
289 + ?assertEqual("0.1",
290 + digits(0.1)),
291 + ?assertEqual("0.01",
292 + digits(0.01)),
293 + ?assertEqual("0.001",
294 + digits(0.001)),
295 + ?assertEqual("1.0e+6",
296 + digits(1000000.0)),
297 + ?assertEqual("0.5",
298 + digits(0.5)),
299 + ?assertEqual("4503599627370496.0",
300 + digits(4503599627370496.0)),
301 + %% small denormalized number
302 + %% 4.94065645841246544177e-324 =:= 5.0e-324
303 + <<SmallDenorm/float>> = <<0,0,0,0,0,0,0,1>>,
304 + ?assertEqual("5.0e-324",
305 + digits(SmallDenorm)),
306 + ?assertEqual(SmallDenorm,
307 + list_to_float(digits(SmallDenorm))),
308 + %% large denormalized number
309 + %% 2.22507385850720088902e-308
310 + <<BigDenorm/float>> = <<0,15,255,255,255,255,255,255>>,
311 + ?assertEqual("2.225073858507201e-308",
312 + digits(BigDenorm)),
313 + ?assertEqual(BigDenorm,
314 + list_to_float(digits(BigDenorm))),
315 + %% small normalized number
316 + %% 2.22507385850720138309e-308
317 + <<SmallNorm/float>> = <<0,16,0,0,0,0,0,0>>,
318 + ?assertEqual("2.2250738585072014e-308",
319 + digits(SmallNorm)),
320 + ?assertEqual(SmallNorm,
321 + list_to_float(digits(SmallNorm))),
322 + %% large normalized number
323 + %% 1.79769313486231570815e+308
324 + <<LargeNorm/float>> = <<127,239,255,255,255,255,255,255>>,
325 + ?assertEqual("1.7976931348623157e+308",
326 + digits(LargeNorm)),
327 + ?assertEqual(LargeNorm,
328 + list_to_float(digits(LargeNorm))),
329 + %% issue #10 - mochinum:frexp(math:pow(2, -1074)).
330 + ?assertEqual("5.0e-324",
331 + digits(math:pow(2, -1074))),
332 + ok.
333 +
334 + frexp_test() ->
335 + %% zero
336 + ?assertEqual({0.0, 0}, frexp(0.0)),
337 + %% one
338 + ?assertEqual({0.5, 1}, frexp(1.0)),
339 + %% negative one
340 + ?assertEqual({-0.5, 1}, frexp(-1.0)),
341 + %% small denormalized number
342 + %% 4.94065645841246544177e-324
343 + <<SmallDenorm/float>> = <<0,0,0,0,0,0,0,1>>,
344 + ?assertEqual({0.5, -1073}, frexp(SmallDenorm)),
345 + %% large denormalized number
346 + %% 2.22507385850720088902e-308
347 + <<BigDenorm/float>> = <<0,15,255,255,255,255,255,255>>,
348 + ?assertEqual(
349 + {0.99999999999999978, -1022},
350 + frexp(BigDenorm)),
351 + %% small normalized number
352 + %% 2.22507385850720138309e-308
353 + <<SmallNorm/float>> = <<0,16,0,0,0,0,0,0>>,
354 + ?assertEqual({0.5, -1021}, frexp(SmallNorm)),
355 + %% large normalized number
356 + %% 1.79769313486231570815e+308
357 + <<LargeNorm/float>> = <<127,239,255,255,255,255,255,255>>,
358 + ?assertEqual(
359 + {0.99999999999999989, 1024},
360 + frexp(LargeNorm)),
361 + %% issue #10 - mochinum:frexp(math:pow(2, -1074)).
362 + ?assertEqual(
363 + {0.5, -1073},
364 + frexp(math:pow(2, -1074))),
365 + ok.
366 +
367 + -endif.
Loading more files…