Packages

floki

0.0.2
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
0 Dependencies
Dependants
Files
Activity
Hide whitespace

Compare versions

3 files changed
+794 additions
-6 deletions
  @@ -4,9 +4,11 @@
4 4 {<<"description">>,
5 5 <<"Floki is useful to search elements inside HTML documents using query selectors (like jQuery).\n">>}.
6 6 {<<"elixir">>,<<"~> 1.0.0">>}.
7 - {<<"files">>,[<<"lib/floki.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>]}.
7 + {<<"files">>,
8 + [<<"lib/floki.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,
9 + <<"src/mochiweb_html.erl">>]}.
8 10 {<<"licenses">>,[<<"MIT">>]}.
9 11 {<<"links">>,#{<<"GitHub">> => <<"https://github.com/philss/floki">>}}.
10 12 {<<"name">>,<<"floki">>}.
11 13 {<<"requirements">>,#{}}.
12 - {<<"version">>,<<"0.0.1">>}.
14 + {<<"version">>,<<"0.0.2">>}.
  @@ -3,7 +3,7 @@ defmodule Floki.Mixfile do
3 3
4 4 def project do
5 5 [app: :floki,
6 - version: "0.0.1",
6 + version: "0.0.2",
7 7 elixir: "~> 1.0.0",
8 8 package: package,
9 9 description: description,
  @@ -28,9 +28,7 @@ defmodule Floki.Mixfile do
28 28 #
29 29 # Type `mix help deps` for more examples and options
30 30 defp deps do
31 - [
32 - {:mochiweb, git: "https://github.com/mochi/mochiweb.git", tag: "v2.9.2"}
33 - ]
31 + []
34 32 end
35 33
36 34 defp description do
  @@ -43,6 +41,7 @@ defmodule Floki.Mixfile do
43 41 %{
44 42 contributors: ["Philip Sampaio Silva"],
45 43 licenses: ["MIT"],
44 + files: ["lib", "priv", "mix.exs", "README*", "readme*", "LICENSE*", "license*", "src"],
46 45 links: %{"GitHub" => "https://github.com/philss/floki"}
47 46 }
48 47 end
  @@ -0,0 +1,787 @@
1 + %% @author Bob Ippolito <bob@mochimedia.com>
2 + %% @copyright 2007 Mochi Media, Inc.
3 +
4 + %% This file was extracted from `https://github.com/mochi/mochiweb/blob/master/src/mochiweb_html.erl`
5 + %% and is under the MIT license as described below.
6 +
7 + %% This is the MIT license.
8 + %%
9 + %% Copyright (c) 2007 Mochi Media, Inc.
10 + %%
11 + %% Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
12 + %%
13 + %% The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
14 + %%
15 + %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
16 +
17 + %% @doc Loosely tokenizes and generates parse trees for HTML 4.
18 + -module(mochiweb_html).
19 + -export([tokens/1, parse/1, parse_tokens/1, to_tokens/1, escape/1,
20 + escape_attr/1, to_html/1]).
21 + -compile([export_all]).
22 + -ifdef(TEST).
23 + -export([destack/1, destack/2, is_singleton/1]).
24 + -endif.
25 +
26 + %% This is a macro to placate syntax highlighters..
27 + -define(QUOTE, $\"). %% $\"
28 + -define(SQUOTE, $\'). %% $\'
29 + -define(ADV_COL(S, N),
30 + S#decoder{column=N+S#decoder.column,
31 + offset=N+S#decoder.offset}).
32 + -define(INC_COL(S),
33 + S#decoder{column=1+S#decoder.column,
34 + offset=1+S#decoder.offset}).
35 + -define(INC_LINE(S),
36 + S#decoder{column=1,
37 + line=1+S#decoder.line,
38 + offset=1+S#decoder.offset}).
39 + -define(INC_CHAR(S, C),
40 + case C of
41 + $\n ->
42 + S#decoder{column=1,
43 + line=1+S#decoder.line,
44 + offset=1+S#decoder.offset};
45 + _ ->
46 + S#decoder{column=1+S#decoder.column,
47 + offset=1+S#decoder.offset}
48 + end).
49 +
50 + -define(IS_WHITESPACE(C),
51 + (C =:= $\s orelse C =:= $\t orelse C =:= $\r orelse C =:= $\n)).
52 + -define(IS_LITERAL_SAFE(C),
53 + ((C >= $A andalso C =< $Z) orelse (C >= $a andalso C =< $z)
54 + orelse (C >= $0 andalso C =< $9))).
55 + -define(PROBABLE_CLOSE(C),
56 + (C =:= $> orelse ?IS_WHITESPACE(C))).
57 +
58 + -record(decoder, {line=1,
59 + column=1,
60 + offset=0}).
61 +
62 + %% @type html_node() = {string(), [html_attr()], [html_node() | string()]}
63 + %% @type html_attr() = {string(), string()}
64 + %% @type html_token() = html_data() | start_tag() | end_tag() | inline_html() | html_comment() | html_doctype()
65 + %% @type html_data() = {data, string(), Whitespace::boolean()}
66 + %% @type start_tag() = {start_tag, Name, [html_attr()], Singleton::boolean()}
67 + %% @type end_tag() = {end_tag, Name}
68 + %% @type html_comment() = {comment, Comment}
69 + %% @type html_doctype() = {doctype, [Doctype]}
70 + %% @type inline_html() = {'=', iolist()}
71 +
72 + %% External API.
73 +
74 + %% @spec parse(string() | binary()) -> html_node()
75 + %% @doc tokenize and then transform the token stream into a HTML tree.
76 + parse(Input) ->
77 + parse_tokens(tokens(Input)).
78 +
79 + %% @spec parse_tokens([html_token()]) -> html_node()
80 + %% @doc Transform the output of tokens(Doc) into a HTML tree.
81 + parse_tokens(Tokens) when is_list(Tokens) ->
82 + %% Skip over doctype, processing instructions
83 + [{start_tag, Tag, Attrs, false} | Rest] = find_document(Tokens, normal),
84 + {Tree, _} = tree(Rest, [norm({Tag, Attrs})]),
85 + Tree.
86 +
87 + find_document(Tokens=[{start_tag, _Tag, _Attrs, false} | _Rest], Mode) ->
88 + maybe_add_html_tag(Tokens, Mode);
89 + find_document([{doctype, [<<"html">>]} | Rest], _Mode) ->
90 + find_document(Rest, html5);
91 + find_document([_T | Rest], Mode) ->
92 + find_document(Rest, Mode);
93 + find_document([], _Mode) ->
94 + [].
95 +
96 + maybe_add_html_tag(Tokens=[{start_tag, Tag, _Attrs, false} | _], html5)
97 + when Tag =/= <<"html">> ->
98 + [{start_tag, <<"html">>, [], false} | Tokens];
99 + maybe_add_html_tag(Tokens, _Mode) ->
100 + Tokens.
101 +
102 + %% @spec tokens(StringOrBinary) -> [html_token()]
103 + %% @doc Transform the input UTF-8 HTML into a token stream.
104 + tokens(Input) ->
105 + tokens(iolist_to_binary(Input), #decoder{}, []).
106 +
107 + %% @spec to_tokens(html_node()) -> [html_token()]
108 + %% @doc Convert a html_node() tree to a list of tokens.
109 + to_tokens({Tag0}) ->
110 + to_tokens({Tag0, [], []});
111 + to_tokens(T={'=', _}) ->
112 + [T];
113 + to_tokens(T={doctype, _}) ->
114 + [T];
115 + to_tokens(T={comment, _}) ->
116 + [T];
117 + to_tokens({Tag0, Acc}) ->
118 + %% This is only allowed in sub-tags: {p, [{"class", "foo"}]}
119 + to_tokens({Tag0, [], Acc});
120 + to_tokens({Tag0, Attrs, Acc}) ->
121 + Tag = to_tag(Tag0),
122 + case is_singleton(Tag) of
123 + true ->
124 + to_tokens([], [{start_tag, Tag, Attrs, true}]);
125 + false ->
126 + to_tokens([{Tag, Acc}], [{start_tag, Tag, Attrs, false}])
127 + end.
128 +
129 + %% @spec to_html([html_token()] | html_node()) -> iolist()
130 + %% @doc Convert a list of html_token() to a HTML document.
131 + to_html(Node) when is_tuple(Node) ->
132 + to_html(to_tokens(Node));
133 + to_html(Tokens) when is_list(Tokens) ->
134 + to_html(Tokens, []).
135 +
136 + %% @spec escape(string() | atom() | binary()) -> binary()
137 + %% @doc Escape a string such that it's safe for HTML (amp; lt; gt;).
138 + escape(B) when is_binary(B) ->
139 + escape(binary_to_list(B), []);
140 + escape(A) when is_atom(A) ->
141 + escape(atom_to_list(A), []);
142 + escape(S) when is_list(S) ->
143 + escape(S, []).
144 +
145 + %% @spec escape_attr(string() | binary() | atom() | integer() | float()) -> binary()
146 + %% @doc Escape a string such that it's safe for HTML attrs
147 + %% (amp; lt; gt; quot;).
148 + escape_attr(B) when is_binary(B) ->
149 + escape_attr(binary_to_list(B), []);
150 + escape_attr(A) when is_atom(A) ->
151 + escape_attr(atom_to_list(A), []);
152 + escape_attr(S) when is_list(S) ->
153 + escape_attr(S, []);
154 + escape_attr(I) when is_integer(I) ->
155 + escape_attr(integer_to_list(I), []);
156 + escape_attr(F) when is_float(F) ->
157 + escape_attr(mochinum:digits(F), []).
158 +
159 + to_html([], Acc) ->
160 + lists:reverse(Acc);
161 + to_html([{'=', Content} | Rest], Acc) ->
162 + to_html(Rest, [Content | Acc]);
163 + to_html([{pi, Bin} | Rest], Acc) ->
164 + Open = [<<"<?">>,
165 + Bin,
166 + <<"?>">>],
167 + to_html(Rest, [Open | Acc]);
168 + to_html([{pi, Tag, Attrs} | Rest], Acc) ->
169 + Open = [<<"<?">>,
170 + Tag,
171 + attrs_to_html(Attrs, []),
172 + <<"?>">>],
173 + to_html(Rest, [Open | Acc]);
174 + to_html([{comment, Comment} | Rest], Acc) ->
175 + to_html(Rest, [[<<"<!--">>, Comment, <<"-->">>] | Acc]);
176 + to_html([{doctype, Parts} | Rest], Acc) ->
177 + Inside = doctype_to_html(Parts, Acc),
178 + to_html(Rest, [[<<"<!DOCTYPE">>, Inside, <<">">>] | Acc]);
179 + to_html([{data, Data, _Whitespace} | Rest], Acc) ->
180 + to_html(Rest, [escape(Data) | Acc]);
181 + to_html([{start_tag, Tag, Attrs, Singleton} | Rest], Acc) ->
182 + Open = [<<"<">>,
183 + Tag,
184 + attrs_to_html(Attrs, []),
185 + case Singleton of
186 + true -> <<" />">>;
187 + false -> <<">">>
188 + end],
189 + to_html(Rest, [Open | Acc]);
190 + to_html([{end_tag, Tag} | Rest], Acc) ->
191 + to_html(Rest, [[<<"</">>, Tag, <<">">>] | Acc]).
192 +
193 + doctype_to_html([], Acc) ->
194 + lists:reverse(Acc);
195 + doctype_to_html([Word | Rest], Acc) ->
196 + case lists:all(fun (C) -> ?IS_LITERAL_SAFE(C) end,
197 + binary_to_list(iolist_to_binary(Word))) of
198 + true ->
199 + doctype_to_html(Rest, [[<<" ">>, Word] | Acc]);
200 + false ->
201 + doctype_to_html(Rest, [[<<" \"">>, escape_attr(Word), ?QUOTE] | Acc])
202 + end.
203 +
204 + attrs_to_html([], Acc) ->
205 + lists:reverse(Acc);
206 + attrs_to_html([{K, V} | Rest], Acc) ->
207 + attrs_to_html(Rest,
208 + [[<<" ">>, escape(K), <<"=\"">>,
209 + escape_attr(V), <<"\"">>] | Acc]).
210 +
211 + escape([], Acc) ->
212 + list_to_binary(lists:reverse(Acc));
213 + escape("<" ++ Rest, Acc) ->
214 + escape(Rest, lists:reverse("&lt;", Acc));
215 + escape(">" ++ Rest, Acc) ->
216 + escape(Rest, lists:reverse("&gt;", Acc));
217 + escape("&" ++ Rest, Acc) ->
218 + escape(Rest, lists:reverse("&amp;", Acc));
219 + escape([C | Rest], Acc) ->
220 + escape(Rest, [C | Acc]).
221 +
222 + escape_attr([], Acc) ->
223 + list_to_binary(lists:reverse(Acc));
224 + escape_attr("<" ++ Rest, Acc) ->
225 + escape_attr(Rest, lists:reverse("&lt;", Acc));
226 + escape_attr(">" ++ Rest, Acc) ->
227 + escape_attr(Rest, lists:reverse("&gt;", Acc));
228 + escape_attr("&" ++ Rest, Acc) ->
229 + escape_attr(Rest, lists:reverse("&amp;", Acc));
230 + escape_attr([?QUOTE | Rest], Acc) ->
231 + escape_attr(Rest, lists:reverse("&quot;", Acc));
232 + escape_attr([C | Rest], Acc) ->
233 + escape_attr(Rest, [C | Acc]).
234 +
235 + to_tag(A) when is_atom(A) ->
236 + norm(atom_to_list(A));
237 + to_tag(L) ->
238 + norm(L).
239 +
240 + to_tokens([], Acc) ->
241 + lists:reverse(Acc);
242 + to_tokens([{Tag, []} | Rest], Acc) ->
243 + to_tokens(Rest, [{end_tag, to_tag(Tag)} | Acc]);
244 + to_tokens([{Tag0, [{T0} | R1]} | Rest], Acc) ->
245 + %% Allow {br}
246 + to_tokens([{Tag0, [{T0, [], []} | R1]} | Rest], Acc);
247 + to_tokens([{Tag0, [T0={'=', _C0} | R1]} | Rest], Acc) ->
248 + %% Allow {'=', iolist()}
249 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
250 + to_tokens([{Tag0, [T0={comment, _C0} | R1]} | Rest], Acc) ->
251 + %% Allow {comment, iolist()}
252 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
253 + to_tokens([{Tag0, [T0={pi, _S0} | R1]} | Rest], Acc) ->
254 + %% Allow {pi, binary()}
255 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
256 + to_tokens([{Tag0, [T0={pi, _S0, _A0} | R1]} | Rest], Acc) ->
257 + %% Allow {pi, binary(), list()}
258 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
259 + to_tokens([{Tag0, [{T0, A0=[{_, _} | _]} | R1]} | Rest], Acc) ->
260 + %% Allow {p, [{"class", "foo"}]}
261 + to_tokens([{Tag0, [{T0, A0, []} | R1]} | Rest], Acc);
262 + to_tokens([{Tag0, [{T0, C0} | R1]} | Rest], Acc) ->
263 + %% Allow {p, "content"} and {p, <<"content">>}
264 + to_tokens([{Tag0, [{T0, [], C0} | R1]} | Rest], Acc);
265 + to_tokens([{Tag0, [{T0, A1, C0} | R1]} | Rest], Acc) when is_binary(C0) ->
266 + %% Allow {"p", [{"class", "foo"}], <<"content">>}
267 + to_tokens([{Tag0, [{T0, A1, binary_to_list(C0)} | R1]} | Rest], Acc);
268 + to_tokens([{Tag0, [{T0, A1, C0=[C | _]} | R1]} | Rest], Acc)
269 + when is_integer(C) ->
270 + %% Allow {"p", [{"class", "foo"}], "content"}
271 + to_tokens([{Tag0, [{T0, A1, [C0]} | R1]} | Rest], Acc);
272 + to_tokens([{Tag0, [{T0, A1, C1} | R1]} | Rest], Acc) ->
273 + %% Native {"p", [{"class", "foo"}], ["content"]}
274 + Tag = to_tag(Tag0),
275 + T1 = to_tag(T0),
276 + case is_singleton(norm(T1)) of
277 + true ->
278 + to_tokens([{Tag, R1} | Rest], [{start_tag, T1, A1, true} | Acc]);
279 + false ->
280 + to_tokens([{T1, C1}, {Tag, R1} | Rest],
281 + [{start_tag, T1, A1, false} | Acc])
282 + end;
283 + to_tokens([{Tag0, [L | R1]} | Rest], Acc) when is_list(L) ->
284 + %% List text
285 + Tag = to_tag(Tag0),
286 + to_tokens([{Tag, R1} | Rest], [{data, iolist_to_binary(L), false} | Acc]);
287 + to_tokens([{Tag0, [B | R1]} | Rest], Acc) when is_binary(B) ->
288 + %% Binary text
289 + Tag = to_tag(Tag0),
290 + to_tokens([{Tag, R1} | Rest], [{data, B, false} | Acc]).
291 +
292 + tokens(B, S=#decoder{offset=O}, Acc) ->
293 + case B of
294 + <<_:O/binary>> ->
295 + lists:reverse(Acc);
296 + _ ->
297 + {Tag, S1} = tokenize(B, S),
298 + case parse_flag(Tag) of
299 + script ->
300 + {Tag2, S2} = tokenize_script(B, S1),
301 + tokens(B, S2, [Tag2, Tag | Acc]);
302 + textarea ->
303 + {Tag2, S2} = tokenize_textarea(B, S1),
304 + tokens(B, S2, [Tag2, Tag | Acc]);
305 + none ->
306 + tokens(B, S1, [Tag | Acc])
307 + end
308 + end.
309 +
310 + parse_flag({start_tag, B, _, false}) ->
311 + case string:to_lower(binary_to_list(B)) of
312 + "script" ->
313 + script;
314 + "textarea" ->
315 + textarea;
316 + _ ->
317 + none
318 + end;
319 + parse_flag(_) ->
320 + none.
321 +
322 + tokenize(B, S=#decoder{offset=O}) ->
323 + case B of
324 + <<_:O/binary, "<!--", _/binary>> ->
325 + tokenize_comment(B, ?ADV_COL(S, 4));
326 + <<_:O/binary, "<!doctype", _/binary>> ->
327 + tokenize_doctype(B, ?ADV_COL(S, 10));
328 + <<_:O/binary, "<!DOCTYPE", _/binary>> ->
329 + tokenize_doctype(B, ?ADV_COL(S, 10));
330 + <<_:O/binary, "<![CDATA[", _/binary>> ->
331 + tokenize_cdata(B, ?ADV_COL(S, 9));
332 + <<_:O/binary, "<?php", _/binary>> ->
333 + {Body, S1} = raw_qgt(B, ?ADV_COL(S, 2)),
334 + {{pi, Body}, S1};
335 + <<_:O/binary, "<?", _/binary>> ->
336 + {Tag, S1} = tokenize_literal(B, ?ADV_COL(S, 2)),
337 + {Attrs, S2} = tokenize_attributes(B, S1),
338 + S3 = find_qgt(B, S2),
339 + {{pi, Tag, Attrs}, S3};
340 + <<_:O/binary, "&", _/binary>> ->
341 + tokenize_charref(B, ?INC_COL(S));
342 + <<_:O/binary, "</", _/binary>> ->
343 + {Tag, S1} = tokenize_literal(B, ?ADV_COL(S, 2)),
344 + {S2, _} = find_gt(B, S1),
345 + {{end_tag, Tag}, S2};
346 + <<_:O/binary, "<", C, _/binary>>
347 + when ?IS_WHITESPACE(C); not ?IS_LITERAL_SAFE(C) ->
348 + %% This isn't really strict HTML
349 + {{data, Data, _Whitespace}, S1} = tokenize_data(B, ?INC_COL(S)),
350 + {{data, <<$<, Data/binary>>, false}, S1};
351 + <<_:O/binary, "<", _/binary>> ->
352 + {Tag, S1} = tokenize_literal(B, ?INC_COL(S)),
353 + {Attrs, S2} = tokenize_attributes(B, S1),
354 + {S3, HasSlash} = find_gt(B, S2),
355 + Singleton = HasSlash orelse is_singleton(Tag),
356 + {{start_tag, Tag, Attrs, Singleton}, S3};
357 + _ ->
358 + tokenize_data(B, S)
359 + end.
360 +
361 + tree_data([{data, Data, Whitespace} | Rest], AllWhitespace, Acc) ->
362 + tree_data(Rest, (Whitespace andalso AllWhitespace), [Data | Acc]);
363 + tree_data(Rest, AllWhitespace, Acc) ->
364 + {iolist_to_binary(lists:reverse(Acc)), AllWhitespace, Rest}.
365 +
366 + tree([], Stack) ->
367 + {destack(Stack), []};
368 + tree([{end_tag, Tag} | Rest], Stack) ->
369 + case destack(norm(Tag), Stack) of
370 + S when is_list(S) ->
371 + tree(Rest, S);
372 + Result ->
373 + {Result, []}
374 + end;
375 + tree([{start_tag, Tag, Attrs, true} | Rest], S) ->
376 + tree(Rest, append_stack_child(norm({Tag, Attrs}), S));
377 + tree([{start_tag, Tag, Attrs, false} | Rest], S) ->
378 + tree(Rest, stack(norm({Tag, Attrs}), S));
379 + tree([T={pi, _Raw} | Rest], S) ->
380 + tree(Rest, append_stack_child(T, S));
381 + tree([T={pi, _Tag, _Attrs} | Rest], S) ->
382 + tree(Rest, append_stack_child(T, S));
383 + tree([T={comment, _Comment} | Rest], S) ->
384 + tree(Rest, append_stack_child(T, S));
385 + tree(L=[{data, _Data, _Whitespace} | _], S) ->
386 + case tree_data(L, true, []) of
387 + {_, true, Rest} ->
388 + tree(Rest, S);
389 + {Data, false, Rest} ->
390 + tree(Rest, append_stack_child(Data, S))
391 + end;
392 + tree([{doctype, _} | Rest], Stack) ->
393 + tree(Rest, Stack).
394 +
395 + norm({Tag, Attrs}) ->
396 + {norm(Tag), [{norm(K), iolist_to_binary(V)} || {K, V} <- Attrs], []};
397 + norm(Tag) when is_binary(Tag) ->
398 + Tag;
399 + norm(Tag) ->
400 + list_to_binary(string:to_lower(Tag)).
401 +
402 + stack(T1={TN, _, _}, Stack=[{TN, _, _} | _Rest])
403 + when TN =:= <<"li">> orelse TN =:= <<"option">> ->
404 + [T1 | destack(TN, Stack)];
405 + stack(T1={TN0, _, _}, Stack=[{TN1, _, _} | _Rest])
406 + when (TN0 =:= <<"dd">> orelse TN0 =:= <<"dt">>) andalso
407 + (TN1 =:= <<"dd">> orelse TN1 =:= <<"dt">>) ->
408 + [T1 | destack(TN1, Stack)];
409 + stack(T1, Stack) ->
410 + [T1 | Stack].
411 +
412 + append_stack_child(StartTag, [{Name, Attrs, Acc} | Stack]) ->
413 + [{Name, Attrs, [StartTag | Acc]} | Stack].
414 +
415 + destack(<<"br">>, Stack) ->
416 + %% This is an ugly hack to make dumb_br_test() pass,
417 + %% this makes it such that br can never have children.
418 + Stack;
419 + destack(TagName, Stack) when is_list(Stack) ->
420 + F = fun (X) ->
421 + case X of
422 + {TagName, _, _} ->
423 + false;
424 + _ ->
425 + true
426 + end
427 + end,
428 + case lists:splitwith(F, Stack) of
429 + {_, []} ->
430 + %% If we're parsing something like XML we might find
431 + %% a <link>tag</link> that is normally a singleton
432 + %% in HTML but isn't here
433 + case {is_singleton(TagName), Stack} of
434 + {true, [{T0, A0, Acc0} | Post0]} ->
435 + case lists:splitwith(F, Acc0) of
436 + {_, []} ->
437 + %% Actually was a singleton
438 + Stack;
439 + {Pre, [{T1, A1, Acc1} | Post1]} ->
440 + [{T0, A0, [{T1, A1, Acc1 ++ lists:reverse(Pre)} | Post1]}
441 + | Post0]
442 + end;
443 + _ ->
444 + %% No match, no state change
445 + Stack
446 + end;
447 + {_Pre, [_T]} ->
448 + %% Unfurl the whole stack, we're done
449 + destack(Stack);
450 + {Pre, [T, {T0, A0, Acc0} | Post]} ->
451 + %% Unfurl up to the tag, then accumulate it
452 + [{T0, A0, [destack(Pre ++ [T]) | Acc0]} | Post]
453 + end.
454 +
455 + destack([{Tag, Attrs, Acc}]) ->
456 + {Tag, Attrs, lists:reverse(Acc)};
457 + destack([{T1, A1, Acc1}, {T0, A0, Acc0} | Rest]) ->
458 + destack([{T0, A0, [{T1, A1, lists:reverse(Acc1)} | Acc0]} | Rest]).
459 +
460 + is_singleton(<<"br">>) -> true;
461 + is_singleton(<<"hr">>) -> true;
462 + is_singleton(<<"img">>) -> true;
463 + is_singleton(<<"input">>) -> true;
464 + is_singleton(<<"base">>) -> true;
465 + is_singleton(<<"meta">>) -> true;
466 + is_singleton(<<"link">>) -> true;
467 + is_singleton(<<"area">>) -> true;
468 + is_singleton(<<"param">>) -> true;
469 + is_singleton(<<"col">>) -> true;
470 + is_singleton(_) -> false.
471 +
472 + tokenize_data(B, S=#decoder{offset=O}) ->
473 + tokenize_data(B, S, O, true).
474 +
475 + tokenize_data(B, S=#decoder{offset=O}, Start, Whitespace) ->
476 + case B of
477 + <<_:O/binary, C, _/binary>> when (C =/= $< andalso C =/= $&) ->
478 + tokenize_data(B, ?INC_CHAR(S, C), Start,
479 + (Whitespace andalso ?IS_WHITESPACE(C)));
480 + _ ->
481 + Len = O - Start,
482 + <<_:Start/binary, Data:Len/binary, _/binary>> = B,
483 + {{data, Data, Whitespace}, S}
484 + end.
485 +
486 + tokenize_attributes(B, S) ->
487 + tokenize_attributes(B, S, []).
488 +
489 + tokenize_attributes(B, S=#decoder{offset=O}, Acc) ->
490 + case B of
491 + <<_:O/binary>> ->
492 + {lists:reverse(Acc), S};
493 + <<_:O/binary, C, _/binary>> when (C =:= $> orelse C =:= $/) ->
494 + {lists:reverse(Acc), S};
495 + <<_:O/binary, "?>", _/binary>> ->
496 + {lists:reverse(Acc), S};
497 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) ->
498 + tokenize_attributes(B, ?INC_CHAR(S, C), Acc);
499 + _ ->
500 + {Attr, S1} = tokenize_literal(B, S),
501 + {Value, S2} = tokenize_attr_value(Attr, B, S1),
502 + tokenize_attributes(B, S2, [{Attr, Value} | Acc])
503 + end.
504 +
505 + tokenize_attr_value(Attr, B, S) ->
506 + S1 = skip_whitespace(B, S),
507 + O = S1#decoder.offset,
508 + case B of
509 + <<_:O/binary, "=", _/binary>> ->
510 + S2 = skip_whitespace(B, ?INC_COL(S1)),
511 + tokenize_quoted_or_unquoted_attr_value(B, S2);
512 + _ ->
513 + {Attr, S1}
514 + end.
515 +
516 + tokenize_quoted_or_unquoted_attr_value(B, S=#decoder{offset=O}) ->
517 + case B of
518 + <<_:O/binary>> ->
519 + { [], S };
520 + <<_:O/binary, Q, _/binary>> when Q =:= ?QUOTE orelse
521 + Q =:= ?SQUOTE ->
522 + tokenize_quoted_attr_value(B, ?INC_COL(S), [], Q);
523 + <<_:O/binary, _/binary>> ->
524 + tokenize_unquoted_attr_value(B, S, [])
525 + end.
526 +
527 + tokenize_quoted_attr_value(B, S=#decoder{offset=O}, Acc, Q) ->
528 + case B of
529 + <<_:O/binary>> ->
530 + { iolist_to_binary(lists:reverse(Acc)), S };
531 + <<_:O/binary, $&, _/binary>> ->
532 + {{data, Data, false}, S1} = tokenize_charref(B, ?INC_COL(S)),
533 + tokenize_quoted_attr_value(B, S1, [Data|Acc], Q);
534 + <<_:O/binary, Q, _/binary>> ->
535 + { iolist_to_binary(lists:reverse(Acc)), ?INC_COL(S) };
536 + <<_:O/binary, C, _/binary>> ->
537 + tokenize_quoted_attr_value(B, ?INC_COL(S), [C|Acc], Q)
538 + end.
539 +
540 + tokenize_unquoted_attr_value(B, S=#decoder{offset=O}, Acc) ->
541 + case B of
542 + <<_:O/binary>> ->
543 + { iolist_to_binary(lists:reverse(Acc)), S };
544 + <<_:O/binary, $&, _/binary>> ->
545 + {{data, Data, false}, S1} = tokenize_charref(B, ?INC_COL(S)),
546 + tokenize_unquoted_attr_value(B, S1, [Data|Acc]);
547 + <<_:O/binary, $/, $>, _/binary>> ->
548 + { iolist_to_binary(lists:reverse(Acc)), S };
549 + <<_:O/binary, C, _/binary>> when ?PROBABLE_CLOSE(C) ->
550 + { iolist_to_binary(lists:reverse(Acc)), S };
551 + <<_:O/binary, C, _/binary>> ->
552 + tokenize_unquoted_attr_value(B, ?INC_COL(S), [C|Acc])
553 + end.
554 +
555 + skip_whitespace(B, S=#decoder{offset=O}) ->
556 + case B of
557 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) ->
558 + skip_whitespace(B, ?INC_CHAR(S, C));
559 + _ ->
560 + S
561 + end.
562 +
563 + tokenize_literal(Bin, S=#decoder{offset=O}) ->
564 + case Bin of
565 + <<_:O/binary, C, _/binary>> when C =:= $>
566 + orelse C =:= $/
567 + orelse C =:= $= ->
568 + %% Handle case where tokenize_literal would consume
569 + %% 0 chars. http://github.com/mochi/mochiweb/pull/13
570 + {[C], ?INC_COL(S)};
571 + _ ->
572 + tokenize_literal(Bin, S, [])
573 + end.
574 +
575 + tokenize_literal(Bin, S=#decoder{offset=O}, Acc) ->
576 + case Bin of
577 + <<_:O/binary, $&, _/binary>> ->
578 + {{data, Data, false}, S1} = tokenize_charref(Bin, ?INC_COL(S)),
579 + tokenize_literal(Bin, S1, [Data | Acc]);
580 + <<_:O/binary, C, _/binary>> when not (?IS_WHITESPACE(C)
581 + orelse C =:= $>
582 + orelse C =:= $/
583 + orelse C =:= $=) ->
584 + tokenize_literal(Bin, ?INC_COL(S), [C | Acc]);
585 + _ ->
586 + {iolist_to_binary(string:to_lower(lists:reverse(Acc))), S}
587 + end.
588 +
589 + raw_qgt(Bin, S=#decoder{offset=O}) ->
590 + raw_qgt(Bin, S, O).
591 +
592 + raw_qgt(Bin, S=#decoder{offset=O}, Start) ->
593 + case Bin of
594 + <<_:O/binary, "?>", _/binary>> ->
595 + Len = O - Start,
596 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
597 + {Raw, ?ADV_COL(S, 2)};
598 + <<_:O/binary, C, _/binary>> ->
599 + raw_qgt(Bin, ?INC_CHAR(S, C), Start);
600 + <<_:O/binary>> ->
601 + <<_:Start/binary, Raw/binary>> = Bin,
602 + {Raw, S}
603 + end.
604 +
605 + find_qgt(Bin, S=#decoder{offset=O}) ->
606 + case Bin of
607 + <<_:O/binary, "?>", _/binary>> ->
608 + ?ADV_COL(S, 2);
609 + <<_:O/binary, ">", _/binary>> ->
610 + ?ADV_COL(S, 1);
611 + <<_:O/binary, "/>", _/binary>> ->
612 + ?ADV_COL(S, 2);
613 + %% tokenize_attributes takes care of this state:
614 + %% <<_:O/binary, C, _/binary>> ->
615 + %% find_qgt(Bin, ?INC_CHAR(S, C));
616 + <<_:O/binary>> ->
617 + S
618 + end.
619 +
620 + find_gt(Bin, S) ->
621 + find_gt(Bin, S, false).
622 +
623 + find_gt(Bin, S=#decoder{offset=O}, HasSlash) ->
624 + case Bin of
625 + <<_:O/binary, $/, _/binary>> ->
626 + find_gt(Bin, ?INC_COL(S), true);
627 + <<_:O/binary, $>, _/binary>> ->
628 + {?INC_COL(S), HasSlash};
629 + <<_:O/binary, C, _/binary>> ->
630 + find_gt(Bin, ?INC_CHAR(S, C), HasSlash);
631 + _ ->
632 + {S, HasSlash}
633 + end.
634 +
635 + tokenize_charref(Bin, S=#decoder{offset=O}) ->
636 + try
637 + tokenize_charref(Bin, S, O)
638 + catch
639 + throw:invalid_charref ->
640 + {{data, <<"&">>, false}, S}
641 + end.
642 +
643 + tokenize_charref(Bin, S=#decoder{offset=O}, Start) ->
644 + case Bin of
645 + <<_:O/binary>> ->
646 + throw(invalid_charref);
647 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C)
648 + orelse C =:= ?SQUOTE
649 + orelse C =:= ?QUOTE
650 + orelse C =:= $/
651 + orelse C =:= $> ->
652 + throw(invalid_charref);
653 + <<_:O/binary, $;, _/binary>> ->
654 + Len = O - Start,
655 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
656 + Data = case mochiweb_charref:charref(Raw) of
657 + undefined ->
658 + throw(invalid_charref);
659 + Unichar when is_integer(Unichar) ->
660 + mochiutf8:codepoint_to_bytes(Unichar);
661 + Unichars when is_list(Unichars) ->
662 + unicode:characters_to_binary(Unichars)
663 + end,
664 + {{data, Data, false}, ?INC_COL(S)};
665 + _ ->
666 + tokenize_charref(Bin, ?INC_COL(S), Start)
667 + end.
668 +
669 + tokenize_doctype(Bin, S) ->
670 + tokenize_doctype(Bin, S, []).
671 +
672 + tokenize_doctype(Bin, S=#decoder{offset=O}, Acc) ->
673 + case Bin of
674 + <<_:O/binary>> ->
675 + {{doctype, lists:reverse(Acc)}, S};
676 + <<_:O/binary, $>, _/binary>> ->
677 + {{doctype, lists:reverse(Acc)}, ?INC_COL(S)};
678 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) ->
679 + tokenize_doctype(Bin, ?INC_CHAR(S, C), Acc);
680 + _ ->
681 + {Word, S1} = tokenize_word_or_literal(Bin, S),
682 + tokenize_doctype(Bin, S1, [Word | Acc])
683 + end.
684 +
685 + tokenize_word_or_literal(Bin, S=#decoder{offset=O}) ->
686 + case Bin of
687 + <<_:O/binary, C, _/binary>> when C =:= ?QUOTE orelse C =:= ?SQUOTE ->
688 + tokenize_word(Bin, ?INC_COL(S), C);
689 + <<_:O/binary, C, _/binary>> when not ?IS_WHITESPACE(C) ->
690 + %% Sanity check for whitespace
691 + tokenize_literal(Bin, S)
692 + end.
693 +
694 + tokenize_word(Bin, S, Quote) ->
695 + tokenize_word(Bin, S, Quote, []).
696 +
697 + tokenize_word(Bin, S=#decoder{offset=O}, Quote, Acc) ->
698 + case Bin of
699 + <<_:O/binary>> ->
700 + {iolist_to_binary(lists:reverse(Acc)), S};
701 + <<_:O/binary, Quote, _/binary>> ->
702 + {iolist_to_binary(lists:reverse(Acc)), ?INC_COL(S)};
703 + <<_:O/binary, $&, _/binary>> ->
704 + {{data, Data, false}, S1} = tokenize_charref(Bin, ?INC_COL(S)),
705 + tokenize_word(Bin, S1, Quote, [Data | Acc]);
706 + <<_:O/binary, C, _/binary>> ->
707 + tokenize_word(Bin, ?INC_CHAR(S, C), Quote, [C | Acc])
708 + end.
709 +
710 + tokenize_cdata(Bin, S=#decoder{offset=O}) ->
711 + tokenize_cdata(Bin, S, O).
712 +
713 + tokenize_cdata(Bin, S=#decoder{offset=O}, Start) ->
714 + case Bin of
715 + <<_:O/binary, "]]>", _/binary>> ->
716 + Len = O - Start,
717 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
718 + {{data, Raw, false}, ?ADV_COL(S, 3)};
719 + <<_:O/binary, C, _/binary>> ->
720 + tokenize_cdata(Bin, ?INC_CHAR(S, C), Start);
721 + _ ->
722 + <<_:O/binary, Raw/binary>> = Bin,
723 + {{data, Raw, false}, S}
724 + end.
725 +
726 + tokenize_comment(Bin, S=#decoder{offset=O}) ->
727 + tokenize_comment(Bin, S, O).
728 +
729 + tokenize_comment(Bin, S=#decoder{offset=O}, Start) ->
730 + case Bin of
731 + <<_:O/binary, "-->", _/binary>> ->
732 + Len = O - Start,
733 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
734 + {{comment, Raw}, ?ADV_COL(S, 3)};
735 + <<_:O/binary, C, _/binary>> ->
736 + tokenize_comment(Bin, ?INC_CHAR(S, C), Start);
737 + <<_:Start/binary, Raw/binary>> ->
738 + {{comment, Raw}, S}
739 + end.
740 +
741 + tokenize_script(Bin, S=#decoder{offset=O}) ->
742 + tokenize_script(Bin, S, O).
743 +
744 + tokenize_script(Bin, S=#decoder{offset=O}, Start) ->
745 + case Bin of
746 + %% Just a look-ahead, we want the end_tag separately
747 + <<_:O/binary, $<, $/, SS, CC, RR, II, PP, TT, ZZ, _/binary>>
748 + when (SS =:= $s orelse SS =:= $S) andalso
749 + (CC =:= $c orelse CC =:= $C) andalso
750 + (RR =:= $r orelse RR =:= $R) andalso
751 + (II =:= $i orelse II =:= $I) andalso
752 + (PP =:= $p orelse PP =:= $P) andalso
753 + (TT=:= $t orelse TT =:= $T) andalso
754 + ?PROBABLE_CLOSE(ZZ) ->
755 + Len = O - Start,
756 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
757 + {{data, Raw, false}, S};
758 + <<_:O/binary, C, _/binary>> ->
759 + tokenize_script(Bin, ?INC_CHAR(S, C), Start);
760 + <<_:Start/binary, Raw/binary>> ->
761 + {{data, Raw, false}, S}
762 + end.
763 +
764 + tokenize_textarea(Bin, S=#decoder{offset=O}) ->
765 + tokenize_textarea(Bin, S, O).
766 +
767 + tokenize_textarea(Bin, S=#decoder{offset=O}, Start) ->
768 + case Bin of
769 + %% Just a look-ahead, we want the end_tag separately
770 + <<_:O/binary, $<, $/, TT, EE, XX, TT2, AA, RR, EE2, AA2, ZZ, _/binary>>
771 + when (TT =:= $t orelse TT =:= $T) andalso
772 + (EE =:= $e orelse EE =:= $E) andalso
773 + (XX =:= $x orelse XX =:= $X) andalso
774 + (TT2 =:= $t orelse TT2 =:= $T) andalso
775 + (AA =:= $a orelse AA =:= $A) andalso
776 + (RR =:= $r orelse RR =:= $R) andalso
777 + (EE2 =:= $e orelse EE2 =:= $E) andalso
778 + (AA2 =:= $a orelse AA2 =:= $A) andalso
779 + ?PROBABLE_CLOSE(ZZ) ->
780 + Len = O - Start,
781 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
782 + {{data, Raw, false}, S};
783 + <<_:O/binary, C, _/binary>> ->
784 + tokenize_textarea(Bin, ?INC_CHAR(S, C), Start);
785 + <<_:Start/binary, Raw/binary>> ->
786 + {{data, Raw, false}, S}
787 + end.