Packages

floki

0.23.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
1 Dependency
Dependants
Files
Activity
Hide whitespace

Compare versions

5 files changed
+842 additions
-17 deletions
  @@ -70,7 +70,7 @@ Add Floki to your `mix.exs`:
70 70 ```elixir
71 71 defp deps do
72 72 [
73 - {:floki, "~> 0.22.0"}
73 + {:floki, "~> 0.23.0"}
74 74 ]
75 75 end
76 76 ```
  @@ -105,7 +105,7 @@ After setup Rust, you need to add `html5ever` NIF to your dependency list:
105 105 ```elixir
106 106 defp deps do
107 107 [
108 - {:floki, "~> 0.22.0"},
108 + {:floki, "~> 0.23.0"},
109 109 {:html5ever, "~> 0.7.0"}
110 110 ]
111 111 end
  @@ -2,7 +2,7 @@
2 2 {<<"build_tools">>,[<<"mix">>]}.
3 3 {<<"description">>,
4 4 <<"Floki is a simple HTML parser that enables search for nodes using CSS selectors.">>}.
5 - {<<"elixir">>,<<">= 1.5.0">>}.
5 + {<<"elixir">>,<<"~> 1.5">>}.
6 6 {<<"files">>,
7 7 [<<"lib">>,<<"lib/floki">>,<<"lib/floki/html_parser.ex">>,
8 8 <<"lib/floki/html_parser">>,<<"lib/floki/html_parser/mochiweb.ex">>,
  @@ -18,20 +18,16 @@
18 18 <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/comment.ex">>,
19 19 <<"lib/floki/html_tree/text.ex">>,<<"lib/floki/html_tree/html_node.ex">>,
20 20 <<"lib/floki/selector.ex">>,<<"lib/floki/html_tree.ex">>,<<"lib/floki.ex">>,
21 - <<"src/floki_selector_lexer.xrl">>,<<"mix.exs">>,<<"README.md">>,
22 - <<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>,<<"CONTRIBUTING.md">>]}.
21 + <<"src/floki_selector_lexer.xrl">>,<<"src/floki_mochi_html.erl">>,
22 + <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,<<"CODE_OF_CONDUCT.md">>,
23 + <<"CONTRIBUTING.md">>]}.
23 24 {<<"licenses">>,[<<"MIT">>]}.
24 25 {<<"links">>,[{<<"GitHub">>,<<"https://github.com/philss/floki">>}]}.
25 26 {<<"name">>,<<"floki">>}.
26 27 {<<"requirements">>,
27 - [[{<<"app">>,<<"mochiweb">>},
28 - {<<"name">>,<<"mochiweb">>},
29 - {<<"optional">>,false},
30 - {<<"repository">>,<<"hexpm">>},
31 - {<<"requirement">>,<<"~> 2.15">>}],
32 - [{<<"app">>,<<"html_entities">>},
28 + [[{<<"app">>,<<"html_entities">>},
33 29 {<<"name">>,<<"html_entities">>},
34 30 {<<"optional">>,false},
35 31 {<<"repository">>,<<"hexpm">>},
36 32 {<<"requirement">>,<<"~> 0.4.0">>}]]}.
37 - {<<"version">>,<<"0.22.0">>}.
33 + {<<"version">>,<<"0.23.0">>}.
  @@ -4,7 +4,7 @@ defmodule Floki.HTMLParser.Mochiweb do
4 4
5 5 def parse(html) do
6 6 html = "<#{@root_node}>#{html}</#{@root_node}>"
7 - {@root_node, [], parsed} = :mochiweb_html.parse(html)
7 + {@root_node, [], parsed} = :floki_mochi_html.parse(html)
8 8
9 9 if length(parsed) == 1, do: hd(parsed), else: parsed
10 10 end
  @@ -2,7 +2,7 @@ defmodule Floki.Mixfile do
2 2 use Mix.Project
3 3
4 4 @description "Floki is a simple HTML parser that enables search for nodes using CSS selectors."
5 - @version "0.22.0"
5 + @version "0.23.0"
6 6
7 7 def project do
8 8 [
  @@ -10,7 +10,7 @@ defmodule Floki.Mixfile do
10 10 name: "Floki",
11 11 version: @version,
12 12 description: @description,
13 - elixir: ">= 1.5.0",
13 + elixir: "~> 1.5",
14 14 package: package(),
15 15 deps: deps(),
16 16 source_url: "https://github.com/philss/floki",
  @@ -19,12 +19,11 @@ defmodule Floki.Mixfile do
19 19 end
20 20
21 21 def application do
22 - [applications: [:logger, :mochiweb, :html_entities]]
22 + [extra_applications: [:logger]]
23 23 end
24 24
25 25 defp deps do
26 26 [
27 - {:mochiweb, "~> 2.15"},
28 27 {:html_entities, "~> 0.4.0"},
29 28 {:earmark, "~> 1.2", only: :dev},
30 29 {:ex_doc, "~> 0.18", only: :dev},
  @@ -40,6 +39,7 @@ defmodule Floki.Mixfile do
40 39 files: [
41 40 "lib",
42 41 "src/*.xrl",
42 + "src/floki_mochi_html.erl",
43 43 "mix.exs",
44 44 "README.md",
45 45 "LICENSE",
  @@ -0,0 +1,829 @@
1 + %% @author Bob Ippolito <bob@mochimedia.com>
2 + %% @copyright 2007 Mochi Media, Inc.
3 + %%
4 + %% Permission is hereby granted, free of charge, to any person obtaining a
5 + %% copy of this software and associated documentation files (the "Software"),
6 + %% to deal in the Software without restriction, including without limitation
7 + %% the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 + %% and/or sell copies of the Software, and to permit persons to whom the
9 + %% Software is furnished to do so, subject to the following conditions:
10 + %%
11 + %% The above copyright notice and this permission notice shall be included in
12 + %% all copies or substantial portions of the Software.
13 + %%
14 + %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 + %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 + %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 + %% THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 + %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 + %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 + %% DEALINGS IN THE SOFTWARE.
21 +
22 + %% @doc Loosely tokenizes and generates parse trees for HTML 4.
23 + -module(floki_mochi_html).
24 + -export([tokens/1, parse/1, parse_tokens/1, to_tokens/1, escape/1,
25 + escape_attr/1, to_html/1]).
26 + -ifdef(TEST).
27 + -export([destack/1, destack/2, is_singleton/1]).
28 + -endif.
29 +
30 + %% This is a macro to placate syntax highlighters..
31 + -define(QUOTE, $\"). %% $\"
32 + -define(SQUOTE, $\'). %% $\'
33 + -define(ADV_COL(S, N),
34 + S#decoder{column=N+S#decoder.column,
35 + offset=N+S#decoder.offset}).
36 + -define(INC_COL(S),
37 + S#decoder{column=1+S#decoder.column,
38 + offset=1+S#decoder.offset}).
39 + -define(INC_LINE(S),
40 + S#decoder{column=1,
41 + line=1+S#decoder.line,
42 + offset=1+S#decoder.offset}).
43 + -define(INC_CHAR(S, C),
44 + case C of
45 + $\n ->
46 + S#decoder{column=1,
47 + line=1+S#decoder.line,
48 + offset=1+S#decoder.offset};
49 + _ ->
50 + S#decoder{column=1+S#decoder.column,
51 + offset=1+S#decoder.offset}
52 + end).
53 +
54 + -define(IS_WHITESPACE(C),
55 + (C =:= $\s orelse C =:= $\t orelse C =:= $\r orelse C =:= $\n)).
56 + -define(IS_LETTER(C),
57 + ((C >= $A andalso C =< $Z) orelse (C >= $a andalso C =< $z))).
58 + -define(IS_LITERAL_SAFE(C),
59 + ((C >= $A andalso C =< $Z) orelse (C >= $a andalso C =< $z)
60 + orelse (C >= $0 andalso C =< $9))).
61 + -define(PROBABLE_CLOSE(C),
62 + (C =:= $> orelse ?IS_WHITESPACE(C))).
63 +
64 + -record(decoder, {line=1,
65 + column=1,
66 + offset=0}).
67 +
68 + %% @type html_node() = {string(), [html_attr()], [html_node() | string()]}
69 + %% @type html_attr() = {string(), string()}
70 + %% @type html_token() = html_data() | start_tag() | end_tag() | inline_html() | html_comment() | html_doctype()
71 + %% @type html_data() = {data, string(), Whitespace::boolean()}
72 + %% @type start_tag() = {start_tag, Name, [html_attr()], Singleton::boolean()}
73 + %% @type end_tag() = {end_tag, Name}
74 + %% @type html_comment() = {comment, Comment}
75 + %% @type html_doctype() = {doctype, [Doctype]}
76 + %% @type inline_html() = {'=', iolist()}
77 +
78 + %% External API.
79 +
80 + %% @spec parse(string() | binary()) -> html_node()
81 + %% @doc tokenize and then transform the token stream into a HTML tree.
82 + parse(Input) ->
83 + parse_tokens(tokens(Input)).
84 +
85 + %% @spec parse_tokens([html_token()]) -> html_node()
86 + %% @doc Transform the output of tokens(Doc) into a HTML tree.
87 + parse_tokens(Tokens) when is_list(Tokens) ->
88 + %% Skip over doctype, processing instructions
89 + [{start_tag, Tag, Attrs, false} | Rest] = find_document(Tokens, normal),
90 + {Tree, _} = tree(Rest, [norm({Tag, Attrs})]),
91 + Tree.
92 +
93 + find_document(Tokens=[{start_tag, _Tag, _Attrs, false} | _Rest], Mode) ->
94 + maybe_add_html_tag(Tokens, Mode);
95 + find_document([{doctype, [<<"html">>]} | Rest], _Mode) ->
96 + find_document(Rest, html5);
97 + find_document([_T | Rest], Mode) ->
98 + find_document(Rest, Mode);
99 + find_document([], _Mode) ->
100 + [].
101 +
102 + maybe_add_html_tag(Tokens=[{start_tag, Tag, _Attrs, false} | _], html5)
103 + when Tag =/= <<"html">> ->
104 + [{start_tag, <<"html">>, [], false} | Tokens];
105 + maybe_add_html_tag(Tokens, _Mode) ->
106 + Tokens.
107 +
108 + %% @spec tokens(StringOrBinary) -> [html_token()]
109 + %% @doc Transform the input UTF-8 HTML into a token stream.
110 + tokens(Input) ->
111 + tokens(iolist_to_binary(Input), #decoder{}, []).
112 +
113 + %% @spec to_tokens(html_node()) -> [html_token()]
114 + %% @doc Convert a html_node() tree to a list of tokens.
115 + to_tokens({Tag0}) ->
116 + to_tokens({Tag0, [], []});
117 + to_tokens(T={'=', _}) ->
118 + [T];
119 + to_tokens(T={doctype, _}) ->
120 + [T];
121 + to_tokens(T={comment, _}) ->
122 + [T];
123 + to_tokens({Tag0, Acc}) ->
124 + %% This is only allowed in sub-tags: {p, [{"class", "foo"}]}
125 + to_tokens({Tag0, [], Acc});
126 + to_tokens({Tag0, Attrs, Acc}) ->
127 + Tag = to_tag(Tag0),
128 + case is_singleton(Tag) of
129 + true ->
130 + to_tokens([], [{start_tag, Tag, Attrs, true}]);
131 + false ->
132 + to_tokens([{Tag, Acc}], [{start_tag, Tag, Attrs, false}])
133 + end.
134 +
135 + %% @spec to_html([html_token()] | html_node()) -> iolist()
136 + %% @doc Convert a list of html_token() to a HTML document.
137 + to_html(Node) when is_tuple(Node) ->
138 + to_html(to_tokens(Node));
139 + to_html(Tokens) when is_list(Tokens) ->
140 + to_html(Tokens, []).
141 +
142 + %% @spec escape(string() | atom() | binary()) -> binary()
143 + %% @doc Escape a string such that it's safe for HTML (amp; lt; gt;).
144 + escape(B) when is_binary(B) ->
145 + escape(binary_to_list(B), []);
146 + escape(A) when is_atom(A) ->
147 + escape(atom_to_list(A), []);
148 + escape(S) when is_list(S) ->
149 + escape(S, []).
150 +
151 + %% @spec escape_attr(string() | binary() | atom() | integer() | float()) -> binary()
152 + %% @doc Escape a string such that it's safe for HTML attrs
153 + %% (amp; lt; gt; quot;).
154 + escape_attr(B) when is_binary(B) ->
155 + escape_attr(binary_to_list(B), []);
156 + escape_attr(A) when is_atom(A) ->
157 + escape_attr(atom_to_list(A), []);
158 + escape_attr(S) when is_list(S) ->
159 + escape_attr(S, []);
160 + escape_attr(I) when is_integer(I) ->
161 + escape_attr(integer_to_list(I), []);
162 + %% CHANGED: Previously it used mochinum:digits/1 but float_to_list/1 is
163 + %% good enough generally. In case it isn't, a similar alternative would
164 + %% be lists:flatten(io_lib:format("~p", [F])).
165 + escape_attr(F) when is_float(F) ->
166 + escape_attr(erlang:float_to_list(F), []).
167 +
168 + to_html([], Acc) ->
169 + lists:reverse(Acc);
170 + to_html([{'=', Content} | Rest], Acc) ->
171 + to_html(Rest, [Content | Acc]);
172 + to_html([{pi, Bin} | Rest], Acc) ->
173 + Open = [<<"<?">>,
174 + Bin,
175 + <<"?>">>],
176 + to_html(Rest, [Open | Acc]);
177 + to_html([{pi, Tag, Attrs} | Rest], Acc) ->
178 + Open = [<<"<?">>,
179 + Tag,
180 + attrs_to_html(Attrs, []),
181 + <<"?>">>],
182 + to_html(Rest, [Open | Acc]);
183 + to_html([{comment, Comment} | Rest], Acc) ->
184 + to_html(Rest, [[<<"<!--">>, Comment, <<"-->">>] | Acc]);
185 + to_html([{doctype, Parts} | Rest], Acc) ->
186 + Inside = doctype_to_html(Parts, Acc),
187 + to_html(Rest, [[<<"<!DOCTYPE">>, Inside, <<">">>] | Acc]);
188 + to_html([{data, Data, _Whitespace} | Rest], Acc) ->
189 + to_html(Rest, [escape(Data) | Acc]);
190 + to_html([{start_tag, Tag, Attrs, Singleton} | Rest], Acc) ->
191 + Open = [<<"<">>,
192 + Tag,
193 + attrs_to_html(Attrs, []),
194 + case Singleton of
195 + true -> <<" />">>;
196 + false -> <<">">>
197 + end],
198 + to_html(Rest, [Open | Acc]);
199 + to_html([{end_tag, Tag} | Rest], Acc) ->
200 + to_html(Rest, [[<<"</">>, Tag, <<">">>] | Acc]).
201 +
202 + doctype_to_html([], Acc) ->
203 + lists:reverse(Acc);
204 + doctype_to_html([Word | Rest], Acc) ->
205 + case lists:all(fun (C) -> ?IS_LITERAL_SAFE(C) end,
206 + binary_to_list(iolist_to_binary(Word))) of
207 + true ->
208 + doctype_to_html(Rest, [[<<" ">>, Word] | Acc]);
209 + false ->
210 + doctype_to_html(Rest, [[<<" \"">>, escape_attr(Word), ?QUOTE] | Acc])
211 + end.
212 +
213 + attrs_to_html([], Acc) ->
214 + lists:reverse(Acc);
215 + attrs_to_html([{K, V} | Rest], Acc) ->
216 + attrs_to_html(Rest,
217 + [[<<" ">>, escape(K), <<"=\"">>,
218 + escape_attr(V), <<"\"">>] | Acc]).
219 +
220 + escape([], Acc) ->
221 + list_to_binary(lists:reverse(Acc));
222 + escape("<" ++ Rest, Acc) ->
223 + escape(Rest, lists:reverse("&lt;", Acc));
224 + escape(">" ++ Rest, Acc) ->
225 + escape(Rest, lists:reverse("&gt;", Acc));
226 + escape("&" ++ Rest, Acc) ->
227 + escape(Rest, lists:reverse("&amp;", Acc));
228 + escape([C | Rest], Acc) ->
229 + escape(Rest, [C | Acc]).
230 +
231 + escape_attr([], Acc) ->
232 + list_to_binary(lists:reverse(Acc));
233 + escape_attr("<" ++ Rest, Acc) ->
234 + escape_attr(Rest, lists:reverse("&lt;", Acc));
235 + escape_attr(">" ++ Rest, Acc) ->
236 + escape_attr(Rest, lists:reverse("&gt;", Acc));
237 + escape_attr("&" ++ Rest, Acc) ->
238 + escape_attr(Rest, lists:reverse("&amp;", Acc));
239 + escape_attr([?QUOTE | Rest], Acc) ->
240 + escape_attr(Rest, lists:reverse("&quot;", Acc));
241 + escape_attr([C | Rest], Acc) ->
242 + escape_attr(Rest, [C | Acc]).
243 +
244 + to_tag(A) when is_atom(A) ->
245 + norm(atom_to_list(A));
246 + to_tag(L) ->
247 + norm(L).
248 +
249 + to_tokens([], Acc) ->
250 + lists:reverse(Acc);
251 + to_tokens([{Tag, []} | Rest], Acc) ->
252 + to_tokens(Rest, [{end_tag, to_tag(Tag)} | Acc]);
253 + to_tokens([{Tag0, [{T0} | R1]} | Rest], Acc) ->
254 + %% Allow {br}
255 + to_tokens([{Tag0, [{T0, [], []} | R1]} | Rest], Acc);
256 + to_tokens([{Tag0, [T0={'=', _C0} | R1]} | Rest], Acc) ->
257 + %% Allow {'=', iolist()}
258 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
259 + to_tokens([{Tag0, [T0={comment, _C0} | R1]} | Rest], Acc) ->
260 + %% Allow {comment, iolist()}
261 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
262 + to_tokens([{Tag0, [T0={pi, _S0} | R1]} | Rest], Acc) ->
263 + %% Allow {pi, binary()}
264 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
265 + to_tokens([{Tag0, [T0={pi, _S0, _A0} | R1]} | Rest], Acc) ->
266 + %% Allow {pi, binary(), list()}
267 + to_tokens([{Tag0, R1} | Rest], [T0 | Acc]);
268 + to_tokens([{Tag0, [{T0, A0=[{_, _} | _]} | R1]} | Rest], Acc) ->
269 + %% Allow {p, [{"class", "foo"}]}
270 + to_tokens([{Tag0, [{T0, A0, []} | R1]} | Rest], Acc);
271 + to_tokens([{Tag0, [{T0, C0} | R1]} | Rest], Acc) ->
272 + %% Allow {p, "content"} and {p, <<"content">>}
273 + to_tokens([{Tag0, [{T0, [], C0} | R1]} | Rest], Acc);
274 + to_tokens([{Tag0, [{T0, A1, C0} | R1]} | Rest], Acc) when is_binary(C0) ->
275 + %% Allow {"p", [{"class", "foo"}], <<"content">>}
276 + to_tokens([{Tag0, [{T0, A1, binary_to_list(C0)} | R1]} | Rest], Acc);
277 + to_tokens([{Tag0, [{T0, A1, C0=[C | _]} | R1]} | Rest], Acc)
278 + when is_integer(C) ->
279 + %% Allow {"p", [{"class", "foo"}], "content"}
280 + to_tokens([{Tag0, [{T0, A1, [C0]} | R1]} | Rest], Acc);
281 + to_tokens([{Tag0, [{T0, A1, C1} | R1]} | Rest], Acc) ->
282 + %% Native {"p", [{"class", "foo"}], ["content"]}
283 + Tag = to_tag(Tag0),
284 + T1 = to_tag(T0),
285 + case is_singleton(norm(T1)) of
286 + true ->
287 + to_tokens([{Tag, R1} | Rest], [{start_tag, T1, A1, true} | Acc]);
288 + false ->
289 + to_tokens([{T1, C1}, {Tag, R1} | Rest],
290 + [{start_tag, T1, A1, false} | Acc])
291 + end;
292 + to_tokens([{Tag0, [L | R1]} | Rest], Acc) when is_list(L) ->
293 + %% List text
294 + Tag = to_tag(Tag0),
295 + to_tokens([{Tag, R1} | Rest], [{data, iolist_to_binary(L), false} | Acc]);
296 + to_tokens([{Tag0, [B | R1]} | Rest], Acc) when is_binary(B) ->
297 + %% Binary text
298 + Tag = to_tag(Tag0),
299 + to_tokens([{Tag, R1} | Rest], [{data, B, false} | Acc]).
300 +
301 + tokens(B, S=#decoder{offset=O}, Acc) ->
302 + case B of
303 + <<_:O/binary>> ->
304 + lists:reverse(Acc);
305 + _ ->
306 + {Tag, S1} = tokenize(B, S),
307 + case parse_flag(Tag) of
308 + script ->
309 + {Tag2, S2} = tokenize_script(B, S1),
310 + tokens(B, S2, [Tag2, Tag | Acc]);
311 + textarea ->
312 + {Tag2, S2} = tokenize_textarea(B, S1),
313 + tokens(B, S2, [Tag2, Tag | Acc]);
314 + none ->
315 + tokens(B, S1, [Tag | Acc])
316 + end
317 + end.
318 +
319 + parse_flag({start_tag, B, _, false}) ->
320 + case string:to_lower(binary_to_list(B)) of
321 + "script" ->
322 + script;
323 + "textarea" ->
324 + textarea;
325 + _ ->
326 + none
327 + end;
328 + parse_flag(_) ->
329 + none.
330 +
331 + tokenize(B, S=#decoder{offset=O}) ->
332 + case B of
333 + <<_:O/binary, "<!--", _/binary>> ->
334 + tokenize_comment(B, ?ADV_COL(S, 4));
335 + <<_:O/binary, "<!doctype", _/binary>> ->
336 + tokenize_doctype(B, ?ADV_COL(S, 10));
337 + <<_:O/binary, "<!DOCTYPE", _/binary>> ->
338 + tokenize_doctype(B, ?ADV_COL(S, 10));
339 + <<_:O/binary, "<![CDATA[", _/binary>> ->
340 + tokenize_cdata(B, ?ADV_COL(S, 9));
341 + <<_:O/binary, "<?php", _/binary>> ->
342 + {Body, S1} = raw_qgt(B, ?ADV_COL(S, 2)),
343 + {{pi, Body}, S1};
344 + <<_:O/binary, "<?", _/binary>> ->
345 + {Tag, S1} = tokenize_literal(B, ?ADV_COL(S, 2)),
346 + {Attrs, S2} = tokenize_attributes(B, S1),
347 + S3 = find_qgt(B, S2),
348 + {{pi, Tag, Attrs}, S3};
349 + <<_:O/binary, "&", _/binary>> ->
350 + tokenize_charref(B, ?INC_COL(S));
351 + <<_:O/binary, "</", _/binary>> ->
352 + {Tag, S1} = tokenize_literal(B, ?ADV_COL(S, 2)),
353 + {S2, _} = find_gt(B, S1),
354 + {{end_tag, Tag}, S2};
355 + <<_:O/binary, "<", C, _/binary>>
356 + when ?IS_WHITESPACE(C); not ?IS_LETTER(C) ->
357 + %% This isn't really strict HTML
358 + {{data, Data, _Whitespace}, S1} = tokenize_data(B, ?INC_COL(S)),
359 + {{data, <<$<, Data/binary>>, false}, S1};
360 + <<_:O/binary, "<", _/binary>> ->
361 + {Tag, S1} = tokenize_literal(B, ?INC_COL(S)),
362 + {Attrs, S2} = tokenize_attributes(B, S1),
363 + {S3, HasSlash} = find_gt(B, S2),
364 + Singleton = HasSlash orelse is_singleton(Tag),
365 + {{start_tag, Tag, Attrs, Singleton}, S3};
366 + _ ->
367 + tokenize_data(B, S)
368 + end.
369 +
370 + tree_data([{data, Data, Whitespace} | Rest], AllWhitespace, Acc) ->
371 + tree_data(Rest, (Whitespace andalso AllWhitespace), [Data | Acc]);
372 + tree_data(Rest, AllWhitespace, Acc) ->
373 + {iolist_to_binary(lists:reverse(Acc)), AllWhitespace, Rest}.
374 +
375 + tree([], Stack) ->
376 + {destack(Stack), []};
377 + tree([{end_tag, Tag} | Rest], Stack) ->
378 + case destack(norm(Tag), Stack) of
379 + S when is_list(S) ->
380 + tree(Rest, S);
381 + Result ->
382 + {Result, []}
383 + end;
384 + tree([{start_tag, Tag, Attrs, true} | Rest], S) ->
385 + tree(Rest, append_stack_child(norm({Tag, Attrs}), S));
386 + tree([{start_tag, Tag, Attrs, false} | Rest], S) ->
387 + tree(Rest, stack(norm({Tag, Attrs}), S));
388 + tree([T={pi, _Raw} | Rest], S) ->
389 + tree(Rest, append_stack_child(T, S));
390 + tree([T={pi, _Tag, _Attrs} | Rest], S) ->
391 + tree(Rest, append_stack_child(T, S));
392 + tree([T={comment, _Comment} | Rest], S) ->
393 + tree(Rest, append_stack_child(T, S));
394 + tree(L=[{data, _Data, _Whitespace} | _], S) ->
395 + case tree_data(L, true, []) of
396 + {_, true, Rest} ->
397 + tree(Rest, S);
398 + {Data, false, Rest} ->
399 + tree(Rest, append_stack_child(Data, S))
400 + end;
401 + tree([{doctype, _} | Rest], Stack) ->
402 + tree(Rest, Stack).
403 +
404 + norm({Tag, Attrs}) ->
405 + {norm(Tag), [{norm(K), iolist_to_binary(V)} || {K, V} <- Attrs], []};
406 + norm(Tag) when is_binary(Tag) ->
407 + Tag;
408 + norm(Tag) ->
409 + list_to_binary(string:to_lower(Tag)).
410 +
411 + stack(T1={TN, _, _}, Stack=[{TN, _, _} | _Rest])
412 + when TN =:= <<"li">> orelse TN =:= <<"option">> ->
413 + [T1 | destack(TN, Stack)];
414 + stack(T1={TN0, _, _}, Stack=[{TN1, _, _} | _Rest])
415 + when (TN0 =:= <<"dd">> orelse TN0 =:= <<"dt">>) andalso
416 + (TN1 =:= <<"dd">> orelse TN1 =:= <<"dt">>) ->
417 + [T1 | destack(TN1, Stack)];
418 + stack(T1, Stack) ->
419 + [T1 | Stack].
420 +
421 + append_stack_child(StartTag, [{Name, Attrs, Acc} | Stack]) ->
422 + [{Name, Attrs, [StartTag | Acc]} | Stack].
423 +
424 + destack(<<"br">>, Stack) ->
425 + %% This is an ugly hack to make dumb_br_test() pass,
426 + %% this makes it such that br can never have children.
427 + Stack;
428 + destack(TagName, Stack) when is_list(Stack) ->
429 + F = fun (X) ->
430 + case X of
431 + {TagName, _, _} ->
432 + false;
433 + _ ->
434 + true
435 + end
436 + end,
437 + case lists:splitwith(F, Stack) of
438 + {_, []} ->
439 + %% If we're parsing something like XML we might find
440 + %% a <link>tag</link> that is normally a singleton
441 + %% in HTML but isn't here
442 + case {is_singleton(TagName), Stack} of
443 + {true, [{T0, A0, Acc0} | Post0]} ->
444 + case lists:splitwith(F, Acc0) of
445 + {_, []} ->
446 + %% Actually was a singleton
447 + Stack;
448 + {Pre, [{T1, A1, Acc1} | Post1]} ->
449 + [{T0, A0, [{T1, A1, Acc1 ++ lists:reverse(Pre)} | Post1]}
450 + | Post0]
451 + end;
452 + _ ->
453 + %% No match, no state change
454 + Stack
455 + end;
456 + {_Pre, [_T]} ->
457 + %% Unfurl the whole stack, we're done
458 + destack(Stack);
459 + {Pre, [T, {T0, A0, Acc0} | Post]} ->
460 + %% Unfurl up to the tag, then accumulate it
461 + [{T0, A0, [destack(Pre ++ [T]) | Acc0]} | Post]
462 + end.
463 +
464 + destack([{Tag, Attrs, Acc}]) ->
465 + {Tag, Attrs, lists:reverse(Acc)};
466 + destack([{T1, A1, Acc1}, {T0, A0, Acc0} | Rest]) ->
467 + destack([{T0, A0, [{T1, A1, lists:reverse(Acc1)} | Acc0]} | Rest]).
468 +
469 + is_singleton(<<"area">>) -> true;
470 + is_singleton(<<"base">>) -> true;
471 + is_singleton(<<"br">>) -> true;
472 + is_singleton(<<"col">>) -> true;
473 + is_singleton(<<"embed">>) -> true;
474 + is_singleton(<<"hr">>) -> true;
475 + is_singleton(<<"img">>) -> true;
476 + is_singleton(<<"input">>) -> true;
477 + is_singleton(<<"keygen">>) -> true;
478 + is_singleton(<<"link">>) -> true;
479 + is_singleton(<<"meta">>) -> true;
480 + is_singleton(<<"param">>) -> true;
481 + is_singleton(<<"source">>) -> true;
482 + is_singleton(<<"track">>) -> true;
483 + is_singleton(<<"wbr">>) -> true;
484 + is_singleton(_) -> false.
485 +
486 + tokenize_data(B, S=#decoder{offset=O}) ->
487 + tokenize_data(B, S, O, true).
488 +
489 + tokenize_data(B, S=#decoder{offset=O}, Start, Whitespace) ->
490 + case B of
491 + <<_:O/binary, C, _/binary>> when (C =/= $< andalso C =/= $&) ->
492 + tokenize_data(B, ?INC_CHAR(S, C), Start,
493 + (Whitespace andalso ?IS_WHITESPACE(C)));
494 + _ ->
495 + Len = O - Start,
496 + <<_:Start/binary, Data:Len/binary, _/binary>> = B,
497 + {{data, Data, Whitespace}, S}
498 + end.
499 +
500 + tokenize_attributes(B, S) ->
501 + tokenize_attributes(B, S, []).
502 +
503 + tokenize_attributes(B, S=#decoder{offset=O}, Acc) ->
504 + case B of
505 + <<_:O/binary>> ->
506 + {lists:reverse(Acc), S};
507 + <<_:O/binary, C, _/binary>> when (C =:= $> orelse C =:= $/) ->
508 + {lists:reverse(Acc), S};
509 + <<_:O/binary, "?>", _/binary>> ->
510 + {lists:reverse(Acc), S};
511 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) ->
512 + tokenize_attributes(B, ?INC_CHAR(S, C), Acc);
513 + _ ->
514 + {Attr, S1} = tokenize_literal(B, S),
515 + {Value, S2} = tokenize_attr_value(Attr, B, S1),
516 + tokenize_attributes(B, S2, [{Attr, Value} | Acc])
517 + end.
518 +
519 + tokenize_attr_value(Attr, B, S) ->
520 + S1 = skip_whitespace(B, S),
521 + O = S1#decoder.offset,
522 + case B of
523 + <<_:O/binary, "=", _/binary>> ->
524 + S2 = skip_whitespace(B, ?INC_COL(S1)),
525 + tokenize_quoted_or_unquoted_attr_value(B, S2);
526 + _ ->
527 + {Attr, S1}
528 + end.
529 +
530 + tokenize_quoted_or_unquoted_attr_value(B, S=#decoder{offset=O}) ->
531 + case B of
532 + <<_:O/binary>> ->
533 + { [], S };
534 + <<_:O/binary, Q, _/binary>> when Q =:= ?QUOTE orelse
535 + Q =:= ?SQUOTE ->
536 + tokenize_quoted_attr_value(B, ?INC_COL(S), [], Q);
537 + <<_:O/binary, _/binary>> ->
538 + tokenize_unquoted_attr_value(B, S, [])
539 + end.
540 +
541 + tokenize_quoted_attr_value(B, S=#decoder{offset=O}, Acc, Q) ->
542 + case B of
543 + <<_:O/binary>> ->
544 + { iolist_to_binary(lists:reverse(Acc)), S };
545 + <<_:O/binary, $&, _/binary>> ->
546 + {{data, Data, false}, S1} = tokenize_charref(B, ?INC_COL(S)),
547 + tokenize_quoted_attr_value(B, S1, [Data|Acc], Q);
548 + <<_:O/binary, Q, _/binary>> ->
549 + { iolist_to_binary(lists:reverse(Acc)), ?INC_COL(S) };
550 + <<_:O/binary, C, _/binary>> ->
551 + tokenize_quoted_attr_value(B, ?INC_COL(S), [C|Acc], Q)
552 + end.
553 +
554 + tokenize_unquoted_attr_value(B, S=#decoder{offset=O}, Acc) ->
555 + case B of
556 + <<_:O/binary>> ->
557 + { iolist_to_binary(lists:reverse(Acc)), S };
558 + <<_:O/binary, $&, _/binary>> ->
559 + {{data, Data, false}, S1} = tokenize_charref(B, ?INC_COL(S)),
560 + tokenize_unquoted_attr_value(B, S1, [Data|Acc]);
561 + <<_:O/binary, $/, $>, _/binary>> ->
562 + { iolist_to_binary(lists:reverse(Acc)), S };
563 + <<_:O/binary, C, _/binary>> when ?PROBABLE_CLOSE(C) ->
564 + { iolist_to_binary(lists:reverse(Acc)), S };
565 + <<_:O/binary, C, _/binary>> ->
566 + tokenize_unquoted_attr_value(B, ?INC_COL(S), [C|Acc])
567 + end.
568 +
569 + skip_whitespace(B, S=#decoder{offset=O}) ->
570 + case B of
571 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) ->
572 + skip_whitespace(B, ?INC_CHAR(S, C));
573 + _ ->
574 + S
575 + end.
576 +
577 + tokenize_literal(Bin, S=#decoder{offset=O}) ->
578 + case Bin of
579 + <<_:O/binary, C, _/binary>> when C =:= $>
580 + orelse C =:= $/
581 + orelse C =:= $= ->
582 + %% Handle case where tokenize_literal would consume
583 + %% 0 chars. http://github.com/mochi/mochiweb/pull/13
584 + {[C], ?INC_COL(S)};
585 + _ ->
586 + tokenize_literal(Bin, S, [])
587 + end.
588 +
589 + tokenize_literal(Bin, S=#decoder{offset=O}, Acc) ->
590 + case Bin of
591 + <<_:O/binary, $&, _/binary>> ->
592 + {{data, Data, false}, S1} = tokenize_charref(Bin, ?INC_COL(S)),
593 + tokenize_literal(Bin, S1, [Data | Acc]);
594 + <<_:O/binary, C, _/binary>> when not (?IS_WHITESPACE(C)
595 + orelse C =:= $>
596 + orelse C =:= $/
597 + orelse C =:= $=) ->
598 + tokenize_literal(Bin, ?INC_COL(S), [C | Acc]);
599 + _ ->
600 + {iolist_to_binary(string:to_lower(lists:reverse(Acc))), S}
601 + end.
602 +
603 + raw_qgt(Bin, S=#decoder{offset=O}) ->
604 + raw_qgt(Bin, S, O).
605 +
606 + raw_qgt(Bin, S=#decoder{offset=O}, Start) ->
607 + case Bin of
608 + <<_:O/binary, "?>", _/binary>> ->
609 + Len = O - Start,
610 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
611 + {Raw, ?ADV_COL(S, 2)};
612 + <<_:O/binary, C, _/binary>> ->
613 + raw_qgt(Bin, ?INC_CHAR(S, C), Start);
614 + <<_:O/binary>> ->
615 + <<_:Start/binary, Raw/binary>> = Bin,
616 + {Raw, S}
617 + end.
618 +
619 + find_qgt(Bin, S=#decoder{offset=O}) ->
620 + case Bin of
621 + <<_:O/binary, "?>", _/binary>> ->
622 + ?ADV_COL(S, 2);
623 + <<_:O/binary, ">", _/binary>> ->
624 + ?ADV_COL(S, 1);
625 + <<_:O/binary, "/>", _/binary>> ->
626 + ?ADV_COL(S, 2);
627 + %% tokenize_attributes takes care of this state:
628 + %% <<_:O/binary, C, _/binary>> ->
629 + %% find_qgt(Bin, ?INC_CHAR(S, C));
630 + <<_:O/binary>> ->
631 + S
632 + end.
633 +
634 + find_gt(Bin, S) ->
635 + find_gt(Bin, S, false).
636 +
637 + find_gt(Bin, S=#decoder{offset=O}, HasSlash) ->
638 + case Bin of
639 + <<_:O/binary, $/, _/binary>> ->
640 + find_gt(Bin, ?INC_COL(S), true);
641 + <<_:O/binary, $>, _/binary>> ->
642 + {?INC_COL(S), HasSlash};
643 + <<_:O/binary, C, _/binary>> ->
644 + find_gt(Bin, ?INC_CHAR(S, C), HasSlash);
645 + _ ->
646 + {S, HasSlash}
647 + end.
648 +
649 + tokenize_charref(Bin, S=#decoder{offset=O}) ->
650 + try
651 + case tokenize_charref_raw(Bin, S, O) of
652 + {C1, S1} when C1 >= 16#D800 andalso C1 =< 16#DFFF ->
653 + %% Surrogate pair
654 + tokeninize_charref_surrogate_pair(Bin, S1, C1);
655 + {Unichar, S1} when is_integer(Unichar) ->
656 + %% CHANGED: Previously this was mochiutf8:codepoint_to_bytes(Unichar)
657 + %% but that is equivalent to the below.
658 + {{data, <<Unichar/utf8>>, false},
659 + S1};
660 + {Unichars, S1} when is_list(Unichars) ->
661 + {{data, unicode:characters_to_binary(Unichars), false},
662 + S1};
663 + {undefined, _} ->
664 + throw(invalid_charref)
665 + end
666 + catch
667 + throw:invalid_charref ->
668 + {{data, <<"&">>, false}, S}
669 + end.
670 +
671 + tokeninize_charref_surrogate_pair(Bin, S=#decoder{offset=O}, C1) ->
672 + case Bin of
673 + <<_:O/binary, $&, _/binary>> ->
674 + case tokenize_charref_raw(Bin, ?INC_COL(S), O + 1) of
675 + {C2, S1} when C2 >= 16#D800 andalso C1 =< 16#DFFF ->
676 + {{data,
677 + unicode:characters_to_binary(
678 + <<C1:16, C2:16>>,
679 + utf16,
680 + utf8),
681 + false},
682 + S1};
683 + _ ->
684 + throw(invalid_charref)
685 + end;
686 + _ ->
687 + throw(invalid_charref)
688 + end.
689 +
690 + tokenize_charref_raw(Bin, S=#decoder{offset=O}, Start) ->
691 + case Bin of
692 + <<_:O/binary>> ->
693 + throw(invalid_charref);
694 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C)
695 + orelse C =:= ?SQUOTE
696 + orelse C =:= ?QUOTE
697 + orelse C =:= $/
698 + orelse C =:= $> ->
699 + throw(invalid_charref);
700 + <<_:O/binary, $;, _/binary>> ->
701 + Len = O - Start,
702 + %% CHANGED: Previously this was mochiweb_charref:charref/1
703 + %% but the functionality below is equivalent;
704 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
705 + <<CP/utf8>> = 'Elixir.HtmlEntities':decode(<<$&, Raw/binary, $;>>),
706 + {CP, ?INC_COL(S)};
707 + _ ->
708 + tokenize_charref_raw(Bin, ?INC_COL(S), Start)
709 + end.
710 +
711 + tokenize_doctype(Bin, S) ->
712 + tokenize_doctype(Bin, S, []).
713 +
714 + tokenize_doctype(Bin, S=#decoder{offset=O}, Acc) ->
715 + case Bin of
716 + <<_:O/binary>> ->
717 + {{doctype, lists:reverse(Acc)}, S};
718 + <<_:O/binary, $>, _/binary>> ->
719 + {{doctype, lists:reverse(Acc)}, ?INC_COL(S)};
720 + <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) ->
721 + tokenize_doctype(Bin, ?INC_CHAR(S, C), Acc);
722 + _ ->
723 + {Word, S1} = tokenize_word_or_literal(Bin, S),
724 + tokenize_doctype(Bin, S1, [Word | Acc])
725 + end.
726 +
727 + tokenize_word_or_literal(Bin, S=#decoder{offset=O}) ->
728 + case Bin of
729 + <<_:O/binary, C, _/binary>> when C =:= ?QUOTE orelse C =:= ?SQUOTE ->
730 + tokenize_word(Bin, ?INC_COL(S), C);
731 + <<_:O/binary, C, _/binary>> when not ?IS_WHITESPACE(C) ->
732 + %% Sanity check for whitespace
733 + tokenize_literal(Bin, S)
734 + end.
735 +
736 + tokenize_word(Bin, S, Quote) ->
737 + tokenize_word(Bin, S, Quote, []).
738 +
739 + tokenize_word(Bin, S=#decoder{offset=O}, Quote, Acc) ->
740 + case Bin of
741 + <<_:O/binary>> ->
742 + {iolist_to_binary(lists:reverse(Acc)), S};
743 + <<_:O/binary, Quote, _/binary>> ->
744 + {iolist_to_binary(lists:reverse(Acc)), ?INC_COL(S)};
745 + <<_:O/binary, $&, _/binary>> ->
746 + {{data, Data, false}, S1} = tokenize_charref(Bin, ?INC_COL(S)),
747 + tokenize_word(Bin, S1, Quote, [Data | Acc]);
748 + <<_:O/binary, C, _/binary>> ->
749 + tokenize_word(Bin, ?INC_CHAR(S, C), Quote, [C | Acc])
750 + end.
751 +
752 + tokenize_cdata(Bin, S=#decoder{offset=O}) ->
753 + tokenize_cdata(Bin, S, O).
754 +
755 + tokenize_cdata(Bin, S=#decoder{offset=O}, Start) ->
756 + case Bin of
757 + <<_:O/binary, "]]>", _/binary>> ->
758 + Len = O - Start,
759 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
760 + {{data, Raw, false}, ?ADV_COL(S, 3)};
761 + <<_:O/binary, C, _/binary>> ->
762 + tokenize_cdata(Bin, ?INC_CHAR(S, C), Start);
763 + _ ->
764 + <<_:O/binary, Raw/binary>> = Bin,
765 + {{data, Raw, false}, S}
766 + end.
767 +
768 + tokenize_comment(Bin, S=#decoder{offset=O}) ->
769 + tokenize_comment(Bin, S, O).
770 +
771 + tokenize_comment(Bin, S=#decoder{offset=O}, Start) ->
772 + case Bin of
773 + <<_:O/binary, "-->", _/binary>> ->
774 + Len = O - Start,
775 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
776 + {{comment, Raw}, ?ADV_COL(S, 3)};
777 + <<_:O/binary, C, _/binary>> ->
778 + tokenize_comment(Bin, ?INC_CHAR(S, C), Start);
779 + <<_:Start/binary, Raw/binary>> ->
780 + {{comment, Raw}, S}
781 + end.
782 +
783 + tokenize_script(Bin, S=#decoder{offset=O}) ->
784 + tokenize_script(Bin, S, O).
785 +
786 + tokenize_script(Bin, S=#decoder{offset=O}, Start) ->
787 + case Bin of
788 + %% Just a look-ahead, we want the end_tag separately
789 + <<_:O/binary, $<, $/, SS, CC, RR, II, PP, TT, ZZ, _/binary>>
790 + when (SS =:= $s orelse SS =:= $S) andalso
791 + (CC =:= $c orelse CC =:= $C) andalso
792 + (RR =:= $r orelse RR =:= $R) andalso
793 + (II =:= $i orelse II =:= $I) andalso
794 + (PP =:= $p orelse PP =:= $P) andalso
795 + (TT=:= $t orelse TT =:= $T) andalso
796 + ?PROBABLE_CLOSE(ZZ) ->
797 + Len = O - Start,
798 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
799 + {{data, Raw, false}, S};
800 + <<_:O/binary, C, _/binary>> ->
801 + tokenize_script(Bin, ?INC_CHAR(S, C), Start);
802 + <<_:Start/binary, Raw/binary>> ->
803 + {{data, Raw, false}, S}
804 + end.
805 +
806 + tokenize_textarea(Bin, S=#decoder{offset=O}) ->
807 + tokenize_textarea(Bin, S, O).
808 +
809 + tokenize_textarea(Bin, S=#decoder{offset=O}, Start) ->
810 + case Bin of
811 + %% Just a look-ahead, we want the end_tag separately
812 + <<_:O/binary, $<, $/, TT, EE, XX, TT2, AA, RR, EE2, AA2, ZZ, _/binary>>
813 + when (TT =:= $t orelse TT =:= $T) andalso
814 + (EE =:= $e orelse EE =:= $E) andalso
815 + (XX =:= $x orelse XX =:= $X) andalso
816 + (TT2 =:= $t orelse TT2 =:= $T) andalso
817 + (AA =:= $a orelse AA =:= $A) andalso
818 + (RR =:= $r orelse RR =:= $R) andalso
819 + (EE2 =:= $e orelse EE2 =:= $E) andalso
820 + (AA2 =:= $a orelse AA2 =:= $A) andalso
821 + ?PROBABLE_CLOSE(ZZ) ->
822 + Len = O - Start,
823 + <<_:Start/binary, Raw:Len/binary, _/binary>> = Bin,
824 + {{data, Raw, false}, S};
825 + <<_:O/binary, C, _/binary>> ->
826 + tokenize_textarea(Bin, ?INC_CHAR(S, C), Start);
827 + <<_:Start/binary, Raw/binary>> ->
828 + {{data, Raw, false}, S}
829 + end.