Packages

floki

0.7.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
1 Dependency
Dependants
Files
Activity
Hide whitespace

Compare versions

6 files changed
+104 additions
-15 deletions
  @@ -109,6 +109,15 @@ end
109 109
110 110 After that, run `mix deps.get`.
111 111
112 + ## Dependencies
113 +
114 + Floki needs the `leex` module in order to compile.
115 + Normally this module is installed with Erlang in a complete installation.
116 +
117 + If you get this [kind of error](https://github.com/philss/floki/issues/35),
118 + you need to install the `erlang-dev` and `erlang-parsetools` packages in order get the `leex` module.
119 + The packages names may be different depending on your OS.
120 +
112 121 ## More about the API
113 122
114 123 To parse a HTML document, try:
  @@ -137,7 +146,7 @@ To convert your node tree back to raw HTML (spaces are ignored):
137 146
138 147 ```elixir
139 148 Floki.find(html, ".example")
140 - |> Flok.raw_html
149 + |> Floki.raw_html
141 150 # => <div class="example"></div>
142 151 ```
  @@ -6,11 +6,11 @@
6 6 {<<"files">>,
7 7 [<<"lib/floki.ex">>,<<"lib/floki/attribute_selector.ex">>,
8 8 <<"lib/floki/combinator.ex">>,<<"lib/floki/deep_text.ex">>,
9 - <<"lib/floki/finder.ex">>,<<"lib/floki/flat_text.ex">>,
10 - <<"lib/floki/parser.ex">>,<<"lib/floki/selector.ex">>,
11 - <<"lib/floki/selector_parser.ex">>,<<"lib/floki/selector_tokenizer.ex">>,
12 - <<"src/floki_selector_lexer.xrl">>,<<"mix.exs">>,<<"README.md">>,
13 - <<"LICENSE">>]}.
9 + <<"lib/floki/filter_out.ex">>,<<"lib/floki/finder.ex">>,
10 + <<"lib/floki/flat_text.ex">>,<<"lib/floki/parser.ex">>,
11 + <<"lib/floki/selector.ex">>,<<"lib/floki/selector_parser.ex">>,
12 + <<"lib/floki/selector_tokenizer.ex">>,<<"src/floki_selector_lexer.xrl">>,
13 + <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>]}.
14 14 {<<"licenses">>,[<<"MIT">>]}.
15 15 {<<"links">>,
16 16 [{<<"Docs">>,<<"http://hexdocs.pm/floki">>},
  @@ -22,4 +22,4 @@
22 22 [{<<"app">>,<<"mochiweb">>},
23 23 {<<"optional">>,false},
24 24 {<<"requirement">>,<<"~> 2.12.2">>}]}]}.
25 - {<<"version">>,<<"0.6.1">>}.
25 + {<<"version">>,<<"0.7.0">>}.
  @@ -148,6 +148,7 @@ defmodule Floki do
148 148 Returns the text nodes from a HTML tree.
149 149 By default, it will perform a deep search through the HTML tree.
150 150 You can disable deep search with the option `deep` assigned to false.
151 + You can include content of script tags with the option `js` assigned to true.
151 152
152 153 ## Examples
153 154
  @@ -157,24 +158,36 @@ defmodule Floki do
157 158 iex> Floki.text("<div><span>hello</span> world</div>", deep: false)
158 159 " world"
159 160
161 + iex> Floki.text("<div><script>hello</script> world</div>")
162 + " world"
163 +
164 + iex> Floki.text("<div><script>hello</script> world</div>", js: true)
165 + "hello world"
166 +
160 167 """
161 168
162 169 @spec text(html_tree | binary) :: binary
163 170
164 - def text(html, opts \\ [deep: true]) do
171 + def text(html, opts \\ [deep: true, js: false]) do
165 172 html_tree =
166 173 case is_binary(html) do
167 174 true -> parse(html)
168 175 false -> html
169 176 end
170 177
171 - search_strategy =
172 - case opts[:deep] do
173 - true -> Floki.DeepText
174 - false -> Floki.FlatText
178 + cleaned_html_tree =
179 + case opts[:js] do
180 + true -> html_tree
181 + _ -> filter_out(html_tree, "script")
175 182 end
176 183
177 - search_strategy.get(html_tree)
184 + search_strategy =
185 + case opts[:deep] do
186 + false -> Floki.FlatText
187 + _ -> Floki.DeepText
188 + end
189 +
190 + search_strategy.get(cleaned_html_tree)
178 191 end
179 192
180 193 @doc """
  @@ -237,4 +250,30 @@ defmodule Floki do
237 250 attr_name == attribute_name
238 251 end
239 252 end
253 +
254 + @doc """
255 + Returns the nodes from a HTML tree that don't match the filter selector.
256 +
257 + ## Examples
258 +
259 + iex> Floki.filter_out("<div><script>hello</script> world</div>", "script")
260 + {"div", [], [" world"]}
261 +
262 + iex> Floki.filter_out([{"body", [], [{"script", [], []},{"div", [], []}]}], "script")
263 + [{"body", [], [{"div", [], []}]}]
264 +
265 + """
266 +
267 + @spec filter_out(binary | html_tree, binary) :: list
268 +
269 + def filter_out(html_tree, selector) when is_binary(html_tree) do
270 + html_tree
271 + |> parse
272 + |> Floki.FilterOut.filter_out(selector)
273 + end
274 + def filter_out(elements, selector) do
275 + Floki.FilterOut.filter_out(elements, selector)
276 + end
277 +
278 +
240 279 end
  @@ -0,0 +1,36 @@
1 + defmodule Floki.FilterOut do
2 + @doc """
3 + Helper functions for filtering out a specific element from the tree.
4 + """
5 +
6 + @type html_tree :: tuple | list
7 + @type selector :: binary
8 +
9 + @spec filter_out(html_tree, selector) :: tuple | list
10 +
11 + def filter_out(html_tree, selector) do
12 + mapper(html_tree, selector)
13 + end
14 +
15 + defp filter({nodetext, _, _}, selector) when nodetext === selector do
16 + false
17 + end
18 +
19 + defp filter(_, _) do
20 + true
21 + end
22 +
23 + defp mapper(nodes, selector) when is_list(nodes) do
24 + Enum.filter_map(nodes, &filter(&1, selector), &mapper(&1, selector))
25 + end
26 +
27 + defp mapper({nodetext, x, y}, selector) do
28 + {nodetext, x, mapper(y, selector)}
29 + end
30 +
31 + defp mapper(nodetext, _) do
32 + nodetext
33 + end
34 +
35 +
36 + end
  @@ -87,7 +87,12 @@ defmodule Floki.Finder do
87 87 end
88 88
89 89 defp traverse_sibling(_nodes, sibling_nodes, selector, acc) do
90 - sibling_node = Enum.drop_while(sibling_nodes, &ignore_node?/1) |> hd
90 + sibling_nodes = Enum.drop_while(sibling_nodes, &ignore_node?/1)
91 +
92 + sibling_node = case sibling_nodes do
93 + [] -> nil
94 + _ -> hd(sibling_nodes)
95 + end
91 96
92 97 if Selector.match?(sibling_node, selector) do
93 98 case selector.combinator do
Loading more files…