Packages

floki

0.13.0
0.38.4 0.38.3 0.38.2 0.38.1 0.38.0 0.37.1 0.37.0 0.36.3 0.36.2 0.36.1 0.36.0 0.35.4 0.35.3 0.35.2 0.35.1 0.35.0 0.34.3 0.34.2 0.34.1 0.34.0 0.33.1 0.33.0 0.32.1 0.32.0 0.31.0 0.30.1 0.30.0 0.29.0 0.28.0 0.27.0 0.26.0 0.25.0 0.24.0 0.23.1 0.23.0 0.22.0 0.21.0 0.20.4 0.20.3 0.20.2 0.20.1 0.20.0 0.19.3 0.19.2 0.19.1 0.19.0 0.18.1 0.18.0 0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.2 0.13.1 0.13.0 0.12.1 0.12.0 0.11.0 0.10.1 0.10.0 0.9.0 0.8.1 0.8.0 0.7.2 0.7.1 0.7.0 0.6.1 0.6.0 0.5.0 0.4.1 0.4.0 0.3.3 0.3.2 0.3.1 0.3.0 0.2.1 0.2.0 0.1.1 0.1.0 0.0.5 0.0.4 0.0.3 0.0.2 0.0.1

Floki is a simple HTML parser that enables search for nodes using CSS selectors.

HexDocs

Current section

86 Versions

Jump to
Readme
86 Versions
1 Dependency
Dependants
Files
Activity
Hide whitespace

Compare versions

9 files changed
+144 additions
-35 deletions
  @@ -8,7 +8,7 @@
8 8 <<"lib/floki/combinator.ex">>,<<"lib/floki/deep_text.ex">>,
9 9 <<"lib/floki/filter_out.ex">>,<<"lib/floki/finder.ex">>,
10 10 <<"lib/floki/flat_text.ex">>,<<"lib/floki/html_tree.ex">>,
11 - <<"lib/floki/html_tree/html_node.ex">>,
11 + <<"lib/floki/html_tree/comment.ex">>,<<"lib/floki/html_tree/html_node.ex">>,
12 12 <<"lib/floki/html_tree/id_seeder.ex">>,<<"lib/floki/html_tree/text.ex">>,
13 13 <<"lib/floki/parser.ex">>,<<"lib/floki/selector.ex">>,
14 14 <<"lib/floki/selector/pseudo_class.ex">>,<<"lib/floki/selector_parser.ex">>,
  @@ -25,4 +25,4 @@
25 25 {<<"name">>,<<"mochiweb">>},
26 26 {<<"optional">>,false},
27 27 {<<"requirement">>,<<"~> 2.15">>}]]}.
28 - {<<"version">>,<<"0.12.1">>}.
28 + {<<"version">>,<<"0.13.0">>}.
  @@ -1,5 +1,5 @@
1 1 defmodule Floki do
2 - alias Floki.{Finder, Parser, FilterOut}
2 + alias Floki.{Finder, Parser, FilterOut, HTMLTree}
3 3
4 4 @moduledoc """
5 5 Floki is a simple HTML parser that enables search for nodes using CSS selectors.
  @@ -141,10 +141,16 @@ defmodule Floki do
141 141 @spec find(binary | html_tree, binary) :: html_tree
142 142
143 143 def find(html, selector) when is_binary(html) do
144 - html |> parse |> Finder.find(selector)
144 + html_as_tuple = parse(html)
145 +
146 + {tree, results} = Finder.find(html_as_tuple, selector)
147 +
148 + Enum.map(results, fn(html_node) -> HTMLTree.to_tuple(tree, html_node) end)
145 149 end
146 - def find(html_tree, selector) do
147 - Finder.find(html_tree, selector)
150 + def find(html_tree_as_tuple, selector) do
151 + {tree, results} = Finder.find(html_tree_as_tuple, selector)
152 +
153 + Enum.map(results, fn(html_node) -> HTMLTree.to_tuple(tree, html_node) end)
148 154 end
149 155
150 156 def transform(html_tree_list, transformation) when is_list(html_tree_list) do
  @@ -1,6 +1,8 @@
1 1 defmodule Floki.FilterOut do
2 2 @moduledoc false
3 3
4 + alias Floki.{HTMLTree, Finder}
5 +
4 6 # Helper functions for filtering out a specific element from the tree.
5 7
6 8 @type html_tree :: tuple | list
  @@ -8,8 +10,31 @@ defmodule Floki.FilterOut do
8 10
9 11 @spec filter_out(html_tree, selector) :: tuple | list
10 12
13 + def filter_out(html_tree, :comment) do
14 + mapper(html_tree, :comment)
15 + end
11 16 def filter_out(html_tree, selector) do
12 - mapper(html_tree, selector)
17 + case Finder.find(html_tree, selector) do
18 + {:empty_tree, _} ->
19 + html_tree
20 + {tree, results} ->
21 + new_tree = Enum.reduce(results, tree, fn(html_node, tree) ->
22 + HTMLTree.delete_node(tree, html_node)
23 + end)
24 +
25 + html_as_tuples = Enum.map(new_tree.root_nodes_ids, fn(node_id) ->
26 + root = Map.get(new_tree.nodes, node_id)
27 +
28 + HTMLTree.to_tuple(new_tree, root)
29 + end)
30 +
31 + case html_tree do
32 + tree when is_tuple(tree) ->
33 + hd(html_as_tuples)
34 + trees when is_list(trees) ->
35 + html_as_tuples
36 + end
37 + end
13 38 end
14 39
15 40 defp filter({nodetext, _, _}, selector) when nodetext === selector, do: false
  @@ -8,8 +8,8 @@ defmodule Floki.Finder do
8 8
9 9 alias Floki.{Combinator, Selector, SelectorParser, SelectorTokenizer}
10 10 alias Floki.HTMLTree
11 + alias Floki.HTMLTree.HTMLNode
11 12 alias Floki.Selector.PseudoClass
12 - alias Floki.HTMLTree.Text
13 13
14 14 @type html_tree :: tuple | list
15 15 @type selector :: binary | %Selector{} | [%Selector{}]
  @@ -19,8 +19,8 @@ defmodule Floki.Finder do
19 19
20 20 @spec find(html_tree, selector) :: html_tree
21 21
22 - def find([], _), do: []
23 - def find(html_as_string, _) when is_binary(html_as_string), do: []
22 + def find([], _), do: {:empty_tree, []}
23 + def find(html_as_string, _) when is_binary(html_as_string), do: {:empty_tree, []}
24 24 def find(html_tree, selector_as_string) when is_binary(selector_as_string) do
25 25 selectors = get_selectors(selector_as_string)
26 26 find_selectors(html_tree, selectors)
  @@ -47,11 +47,13 @@ defmodule Floki.Finder do
47 47 defp find_selectors(html_tuple_or_list, selectors) do
48 48 tree = HTMLTree.build(html_tuple_or_list)
49 49
50 - tree.node_ids
51 - |> Enum.reverse
52 - |> get_nodes(tree)
53 - |> Enum.flat_map(fn(html_node) -> get_matches_for_selectors(tree, html_node, selectors) end)
54 - |> Enum.map(fn(html_node) -> as_tuple(tree, html_node) end)
50 + results =
51 + tree.node_ids
52 + |> Enum.reverse
53 + |> get_nodes(tree)
54 + |> Enum.flat_map(fn(html_node) -> get_matches_for_selectors(tree, html_node, selectors) end)
55 +
56 + {tree, results}
55 57 end
56 58
57 59 defp get_selectors(selector_as_string) do
  @@ -171,17 +173,28 @@ defmodule Floki.Finder do
171 173 Map.get(tree.nodes, id)
172 174 end
173 175
176 + defp get_sibling_ids_from([], _html_node), do: []
177 + defp get_sibling_ids_from(ids, html_node) do
178 + ids
179 + |> Enum.reverse
180 + |> Enum.drop_while(fn(id) -> id != html_node.node_id end)
181 + |> tl()
182 + end
174 183 defp get_siblings(html_node, tree) do
175 184 parent = get_node(html_node.parent_node_id, tree)
176 185
177 - if parent do
178 - [_html_node_id | sibling_ids] = parent.children_nodes_ids
179 - |> Enum.reverse
180 - |> Enum.drop_while(fn(id) -> id != html_node.node_id end)
181 - sibling_ids
182 - else
183 - []
184 - end
186 + ids = if parent do
187 + get_sibling_ids_from(parent.children_nodes_ids, html_node)
188 + else
189 + get_sibling_ids_from(tree.root_nodes_ids, html_node)
190 + end
191 +
192 + Enum.filter(ids, fn(id) ->
193 + case get_node(id, tree) do
194 + %HTMLNode{} -> true
195 + _ -> false
196 + end
197 + end)
185 198 end
186 199
187 200 # finds all descendant node ids recursively through the tree
  @@ -193,13 +206,4 @@ defmodule Floki.Finder do
193 206 []
194 207 end
195 208 end
196 -
197 - defp as_tuple(_tree, %Text{content: text}), do: text
198 - defp as_tuple(tree, html_node) do
199 - children = html_node.children_nodes_ids
200 - |> Enum.reverse
201 - |> Enum.map(fn(id) -> as_tuple(tree, get_node(id, tree)) end)
202 -
203 - {html_node.type, html_node.attributes, children}
204 - end
205 209 end
  @@ -9,7 +9,7 @@ defmodule Floki.HTMLTree do
9 9 defstruct nodes: %{}, root_nodes_ids: [], node_ids: []
10 10
11 11 alias Floki.HTMLTree
12 - alias Floki.HTMLTree.{HTMLNode, Text, IDSeeder}
12 + alias Floki.HTMLTree.{HTMLNode, Text, Comment, IDSeeder}
13 13
14 14 def build({tag, attrs, children}) do
15 15 root_id = IDSeeder.seed([])
  @@ -38,6 +38,55 @@ defmodule Floki.HTMLTree do
38 38 Enum.reduce(html_tuples, %HTMLTree{}, reducer)
39 39 end
40 40
41 + def delete_node(tree, html_node) do
42 + do_delete(tree, [html_node], [])
43 + end
44 +
45 + def to_tuple(_tree, %Text{content: text}), do: text
46 + def to_tuple(_tree, %Comment{content: comment}), do: {:comment, comment}
47 + def to_tuple(tree, html_node) do
48 + children = html_node.children_nodes_ids
49 + |> Enum.reverse
50 + |> Enum.map(fn(id) -> to_tuple(tree, Map.get(tree.nodes, id)) end)
51 +
52 + {html_node.type, html_node.attributes, children}
53 + end
54 +
55 + defp do_delete(tree, [], []), do: tree
56 + defp do_delete(tree, [html_node | t], stack_ids) do
57 + new_tree_nodes = delete_node_from_nodes(tree.nodes, html_node)
58 +
59 + ids_for_stack = get_ids_for_delete_stack(html_node)
60 +
61 + do_delete(%{tree | nodes: new_tree_nodes,
62 + node_ids: List.delete(tree.node_ids, html_node.node_id),
63 + root_nodes_ids: List.delete(tree.root_nodes_ids, html_node.node_id)},
64 + t, ids_for_stack ++ stack_ids)
65 + end
66 + defp do_delete(tree, [], stack_ids) do
67 + html_nodes = tree.nodes
68 + |> Map.take(stack_ids)
69 + |> Map.values
70 +
71 + do_delete(tree, html_nodes, [])
72 + end
73 +
74 + defp delete_node_from_nodes(nodes, html_node) do
75 + tree_nodes = Map.delete(nodes, html_node.node_id)
76 + parent_node = Map.get(nodes, html_node.parent_node_id)
77 +
78 + if parent_node do
79 + children_ids = List.delete(parent_node.children_nodes_ids, html_node.node_id)
80 + new_parent = %{parent_node | children_nodes_ids: children_ids}
81 + %{tree_nodes | new_parent.node_id => new_parent}
82 + else
83 + tree_nodes
84 + end
85 + end
86 +
87 + defp get_ids_for_delete_stack(%HTMLNode{children_nodes_ids: ids}), do: ids
88 + defp get_ids_for_delete_stack(_), do: []
89 +
41 90 defp build_tree(tree, [], _, []), do: tree
42 91 defp build_tree(tree, [{tag, attrs, child_children} | children], parent_id, stack) do
43 92 new_id = IDSeeder.seed(tree.node_ids)
  @@ -51,7 +100,15 @@ defmodule Floki.HTMLTree do
51 100 %{tree | nodes: nodes, node_ids: [new_id | tree.node_ids]}
52 101 |> build_tree(child_children, new_id, [{parent_id, children} | stack])
53 102 end
103 + defp build_tree(tree, [{:comment, comment} | children], parent_id, stack) do
104 + new_id = IDSeeder.seed(tree.node_ids)
105 + new_node = %Comment{content: comment, node_id: new_id, parent_node_id: parent_id}
54 106
107 + nodes = put_new_node(tree.nodes, new_node)
108 +
109 + %{tree | nodes: nodes, node_ids: [new_id | tree.node_ids]}
110 + |> build_tree(children, parent_id, stack)
111 + end
55 112 defp build_tree(tree, [text | children], parent_id, stack) when is_binary(text) do
56 113 new_id = IDSeeder.seed(tree.node_ids)
57 114 new_node = %Text{content: text, node_id: new_id, parent_node_id: parent_id}
Loading more files…