Packages

microdata

0.2.1
0.2.4 0.2.3 0.2.2 0.2.1 0.2.0 0.1.9 0.1.8 0.1.7 0.1.6 0.1.5 0.1.3 0.1.2 0.1.1 0.1.0 0.0.1

Microdata is an Elixir library for parsing microdata from a provided document. Includes helper functions, eg for looking up typed items in a document, and named properties in an item.

HexDocs

Current section

15 Versions

Jump to
Readme
15 Versions
2 Dependencies
0 Dependants
Files
Activity
Hide whitespace

Compare versions

6 files changed
+37 additions
-26 deletions
  @@ -26,4 +26,4 @@
26 26 {<<"optional">>,false},
27 27 {<<"repository">>,<<"hexpm">>},
28 28 {<<"requirement">>,<<"~> 0.11.0">>}]]}.
29 - {<<"version">>,<<"0.2.0">>}.
29 + {<<"version">>,<<"0.2.1">>}.
  @@ -119,17 +119,19 @@ defmodule Microdata do
119 119 @spec parse(file: String.t()) :: {:ok, Document.t()} | {:error, Error.t()}
120 120 @spec parse(url: String.t()) :: {:ok, Document.t()} | {:error, Error.t()}
121 121 @spec parse(String.t()) :: {:ok, Document.t()} | {:error, Error.t()}
122 + @spec parse(String.t(), base_uri: String.t()) :: {:ok, Document.t()} | {:error, Error.t()}
122 123 # credo:disable-for-next-line Credo.Check.Refactor.PipeChainStart
123 - def parse(file: path), do: File.read!(path) |> parse
124 + def parse(file: path), do: File.read!(path) |> parse(base_uri: path)
124 125
125 126 # credo:disable-for-next-line Credo.Check.Refactor.PipeChainStart
126 - def parse(url: url), do: HTTPoison.get!(url).body |> parse
127 + def parse(url: url), do: HTTPoison.get!(url).body |> parse(base_uri: url)
128 + def parse(html), do: parse(html, base_uri: nil)
127 129
128 - def parse(html) do
130 + def parse(html, base_uri: base_uri) do
129 131 doc = html |> Meeseeks.parse()
130 132
131 133 strategies()
132 - |> Enum.flat_map(& &1.parse_items(doc))
134 + |> Enum.flat_map(& &1.parse_items(doc, base_uri))
133 135 |> case do
134 136 items when items != [] ->
135 137 {:ok, %Document{items: items}}
  @@ -6,4 +6,5 @@ defmodule Microdata.Strategy do
6 6 """
7 7
8 8 @callback parse_items(Meeseeks.Document.t()) :: [Microdata.Item.t()]
9 + @callback parse_items(Meeseeks.Document.t(), String.t()) :: [Microdata.Item.t()]
9 10 end
  @@ -19,11 +19,11 @@ defmodule Microdata.Strategy.HTMLMicrodata do
19 19 @tags_datetime ~w(datetime)
20 20
21 21 @impl true
22 - def parse_items(doc) do
23 - parse_items(doc, 0)
22 + def parse_items(doc, base_uri \\ nil) do
23 + parse_items(doc, 0, base_uri)
24 24 end
25 25
26 - defp parse_items(doc, nest_level, items \\ []) do
26 + defp parse_items(doc, nest_level, base_uri, items \\ []) do
27 27 selector =
28 28 if nest_level == 0 do
29 29 "/*[@itemscope]|//*[@itemscope and not(@itemprop) and count(ancestor::*[@itemscope]) = 0]"
  @@ -33,17 +33,17 @@ defmodule Microdata.Strategy.HTMLMicrodata do
33 33
34 34 doc
35 35 |> Meeseeks.all(xpath(selector))
36 - |> Enum.map(&parse_item(&1, nest_level))
36 + |> Enum.map(&parse_item(&1, nest_level, base_uri))
37 37 |> case do
38 38 new_items when new_items != [] ->
39 - parse_items(doc, nest_level + 1, new_items ++ items)
39 + parse_items(doc, nest_level + 1, base_uri, new_items ++ items)
40 40
41 41 _ ->
42 42 items
43 43 end
44 44 end
45 45
46 - defp parse_item(item, nest_level) do
46 + defp parse_item(item, nest_level, base_uri) do
47 47 item_model = %Item{
48 48 id: item |> Meeseeks.attr("itemid") |> Helpers.parse_item_id(),
49 49 types:
  @@ -53,21 +53,21 @@ defmodule Microdata.Strategy.HTMLMicrodata do
53 53 |> MapSet.new()
54 54 }
55 55
56 - %{item_model | properties: parse_properties(item, item_model, nest_level)}
56 + %{item_model | properties: parse_properties(item, item_model, nest_level, base_uri)}
57 57 end
58 58
59 - defp parse_properties(item, item_model, nest_level) do
59 + defp parse_properties(item, item_model, nest_level, base_uri) do
60 60 selector = ".//*[@itemprop and count(ancestor::*[@itemscope]) = #{nest_level + 1}]"
61 61
62 62 item
63 63 |> Meeseeks.all(xpath(selector))
64 - |> Enum.map(fn prop -> parse_property(prop, item_model, nest_level) end)
64 + |> Enum.map(fn prop -> parse_property(prop, item_model, nest_level, base_uri) end)
65 65 end
66 66
67 - defp parse_property(property, item, nest_level) do
67 + defp parse_property(property, item, nest_level, base_uri) do
68 68 %Property{
69 69 names: property |> parse_property_names(item) |> MapSet.new(),
70 - value: parse_property_value(property, nest_level),
70 + value: parse_property_value(property, nest_level, base_uri),
71 71 html: Meeseeks.html(property)
72 72 }
73 73 end
  @@ -79,29 +79,29 @@ defmodule Microdata.Strategy.HTMLMicrodata do
79 79 end
80 80
81 81 # credo:disable-for-lines:35 Credo.Check.Refactor.CyclomaticComplexity
82 - defp parse_property_value(property, nest_level) do
82 + defp parse_property_value(property, nest_level, base_uri) do
83 83 tag = Meeseeks.tag(property)
84 84 itemscope = Meeseeks.attr(property, "itemscope")
85 85 content = Meeseeks.attr(property, "content")
86 86
87 87 cond do
88 88 itemscope != nil ->
89 - parse_item(property, nest_level + 1)
89 + parse_item(property, nest_level + 1, base_uri)
90 90
91 91 content != nil ->
92 92 content
93 93
94 94 Enum.member?(@tags_src, tag) ->
95 - url = Meeseeks.attr(property, "src")
96 - if Helpers.absolute_url?(url), do: url, else: ""
95 + Meeseeks.attr(property, "src")
96 + |> parse_property_uri(base_uri)
97 97
98 98 Enum.member?(@tags_href, tag) ->
99 - url = Meeseeks.attr(property, "href")
100 - if Helpers.absolute_url?(url), do: url, else: ""
99 + Meeseeks.attr(property, "href")
100 + |> parse_property_uri(base_uri)
101 101
102 102 Enum.member?(@tags_data, tag) ->
103 - url = Meeseeks.attr(property, "data")
104 - if Helpers.absolute_url?(url), do: url, else: ""
103 + Meeseeks.attr(property, "data")
104 + |> parse_property_uri(base_uri)
105 105
106 106 Enum.member?(@tags_value, tag) ->
107 107 value = Meeseeks.attr(property, "value")
  @@ -115,4 +115,12 @@ defmodule Microdata.Strategy.HTMLMicrodata do
115 115 Meeseeks.text(property)
116 116 end
117 117 end
118 +
119 + defp parse_property_uri(uri, base) do
120 + cond do
121 + Helpers.absolute_url?(uri) -> uri
122 + Helpers.absolute_url?(base) -> URI.merge(base, uri) |> URI.to_string()
123 + true -> ""
124 + end
125 + end
118 126 end
  @@ -13,7 +13,7 @@ defmodule Microdata.Strategy.JSONLD do
13 13 alias Microdata.{Item, Property}
14 14
15 15 @impl true
16 - def parse_items(doc) do
16 + def parse_items(doc, _base_uri \\ nil) do
17 17 doc
18 18 |> Meeseeks.all(xpath("//script[@type=\"application/ld+json\"]"))
19 19 |> Enum.map(&parse_result(&1))
Loading more files…