funkspector

defmodule Funkspector.PageScraper do
  @moduledoc """
  Provides a method to scrape a page, given its URL.
  """

  import Funkspector, only: [default_options: 0]
  import Funkspector.Utils
  alias Funkspector.Resolver

  @doc """
  Fetches the given URL, follows redirections, and returns the data scraped from its HTML.

  ## Examples

      iex> { :ok, data } = Funkspector.PageScraper.scrape("http://jaimeiniesta.com")
      iex> data.scheme
      "http"
      iex> data.host
      "jaimeiniesta.com"
      iex> data.root_url
      "http://jaimeiniesta.com/"
      iex> data.links.http.internal
      ["http://jaimeiniesta.com/",
       "http://jaimeiniesta.com/about/",
       "http://jaimeiniesta.com/archive/",
       "http://jaimeiniesta.com/portfolio/",
       "http://jaimeiniesta.com",
       "http://jaimeiniesta.com/articles/questions-about-getting-into-freelancing/",
       "http://jaimeiniesta.com/articles/building-a-disqus-recent-comments-widget-with-javascript/",
       "http://jaimeiniesta.com/articles/tips-for-a-new-rails-developer/",
       "http://jaimeiniesta.com/articles/fifteen-servers/",
       "http://jaimeiniesta.com/atom.xml"]
      iex> data.links.http.external
      ["http://jekyllrb.com",
       "http://mademistakes.com/so-simple/",
       "http://twitter.com/jaimeiniesta",
       "http://facebook.com/jaime.iniesta.7",
       "http://plus.google.com/+jaimeiniesta",
       "http://linkedin.com/in/jaimeiniesta",
       "http://instagram.com/jaimeiniesta",
       "http://www.flickr.com/photos/jaimeiniesta", "http://github.com/jaimeiniesta"]
      iex> data.links.non_http
      ["mailto:jaimeiniesta@gmail.com"]

      iex> { :ok, data } = Funkspector.PageScraper.scrape("http://github.com")
      iex> data.original_url
      "http://github.com"
      iex> data.final_url
      "https://github.com/"
  """
  def scrape(original_url, options \\ %{}) do
    options = Map.merge(default_options(), options)

    case Resolver.resolve(original_url, options) do
      {:ok, final_url, response} ->
        handle_response(response, original_url, final_url)

      {_, url, response} ->
        {:error, url, response}
    end
  end

  #####################
  # Private functions #
  #####################

  defp handle_response(response = %{status_code: status, body: _body}, original_url, _final_url)
       when status not in 200..299 do
    {:error, original_url, response}
  end

  defp handle_response(%{status_code: status, body: body}, original_url, final_url)
       when status in 200..299 do
    {:ok, scraped_data(body, original_url, final_url)}
  end

  defp scraped_data(body, original_url, final_url) do
    base_url = base_href(body) || final_url

    %{scheme: scheme, host: host} = URI.parse(final_url)

    root_url = "#{scheme}://#{host}/"
    raw_links = raw_links(body)

    {http_links, non_http_links} =
      raw_links
      |> absolutify(base_url)
      |> http_and_non_http

    {internal_links, external_links} = internal_and_external(http_links, host)

    %{
      scheme: scheme,
      host: host,
      original_url: original_url,
      final_url: final_url,
      root_url: root_url,
      body: body,
      links: %{
        raw: raw_links,
        http: %{
          internal: internal_links,
          external: external_links
        },
        non_http: non_http_links
      }
    }
  end

  defp base_href(html) do
    html
    |> Floki.find("base")
    |> Floki.attribute("href")
    |> List.first()
  end

  defp raw_links(html) do
    html
    |> Floki.find("a")
    |> Floki.attribute("href")
    |> Enum.uniq()
  end

  defp http_and_non_http(links) do
    Enum.split_with(links, &(&1 =~ ~r/^http(s)?:\/\//i))
  end

  defp internal_and_external(links, host) do
    Enum.split_with(links, &(&1 =~ ~r/^http(s)?:\/\/#{host}/i))
  end
end