Packages

scrape

1.0.4
3.1.0 3.0.3 3.0.2 3.0.1 3.0.0 2.0.0 1.2.7 1.2.6 1.2.5 1.2.4 1.2.3 1.2.2 1.2.1 1.0.4 1.0.3 1.0.2 1.0.1 1.0.0 0.2.0 0.1.1 0.1.0 0.0.4 0.0.3 0.0.2 0.0.1

Scrape any website, article or RSS/Atom feed with ease!

HexDocs

Current section

Files

Jump to
Readme
25 Versions
5 Dependencies
Dependants
Files
Activity
scrape lib scrape fetch.ex
Raw

lib/scrape/fetch.ex 

defmodule Scrape.Fetch do



  @opts [

    follow_redirect: true,

    timeout: 33_000,

    recv_timeout: 30_000

  ]



  def run(url) do

    url

    |> HTTPoison.get([], @opts)

    |> evaluate

  end



  defp evaluate({:error, _}), do: ""

  defp evaluate({:ok, response}) do

    cs = charset(response.headers)

    if cs do

      encoding = cs

      |> String.replace("-","_")

      |> String.downcase

      |> to_char_list

      |> List.to_atom

      {_status, result} = Codepagex.to_string response.body, encoding

      result

    else

      response.body

    end

  end



  defp charset(headers) do

    header = headers

    |> Enum.filter(fn({k,_}) -> k == "Content-Type" end)

    |> first

    if header do

      {_name, content} = header

      ~r/charset=(ISO-8859-[1-9])/i

      |> Regex.run(content, capture: :all_but_first)

      |> first

    else

      nil

    end

  end



  defp first([h|_]), do: h

  defp first(_), do: nil

end