Packages

manga_ex

0.4.7
0.5.0 0.4.10 0.4.9 0.4.8 0.4.7 0.4.6 0.4.5 0.4.4 0.4.3 0.4.2 0.4.1 0.3.3 0.3.2 0.3.1 0.2.7 0.2.6 0.2.5 0.2.4 0.2.3 0.2.2 0.2.1 0.1.13 0.1.12 0.1.11 0.1.10 0.1.9 0.1.8 0.1.7 0.1.6 0.1.5 0.1.4 0.1.3 0.1.2 0.1.1 0.1.0

Scrapper that download chapters whitout limits from some providers

HexDocs

Current section

Files

Jump to
Readme
35 Versions
3 Dependencies
Dependants
Files
Activity
manga_ex lib http_client curl.ex
Raw

lib/http_client/curl.ex 

defmodule MangaEx.HttpClient.Curl do

  require Logger



  def get_curl(url, opts \\ []) do

    args =

      opts ++

        [

          "-H",

          "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36",

          "-s"

        ]



    "curl"

    |> System.cmd(args ++ [URI.encode(url)], [])

    |> case do

      {body, _} when is_binary(body) ->

        body



      {error, _} when is_binary(error) ->

        :timer.sleep(:timer.seconds(1))

        get_curl(url)



      errors ->

        handle_errors(errors)

    end

  end



  def curl_expected_errors do

    [

      "error code: 1007",

      "<html>\r\n<head><title>403 Forbidden</title></head>\r\n<body>\r\n<center><h1>403 Forbidden</h1></center>\r\n<hr><center>nginx</center>\r\n</body>\r\n</html>\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n"

    ]

  end



  defp handle_errors(errors) do

    case errors do

      {:ok, %{status: status}} when status in 400..499 ->

        {:error, :client_error}



      {:ok, %{status: status}} when status in 500..599 ->

        {:error, :server_error}



      error ->

        Logger.error("unexpected error")

        {:error, error}

    end

  end

end