Packages

spider_man

0.3.1
0.6.4 0.6.3 0.6.2 0.6.1 retired 0.6.0 0.5.1 0.5.0 0.4.7 0.4.6 0.4.5 0.4.4 0.4.3 0.4.2 0.4.1 0.4.0 0.3.6 0.3.5 0.3.4 0.3.2 0.3.1 0.3.0 0.2.3 0.2.2 0.2.1 0.2.0 0.1.2 0.1.1 0.1.0

SpiderMan,a fast high-level web crawling & scraping framework for Elixir

HexDocs

Current section

Files

Jump to
Readme
28 Versions
9 Dependencies
Dependants
Files
Activity
spider_man lib spider_man requester hackney.ex
Raw

lib/spider_man/requester/hackney.ex 

defmodule SpiderMan.Requester.Hackney do

  @moduledoc false

  alias SpiderMan.Requester

  @behaviour Requester



  @default_options [

    adapter_options: [recv_timeout: 10_000],

    request_options: [],

    append_default_middlewares?: true,

    middlewares: [],

    logging?: false

  ]



  @impl true

  def request(url, options, %{request_options: request_options, client: client}) do

    options = Map.merge(request_options, options)

    Tesla.request(client, [{:url, url} | options])

  end



  @impl true

  def prepare_for_start(options, downloader_options) do

    options = Keyword.merge(@default_options, options || [])



    client =

      options[:append_default_middlewares?]

      |> Requester.append_default_middlewares(options)

      |> Tesla.client({Tesla.Adapter.Hackney, options[:adapter_options]})



    context = %{

      requester: __MODULE__,

      request_options: options[:request_options],

      client: client

    }



    Keyword.update(downloader_options, :context, context, &Map.merge(&1, context))

  end

end