Packages

spider_man

0.4.4
0.6.4 0.6.3 0.6.2 0.6.1 retired 0.6.0 0.5.1 0.5.0 0.4.7 0.4.6 0.4.5 0.4.4 0.4.3 0.4.2 0.4.1 0.4.0 0.3.6 0.3.5 0.3.4 0.3.2 0.3.1 0.3.0 0.2.3 0.2.2 0.2.1 0.2.0 0.1.2 0.1.1 0.1.0

SpiderMan,a fast high-level web crawling & scraping framework for Elixir

HexDocs

Current section

Files

Jump to
Readme
28 Versions
9 Dependencies
Dependants
Files
Activity
spider_man lib spider_man middleware user_agent.ex
Raw

lib/spider_man/middleware/user_agent.ex 

defmodule SpiderMan.Middleware.UserAgent do

  @moduledoc """

  Setting user-agent for request



  ## Usage



      settings = [

        downloader_options: [requester: {SpiderMan.Requester.Finch, [

          middlewares: [

          {#{inspect(__MODULE__)},

           [

             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36",

             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",

             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",

             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4389.82 Safari/537.36",

             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4389.82 Safari/537.36",

             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"

           ]}],

           ...

        ]}]

      ]

  """

  @behaviour Tesla.Middleware



  @impl true

  def call(env, next, user_agents) do

    user_agent = Enum.random(user_agents)



    env

    |> Tesla.put_headers([{"user-agent", user_agent}])

    |> Tesla.run(next)

  end

end