Packages

crawly

0.10.0
0.17.2 0.17.1 0.17.0 0.16.0 0.15.0 0.14.0 0.13.0 0.12.0 0.11.0 0.10.0 0.9.0 0.8.0 0.7.0 0.6.0 0.5.0 0.4.0 0.3.0 0.2.0 0.1.2 0.1.1 0.1.0

High-level web crawling & scraping framework for Elixir.

HexDocs

Current section

Files

Jump to
Readme
21 Versions
6 Dependencies
Dependants
Files
Activity
crawly lib crawly middlewares unique_request.ex
Raw

lib/crawly/middlewares/unique_request.ex 

defmodule Crawly.Middlewares.UniqueRequest do

  @moduledoc """

  Avoid scheduling requests for the same pages.

  """

  require Logger



  def run(request, state) do

    unique_request_seen_requests =

      Map.get(state, :unique_request_seen_requests, %{})



    case Map.get(unique_request_seen_requests, request.url) do

      nil ->

        unique_request_seen_requests =

          Map.put(unique_request_seen_requests, request.url, true)



        new_state =

          Map.put(

            state,

            :unique_request_seen_requests,

            unique_request_seen_requests

          )



        {request, new_state}



      _ ->

        Logger.debug(

          "Dropping request: #{request.url}, as it's already processed"

        )



        {false, state}

    end

  end

end