Packages

A dead-simple, concurrent web crawler which focuses on ease of use and speed.

Retired package: I was just trying out hex features - package wasn't ready :)

Current section

Files

Jump to
spidey lib spidey storage queue.ex
Raw

lib/spidey/storage/queue.ex

defmodule Spidey.Storage.Queue do
use Agent
def start_link(urls) do
queue = :queue.from_list(urls)
Agent.start_link(fn -> queue end, name: __MODULE__)
end
def pop do
queue = Agent.get(__MODULE__, & &1)
{value, queue} = pop_value(queue)
Agent.update(__MODULE__, fn _ -> queue end)
value
end
def take(n) do
queue = Agent.get(__MODULE__, & &1)
{queue, elems} = pop_multiple(queue, n)
Agent.update(__MODULE__, fn _ -> queue end)
elems
end
def push(url) do
Agent.update(__MODULE__, &:queue.in(url, &1))
end
defp pop_value(queue) do
case :queue.out(queue) do
{{:value, value}, queue} -> {value, queue}
{:empty, queue} -> {:empty, queue}
end
end
defp pop_multiple(queue, n, elems \\ [])
defp pop_multiple(queue, 0, elems), do: {queue, elems}
defp pop_multiple(queue, n, elems) do
case :queue.out(queue) do
{{:value, value}, queue} -> pop_multiple(queue, n - 1, [value | elems])
{:empty, queue} -> {queue, elems}
end
end
def length do
queue = Agent.get(__MODULE__, & &1)
:queue.len(queue)
end
end