Current section
Files
Jump to
Current section
Files
lib/exalice/stream_runner.ex
defmodule ExAlice.StreamRunner do
use GenServer
defstruct workers: %{}, cont: nil, capacity: 4, fun: nil, reply_to: nil
@moduledoc """
Concurrently work through a stream.
See the docs for run/3 for usage
"""
def start_link(capacity, opts) do
GenServer.start_link(__MODULE__, capacity, opts)
end
@doc """
Run a stream with a given function.
The specified GenServer process will successively go through a stream, handing
each item off to a worker, which executes the function on the item.
It cannot process more than one stream at a time, so calling run/3 again prior
to its completion will abandon the prior stream and associated workers.
To wait until the stream is done, use await/2
"""
def run(name, stream, fun) do
GenServer.call(name, {:run, stream, fun})
end
@doc """
Will block until the stream finishes, or the timeout is reached
"""
def await(name, timeout \\ :infinity) do
GenServer.call(name, :await, timeout)
end
def init(capacity) do
{:ok, %__MODULE__{capacity: capacity}}
end
def handle_call({:run, stream, fun}, _, state) do
# This places the stream in a suspended state
# where we can choose to take additional items from it or not.
# We'll take $capacity from it initially, and then one after each
# task finishes.
#
# Ideally each task gets a big enough chunk that the task starting / stopping isn't
# bottleneck.
{:suspended, nil, cont} = Enumerable.reduce(stream, {:suspend, nil}, fn(v, _) -> {:suspend, v} end)
state = start_workers(%{state | fun: fun, cont: cont, workers: %{}})
{:reply, :started, state}
end
def handle_call(:await, from, state) do
{:noreply, %{state | reply_to: from}}
end
def handle_info({:DOWN, ref, _, proc, reason}, %{workers: workers} = state) do
state = case Map.has_key?(workers, ref) do
true ->
start_workers(%{state | workers: Map.delete(workers, ref)})
false ->
state
end
{:noreply, state}
end
def start_workers(%{cont: nil} = state), do: state |> reply_to
def start_workers(%{workers: workers, capacity: capacity, cont: cont, fun: fun} = state) when map_size(workers) < capacity do
case cont.({:cont, nil}) do
{:suspended, item, cont} ->
{:ok, pid} = Task.start_link(fn -> fun.(item) end)
ref = Process.monitor(pid)
workers = Map.put(workers, ref, pid)
start_workers %{state | workers: workers, cont: cont}
_ ->
%{state | cont: nil, fun: nil}
end
end
def start_workers(state), do: state
def reply_to(%{reply_to: nil} = state), do: state
def reply_to(%{reply_to: pid} = state) do
GenServer.reply(pid, :done)
state
end
end