Current section

Files

Jump to
electric lib electric postgres replication_client connection_setup.ex
Raw

lib/electric/postgres/replication_client/connection_setup.ex

defmodule Electric.Postgres.ReplicationClient.ConnectionSetup do
@moduledoc """
This module encapsulates the initial setup of a replication connection opened by
`Electric.Postgres.ReplicationClient`.
A state machine is implemented to run a series of SQL queries prior to switching the
connection into the logical streaming mode. This helps keep the main `ReplicationClient`
module focused on the handling of logical messages.
"""
alias Electric.Utils
require Logger
@type state :: Electric.Postgres.ReplicationClient.State.t()
@type step :: Electric.Postgres.ReplicationClient.step()
@type callback_return ::
{:noreply, state}
| {:query, iodata, state}
| {:stream, iodata, Postgrex.ReplicationConnection.stream_opts(), state}
@type query_result :: [Postgrex.Result.t()] | Postgrex.Error.t()
# The entrypoint to the connection setup that picks the first step to run and returns the
# `{:query, ...}` tuple for it.
@spec start(state) :: callback_return
def start(%{step: :connected} = state) do
next_step = next_step(state)
query_for_step(next_step, %{state | step: next_step})
end
# Process the result of executing the query, pick the next step and return the `{:query, ...}`
# tuple for it.
@spec process_query_result(query_result, state) :: callback_return
def process_query_result(result, %{step: step} = state) do
state = dispatch_query_result(step, result, state)
next_step = next_step(state)
query_for_step(next_step, %{state | step: next_step})
end
# Instruct `Postgrex.ReplicationConnection` to switch the connection into the logical
# streaming mode.
@spec start_streaming(state) :: callback_return
def start_streaming(%{step: :ready_to_stream} = state) do
query_for_step(:streaming, %{state | step: :streaming})
end
###
defp create_publication_query(state) do
# We're creating an "empty" publication because first snapshot creation should add the table
query = "CREATE PUBLICATION #{Utils.quote_name(state.publication_name)}"
{:query, query, state}
end
# Successfully created the publication.
defp create_publication_result([%Postgrex.Result{}], state) do
state
end
defp create_publication_result(%Postgrex.Error{} = error, state) do
error_message = "publication \"#{state.publication_name}\" already exists"
case error.postgres do
%{code: :duplicate_object, pg_code: "42710", message: ^error_message} ->
# Publication already exists, proceed to the next step.
state
_ ->
# Unexpected error, fail loudly.
raise error
end
end
###
defp create_slot_query(state) do
query =
"CREATE_REPLICATION_SLOT #{Utils.quote_name(state.slot_name)} LOGICAL pgoutput NOEXPORT_SNAPSHOT"
{:query, query, state}
end
# Sucessfully created the replication slot.
defp create_slot_result([%Postgrex.Result{} = result], state) do
log_slot_creation_result(result)
state
end
defp create_slot_result(%Postgrex.Error{} = error, state) do
error_msg = "replication slot \"#{state.slot_name}\" already exists"
case error.postgres do
%{code: :duplicate_object, pg_code: "42710", message: ^error_msg} ->
# Slot already exists, proceed to the next step.
Logger.debug("Found existing replication slot")
state
_ ->
# Unexpected error, fail loudly.
raise error
end
end
defp log_slot_creation_result(result) do
Logger.debug(fn ->
%Postgrex.Result{
command: :create,
columns: ["slot_name", "consistent_point", "snapshot_name", "output_plugin"],
rows: [[_, lsn_str, nil, _]],
num_rows: 1,
connection_id: _,
messages: []
} = result
"Created new slot at lsn=#{lsn_str}"
end)
end
###
defp set_display_setting_query(%{display_settings: [query | rest]} = state) do
{:query, query, %{state | display_settings: rest}}
end
defp set_display_setting_result([%Postgrex.Result{}], state) do
state
end
defp set_display_setting_result(%Postgrex.Error{} = error, _state) do
# Unexpected error, fail loudly.
raise error
end
###
# This is one of the two terminal states of our state machine that puts the process into an
# idling state until it receives a `:start_streaming` message.
#
# The other terminal state is implemented in `start_replication_slot_query/1`.
defp ready_to_stream(state) do
{:noreply, state}
end
###
# After Postgres executes the `START_REPLICATION` command, it will switch to the logical
# streaming mode. That's why this function must return a `{:stream, ...}` and no queries
# will be executed after this.
defp start_replication_slot_query(state) do
query =
"START_REPLICATION SLOT #{Utils.quote_name(state.slot_name)} LOGICAL 0/0 (proto_version '1', publication_names '#{state.publication_name}')"
Logger.info("Starting replication from postgres")
{:stream, query, [], state}
end
### Below you'll find the boilerplate needed to put the state machine together.
# This function defines the transition table for our ad-hoc state machine that determines which
# step leads to which next one.
#
# This is how we order the queries to be executed prior to switching into the logical streaming mode.
@spec next_step(state) :: step
defp next_step(%{step: :connected, try_creating_publication?: true}), do: :create_publication
defp next_step(%{step: :connected}), do: :create_slot
defp next_step(%{step: :create_publication}), do: :create_slot
defp next_step(%{step: :create_slot}), do: :set_display_setting
defp next_step(%{step: :set_display_setting, display_settings: queries}) when queries != [],
do: :set_display_setting
defp next_step(%{step: :set_display_setting, start_streaming?: true}), do: :streaming
defp next_step(%{step: :set_display_setting}), do: :ready_to_stream
###
# Helper function that dispatches each step to a function specific to it. This is done so
# that query and result processing functions for the same step can be grouped together in
# this module.
@spec query_for_step(step, state) :: callback_return
defp query_for_step(:create_publication, state), do: create_publication_query(state)
defp query_for_step(:create_slot, state), do: create_slot_query(state)
defp query_for_step(:set_display_setting, state), do: set_display_setting_query(state)
defp query_for_step(:ready_to_stream, state), do: ready_to_stream(state)
defp query_for_step(:streaming, state), do: start_replication_slot_query(state)
###
# Helper function that dispatches processing of a query result to a function specific to
# that query's step. This is again done to facilitate grouping functions for the same step.
@spec dispatch_query_result(step, query_result, state) :: state | no_return
defp dispatch_query_result(:create_publication, result, state),
do: create_publication_result(result, state)
defp dispatch_query_result(:create_slot, result, state),
do: create_slot_result(result, state)
defp dispatch_query_result(:set_display_setting, result, state),
do: set_display_setting_result(result, state)
end