Packages
electric
0.7.1
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.6.2
1.6.1
1.6.0
1.5.1
1.5.0
1.4.16
1.4.16-beta-1
1.4.15
1.4.14
1.4.13
1.4.12
1.4.11
1.4.10
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.4
1.3.3
1.3.2
1.2.4
1.2.3
1.2.2
1.2.1
1.2.0
1.1.14
1.1.13
1.1.12
1.1.11
1.1.10
1.1.9
1.1.8
1.1.7
1.1.6
retired
1.1.5
retired
1.1.4
retired
1.1.3
retired
1.1.2
1.1.1
1.1.0
1.0.24
1.0.23
1.0.22
1.0.21
1.0.20
1.0.19
1.0.18
1.0.17
1.0.15
1.0.13
1.0.12
1.0.11
1.0.10
1.0.9
1.0.5
1.0.4
1.0.3
1.0.2
1.0.1
1.0.0
1.0.0-beta.23
1.0.0-beta.22
1.0.0-beta.20
1.0.0-beta.19
1.0.0-beta.18
1.0.0-beta.17
1.0.0-beta.16
1.0.0-beta.15
1.0.0-beta.14
1.0.0-beta.13
1.0.0-beta.12
1.0.0-beta.11
1.0.0-beta.10
1.0.0-beta.9
1.0.0-beta.8
1.0.0-beta.7
1.0.0-beta.6
1.0.0-beta.5
1.0.0-beta.4
1.0.0-beta.3
1.0.0-beta.2
1.0.0-beta.1
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.1
0.8.0
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.3
0.6.2
0.6.1
0.5.2
0.4.4
Postgres sync engine. Sync little subsets of your Postgres data into local apps and services.
Current section
Files
Jump to
Current section
Files
lib/electric/postgres/replication_client/connection_setup.ex
defmodule Electric.Postgres.ReplicationClient.ConnectionSetup do
@moduledoc """
This module encapsulates the initial setup of a replication connection opened by
`Electric.Postgres.ReplicationClient`.
A state machine is implemented to run a series of SQL queries prior to switching the
connection into the logical streaming mode. This helps keep the main `ReplicationClient`
module focused on the handling of logical messages.
"""
alias Electric.Utils
require Logger
@type state :: Electric.Postgres.ReplicationClient.State.t()
@type step :: Electric.Postgres.ReplicationClient.step()
@type callback_return ::
{:noreply, state}
| {:query, iodata, state}
| {:stream, iodata, Postgrex.ReplicationConnection.stream_opts(), state}
@type query_result :: [Postgrex.Result.t()] | Postgrex.Error.t()
# The entrypoint to the connection setup that picks the first step to run and returns the
# `{:query, ...}` tuple for it.
@spec start(state) :: callback_return
def start(%{step: :connected} = state) do
next_step = next_step(state)
query_for_step(next_step, %{state | step: next_step})
end
# Process the result of executing the query, pick the next step and return the `{:query, ...}`
# tuple for it.
@spec process_query_result(query_result, state) :: callback_return
def process_query_result(result, %{step: step} = state) do
state = dispatch_query_result(step, result, state)
next_step = next_step(state)
query_for_step(next_step, %{state | step: next_step})
end
# Instruct `Postgrex.ReplicationConnection` to switch the connection into the logical
# streaming mode.
@spec start_streaming(state) :: callback_return
def start_streaming(%{step: :ready_to_stream} = state) do
query_for_step(:streaming, %{state | step: :streaming})
end
###
defp create_publication_query(state) do
# We're creating an "empty" publication because first snapshot creation should add the table
query = "CREATE PUBLICATION #{Utils.quote_name(state.publication_name)}"
{:query, query, state}
end
# Successfully created the publication.
defp create_publication_result([%Postgrex.Result{}], state) do
state
end
defp create_publication_result(%Postgrex.Error{} = error, state) do
error_message = "publication \"#{state.publication_name}\" already exists"
case error.postgres do
%{code: :duplicate_object, pg_code: "42710", message: ^error_message} ->
# Publication already exists, proceed to the next step.
state
_ ->
# Unexpected error, fail loudly.
raise error
end
end
###
defp create_slot_query(state) do
query =
"CREATE_REPLICATION_SLOT #{Utils.quote_name(state.slot_name)} LOGICAL pgoutput NOEXPORT_SNAPSHOT"
{:query, query, state}
end
# Sucessfully created the replication slot.
defp create_slot_result([%Postgrex.Result{} = result], state) do
log_slot_creation_result(result)
state
end
defp create_slot_result(%Postgrex.Error{} = error, state) do
error_msg = "replication slot \"#{state.slot_name}\" already exists"
case error.postgres do
%{code: :duplicate_object, pg_code: "42710", message: ^error_msg} ->
# Slot already exists, proceed to the next step.
Logger.debug("Found existing replication slot")
state
_ ->
# Unexpected error, fail loudly.
raise error
end
end
defp log_slot_creation_result(result) do
Logger.debug(fn ->
%Postgrex.Result{
command: :create,
columns: ["slot_name", "consistent_point", "snapshot_name", "output_plugin"],
rows: [[_, lsn_str, nil, _]],
num_rows: 1,
connection_id: _,
messages: []
} = result
"Created new slot at lsn=#{lsn_str}"
end)
end
###
defp set_display_setting_query(%{display_settings: [query | rest]} = state) do
{:query, query, %{state | display_settings: rest}}
end
defp set_display_setting_result([%Postgrex.Result{}], state) do
state
end
defp set_display_setting_result(%Postgrex.Error{} = error, _state) do
# Unexpected error, fail loudly.
raise error
end
###
# This is one of the two terminal states of our state machine that puts the process into an
# idling state until it receives a `:start_streaming` message.
#
# The other terminal state is implemented in `start_replication_slot_query/1`.
defp ready_to_stream(state) do
{:noreply, state}
end
###
# After Postgres executes the `START_REPLICATION` command, it will switch to the logical
# streaming mode. That's why this function must return a `{:stream, ...}` and no queries
# will be executed after this.
defp start_replication_slot_query(state) do
query =
"START_REPLICATION SLOT #{Utils.quote_name(state.slot_name)} LOGICAL 0/0 (proto_version '1', publication_names '#{state.publication_name}')"
Logger.info("Starting replication from postgres")
{:stream, query, [], state}
end
### Below you'll find the boilerplate needed to put the state machine together.
# This function defines the transition table for our ad-hoc state machine that determines which
# step leads to which next one.
#
# This is how we order the queries to be executed prior to switching into the logical streaming mode.
@spec next_step(state) :: step
defp next_step(%{step: :connected, try_creating_publication?: true}), do: :create_publication
defp next_step(%{step: :connected}), do: :create_slot
defp next_step(%{step: :create_publication}), do: :create_slot
defp next_step(%{step: :create_slot}), do: :set_display_setting
defp next_step(%{step: :set_display_setting, display_settings: queries}) when queries != [],
do: :set_display_setting
defp next_step(%{step: :set_display_setting, start_streaming?: true}), do: :streaming
defp next_step(%{step: :set_display_setting}), do: :ready_to_stream
###
# Helper function that dispatches each step to a function specific to it. This is done so
# that query and result processing functions for the same step can be grouped together in
# this module.
@spec query_for_step(step, state) :: callback_return
defp query_for_step(:create_publication, state), do: create_publication_query(state)
defp query_for_step(:create_slot, state), do: create_slot_query(state)
defp query_for_step(:set_display_setting, state), do: set_display_setting_query(state)
defp query_for_step(:ready_to_stream, state), do: ready_to_stream(state)
defp query_for_step(:streaming, state), do: start_replication_slot_query(state)
###
# Helper function that dispatches processing of a query result to a function specific to
# that query's step. This is again done to facilitate grouping functions for the same step.
@spec dispatch_query_result(step, query_result, state) :: state | no_return
defp dispatch_query_result(:create_publication, result, state),
do: create_publication_result(result, state)
defp dispatch_query_result(:create_slot, result, state),
do: create_slot_result(result, state)
defp dispatch_query_result(:set_display_setting, result, state),
do: set_display_setting_result(result, state)
end