Current section
28 Versions
Jump to
Current section
28 Versions
Compare versions
5
files changed
+157
additions
-3
deletions
| @@ -1,6 +1,6 @@ | |
| 1 1 | {<<"links">>,[{<<"GitHub">>,<<"https://github.com/pinetops/sandbox_case">>}]}. |
| 2 2 | {<<"name">>,<<"sandbox_case">>}. |
| 3 | - {<<"version">>,<<"0.3.5">>}. |
| 3 | + {<<"version">>,<<"0.3.7">>}. |
| 4 4 | {<<"description">>, |
| 5 5 | <<"Batteries-included test isolation for Elixir and Phoenix.">>}. |
| 6 6 | {<<"elixir">>,<<"~> 1.15">>}. |
| @@ -15,6 +15,7 @@ | |
| 15 15 | <<"lib/sandbox_case/sandbox/cachex_patcher.ex">>, |
| 16 16 | <<"lib/sandbox_case/sandbox/ecto.ex">>, |
| 17 17 | <<"lib/sandbox_case/sandbox/fun_with_flags.ex">>, |
| 18 | + <<"lib/sandbox_case/sandbox/lock_monitor.ex">>, |
| 18 19 | <<"lib/sandbox_case/sandbox/propagator.ex">>, |
| 19 20 | <<"lib/sandbox_case/sandbox/mox.ex">>, |
| 20 21 | <<"lib/sandbox_case/sandbox/mimic.ex">>, |
| @@ -68,6 +68,26 @@ defmodule SandboxCase.Sandbox do | |
| 68 68 | SandboxCase.Sandbox.DeadlockDetector.setup(config) |
| 69 69 | end |
| 70 70 | |
| 71 | + if lm_config = sandbox_config[:lock_monitor] do |
| 72 | + config = if is_list(lm_config), do: lm_config, else: [] |
| 73 | + |
| 74 | + # Auto-discover repo if not specified |
| 75 | + config = |
| 76 | + if config[:repo] do |
| 77 | + config |
| 78 | + else |
| 79 | + otp_app = opts[:otp_app] || Application.get_env(:sandbox_case, :otp_app) |
| 80 | + repos = if otp_app, do: Application.get_env(otp_app, :ecto_repos, []), else: [] |
| 81 | + |
| 82 | + case List.first(repos) do |
| 83 | + nil -> config |
| 84 | + repo -> Keyword.put(config, :repo, repo) |
| 85 | + end |
| 86 | + end |
| 87 | + |
| 88 | + SandboxCase.Sandbox.LockMonitor.start_link(config) |
| 89 | + end |
| 90 | + |
| 71 91 | :ok |
| 72 92 | end |
| @@ -0,0 +1,132 @@ | |
| 1 | + defmodule SandboxCase.Sandbox.LockMonitor do |
| 2 | + @moduledoc """ |
| 3 | + Background monitor that periodically polls Postgres for blocked queries |
| 4 | + and reports the lock chain. |
| 5 | + |
| 6 | + Catches lock contention between async tests as it happens, instead of |
| 7 | + waiting for the ownership timeout (15-120s). |
| 8 | + |
| 9 | + ## Configuration |
| 10 | + |
| 11 | + config :sandbox_case, |
| 12 | + sandbox: [ |
| 13 | + lock_monitor: true |
| 14 | + # or with options: |
| 15 | + lock_monitor: [interval: 2_000, repo: MyApp.Repo] |
| 16 | + ] |
| 17 | + |
| 18 | + ## What it reports |
| 19 | + |
| 20 | + When a query is blocked waiting for a lock, the monitor logs: |
| 21 | + - The blocked query and how long it's been waiting |
| 22 | + - The blocking query and its state |
| 23 | + - Both Postgres PIDs for further investigation |
| 24 | + """ |
| 25 | + use GenServer |
| 26 | + |
| 27 | + require Logger |
| 28 | + |
| 29 | + @default_interval 2_000 |
| 30 | + |
| 31 | + @lock_query """ |
| 32 | + SELECT |
| 33 | + blocked_activity.pid AS blocked_pid, |
| 34 | + blocked_activity.query AS blocked_query, |
| 35 | + blocked_activity.wait_event_type AS wait_type, |
| 36 | + blocked_activity.state AS blocked_state, |
| 37 | + extract(epoch from now() - blocked_activity.query_start)::float AS blocked_seconds, |
| 38 | + blocking_activity.pid AS blocking_pid, |
| 39 | + blocking_activity.query AS blocking_query, |
| 40 | + blocking_activity.state AS blocking_state, |
| 41 | + extract(epoch from now() - blocking_activity.query_start)::float AS blocking_seconds |
| 42 | + FROM pg_stat_activity blocked_activity |
| 43 | + JOIN pg_locks blocked_locks ON blocked_locks.pid = blocked_activity.pid |
| 44 | + JOIN pg_locks blocking_locks |
| 45 | + ON blocking_locks.locktype = blocked_locks.locktype |
| 46 | + AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database |
| 47 | + AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation |
| 48 | + AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page |
| 49 | + AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple |
| 50 | + AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid |
| 51 | + AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid |
| 52 | + AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid |
| 53 | + AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid |
| 54 | + AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid |
| 55 | + AND blocking_locks.pid != blocked_locks.pid |
| 56 | + AND blocking_locks.granted |
| 57 | + JOIN pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid |
| 58 | + WHERE NOT blocked_locks.granted |
| 59 | + """ |
| 60 | + |
| 61 | + def start_link(opts) do |
| 62 | + name = Keyword.get(opts, :name, __MODULE__) |
| 63 | + GenServer.start_link(__MODULE__, opts, name: name) |
| 64 | + end |
| 65 | + |
| 66 | + @impl true |
| 67 | + def init(opts) do |
| 68 | + repo = opts[:repo] |
| 69 | + interval = opts[:interval] || @default_interval |
| 70 | + |
| 71 | + if repo do |
| 72 | + schedule(interval) |
| 73 | + {:ok, %{repo: repo, interval: interval}} |
| 74 | + else |
| 75 | + :ignore |
| 76 | + end |
| 77 | + end |
| 78 | + |
| 79 | + @impl true |
| 80 | + def handle_info(:poll, state) do |
| 81 | + check_for_locks(state.repo) |
| 82 | + schedule(state.interval) |
| 83 | + {:noreply, state} |
| 84 | + end |
| 85 | + |
| 86 | + def handle_info(_, state), do: {:noreply, state} |
| 87 | + |
| 88 | + defp check_for_locks(repo) do |
| 89 | + case repo.query(@lock_query, [], log: false, timeout: 5_000) do |
| 90 | + {:ok, %{rows: rows}} when rows != [] -> |
| 91 | + report_locks(rows) |
| 92 | + |
| 93 | + _ -> |
| 94 | + :ok |
| 95 | + end |
| 96 | + rescue |
| 97 | + _ -> :ok |
| 98 | + end |
| 99 | + |
| 100 | + defp report_locks(rows) do |
| 101 | + details = |
| 102 | + Enum.map_join(rows, "\n\n", fn row -> |
| 103 | + [blocked_pid, blocked_query, wait_type, blocked_state, blocked_secs, |
| 104 | + blocking_pid, blocking_query, blocking_state, blocking_secs] = row |
| 105 | + |
| 106 | + """ |
| 107 | + BLOCKED: PG pid #{blocked_pid} (#{blocked_state}, waiting #{format_secs(blocked_secs)}, #{wait_type}) |
| 108 | + Query: #{truncate(blocked_query)} |
| 109 | + HELD BY: PG pid #{blocking_pid} (#{blocking_state}, #{format_secs(blocking_secs)}) |
| 110 | + Query: #{truncate(blocking_query)}\ |
| 111 | + """ |
| 112 | + end) |
| 113 | + |
| 114 | + Logger.warning(""" |
| 115 | + SandboxCase LockMonitor: #{length(rows)} blocked query(ies) detected: |
| 116 | + |
| 117 | + #{details} |
| 118 | + """) |
| 119 | + end |
| 120 | + |
| 121 | + defp schedule(interval) do |
| 122 | + Process.send_after(self(), :poll, interval) |
| 123 | + end |
| 124 | + |
| 125 | + defp format_secs(nil), do: "?" |
| 126 | + defp format_secs(secs) when is_float(secs), do: "#{Float.round(secs, 1)}s" |
| 127 | + defp format_secs(secs), do: "#{secs}s" |
| 128 | + |
| 129 | + defp truncate(nil), do: "(none)" |
| 130 | + defp truncate(s) when byte_size(s) > 200, do: String.slice(s, 0..197) <> "..." |
| 131 | + defp truncate(s), do: s |
| 132 | + end |
| @@ -267,7 +267,8 @@ defmodule SandboxCase.Sandbox.Logger do | |
| 267 267 | cleanup_started != nil and |
| 268 268 | logged_at >= cleanup_started and |
| 269 269 | (String.contains?(message, "OwnershipError") or |
| 270 | - String.contains?(message, "cannot find ownership process")) |
| 270 | + String.contains?(message, "cannot find ownership process") or |
| 271 | + String.contains?(message, "owner") and String.contains?(message, "exited")) |
| 271 272 | end |
| 272 273 | |
| 273 274 | defp ownership_error_during_cleanup?(_), do: false |
| @@ -2,7 +2,7 @@ defmodule SandboxCase.MixProject do | |
| 2 2 | use Mix.Project |
| 3 3 | |
| 4 4 | @source_url "https://github.com/pinetops/sandbox_case" |
| 5 | - @version "0.3.5" |
| 5 | + @version "0.3.7" |
| 6 6 | |
| 7 7 | def project do |
| 8 8 | [ |