Current section

28 Versions

Jump to

Compare versions

5 files changed
+157 additions
-3 deletions
  @@ -1,6 +1,6 @@
1 1 {<<"links">>,[{<<"GitHub">>,<<"https://github.com/pinetops/sandbox_case">>}]}.
2 2 {<<"name">>,<<"sandbox_case">>}.
3 - {<<"version">>,<<"0.3.5">>}.
3 + {<<"version">>,<<"0.3.7">>}.
4 4 {<<"description">>,
5 5 <<"Batteries-included test isolation for Elixir and Phoenix.">>}.
6 6 {<<"elixir">>,<<"~> 1.15">>}.
  @@ -15,6 +15,7 @@
15 15 <<"lib/sandbox_case/sandbox/cachex_patcher.ex">>,
16 16 <<"lib/sandbox_case/sandbox/ecto.ex">>,
17 17 <<"lib/sandbox_case/sandbox/fun_with_flags.ex">>,
18 + <<"lib/sandbox_case/sandbox/lock_monitor.ex">>,
18 19 <<"lib/sandbox_case/sandbox/propagator.ex">>,
19 20 <<"lib/sandbox_case/sandbox/mox.ex">>,
20 21 <<"lib/sandbox_case/sandbox/mimic.ex">>,
  @@ -68,6 +68,26 @@ defmodule SandboxCase.Sandbox do
68 68 SandboxCase.Sandbox.DeadlockDetector.setup(config)
69 69 end
70 70
71 + if lm_config = sandbox_config[:lock_monitor] do
72 + config = if is_list(lm_config), do: lm_config, else: []
73 +
74 + # Auto-discover repo if not specified
75 + config =
76 + if config[:repo] do
77 + config
78 + else
79 + otp_app = opts[:otp_app] || Application.get_env(:sandbox_case, :otp_app)
80 + repos = if otp_app, do: Application.get_env(otp_app, :ecto_repos, []), else: []
81 +
82 + case List.first(repos) do
83 + nil -> config
84 + repo -> Keyword.put(config, :repo, repo)
85 + end
86 + end
87 +
88 + SandboxCase.Sandbox.LockMonitor.start_link(config)
89 + end
90 +
71 91 :ok
72 92 end
  @@ -0,0 +1,132 @@
1 + defmodule SandboxCase.Sandbox.LockMonitor do
2 + @moduledoc """
3 + Background monitor that periodically polls Postgres for blocked queries
4 + and reports the lock chain.
5 +
6 + Catches lock contention between async tests as it happens, instead of
7 + waiting for the ownership timeout (15-120s).
8 +
9 + ## Configuration
10 +
11 + config :sandbox_case,
12 + sandbox: [
13 + lock_monitor: true
14 + # or with options:
15 + lock_monitor: [interval: 2_000, repo: MyApp.Repo]
16 + ]
17 +
18 + ## What it reports
19 +
20 + When a query is blocked waiting for a lock, the monitor logs:
21 + - The blocked query and how long it's been waiting
22 + - The blocking query and its state
23 + - Both Postgres PIDs for further investigation
24 + """
25 + use GenServer
26 +
27 + require Logger
28 +
29 + @default_interval 2_000
30 +
31 + @lock_query """
32 + SELECT
33 + blocked_activity.pid AS blocked_pid,
34 + blocked_activity.query AS blocked_query,
35 + blocked_activity.wait_event_type AS wait_type,
36 + blocked_activity.state AS blocked_state,
37 + extract(epoch from now() - blocked_activity.query_start)::float AS blocked_seconds,
38 + blocking_activity.pid AS blocking_pid,
39 + blocking_activity.query AS blocking_query,
40 + blocking_activity.state AS blocking_state,
41 + extract(epoch from now() - blocking_activity.query_start)::float AS blocking_seconds
42 + FROM pg_stat_activity blocked_activity
43 + JOIN pg_locks blocked_locks ON blocked_locks.pid = blocked_activity.pid
44 + JOIN pg_locks blocking_locks
45 + ON blocking_locks.locktype = blocked_locks.locktype
46 + AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database
47 + AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
48 + AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
49 + AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
50 + AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
51 + AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
52 + AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
53 + AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
54 + AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
55 + AND blocking_locks.pid != blocked_locks.pid
56 + AND blocking_locks.granted
57 + JOIN pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
58 + WHERE NOT blocked_locks.granted
59 + """
60 +
61 + def start_link(opts) do
62 + name = Keyword.get(opts, :name, __MODULE__)
63 + GenServer.start_link(__MODULE__, opts, name: name)
64 + end
65 +
66 + @impl true
67 + def init(opts) do
68 + repo = opts[:repo]
69 + interval = opts[:interval] || @default_interval
70 +
71 + if repo do
72 + schedule(interval)
73 + {:ok, %{repo: repo, interval: interval}}
74 + else
75 + :ignore
76 + end
77 + end
78 +
79 + @impl true
80 + def handle_info(:poll, state) do
81 + check_for_locks(state.repo)
82 + schedule(state.interval)
83 + {:noreply, state}
84 + end
85 +
86 + def handle_info(_, state), do: {:noreply, state}
87 +
88 + defp check_for_locks(repo) do
89 + case repo.query(@lock_query, [], log: false, timeout: 5_000) do
90 + {:ok, %{rows: rows}} when rows != [] ->
91 + report_locks(rows)
92 +
93 + _ ->
94 + :ok
95 + end
96 + rescue
97 + _ -> :ok
98 + end
99 +
100 + defp report_locks(rows) do
101 + details =
102 + Enum.map_join(rows, "\n\n", fn row ->
103 + [blocked_pid, blocked_query, wait_type, blocked_state, blocked_secs,
104 + blocking_pid, blocking_query, blocking_state, blocking_secs] = row
105 +
106 + """
107 + BLOCKED: PG pid #{blocked_pid} (#{blocked_state}, waiting #{format_secs(blocked_secs)}, #{wait_type})
108 + Query: #{truncate(blocked_query)}
109 + HELD BY: PG pid #{blocking_pid} (#{blocking_state}, #{format_secs(blocking_secs)})
110 + Query: #{truncate(blocking_query)}\
111 + """
112 + end)
113 +
114 + Logger.warning("""
115 + SandboxCase LockMonitor: #{length(rows)} blocked query(ies) detected:
116 +
117 + #{details}
118 + """)
119 + end
120 +
121 + defp schedule(interval) do
122 + Process.send_after(self(), :poll, interval)
123 + end
124 +
125 + defp format_secs(nil), do: "?"
126 + defp format_secs(secs) when is_float(secs), do: "#{Float.round(secs, 1)}s"
127 + defp format_secs(secs), do: "#{secs}s"
128 +
129 + defp truncate(nil), do: "(none)"
130 + defp truncate(s) when byte_size(s) > 200, do: String.slice(s, 0..197) <> "..."
131 + defp truncate(s), do: s
132 + end
  @@ -267,7 +267,8 @@ defmodule SandboxCase.Sandbox.Logger do
267 267 cleanup_started != nil and
268 268 logged_at >= cleanup_started and
269 269 (String.contains?(message, "OwnershipError") or
270 - String.contains?(message, "cannot find ownership process"))
270 + String.contains?(message, "cannot find ownership process") or
271 + String.contains?(message, "owner") and String.contains?(message, "exited"))
271 272 end
272 273
273 274 defp ownership_error_during_cleanup?(_), do: false
  @@ -2,7 +2,7 @@ defmodule SandboxCase.MixProject do
2 2 use Mix.Project
3 3
4 4 @source_url "https://github.com/pinetops/sandbox_case"
5 - @version "0.3.5"
5 + @version "0.3.7"
6 6
7 7 def project do
8 8 [