Current section

Files

Jump to
exmobiledevice lib ex_mobile_device web_inspector.ex
Raw

lib/ex_mobile_device/web_inspector.ex

defmodule ExMobileDevice.WebInspector do
@moduledoc """
Functions for controlling Mobile Safari.
This module provides a means of controlling Safari, without requiring SafariDriver.
Starting an instance of this process will create a connection to an automated safari
instance, on the specified device.
## Example
iex(1)> alias ExMobileDevice.WebInspector
ExMobileDevice.WebInspector
iex(2)> {:ok, pid} = WebInspector.start_supervised("00008120-0018DEADC0DEFACE")
{:ok, #PID<0.208.0>}
iex(3)> {:ok, page} = WebInspector.create_page(pid)
{:ok, "page-7102B011-5BC0-4785-87DF-ADBA671EAD74"}
iex(4)> WebInspector.navigate_to(pid, page, "https://elixir-lang.org")
:ok
"""
use GenStateMachine, restart: :temporary
use TypedStruct
require Logger
@doc """
Create a supervised connection to the 'webinspector' service on the device.
The connection is supervised by an internal supervision tree of `exmobiledevice`.
To manually stop the connection, call `stop/1`.
> #### Controlling Process {: .info}
>
> On success, the caller is the 'controlling process' of the connection. Should
> the caller exit for any reason, the connection will also exit, which will
> in turn tear down the automation pages.
>
> To transfer control to another process, call `set_controlling_process/2`.
"""
@spec start_supervised(String.t()) :: DynamicSupervisor.on_start_child()
def start_supervised(udid) when is_binary(udid) do
args = [udid: udid, controlling_process: self()]
DynamicSupervisor.start_child(ExMobileDevice.WebInspector.Supervisor, {__MODULE__, args})
end
@doc """
Stop the webinspector process.
Stopping the process will terminate the connection to the webinspector service
on the device, which will remove any automation pages from Safari.
"""
@spec stop(pid) :: :ok
def stop(pid) do
GenStateMachine.stop(pid)
end
@doc """
Transfer ownership to the specified process.
Transfers controlling ownership of `pid` to `cp`. The caller must be the
current owner.
"""
@spec set_controlling_process(:gen_statem.server_ref(), pid()) :: :ok | {:error, any()}
def set_controlling_process(pid, cp) when is_pid(cp) do
GenStateMachine.call(pid, {:set_cp, cp})
end
@doc """
Create a new automation page.
"""
@spec create_page(:gen_statem.server_ref()) :: {:ok, String.t()} | {:error, any()}
def create_page(pid) do
GenStateMachine.call(pid, :create_page)
end
@doc """
List the current automation pages.
"""
@spec list_pages(:gen_statem.server_ref()) :: {:ok, list(map())} | {:error, any()}
def list_pages(pid) do
GenStateMachine.call(pid, :list_pages)
end
@doc """
Switch to the specified page.
"""
@spec switch_to_page(:gen_statem.server_ref(), String.t()) :: :ok | {:error, any()}
def switch_to_page(pid, page) do
GenStateMachine.call(pid, {:switch_to_page, page})
end
@doc """
Navigate the specified page to the provided url.
Supported options are:
- `timeout`: The page-load timeout in milliseconds. Defaults to 30_000 (30 seconds)
"""
@spec navigate_to(:gen_statem.server_ref(), String.t(), String.t(), Keyword.t()) ::
:ok | {:error, any()}
def navigate_to(pid, page, url, opts \\ []) do
GenStateMachine.call(pid, {:navigate_to, page, url, opts})
end
@doc """
Go to the previous url in the page's history.
"""
@spec go_back(:gen_statem.server_ref(), String.t()) :: :ok | {:error, any()}
def go_back(pid, page) do
GenStateMachine.call(pid, {:go_back, page})
end
@doc """
Go to the next url in the page's history.
"""
@spec go_forward(:gen_statem.server_ref(), String.t()) :: :ok | {:error, any()}
def go_forward(pid, page) do
GenStateMachine.call(pid, {:go_forward, page})
end
@doc """
Reload the page's current url.
"""
@spec reload(:gen_statem.server_ref(), String.t()) :: :ok | {:error, any()}
def reload(pid, page) do
GenStateMachine.call(pid, {:reload, page})
end
@doc """
Take a screenshot of the current page, returning the bytes in PNG format.
"""
@spec take_screenshot(:gen_statem.server_ref(), String.t()) :: {:ok, binary()} | {:error, any()}
def take_screenshot(pid, page) do
GenStateMachine.call(pid, {:take_screenshot, page})
end
@doc """
Close the specified page.
"""
@spec close_page(:gen_statem.server_ref(), String.t()) :: :ok | {:error, any()}
def close_page(pid, page) do
GenStateMachine.call(pid, {:close_page, page})
end
@doc false
def start_link(args) do
GenStateMachine.start_link(__MODULE__, args, hibernate_after: 15_000)
end
@service "com.apple.webinspector"
@bundle "com.apple.mobilesafari"
@selector "__selector"
@argument "__argument"
@rpcReportIdentifier "_rpc_reportIdentifier:"
@rpcReportCurrentState "_rpc_reportCurrentState:"
@rpcReportConnectedApplications "_rpc_reportConnectedApplicationList:"
@rpcApplicationConnected "_rpc_applicationConnected:"
@rpcApplicationUpdated "_rpc_applicationUpdated:"
@rpcApplicationDisconnected "_rpc_applicationDisconnected:"
@rpcApplicationSentListing "_rpc_applicationSentListing:"
@rpcApplicationSentData "_rpc_applicationSentData:"
@rpcRequestApplicationLaunch "_rpc_requestApplicationLaunch:"
@rpcForwardAutomationSessionRequest "_rpc_forwardAutomationSessionRequest:"
@rpcForwardSocketSetup "_rpc_forwardSocketSetup:"
@rpcForwardSocketData "_rpc_forwardSocketData:"
@wirConnectionIdentifierKey "WIRConnectionIdentifierKey"
@wirAutomationAvailabilityKey "WIRAutomationAvailabilityKey"
@wirAutomationAvailable "WIRAutomationAvailabilityAvailable"
@wirApplicationDictionaryKey "WIRApplicationDictionaryKey"
@wirApplicationIdentifierKey "WIRApplicationIdentifierKey"
@wirApplicationBundleIdentifierKey "WIRApplicationBundleIdentifierKey"
@wirIsApplicationReadyKey "WIRIsApplicationReadyKey"
@wirSessionIdentifierKey "WIRSessionIdentifierKey"
@wirSessionCapabilitiesKey "WIRSessionCapabilitiesKey"
@wirListingKey "WIRListingKey"
@wirTypeKey "WIRTypeKey"
@wirTypeAutomation "WIRTypeAutomation"
@wirPageIdentifierKey "WIRPageIdentifierKey"
@wirSenderKey "WIRSenderKey"
@wirSocketDataKey "WIRSocketDataKey"
@wirDestinationKey "WIRDestinationKey"
@wirMessageDataKey "WIRMessageDataKey"
typedstruct do
@typedoc false
# The socket connected to the web-inspector service
field(:ssl_sock, :ssl.sslsocket())
# The unique session id identifying this connection
field(:session_id, String.t())
# The pid of the controlling process
field(:cp_pid, reference())
# The monitor reference to the controlling process
field(:cp_mref, reference())
# The reported state of the safari process, if any
field(:safari, %{String.t() => any} | nil)
# The id of the currently 'automatable' page.
field(:page_id, String.t() | nil)
# The sequence number of the next request to the page
field(:page_out, non_neg_integer(), default: 0)
# A map of pending replies to page requests. The key is the
# sequence number and the value is an arbitrary context.
field(:page_in, %{non_neg_integer() => any()}, default: %{})
end
@impl true
def init(args) do
proc = Keyword.fetch!(args, :controlling_process)
udid = Keyword.fetch!(args, :udid)
with {:ok, ssl_sock} <- ExMobileDevice.Services.connect(udid, @service) do
session_id = UUID.uuid4() |> String.upcase()
if automation_enabled?(ssl_sock, session_id) do
:ok = :ssl.setopts(ssl_sock, active: :once)
mref = Process.monitor(proc)
timeout = Keyword.get(args, :timeout, 30_000)
next_events = [
{{:timeout, :start_session}, timeout, nil},
{:next_event, :internal, :start_session}
]
{:ok, :created,
%__MODULE__{ssl_sock: ssl_sock, session_id: session_id, cp_pid: proc, cp_mref: mref},
next_events}
else
{:stop, :no_automation}
end
end
end
@impl true
#
# Handle api calls
#
def handle_event({:call, {caller, _} = from}, {:set_cp, cp}, _, %__MODULE__{} = data) do
if caller == data.cp_pid do
# The controlling process can transfer control to another process
mref = Process.monitor(cp)
Process.demonitor(data.cp_mref, [:flush])
{:keep_state, %__MODULE__{data | cp_pid: cp, cp_mref: mref}, {:reply, from, :ok}}
else
{:keep_state_and_data, {:reply, from, {:error, :not_controlling_process}}}
end
end
def handle_event({:call, from}, _, :failed, %__MODULE__{}) do
# If the process failed to initialize, all requests will fail
{:keep_state_and_data, {:reply, from, {:error, :failed}}}
end
def handle_event({:call, _}, _, state, %__MODULE__{}) when state != :connected do
# All requests must otherwise wait for the process to connect to safari
{:keep_state_and_data, :postpone}
end
def handle_event({:call, from}, :create_page, _, %__MODULE__{} = data) do
send_rpc("createBrowsingContext", [], from, data, fn
%{"result" => %{"handle" => handle}}, data ->
{:keep_state, data, {:reply, from, {:ok, handle}}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, :list_pages, _, %__MODULE__{} = data) do
send_rpc("getBrowsingContexts", [], from, data, fn
%{"result" => %{"contexts" => contexts}}, data ->
pages =
for %{"active" => active, "handle" => handle, "url" => url} <- contexts do
%{active: active, id: handle, url: url}
end
{:keep_state, data, {:reply, from, {:ok, pages}}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, {:navigate_to, page, url, opts}, _, %__MODULE__{} = data) do
args = [handle: page, url: url] ++ page_load_timeout(opts)
send_rpc("navigateBrowsingContext", args, from, data, fn
%{"result" => %{}}, data ->
{:keep_state, data, {:reply, from, :ok}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, {:switch_to_page, page}, _, %__MODULE__{} = data) do
args = [browsingContextHandle: page, frameHandle: ""]
send_rpc("switchToBrowsingContext", args, from, data, fn
%{"result" => %{}}, data ->
{:keep_state, data, {:reply, from, :ok}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, {:take_screenshot, page}, _, %__MODULE__{} = data) do
args = [handle: page, scrollIntoViewIfNeeded: true, clipToViewport: true]
send_rpc("takeScreenshot", args, from, data, fn
%{"result" => %{"data" => base64}}, data ->
{:keep_state, data, {:reply, from, {:ok, Base.decode64!(base64)}}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, {:go_forward, page}, _, %__MODULE__{} = data) do
send_rpc("goForwardInBrowsingContext", [handle: page], from, data, fn
%{"result" => %{}}, data ->
{:keep_state, data, {:reply, from, :ok}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, {:go_back, page}, _, %__MODULE__{} = data) do
send_rpc("goBackInBrowsingContext", [handle: page], from, data, fn
%{"result" => %{}}, data ->
{:keep_state, data, {:reply, from, :ok}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, {:reload, page}, _, %__MODULE__{} = data) do
send_rpc("reloadBrowsingContext", [handle: page], from, data, fn
%{"result" => %{}}, data ->
{:keep_state, data, {:reply, from, :ok}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
def handle_event({:call, from}, {:close_page, page}, _, %__MODULE__{} = data) do
send_rpc("closeBrowsingContext", [handle: page], from, data, fn
%{"result" => %{}}, data ->
{:keep_state, data, {:reply, from, :ok}}
%{"error" => error}, data ->
{:keep_state, data, {:reply, from, {:error, error}}}
end)
end
#
# Event handling
#
def handle_event(:internal, :start_session, state, %__MODULE__{} = data) do
case state do
:created ->
{:keep_state_and_data, :postpone}
:initialized when is_nil(data.safari) ->
# Start safari if it is not already started
bundle_key = %{@wirApplicationBundleIdentifierKey => @bundle}
case send_msg(data.ssl_sock, data.session_id, @rpcRequestApplicationLaunch, bundle_key) do
:ok ->
{:keep_state_and_data, :postpone}
_error ->
{:next_state, :failed, data}
end
:initialized ->
if data.safari[@wirAutomationAvailabilityKey] == @wirAutomationAvailable do
{:next_state, :ready, data, :postpone}
else
{:next_state, :failed, data}
end
:ready ->
next_event = [{:next_event, :internal, :start_automation_session}]
{:keep_state_and_data, next_event}
end
end
def handle_event(:internal, :start_automation_session, _, %__MODULE__{} = data) do
if data.safari[@wirAutomationAvailabilityKey] == @wirAutomationAvailable do
params = %{
@wirSessionIdentifierKey => data.session_id,
@wirApplicationIdentifierKey => data.safari[@wirApplicationIdentifierKey],
@wirSessionCapabilitiesKey => %{
"org.webkit.webdriver.webrtc.allow-insecure-media-capture" => true,
"org.webkit.webdriver.webrtc.suppress-ice-candidate-filtering" => false
}
}
case send_msg(data.ssl_sock, data.session_id, @rpcForwardAutomationSessionRequest, params) do
:ok ->
:keep_state_and_data
_error ->
{:next_state, :failed, data}
end
else
{:next_state, :failed, data}
end
end
def handle_event(:internal, {:recv, msg}, state, %__MODULE__{} = data) do
args = msg[@argument]
case msg[@selector] do
@rpcReportConnectedApplications when state == :created ->
update = find_safari(Map.values(args[@wirApplicationDictionaryKey]))
{:next_state, :initialized, %__MODULE__{data | safari: update}}
@rpcApplicationSentData ->
if args[@wirDestinationKey] == data.session_id do
case Jason.decode!(args[@wirMessageDataKey]) do
%{"id" => id} = response ->
case Map.pop(data.page_in, id) do
{nil, _} ->
:keep_state_and_data
{fun, pending} ->
fun.(response, %__MODULE__{data | page_in: pending})
end
_ ->
:keep_state_and_data
end
else
:keep_state_and_data
end
@rpcApplicationSentListing ->
app_id = get_in(data.safari, [@wirApplicationIdentifierKey])
if app_id == args[@wirApplicationIdentifierKey] do
find_automation_page(args[@wirListingKey], data)
else
:keep_state_and_data
end
@rpcApplicationDisconnected ->
app_id = get_in(data.safari, [@wirApplicationIdentifierKey])
if app_id == args[@wirApplicationIdentifierKey] do
{:keep_state, %__MODULE__{data | safari: nil, page_id: nil}}
else
:keep_state_and_data
end
rpc when rpc in [@rpcApplicationConnected, @rpcApplicationUpdated] ->
if args[@wirApplicationBundleIdentifierKey] == @bundle do
if state == :initialized && safari_automated?(args) && safari_ready?(args) do
{:next_state, :ready, %__MODULE__{data | safari: args}}
else
{:keep_state, %__MODULE__{data | safari: args}}
end
else
:keep_state_and_data
end
_ ->
:keep_state_and_data
end
end
def handle_event(:internal, :connect_page, _, %__MODULE__{} = data) do
params = %{
@wirSenderKey => data.session_id,
@wirApplicationIdentifierKey => data.safari[@wirApplicationIdentifierKey],
@wirPageIdentifierKey => data.page_id
}
case send_msg(data.ssl_sock, data.session_id, @rpcForwardSocketSetup, params) do
:ok -> :keep_state_and_data
end
end
#
# Socket handling
#
def handle_event(:info, {:ssl, socket, data}, _, %__MODULE__{ssl_sock: socket}) do
:ok = :ssl.setopts(socket, active: :once)
{:keep_state_and_data, {:next_event, :internal, {:recv, ExMobileDevice.Plist.decode(data)}}}
end
def handle_event(:info, {:ssl_closed, socket}, _, %__MODULE__{ssl_sock: socket}) do
{:stop, :shutdown}
end
def handle_event(:info, {:DOWN, ref, :process, _, _}, _, %__MODULE__{cp_mref: ref}) do
{:stop, :shutdown}
end
#
# Session creation timeout
#
def handle_event({:timeout, :start_session}, _, _, %__MODULE__{} = data) do
{:next_state, :failed, data}
end
defp find_safari(applications) do
Enum.find(applications, &match?(%{@wirApplicationBundleIdentifierKey => @bundle}, &1))
end
defp safari_automated?(%{@wirAutomationAvailabilityKey => @wirAutomationAvailable}), do: true
defp safari_automated?(_), do: false
defp safari_ready?(safari), do: safari[@wirIsApplicationReadyKey]
defp send_rpc(method, args, from, data, fun) do
seqno = data.page_out
case send_cmd(data, seqno, method, args) do
:ok ->
page_in = Map.put(data.page_in, seqno, fun)
{:keep_state, %__MODULE__{data | page_out: seqno + 1, page_in: page_in}}
error ->
stop_and_reply(error, from)
end
end
defp send_cmd(%__MODULE__{} = data, id, method, args) do
call_args = %{
"method" => "Automation.#{method}",
"params" => Map.new(args),
"id" => id
}
call_params = %{
@wirApplicationIdentifierKey => data.safari[@wirApplicationIdentifierKey],
@wirPageIdentifierKey => data.page_id,
@wirSessionIdentifierKey => data.session_id,
@wirSocketDataKey => {:data, Jason.encode!(call_args)}
}
send_msg(data.ssl_sock, data.session_id, @rpcForwardSocketData, call_params)
end
defp find_automation_page(pages, %__MODULE__{session_id: sid} = data) do
candidates =
for {_, %{@wirTypeKey => @wirTypeAutomation, @wirSessionIdentifierKey => ^sid} = page} <-
pages do
page[@wirPageIdentifierKey]
end
if pid = List.first(candidates) do
cond do
is_nil(data.page_id) ->
{:keep_state, %__MODULE__{data | page_id: pid}, {:next_event, :internal, :connect_page}}
get_in(pages, [to_string(data.page_id), @wirConnectionIdentifierKey]) == sid ->
{:next_state, :connected, data, {{:timeout, :start_session}, :cancel}}
end
else
:keep_state_and_data
end
end
defp page_load_timeout(opts) do
if t = Keyword.get(opts, :timeout) do
[pageLoadTimeout: t]
else
[]
end
end
defp automation_enabled?(ssl_sock, session_id) do
with :ok <- send_msg(ssl_sock, session_id, @rpcReportIdentifier),
{:ok, %{@selector => @rpcReportCurrentState} = reply} <- recv_msg(ssl_sock, 5000) do
get_in(reply, [@argument, @wirAutomationAvailabilityKey]) == @wirAutomationAvailable
else
_ -> false
end
end
defp recv_msg(ssl_sock, timeout) do
with {:ok, packet} <- :ssl.recv(ssl_sock, 0, timeout) do
{:ok, ExMobileDevice.Plist.decode(packet)}
end
end
defp send_msg(ssl_sock, conn_id, selector, args \\ %{}) do
send_plist(ssl_sock, %{
@selector => selector,
@argument => Map.put(args, @wirConnectionIdentifierKey, conn_id)
})
end
defp stop_and_reply(error, to) do
if to do
{:stop_and_reply, {:shutdown, error}, {:reply, to, error}}
else
{:stop, {:shutdown, error}}
end
end
defp send_plist(ssl_sock, plist) do
:ssl.send(ssl_sock, ExMobileDevice.Plist.encode(plist))
end
end