Current section

Files

Jump to
soulless lib soulless http.ex
Raw

lib/soulless/http.ex

defmodule Soulless.HTTP do
def default_user_agent() do
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
end
def get(url, headers \\ [], proxy \\ nil) do
request(:get, url, headers, nil, proxy)
end
def post(url, body, headers \\ [], proxy \\ nil) do
request(:post, url, headers, body, proxy)
end
defp request(method, url, headers, body, proxy) do
default_headers = [{"user-agent", default_user_agent()}]
headers = merge_headers(headers, default_headers)
{host, path, port, opts} = gun_args(url, proxy)
# ugh
case :gun.open(host, port, opts) do
{:ok, conn} ->
try do
with {:ok, _} <- :gun.await_up(conn),
{:ok, tunnel_ref} <- setup_tunnel(conn, url, proxy) do
stream = dispatch(conn, method, path, headers, body, tunnel_ref)
case :gun.await(conn, stream) do
{:response, :fin, status, resp_headers} ->
{:ok, %{status: status, headers: resp_headers, body: nil}}
{:response, :nofin, status, resp_headers} ->
case :gun.await_body(conn, stream) do
{:ok, resp_body} ->
{:ok, %{status: status, headers: resp_headers, body: resp_body}}
{:error, reason} ->
{:error, reason}
end
{:error, reason} ->
{:error, reason}
end
end
after
:gun.close(conn)
end
{:error, reason} ->
{:error, reason}
end
end
defp setup_tunnel(_conn, _url, nil), do: {:ok, nil}
defp setup_tunnel(conn, url, proxy) do
case parse_proxy(proxy) do
{:http, _, _} ->
stream_ref = :gun.connect(conn, gun_connect_dest(URI.parse(url)))
case await_tunnel_up(conn, stream_ref) do
:ok -> {:ok, stream_ref}
{:error, _} = error -> error
end
_ ->
case await_tunnel_up(conn) do
:ok -> {:ok, nil}
{:error, _} = error -> error
end
end
end
defp await_tunnel_up(conn, stream_ref \\ nil, timeout \\ 30_000) do
receive do
{:gun_tunnel_up, ^conn, ref, _protocol} when stream_ref == nil or ref == stream_ref ->
:ok
# an HTTP proxy rejects CONNECT with a normal response instead of tunneling
{:gun_response, ^conn, ref, _is_fin, status, _headers}
when stream_ref == nil or ref == stream_ref ->
{:error, {:proxy_connect_failed, status}}
# transport died / proxy errored before the tunnel came up
{:gun_down, ^conn, _protocol, reason, _killed} ->
{:error, reason}
{:gun_error, ^conn, _stream_ref, reason} ->
{:error, reason}
{:gun_error, ^conn, reason} ->
{:error, reason}
after
timeout ->
{:error, :tunnel_timeout}
end
end
defp dispatch(conn, method, path, headers, body, tunnel_ref) do
req_opts = if tunnel_ref, do: %{tunnel: tunnel_ref}, else: %{}
case method do
:get -> :gun.get(conn, path, headers, req_opts)
:post -> :gun.post(conn, path, headers, body, req_opts)
end
end
def gun_args(uri, proxy \\ nil)
def gun_args(uri, proxy) when is_binary(uri) do
uri |> URI.parse() |> gun_args(proxy)
end
def gun_args(%URI{} = uri, nil) do
host = to_charlist(uri.host)
path = build_path(uri)
port = uri.port
opts = direct_gun_opts(uri.scheme)
{host, path, port, opts}
end
def gun_args(%URI{} = uri, proxy) do
{proxy_type, proxy_host, proxy_port} = parse_proxy(proxy)
open_opts =
case proxy_type do
:socks5 ->
dest_host = to_charlist(uri.host)
dest_port = uri.port
dest_transport = scheme_transport(uri.scheme)
tunnel_dest =
%{host: dest_host, port: dest_port, transport: dest_transport}
|> Map.put(:protocols, [:http])
|> maybe_add_tls_opts(dest_transport)
%{transport: :tcp, protocols: [{:socks, tunnel_dest}]}
:http ->
%{transport: :tcp, protocols: [:http]}
end
{proxy_host, build_path(uri), proxy_port, open_opts}
end
def gun_connect_dest(uri) when is_binary(uri), do: gun_connect_dest(URI.parse(uri))
def gun_connect_dest(%URI{} = uri) do
dest_transport = scheme_transport(uri.scheme)
%{host: to_charlist(uri.host), port: uri.port, transport: dest_transport, protocols: [:http]}
|> maybe_add_tls_opts(dest_transport)
end
defp direct_gun_opts(scheme) when scheme in ["https", "wss"] do
%{
protocols: [:http],
transport: :tls,
tls_opts: tls_opts()
}
end
defp direct_gun_opts(_scheme), do: %{protocols: [:http]}
defp tls_opts do
[
verify: :verify_peer,
cacerts: :public_key.cacerts_get(),
depth: 4,
customize_hostname_check: [
match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
]
]
end
defp maybe_add_tls_opts(opts, :tls), do: Map.put(opts, :tls_opts, tls_opts())
defp maybe_add_tls_opts(opts, _), do: opts
defp scheme_transport(scheme) when scheme in ["https", "wss"], do: :tls
defp scheme_transport(_scheme), do: :tcp
defp parse_proxy({type, host, port}) when type in [:socks5, :http] and is_binary(host) do
{type, to_charlist(host), port}
end
defp parse_proxy({type, host, port}) when type in [:socks5, :http] and is_list(host) do
{type, host, port}
end
defp build_path(%URI{path: nil}), do: "/"
defp build_path(%URI{path: "", query: nil}), do: "/"
defp build_path(%URI{path: path, query: nil}), do: path
defp build_path(%URI{path: path, query: query}), do: "#{path}?#{query}"
defp merge_headers(headers1, headers2) do
Enum.concat(headers1, headers2)
|> Enum.uniq_by(fn {k, _} -> String.downcase(k) end)
end
end