Current section
40 Versions
Jump to
Current section
40 Versions
Compare versions
16
files changed
+1301
additions
-458
deletions
| @@ -1,5 +1,23 @@ | |
| 1 1 | # Changelog |
| 2 2 | |
| 3 | + ## v0.17.0 (2024-01-07) |
| 4 | + |
| 5 | + ### Enhancements |
| 6 | + |
| 7 | + - Add support for async requests #228, #231 |
| 8 | + - Add stream example to docs #230 |
| 9 | + - Fix calls to deprecated Logger.warn/2 #232 |
| 10 | + - Fix typos #233 |
| 11 | + - Docs: do not use streams with async_request #238 |
| 12 | + - Add Finch.stream_while/5 #239 |
| 13 | + - Set MIX_ENV=test on CI #241 |
| 14 | + - Update HTTP/2 pool log level to warning for retried action #240 |
| 15 | + - Split trailers from headers #242 |
| 16 | + - Introduce :request_timeout option #244 |
| 17 | + - Support ALPN over HTTP1 pools #250 |
| 18 | + - Deprecate :protocol in favour of :protocols #251 |
| 19 | + - Implement pool telemetry #248 |
| 20 | + |
| 3 21 | ## v0.16.0 (2023-04-13) |
| 4 22 | |
| 5 23 | ### Enhancements |
| @@ -8,7 +8,9 @@ | |
| 8 8 | An HTTP client with a focus on performance, built on top of |
| 9 9 | [Mint](https://github.com/elixir-mint/mint) and [NimblePool](https://github.com/dashbitco/nimble_pool). |
| 10 10 | |
| 11 | - We try to achieve this goal by providing efficient connection pooling strategies and avoiding copying wherever possible. |
| 11 | + We attempt to achieve this goal by providing efficient connection pooling strategies and avoiding copying of memory wherever possible. |
| 12 | + |
| 13 | + Most developers will most likely prefer to use the fabulous HTTP client [Req](https://github.com/wojtekmach/req) which takes advantage of Finch's pooling and provides an extremely friendly and pleasant to use API. |
| 12 14 | |
| 13 15 | ## Usage |
| 14 16 | |
| @@ -105,7 +107,7 @@ The package can be installed by adding `finch` to your list of dependencies in ` | |
| 105 107 | ```elixir |
| 106 108 | def deps do |
| 107 109 | [ |
| 108 | - {:finch, "~> 0.16"} |
| 110 | + {:finch, "~> 0.17"} |
| 109 111 | ] |
| 110 112 | end |
| 111 113 | ``` |
| @@ -1,51 +1,51 @@ | |
| 1 | - {<<"app">>,<<"finch">>}. |
| 2 | - {<<"build_tools">>,[<<"mix">>]}. |
| 3 | - {<<"description">>,<<"An HTTP client focused on performance.">>}. |
| 4 | - {<<"elixir">>,<<"~> 1.7">>}. |
| 5 | - {<<"files">>, |
| 6 | - [<<"lib">>,<<"lib/finch">>,<<"lib/finch/error.ex">>,<<"lib/finch/http1">>, |
| 7 | - <<"lib/finch/http1/conn.ex">>,<<"lib/finch/http1/mint_http1.ex">>, |
| 8 | - <<"lib/finch/http1/pool.ex">>,<<"lib/finch/http2">>, |
| 9 | - <<"lib/finch/http2/request_stream.ex">>,<<"lib/finch/http2/pool.ex">>, |
| 10 | - <<"lib/finch/pool.ex">>,<<"lib/finch/pool_manager.ex">>, |
| 11 | - <<"lib/finch/request.ex">>,<<"lib/finch/response.ex">>, |
| 12 | - <<"lib/finch/ssl.ex">>,<<"lib/finch/telemetry.ex">>,<<"lib/finch.ex">>, |
| 13 | - <<".formatter.exs">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE.md">>, |
| 14 | - <<"CHANGELOG.md">>]}. |
| 15 | - {<<"licenses">>,[<<"MIT">>]}. |
| 16 1 | {<<"links">>, |
| 17 2 | [{<<"Changelog">>,<<"https://hexdocs.pm/finch/changelog.html">>}, |
| 18 3 | {<<"GitHub">>,<<"https://github.com/sneako/finch">>}]}. |
| 19 4 | {<<"name">>,<<"finch">>}. |
| 5 | + {<<"version">>,<<"0.17.0">>}. |
| 6 | + {<<"description">>,<<"An HTTP client focused on performance.">>}. |
| 7 | + {<<"elixir">>,<<"~> 1.11">>}. |
| 8 | + {<<"app">>,<<"finch">>}. |
| 9 | + {<<"licenses">>,[<<"MIT">>]}. |
| 20 10 | {<<"requirements">>, |
| 21 | - [[{<<"app">>,<<"mint">>}, |
| 22 | - {<<"name">>,<<"mint">>}, |
| 11 | + [[{<<"name">>,<<"mint">>}, |
| 12 | + {<<"app">>,<<"mint">>}, |
| 23 13 | {<<"optional">>,false}, |
| 24 | - {<<"repository">>,<<"hexpm">>}, |
| 25 | - {<<"requirement">>,<<"~> 1.3">>}], |
| 26 | - [{<<"app">>,<<"castore">>}, |
| 27 | - {<<"name">>,<<"castore">>}, |
| 14 | + {<<"requirement">>,<<"~> 1.3">>}, |
| 15 | + {<<"repository">>,<<"hexpm">>}], |
| 16 | + [{<<"name">>,<<"castore">>}, |
| 17 | + {<<"app">>,<<"castore">>}, |
| 28 18 | {<<"optional">>,false}, |
| 29 | - {<<"repository">>,<<"hexpm">>}, |
| 30 | - {<<"requirement">>,<<"~> 0.1 or ~> 1.0">>}], |
| 31 | - [{<<"app">>,<<"nimble_pool">>}, |
| 32 | - {<<"name">>,<<"nimble_pool">>}, |
| 19 | + {<<"requirement">>,<<"~> 0.1 or ~> 1.0">>}, |
| 20 | + {<<"repository">>,<<"hexpm">>}], |
| 21 | + [{<<"name">>,<<"nimble_pool">>}, |
| 22 | + {<<"app">>,<<"nimble_pool">>}, |
| 33 23 | {<<"optional">>,false}, |
| 34 | - {<<"repository">>,<<"hexpm">>}, |
| 35 | - {<<"requirement">>,<<"~> 0.2.6 or ~> 1.0">>}], |
| 36 | - [{<<"app">>,<<"nimble_options">>}, |
| 37 | - {<<"name">>,<<"nimble_options">>}, |
| 24 | + {<<"requirement">>,<<"~> 0.2.6 or ~> 1.0">>}, |
| 25 | + {<<"repository">>,<<"hexpm">>}], |
| 26 | + [{<<"name">>,<<"nimble_options">>}, |
| 27 | + {<<"app">>,<<"nimble_options">>}, |
| 38 28 | {<<"optional">>,false}, |
| 39 | - {<<"repository">>,<<"hexpm">>}, |
| 40 | - {<<"requirement">>,<<"~> 0.4 or ~> 1.0">>}], |
| 41 | - [{<<"app">>,<<"telemetry">>}, |
| 42 | - {<<"name">>,<<"telemetry">>}, |
| 29 | + {<<"requirement">>,<<"~> 0.4 or ~> 1.0">>}, |
| 30 | + {<<"repository">>,<<"hexpm">>}], |
| 31 | + [{<<"name">>,<<"telemetry">>}, |
| 32 | + {<<"app">>,<<"telemetry">>}, |
| 43 33 | {<<"optional">>,false}, |
| 44 | - {<<"repository">>,<<"hexpm">>}, |
| 45 | - {<<"requirement">>,<<"~> 0.4 or ~> 1.0">>}], |
| 46 | - [{<<"app">>,<<"mime">>}, |
| 47 | - {<<"name">>,<<"mime">>}, |
| 34 | + {<<"requirement">>,<<"~> 0.4 or ~> 1.0">>}, |
| 35 | + {<<"repository">>,<<"hexpm">>}], |
| 36 | + [{<<"name">>,<<"mime">>}, |
| 37 | + {<<"app">>,<<"mime">>}, |
| 48 38 | {<<"optional">>,false}, |
| 49 | - {<<"repository">>,<<"hexpm">>}, |
| 50 | - {<<"requirement">>,<<"~> 1.0 or ~> 2.0">>}]]}. |
| 51 | - {<<"version">>,<<"0.16.0">>}. |
| 39 | + {<<"requirement">>,<<"~> 1.0 or ~> 2.0">>}, |
| 40 | + {<<"repository">>,<<"hexpm">>}]]}. |
| 41 | + {<<"files">>, |
| 42 | + [<<"lib">>,<<"lib/finch">>,<<"lib/finch/error.ex">>,<<"lib/finch/http1">>, |
| 43 | + <<"lib/finch/http1/conn.ex">>,<<"lib/finch/http1/pool.ex">>, |
| 44 | + <<"lib/finch/http1/pool_metrics.ex">>,<<"lib/finch/http2">>, |
| 45 | + <<"lib/finch/http2/request_stream.ex">>,<<"lib/finch/http2/pool.ex">>, |
| 46 | + <<"lib/finch/http2/pool_metrics.ex">>,<<"lib/finch/request.ex">>, |
| 47 | + <<"lib/finch/ssl.ex">>,<<"lib/finch/telemetry.ex">>, |
| 48 | + <<"lib/finch/response.ex">>,<<"lib/finch/pool.ex">>, |
| 49 | + <<"lib/finch/pool_manager.ex">>,<<"lib/finch.ex">>,<<".formatter.exs">>, |
| 50 | + <<"mix.exs">>,<<"README.md">>,<<"LICENSE.md">>,<<"CHANGELOG.md">>]}. |
| 51 | + {<<"build_tools">>,[<<"mix">>]}. |
| @@ -6,6 +6,7 @@ defmodule Finch do | |
| 6 6 | |> Enum.fetch!(1) |
| 7 7 | |
| 8 8 | alias Finch.{PoolManager, Request, Response} |
| 9 | + require Finch.Pool |
| 9 10 | |
| 10 11 | use Supervisor |
| 11 12 | |
| @@ -17,8 +18,19 @@ defmodule Finch do | |
| 17 18 | @pool_config_schema [ |
| 18 19 | protocol: [ |
| 19 20 | type: {:in, [:http2, :http1]}, |
| 20 | - doc: "The type of connection and pool to use.", |
| 21 | - default: :http1 |
| 21 | + deprecated: "Use :protocols instead." |
| 22 | + ], |
| 23 | + protocols: [ |
| 24 | + type: {:list, {:in, [:http1, :http2]}}, |
| 25 | + doc: """ |
| 26 | + The type of connections to support. |
| 27 | + |
| 28 | + If using `:http1` only, an HTTP1 pool without multiplexing is used. \ |
| 29 | + If using `:http2` only, an HTTP2 pool with multiplexing is used. \ |
| 30 | + If both are listed, then both HTTP1/HTTP2 connections are \ |
| 31 | + supported (via ALPN), but there is no multiplexing. |
| 32 | + """, |
| 33 | + default: [:http1] |
| 22 34 | ], |
| 23 35 | size: [ |
| 24 36 | type: :pos_integer, |
| @@ -75,6 +87,11 @@ defmodule Finch do | |
| 75 87 | before being closed during a checkout attempt. |
| 76 88 | """, |
| 77 89 | default: :infinity |
| 90 | + ], |
| 91 | + start_pool_metrics?: [ |
| 92 | + type: :boolean, |
| 93 | + doc: "When true, pool metrics will be collected and avaiable through Finch.pool_status/2", |
| 94 | + default: false |
| 78 95 | ] |
| 79 96 | ] |
| 80 97 | |
| @@ -83,11 +100,43 @@ defmodule Finch do | |
| 83 100 | """ |
| 84 101 | @type name() :: atom() |
| 85 102 | |
| 103 | + @type scheme() :: :http | :https |
| 104 | + |
| 105 | + @type scheme_host_port() :: {scheme(), host :: String.t(), port :: :inet.port_number()} |
| 106 | + |
| 107 | + @type request_opt() :: {:pool_timeout, pos_integer()} | {:receive_timeout, pos_integer()} |
| 108 | + |
| 109 | + @typedoc """ |
| 110 | + Options used by request functions. |
| 111 | + """ |
| 112 | + @type request_opts() :: [request_opt()] |
| 113 | + |
| 114 | + @typedoc """ |
| 115 | + The reference used to identify a request sent using `async_request/3`. |
| 116 | + """ |
| 117 | + @opaque request_ref() :: Finch.Pool.request_ref() |
| 118 | + |
| 86 119 | @typedoc """ |
| 87 120 | The stream function given to `stream/5`. |
| 88 121 | """ |
| 89 122 | @type stream(acc) :: |
| 90 | - ({:status, integer} | {:headers, Mint.Types.headers()} | {:data, binary}, acc -> acc) |
| 123 | + ({:status, integer} |
| 124 | + | {:headers, Mint.Types.headers()} |
| 125 | + | {:data, binary} |
| 126 | + | {:trailers, Mint.Types.headers()}, |
| 127 | + acc -> |
| 128 | + acc) |
| 129 | + |
| 130 | + @typedoc """ |
| 131 | + The stream function given to `stream_while/5`. |
| 132 | + """ |
| 133 | + @type stream_while(acc) :: |
| 134 | + ({:status, integer} |
| 135 | + | {:headers, Mint.Types.headers()} |
| 136 | + | {:data, binary} |
| 137 | + | {:trailers, Mint.Types.headers()}, |
| 138 | + acc -> |
| 139 | + {:cont, acc} | {:halt, acc}) |
| 91 140 | |
| 92 141 | @doc """ |
| 93 142 | Start an instance of Finch. |
| @@ -206,14 +255,33 @@ defmodule Finch do | |
| 206 255 | conn_opts |
| 207 256 | |> Keyword.put(:ssl_key_log_file_device, ssl_key_log_file_device) |
| 208 257 | |> Keyword.put(:transport_opts, transport_opts) |
| 258 | + |> Keyword.put(:protocols, valid[:protocols]) |
| 259 | + |
| 260 | + # TODO: Remove :protocol on v0.18 |
| 261 | + mod = |
| 262 | + case valid[:protocol] do |
| 263 | + :http1 -> |
| 264 | + Finch.HTTP1.Pool |
| 265 | + |
| 266 | + :http2 -> |
| 267 | + Finch.HTTP2.Pool |
| 268 | + |
| 269 | + nil -> |
| 270 | + if :http1 in valid[:protocols] do |
| 271 | + Finch.HTTP1.Pool |
| 272 | + else |
| 273 | + Finch.HTTP2.Pool |
| 274 | + end |
| 275 | + end |
| 209 276 | |
| 210 277 | %{ |
| 278 | + mod: mod, |
| 211 279 | size: valid[:size], |
| 212 280 | count: valid[:count], |
| 213 281 | conn_opts: conn_opts, |
| 214 | - protocol: valid[:protocol], |
| 215 282 | conn_max_idle_time: to_native(valid[:max_idle_time] || valid[:conn_max_idle_time]), |
| 216 | - pool_max_idle_time: valid[:pool_max_idle_time] |
| 283 | + pool_max_idle_time: valid[:pool_max_idle_time], |
| 284 | + start_pool_metrics?: valid[:start_pool_metrics?] |
| 217 285 | } |
| 218 286 | end |
| 219 287 | |
| @@ -255,45 +323,113 @@ defmodule Finch do | |
| 255 323 | accumulator. The function must return a potentially updated |
| 256 324 | accumulator. |
| 257 325 | |
| 326 | + See also `stream_while/5`. |
| 327 | + |
| 258 328 | ## Stream commands |
| 259 329 | |
| 260 | - * `{:status, status}` - the status of the http response |
| 261 | - * `{:headers, headers}` - the headers of the http response |
| 262 | - * `{:data, data}` - a streaming section of the http body |
| 330 | + * `{:status, status}` - the http response status |
| 331 | + * `{:headers, headers}` - the http response headers |
| 332 | + * `{:data, data}` - a streaming section of the http response body |
| 333 | + * `{:trailers, trailers}` - the http response trailers |
| 263 334 | |
| 264 335 | ## Options |
| 265 336 | |
| 266 | - * `:pool_timeout` - This timeout is applied when we check out a connection from the pool. |
| 267 | - Default value is `5_000`. |
| 337 | + Shares options with `request/3`. |
| 268 338 | |
| 269 | - * `:receive_timeout` - The maximum time to wait for a response before returning an error. |
| 270 | - Default value is `15_000`. |
| 339 | + ## Examples |
| 271 340 | |
| 341 | + path = "/tmp/archive.zip" |
| 342 | + file = File.open!(path, [:write, :exclusive]) |
| 343 | + url = "https://example.com/archive.zip" |
| 344 | + request = Finch.build(:get, url) |
| 345 | + |
| 346 | + Finch.stream(request, MyFinch, nil, fn |
| 347 | + {:status, status}, _acc -> |
| 348 | + IO.inspect(status) |
| 349 | + |
| 350 | + {:headers, headers}, _acc -> |
| 351 | + IO.inspect(headers) |
| 352 | + |
| 353 | + {:data, data}, _acc -> |
| 354 | + IO.binwrite(file, data) |
| 355 | + end) |
| 356 | + |
| 357 | + File.close(file) |
| 272 358 | """ |
| 273 | - @spec stream(Request.t(), name(), acc, stream(acc), keyword) :: |
| 359 | + @spec stream(Request.t(), name(), acc, stream(acc), request_opts()) :: |
| 274 360 | {:ok, acc} | {:error, Exception.t()} |
| 275 361 | when acc: term() |
| 276 362 | def stream(%Request{} = req, name, acc, fun, opts \\ []) when is_function(fun, 2) do |
| 363 | + fun = |
| 364 | + fn entry, acc -> |
| 365 | + {:cont, fun.(entry, acc)} |
| 366 | + end |
| 367 | + |
| 368 | + stream_while(req, name, acc, fun, opts) |
| 369 | + end |
| 370 | + |
| 371 | + @doc """ |
| 372 | + Streams an HTTP request until it finishes or `fun` returns `{:halt, acc}`. |
| 373 | + |
| 374 | + A function of arity 2 is expected as argument. The first argument |
| 375 | + is a tuple, as listed below, and the second argument is the |
| 376 | + accumulator. |
| 377 | + |
| 378 | + The function must return: |
| 379 | + |
| 380 | + * `{:cont, acc}` to continue streaming |
| 381 | + * `{:halt, acc}` to halt streaming |
| 382 | + |
| 383 | + See also `stream/5`. |
| 384 | + |
| 385 | + ## Stream commands |
| 386 | + |
| 387 | + * `{:status, status}` - the http response status |
| 388 | + * `{:headers, headers}` - the http response headers |
| 389 | + * `{:data, data}` - a streaming section of the http response body |
| 390 | + * `{:trailers, trailers}` - the http response trailers |
| 391 | + |
| 392 | + ## Options |
| 393 | + |
| 394 | + Shares options with `request/3`. |
| 395 | + |
| 396 | + ## Examples |
| 397 | + |
| 398 | + path = "/tmp/archive.zip" |
| 399 | + file = File.open!(path, [:write, :exclusive]) |
| 400 | + url = "https://example.com/archive.zip" |
| 401 | + request = Finch.build(:get, url) |
| 402 | + |
| 403 | + Finch.stream_while(request, MyFinch, nil, fn |
| 404 | + {:status, status}, acc -> |
| 405 | + IO.inspect(status) |
| 406 | + {:cont, acc} |
| 407 | + |
| 408 | + {:headers, headers}, acc -> |
| 409 | + IO.inspect(headers) |
| 410 | + {:cont, acc} |
| 411 | + |
| 412 | + {:data, data}, acc -> |
| 413 | + IO.binwrite(file, data) |
| 414 | + {:cont, acc} |
| 415 | + end) |
| 416 | + |
| 417 | + File.close(file) |
| 418 | + """ |
| 419 | + @spec stream_while(Request.t(), name(), acc, stream_while(acc), request_opts()) :: |
| 420 | + {:ok, acc} | {:error, Exception.t()} |
| 421 | + when acc: term() |
| 422 | + def stream_while(%Request{} = req, name, acc, fun, opts \\ []) when is_function(fun, 2) do |
| 277 423 | request_span req, name do |
| 278 424 | __stream__(req, name, acc, fun, opts) |
| 279 425 | end |
| 280 426 | end |
| 281 427 | |
| 282 | - defp __stream__(%Request{} = req, name, acc, fun, opts) when is_function(fun, 2) do |
| 283 | - shp = build_shp(req) |
| 284 | - {pool, pool_mod} = PoolManager.get_pool(name, shp) |
| 428 | + defp __stream__(%Request{} = req, name, acc, fun, opts) do |
| 429 | + {pool, pool_mod} = get_pool(req, name) |
| 285 430 | pool_mod.request(pool, req, acc, fun, opts) |
| 286 431 | end |
| 287 432 | |
| 288 | - defp build_shp(%Request{scheme: scheme, unix_socket: unix_socket}) |
| 289 | - when is_binary(unix_socket) do |
| 290 | - {scheme, {:local, unix_socket}, 0} |
| 291 | - end |
| 292 | - |
| 293 | - defp build_shp(%Request{scheme: scheme, host: host, port: port}) do |
| 294 | - {scheme, host, port} |
| 295 | - end |
| 296 | - |
| 297 433 | @doc """ |
| 298 434 | Sends an HTTP request and returns a `Finch.Response` struct. |
| 299 435 | |
| @@ -302,31 +438,45 @@ defmodule Finch do | |
| 302 438 | * `:pool_timeout` - This timeout is applied when we check out a connection from the pool. |
| 303 439 | Default value is `5_000`. |
| 304 440 | |
| 305 | - * `:receive_timeout` - The maximum time to wait for a response before returning an error. |
| 441 | + * `:receive_timeout` - The maximum time to wait for each chunk to be received before returning an error. |
| 306 442 | Default value is `15_000`. |
| 307 443 | |
| 444 | + * `:request_timeout` - The amount of time to wait for a complete response before returning an error. |
| 445 | + This timeout only applies to HTTP/1, and its current implementation is a best effort timeout, |
| 446 | + it does not guarantee the call will return precisely when the time has elapsed. |
| 447 | + Default value is `:infinity`. |
| 448 | + |
| 308 449 | """ |
| 309 | - @spec request(Request.t(), name(), keyword()) :: |
| 450 | + @spec request(Request.t(), name(), request_opts()) :: |
| 310 451 | {:ok, Response.t()} |
| 311 452 | | {:error, Exception.t()} |
| 312 453 | def request(req, name, opts \\ []) |
| 313 454 | |
| 314 455 | def request(%Request{} = req, name, opts) do |
| 315 456 | request_span req, name do |
| 316 | - acc = {nil, [], []} |
| 457 | + acc = {nil, [], [], []} |
| 317 458 | |
| 318 459 | fun = fn |
| 319 | - {:status, value}, {_, headers, body} -> {value, headers, body} |
| 320 | - {:headers, value}, {status, headers, body} -> {status, headers ++ value, body} |
| 321 | - {:data, value}, {status, headers, body} -> {status, headers, [value | body]} |
| 460 | + {:status, value}, {_, headers, body, trailers} -> |
| 461 | + {:cont, {value, headers, body, trailers}} |
| 462 | + |
| 463 | + {:headers, value}, {status, headers, body, trailers} -> |
| 464 | + {:cont, {status, headers ++ value, body, trailers}} |
| 465 | + |
| 466 | + {:data, value}, {status, headers, body, trailers} -> |
| 467 | + {:cont, {status, headers, [value | body], trailers}} |
| 468 | + |
| 469 | + {:trailers, value}, {status, headers, body, trailers} -> |
| 470 | + {:cont, {status, headers, body, trailers ++ value}} |
| 322 471 | end |
| 323 472 | |
| 324 | - with {:ok, {status, headers, body}} <- __stream__(req, name, acc, fun, opts) do |
| 473 | + with {:ok, {status, headers, body, trailers}} <- __stream__(req, name, acc, fun, opts) do |
| 325 474 | {:ok, |
| 326 475 | %Response{ |
| 327 476 | status: status, |
| 328 477 | headers: headers, |
| 329 | - body: body |> Enum.reverse() |> IO.iodata_to_binary() |
| 478 | + body: body |> Enum.reverse() |> IO.iodata_to_binary(), |
| 479 | + trailers: trailers |
| 330 480 | }} |
| 331 481 | end |
| 332 482 | end |
| @@ -351,7 +501,7 @@ defmodule Finch do | |
| 351 501 | |
| 352 502 | See `request/3` for more detailed information. |
| 353 503 | """ |
| 354 | - @spec request!(Request.t(), name(), keyword()) :: |
| 504 | + @spec request!(Request.t(), name(), request_opts()) :: |
| 355 505 | Response.t() |
| 356 506 | def request!(%Request{} = req, name, opts \\ []) do |
| 357 507 | case request(req, name, opts) do |
| @@ -359,4 +509,127 @@ defmodule Finch do | |
| 359 509 | {:error, exception} -> raise exception |
| 360 510 | end |
| 361 511 | end |
| 512 | + |
| 513 | + @doc """ |
| 514 | + Sends an HTTP request asynchronously, returning a request reference. |
| 515 | + |
| 516 | + If the request is sent using HTTP1, an extra process is spawned to |
| 517 | + consume messages from the underlying socket. The messages are sent |
| 518 | + to the current process as soon as they arrive, as a firehose. If |
| 519 | + you wish to maximize request rate or have more control over how |
| 520 | + messages are streamed, a strategy using `request/3` or `stream/5` |
| 521 | + should be used instead. |
| 522 | + |
| 523 | + ## Receiving the response |
| 524 | + |
| 525 | + Response information is sent to the calling process as it is received |
| 526 | + in `{ref, response}` tuples. |
| 527 | + |
| 528 | + If the calling process exits before the request has completed, the |
| 529 | + request will be canceled. |
| 530 | + |
| 531 | + Responses include: |
| 532 | + |
| 533 | + * `{:status, status}` - HTTP response status |
| 534 | + * `{:headers, headers}` - HTTP response headers |
| 535 | + * `{:data, data}` - section of the HTTP response body |
| 536 | + * `{:error, exception}` - an error occurred during the request |
| 537 | + * `:done` - request has completed successfully |
| 538 | + |
| 539 | + On a successful request, a single `:status` message will be followed |
| 540 | + by a single `:headers` message, after which more than one `:data` |
| 541 | + messages may be sent. If trailing headers are present, a final |
| 542 | + `:headers` message may be sent. Any `:done` or `:error` message |
| 543 | + indicates that the request has succeeded or failed and no further |
| 544 | + messages are expected. |
| 545 | + |
| 546 | + ## Example |
| 547 | + |
| 548 | + iex> req = Finch.build(:get, "https://httpbin.org/stream/5") |
| 549 | + iex> ref = Finch.async_request(req, MyFinch) |
| 550 | + iex> flush() |
| 551 | + {ref, {:status, 200}} |
| 552 | + {ref, {:headers, [...]}} |
| 553 | + {ref, {:data, "..."}} |
| 554 | + {ref, :done} |
| 555 | + |
| 556 | + ## Options |
| 557 | + |
| 558 | + Shares options with `request/3`. |
| 559 | + """ |
| 560 | + @spec async_request(Request.t(), name(), request_opts()) :: request_ref() |
| 561 | + def async_request(%Request{} = req, name, opts \\ []) do |
| 562 | + {pool, pool_mod} = get_pool(req, name) |
| 563 | + pool_mod.async_request(pool, req, opts) |
| 564 | + end |
| 565 | + |
| 566 | + @doc """ |
| 567 | + Cancels a request sent with `async_request/3`. |
| 568 | + """ |
| 569 | + @spec cancel_async_request(request_ref()) :: :ok |
| 570 | + def cancel_async_request(request_ref) when Finch.Pool.is_request_ref(request_ref) do |
| 571 | + {pool_mod, _cancel_ref} = request_ref |
| 572 | + pool_mod.cancel_async_request(request_ref) |
| 573 | + end |
| 574 | + |
| 575 | + defp get_pool(%Request{scheme: scheme, unix_socket: unix_socket}, name) |
| 576 | + when is_binary(unix_socket) do |
| 577 | + PoolManager.get_pool(name, {scheme, {:local, unix_socket}, 0}) |
| 578 | + end |
| 579 | + |
| 580 | + defp get_pool(%Request{scheme: scheme, host: host, port: port}, name) do |
| 581 | + PoolManager.get_pool(name, {scheme, host, port}) |
| 582 | + end |
| 583 | + |
| 584 | + @doc """ |
| 585 | + Get pool metrics list. |
| 586 | + |
| 587 | + The number of items present on the metrics list depends on the `:count` option |
| 588 | + each metric will have a `pool_index` going from 1 to `:count`. |
| 589 | + |
| 590 | + The metrics struct depends on the pool scheme defined on the `:protocols` option |
| 591 | + `Finch.HTTP1.PoolMetrics` for `:http1` and `Finch.HTTP2.PoolMetrics` for `:http2`. |
| 592 | + |
| 593 | + See the `Finch.HTTP1.PoolMetrics` and `Finch.HTTP2.PoolMetrics` for more details. |
| 594 | + |
| 595 | + `{:error, :not_found}` may return on 2 scenarios: |
| 596 | + - There is no pool registered for the given pair finch instance and url |
| 597 | + - The pool is configured with `start_pool_metrics?` option false (default) |
| 598 | + |
| 599 | + ## Example |
| 600 | + |
| 601 | + iex> Finch.get_pool_status(MyFinch, "https://httpbin.org") |
| 602 | + {:ok, [ |
| 603 | + %Finch.HTTP1.PoolMetrics{ |
| 604 | + pool_index: 1, |
| 605 | + pool_size: 50, |
| 606 | + available_connections: 43, |
| 607 | + in_use_connections: 7 |
| 608 | + }, |
| 609 | + %Finch.HTTP1.PoolMetrics{ |
| 610 | + pool_index: 2, |
| 611 | + pool_size: 50, |
| 612 | + available_connections: 37, |
| 613 | + in_use_connections: 13 |
| 614 | + }] |
| 615 | + } |
| 616 | + """ |
| 617 | + @spec get_pool_status(name(), url :: String.t() | scheme_host_port()) :: |
| 618 | + {:ok, list(Finch.HTTP1.PoolMetrics.t())} |
| 619 | + | {:ok, list(Finch.HTTP2.PoolMetrics.t())} |
| 620 | + | {:error, :not_found} |
| 621 | + def get_pool_status(finch_name, url) when is_binary(url) do |
| 622 | + {s, h, p, _, _} = Request.parse_url(url) |
| 623 | + get_pool_status(finch_name, {s, h, p}) |
| 624 | + end |
| 625 | + |
| 626 | + def get_pool_status(finch_name, shp) when is_tuple(shp) do |
| 627 | + case PoolManager.get_pool(finch_name, shp, auto_start?: false) do |
| 628 | + {_pool, pool_mod} -> |
| 629 | + pool_mod.get_pool_status(finch_name, shp) |
| 630 | + |
| 631 | + :not_found -> |
| 632 | + {:error, :not_found} |
| 633 | + end |
| 634 | + end |
| 362 635 | end |
| @@ -1,7 +1,6 @@ | |
| 1 | - defmodule Finch.Conn do |
| 1 | + defmodule Finch.HTTP1.Conn do |
| 2 2 | @moduledoc false |
| 3 3 | |
| 4 | - alias Finch.MintHTTP1 |
| 5 4 | alias Finch.SSL |
| 6 5 | alias Finch.Telemetry |
| 7 6 | |
| @@ -38,11 +37,17 @@ defmodule Finch.Conn do | |
| 38 37 | |
| 39 38 | start_time = Telemetry.start(:connect, meta) |
| 40 39 | |
| 41 | - # We have to use Mint's top-level connect function or else proxying won't work. So we |
| 42 | - # force the connection to use http1 and call it in this roundabout way. |
| 43 | - conn_opts = Keyword.merge(conn.opts, mode: :passive, protocols: [:http1]) |
| 40 | + # By default we force HTTP1, but we allow someone to set |
| 41 | + # custom protocols in case they don't know if a connection |
| 42 | + # is HTTP1/HTTP2, but they are fine as treating HTTP2 |
| 43 | + # connections has HTTP2. |
| 44 44 | |
| 45 | - case MintHTTP1.connect(conn.scheme, conn.host, conn.port, conn_opts) do |
| 45 | + conn_opts = |
| 46 | + conn.opts |
| 47 | + |> Keyword.put(:mode, :passive) |
| 48 | + |> Keyword.put_new(:protocols, [:http1]) |
| 49 | + |
| 50 | + case Mint.HTTP.connect(conn.scheme, conn.host, conn.port, conn_opts) do |
| 46 51 | {:ok, mint} -> |
| 47 52 | Telemetry.stop(:connect, start_time, meta) |
| 48 53 | SSL.maybe_log_secrets(conn.scheme, conn_opts, mint) |
| @@ -56,8 +61,8 @@ defmodule Finch.Conn do | |
| 56 61 | end |
| 57 62 | |
| 58 63 | def transfer(conn, pid) do |
| 59 | - case MintHTTP1.controlling_process(conn.mint, pid) do |
| 60 | - # MintHTTP1.controlling_process causes a side-effect, but it doesn't actually |
| 64 | + case Mint.HTTP.controlling_process(conn.mint, pid) do |
| 65 | + # Mint.HTTP.controlling_process causes a side-effect, but it doesn't actually |
| 61 66 | # change the conn, so we can ignore the value returned above. |
| 62 67 | {:ok, _} -> {:ok, conn} |
| 63 68 | {:error, error} -> {:error, conn, error} |
| @@ -65,7 +70,7 @@ defmodule Finch.Conn do | |
| 65 70 | end |
| 66 71 | |
| 67 72 | def open?(%{mint: nil}), do: false |
| 68 | - def open?(%{mint: mint}), do: MintHTTP1.open?(mint) |
| 73 | + def open?(%{mint: mint}), do: Mint.HTTP.open?(mint) |
| 69 74 | |
| 70 75 | def idle_time(conn, unit \\ :native) do |
| 71 76 | idle_time = System.monotonic_time() - conn.last_checkin |
| @@ -77,7 +82,7 @@ defmodule Finch.Conn do | |
| 77 82 | def reusable?(%{max_idle_time: max_idle_time}, idle_time), do: idle_time <= max_idle_time |
| 78 83 | |
| 79 84 | def set_mode(conn, mode) when mode in [:active, :passive] do |
| 80 | - case MintHTTP1.set_mode(conn.mint, mode) do |
| 85 | + case Mint.HTTP.set_mode(conn.mint, mode) do |
| 81 86 | {:ok, mint} -> {:ok, %{conn | mint: mint}} |
| 82 87 | _ -> {:error, "Connection is dead"} |
| 83 88 | end |
| @@ -86,16 +91,16 @@ defmodule Finch.Conn do | |
| 86 91 | def discard(%{mint: nil}, _), do: :unknown |
| 87 92 | |
| 88 93 | def discard(conn, message) do |
| 89 | - case MintHTTP1.stream(conn.mint, message) do |
| 94 | + case Mint.HTTP.stream(conn.mint, message) do |
| 90 95 | {:ok, mint, _responses} -> {:ok, %{conn | mint: mint}} |
| 91 96 | {:error, _, reason, _} -> {:error, reason} |
| 92 97 | :unknown -> :unknown |
| 93 98 | end |
| 94 99 | end |
| 95 100 | |
| 96 | - def request(%{mint: nil} = conn, _, _, _, _, _), do: {:error, conn, "Could not connect"} |
| 101 | + def request(%{mint: nil} = conn, _, _, _, _, _, _), do: {:error, conn, "Could not connect"} |
| 97 102 | |
| 98 | - def request(conn, req, acc, fun, receive_timeout, idle_time) do |
| 103 | + def request(conn, req, acc, fun, receive_timeout, request_timeout, idle_time) do |
| 99 104 | full_path = Finch.Request.request_path(req) |
| 100 105 | |
| 101 106 | metadata = %{request: req} |
| @@ -105,7 +110,7 @@ defmodule Finch.Conn do | |
| 105 110 | start_time = Telemetry.start(:send, metadata, extra_measurements) |
| 106 111 | |
| 107 112 | try do |
| 108 | - case MintHTTP1.request( |
| 113 | + case Mint.HTTP.request( |
| 109 114 | conn.mint, |
| 110 115 | req.method, |
| 111 116 | full_path, |
| @@ -113,11 +118,25 @@ defmodule Finch.Conn do | |
| 113 118 | stream_or_body(req.body) |
| 114 119 | ) do |
| 115 120 | {:ok, mint, ref} -> |
| 116 | - case maybe_stream_request_body(mint, ref, req.body, receive_timeout) do |
| 121 | + case maybe_stream_request_body(mint, ref, req.body) do |
| 117 122 | {:ok, mint} -> |
| 118 123 | Telemetry.stop(:send, start_time, metadata, extra_measurements) |
| 119 124 | start_time = Telemetry.start(:recv, metadata, extra_measurements) |
| 120 | - response = receive_response([], acc, fun, mint, ref, receive_timeout) |
| 125 | + resp_metadata = %{status: nil, headers: [], trailers: []} |
| 126 | + timeouts = %{receive_timeout: receive_timeout, request_timeout: request_timeout} |
| 127 | + |
| 128 | + response = |
| 129 | + receive_response( |
| 130 | + [], |
| 131 | + acc, |
| 132 | + fun, |
| 133 | + mint, |
| 134 | + ref, |
| 135 | + timeouts, |
| 136 | + :headers, |
| 137 | + resp_metadata |
| 138 | + ) |
| 139 | + |
| 121 140 | handle_response(response, conn, metadata, start_time, extra_measurements) |
| 122 141 | |
| 123 142 | {:error, mint, error} -> |
| @@ -151,17 +170,17 @@ defmodule Finch.Conn do | |
| 151 170 | {:error, %{conn | mint: mint}, error} |
| 152 171 | end |
| 153 172 | |
| 154 | - defp maybe_stream_request_body(mint, ref, {:stream, stream}, _timeout) do |
| 173 | + defp maybe_stream_request_body(mint, ref, {:stream, stream}) do |
| 155 174 | with {:ok, mint} <- stream_request_body(mint, ref, stream) do |
| 156 | - MintHTTP1.stream_request_body(mint, ref, :eof) |
| 175 | + Mint.HTTP.stream_request_body(mint, ref, :eof) |
| 157 176 | end |
| 158 177 | end |
| 159 178 | |
| 160 | - defp maybe_stream_request_body(mint, _, _, _), do: {:ok, mint} |
| 179 | + defp maybe_stream_request_body(mint, _, _), do: {:ok, mint} |
| 161 180 | |
| 162 181 | defp stream_request_body(mint, ref, stream) do |
| 163 182 | Enum.reduce_while(stream, {:ok, mint}, fn |
| 164 | - chunk, {:ok, mint} -> {:cont, MintHTTP1.stream_request_body(mint, ref, chunk)} |
| 183 | + chunk, {:ok, mint} -> {:cont, Mint.HTTP.stream_request_body(mint, ref, chunk)} |
| 165 184 | _chunk, error -> {:halt, error} |
| 166 185 | end) |
| 167 186 | end |
| @@ -169,79 +188,182 @@ defmodule Finch.Conn do | |
| 169 188 | def close(%{mint: nil} = conn), do: conn |
| 170 189 | |
| 171 190 | def close(conn) do |
| 172 | - {:ok, mint} = MintHTTP1.close(conn.mint) |
| 191 | + {:ok, mint} = Mint.HTTP.close(conn.mint) |
| 173 192 | %{conn | mint: mint} |
| 174 193 | end |
| 175 194 | |
| 176 195 | defp handle_response(response, conn, metadata, start_time, extra_measurements) do |
| 177 196 | case response do |
| 178 | - {:ok, mint, acc, {status, headers}} -> |
| 179 | - metadata = Map.merge(metadata, %{status: status, headers: headers}) |
| 197 | + {:ok, mint, acc, resp_metadata} -> |
| 198 | + metadata = Map.merge(metadata, resp_metadata) |
| 180 199 | Telemetry.stop(:recv, start_time, metadata, extra_measurements) |
| 181 200 | {:ok, %{conn | mint: mint}, acc} |
| 182 201 | |
| 183 | - {:error, mint, error, {status, headers}} -> |
| 184 | - metadata = Map.merge(metadata, %{error: error, status: status, headers: headers}) |
| 202 | + {:error, mint, error, resp_metadata} -> |
| 203 | + metadata = Map.merge(metadata, Map.put(resp_metadata, :error, error)) |
| 185 204 | Telemetry.stop(:recv, start_time, metadata, extra_measurements) |
| 186 205 | {:error, %{conn | mint: mint}, error} |
| 187 206 | end |
| 188 207 | end |
| 189 208 | |
| 190 | - defp receive_response(entries, acc, fun, mint, ref, timeout, status \\ nil, headers \\ []) |
| 209 | + defp receive_response( |
| 210 | + entries, |
| 211 | + acc, |
| 212 | + fun, |
| 213 | + mint, |
| 214 | + ref, |
| 215 | + timeouts, |
| 216 | + fields, |
| 217 | + resp_metadata |
| 218 | + ) |
| 191 219 | |
| 192 | - defp receive_response([], acc, fun, mint, ref, timeout, status, headers) do |
| 193 | - case MintHTTP1.recv(mint, 0, timeout) do |
| 220 | + defp receive_response( |
| 221 | + [{:done, ref} | _], |
| 222 | + acc, |
| 223 | + _fun, |
| 224 | + mint, |
| 225 | + ref, |
| 226 | + _timeouts, |
| 227 | + _fields, |
| 228 | + resp_metadata |
| 229 | + ) do |
| 230 | + {:ok, mint, acc, resp_metadata} |
| 231 | + end |
| 232 | + |
| 233 | + defp receive_response( |
| 234 | + _, |
| 235 | + _acc, |
| 236 | + _fun, |
| 237 | + mint, |
| 238 | + _ref, |
| 239 | + timeouts, |
| 240 | + _fields, |
| 241 | + resp_metadata |
| 242 | + ) |
| 243 | + when timeouts.request_timeout < 0 do |
| 244 | + {:ok, mint} = Mint.HTTP1.close(mint) |
| 245 | + {:error, mint, %Mint.TransportError{reason: :timeout}, resp_metadata} |
| 246 | + end |
| 247 | + |
| 248 | + defp receive_response( |
| 249 | + [], |
| 250 | + acc, |
| 251 | + fun, |
| 252 | + mint, |
| 253 | + ref, |
| 254 | + timeouts, |
| 255 | + fields, |
| 256 | + resp_metadata |
| 257 | + ) do |
| 258 | + start_time = System.monotonic_time(:millisecond) |
| 259 | + |
| 260 | + case Mint.HTTP.recv(mint, 0, timeouts.receive_timeout) do |
| 194 261 | {:ok, mint, entries} -> |
| 195 | - receive_response(entries, acc, fun, mint, ref, timeout, status, headers) |
| 262 | + timeouts = |
| 263 | + if is_integer(timeouts.request_timeout) do |
| 264 | + elapsed_time = System.monotonic_time(:millisecond) - start_time |
| 265 | + update_in(timeouts.request_timeout, &(&1 - elapsed_time)) |
| 266 | + else |
| 267 | + timeouts |
| 268 | + end |
| 269 | + |
| 270 | + receive_response( |
| 271 | + entries, |
| 272 | + acc, |
| 273 | + fun, |
| 274 | + mint, |
| 275 | + ref, |
| 276 | + timeouts, |
| 277 | + fields, |
| 278 | + resp_metadata |
| 279 | + ) |
| 196 280 | |
| 197 281 | {:error, mint, error, _responses} -> |
| 198 | - {:error, mint, error, {status, headers}} |
| 282 | + {:error, mint, error, resp_metadata} |
| 199 283 | end |
| 200 284 | end |
| 201 285 | |
| 202 | - defp receive_response([entry | entries], acc, fun, mint, ref, timeout, status, headers) do |
| 286 | + defp receive_response( |
| 287 | + [entry | entries], |
| 288 | + acc, |
| 289 | + fun, |
| 290 | + mint, |
| 291 | + ref, |
| 292 | + timeouts, |
| 293 | + fields, |
| 294 | + resp_metadata |
| 295 | + ) do |
| 203 296 | case entry do |
| 204 297 | {:status, ^ref, value} -> |
| 205 | - receive_response( |
| 206 | - entries, |
| 207 | - fun.({:status, value}, acc), |
| 208 | - fun, |
| 209 | - mint, |
| 210 | - ref, |
| 211 | - timeout, |
| 212 | - value, |
| 213 | - headers |
| 214 | - ) |
| 298 | + case fun.({:status, value}, acc) do |
| 299 | + {:cont, acc} -> |
| 300 | + receive_response( |
| 301 | + entries, |
| 302 | + acc, |
| 303 | + fun, |
| 304 | + mint, |
| 305 | + ref, |
| 306 | + timeouts, |
| 307 | + fields, |
| 308 | + %{resp_metadata | status: value} |
| 309 | + ) |
| 310 | + |
| 311 | + {:halt, acc} -> |
| 312 | + {:ok, mint} = Mint.HTTP1.close(mint) |
| 313 | + {:ok, mint, acc, resp_metadata} |
| 314 | + |
| 315 | + other -> |
| 316 | + raise ArgumentError, "expected {:cont, acc} or {:halt, acc}, got: #{inspect(other)}" |
| 317 | + end |
| 215 318 | |
| 216 319 | {:headers, ^ref, value} -> |
| 217 | - receive_response( |
| 218 | - entries, |
| 219 | - fun.({:headers, value}, acc), |
| 220 | - fun, |
| 221 | - mint, |
| 222 | - ref, |
| 223 | - timeout, |
| 224 | - status, |
| 225 | - headers ++ value |
| 226 | - ) |
| 320 | + resp_metadata = update_in(resp_metadata, [fields], &(&1 ++ value)) |
| 321 | + |
| 322 | + case fun.({fields, value}, acc) do |
| 323 | + {:cont, acc} -> |
| 324 | + receive_response( |
| 325 | + entries, |
| 326 | + acc, |
| 327 | + fun, |
| 328 | + mint, |
| 329 | + ref, |
| 330 | + timeouts, |
| 331 | + fields, |
| 332 | + resp_metadata |
| 333 | + ) |
| 334 | + |
| 335 | + {:halt, acc} -> |
| 336 | + {:ok, mint} = Mint.HTTP1.close(mint) |
| 337 | + {:ok, mint, acc, resp_metadata} |
| 338 | + |
| 339 | + other -> |
| 340 | + raise ArgumentError, "expected {:cont, acc} or {:halt, acc}, got: #{inspect(other)}" |
| 341 | + end |
| 227 342 | |
| 228 343 | {:data, ^ref, value} -> |
| 229 | - receive_response( |
| 230 | - entries, |
| 231 | - fun.({:data, value}, acc), |
| 232 | - fun, |
| 233 | - mint, |
| 234 | - ref, |
| 235 | - timeout, |
| 236 | - status, |
| 237 | - headers |
| 238 | - ) |
| 344 | + case fun.({:data, value}, acc) do |
| 345 | + {:cont, acc} -> |
| 346 | + receive_response( |
| 347 | + entries, |
| 348 | + acc, |
| 349 | + fun, |
| 350 | + mint, |
| 351 | + ref, |
| 352 | + timeouts, |
| 353 | + :trailers, |
| 354 | + resp_metadata |
| 355 | + ) |
| 239 356 | |
| 240 | - {:done, ^ref} -> |
| 241 | - {:ok, mint, acc, {status, headers}} |
| 357 | + {:halt, acc} -> |
| 358 | + {:ok, mint} = Mint.HTTP1.close(mint) |
| 359 | + {:ok, mint, acc, resp_metadata} |
| 360 | + |
| 361 | + other -> |
| 362 | + raise ArgumentError, "expected {:cont, acc} or {:halt, acc}, got: #{inspect(other)}" |
| 363 | + end |
| 242 364 | |
| 243 365 | {:error, ^ref, error} -> |
| 244 | - {:error, mint, error, {status, headers}} |
| 366 | + {:error, mint, error, resp_metadata} |
| 245 367 | end |
| 246 368 | end |
| 247 369 | end |
Loading more files…