Current section

6 Versions

Jump to

Compare versions

7 files changed
+332 additions
-36 deletions
  @@ -1,5 +1,15 @@
1 1 # Changelog for Rackla
2 2
3 + ## 1.1
4 + - Added Rackla.Proxy to support SOCKS5 and HTTP tunnel proxies.
5 + - Added option :max_redirect to set the number of redirects.
6 + - Added option :force_redirect to force redirect (e.g. POST).
7 + - Added incoming_request to convert an incoming Plug request to a Rackla.Request.
8 + - Removed compiler warnings caused by Elixir 1.3.
9 + - Removed compiler warnings from tests.
10 + - Updated all dependencies.
11 + - Thanks JustMikey & beno!
12 +
3 13 ## 1.0.1
4 14 - Added option to follow redirect. Thanks bAmpT!
5 15 - Updated all dependencies.
  @@ -18,7 +18,7 @@ You can add Rackla to your existing application by adding the following Mix depe
18 18 ```elixir
19 19 defp deps do
20 20 [
21 - {:rackla, "~> 1.0"},
21 + {:rackla, "~> 1.1"},
22 22 {:cowboy, "~> 1.0"} # Or your web server of choice (which works with Plug)
23 23 ]
24 24 end
  @@ -226,8 +226,101 @@ What is cool about this approach is that the image requests are executed concurr
226 226
227 227 We will also notice, in this example, that the order is nondeterministic - meaning that we will (most likely) get a different order in which the images are sent every time we refresh the page. If we wanted to preserve the ordering of the images, we could either send the ordering with the chunks and let the client code render the images on the correct position - or we could set the `:sync` option to `true` in `response` which would then wait for the responses and then send them in the appropriate order.
228 228
229 + ### Basic authentication
230 + Sending the username and password as headers:
231 +
232 + ```elixir
233 + get "/auth-example" do
234 + url = "http://some-url.com"
235 + headers = %{"Authorization" => "Basic #{Base.encode64("username:password")}"}
236 +
237 + %Rackla.Request{url: url, headers: headers}
238 + |> request
239 + |> response
240 + end
241 + ```
242 +
243 + Adding them as part of the URL:
244 +
245 + ```elixir
246 + get "/auth-example" do
247 + username = "my_username"
248 + password = "my_password"
249 +
250 + "http://#{username}:#{password}@some-url.com"
251 + |> request
252 + |> response
253 + end
254 + ```
255 +
256 + ### Using a SOCKS5 Proxy or HTTP (Connect) Tunnel
257 + For more detailed information about using proxies, see the documentation for `Rackla.Proxy`.
258 +
259 + #### SOCKS5 using request setting
260 + ```elixir
261 + %Rackla.Request{url: "http://api.ipify.org", options: %{proxy: %Rackla.Proxy{type: :socks5, host: "localhost", port: 8080}}}
262 + |> Rackla.request
263 + |> Rackla.collect
264 + ```
265 +
266 + #### HTTP Tunnel using global setting
267 + ```elixir
268 + "http://api.ipify.org"
269 + |> Rackla.request(proxy: %Rackla.Proxy{type: :connect, host: "localhost", port: 8080})
270 + |> Rackla.collect
271 + ```
272 +
273 + ### Adding processing time to a JSON response
274 + ```elixir
275 + get "/resp-time" do
276 + current_millis = fn() ->
277 + {mega, sec, micro} = :os.timestamp
278 + (mega*1000000 + sec)*1000 + round(micro/1000)
279 + end
280 +
281 + urls = ["http://date.jsontest.com/", "http://api.openweathermap.org/data/2.5/weather?q=Malmo,se"]
282 +
283 + start_time = current_millis.()
284 +
285 + urls
286 + |> request
287 + |> map(fn(http_response) ->
288 +
289 + end_time = current_millis.() - start_time
290 +
291 + case http_response do
292 + {:error, reason} ->
293 + "HTTP request failed because: #{reason} in time #{end_time}"
294 +
295 + ok_response ->
296 + case Poison.decode(ok_response) do
297 + {:ok, json_decoded} ->
298 + Map.put(json_decoded, "response_time", end_time)
299 +
300 + {:error, reason} ->
301 + "Failed to decode response because: #{inspect(reason)} in time #{end_time}"
302 + end
303 + end
304 + end)
305 + |> response(json: true)
306 + end
307 + ```
308 +
309 + ### Reverse proxy / Request forwarding
310 + ```elixir
311 + get "/test/a-simple-request-proxy" do
312 + # You should check for errors
313 + {:ok, the_request} = incoming_request()
314 +
315 + the_request
316 + |> Map.put(:url, "http://new-url.com")
317 + |> request
318 + |> response
319 + end
320 + ```
321 +
229 322 ### More examples
230 - A collection of smaller example end-points can be found in found [lib/rackla/rackla.ex](https://github.com/AntonFagerberg/rackla/blob/master/lib/router.ex) which illustrates additional techniques that can be used in Rackla.
323 + A collection of smaller example end-points can be found in found [lib/rackla/router.ex](https://github.com/AntonFagerberg/rackla/blob/master/lib/router.ex) which illustrates additional techniques that can be used in Rackla.
231 324
232 325 ## The Rackla type
233 326 `Rackla` is also the name of the type used in all of Rackla's functions. Internally, it consists of a list of Elixir processes which communicate with message passing according to a protocol defined inside Rackla (these processes can themselves contain even more nested `Rackla` types). The `Rackla` type should never be modified directly!
  @@ -396,6 +489,28 @@ Options:
396 489 * `:json` - If set to true, the encapsulated elements will be converted into
397 490 a JSON encoded string before they are sent to the client. This will also set
398 491 the header "Content-Type" to the appropriate "application/json; charset=utf-8".
492 +
493 + ### incoming_request
494 + Convert an incoming request (from `Plug`) to a `Rackla.Request`.
495 + If `options` is specified, it will be added to the `Rackla.Request`.
496 + For valid options, see documentation for `Rackla.Request`.
497 +
498 + Returns either `{:ok, Rackla.Request}` or `{:error, reason}` as per
499 + `:gen_tcp.recv/2`.
500 +
501 + The `Plug.Conn` will be taken implicitly by looking for a variable named
502 + `conn`. If you want to specify which `Plug.Conn` to use, you can use
503 + `Rackla.incoming_request_conn`.
504 +
505 + Using this macro is the same as writing:
506 + `conn = incoming_request_conn(conn, options)`
507 +
508 + From `Plug.Conn` documentation:
509 + Because the request body can be of any size, reading the body will only work
510 + once, as Plug will not cache the result of these operations. If you need to
511 + access the body multiple times, it is your responsibility to store it. Finally
512 + keep in mind some plugs like Plug.Parsers may read the body, so the body may
513 + be unavailable after being accessed by such plugs.
399 514
400 515 ## License
401 516 Rackla source code is released under Apache 2 License. Check LICENSE file for more information.
  @@ -4,8 +4,8 @@
4 4 {<<"elixir">>,<<"~> 1.0">>}.
5 5 {<<"files">>,
6 6 [<<"lib/rackla/rackla.ex">>,<<"lib/rackla/request.ex">>,
7 - <<"lib/rackla/response.ex">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE">>,
8 - <<"CHANGELOG.md">>]}.
7 + <<"lib/rackla/response.ex">>,<<"lib/rackla/proxy.ex">>,<<"mix.exs">>,
8 + <<"README.md">>,<<"LICENSE">>,<<"CHANGELOG.md">>]}.
9 9 {<<"licenses">>,[<<"Apache 2">>]}.
10 10 {<<"links">>,[{<<"GitHub">>,<<"https://github.com/AntonFagerberg/rackla">>}]}.
11 11 {<<"maintainers">>,[<<"Anton Fagerberg">>]}.
  @@ -14,7 +14,7 @@
14 14 [[{<<"app">>,<<"poison">>},
15 15 {<<"name">>,<<"poison">>},
16 16 {<<"optional">>,false},
17 - {<<"requirement">>,<<"~> 2.1">>}],
17 + {<<"requirement">>,<<"~> 2.2">>}],
18 18 [{<<"app">>,<<"hackney">>},
19 19 {<<"name">>,<<"hackney">>},
20 20 {<<"optional">>,false},
  @@ -26,5 +26,5 @@
26 26 [{<<"app">>,<<"plug">>},
27 27 {<<"name">>,<<"plug">>},
28 28 {<<"optional">>,false},
29 - {<<"requirement">>,<<"~> 1.1">>}]]}.
30 - {<<"version">>,<<"1.0.1">>}.
29 + {<<"requirement">>,<<"~> 1.2">>}]]}.
30 + {<<"version">>,<<"1.1.0">>}.
  @@ -0,0 +1,31 @@
1 + defmodule Rackla.Proxy do
2 + @moduledoc """
3 + `Rackla.Proxy` settings struct for using a proxy in outbound requests.
4 +
5 + Required:
6 + * `type` - `:socks5` (SOCKS5) or `:connect` (HTTP tunnel), default: `:socks5`.
7 + * `host` - Host, example: `"127.0.0.1"` or `"localhost"`, default: `"localhost"`.
8 + * `port` - Port, example: `8080`, default: `8080`.
9 +
10 + Optional:
11 + * `username` - Username for proxy, default: `nil`.
12 + * `password` - Password for proxy, default: `nil`.
13 + * `pool` - [Experimental!] Define one pool per proxy if you wish to use
14 + several proxies simultaneously, default: `nil`.
15 + """
16 +
17 + @type t :: %__MODULE__{
18 + type: atom,
19 + host: binary,
20 + port: integer,
21 + username: binary,
22 + password: binary,
23 + pool: atom}
24 +
25 + defstruct type: :socks5,
26 + host: "localhost",
27 + port: 8080,
28 + username: nil,
29 + password: nil,
30 + pool: nil
31 + end
\ No newline at end of file
  @@ -1,5 +1,5 @@
1 1 defmodule Rackla do
2 - @moduledoc Regex.replace(~r/```(elixir|json)(\n|.*)```/rs, File.read!("README.md"),
2 + @moduledoc Regex.replace(~r/```(elixir|json)(\n|.*)```/Us, File.read!("README.md"),
3 3 fn(_, _, code) -> Regex.replace(~r/^/m, code, " ") end)
4 4
5 5 import Plug.Conn
  @@ -33,6 +33,9 @@ defmodule Rackla do
33 33 default: `false`.
34 34 * `:follow_redirect` - If set to true, Rackla will follow redirects,
35 35 default: `false`.
36 + * `:max_redirect` - Maximum number of redirects, default: `5`.
37 + * `:force_redirect` - Force follow redirect (e.g. POST), default: `false`.
38 + * `:proxy` - Proxy to use, see `Rackla.Proxy`, default: `nil`.
36 39
37 40 If you specify any options in a `Rackla.Request` struct, these will overwrite
38 41 the options passed to the `request` function for that specific request.
  @@ -43,7 +46,12 @@ defmodule Rackla do
43 46 def request(requests, options) when is_list(requests) do
44 47 producers =
45 48 Enum.map(requests, fn(request) ->
46 - if is_binary(request), do: request = %Rackla.Request{url: request}
49 + request =
50 + if is_binary(request) do
51 + %Rackla.Request{url: request}
52 + else
53 + request
54 + end
47 55
48 56 {:ok, producer} =
49 57 Task.start_link(fn ->
  @@ -52,6 +60,45 @@ defmodule Rackla do
52 60 global_connect_timeout = Keyword.get(options, :connect_timeout, 5_000)
53 61 global_receive_timeout = Keyword.get(options, :receive_timeout, 5_000)
54 62 global_follow_redirect = Keyword.get(options, :follow_redirect, false)
63 + global_max_redirect = Keyword.get(options, :max_redirect, 5)
64 + global_force_redirect = Keyword.get(options, :force_redirect, false)
65 +
66 + global_proxy = Keyword.get(options, :proxy)
67 + request_proxy = Map.get(request_options, :proxy)
68 +
69 + rackla_proxy =
70 + cond do
71 + request_proxy -> request_proxy
72 + global_proxy -> global_proxy
73 + true -> nil
74 + end
75 +
76 + proxy_options =
77 + case rackla_proxy do
78 + %Rackla.Proxy{type: type, host: host, port: port, username: username, password: password, pool: pool} ->
79 + proxy_basic_setting = [proxy: {type, String.to_char_list(host), port}]
80 +
81 + auth_settings =
82 + case type do
83 + :socks5 ->
84 + socks5_user = if username, do: [socks5_user: username], else: []
85 + socks5_pass = if password, do: [socks5_pass: password], else: []
86 +
87 + socks5_user ++ socks5_pass
88 + :connect ->
89 + if username && password do
90 + [proxy_auth: {username, password}]
91 + else
92 + []
93 + end
94 + end
95 +
96 + pool_setting = if pool, do: [pool: pool], else: []
97 +
98 + proxy_basic_setting ++ auth_settings ++ pool_setting
99 +
100 + nil -> []
101 + end
55 102
56 103 hackney_request =
57 104 :hackney.request(
  @@ -63,11 +110,16 @@ defmodule Rackla do
63 110 insecure: Map.get(request_options, :insecure, global_insecure),
64 111 connect_timeout: Map.get(request_options, :connect_timeout, global_connect_timeout),
65 112 recv_timeout: Map.get(request_options, :receive_timeout, global_receive_timeout),
66 - follow_redirect: Map.get(request_options, :follow_redirect, global_follow_redirect)
67 - ]
113 + follow_redirect: Map.get(request_options, :follow_redirect, global_follow_redirect),
114 + max_redirect: Map.get(request_options, :max_redirect, global_max_redirect),
115 + force_redirect: Map.get(request_options, :force_redirect, global_force_redirect)
116 + ] ++ proxy_options
68 117 )
69 118
70 119 case hackney_request do
120 + {:ok, {:maybe_redirect, _, _, _}} ->
121 + warn_request(:force_redirect_disabled)
122 +
71 123 {:ok, status, headers, body_ref} ->
72 124 case :hackney.body(body_ref) do
73 125 {:ok, body} ->
  @@ -410,6 +462,83 @@ defmodule Rackla do
410 462 response_async(rackla, conn, options)
411 463 end
412 464 end
465 +
466 + @doc """
467 + Convert an incoming request (from `Plug`) to a `Rackla.Request`.
468 + If `options` is specified, it will be added to the `Rackla.Request`.
469 + For valid options, see documentation for `Rackla.Request`.
470 +
471 + Returns either `{:ok, Rackla.Request}` or `{:error, reason}` as per
472 + `:gen_tcp.recv/2`.
473 +
474 + The `Plug.Conn` will be taken implicitly by looking for a variable named
475 + `conn`. If you want to specify which `Plug.Conn` to use, you can use
476 + `Rackla.incoming_request_conn`.
477 +
478 + Using this macro is the same as writing:
479 + `conn = incoming_request_conn(conn, options)`
480 +
481 + From `Plug.Conn` documentation:
482 + Because the request body can be of any size, reading the body will only work
483 + once, as Plug will not cache the result of these operations. If you need to
484 + access the body multiple times, it is your responsibility to store it. Finally
485 + keep in mind some plugs like Plug.Parsers may read the body, so the body may
486 + be unavailable after being accessed by such plugs.
487 + """
488 + @spec incoming_request(%{}) :: {:ok, Rackla.Request.t} | {:error, atom}
489 + defmacro incoming_request(options \\ %{}) do
490 + quote do
491 + {var!(conn), rackla_request} = incoming_request_conn(var!(conn), unquote(options))
492 + _ = var!(conn) # hack to get rid of "unused variable" compiler warning
493 + rackla_request
494 + end
495 + end
496 +
497 + @doc """
498 + See documentation for `Rackla.incoming_request`.
499 + """
500 + @spec incoming_request_conn(Plug.Conn.t, %{}) :: {Plug.Conn.t, {:ok, Rackla.Request.t}} | {Plug.Conn.t, {:error, atom}}
501 + def incoming_request_conn(conn, options \\ %{}) do
502 + response_body =
503 + Stream.unfold(Plug.Conn.read_body(conn), fn
504 + :done ->
505 + nil;
506 +
507 + {:ok, body, new_conn} ->
508 + {{new_conn, body}, :done};
509 +
510 + {:more, partial_body, new_conn} ->
511 + {partial_body, Plug.Conn.read_body(new_conn)};
512 +
513 + {:error, term} ->
514 + {{:error, term}, :done}
515 + end)
516 + |> Enum.reduce({"", conn}, fn
517 + ({:error, term}, {_body_acc, conn_acc}) -> {{:error, term}, conn_acc};
518 + ({new_conn, body}, {body_acc, _conn_acc}) -> {{:ok, body_acc <> body}, new_conn};
519 + (partial_body, {body_acc, conn_acc}) -> {body_acc <> partial_body, conn_acc}
520 + end)
521 +
522 + case response_body do
523 + {{:error, term}, final_conn} -> {final_conn, {:error, term}}
524 +
525 + {{:ok, body}, final_conn} ->
526 + method = conn.method |> String.downcase |> String.to_atom
527 + url = "#{Atom.to_string(conn.scheme)}://#{conn.host}#{conn.request_path}"
528 + headers = Enum.into(conn.req_headers, %{})
529 +
530 + rackla_request =
531 + %Rackla.Request{
532 + method: method,
533 + url: url,
534 + headers: headers,
535 + body: body,
536 + options: options
537 + }
538 +
539 + {final_conn, {:ok, rackla_request}}
540 + end
541 + end
413 542
414 543 @spec response_async(t, Plug.Conn.t, Keyword.t) :: Plug.Conn.t
415 544 defp response_async(%Rackla{} = rackla, conn, options) do
  @@ -433,7 +562,7 @@ defmodule Rackla do
433 562 defp send_chunks(producers, conn) when is_list(producers) do
434 563 send_thing =
435 564 fn(thing, remaining_producers, conn) ->
436 - unless is_binary(thing), do: thing = inspect(thing)
565 + thing = if is_binary(thing), do: thing, else: inspect(thing)
437 566
438 567 case chunk(conn, thing) do
439 568 {:ok, new_conn} ->
  @@ -488,8 +617,8 @@ defmodule Rackla do
488 617 response_sync_chunk(nested_rackla, conn, options)
489 618
490 619 {^pid, thing} ->
491 - if elem(thing, 0) == :ok, do: thing = elem(thing, 1)
492 - unless is_binary(thing), do: thing = inspect(thing)
620 + thing = if elem(thing, 0) == :ok, do: elem(thing, 1), else: thing
621 + thing = if is_binary(thing), do: thing, else: inspect(thing)
493 622
494 623 case chunk(conn, thing) do
495 624 {:ok, new_conn} ->
  @@ -544,24 +673,31 @@ defmodule Rackla do
544 673 headers = Keyword.get(options, :headers, %{})
545 674 compress = Keyword.get(options, :compress, false)
546 675
547 - if compress do
548 - allow_gzip =
549 - Plug.Conn.get_req_header(conn, "accept-encoding")
550 - |> Enum.flat_map(fn(encoding) ->
551 - String.split(encoding, ",", trim: true)
552 - |> Enum.map(&String.strip/1)
553 - end)
554 - |> Enum.any?(&(Regex.match?(~r/(^(\*|gzip)(;q=(1$|1\.0{1,3}$|0\.[1-9]{1,3}$)|$))/, &1)))
555 -
556 - if allow_gzip || compress == :force do
557 - response_binary = :zlib.gzip(response_binary)
558 - headers = Map.merge(headers, %{"content-encoding" => "gzip"})
559 - end
560 - end
676 + {response_binary, headers} =
677 + if compress do
678 + allow_gzip =
679 + Plug.Conn.get_req_header(conn, "accept-encoding")
680 + |> Enum.flat_map(fn(encoding) ->
681 + String.split(encoding, ",", trim: true)
682 + |> Enum.map(&String.strip/1)
683 + end)
684 + |> Enum.any?(&(Regex.match?(~r/(^(\*|gzip)(;q=(1$|1\.0{1,3}$|0\.[1-9]{1,3}$)|$))/, &1)))
561 685
562 - if Keyword.get(options, :json, false) do
563 - conn = put_resp_content_type(conn, "application/json")
564 - end
686 + if allow_gzip || compress == :force do
687 + {:zlib.gzip(response_binary), Map.merge(headers, %{"content-encoding" => "gzip"})}
688 + else
689 + {response_binary, headers}
690 + end
691 + else
692 + {response_binary, headers}
693 + end
694 +
695 + conn =
696 + if Keyword.get(options, :json, false) do
697 + put_resp_content_type(conn, "application/json")
698 + else
699 + conn
700 + end
565 701
566 702 chunk_status =
567 703 prepare_conn(conn, Keyword.get(options, :status, 200), headers)
Loading more files…