Current section

Files

Jump to
paraxial lib paraxial fetcher.ex
Raw

lib/paraxial/fetcher.ex

defmodule Paraxial.Fetcher do
@moduledoc false
require Logger
def add_cloud_ips() do
providers = %{
aws: "https://ip-ranges.amazonaws.com/ip-ranges.json",
azure: "https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519",
gcp: "https://www.gstatic.com/ipranges/cloud.json",
digital_ocean: "https://digitalocean.com/geo/google.csv",
oracle: "https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json"
}
get_prefixes_update_term(providers)
end
def get_prefixes_update_term(providers) do
# 1. Get the IP prefixes for each provider
# 2. Return a list of prefixes for each provider, if the HTTP request fails return []
new_prefixes =
Enum.map(providers, fn {cloud_provider, url} ->
Task.async(fn -> {cloud_provider, url_to_prefixes(url, cloud_provider)} end)
end)
|> Enum.map(&Task.await/1)
iptrie = prefixes_to_trie(new_prefixes)
:persistent_term.put({__MODULE__, :cloud_trie}, iptrie)
Logger.debug("[Paraxial] Iptrie count - #{Iptrie.count(iptrie)}")
iptrie_size = iptrie |> :erlang.term_to_binary() |> :erlang.byte_size()
Logger.debug("[Paraxial] Iptrie size in MB: #{iptrie_size / 1_000_000}")
end
def prefixes_to_trie(pl) do
prefixes =
pl
|> Enum.map(fn {c, prefixes} ->
Logger.debug("[Paraxial] Prefixes downloaded for #{c}: #{length(prefixes)}")
Enum.map(prefixes, fn p -> {p, c} end)
end)
|> List.flatten()
|> Enum.map(fn {prefix, c} -> {prefix, c} end)
Logger.debug("[Paraxial] Prefixes length with duplicates: #{length(prefixes)}")
Iptrie.new(prefixes)
end
def url_to_prefixes(url, :digital_ocean) do
with {:ok, %{status_code: 200, body: body}} <- HTTPoison.get(url, [], follow_redirect: true) do
body
|> String.split("\n")
|> Enum.map(fn l -> String.split(l, ",") |> hd() end)
|> Enum.filter(fn x -> String.length(x) > 0 end)
else
_ ->
[]
end
end
def url_to_prefixes(url, :oracle) do
with {:ok, %{status_code: 200, body: body}} <- HTTPoison.get(url),
{:ok, j_body} <- Jason.decode(body) do
j_body
|> Map.get("regions")
|> Enum.map(fn %{"cidrs" => c_maps} ->
Enum.map(c_maps, fn %{"cidr" => prefix} -> prefix end)
end)
|> List.flatten()
else
_ ->
[]
end
end
def url_to_prefixes(url, :gcp) do
with {:ok, %{status_code: 200, body: body}} <- HTTPoison.get(url),
{:ok, j_body} <- Jason.decode(body) do
j_body
|> Map.get("prefixes")
|> Enum.map(fn
%{"ipv4Prefix" => v} -> v
%{"ipv6Prefix" => v} -> v
end)
else
_ ->
[]
end
end
def url_to_prefixes(url, :aws) do
with {:ok, %{status_code: 200, body: body}} <- HTTPoison.get(url),
{:ok, j_body} <- Jason.decode(body) do
ipv4 = extract_prefixes_aws(j_body, "prefixes", "ip_prefix")
ipv6 = extract_prefixes_aws(j_body, "ipv6_prefixes", "ipv6_prefix")
Enum.concat([ipv4, ipv6])
else
_ ->
[]
end
end
def url_to_prefixes(url, :azure) do
with {:ok, %{status_code: 200, body: body}} <- HTTPoison.get(url),
{:ok, json_url} <- get_json_url(body),
{:ok, %{status_code: 200, body: json_body}} <-
HTTPoison.get(json_url, [], follow_redirect: true),
{:ok, jd_body} <- Jason.decode(json_body) do
extract_prefixes_azure(jd_body)
else
_ ->
[]
end
end
def get_json_url(body) do
json_url =
Regex.scan(
~r/(http|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-]\.json)/,
body
)
|> hd()
|> hd()
{:ok, json_url}
end
def extract_prefixes_aws(body, k1, k2) do
body
|> Map.get(k1)
|> Enum.map(fn %{^k2 => prefix} -> prefix end)
end
def extract_prefixes_azure(body) do
body
|> Map.get("values")
|> Enum.map(fn x -> get_in(x, ["properties", "addressPrefixes"]) end)
|> List.flatten()
end
end