Current section
Files
Jump to
Current section
Files
lib/loader.ex
defmodule Ports.Loader do
@moduledoc false
alias Ports.{
Country,
Function,
Port,
Status,
Subdivision
}
NimbleCSV.define(CSVParser, separator: ",", escape: "\"")
for path <- Path.wildcard("priv/data/*.csv") do
@external_resource path
end
defguard is_empty(data) when data in [nil, ""]
@code_list_sources [
"2023-2 UNLOCODE CodeListPart1.csv",
"2023-2 UNLOCODE CodeListPart2.csv",
"2023-2 UNLOCODE CodeListPart3.csv"
]
def load_code_list do
@code_list_sources
|> csv_decode()
|> Stream.map(fn [
_change,
country,
location,
name,
_name_wo_diacritics,
subdivision,
function,
status,
date,
iata,
coordinates,
_remarks
] ->
%Port{
country: country,
location: location,
name: name,
subdivision: subdivision,
status: status,
function: function,
date: date,
iata: iata,
coordinates: coordinates
}
end)
|> Stream.reject(fn
%{country: country, location: location} when is_empty(country) or is_empty(location) -> true
_ -> false
end)
|> Stream.map(fn port -> Map.update!(port, :name, &to_valid_string/1) end)
|> Enum.to_list()
end
defp to_valid_string(string) do
string |> :binary.bin_to_list() |> :unicode.characters_to_binary()
end
def load_countries do
"country-codes.csv"
|> csv_decode()
|> Stream.map(fn [
country_code,
country_name
] ->
%Country{
code: country_code,
name: country_name
}
end)
|> Enum.to_list()
end
def load_functions do
"function-classifiers.csv"
|> csv_decode()
|> Stream.map(fn [
function_code,
function_description
] ->
%Function{
code: function_code,
description: function_description
}
end)
|> Enum.to_list()
end
def load_statuses do
"status-indicators.csv"
|> csv_decode()
|> Stream.map(fn [
status_code,
status_description
] ->
%Status{
code: status_code,
description: status_description
}
end)
|> Enum.to_list()
end
def load_subdivisions do
"subdivision-codes.csv"
|> csv_decode()
|> Stream.map(fn [
status_country,
status_code,
status_name
] ->
%Subdivision{
country: status_country,
code: status_code,
name: status_name
}
end)
|> Enum.to_list()
end
defp csv_decode(file_names) do
file_names = List.wrap(file_names)
streams =
Enum.map(file_names, fn file_name ->
[:code.priv_dir(:ports), "data", file_name]
|> Path.join()
|> File.stream!()
end)
streams
|> Stream.concat()
|> CSVParser.parse_stream(skip_headers: false)
end
end