Packages

Parse domain names into component parts using the Public Suffix List

Current section

Files

Jump to
psl src psl.gleam
Raw

src/psl.gleam

import gleam/dict.{type Dict}
import gleam/erlang/application
import gleam/int
import gleam/list
import gleam/option.{type Option, None, Some}
import gleam/result
import gleam/string
import gleam/uri
import simplifile
const splat = "*"
const dot = "."
const bang = "!"
const comment_marker = "//"
const private_marker = "===BEGIN PRIVATE DOMAINS==="
/// Represents the parsed components of a domain
///
/// Definitions
///
/// **top_level_domain** or TLD is the last part of a domain name before the paths.
/// For example, in https://packages.gleam.run, "run" is the TLD.
///
/// **second_level_domain** or SLD is the part of a domain name before the TLD,
/// separated by a ".". For example, in https://packages.gleam.run, "gleam" is
/// the SLD.
///
/// **transit_routing_domain** or TRD is the first part of a domain name and may
/// have more than one part. For example, in https://packages.gleam.run,
/// "packages" is the TRD and in https://cool.packages.gleam.run
/// "cool.packages" is the TRD.
///
/// We return "subdomain_parts" that splits the TRD on ".".
pub type DomainParts {
DomainParts(
top_level_domain: String,
second_level_domain: String,
transit_routing_domain: String,
subdomain_parts: List(String),
)
}
/// Errors that can occur during parsing
pub type ParseError {
InvalidUri
NoHost
InvalidDomain
UnknownSuffix
}
/// The suffix list data structure
pub opaque type SuffixList {
SuffixList(
normal: Dict(String, Suffix),
wildcards: List(String),
exceptions: Dict(String, Suffix),
)
}
// An individual suffix record.
type Suffix {
Suffix(suffix: String, is_public: Bool, length: Int)
}
/// Load the public suffix list from the a data file and choose to
/// include private domains or not. Pass a file path to include your own list
/// or leave `path` empty to load the one included in the package.
pub fn load_suffix_list(
include_private: Bool,
path: Option(String),
) -> SuffixList {
let file_path = case path {
Some(p) -> p
None -> {
let assert Ok(priv_path) = application.priv_directory("psl")
priv_path <> "/public_suffix_list.dat"
}
}
let assert Ok(content) = simplifile.read(from: file_path)
let domain_data = case include_private {
True -> content
False -> {
case string.split_once(content, private_marker) {
Ok(#(public_domains, _private_domains)) -> public_domains
Error(_) -> content
}
}
}
let #(suffix_list, _) =
domain_data
|> string.split("\n")
|> list.fold(
#(SuffixList(dict.new(), list.new(), dict.new()), True),
fn(acc, line) {
let #(sl, is_public) = acc
let trimmed = string.trim(line)
// Check if we've entered the private domains section
case string.contains(trimmed, private_marker) {
True ->
case include_private {
// Toggle is_public to False
True -> #(sl, False)
// No Need to continue
False -> acc
}
False -> {
// Skip empty lines and comments
case trimmed == "" || string.starts_with(trimmed, comment_marker) {
True -> acc
False -> #(add_rule(sl, trimmed, is_public), is_public)
}
}
}
},
)
suffix_list
}
/// Parse a URI and extract domain parts
///
/// ```gleam
/// let list = load_suffix_list(True, None)
/// let result1 = parse("https://example.com", list)
/// let result2 = parse("https://test.co.uk", list)
/// ```
pub fn parse(
uri_string: String,
list: SuffixList,
) -> Result(DomainParts, ParseError) {
use parsed_uri <- result.try(
uri.parse(uri_string)
|> result.replace_error(InvalidUri),
)
use host <- result.try(case parsed_uri.host {
Some(h) -> Ok(h)
None -> Error(NoHost)
})
// Decode punycode to UTF-8 for suffix lookup
let decoded_host = decode_domain(host)
use suffix <- result.try(
find_suffix(decoded_host, list)
|> result.replace_error(UnknownSuffix),
)
extract_parts(decoded_host, suffix)
|> result.map_error(fn(_) { InvalidDomain })
}
/// Add a single rule to the suffix list. This is useful if a suffix is not yet
/// in the list provided by publicsuffix.org.
///
/// Rules are formatted as follows:
///
/// * Normal domaains do not start with a special character.
/// * Exceptions start with a "!".
/// * Wildcards start with a "*".
pub fn add_rule(sl: SuffixList, rule: String, is_public: Bool) -> SuffixList {
case string.starts_with(rule, bang) {
True -> {
let suffix_str = string.drop_start(rule, 1)
let suffix = Suffix(suffix_str, is_public, string.length(suffix_str))
SuffixList(
..sl,
exceptions: dict.insert(sl.exceptions, suffix_str, suffix),
)
}
False ->
case string.starts_with(rule, splat <> ".") {
True -> {
let pattern = string.drop_start(rule, 2)
let suffix = Suffix(pattern, is_public, string.length(pattern))
SuffixList(
normal: dict.insert(sl.normal, pattern, suffix),
wildcards: [pattern, ..sl.wildcards],
exceptions: sl.exceptions,
)
}
False -> {
let suffix = Suffix(rule, is_public, string.length(rule))
SuffixList(..sl, normal: dict.insert(sl.normal, rule, suffix))
}
}
}
}
/// Extract domain parts from hostname and suffix
fn extract_parts(
host: String,
suffix: String,
) -> Result(DomainParts, ParseError) {
let host_labels = string.split(host, dot)
let suffix_labels = string.split(suffix, dot)
case list.length(host_labels) <= list.length(suffix_labels) {
True -> Error(InvalidDomain)
False -> {
let remaining_count =
list.length(host_labels) - list.length(suffix_labels)
case remaining_count {
0 -> Error(InvalidDomain)
_ -> {
let remaining = list.take(host_labels, remaining_count)
case list.reverse(remaining) {
[] -> Error(InvalidDomain)
[domain, ..rest] -> {
let subdomains = list.reverse(rest)
Ok(DomainParts(
top_level_domain: suffix,
second_level_domain: domain,
transit_routing_domain: string.join(subdomains, "."),
subdomain_parts: subdomains,
))
}
}
}
}
}
}
}
// External call to Erlang's idna library - returns a charlist
@external(erlang, "idna", "to_unicode")
fn idna_to_unicode(domain: String) -> String
// Decode a domain that may contain punycode labels
fn decode_domain(domain: String) -> String {
// Check if domain contains any punycode (xn--)
case string.contains(domain, "xn--") {
False -> domain
True -> idna_to_unicode(domain)
}
}
/// Find the matching public suffix for a hostname
fn find_suffix(host: String, suffix_list: SuffixList) -> Result(String, Nil) {
let labels = string.split(host, dot)
// Try to find the longest matching suffix
let matches = find_all_matches(labels, suffix_list)
case matches {
[] -> Error(Nil)
_ -> {
// Return the longest match
matches
|> list.sort(fn(a, b) { int.compare(string.length(b), string.length(a)) })
|> list.first()
|> result.replace_error(Nil)
}
}
}
/// Find all matching suffixes for the given labels
fn find_all_matches(
labels: List(String),
suffix_list: SuffixList,
) -> List(String) {
let reversed = list.reverse(labels)
// Generate all possible suffix combinations
list.range(1, list.length(labels))
|> list.filter_map(fn(i) {
let suffix_labels = list.take(reversed, i)
let suffix = suffix_labels |> list.reverse() |> string.join(".")
// Check for exception rules first - exceptions mean "NOT a public suffix"
// so we skip them and don't include them in matches
case dict.has_key(suffix_list.exceptions, suffix) {
True -> Error(Nil)
False -> {
// Check for exact match
case dict.has_key(suffix_list.normal, suffix) {
True -> Ok(suffix)
False -> {
// Check for wildcard match
case check_wildcard_match(suffix_labels, suffix_list.wildcards) {
Ok(matched) -> Ok(matched)
Error(_) -> Error(Nil)
}
}
}
}
}
})
}
/// Check if a suffix matches any wildcard pattern
fn check_wildcard_match(
suffix_labels: List(String),
wildcards: List(String),
) -> Result(String, Nil) {
// suffix_labels is in reverse order (TLD first)
// For *.ck matching "something.ck", suffix_labels = ["ck", "something"]
// We need to extract the parent "ck" to match against the wildcard pattern
let normal_order = list.reverse(suffix_labels)
case normal_order {
[] -> Error(Nil)
[_] -> Error(Nil)
[_, ..rest] -> {
// rest is the parent domain labels, e.g., ["ck"] for "something.ck"
let parent = string.join(list.reverse(rest), ".")
case list.any(wildcards, fn(w) { w == parent }) {
True -> {
let matched = string.join(normal_order, ".")
Ok(matched)
}
False -> Error(Nil)
}
}
}
}