Packages
magic_string
1.0.0
Edit strings by byte offset and get a source map for free. A Gleam port of Rich Harris's magic-string.
Current section
Files
Jump to
Current section
Files
src/magic_string/codec.gleam
//// Source Map v3 model and base64 VLQ codec.
////
//// `generate_mappings` turns a flat `Segment` list into the v3 `mappings`
//// string; `decode_mappings` is its inverse. `to_json` serializes the whole
//// `SourceMap`. See https://tc39.es/ecma426/ for the format.
import gleam/bit_array
import gleam/dict
import gleam/int
import gleam/list
import gleam/option.{type Option, None, Some}
import gleam/order
import gleam/result
import gleam/string
/// A Source Map v3 document.
///
/// `sources_content` is positional with `sources`: each entry is the full
/// original text of the source at the same index, or `None` when the content
/// is not embedded (serialized as JSON `null`).
pub type SourceMap {
SourceMap(
version: Int,
file: Option(String),
source_root: Option(String),
sources: List(String),
sources_content: List(Option(String)),
names: List(String),
mappings: String,
)
}
/// How a generated file should reference its source map.
pub type MapMode {
/// Emit a `sourceMappingURL` comment pointing at a separate `.map` file.
External(url: String)
/// Emit a `sourceMappingURL` comment with the whole map inlined as a
/// base64 `data:` URI.
Inline
/// Emit no comment (the map exists but is not referenced from the output).
Hidden
}
/// One mapping from a position in the generated output to a position in an
/// original source. Half-open columns are UTF-16 code units; lines are
/// zero-based. Inserted (synthesized) output produces no `Segment`.
pub type Segment {
Segment(
gen_line: Int,
gen_col: Int,
source_idx: Int,
orig_line: Int,
orig_col: Int,
)
}
/// Base64 VLQ alphabet (Source Map v3): A-Z, a-z, 0-9, '+', '/'.
const base64_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
/// Encode a single signed integer as a base64 VLQ string.
///
/// Sign is folded into the low bit (`value << 1`, or `(-value << 1) | 1` for
/// negatives), then the magnitude is split into 5-bit groups emitted
/// low-group-first. Every group except the last carries the continuation bit
/// (32); each resulting 6-bit value indexes into `base64_alphabet`.
pub fn encode_vlq(value: Int) -> String {
let vlq = case value < 0 {
True -> int.bitwise_or(int.bitwise_shift_left(-value, 1), 1)
False -> int.bitwise_shift_left(value, 1)
}
encode_vlq_loop(vlq, "")
}
fn encode_vlq_loop(vlq: Int, acc: String) -> String {
let digit = int.bitwise_and(vlq, 31)
let rest = int.bitwise_shift_right(vlq, 5)
case rest > 0 {
// More groups follow: set the continuation bit on this one.
True -> encode_vlq_loop(rest, acc <> base64_char(int.bitwise_or(digit, 32)))
False -> acc <> base64_char(digit)
}
}
fn base64_char(n: Int) -> String {
string.slice(base64_alphabet, n, 1)
}
/// Decode a single base64-VLQ field string back to its list of signed
/// integers. This is the exact inverse of concatenating `encode_vlq` calls:
/// `decode_vlq(encode_vlq(a) <> encode_vlq(b)) == [a, b]`.
///
/// Returns `Error` when `field` contains a character outside the base64
/// alphabet, or ends mid-value (a continuation bit with no following digit).
pub fn decode_vlq(field: String) -> Result(List(Int), String) {
decode_vlq_loop(string.to_graphemes(field), 0, 0, [])
}
fn decode_vlq_loop(
chars: List(String),
partial: Int,
shift: Int,
acc: List(Int),
) -> Result(List(Int), String) {
case chars {
[] ->
case shift {
0 -> Ok(list.reverse(acc))
_ ->
Error(
"truncated VLQ: continuation bit set on final digit (shift "
<> int.to_string(shift)
<> ")",
)
}
[c, ..rest] ->
case base64_value(c) {
Error(msg) -> Error(msg)
Ok(digit) -> {
let chunk = int.bitwise_and(digit, 31)
let partial = partial + int.bitwise_shift_left(chunk, shift)
case int.bitwise_and(digit, 32) {
// Continuation bit set: accumulate and keep going.
0 -> decode_vlq_loop(rest, 0, 0, [from_vlq(partial), ..acc])
_ -> decode_vlq_loop(rest, partial, shift + 5, acc)
}
}
}
}
}
/// Undo the sign-in-low-bit folding `encode_vlq` applies.
fn from_vlq(value: Int) -> Int {
let magnitude = int.bitwise_shift_right(value, 1)
case int.bitwise_and(value, 1) {
0 -> magnitude
_ -> -magnitude
}
}
/// Position of a base64 character in the Source Map v3 alphabet.
fn base64_value(char: String) -> Result(Int, String) {
case char_code(char) {
code if code >= 65 && code <= 90 -> Ok(code - 65)
code if code >= 97 && code <= 122 -> Ok(code - 97 + 26)
code if code >= 48 && code <= 57 -> Ok(code - 48 + 52)
43 -> Ok(62)
47 -> Ok(63)
code ->
Error(
"invalid base64 VLQ character '"
<> char
<> "' (codepoint "
<> int.to_string(code)
<> ")",
)
}
}
fn char_code(char: String) -> Int {
char
|> string.to_utf_codepoints
|> list.first
|> result.map(string.utf_codepoint_to_int)
|> result.unwrap(0)
}
/// Decode a Source Map v3 `mappings` string back to a flat `Segment` list.
/// This is the inverse of `generate_mappings`:
/// `decode_mappings(generate_mappings(segs))` returns the same segments
/// (in `(gen_line, gen_col)` order).
///
/// Only 4- and 5-field segments (those carrying a source pointer) are
/// returned; 1-field segments (generated column only, no source) are skipped
/// since `Segment` requires a source position. The optional 5th `name` field
/// is decoded for delta-tracking but discarded (this codec does not model
/// names).
pub fn decode_mappings(mappings: String) -> Result(List(Segment), String) {
let lines = string.split(mappings, ";")
use #(segs_rev, _delta) <- result.map(
list.try_fold(
list.index_map(lines, fn(line, idx) { #(idx, line) }),
#([], DeltaState(0, 0, 0)),
fn(acc, entry) {
let #(gen_line, line) = entry
decode_line(line, gen_line, acc)
},
),
)
list.reverse(segs_rev)
}
fn decode_line(
line: String,
gen_line: Int,
acc: #(List(Segment), DeltaState),
) -> Result(#(List(Segment), DeltaState), String) {
case line {
"" -> Ok(acc)
_ ->
string.split(line, ",")
|> list.try_fold(#(acc.0, acc.1, 0), fn(state, field) {
let #(segs, delta, prev_gen_col) = state
use values <- result.try(
decode_vlq(field)
|> result.map_error(fn(err) {
"line " <> int.to_string(gen_line) <> ": " <> err
}),
)
case values {
[gc, si, ol, oc, ..] -> {
let gen_col = prev_gen_col + gc
let source_idx = delta.source_idx + si
let orig_line = delta.orig_line + ol
let orig_col = delta.orig_col + oc
let seg =
Segment(gen_line:, gen_col:, source_idx:, orig_line:, orig_col:)
Ok(#(
[seg, ..segs],
DeltaState(source_idx:, orig_line:, orig_col:),
gen_col,
))
}
// 1-field segment: generated column only, no source pointer. Update
// gen_col so the next segment's delta is correct, but emit nothing.
[gc] -> Ok(#(segs, delta, prev_gen_col + gc))
other ->
Error(
"line "
<> int.to_string(gen_line)
<> ": segment has "
<> int.to_string(list.length(other))
<> " fields; expected 1, 4, or 5",
)
}
})
|> result.map(fn(state) { #(state.0, state.1) })
}
}
/// Running delta state for the source-pointing fields. `gen_col` resets per
/// output line, so it is threaded separately.
type DeltaState {
DeltaState(source_idx: Int, orig_line: Int, orig_col: Int)
}
/// Build the Source Map v3 `mappings` string from a flat segment list.
///
/// Output lines are `;`-separated and segments within a line are
/// `,`-separated. Each segment is the VLQ of four deltas:
/// `[gen_col, source_idx, orig_line, orig_col]`. `gen_col` is delta'd within
/// the line (reset to 0 per line); the other three are delta'd across the
/// whole map. Inserted text contributes no segment (so a line with only
/// inserted output is empty). `hires` is false: one segment per chunk.
pub fn generate_mappings(segments: List(Segment)) -> String {
case segments {
[] -> ""
_ -> {
let sorted = list.sort(segments, by: compare_segment)
let max_line =
list.fold(sorted, 0, fn(acc, s) {
case s.gen_line > acc {
True -> s.gen_line
False -> acc
}
})
let by_line = list.group(sorted, by: fn(s) { s.gen_line })
// int.range's `to` is exclusive, so go to max_line + 1 to cover the
// last output line.
let #(lines_rev, _final) =
int.range(0, max_line + 1, #([], DeltaState(0, 0, 0)), fn(acc, line) {
let #(lines, delta) = acc
// `list.group` does not preserve order within a group, so re-sort
// this line's segments by generated column before encoding.
let segs =
dict.get(by_line, line)
|> result.unwrap([])
|> list.sort(by: fn(a, b) { int.compare(a.gen_col, b.gen_col) })
let #(line_str, next_delta) = emit_line(segs, delta)
#([line_str, ..lines], next_delta)
})
lines_rev |> list.reverse |> string.join(";")
}
}
}
/// Encode one output line's segments, threading the cross-map delta state in
/// and out. `gen_col` resets to 0 at the start of every line.
fn emit_line(segs: List(Segment), delta: DeltaState) -> #(String, DeltaState) {
let #(parts_rev, _gen_col, next_delta) =
list.fold(segs, #([], 0, delta), fn(acc, seg) {
let #(parts, prev_gen_col, d) = acc
let field =
string.concat([
encode_vlq(seg.gen_col - prev_gen_col),
encode_vlq(seg.source_idx - d.source_idx),
encode_vlq(seg.orig_line - d.orig_line),
encode_vlq(seg.orig_col - d.orig_col),
])
let next = DeltaState(seg.source_idx, seg.orig_line, seg.orig_col)
#([field, ..parts], seg.gen_col, next)
})
#(parts_rev |> list.reverse |> string.join(","), next_delta)
}
fn compare_segment(a: Segment, b: Segment) -> order.Order {
case int.compare(a.gen_line, b.gen_line) {
order.Eq -> int.compare(a.gen_col, b.gen_col)
other -> other
}
}
/// Serialize a `SourceMap` to a JSON string. `file` and `sourceRoot` are
/// omitted when `None`; `sourcesContent` entries are `null` when `None`.
pub fn to_json(map: SourceMap) -> String {
let fields =
[
Some("\"version\":" <> int.to_string(map.version)),
option.map(map.file, fn(f) { "\"file\":" <> json_string(f) }),
option.map(map.source_root, fn(r) { "\"sourceRoot\":" <> json_string(r) }),
Some("\"sources\":" <> json_array(list.map(map.sources, json_string))),
Some(
"\"sourcesContent\":"
<> json_array(list.map(map.sources_content, json_nullable)),
),
Some("\"names\":" <> json_array(list.map(map.names, json_string))),
Some("\"mappings\":" <> json_string(map.mappings)),
]
|> option.values
"{" <> string.join(fields, ",") <> "}"
}
fn json_array(items: List(String)) -> String {
"[" <> string.join(items, ",") <> "]"
}
fn json_nullable(value: Option(String)) -> String {
case value {
Some(s) -> json_string(s)
None -> "null"
}
}
fn json_string(value: String) -> String {
let escaped =
value
|> string.replace("\\", "\\\\")
|> string.replace("\"", "\\\"")
|> string.replace("\n", "\\n")
|> string.replace("\r", "\\r")
|> string.replace("\t", "\\t")
"\"" <> escaped <> "\""
}
/// Produce the `sourceMappingURL` comment for the given map and mode.
///
/// `External` points at a sibling `.map` URL, `Inline` embeds the whole map
/// as a base64 `data:` URI, and `Hidden` emits nothing.
pub fn url_comment(map: SourceMap, mode: MapMode) -> String {
case mode {
External(url:) -> "//# sourceMappingURL=" <> url
Hidden -> ""
Inline -> {
let b64 =
bit_array.base64_encode(bit_array.from_string(to_json(map)), True)
"//# sourceMappingURL=data:application/json;charset=utf-8;base64," <> b64
}
}
}