Current section
Files
Jump to
Current section
Files
src/sqlode/internal/lexer.gleam
import gleam/list
import gleam/option
import gleam/string
import sqlode/internal/char_utils
import sqlode/internal/model
/// Options for controlling how tokens are rendered back to text.
pub type TokenRenderOptions {
TokenRenderOptions(
/// When True, SQL keywords are rendered in UPPERCASE.
uppercase_keywords: Bool,
/// When True, quoted identifiers keep their quotes and string
/// literals escape embedded single-quotes.
preserve_quotes: Bool,
/// When set, use engine-specific quote style for identifiers:
/// MySQL → backticks, PostgreSQL/SQLite → double quotes.
engine: option.Option(model.Engine),
)
}
/// A SQL token produced by the lexer.
pub type Token {
/// SQL keyword (lowercased): SELECT, FROM, CREATE, etc.
Keyword(String)
/// Unquoted identifier: table_name, column_name
Ident(String)
/// Quoted identifier: "name", `name`, [name] (quotes stripped)
QuotedIdent(String)
/// String literal (quotes stripped): 'value', $$dollar$$
StringLit(String)
/// Numeric literal: 42, 3.14
NumberLit(String)
/// Parameter placeholder: $1, ?, :name, @name
Placeholder(String)
/// Operator: =, <>, ::, ->, ||, etc.
Operator(String)
LParen
RParen
Comma
Semicolon
Dot
Star
}
/// Tokenize a SQL string into a list of tokens.
/// Comments are stripped. String literals and quoted identifiers are preserved
/// as single tokens with their content.
pub fn tokenize(sql: String, engine: model.Engine) -> List(Token) {
let graphemes = string.to_graphemes(sql)
do_tokenize(graphemes, engine, [])
|> list.reverse
}
fn do_tokenize(
input: List(String),
engine: model.Engine,
acc: List(Token),
) -> List(Token) {
case input {
[] -> acc
[g, ..rest] ->
case g {
// Whitespace — skip
" " | "\t" | "\n" | "\r" -> do_tokenize(rest, engine, acc)
// Line comment: --
"-" ->
case rest {
["-", ..after_dash] -> {
let remaining = skip_line_comment(after_dash)
do_tokenize(remaining, engine, acc)
}
_ -> {
let #(op, remaining) = read_operator([g, ..rest])
do_tokenize(remaining, engine, [Operator(op), ..acc])
}
}
// Block comment: /* ... */ or MySQL # comment
"/" ->
case rest {
["*", ..after_star] -> {
let remaining = skip_block_comment(after_star)
do_tokenize(remaining, engine, acc)
}
_ -> do_tokenize(rest, engine, [Operator("/"), ..acc])
}
"#" ->
case engine {
model.MySQL -> {
let remaining = skip_line_comment(rest)
do_tokenize(remaining, engine, acc)
}
// PostgreSQL JSON path operators: #>> (text), #> (json)
_ ->
case rest {
[">", ">", ..after] ->
do_tokenize(after, engine, [Operator("#>>"), ..acc])
[">", ..after] ->
do_tokenize(after, engine, [Operator("#>"), ..acc])
_ -> {
let #(op, remaining) = read_operator([g, ..rest])
do_tokenize(remaining, engine, [Operator(op), ..acc])
}
}
}
// Single-quoted string literal
"'" -> {
let #(content, remaining) = read_single_quoted_string(rest, [])
do_tokenize(remaining, engine, [StringLit(content), ..acc])
}
// Dollar-quoted string (PostgreSQL)
"$" ->
case engine {
model.PostgreSQL ->
case try_dollar_tag_lex(rest) {
Ok(#(tag, after_tag)) -> {
let #(content, remaining) =
read_dollar_quoted_string(after_tag, tag, [])
do_tokenize(remaining, engine, [StringLit(content), ..acc])
}
Error(_) -> {
// Could be placeholder $1, $name
let #(token, remaining) =
read_placeholder_or_ident(["$", ..rest])
do_tokenize(remaining, engine, [token, ..acc])
}
}
_ -> {
// SQLite $name placeholder
let #(token, remaining) = read_placeholder_or_ident(["$", ..rest])
do_tokenize(remaining, engine, [token, ..acc])
}
}
// Double-quoted identifier (PostgreSQL/SQLite) or string (MySQL)
"\"" ->
case engine {
model.MySQL -> {
let #(content, remaining) = read_quoted(rest, "\"", [])
do_tokenize(remaining, engine, [StringLit(content), ..acc])
}
_ -> {
let #(content, remaining) = read_quoted(rest, "\"", [])
do_tokenize(remaining, engine, [QuotedIdent(content), ..acc])
}
}
// Backtick-quoted identifier (MySQL)
"`" -> {
let #(content, remaining) = read_quoted(rest, "`", [])
do_tokenize(remaining, engine, [QuotedIdent(content), ..acc])
}
// Bracket-quoted identifier (SQLite)
"[" ->
case engine {
model.SQLite -> {
let #(content, remaining) = read_quoted(rest, "]", [])
do_tokenize(remaining, engine, [QuotedIdent(content), ..acc])
}
_ -> {
let #(op, remaining) = read_operator([g, ..rest])
do_tokenize(remaining, engine, [Operator(op), ..acc])
}
}
// Single-char tokens
"(" -> do_tokenize(rest, engine, [LParen, ..acc])
")" -> do_tokenize(rest, engine, [RParen, ..acc])
"," -> do_tokenize(rest, engine, [Comma, ..acc])
";" -> do_tokenize(rest, engine, [Semicolon, ..acc])
"*" -> do_tokenize(rest, engine, [Star, ..acc])
// Dot
"." ->
case rest {
[next, ..] ->
case char_utils.is_digit(next) {
// .5 → numeric literal
True -> {
let #(num, remaining) = read_number(rest, [g])
do_tokenize(remaining, engine, [NumberLit(num), ..acc])
}
False -> do_tokenize(rest, engine, [Dot, ..acc])
}
_ -> do_tokenize(rest, engine, [Dot, ..acc])
}
// Placeholder: ?, :name, @name. Also JSONB key-existence operators
// ?| (any) and ?& (all) take precedence over the SQLite/MySQL
// bare-`?` placeholder.
"?" ->
case rest {
["|", ..after] ->
do_tokenize(after, engine, [Operator("?|"), ..acc])
["&", ..after] ->
do_tokenize(after, engine, [Operator("?&"), ..acc])
_ -> {
let #(ph, remaining) = read_placeholder_question(rest, [g])
do_tokenize(remaining, engine, [Placeholder(ph), ..acc])
}
}
":" ->
case rest {
// PostgreSQL :: cast operator
[":", ..after_colon] ->
do_tokenize(after_colon, engine, [Operator("::"), ..acc])
[next, ..] ->
case char_utils.is_alpha_or_underscore(next) {
True -> {
let #(ph, remaining) = read_word(rest, [g])
do_tokenize(remaining, engine, [Placeholder(ph), ..acc])
}
False -> do_tokenize(rest, engine, [Operator(":"), ..acc])
}
_ -> do_tokenize(rest, engine, [Operator(":"), ..acc])
}
"@" ->
case rest {
// JSONB/Array containment: @>
[">", ..after] ->
do_tokenize(after, engine, [Operator("@>"), ..acc])
[next, ..] ->
case char_utils.is_alpha_or_underscore(next) {
True -> {
let #(ph, remaining) = read_word(rest, [g])
do_tokenize(remaining, engine, [Placeholder(ph), ..acc])
}
False -> do_tokenize(rest, engine, [Operator("@"), ..acc])
}
_ -> do_tokenize(rest, engine, [Operator("@"), ..acc])
}
// Numbers
_ ->
case char_utils.is_digit(g) {
True ->
case g, rest {
// 0x/0X hex literal (MySQL, SQLite). Treat as a single
// numeric token so downstream sees one NumberLit, not
// 0 followed by an identifier.
"0", [x, ..hex_rest] if x == "x" || x == "X" -> {
let #(digits, remaining) = read_hex_digits(hex_rest, [])
do_tokenize(remaining, engine, [
NumberLit("0" <> x <> digits),
..acc
])
}
_, _ -> {
let #(num, remaining) = read_number(rest, [g])
do_tokenize(remaining, engine, [NumberLit(num), ..acc])
}
}
False ->
case char_utils.is_alpha_or_underscore(g) {
True ->
case detect_string_literal_prefix(g, rest) {
option.Some(#(after_prefix, _kind)) -> {
let #(content, remaining) =
read_single_quoted_string(after_prefix, [])
do_tokenize(remaining, engine, [StringLit(content), ..acc])
}
option.None -> {
let #(word, remaining) = read_word(rest, [g])
let token = classify_word(word)
do_tokenize(remaining, engine, [token, ..acc])
}
}
// Operators and other characters
False -> {
let #(op, remaining) = read_operator([g, ..rest])
do_tokenize(remaining, engine, [Operator(op), ..acc])
}
}
}
}
}
}
// --- Comment helpers ---
fn skip_line_comment(input: List(String)) -> List(String) {
case input {
[] -> []
["\n", ..rest] -> rest
[_, ..rest] -> skip_line_comment(rest)
}
}
fn skip_block_comment(input: List(String)) -> List(String) {
skip_block_comment_loop(input, 1)
}
fn skip_block_comment_loop(input: List(String), depth: Int) -> List(String) {
case input {
[] -> []
_ ->
case depth <= 0 {
True -> input
False ->
case input {
["*", "/", ..rest] -> skip_block_comment_loop(rest, depth - 1)
["/", "*", ..rest] -> skip_block_comment_loop(rest, depth + 1)
[_, ..rest] -> skip_block_comment_loop(rest, depth)
[] -> []
}
}
}
}
// --- String literal helpers ---
fn read_single_quoted_string(
input: List(String),
acc: List(String),
) -> #(String, List(String)) {
case input {
[] -> #(acc |> list.reverse |> string.concat, [])
["'", "'", ..rest] ->
// Escaped quote '' → single '
read_single_quoted_string(rest, ["'", ..acc])
["'", ..rest] ->
// End of string
#(acc |> list.reverse |> string.concat, rest)
[g, ..rest] -> read_single_quoted_string(rest, [g, ..acc])
}
}
fn read_dollar_quoted_string(
input: List(String),
tag: String,
acc: List(String),
) -> #(String, List(String)) {
case input {
[] -> #(acc |> list.reverse |> string.concat, [])
["$", ..rest] ->
case try_match_closing_dollar_tag_lex(rest, tag) {
Ok(remaining) -> #(acc |> list.reverse |> string.concat, remaining)
Error(_) -> read_dollar_quoted_string(rest, tag, ["$", ..acc])
}
[g, ..rest] -> read_dollar_quoted_string(rest, tag, [g, ..acc])
}
}
fn try_dollar_tag_lex(
chars: List(String),
) -> Result(#(String, List(String)), Nil) {
case chars {
["$", ..rest] -> Ok(#("", rest))
[c, ..rest] ->
case char_utils.is_alpha_or_underscore(c) {
True -> read_dollar_tag_chars_lex(rest, [c])
False -> Error(Nil)
}
[] -> Error(Nil)
}
}
fn read_dollar_tag_chars_lex(
chars: List(String),
acc: List(String),
) -> Result(#(String, List(String)), Nil) {
case chars {
[] -> Error(Nil)
["$", ..rest] -> {
let tag = acc |> list.reverse |> string.concat
Ok(#(tag, rest))
}
[c, ..rest] ->
case char_utils.is_alnum_or_underscore(c) {
True -> read_dollar_tag_chars_lex(rest, [c, ..acc])
False -> Error(Nil)
}
}
}
fn try_match_closing_dollar_tag_lex(
chars: List(String),
tag: String,
) -> Result(List(String), Nil) {
let tag_chars = string.to_graphemes(tag)
match_tag_then_dollar_lex(chars, tag_chars)
}
fn match_tag_then_dollar_lex(
chars: List(String),
tag_chars: List(String),
) -> Result(List(String), Nil) {
case tag_chars {
[] ->
case chars {
["$", ..rest] -> Ok(rest)
_ -> Error(Nil)
}
[expected, ..tag_rest] ->
case chars {
[actual, ..chars_rest] if actual == expected ->
match_tag_then_dollar_lex(chars_rest, tag_rest)
_ -> Error(Nil)
}
}
}
fn read_quoted(
input: List(String),
closer: String,
acc: List(String),
) -> #(String, List(String)) {
case input {
[] -> #(acc |> list.reverse |> string.concat, [])
// SQL standard: a doubled closer inside the quotes is an escaped literal.
// SQLite bracket identifiers ([...]) have no escape mechanism for ].
[g, g2, ..rest] if g == closer && g2 == closer && closer != "]" ->
read_quoted(rest, closer, [g, ..acc])
[g, ..rest] ->
case g == closer {
True -> #(acc |> list.reverse |> string.concat, rest)
False -> read_quoted(rest, closer, [g, ..acc])
}
}
}
/// Detect SQL prefixed string literals: E'..', U&'..', B'..', X'..', N'..'.
/// Returns the tokens after the prefix and a tag describing the kind, so
/// the caller can read the body via read_single_quoted_string. Case
/// insensitive on the prefix character.
fn detect_string_literal_prefix(
g: String,
rest: List(String),
) -> option.Option(#(List(String), String)) {
case g, rest {
p, ["'", ..after] if p == "E" || p == "e" -> option.Some(#(after, "escape"))
p, ["'", ..after] if p == "B" || p == "b" -> option.Some(#(after, "bit"))
p, ["'", ..after] if p == "X" || p == "x" -> option.Some(#(after, "hex"))
p, ["'", ..after] if p == "N" || p == "n" ->
option.Some(#(after, "national"))
p, ["&", "'", ..after] if p == "U" || p == "u" ->
option.Some(#(after, "unicode"))
_, _ -> option.None
}
}
fn read_hex_digits(
input: List(String),
acc: List(String),
) -> #(String, List(String)) {
case input {
[g, ..rest] ->
case is_hex_digit(g) {
True -> read_hex_digits(rest, [g, ..acc])
False -> #(acc |> list.reverse |> string.concat, input)
}
[] -> #(acc |> list.reverse |> string.concat, [])
}
}
fn is_hex_digit(g: String) -> Bool {
case g {
"0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" -> True
"a" | "b" | "c" | "d" | "e" | "f" -> True
"A" | "B" | "C" | "D" | "E" | "F" -> True
_ -> False
}
}
// --- Word/identifier helpers ---
fn read_word(input: List(String), acc: List(String)) -> #(String, List(String)) {
case input {
[g, ..rest] ->
case char_utils.is_alnum_or_underscore(g) {
True -> read_word(rest, [g, ..acc])
False -> #(acc |> list.reverse |> string.concat, input)
}
[] -> #(acc |> list.reverse |> string.concat, [])
}
}
fn classify_word(word: String) -> Token {
let lowered = string.lowercase(word)
case is_sqlode_marker(word) {
True -> Placeholder(word)
False ->
case is_sql_keyword(lowered) {
True -> Keyword(lowered)
False -> Ident(word)
}
}
}
fn is_sqlode_marker(word: String) -> Bool {
{
string.starts_with(word, "__sqlode_param_")
|| string.starts_with(word, "__sqlode_slice_")
}
&& string.ends_with(word, "__")
}
fn is_sql_keyword(word: String) -> Bool {
case word {
"select"
| "from"
| "where"
| "and"
| "or"
| "not"
| "in"
| "is"
| "null"
| "as"
| "on"
| "join"
| "inner"
| "left"
| "right"
| "full"
| "outer"
| "cross"
| "natural"
| "using"
| "order"
| "by"
| "group"
| "rollup"
| "cube"
| "sets"
| "having"
| "limit"
| "offset"
| "union"
| "intersect"
| "except"
| "all"
| "distinct"
| "insert"
| "into"
| "values"
| "update"
| "set"
| "delete"
| "create"
| "table"
| "view"
| // `type` is intentionally NOT a keyword. It is a common
// column name (`account_type`, `event_type`, BLOB-related
// `type` for content-type, etc.) and SQLite / MySQL /
// PostgreSQL all treat it as a non-reserved identifier in
// SELECT contexts. The DDL paths that legitimately need to
// recognise it (`CREATE TYPE`, `DROP TYPE`,
// `ALTER COLUMN ... TYPE`, `ALTER COLUMN ... SET DATA TYPE`)
// pattern-match on `Ident("type")` instead. (#479)
"index"
| "alter"
| "add"
| "column"
| "drop"
| "if"
| "exists"
| "primary"
| "key"
| "foreign"
| "references"
| "unique"
| "check"
| "default"
| "constraint"
| "cascade"
| "restrict"
| "no"
| "action"
| "autoincrement"
| "serial"
| "bigserial"
| "smallserial"
| "returning"
| "case"
| "when"
| "then"
| "else"
| "end"
| "cast"
| "between"
| "like"
| "ilike"
| "with"
| "recursive"
| "temporary"
| "temp"
| "unlogged"
| "true"
| "false"
| "asc"
| "desc"
| "nulls"
| "first"
| "last"
| "over"
| "partition"
| "window"
| "row"
| "rows"
| "range"
| "preceding"
| "following"
| "current"
| "unbounded"
| "enum"
| "replace"
| "conflict"
| "do"
| "nothing"
| "begin"
| "commit"
| "rollback"
| "transaction"
| "for"
| "each"
| "trigger"
| "execute"
| "procedure"
| "function"
| "returns"
| "language"
| "volatile"
| "stable"
| "immutable"
| "security"
| "definer"
| "invoker"
| "grant"
| "revoke"
| "to"
| "schema"
| "database"
| "extension"
| "only"
| "lateral"
| "any"
| "some"
| "array"
| "of"
| "collate"
| // Issue #513: SQLite / MySQL conflict-resolution keywords
// (`INSERT OR IGNORE`, `INSERT OR ABORT`, `INSERT OR FAIL`,
// MySQL's bare `INSERT IGNORE`). Without these the surrounding
// `INSERT` / `OR` / `INTO` lowercase via the `Keyword` token
// path while `IGNORE` / `ABORT` / `FAIL` survive as `Ident`
// (preserved-case), producing the `insert or IGNORE into ...`
// mixed casing the issue reports. None of these is a
// realistic identifier name, so promoting them to keywords is
// safe and restores consistent lowercasing across the
// generated SQL string.
"ignore"
| "abort"
| "fail" -> True
_ -> False
}
}
// --- Number helpers ---
fn read_number(
input: List(String),
acc: List(String),
) -> #(String, List(String)) {
case input {
[g, ..rest] ->
case char_utils.is_digit(g) || g == "." {
True -> read_number(rest, [g, ..acc])
False ->
case g == "e" || g == "E" {
True ->
case rest {
["+", ..] | ["-", ..] ->
read_number(list.drop(rest, 1), [
case rest {
[s, ..] -> s
_ -> ""
},
g,
..acc
])
_ -> read_number(rest, [g, ..acc])
}
False -> #(acc |> list.reverse |> string.concat, input)
}
}
[] -> #(acc |> list.reverse |> string.concat, [])
}
}
// --- Placeholder helpers ---
fn read_placeholder_or_ident(input: List(String)) -> #(Token, List(String)) {
case input {
["$", ..rest] -> {
let #(word, remaining) = read_word(rest, ["$"])
#(Placeholder(word), remaining)
}
_ -> #(Operator("$"), input)
}
}
fn read_placeholder_question(
input: List(String),
acc: List(String),
) -> #(String, List(String)) {
case input {
[g, ..rest] ->
case char_utils.is_digit(g) {
True -> read_placeholder_question(rest, [g, ..acc])
False -> #(acc |> list.reverse |> string.concat, input)
}
[] -> #(acc |> list.reverse |> string.concat, [])
}
}
// --- Operator helpers ---
fn read_operator(input: List(String)) -> #(String, List(String)) {
case input {
// Multi-char operators
["<", ">", ..rest] -> #("<>", rest)
["<", "=", ..rest] -> #("<=", rest)
["<", "@", ..rest] -> #("<@", rest)
[">", "=", ..rest] -> #(">=", rest)
["!", "=", ..rest] -> #("!=", rest)
["|", "|", ..rest] -> #("||", rest)
["&", "&", ..rest] -> #("&&", rest)
["-", ">", ">", ..rest] -> #("->>", rest)
["-", ">", ..rest] -> #("->", rest)
// Single-char operators
[g, ..rest] -> #(g, rest)
[] -> #("", [])
}
}
/// Render a list of tokens back to a SQL string with smart spacing.
pub fn tokens_to_string(
tokens: List(Token),
options: TokenRenderOptions,
) -> String {
tokens_to_string_loop(tokens, [], options)
|> list.reverse
|> string.concat
}
fn tokens_to_string_loop(
tokens: List(Token),
acc: List(String),
options: TokenRenderOptions,
) -> List(String) {
case tokens {
[] -> acc
[token, ..rest] -> {
let token_text = token_to_string(token, options)
let with_space = case acc, token {
_, Comma | _, Semicolon | _, LParen | _, RParen | _, Dot | _, Star -> [
token_text,
..acc
]
["(", ..], _ | [".", ..], _ -> [token_text, ..acc]
_, Operator("[") -> [token_text, ..acc]
["]", ..], _ | ["[", ..], _ -> [token_text, ..acc]
[], _ -> [token_text]
_, _ -> [token_text, " ", ..acc]
}
tokens_to_string_loop(rest, with_space, options)
}
}
}
fn token_to_string(token: Token, options: TokenRenderOptions) -> String {
case token {
Keyword(k) ->
case options.uppercase_keywords {
True -> string.uppercase(k)
False -> k
}
Ident(name) -> name
QuotedIdent(name) ->
case options.preserve_quotes {
True ->
case options.engine {
option.Some(model.MySQL) -> "`" <> name <> "`"
_ -> "\"" <> name <> "\""
}
False -> name
}
StringLit(value) ->
case options.preserve_quotes {
True -> "'" <> string.replace(value, "'", "''") <> "'"
False -> "'" <> value <> "'"
}
NumberLit(n) -> n
Placeholder(p) -> p
Operator(op) -> op
LParen -> "("
RParen -> ")"
Comma -> ","
Semicolon -> ";"
Dot -> "."
Star -> "*"
}
}