dymokomi/luce-base
Source of release 0.42.0
0.42.0 / src / front / lexer.lucb
#==============================================================================================
#
# lexer - Bytes to tokens, then layout
#
# DESCRIPTION:
# Tokens are spans; the literal decoder of a later slice turns spellings into values.
# Layout is part of lexing: a newline at the top level becomes `newline`, a deeper first
# column becomes `indent`, and a shallower one becomes one `dedent` per closed block.
# Inside `()`, `[]`, and `{}` newlines are spacing, except that a `:` ending a line
# reopens layout and the bracket that closes the suite ends it (base.md §3.2). A
# formatted string lexes as `format_start`, `format_text` runs, ordinary tokens for each
# `{field}`, then `format_end`. An `asm` suite is captured as `raw_line` tokens.
#
#==============================================================================================
import front.source
import front.token
## Indentation is not a multiple of four spaces, or misaligns a block.
pub let bad_indent: ErrorCode = ErrorCode.package(3)
## A string, character, or comment reaches end of file unterminated.
pub let unterminated: ErrorCode = ErrorCode.package(4)
## A byte that is not valid in the source at this point.
pub let bad_character: ErrorCode = ErrorCode.package(5)
## The suffixes a number may carry (§4.2, §4.3): the exact type's name.
func is_literal_suffix(word: str) -> bool:
let names: str[13] = ["u8", "u16", "u32", "u64", "usize", "i8", "i16", "i32", "i64", "isize", "f16", "f32", "f64"]
for name in names:
if word == name:
return true
return false
## The longest identifier (§3.1): a diagnostic quoting one fits its buffer.
pub let longest_name: usize = 128
## A tab character in the source, which is not allowed.
pub let tab: ErrorCode = ErrorCode.package(6)
let max_blocks: usize = 64
let max_modes: usize = 32
## The lexer over one source, producing tokens on demand.
pub struct Lexer:
## The source being lexed.
pub var src: source.Source
var offset: usize
var line: u32
var line_start: usize
var at_line_start: bool
var levels: u32[64]
var depth: usize
var pending_dedents: usize
var delimiters: i32
## 0: code; 1: inside a formatted string's literal part; 2: inside a `{field}`
var modes: u8[32]
## For each field open in `modes`, how many brackets were open before it: a `:` with no
## more open than that starts the field's specification (§5.5).
var field_brackets: i32[32]
var mode_depth: usize
var asm_header: bool
var raw_level: u32
var in_raw: bool
## Offsets of the last comment's `#` and just past it, so a `:` inside it does not open
## a suite while one before it still does.
var comment_start: usize
var comment_end: usize
## The block depth when each open bracket was seen; a closer ends any suite
## opened inside it (base.md §3.2: closers on the body's last line).
var bracket_base: usize[64]
## A colon ended a line inside brackets: the next line's indentation counts.
var reopened: bool
## Where the token being read began, for a diagnostic (`error_position`).
var token_start: usize
## A lexer positioned at the start of `src`.
pub static func over(src: source.Source) -> Lexer:
return Lexer(src = src, offset = 0, line = 1, line_start = 0, at_line_start = true,
depth = 0, pending_dedents = 0, delimiters = 0, mode_depth = 0,
asm_header = false, raw_level = 0, in_raw = false, comment_start = 0, comment_end = 0,
reopened = false, token_start = 0)
## The one-based line and column of the byte about to be read, for diagnostics.
pub func position() -> (u32, u32):
return (self.line, u32(self.offset - self.line_start) + 1)
## Where the token `next` failed in begins, when it began on the current line; else
## where the lexer stopped.
pub func error_position() -> (u32, u32):
if self.token_start >= self.line_start:
return (self.line, u32(self.token_start - self.line_start) + 1)
return self.position()
func peek() -> u8:
if self.offset >= self.src.bytes.length:
return 0
return self.src.bytes[self.offset]
func peek_at(ahead: usize) -> u8:
if self.offset + ahead >= self.src.bytes.length:
return 0
return self.src.bytes[self.offset + ahead]
func make(kind: token.Kind, start: usize) -> token.Token:
return token.Token(kind = kind, start = start, end = self.offset, line = self.line,
column = u32(start - self.line_start) + 1)
func mode() -> u8:
if self.mode_depth == 0:
return 0
return self.modes[self.mode_depth - 1]
func push_mode(m: u8) -> !:
if self.mode_depth >= max_modes:
error(unterminated, "formatted strings nest too deeply")
self.modes[self.mode_depth] = m
self.mode_depth += 1
func pop_mode():
if self.mode_depth > 0:
self.mode_depth -= 1
func newline_here():
self.offset += 1
self.line += 1
self.line_start = self.offset
self.at_line_start = true
## Skip spaces and a trailing comment; a `##` at a line's start is a token.
func skip_trivia() -> token.Token?!:
while true:
let c = self.peek()
if c == ' ':
self.offset += 1
elif c == '\t':
error(tab, "tabs are not allowed; use four spaces")
elif c == '\r' and self.peek_at(1) == '\n':
self.offset += 1
elif c == '#':
let start = self.offset
let doc = self.peek_at(1) == '#' and start == self.line_start
while self.peek() != '\n' and self.peek() != 0:
self.offset += 1
self.comment_start = start
self.comment_end = self.offset
if doc:
return self.make(token.Kind.doc_comment, start)
else:
return none
## Measure the current line's indentation and queue layout tokens.
func layout() -> token.Token?!:
var column: u32 = 0
while true:
column = 0
while self.peek() == ' ':
self.offset += 1
column += 1
let c = self.peek()
if c == '#' and not (self.peek_at(1) == '#' and column == 0):
while self.peek() != '\n' and self.peek() != 0:
self.offset += 1 # a comment-only line
continue
if c == '\n':
self.newline_here() # a blank line
continue
if c == '\r' and self.peek_at(1) == '\n':
self.offset += 1
continue
break
if self.peek() == 0 or self.peek() == '#':
return none # end of input, or a doc comment
self.at_line_start = false
if self.delimiters > 0 and not self.reopened and self.depth <= self.bracket_base[(usize)self.delimiters - 1]:
return none # spacing inside brackets
self.reopened = false
let current = self.levels[self.depth]
if column > current:
if self.depth + 1 >= max_blocks:
error(bad_indent, "blocks nest too deeply")
if column != current + 4:
error(bad_indent, "a block indents by exactly four spaces")
self.depth += 1
self.levels[self.depth] = column
if self.asm_header:
self.asm_header = false
self.in_raw = true
self.raw_level = column
return self.make(token.Kind.indent, self.offset)
while column < self.levels[self.depth]:
self.depth -= 1
self.pending_dedents += 1
if column != self.levels[self.depth]:
error(bad_indent, "indentation does not match an open block")
if self.pending_dedents > 0:
self.pending_dedents -= 1
return self.make(token.Kind.dedent, self.offset)
return none
## One physical line of an `asm` suite, or the dedent that ends it.
func raw_line() -> token.Token!:
# layout() may already have consumed the indentation; measure from the line start
var i = self.offset
while i < self.src.bytes.length and self.src.bytes[i] == ' ':
i += 1
let column = u32(i - self.line_start)
var c = self.src.bytes[i] if i < self.src.bytes.length else 0
if c == '\r' and i + 1 < self.src.bytes.length and self.src.bytes[i + 1] == '\n':
# a blank line ending in CRLF is a blank line (§3.1)
i += 1
c = '\n'
if c == '\n' or c == 0:
if c == '\n':
# past the blank line's spaces, then its end
self.offset = i
self.newline_here()
return try self.raw_line()
self.in_raw = false
return try self.next()
if column < self.raw_level:
self.in_raw = false
# End the suite by dedenting to the line's level now, so a comment on
# that line does not defer the dedent past the asm node (layout() would
# return none for a `##`/`#` line, hiding the dedent the parser needs).
while column < self.levels[self.depth]:
self.depth -= 1
self.pending_dedents += 1
return try self.next()
let start = i
self.offset = i
while self.peek() != '\n' and self.peek() != 0:
self.offset += 1
var t = self.make(token.Kind.raw_line, start)
if t.end > t.start and self.src.bytes[t.end - 1] == '\r' and self.peek() == '\n':
t.end -= 1
if self.peek() == '\n':
self.newline_here()
return t
func number(start: usize) -> token.Token!:
var kind = token.Kind.integer
if self.peek() == '0' and (self.peek_at(1) == 'X' or self.peek_at(1) == 'O' or self.peek_at(1) == 'B'):
error(bad_character, "a based integer prefix is lowercase: `0x`, `0o`, or `0b`")
if self.peek() == '0' and (self.peek_at(1) == 'x' or self.peek_at(1) == 'o' or self.peek_at(1) == 'b'):
let base: u8 = 16 if self.peek_at(1) == 'x' else (8 if self.peek_at(1) == 'o' else 2)
self.offset += 2
try self.digits(base)
else:
try self.digits(10)
if self.peek() == '.' and is_digit(self.peek_at(1)):
kind = token.Kind.float
self.offset += 1
try self.digits(10)
if (self.peek() == 'e' or self.peek() == 'E') and (is_digit(self.peek_at(1)) or ((self.peek_at(1) == '+' or self.peek_at(1) == '-') and is_digit(self.peek_at(2)))):
kind = token.Kind.float
self.offset += 1
if self.peek() == '+' or self.peek() == '-':
self.offset += 1
try self.digits(10)
let digits_end = self.offset
while is_letter(self.peek()) or is_digit(self.peek()):
self.offset += 1 # a suffix such as u8 or f32
if self.offset > digits_end and not is_literal_suffix(self.src.text(digits_end, self.offset)):
error(bad_character, "a literal suffix names an integer or float type: `255u8`, `0.5f32`")
return self.make(kind, start)
## The digits of a number in `base` with `_` between digits (§4.2): at least one digit,
## no separator at either end or doubled, and no digit of a wider base after them.
func digits(base: u8) -> !:
var count: usize = 0
var separator = false
while true:
let c = self.peek()
if c == '_':
if count == 0 or separator:
error(bad_character, "a `_` in a number separates two digits")
separator = true
elif digit_value(c) < base:
count += 1
separator = false
else:
break
self.offset += 1
if separator:
error(bad_character, "a `_` in a number separates two digits")
if count == 0:
error(bad_character, "a number needs a digit")
if is_digit(self.peek()):
error(bad_character, "a digit outside the number's base")
## The escapes of §4.2: `\\ \" \' \n \r \t \0 \u{HEX}`, and `\xNN` in a byte literal only.
func escape(bytes: bool) -> !:
self.offset += 1 # the backslash
let c = self.peek()
if c == 'u' and self.peek_at(1) == '{':
self.offset += 2
var digits: usize = 0
while is_hex(self.peek()):
self.offset += 1
digits += 1
if self.peek() != '}' or digits == 0 or digits > 6:
error(bad_character, "a Unicode escape is \\u{HEX} with one to six digits")
self.offset += 1
return
if c == 0 or c == '\n':
error(unterminated, "escape at the end of the line")
if bytes and c == 'x':
if not (is_hex(self.peek_at(1)) and is_hex(self.peek_at(2))):
error(bad_character, "a byte escape is \\xNN")
self.offset += 3
return
if c != '\\' and c != '"' and c != '\'' and c != 'n' and c != 'r' and c != 't' and c != '0':
error(bad_character, "unknown escape")
self.offset += 1
## A quoted literal after its opening quote: text, bytes, raw, or triple.
func quoted(kind: token.Kind, start: usize, triple: bool, raw: bool) -> token.Token!:
# a triple-quoted literal spans lines; its token keeps the line and column it opened on
let first_line = self.line
let first_start = self.line_start
while true:
let c = self.peek()
if c == 0:
error(unterminated, "unterminated string literal")
if c == '\n' and not triple:
error(unterminated, "a string literal does not span lines")
if c == '\\' and not raw:
try self.escape(kind == token.Kind.bytes)
continue
if kind == token.Kind.bytes and c >= 0x80:
error(bad_character, "a byte literal holds ASCII text and `\\xNN` escapes")
if c == '"':
if not triple:
self.offset += 1
return self.make(kind, start)
if self.peek_at(1) == '"' and self.peek_at(2) == '"':
self.offset += 3
return token.Token(kind = kind, start = start, end = self.offset, line = first_line,
column = u32(start - first_start) + 1)
if c == '\n':
self.line += 1
self.line_start = self.offset + 1
self.offset += 1
## The literal part of a formatted string up to the next field or its end.
func format_part() -> token.Token!:
let start = self.offset
while true:
let c = self.peek()
if c == 0 or c == '\n':
error(unterminated, "unterminated formatted string")
if c == '"':
if self.offset > start:
return self.make(token.Kind.format_text, start)
self.offset += 1
self.pop_mode()
return self.make(token.Kind.format_end, start)
if c == '{':
if self.peek_at(1) == '{':
self.offset += 2
continue
if self.offset > start:
return self.make(token.Kind.format_text, start)
self.offset += 1
try self.push_mode(2)
self.field_brackets[self.mode_depth - 1] = self.delimiters
return self.make(token.Kind.operator, start)
if c == '}' and self.peek_at(1) == '}':
self.offset += 2
continue
if c == '\\':
try self.escape(false)
continue
self.offset += 1
## A field's specification, from after its `:` up to the field's `}`: plain text, no
## field or escape of its own (§5.5).
func format_spec() -> token.Token!:
self.offset += 1
let start = self.offset
while self.peek() != '}':
let c = self.peek()
if c == 0 or c == '\n' or c == '"':
error(unterminated, "a formatted string's field ends with `}`")
if c == '{' or c == '\\':
error(bad_character, "a format specification holds no field and no escape")
self.offset += 1
return self.make(token.Kind.format_spec, start)
func operator(start: usize) -> token.Token!:
let first = self.peek()
for op in token.operators:
# the first byte rules out all but a few, before any text is compared
if op.bytes[0] == first and self.offset + op.length <= self.src.bytes.length and (str)self.src.bytes[self.offset..<self.offset + op.length] == op:
self.offset += op.length
return self.make(token.Kind.operator, start)
for op in token.singles:
if self.peek() == op.bytes[0]:
self.offset += 1
return self.make(token.Kind.operator, start)
error(bad_character, "unexpected character")
## The next token, advancing past it; the `eof` token at the end.
pub func next() -> token.Token!:
self.token_start = self.offset
if self.in_raw:
return try self.raw_line()
if self.pending_dedents > 0:
self.pending_dedents -= 1
return self.make(token.Kind.dedent, self.offset)
if self.mode() == 1:
return try self.format_part()
if self.at_line_start:
if let layout_token = try self.layout():
return layout_token
if let comment = try self.skip_trivia():
return comment
let start = self.offset
self.token_start = start
let c = self.peek()
if c == 0:
if self.depth > 0:
self.depth -= 1
return self.make(token.Kind.dedent, start)
return self.make(token.Kind.eof, start)
if c == '\n':
self.newline_here()
if self.delimiters > 0 and self.depth <= self.bracket_base[(usize)self.delimiters - 1]:
if not self.colon_ended(start):
return try self.next() # spacing inside brackets
self.reopened = true
let t = token.Token(kind = token.Kind.newline, start = start, end = start + 1,
line = self.line - 1, column = u32(start - self.line_start_before(start)) + 1)
return t
if is_letter(c):
return try self.name_or_keyword(start, c)
if is_digit(c):
return try self.number(start)
if c == '"':
self.offset += 1
if self.peek() == '"' and self.peek_at(1) == '"':
self.offset += 2
return try self.quoted(token.Kind.text, start, true, false)
return try self.quoted(token.Kind.text, start, false, false)
if c == '\'':
return try self.character(start)
if c == ':' and self.mode() == 2 and self.delimiters == self.field_brackets[self.mode_depth - 1]:
return try self.format_spec()
if c == '(' or c == '[' or c == '{':
# the depth a bracket was opened at is kept for each; more open brackets than
# the table holds is a diagnostic, never a fault
if (usize)self.delimiters >= self.bracket_base.length:
error(bad_indent, "brackets nest too deeply")
self.bracket_base[(usize)self.delimiters] = self.depth
self.delimiters += 1
elif c == ')' or c == ']' or (c == '}' and self.mode() != 2):
if self.delimiters > 0:
# a closer on a suite's last line ends the suite first
let base = self.bracket_base[(usize)self.delimiters - 1]
if self.depth > base:
while self.depth > base:
self.depth -= 1
self.pending_dedents += 1
self.pending_dedents -= 1
self.reopened = false
return self.make(token.Kind.dedent, start)
self.delimiters -= 1
elif c == '}':
self.offset += 1
self.pop_mode()
return self.make(token.Kind.operator, start)
return try self.operator(start)
## A token starting with the letter `c` at `start`: a name, a keyword, or a `b"..."`,
## `r"..."`, or `f"..."` literal; `asm ARCH` at the start of a line opens a raw suite.
func name_or_keyword(start: usize, c: u8) -> token.Token!:
# b"...", r"...", f"..." are literals, not names
if (c == 'b' or c == 'r' or c == 'f') and self.peek_at(1) == '"':
self.offset += 2
if c == 'f':
try self.push_mode(1)
return self.make(token.Kind.format_start, start)
let triple = self.peek() == '"' and self.peek_at(1) == '"'
if triple:
self.offset += 2
return try self.quoted(token.Kind.bytes if c == 'b' else token.Kind.raw, start, triple, c == 'r')
while is_letter(self.peek()) or is_digit(self.peek()):
self.offset += 1
if self.offset - start > longest_name:
error(bad_character, "an identifier is at most 128 bytes")
let word = self.src.text(start, self.offset)
if not token.is_keyword(word):
return self.make(token.Kind.name, start)
# `asm ARCH ... :` at the start of a line opens a raw suite;
# `asm = 19` (an enum case) or `.asm` does not.
if word == "asm" and start == self.line_start + (usize)self.levels[self.depth]:
var look = self.offset
while look < self.src.bytes.length and self.src.bytes[look] == ' ':
look += 1
if look < self.src.bytes.length and is_letter(self.src.bytes[look]):
self.asm_header = true
return self.make(token.Kind.keyword, start)
## A character literal at `start`: one scalar, or an escape, between single quotes.
func character(start: usize) -> token.Token!:
self.offset += 1
if self.peek() == '\n' or self.peek() == 0:
error(unterminated, "unterminated character literal")
if self.peek() == '\\':
try self.escape(false)
else:
self.offset += 1
# a multi-byte UTF-8 scalar continues with 10xxxxxx bytes
while (self.peek() & 0xC0) == 0x80:
self.offset += 1
if self.peek() != '\'':
error(unterminated, "unterminated character literal")
self.offset += 1
return self.make(token.Kind.character, start)
## Whether the physical line ending at `end` ends with `:` (a suite inside brackets); a
## comment ending the line is read past, as the code before it is what ends.
func colon_ended(end: usize) -> bool:
var i = self.comment_start if self.comment_end == end else end
while i > 0:
i -= 1
let c = self.src.bytes[i]
if c == ' ' or c == '\r':
continue
return c == ':'
return false
## The start of the line containing `offset`, for a token made after a newline.
func line_start_before(offset: usize) -> usize:
var i = offset
while i > 0 and self.src.bytes[i - 1] != '\n':
i -= 1
return i
## The source text token `t` spans.
pub func text(t: token.Token) -> str:
return self.src.text(t.start, t.end)
func is_hex(c: u8) -> bool:
return is_digit(c) or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')
func is_letter(c: u8) -> bool:
return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or c == '_'
func is_digit(c: u8) -> bool:
return c >= '0' and c <= '9'
# mark: Entry point and tests =================================================================
## Tokenize `text` into `out`; the number of tokens, `eof` excluded.
pub func tokenize(text: str, out: token.Token[]) -> usize!:
let src = try source.Source.over("test", text.bytes)
var lexer = Lexer.over(src)
var n: usize = 0
while true:
let t = try lexer.next()
if t.kind == token.Kind.eof:
return n
if n >= out.length:
error(unterminated, "token buffer is full")
out[n] = t
n += 1
test "names keywords and operators":
var toks: token.Token[16] = ---
let n = tokenize("let x +%= 40_000u8", toks) catch failure: recover 0
assert(n == 4)
assert(toks[0].kind == token.Kind.keyword)
assert(toks[1].kind == token.Kind.name)
assert(toks[2].kind == token.Kind.operator)
assert(toks[2].end - toks[2].start == 3)
assert(toks[3].kind == token.Kind.integer)
test "layout makes indent and dedent":
var toks: token.Token[32] = ---
let n = tokenize("if a:\n b\n c\nd\n", toks) catch failure: recover 0
# if a : newline indent b newline c newline dedent d newline
assert(n == 12)
assert(toks[4].kind == token.Kind.indent)
assert(toks[9].kind == token.Kind.dedent)
test "newlines inside brackets are spacing":
var toks: token.Token[32] = ---
let n = tokenize("f(1,\n 2)\n", toks) catch failure: recover 0
assert(n == 7)
test "formatted strings split into parts":
var toks: token.Token[32] = ---
let n = tokenize("f\"a {b} c\"", toks) catch failure: recover 0
# format_start format_text { b } format_text format_end
assert(n == 7)
assert(toks[0].kind == token.Kind.format_start)
assert(toks[3].kind == token.Kind.name)
assert(toks[6].kind == token.Kind.format_end)
test "literals":
var toks: token.Token[32] = ---
let n = tokenize("'\\n' \"s\" b\"\\x00\" r\"\\\" 1.5e3 0xff", toks) catch failure: recover 0
assert(n == 6)
assert(toks[0].kind == token.Kind.character)
assert(toks[2].kind == token.Kind.bytes)
assert(toks[3].kind == token.Kind.raw)
assert(toks[4].kind == token.Kind.float)
assert(toks[5].kind == token.Kind.integer)
test "asm suites are raw":
var toks: token.Token[32] = ---
let n = tokenize("asm arm64:\n mov x0, #1\n ret\nx\n", toks) catch failure: recover 0
# asm arm64 : newline indent raw raw dedent x newline
assert(n == 10)
assert(toks[5].kind == token.Kind.raw_line)
assert(toks[6].kind == token.Kind.raw_line)
test "a comment after an asm suite ends the suite":
var toks: token.Token[32] = ---
let n = tokenize("asm arm64:\n ret\n## doc\nx\n", toks) catch failure: recover 0
# asm arm64 : newline indent raw dedent doc_comment x newline
assert(n == 10)
assert(toks[5].kind == token.Kind.raw_line)
assert(toks[6].kind == token.Kind.dedent)
assert(toks[7].kind == token.Kind.doc_comment)
test "a blank CRLF line inside or after an asm suite keeps the suite's block":
let src = try source.Source.over("test", "func f():\r\n asm arm64:\r\n ret\r\n\r\n nop\r\n asm x86_64:\r\n ret\r\n\r\n # a comment\r\n g()\r\n".bytes)
var lexer = Lexer.over(src)
var raw = 0
var dedents = 0
var g_line: u32 = 0
for index in 0..<40:
_ = index
let t = try lexer.next()
if t.kind == token.Kind.raw_line:
raw += 1
if t.kind == token.Kind.dedent:
dedents += 1
if t.kind == token.Kind.name and lexer.text(t) == "g":
g_line = t.line
break
# three raw lines, each suite's dedent and none past the function's body, `g` on line 10
assert(raw == 3)
assert(dedents == 2)
assert(g_line == 10)
test "CRLF asm lines exclude the physical line ending":
let src = try source.Source.over("test", "asm x86_64:\r\n cpuid\r\nx\r\n".bytes)
var lexer = Lexer.over(src)
var found = false
for index in 0..<8:
_ = index
let t = try lexer.next()
if t.kind == token.Kind.raw_line:
assert(lexer.text(t) == "cpuid")
found = true
assert(found)
test "a tab is an error":
var toks: token.Token[8] = ---
var refused = false
_ = tokenize("\tx", toks) catch failure:
refused = failure.code == tab
recover 0
assert(refused)
test "a suite inside brackets closes with the bracket":
var toks: token.Token[64] = ---
let n = tokenize("f(x catch e:\n recover 0)\ny\n", toks) catch failure: recover 0
# f ( x catch e : newline indent recover 0 dedent ) newline y newline
assert(n == 15)
assert(toks[7].kind == token.Kind.indent)
assert(toks[10].kind == token.Kind.dedent)
assert(toks[11].kind == token.Kind.operator)
test "a CRLF suite inside brackets closes with the bracket":
var toks: token.Token[64] = ---
let n = tokenize("f(x catch e:\r\n recover 0)\r\ny\r\n", toks) catch failure: recover 0
assert(n == 15)
assert(toks[7].kind == token.Kind.indent)
assert(toks[10].kind == token.Kind.dedent)
assert(toks[11].kind == token.Kind.operator)
test "a colon in a trailing comment does not open a suite":
var toks: token.Token[64] = ---
let n = tokenize("f(1, # note:\n 2)\n", toks) catch failure: recover 0
assert(n == 7)
test "a colon before a trailing comment opens a suite":
var toks: token.Token[64] = ---
let n = tokenize("f(x catch e: # note\n recover 0)\ny\n", toks) catch failure: recover 0
assert(n == 15)
assert(toks[7].kind == token.Kind.indent)
## The value of a digit in any base up to 16, or 16 for a byte that is no digit.
func digit_value(c: u8) -> u8:
if c >= '0' and c <= '9':
return c - '0'
if c >= 'a' and c <= 'f':
return c - 'a' + 10
if c >= 'A' and c <= 'F':
return c - 'A' + 10
return 16