Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.43.0

0.43.0 / src / front / lexer.lucb

#==============================================================================================
#
#   lexer - Bytes to tokens, then layout
#
#   DESCRIPTION:
#       Tokens are spans; the literal decoder of a later slice turns spellings into values.
#       Layout is part of lexing: a newline at the top level becomes `newline`, a deeper first
#       column becomes `indent`, and a shallower one becomes one `dedent` per closed block.
#       Inside `()`, `[]`, and `{}` newlines are spacing, except that a `:` ending a line
#       reopens layout and the bracket that closes the suite ends it (base.md §3.2). A
#       formatted string lexes as `format_start`, `format_text` runs, ordinary tokens for each
#       `{field}`, then `format_end`. An `asm` suite is captured as `raw_line` tokens.
#
#==============================================================================================

import front.source
import front.token
## Indentation is not a multiple of four spaces, or misaligns a block.
pub let bad_indent: ErrorCode = ErrorCode.package(3)
## A string, character, or comment reaches end of file unterminated.
pub let unterminated: ErrorCode = ErrorCode.package(4)
## A byte that is not valid in the source at this point.
pub let bad_character: ErrorCode = ErrorCode.package(5)

## The suffixes a number may carry (§4.2, §4.3): the exact type's name.
func is_literal_suffix(word: str) -> bool:
    let names: str[13] = ["u8", "u16", "u32", "u64", "usize", "i8", "i16", "i32", "i64", "isize", "f16", "f32", "f64"]
    for name in names:
        if word == name:
            return true
    return false

## The longest identifier (§3.1): a diagnostic quoting one fits its buffer.
pub let longest_name: usize = 128
## A tab character in the source, which is not allowed.
pub let tab: ErrorCode = ErrorCode.package(6)

let max_blocks: usize = 64
let max_modes: usize = 32

## The lexer over one source, producing tokens on demand.
pub struct Lexer:
    ## The source being lexed.
    pub var src: source.Source
    var offset: usize
    var line: u32
    var line_start: usize
    var at_line_start: bool
    var levels: u32[64]
    var depth: usize
    var pending_dedents: usize
    var delimiters: i32
    ## 0: code; 1: inside a formatted string's literal part; 2: inside a `{field}`
    var modes: u8[32]
    ## For each field open in `modes`, how many brackets were open before it: a `:` with no
    ## more open than that starts the field's specification (§5.5).
    var field_brackets: i32[32]
    var mode_depth: usize
    var asm_header: bool
    var raw_level: u32
    var in_raw: bool
    ## Offsets of the last comment's `#` and just past it, so a `:` inside it does not open
    ## a suite while one before it still does.
    var comment_start: usize
    var comment_end: usize
    ## The block depth when each open bracket was seen; a closer ends any suite
    ## opened inside it (base.md §3.2: closers on the body's last line).
    var bracket_base: usize[64]
    ## A colon ended a line inside brackets: the next line's indentation counts.
    var reopened: bool
    ## Where the token being read began, for a diagnostic (`error_position`).
    var token_start: usize

    ## A lexer positioned at the start of `src`.
    pub static func over(src: source.Source) -> Lexer:
        return Lexer(src = src, offset = 0, line = 1, line_start = 0, at_line_start = true,
                     depth = 0, pending_dedents = 0, delimiters = 0, mode_depth = 0,
                     asm_header = false, raw_level = 0, in_raw = false, comment_start = 0, comment_end = 0,
                     reopened = false, token_start = 0)

    ## The one-based line and column of the byte about to be read, for diagnostics.
    pub func position() -> (u32, u32):
        return (self.line, u32(self.offset - self.line_start) + 1)

    ## Where the token `next` failed in begins, when it began on the current line; else
    ## where the lexer stopped.
    pub func error_position() -> (u32, u32):
        if self.token_start >= self.line_start:
            return (self.line, u32(self.token_start - self.line_start) + 1)
        return self.position()

    func peek() -> u8:
        if self.offset >= self.src.bytes.length:
            return 0
        return self.src.bytes[self.offset]

    func peek_at(ahead: usize) -> u8:
        if self.offset + ahead >= self.src.bytes.length:
            return 0
        return self.src.bytes[self.offset + ahead]

    func make(kind: token.Kind, start: usize) -> token.Token:
        return token.Token(kind = kind, start = start, end = self.offset, line = self.line,
                           column = u32(start - self.line_start) + 1)

    func mode() -> u8:
        if self.mode_depth == 0:
            return 0
        return self.modes[self.mode_depth - 1]

    func push_mode(m: u8) -> !:
        if self.mode_depth >= max_modes:
            error(unterminated, "formatted strings nest too deeply")
        self.modes[self.mode_depth] = m
        self.mode_depth += 1

    func pop_mode():
        if self.mode_depth > 0:
            self.mode_depth -= 1

    func newline_here():
        self.offset += 1
        self.line += 1
        self.line_start = self.offset
        self.at_line_start = true

    ## Skip spaces and a trailing comment; a `##` at a line's start is a token.
    func skip_trivia() -> token.Token?!:
        while true:
            let c = self.peek()
            if c == ' ':
                self.offset += 1
            elif c == '\t':
                error(tab, "tabs are not allowed; use four spaces")
            elif c == '\r' and self.peek_at(1) == '\n':
                self.offset += 1
            elif c == '#':
                let start = self.offset
                let doc = self.peek_at(1) == '#' and start == self.line_start
                while self.peek() != '\n' and self.peek() != 0:
                    self.offset += 1
                self.comment_start = start
                self.comment_end = self.offset
                if doc:
                    return self.make(token.Kind.doc_comment, start)
            else:
                return none

    ## Measure the current line's indentation and queue layout tokens.
    func layout() -> token.Token?!:
        var column: u32 = 0
        while true:
            column = 0
            while self.peek() == ' ':
                self.offset += 1
                column += 1
            let c = self.peek()
            if c == '#' and not (self.peek_at(1) == '#' and column == 0):
                while self.peek() != '\n' and self.peek() != 0:
                    self.offset += 1  # a comment-only line
                continue
            if c == '\n':
                self.newline_here()  # a blank line
                continue
            if c == '\r' and self.peek_at(1) == '\n':
                self.offset += 1
                continue
            break
        if self.peek() == 0 or self.peek() == '#':
            return none  # end of input, or a doc comment
        self.at_line_start = false
        if self.delimiters > 0 and not self.reopened and self.depth <= self.bracket_base[(usize)self.delimiters - 1]:
            return none  # spacing inside brackets
        self.reopened = false
        let current = self.levels[self.depth]
        if column > current:
            if self.depth + 1 >= max_blocks:
                error(bad_indent, "blocks nest too deeply")
            if column != current + 4:
                error(bad_indent, "a block indents by exactly four spaces")
            self.depth += 1
            self.levels[self.depth] = column
            if self.asm_header:
                self.asm_header = false
                self.in_raw = true
                self.raw_level = column
            return self.make(token.Kind.indent, self.offset)
        while column < self.levels[self.depth]:
            self.depth -= 1
            self.pending_dedents += 1
        if column != self.levels[self.depth]:
            error(bad_indent, "indentation does not match an open block")
        if self.pending_dedents > 0:
            self.pending_dedents -= 1
            return self.make(token.Kind.dedent, self.offset)
        return none

    ## One physical line of an `asm` suite, or the dedent that ends it.
    func raw_line() -> token.Token!:
        # layout() may already have consumed the indentation; measure from the line start
        var i = self.offset
        while i < self.src.bytes.length and self.src.bytes[i] == ' ':
            i += 1
        let column = u32(i - self.line_start)
        var c = self.src.bytes[i] if i < self.src.bytes.length else 0
        if c == '\r' and i + 1 < self.src.bytes.length and self.src.bytes[i + 1] == '\n':
            # a blank line ending in CRLF is a blank line (§3.1)
            i += 1
            c = '\n'
        if c == '\n' or c == 0:
            if c == '\n':
                # past the blank line's spaces, then its end
                self.offset = i
                self.newline_here()
                return try self.raw_line()
            self.in_raw = false
            return try self.next()
        if column < self.raw_level:
            self.in_raw = false
            # End the suite by dedenting to the line's level now, so a comment on
            # that line does not defer the dedent past the asm node (layout() would
            # return none for a `##`/`#` line, hiding the dedent the parser needs).
            while column < self.levels[self.depth]:
                self.depth -= 1
                self.pending_dedents += 1
            return try self.next()
        let start = i
        self.offset = i
        while self.peek() != '\n' and self.peek() != 0:
            self.offset += 1
        var t = self.make(token.Kind.raw_line, start)
        if t.end > t.start and self.src.bytes[t.end - 1] == '\r' and self.peek() == '\n':
            t.end -= 1
        if self.peek() == '\n':
            self.newline_here()
        return t

    func number(start: usize) -> token.Token!:
        var kind = token.Kind.integer
        if self.peek() == '0' and (self.peek_at(1) == 'X' or self.peek_at(1) == 'O' or self.peek_at(1) == 'B'):
            error(bad_character, "a based integer prefix is lowercase: `0x`, `0o`, or `0b`")
        if self.peek() == '0' and (self.peek_at(1) == 'x' or self.peek_at(1) == 'o' or self.peek_at(1) == 'b'):
            let base: u8 = 16 if self.peek_at(1) == 'x' else (8 if self.peek_at(1) == 'o' else 2)
            self.offset += 2
            try self.digits(base)
        else:
            try self.digits(10)
            if self.peek() == '.' and is_digit(self.peek_at(1)):
                kind = token.Kind.float
                self.offset += 1
                try self.digits(10)
            if (self.peek() == 'e' or self.peek() == 'E') and (is_digit(self.peek_at(1)) or ((self.peek_at(1) == '+' or self.peek_at(1) == '-') and is_digit(self.peek_at(2)))):
                kind = token.Kind.float
                self.offset += 1
                if self.peek() == '+' or self.peek() == '-':
                    self.offset += 1
                try self.digits(10)
        let digits_end = self.offset
        while is_letter(self.peek()) or is_digit(self.peek()):
            self.offset += 1  # a suffix such as u8 or f32
        if self.offset > digits_end and not is_literal_suffix(self.src.text(digits_end, self.offset)):
            error(bad_character, "a literal suffix names an integer or float type: `255u8`, `0.5f32`")
        return self.make(kind, start)

    ## The digits of a number in `base` with `_` between digits (§4.2): at least one digit,
    ## no separator at either end or doubled, and no digit of a wider base after them.
    func digits(base: u8) -> !:
        var count: usize = 0
        var separator = false
        while true:
            let c = self.peek()
            if c == '_':
                if count == 0 or separator:
                    error(bad_character, "a `_` in a number separates two digits")
                separator = true
            elif digit_value(c) < base:
                count += 1
                separator = false
            else:
                break
            self.offset += 1
        if separator:
            error(bad_character, "a `_` in a number separates two digits")
        if count == 0:
            error(bad_character, "a number needs a digit")
        if is_digit(self.peek()):
            error(bad_character, "a digit outside the number's base")

    ## The escapes of §4.2: `\\ \" \' \n \r \t \0 \u{HEX}`, and `\xNN` in a byte literal only.
    func escape(bytes: bool) -> !:
        self.offset += 1  # the backslash
        let c = self.peek()
        if c == 'u' and self.peek_at(1) == '{':
            self.offset += 2
            var digits: usize = 0
            while is_hex(self.peek()):
                self.offset += 1
                digits += 1
            if self.peek() != '}' or digits == 0 or digits > 6:
                error(bad_character, "a Unicode escape is \\u{HEX} with one to six digits")
            self.offset += 1
            return
        if c == 0 or c == '\n':
            error(unterminated, "escape at the end of the line")
        if bytes and c == 'x':
            if not (is_hex(self.peek_at(1)) and is_hex(self.peek_at(2))):
                error(bad_character, "a byte escape is \\xNN")
            self.offset += 3
            return
        if c != '\\' and c != '"' and c != '\'' and c != 'n' and c != 'r' and c != 't' and c != '0':
            error(bad_character, "unknown escape")
        self.offset += 1

    ## A quoted literal after its opening quote: text, bytes, raw, or triple.
    func quoted(kind: token.Kind, start: usize, triple: bool, raw: bool) -> token.Token!:
        # a triple-quoted literal spans lines; its token keeps the line and column it opened on
        let first_line = self.line
        let first_start = self.line_start
        while true:
            let c = self.peek()
            if c == 0:
                error(unterminated, "unterminated string literal")
            if c == '\n' and not triple:
                error(unterminated, "a string literal does not span lines")
            if c == '\\' and not raw:
                try self.escape(kind == token.Kind.bytes)
                continue
            if kind == token.Kind.bytes and c >= 0x80:
                error(bad_character, "a byte literal holds ASCII text and `\\xNN` escapes")
            if c == '"':
                if not triple:
                    self.offset += 1
                    return self.make(kind, start)
                if self.peek_at(1) == '"' and self.peek_at(2) == '"':
                    self.offset += 3
                    return token.Token(kind = kind, start = start, end = self.offset, line = first_line,
                                       column = u32(start - first_start) + 1)
            if c == '\n':
                self.line += 1
                self.line_start = self.offset + 1
            self.offset += 1

    ## The literal part of a formatted string up to the next field or its end.
    func format_part() -> token.Token!:
        let start = self.offset
        while true:
            let c = self.peek()
            if c == 0 or c == '\n':
                error(unterminated, "unterminated formatted string")
            if c == '"':
                if self.offset > start:
                    return self.make(token.Kind.format_text, start)
                self.offset += 1
                self.pop_mode()
                return self.make(token.Kind.format_end, start)
            if c == '{':
                if self.peek_at(1) == '{':
                    self.offset += 2
                    continue
                if self.offset > start:
                    return self.make(token.Kind.format_text, start)
                self.offset += 1
                try self.push_mode(2)
                self.field_brackets[self.mode_depth - 1] = self.delimiters
                return self.make(token.Kind.operator, start)
            if c == '}' and self.peek_at(1) == '}':
                self.offset += 2
                continue
            if c == '\\':
                try self.escape(false)
                continue
            self.offset += 1

    ## A field's specification, from after its `:` up to the field's `}`: plain text, no
    ## field or escape of its own (§5.5).
    func format_spec() -> token.Token!:
        self.offset += 1
        let start = self.offset
        while self.peek() != '}':
            let c = self.peek()
            if c == 0 or c == '\n' or c == '"':
                error(unterminated, "a formatted string's field ends with `}`")
            if c == '{' or c == '\\':
                error(bad_character, "a format specification holds no field and no escape")
            self.offset += 1
        return self.make(token.Kind.format_spec, start)

    func operator(start: usize) -> token.Token!:
        let first = self.peek()
        for op in token.operators:
            # the first byte rules out all but a few, before any text is compared
            if op.bytes[0] == first and self.offset + op.length <= self.src.bytes.length and (str)self.src.bytes[self.offset..<self.offset + op.length] == op:
                self.offset += op.length
                return self.make(token.Kind.operator, start)
        for op in token.singles:
            if self.peek() == op.bytes[0]:
                self.offset += 1
                return self.make(token.Kind.operator, start)
        error(bad_character, "unexpected character")

    ## The next token, advancing past it; the `eof` token at the end.
    pub func next() -> token.Token!:
        self.token_start = self.offset
        if self.in_raw:
            return try self.raw_line()
        if self.pending_dedents > 0:
            self.pending_dedents -= 1
            return self.make(token.Kind.dedent, self.offset)
        if self.mode() == 1:
            return try self.format_part()
        if self.at_line_start:
            if let layout_token = try self.layout():
                return layout_token
        if let comment = try self.skip_trivia():
            return comment
        let start = self.offset
        self.token_start = start
        let c = self.peek()
        if c == 0:
            if self.depth > 0:
                self.depth -= 1
                return self.make(token.Kind.dedent, start)
            return self.make(token.Kind.eof, start)
        if c == '\n':
            self.newline_here()
            if self.delimiters > 0 and self.depth <= self.bracket_base[(usize)self.delimiters - 1]:
                if not self.colon_ended(start):
                    return try self.next()  # spacing inside brackets
                self.reopened = true
            let t = token.Token(kind = token.Kind.newline, start = start, end = start + 1,
                                line = self.line - 1, column = u32(start - self.line_start_before(start)) + 1)
            return t
        if is_letter(c):
            return try self.name_or_keyword(start, c)
        if is_digit(c):
            return try self.number(start)
        if c == '"':
            self.offset += 1
            if self.peek() == '"' and self.peek_at(1) == '"':
                self.offset += 2
                return try self.quoted(token.Kind.text, start, true, false)
            return try self.quoted(token.Kind.text, start, false, false)
        if c == '\'':
            return try self.character(start)
        if c == ':' and self.mode() == 2 and self.delimiters == self.field_brackets[self.mode_depth - 1]:
            return try self.format_spec()
        if c == '(' or c == '[' or c == '{':
            # the depth a bracket was opened at is kept for each; more open brackets than
            # the table holds is a diagnostic, never a fault
            if (usize)self.delimiters >= self.bracket_base.length:
                error(bad_indent, "brackets nest too deeply")
            self.bracket_base[(usize)self.delimiters] = self.depth
            self.delimiters += 1
        elif c == ')' or c == ']' or (c == '}' and self.mode() != 2):
            if self.delimiters > 0:
                # a closer on a suite's last line ends the suite first
                let base = self.bracket_base[(usize)self.delimiters - 1]
                if self.depth > base:
                    while self.depth > base:
                        self.depth -= 1
                        self.pending_dedents += 1
                    self.pending_dedents -= 1
                    self.reopened = false
                    return self.make(token.Kind.dedent, start)
                self.delimiters -= 1
        elif c == '}':
            self.offset += 1
            self.pop_mode()
            return self.make(token.Kind.operator, start)
        return try self.operator(start)

    ## A token starting with the letter `c` at `start`: a name, a keyword, or a `b"..."`,
    ## `r"..."`, or `f"..."` literal; `asm ARCH` at the start of a line opens a raw suite.
    func name_or_keyword(start: usize, c: u8) -> token.Token!:
        # b"...", r"...", f"..." are literals, not names
        if (c == 'b' or c == 'r' or c == 'f') and self.peek_at(1) == '"':
            self.offset += 2
            if c == 'f':
                try self.push_mode(1)
                return self.make(token.Kind.format_start, start)
            let triple = self.peek() == '"' and self.peek_at(1) == '"'
            if triple:
                self.offset += 2
            return try self.quoted(token.Kind.bytes if c == 'b' else token.Kind.raw, start, triple, c == 'r')
        while is_letter(self.peek()) or is_digit(self.peek()):
            self.offset += 1
        if self.offset - start > longest_name:
            error(bad_character, "an identifier is at most 128 bytes")
        let word = self.src.text(start, self.offset)
        if not token.is_keyword(word):
            return self.make(token.Kind.name, start)
        # `asm ARCH ... :` at the start of a line opens a raw suite;
        # `asm = 19` (an enum case) or `.asm` does not.
        if word == "asm" and start == self.line_start + (usize)self.levels[self.depth]:
            var look = self.offset
            while look < self.src.bytes.length and self.src.bytes[look] == ' ':
                look += 1
            if look < self.src.bytes.length and is_letter(self.src.bytes[look]):
                self.asm_header = true
        return self.make(token.Kind.keyword, start)

    ## A character literal at `start`: one scalar, or an escape, between single quotes.
    func character(start: usize) -> token.Token!:
        self.offset += 1
        if self.peek() == '\n' or self.peek() == 0:
            error(unterminated, "unterminated character literal")
        if self.peek() == '\\':
            try self.escape(false)
        else:
            self.offset += 1
            # a multi-byte UTF-8 scalar continues with 10xxxxxx bytes
            while (self.peek() & 0xC0) == 0x80:
                self.offset += 1
        if self.peek() != '\'':
            error(unterminated, "unterminated character literal")
        self.offset += 1
        return self.make(token.Kind.character, start)

    ## Whether the physical line ending at `end` ends with `:` (a suite inside brackets); a
    ## comment ending the line is read past, as the code before it is what ends.
    func colon_ended(end: usize) -> bool:
        var i = self.comment_start if self.comment_end == end else end
        while i > 0:
            i -= 1
            let c = self.src.bytes[i]
            if c == ' ' or c == '\r':
                continue
            return c == ':'
        return false

    ## The start of the line containing `offset`, for a token made after a newline.
    func line_start_before(offset: usize) -> usize:
        var i = offset
        while i > 0 and self.src.bytes[i - 1] != '\n':
            i -= 1
        return i

    ## The source text token `t` spans.
    pub func text(t: token.Token) -> str:
        return self.src.text(t.start, t.end)

func is_hex(c: u8) -> bool:
    return is_digit(c) or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')

func is_letter(c: u8) -> bool:
    return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or c == '_'

func is_digit(c: u8) -> bool:
    return c >= '0' and c <= '9'

# mark: Entry point and tests =================================================================

## Tokenize `text` into `out`; the number of tokens, `eof` excluded.
pub func tokenize(text: str, out: token.Token[]) -> usize!:
    let src = try source.Source.over("test", text.bytes)
    var lexer = Lexer.over(src)
    var n: usize = 0
    while true:
        let t = try lexer.next()
        if t.kind == token.Kind.eof:
            return n
        if n >= out.length:
            error(unterminated, "token buffer is full")
        out[n] = t
        n += 1

test "names keywords and operators":
    var toks: token.Token[16] = ---
    let n = tokenize("let x +%= 40_000u8", toks) catch failure: recover 0
    assert(n == 4)
    assert(toks[0].kind == token.Kind.keyword)
    assert(toks[1].kind == token.Kind.name)
    assert(toks[2].kind == token.Kind.operator)
    assert(toks[2].end - toks[2].start == 3)
    assert(toks[3].kind == token.Kind.integer)

test "layout makes indent and dedent":
    var toks: token.Token[32] = ---
    let n = tokenize("if a:\n    b\n    c\nd\n", toks) catch failure: recover 0
    # if a : newline indent b newline c newline dedent d newline
    assert(n == 12)
    assert(toks[4].kind == token.Kind.indent)
    assert(toks[9].kind == token.Kind.dedent)

test "newlines inside brackets are spacing":
    var toks: token.Token[32] = ---
    let n = tokenize("f(1,\n  2)\n", toks) catch failure: recover 0
    assert(n == 7)

test "formatted strings split into parts":
    var toks: token.Token[32] = ---
    let n = tokenize("f\"a {b} c\"", toks) catch failure: recover 0
    # format_start format_text { b } format_text format_end
    assert(n == 7)
    assert(toks[0].kind == token.Kind.format_start)
    assert(toks[3].kind == token.Kind.name)
    assert(toks[6].kind == token.Kind.format_end)

test "literals":
    var toks: token.Token[32] = ---
    let n = tokenize("'\\n' \"s\" b\"\\x00\" r\"\\\" 1.5e3 0xff", toks) catch failure: recover 0
    assert(n == 6)
    assert(toks[0].kind == token.Kind.character)
    assert(toks[2].kind == token.Kind.bytes)
    assert(toks[3].kind == token.Kind.raw)
    assert(toks[4].kind == token.Kind.float)
    assert(toks[5].kind == token.Kind.integer)

test "asm suites are raw":
    var toks: token.Token[32] = ---
    let n = tokenize("asm arm64:\n    mov x0, #1\n    ret\nx\n", toks) catch failure: recover 0
    # asm arm64 : newline indent raw raw dedent x newline
    assert(n == 10)
    assert(toks[5].kind == token.Kind.raw_line)
    assert(toks[6].kind == token.Kind.raw_line)

test "a comment after an asm suite ends the suite":
    var toks: token.Token[32] = ---
    let n = tokenize("asm arm64:\n    ret\n## doc\nx\n", toks) catch failure: recover 0
    # asm arm64 : newline indent raw dedent doc_comment x newline
    assert(n == 10)
    assert(toks[5].kind == token.Kind.raw_line)
    assert(toks[6].kind == token.Kind.dedent)
    assert(toks[7].kind == token.Kind.doc_comment)

test "a blank CRLF line inside or after an asm suite keeps the suite's block":
    let src = try source.Source.over("test", "func f():\r\n    asm arm64:\r\n        ret\r\n\r\n        nop\r\n    asm x86_64:\r\n        ret\r\n\r\n    # a comment\r\n    g()\r\n".bytes)
    var lexer = Lexer.over(src)
    var raw = 0
    var dedents = 0
    var g_line: u32 = 0
    for index in 0..<40:
        _ = index
        let t = try lexer.next()
        if t.kind == token.Kind.raw_line:
            raw += 1
        if t.kind == token.Kind.dedent:
            dedents += 1
        if t.kind == token.Kind.name and lexer.text(t) == "g":
            g_line = t.line
            break
    # three raw lines, each suite's dedent and none past the function's body, `g` on line 10
    assert(raw == 3)
    assert(dedents == 2)
    assert(g_line == 10)

test "CRLF asm lines exclude the physical line ending":
    let src = try source.Source.over("test", "asm x86_64:\r\n    cpuid\r\nx\r\n".bytes)
    var lexer = Lexer.over(src)
    var found = false
    for index in 0..<8:
        _ = index
        let t = try lexer.next()
        if t.kind == token.Kind.raw_line:
            assert(lexer.text(t) == "cpuid")
            found = true
    assert(found)

test "a tab is an error":
    var toks: token.Token[8] = ---
    var refused = false
    _ = tokenize("\tx", toks) catch failure:
        refused = failure.code == tab
        recover 0
    assert(refused)

test "a suite inside brackets closes with the bracket":
    var toks: token.Token[64] = ---
    let n = tokenize("f(x catch e:\n    recover 0)\ny\n", toks) catch failure: recover 0
    # f ( x catch e : newline indent recover 0 dedent ) newline y newline
    assert(n == 15)
    assert(toks[7].kind == token.Kind.indent)
    assert(toks[10].kind == token.Kind.dedent)
    assert(toks[11].kind == token.Kind.operator)

test "a CRLF suite inside brackets closes with the bracket":
    var toks: token.Token[64] = ---
    let n = tokenize("f(x catch e:\r\n    recover 0)\r\ny\r\n", toks) catch failure: recover 0
    assert(n == 15)
    assert(toks[7].kind == token.Kind.indent)
    assert(toks[10].kind == token.Kind.dedent)
    assert(toks[11].kind == token.Kind.operator)

test "a colon in a trailing comment does not open a suite":
    var toks: token.Token[64] = ---
    let n = tokenize("f(1,   # note:\n  2)\n", toks) catch failure: recover 0
    assert(n == 7)

test "a colon before a trailing comment opens a suite":
    var toks: token.Token[64] = ---
    let n = tokenize("f(x catch e:   # note\n    recover 0)\ny\n", toks) catch failure: recover 0
    assert(n == 15)
    assert(toks[7].kind == token.Kind.indent)

## The value of a digit in any base up to 16, or 16 for a byte that is no digit.
func digit_value(c: u8) -> u8:
    if c >= '0' and c <= '9':
        return c - '0'
    if c >= 'a' and c <= 'f':
        return c - 'a' + 10
    if c >= 'A' and c <= 'F':
        return c - 'A' + 10
    return 16