Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.45.0

0.45.0 / src / bind / pre.lucb

#==============================================================================================
#
#   pre - A C preprocessor for `luce-base bind`
#
#   DESCRIPTION:
#       The header a binding is generated from is read the way the host C compiler reads
#       it: the same system include directories and the same predefined macros, both asked
#       of `cc` once, then C's preprocessing done here. Tokens are a linked list; a macro
#       expansion splices its result in front of what follows, with hide sets so that a
#       macro never expands inside its own expansion. What comes out is one token list of
#       the whole translation unit, each token knowing its file and line, for the
#       declaration parser.
#
#       The directives: `#include` and `#include_next` through the directories, with
#       `#pragma once`; `#define` of object-like and function-like macros with `#`, `##`,
#       `__VA_ARGS__` and `__VA_OPT__`; `#undef`; the `#if` family over C's constant
#       expressions with `defined`, `__has_include` and the `__has_*` queries; `#error`
#       fails the run; the rest is ignored.
#
#==============================================================================================

import c
import luce_std.files
import memory
import luce_std.paths
import platform
import luce_std.process
import strings
import bind.table
import support.buffer
import support.in_arena
import support.list

## A preprocessing error.
pub let failed: ErrorCode = ErrorCode.package(119)

## The lexical kind of a preprocessing token.
pub enum Kind:
    ident
    number
    text
    character
    punct
    other
    eof

## A preprocessing token: its spelling, where it came from, and the spacing around it.
pub struct Tok:
    ## The token's lexical kind.
    pub var kind: Kind
    ## The token's spelling.
    pub var text: str
    ## The source file the token came from.
    pub var file: u32
    ## The line the token is on.
    pub var line: u32
    ## Whitespace before it, and whether it is the first token of its line.
    pub var space: bool
    ## Whether it is the first token of its line.
    pub var bol: bool
    var hide: Hide*?
    ## The next token in the list, or none at the end.
    pub var next: Tok*?

    ## Whether the token has this kind and spelling.
    pub func is(kind: Kind, text: str) -> bool:
        return self.kind == kind and self.text == text

    ## Whether the token is this punctuator.
    pub func is_punct(text: str) -> bool:
        return self.kind == Kind.punct and self.text == text

## The macros a token must not expand: those it came out of.
struct Hide:
    var name: str
    var next: Hide*?

## A macro. `body` is a token list ending in `eof`; a builtin computes its value.
pub struct Macro:
    ## The macro's name.
    pub var name: str
    ## Whether it takes arguments.
    pub var function_like: bool
    ## The parameter names, for a function-like macro.
    pub var params: str[]
    ## Whether it ends in `...`.
    pub var variadic: bool
    ## The replacement tokens, ending in `eof`.
    pub var body: Tok*?
    var builtin: u8
    ## The file it was defined in.
    pub var file: u32
    ## The line it was defined on.
    pub var line: u32
    ## Whether a later `#undef` removed it.
    pub var deleted: bool

## A file the preprocessor opened: its canonical path and which directory found it.
pub struct FileEntry:
    ## The file's canonical path.
    pub var path: str
    var dir: usize

struct Cond:
    var included: bool
    var in_else: bool
    var line: u32
    var file: u32

struct Found:
    var path: str
    var dir: usize

let no_dir: usize = 18446744073709551615
let builtin_file: u8 = 1
let builtin_line: u8 = 2
let builtin_counter: u8 = 3

var scratch: u8[4096]
var place: u8[4096]

## The C preprocessor: include search, macros, and the files it opened.
pub struct Preprocessor:
    ## Include directories: the `-I` ones first, then the compiler's system directories.
    var dirs: list.List[str]
    var user_dirs: usize
    var frameworks: list.List[str]
    ## Every file opened, in order; a token's `file` indexes this list.
    pub var files: list.List[FileEntry]
    var once: list.List[str]
    ## Each macro's current definition by name; `#undef` marks it deleted.
    var macros: table.Table[Macro*]
    ## Every macro defined, in order, redefinitions and undefined ones included.
    pub var defined: list.List[Macro*]
    var conds: list.List[Cond]
    var counter: u32
    ## How many files were included; a header that includes itself without end is stopped.
    var depth: u32
    ## The file `run` was given.
    pub var main_file: u32
    ## What the run had to say: warnings, ignored pragmas, and the like.
    pub var report: buffer.Buffer

    # mark: Setup ==============================================================================

    ## A preprocessor with `compiler`'s include directories and predefined macros.
    pub static func create(compiler: c.str) -> Preprocessor!:
        var p = Preprocessor(dirs = try list.List[str].create(16), user_dirs = 0,
                             frameworks = try list.List[str].create(4),
                             files = try list.List[FileEntry].create(64),
                             once = try list.List[str].create(64), macros = try table.Table[Macro*].create(512),
                             defined = try list.List[Macro*].create(1024),
                             conds = try list.List[Cond].create(32), counter = 0, depth = 0, main_file = 0,
                             report = try buffer.Buffer.create(1024))
        try p.ask_directories(compiler)
        try p.ask_macros(compiler)
        try p.add_builtin("__FILE__", builtin_file)
        try p.add_builtin("__LINE__", builtin_line)
        try p.add_builtin("__COUNTER__", builtin_counter)
        return p

    ## The system include directories, from `cc -E -Wp,-v` on an empty input.
    func ask_directories(compiler: c.str) -> !:
        let empty: c.str = "nul" if platform.windows else "/dev/null"
        let arguments: c.str[4] = ["-E", "-Wp,-v", "-xc", empty]
        let asked = try process.run(compiler, arguments)
        defer asked.release()
        if asked.value.exit_code != 0:
            error(failed, "the C compiler does not report its include directories")
        var inside = false
        for line in try lines(asked.value.error_output):
            let t = strings.trim(line)
            if t == "#include <...> search starts here:":
                inside = true
                continue
            if t == "End of search list.":
                break
            if not inside or t.length == 0 or t.bytes[0] == '#':
                continue
            if strings.ends_with(t, " (framework directory)"):
                try self.frameworks.push(try strings.copy((str)t.bytes[..<t.length - 22]))
            else:
                try self.dirs.push(try strings.copy(t))

    ## The predefined macros, from `cc -dM -E`, read as `#define` lines.
    func ask_macros(compiler: c.str) -> !:
        let empty: c.str = "nul" if platform.windows else "/dev/null"
        let arguments: c.str[4] = ["-dM", "-E", "-xc", empty]
        let asked = try process.run(compiler, arguments)
        defer asked.release()
        if asked.value.exit_code != 0:
            error(failed, "the C compiler does not report its predefined macros")
        try self.files.push(FileEntry(path = "<built-in>", dir = no_dir))
        # the tokens point into their text, which outlives the answer
        let macros = try strings.copy(asked.value.output)
        let head = try tokenize(macros.bytes, u32(self.files.count - 1))
        _ = try self.process(head)

    ## A `-I` directory, searched before the system ones.
    pub func add_include_dir(dir: str) -> !:
        try self.dirs.push("")
        var i = self.dirs.count - 1
        while i > self.user_dirs:
            self.dirs.items[i] = self.dirs.items[i - 1]
            i -= 1
        self.dirs.items[self.user_dirs] = try strings.copy(dir)
        self.user_dirs += 1

    ## A `-D` definition: `NAME` or `NAME=VALUE`, as `#define` would.
    pub func define(text: str) -> !:
        var line = try buffer.Buffer.create(text.length + 16)
        try line.put("#define ")
        if let pair = split_at(text, '='):
            let (name, value) = pair
            try line.put(name)
            try line.put(" ")
            try line.put(value)
        else:
            try line.put(text)
            try line.put(" 1")
        try line.put("\n")
        let head = try tokenize(line.bytes(), 0)
        _ = try self.process(head)

    ## Preprocess `path` and everything it includes: the token list of the whole unit.
    pub func run(path: str) -> Tok*!:
        let file = try self.open(path, no_dir)
        self.main_file = file
        let bytes = try in_arena.read((c.str)self.files.items[file].path)
        let head = try tokenize(bytes, file)
        let out = try self.process(head)
        if self.conds.count > 0:
            let top = self.conds.items[self.conds.count - 1]
            error(failed, try at(self.path_of(top.file), top.line, "the conditional is not closed"))
        return out

    ## A file that is not on disk, a recipe's prototypes say: its index for tokens.
    pub func add_virtual_file(name: str) -> u32!:
        try self.files.push(FileEntry(path = name, dir = no_dir))
        return u32(self.files.count - 1)

    ## Where `#include <name>` would find `name`, for a header given by its include name.
    pub func locate(name: str) -> str?!:
        let found = try self.search(name, true, 0, false) else return none
        return found.path

    ## The canonical path of file index `file`.
    pub func path_of(file: u32) -> str:
        return self.files.items[file].path

    ## The macro `name` as it stands, when it is defined.
    pub func lookup(name: str) -> Macro*?:
        let m = self.macros.get(name) else return none
        return none if m.deleted else m

    ## `m` as its name's definition, replacing any earlier one.
    func insert(m: Macro*) -> !:
        try self.macros.put(m.name, m)

    func add_builtin(name: str, which: u8) -> !:
        let m = try new Macro(name = name, function_like = false, params = try new str[0] ---, variadic = false,
                              body = none, builtin = which, file = 0, line = 0, deleted = false)
        try self.insert(m)

    # mark: Files ==============================================================================

    ## The index of `path` in the file table, canonical, opened once.
    func open(path: str, dir: usize) -> u32!:
        let terminated = try strings.copy(path)
        let canonical = in_arena.canonical((c.str)terminated) catch failure:
            _ = failure
            recover terminated
        for i in 0..<self.files.count:
            if self.files.items[i].path == canonical:
                return u32(i)
        try self.files.push(FileEntry(path = canonical, dir = dir))
        return u32(self.files.count - 1)

    ## Where `#include` finds `name`: beside the including file for a quoted name, then the
    ## directories in order, from the one after the including file's for `#include_next`.
    func search(name: str, angled: bool, source: u32, next: bool) -> Found?!:
        if paths.is_absolute(name):
            if try exists(name):
                return Found(path = name, dir = no_dir)
            return none
        if not angled and not next:
            let beside = try in_arena.join(paths.directory(self.files.items[source].path), name)
            if try exists(beside):
                return Found(path = beside, dir = no_dir)
        var start: usize = 0
        if next and self.files.items[source].dir != no_dir:
            start = self.files.items[source].dir + 1
        for i in start..<self.dirs.count:
            let candidate = try in_arena.join(self.dirs.items[i], name)
            if try exists(candidate):
                return Found(path = candidate, dir = i)
        if let pair = split_at(name, '/'):
            let (framework, rest) = pair
            for dir in self.frameworks.span():
                let root = try in_arena.join(dir, framework)
                let headers = try in_arena.join(root, ".framework/Headers")
                let candidate = try in_arena.join(headers, rest)
                if try exists(candidate):
                    return Found(path = candidate, dir = no_dir)
        return none

    # mark: The loop ===========================================================================

    ## The tokens of `start` with directives obeyed and macros expanded.
    pub func process(start: Tok*) -> Tok*!:
        let head = try new Tok(kind = Kind.eof, text = "", file = 0, line = 0, space = false, bol = true, hide = none, next = none)
        var cur = head
        var tok = start
        while tok.kind != Kind.eof:
            if let expanded = try self.expand(tok):
                tok = expanded
                continue
            if not (tok.bol and tok.is_punct("#")):
                cur.next = tok
                cur = tok
                tok = tok.next else trap("a token list without its end")
                continue
            let sharp = tok
            tok = tok.next else trap("a token list without its end")
            if tok.bol:
                continue
            tok = try self.directive(sharp, tok)
        cur.next = tok
        return head.next else trap("unreachable")

    ## One directive, `tok` its name; the token after its line, or after the group it skips.
    func directive(sharp: Tok*, tok: Tok*) -> Tok*!:
        let name = tok.text
        if name == "include" or name == "include_next" or name == "import":
            return try self.include(sharp, tok)
        if name == "define":
            return try self.define_macro(tok.next else trap("unreachable"))
        if name == "undef":
            let target = tok.next else trap("unreachable")
            if target.kind == Kind.ident:
                if let m = self.lookup(target.text):
                    m.deleted = true
            return skip_line(target)
        if name == "if":
            let (value, rest) = try self.const_expr(tok.next else trap("unreachable"))
            try self.push_cond(sharp, value != 0)
            return rest if value != 0 else try skip_group(rest)
        if name == "ifdef" or name == "ifndef":
            let target = tok.next else trap("unreachable")
            let present = target.kind == Kind.ident and self.lookup(target.text) != none
            let included = present if name == "ifdef" else not present
            try self.push_cond(sharp, included)
            let rest = skip_line(target)
            return rest if included else try skip_group(rest)
        if name == "elif":
            if self.conds.count == 0 or self.conds.items[self.conds.count - 1].in_else:
                error(failed, try at(self.path_of(sharp.file), sharp.line, "a stray #elif"))
            if self.conds.items[self.conds.count - 1].included:
                return try skip_group(skip_line(tok))
            let (value, rest) = try self.const_expr(tok.next else trap("unreachable"))
            if value != 0:
                self.conds.items[self.conds.count - 1].included = true
                return rest
            return try skip_group(rest)
        if name == "else":
            if self.conds.count == 0 or self.conds.items[self.conds.count - 1].in_else:
                error(failed, try at(self.path_of(sharp.file), sharp.line, "a stray #else"))
            self.conds.items[self.conds.count - 1].in_else = true
            let rest = skip_line(tok)
            if self.conds.items[self.conds.count - 1].included:
                return try skip_group(rest)
            return rest
        if name == "endif":
            if self.conds.count == 0:
                error(failed, try at(self.path_of(sharp.file), sharp.line, "a stray #endif"))
            self.conds.count -= 1
            return skip_line(tok)
        if name == "error":
            let (line, rest) = try copy_line(tok.next else trap("unreachable"))
            _ = rest
            error(failed, try at(self.path_of(sharp.file), sharp.line, try spelled(line)))
        if name == "pragma":
            let what = tok.next else trap("unreachable")
            if what.is(Kind.ident, "once"):
                try self.once.push(self.path_of(sharp.file))
            return skip_line(what)
        if name == "warning" or name == "line" or name == "ident" or name == "sccs" or name == "assert" or name == "unassert":
            return skip_line(tok)
        if tok.kind == Kind.number:
            return skip_line(tok)
        try self.report.put(try at(self.path_of(sharp.file), sharp.line, try strings.format(scratch[0..], f"an unknown directive #{name} is ignored\n")))
        return skip_line(tok)

    func push_cond(sharp: Tok*, included: bool) -> !:
        try self.conds.push(Cond(included = included, in_else = false, line = sharp.line, file = sharp.file))

    # mark: Including ==========================================================================

    func include(sharp: Tok*, tok: Tok*) -> Tok*!:
        let next = tok.text == "include_next"
        let once_only = tok.text == "import"
        var first = tok.next else trap("unreachable")
        var name = ""
        var angled = false
        var rest = first
        if first.kind == Kind.text:
            name = (str)first.text.bytes[1..<first.text.length - 1]
            rest = skip_line(first)
        elif first.is_punct("<"):
            let (joined, after) = try angle_name(first)
            name = joined
            angled = true
            rest = after
        else:
            let (line, after) = try copy_line(first)
            let expanded = try self.process(line)
            if expanded.kind == Kind.text:
                name = (str)expanded.text.bytes[1..<expanded.text.length - 1]
            elif expanded.is_punct("<"):
                let (joined, tail) = try angle_name(expanded)
                _ = tail
                name = joined
                angled = true
            else:
                error(failed, try at(self.path_of(sharp.file), sharp.line, "#include expects a file name"))
            rest = after
        let found = try self.search(name, angled, sharp.file, next) else error(failed, try at(self.path_of(sharp.file), sharp.line, try strings.format(scratch[0..], f"{name} is not in any include directory")))
        let file = try self.open(found.path, found.dir)
        let path = self.path_of(file)
        for seen in self.once.span():
            if seen == path:
                return rest
        if once_only:
            try self.once.push(path)
        self.depth += 1
        if self.depth >= 100000:
            error(failed, try at(self.path_of(sharp.file), sharp.line, "the header includes without end"))
        let bytes = try in_arena.read((c.str)path)
        let head = try tokenize(bytes, file)
        return append(head, rest)

    # mark: Defining ===========================================================================

    ## `#define NAME body` or `#define NAME(params) body`, the `(` touching the name.
    func define_macro(tok: Tok*) -> Tok*!:
        if tok.kind != Kind.ident:
            error(failed, try at(self.path_of(tok.file), tok.line, "#define expects a name"))
        let name = tok.text
        var cursor = tok.next else trap("unreachable")
        var params = try list.List[str].create(8)
        var variadic = false
        var function_like = false
        if cursor.is_punct("(") and not cursor.space and not cursor.bol:
            function_like = true
            cursor = cursor.next else trap("unreachable")
            while not cursor.is_punct(")"):
                if cursor.bol:
                    error(failed, try at(self.path_of(tok.file), tok.line, "the parameter list is not closed"))
                if cursor.is_punct("..."):
                    variadic = true
                    try params.push("__VA_ARGS__")
                elif cursor.kind == Kind.ident:
                    let after = cursor.next else trap("unreachable")
                    if after.is_punct("..."):
                        variadic = true
                        try params.push(cursor.text)
                        cursor = after
                    else:
                        try params.push(cursor.text)
                elif not cursor.is_punct(","):
                    error(failed, try at(self.path_of(tok.file), tok.line, "a macro parameter must be a name"))
                cursor = cursor.next else trap("unreachable")
            cursor = cursor.next else trap("unreachable")
        let (body, rest) = try copy_line(cursor)
        let m = try new Macro(name = name, function_like = function_like, params = params.span(),
                              variadic = variadic, body = body, builtin = 0, file = tok.file,
                              line = tok.line, deleted = false)
        try self.insert(m)
        try self.defined.push(m)
        return rest

    # mark: Expanding ==========================================================================

    ## The expansion of a macro invocation at `tok`, spliced before what follows it; none
    ## when `tok` is not one.
    func expand(tok: Tok*) -> Tok*?!:
        if tok.kind != Kind.ident or hidden(tok.hide, tok.text):
            return none
        let m = self.lookup(tok.text) else return none
        let rest = tok.next else trap("unreachable")
        if m.builtin != 0:
            return try self.expand_builtin(m, tok, rest)
        if not m.function_like:
            let body = m.body else trap("a macro without a body")
            let out = try copy_with(body, add_hide(tok.hide, m.name), tok)
            return append(out, rest)
        if not rest.is_punct("("):
            return none
        let (arguments, close) = try self.read_arguments(rest, m, tok)
        var out = try self.substitute(m, arguments, tok)
        let hide = add_hide(intersect(tok.hide, close.hide), m.name)
        var cursor: Tok*? = out
        while let t = cursor:
            if t.kind == Kind.eof:
                break
            t.hide = union_of(t.hide, hide)
            cursor = t.next
        if out.kind != Kind.eof:
            out.bol = tok.bol
            out.space = tok.space
        return append(out, close.next else trap("unreachable"))

    func expand_builtin(m: Macro*, tok: Tok*, rest: Tok*) -> Tok*!:
        var made: Tok* = tok
        if m.builtin == builtin_file:
            var quoted = try buffer.Buffer.create(64)
            try quoted.put_byte('"')
            try quoted.put(self.path_of(tok.file))
            try quoted.put_byte('"')
            made = try new Tok(kind = Kind.text, text = try strings.copy(quoted.view()), file = tok.file, line = tok.line,
                               space = tok.space, bol = tok.bol, hide = none, next = rest)
        elif m.builtin == builtin_line:
            let number = try strings.format(scratch[0..], f"{tok.line}")
            made = try new Tok(kind = Kind.number, text = try strings.copy(number), file = tok.file, line = tok.line,
                               space = tok.space, bol = tok.bol, hide = none, next = rest)
        else:
            let number = try strings.format(scratch[0..], f"{self.counter}")
            self.counter += 1
            made = try new Tok(kind = Kind.number, text = try strings.copy(number), file = tok.file, line = tok.line,
                               space = tok.space, bol = tok.bol, hide = none, next = rest)
        return made

    ## The arguments of a function-like invocation, `open` its `(`: one token list per
    ## parameter, each ending in `eof`, and the closing `)`.
    func read_arguments(open: Tok*, m: Macro*, at_tok: Tok*) -> (Tok*[], Tok*)!:
        var arguments = try list.List[Tok*].create(8)
        var tok = open.next else trap("unreachable")
        var index: usize = 0
        while true:
            let last = index + 1 >= m.params.length
            let variadic_now = m.variadic and last
            let head = try new Tok(kind = Kind.eof, text = "", file = tok.file, line = tok.line, space = false, bol = false, hide = none, next = none)
            var cur = head
            var level: u32 = 0
            while true:
                if tok.kind == Kind.eof:
                    error(failed, try at(self.path_of(at_tok.file), at_tok.line, try strings.format(scratch[0..], f"the arguments of {m.name} are not closed")))
                if level == 0 and tok.is_punct(")"):
                    break
                if level == 0 and tok.is_punct(",") and not variadic_now:
                    break
                if tok.is_punct("(") or tok.is_punct("[") or tok.is_punct("{"):
                    level += 1
                elif tok.is_punct(")") or tok.is_punct("]") or tok.is_punct("}"):
                    level -= 1
                let copied = try copy(tok)
                copied.bol = false
                cur.next = copied
                cur = copied
                tok = tok.next else trap("unreachable")
            cur.next = try new Tok(kind = Kind.eof, text = "", file = tok.file, line = tok.line, space = false, bol = false, hide = none, next = none)
            let first = head.next else trap("unreachable")
            first.space = false
            try arguments.push(first)
            index += 1
            if tok.is_punct(")"):
                break
            tok = tok.next else trap("unreachable")
        # `f()` with one parameter is one empty argument; with none it is no argument
        if m.params.length == 0 and arguments.count == 1 and arguments.items[0].kind == Kind.eof:
            arguments.count = 0
        while m.variadic and arguments.count < m.params.length:
            try arguments.push(try new Tok(kind = Kind.eof, text = "", file = tok.file, line = tok.line, space = false, bol = false, hide = none, next = none))
        if arguments.count != m.params.length:
            error(failed, try at(self.path_of(at_tok.file), at_tok.line, try strings.format(scratch[0..], f"{m.name} takes {m.params.length} arguments, not {arguments.count}")))
        return (arguments.span(), tok)

    ## The body with parameters replaced: `#` stringises, `##` pastes, a parameter beside
    ## neither is its argument fully expanded, `__VA_OPT__(...)` only with variadic arguments.
    func substitute(m: Macro*, arguments: Tok*[], origin: Tok*) -> Tok*!:
        let head = try new Tok(kind = Kind.eof, text = "", file = origin.file, line = origin.line, space = false, bol = false, hide = none, next = none)
        var cur = head
        var tok = m.body else trap("a macro without a body")
        while tok.kind != Kind.eof:
            let next = tok.next else trap("unreachable")
            if tok.is_punct("#"):
                if let p = param_index(m, next):
                    let made = try stringize(arguments[p], tok)
                    cur.next = made
                    cur = made
                    tok = next.next else trap("unreachable")
                    continue
            if tok.is(Kind.ident, "__VA_OPT__") and next.is_punct("("):
                let (inner, after) = try group_after(next)
                if m.variadic and arguments[m.params.length - 1].kind != Kind.eof:
                    let sub = try self.substitute_tokens(m, arguments, inner, origin)
                    cur = link(cur, sub)
                tok = after
                continue
            if tok.is_punct("##"):
                if cur == head:
                    error(failed, try at(self.path_of(origin.file), origin.line, "## at the start of a macro body"))
                if let p = param_index(m, next):
                    let argument = arguments[p]
                    if argument.kind != Kind.eof:
                        let pasted = try self.paste(cur, argument)
                        *cur = *pasted
                        cur = link(cur, try copy_list(argument.next else trap("unreachable")))
                    tok = next.next else trap("unreachable")
                    continue
                let pasted = try self.paste(cur, next)
                *cur = *pasted
                tok = next.next else trap("unreachable")
                continue
            if let p = param_index(m, tok):
                let argument = arguments[p]
                if next.is_punct("##"):
                    let after = next.next else trap("unreachable")
                    if argument.kind == Kind.eof:
                        if let q = param_index(m, after):
                            cur = link(cur, try copy_list(arguments[q]))
                            tok = after.next else trap("unreachable")
                        else:
                            tok = after
                        continue
                    cur = link(cur, try copy_list(argument))
                    tok = next
                    continue
                let expanded = try self.process(try copy_list(argument))
                let first = expanded
                if first.kind != Kind.eof:
                    first.space = tok.space
                cur = link(cur, expanded)
                tok = next
                continue
            let copied = try copy(tok)
            cur.next = copied
            cur = copied
            tok = next
        cur.next = try new Tok(kind = Kind.eof, text = "", file = origin.file, line = origin.line, space = false, bol = false, hide = none, next = none)
        return head.next else trap("unreachable")

    ## `substitute` over a token list rather than the macro's own body.
    func substitute_tokens(m: Macro*, arguments: Tok*[], body: Tok*, origin: Tok*) -> Tok*!:
        let saved = m.body
        m.body = body
        let out = try self.substitute(m, arguments, origin)
        m.body = saved
        return out

    ## `left ## right`: their spellings joined and read as one token.
    func paste(left: Tok*, right: Tok*) -> Tok*!:
        let joined = try strings.format(scratch[0..], f"{left.text}{right.text}")
        let head = try tokenize(try strings.copy(joined), left.file)
        if head.kind == Kind.eof:
            error(failed, try at(self.path_of(left.file), left.line, "pasting produces nothing"))
        let after = head.next else trap("unreachable")
        if after.kind != Kind.eof:
            error(failed, try at(self.path_of(left.file), left.line, try strings.format(scratch[0..], f"pasting \"{left.text}\" and \"{right.text}\" does not give one token")))
        head.space = left.space
        head.bol = left.bol
        head.line = left.line
        head.next = left.next
        head.hide = none
        return head

    # mark: Conditions =========================================================================

    ## The value of the `#if` expression at `tok`, and the token after its line.
    func const_expr(tok: Tok*) -> (i64, Tok*)!:
        let (line, rest) = try copy_line(tok)
        let prepared = try self.replace_queries(line)
        let expanded = try self.process(prepared)
        let again = try self.replace_queries(expanded)
        var cursor: Tok*? = again
        while let t = cursor:
            if t.kind == Kind.ident:
                t.kind = Kind.number
                t.text = "0"
            cursor = t.next
        var e = Evaluator(tok = again, path = self.path_of(tok.file), line = tok.line)
        let value = try e.conditional()
        if e.tok.kind != Kind.eof:
            error(failed, try at(self.path_of(tok.file), tok.line, try strings.format(scratch[0..], f"the #if expression stops at {e.tok.text}")))
        return (value.value, rest)

    ## `defined X`, `defined(X)`, `__has_include(...)` and the other `__has_*` queries,
    ## replaced by their numbers before the line is expanded.
    func replace_queries(head: Tok*) -> Tok*!:
        let anchor = try new Tok(kind = Kind.eof, text = "", file = head.file, line = head.line, space = false, bol = false, hide = none, next = head)
        var prev = anchor
        var tok = head
        while tok.kind != Kind.eof:
            if tok.is(Kind.ident, "defined"):
                var name = tok.next else trap("unreachable")
                var after = name.next else trap("unreachable")
                if name.is_punct("("):
                    name = after
                    after = name.next else trap("unreachable")
                    if not after.is_punct(")"):
                        error(failed, try at(self.path_of(tok.file), tok.line, "defined( expects a name and )"))
                    after = after.next else trap("unreachable")
                let known = name.kind == Kind.ident and (self.lookup(name.text) != none or is_query(name.text))
                tok.kind = Kind.number
                tok.text = "1" if known else "0"
                tok.next = after
                prev = tok
                tok = after
                continue
            if tok.kind == Kind.ident and is_query(tok.text):
                let open = tok.next else trap("unreachable")
                if not open.is_punct("("):
                    prev = tok
                    tok = open
                    continue
                let (inner, after) = try group_after(open)
                let value = try self.query(tok, inner)
                tok.kind = Kind.number
                tok.text = "1" if value else "0"
                tok.next = after
                prev = tok
                tok = after
                continue
            prev = tok
            tok = tok.next else trap("unreachable")
        _ = prev
        return anchor.next else trap("unreachable")

    ## `__has_include` looks; `__has_feature(nullability)` is yes, because the declaration
    ## parser reads the annotations; every other query is no.
    func query(name: Tok*, inner: Tok*) -> bool!:
        if name.text == "__has_include" or name.text == "__has_include_next":
            var target = ""
            var angled = false
            if inner.kind == Kind.text:
                target = (str)inner.text.bytes[1..<inner.text.length - 1]
            elif inner.is_punct("<"):
                let (joined, after) = try angle_name(inner)
                _ = after
                target = joined
                angled = true
            else:
                let expanded = try self.process(inner)
                if expanded.kind == Kind.text:
                    target = (str)expanded.text.bytes[1..<expanded.text.length - 1]
                elif expanded.is_punct("<"):
                    let (joined, after) = try angle_name(expanded)
                    _ = after
                    target = joined
                    angled = true
                else:
                    error(failed, try at(self.path_of(name.file), name.line, "__has_include expects a file name"))
            let found = try self.search(target, angled, name.file, name.text == "__has_include_next")
            return found != none
        if name.text == "__has_feature" or name.text == "__has_extension":
            return inner.is(Kind.ident, "nullability") or inner.is(Kind.ident, "assume_nonnull") or inner.is(Kind.ident, "nullability_on_arrays")
        if name.text == "__is_identifier":
            return true
        return false

## Past a group that is not taken: to its `#elif`, `#else`, or `#endif`, nested ones
## skipped whole; the directive is left for the loop.
func skip_group(start: Tok*) -> Tok*!:
    var tok = start
    var depth: u32 = 0
    while tok.kind != Kind.eof:
        if tok.bol and tok.is_punct("#"):
            let name = tok.next else trap("unreachable")
            if name.kind == Kind.ident:
                if name.text == "if" or name.text == "ifdef" or name.text == "ifndef":
                    depth += 1
                elif name.text == "endif":
                    if depth == 0:
                        return tok
                    depth -= 1
                elif (name.text == "elif" or name.text == "else") and depth == 0:
                    return tok
            tok = skip_line(name)
            continue
        tok = tok.next else trap("unreachable")
    return tok

func param_index(m: Macro*, tok: Tok*) -> usize?:
    if tok.kind != Kind.ident:
        return none
    for i in 0..<m.params.length:
        if m.params[i] == tok.text:
            return i
    return none

## The queries clang answers as if they were macros.
func is_query(name: str) -> bool:
    return strings.starts_with(name, "__has_") or name == "__is_identifier" or name == "__building_module" or strings.starts_with(name, "__is_target_")

# mark: Constant expressions ===================================================================

## A C integer in a constant expression: 64 bits, unsigned when either operand is.
pub struct Value:
    ## The 64-bit value.
    pub var value: i64
    ## Whether the value is unsigned.
    pub var unsigned: bool

## `#if`'s expression grammar over a token line, C's precedence, `?:` included.
struct Evaluator:
    var tok: Tok*
    var path: str
    var line: u32

    func advance():
        self.tok = self.tok.next else trap("unreachable")

    func expect(text: str) -> !:
        if not self.tok.is_punct(text):
            error(failed, try at(self.path, self.line, try strings.format(scratch[0..], f"the #if expression expects {text}")))
        self.advance()

    func conditional() -> Value!:
        let condition = try self.binary(0)
        if not self.tok.is_punct("?"):
            return condition
        self.advance()
        let yes = try self.conditional()
        try self.expect(":")
        let no = try self.conditional()
        return yes if condition.value != 0 else no

    func binary(minimum: u32) -> Value!:
        var left = try self.unary()
        while self.tok.kind == Kind.punct:
            let p = precedence(self.tok.text)
            if p == 0 or p <= minimum:
                break
            let op = self.tok.text
            self.advance()
            let right = try self.binary(p)
            left = try self.apply(op, left, right)
        return left

    func apply(op: str, a: Value, b: Value) -> Value!:
        let unsigned = a.unsigned or b.unsigned
        let x = a.value
        let y = b.value
        if op == "||": return Value(value = 1 if x != 0 or y != 0 else 0, unsigned = false)
        if op == "&&": return Value(value = 1 if x != 0 and y != 0 else 0, unsigned = false)
        if op == "|": return Value(value = x | y, unsigned = unsigned)
        if op == "^": return Value(value = x ^ y, unsigned = unsigned)
        if op == "&": return Value(value = x & y, unsigned = unsigned)
        if op == "==": return Value(value = 1 if x == y else 0, unsigned = false)
        if op == "!=": return Value(value = 1 if x != y else 0, unsigned = false)
        if op == "<": return Value(value = 1 if less(x, y, unsigned) else 0, unsigned = false)
        if op == ">": return Value(value = 1 if less(y, x, unsigned) else 0, unsigned = false)
        if op == "<=": return Value(value = 1 if not less(y, x, unsigned) else 0, unsigned = false)
        if op == ">=": return Value(value = 1 if not less(x, y, unsigned) else 0, unsigned = false)
        if op == "<<": return Value(value = (i64)((u64)x << (u64)(y & 63)), unsigned = unsigned)
        if op == ">>":
            if unsigned:
                return Value(value = (i64)((u64)x >> (u64)(y & 63)), unsigned = true)
            return Value(value = x >> (y & 63), unsigned = false)
        if op == "+": return Value(value = x +% y, unsigned = unsigned)
        if op == "-": return Value(value = x -% y, unsigned = unsigned)
        if op == "*": return Value(value = x *% y, unsigned = unsigned)
        if y == 0:
            error(failed, try at(self.path, self.line, "the #if expression divides by zero"))
        if op == "/":
            if unsigned:
                return Value(value = (i64)((u64)x // (u64)y), unsigned = true)
            return Value(value = x // y, unsigned = false)
        if unsigned:
            return Value(value = (i64)((u64)x % (u64)y), unsigned = true)
        return Value(value = x % y, unsigned = false)

    func unary() -> Value!:
        if self.tok.is_punct("+"):
            self.advance()
            return try self.unary()
        if self.tok.is_punct("-"):
            self.advance()
            let v = try self.unary()
            return Value(value = 0 -% v.value, unsigned = v.unsigned)
        if self.tok.is_punct("!"):
            self.advance()
            let v = try self.unary()
            return Value(value = 1 if v.value == 0 else 0, unsigned = false)
        if self.tok.is_punct("~"):
            self.advance()
            let v = try self.unary()
            return Value(value = ~v.value, unsigned = v.unsigned)
        return try self.primary()

    func primary() -> Value!:
        if self.tok.is_punct("("):
            self.advance()
            let v = try self.conditional()
            try self.expect(")")
            return v
        if self.tok.kind == Kind.number:
            let v = try integer(self.tok.text) else error(failed, try at(self.path, self.line, try strings.format(scratch[0..], f"{self.tok.text} is not an integer the #if expression can use")))
            self.advance()
            return v
        if self.tok.kind == Kind.character:
            let v = character_value(self.tok.text)
            self.advance()
            return Value(value = v, unsigned = false)
        error(failed, try at(self.path, self.line, try strings.format(scratch[0..], f"the #if expression does not understand {self.tok.text}")))

func less(x: i64, y: i64, unsigned: bool) -> bool:
    if unsigned:
        return (u64)x < (u64)y
    return x < y

## A C integer literal: decimal, `0x`, `0b`, or octal, with `u` and `l` suffixes.
pub func integer(text: str) -> Value?!:
    var i: usize = 0
    var radix: u64 = 10
    let b = text.bytes
    if b.length >= 2 and b[0] == '0' and (b[1] == 'x' or b[1] == 'X'):
        radix = 16
        i = 2
    elif b.length >= 2 and b[0] == '0' and (b[1] == 'b' or b[1] == 'B'):
        radix = 2
        i = 2
    elif b.length >= 2 and b[0] == '0':
        radix = 8
        i = 1
    var value: u64 = 0
    var digits: usize = 1 if radix == 8 else 0
    while i < b.length:
        let d = digit_value(b[i])
        if d >= radix:
            break
        value = value *% radix +% d
        digits += 1
        i += 1
    if digits == 0:
        return none
    var unsigned = false
    while i < b.length:
        let s = b[i]
        if s == 'u' or s == 'U':
            unsigned = true
        elif s != 'l' and s != 'L':
            return none
        i += 1
    if value > 9223372036854775807 and radix != 10:
        unsigned = true
    return Value(value = (i64)value, unsigned = unsigned)

func digit_value(b: u8) -> u64:
    if b >= '0' and b <= '9': return (u64)(b - '0')
    if b >= 'a' and b <= 'z': return (u64)(b - 'a') + 10
    if b >= 'A' and b <= 'Z': return (u64)(b - 'A') + 10
    return 99

## The value of a character constant's first character, escapes decoded.
pub func character_value(text: str) -> i64:
    var i: usize = 0
    let b = text.bytes
    while i < b.length and b[i] != '\'':
        i += 1
    i += 1
    if i >= b.length:
        return 0
    if b[i] != '\\':
        return (i64)b[i]
    i += 1
    if i >= b.length:
        return 0
    let e = b[i]
    if e == 'n': return 10
    if e == 't': return 9
    if e == 'r': return 13
    if e == 'a': return 7
    if e == 'b': return 8
    if e == 'f': return 12
    if e == 'v': return 11
    if e == 'e': return 27
    if e == 'x':
        var v: i64 = 0
        i += 1
        while i < b.length and digit_value(b[i]) < 16:
            v = v * 16 + (i64)digit_value(b[i])
            i += 1
        return v
    if e >= '0' and e <= '7':
        var v: i64 = 0
        while i < b.length and b[i] >= '0' and b[i] <= '7':
            v = v * 8 + (i64)(b[i] - '0')
            i += 1
        return v
    return (i64)e

# mark: Text ===================================================================================

## The lines of a text, without their newlines; a last line without one counts.
pub func lines(text: str) -> str[]!:
    var out = try list.List[str].create(64)
    var start: usize = 0
    var i: usize = 0
    while i < text.length:
        if text.bytes[i] == '\n':
            try out.push((str)text.bytes[start..<i])
            start = i + 1
        i += 1
    if start < text.length:
        try out.push((str)text.bytes[start..])
    return out.span()

## The text before and after the first `separator`, or none when there is none.
pub func split_at(text: str, separator: u8) -> (str, str)?:
    var i: usize = 0
    while i < text.length:
        if text.bytes[i] == separator:
            return ((str)text.bytes[..<i], (str)text.bytes[i + 1..])
        i += 1
    return none

# mark: Token lists ============================================================================

## A message placed at a file and line.
pub func at(path: str, line: u32, what: str) -> str!:
    let text = try strings.format(place[0..], f"{path}:{line}: {what}")
    return try strings.copy(text)

func exists(path: str) -> bool!:
    let terminated = try strings.copy(path)
    return files.exists((c.str)terminated)

func hidden(hide: Hide*?, name: str) -> bool:
    var cursor = hide
    while let h = cursor:
        if h.name == name:
            return true
        cursor = h.next
    return false

func add_hide(hide: Hide*?, name: str) -> Hide*?:
    let h: Hide*? = new Hide(name = name, next = hide) catch failure:
        _ = failure
        recover none
    return h

func union_of(a: Hide*?, b: Hide*?) -> Hide*?:
    var out = a
    var cursor = b
    while let h = cursor:
        if not hidden(out, h.name):
            out = add_hide(out, h.name)
        cursor = h.next
    return out

func intersect(a: Hide*?, b: Hide*?) -> Hide*?:
    var out: Hide*? = none
    var cursor = a
    while let h = cursor:
        if hidden(b, h.name):
            out = add_hide(out, h.name)
        cursor = h.next
    return out

## A shallow copy of `t` with no successor.
pub func copy(t: Tok*) -> Tok*!:
    return try new Tok(kind = t.kind, text = t.text, file = t.file, line = t.line, space = t.space,
                       bol = t.bol, hide = t.hide, next = none)

## A copy of the list up to and including its `eof`.
pub func copy_list(head: Tok*) -> Tok*!:
    let anchor = try new Tok(kind = Kind.eof, text = "", file = head.file, line = head.line, space = false, bol = false, hide = none, next = none)
    var cur = anchor
    var tok = head
    while true:
        let copied = try copy(tok)
        cur.next = copied
        cur = copied
        if tok.kind == Kind.eof:
            break
        tok = tok.next else trap("unreachable")
    return anchor.next else trap("unreachable")

## A copy of a macro body with `hide` on every token and `origin`'s place on the first.
func copy_with(body: Tok*, hide: Hide*?, origin: Tok*) -> Tok*!:
    let out = try copy_list(body)
    var cursor: Tok*? = out
    while let t = cursor:
        t.hide = union_of(t.hide, hide)
        cursor = t.next
    if out.kind != Kind.eof:
        out.bol = origin.bol
        out.space = origin.space
    return out

## `b` in place of `a`'s `eof`; `a` when it has tokens, else `b`.
pub func append(a: Tok*, b: Tok*) -> Tok*:
    if a.kind == Kind.eof:
        return b
    var cur = a
    while let n = cur.next:
        if n.kind == Kind.eof:
            break
        cur = n
    cur.next = b
    return a

## `list`, without its `eof`, after `cur`; the new last token.
func link(cur: Tok*, items: Tok*) -> Tok*:
    if items.kind == Kind.eof:
        return cur
    cur.next = items
    var last = items
    while let n = last.next:
        if n.kind == Kind.eof:
            break
        last = n
    return last

## The first token at or after `start` that begins a line.
pub func skip_line(start: Tok*) -> Tok*:
    var tok = start
    while not tok.bol:
        tok = tok.next else return tok
    return tok

## The rest of the line as a fresh list ending in `eof`, and the token after the line.
pub func copy_line(start: Tok*) -> (Tok*, Tok*)!:
    let anchor = try new Tok(kind = Kind.eof, text = "", file = start.file, line = start.line, space = false, bol = false, hide = none, next = none)
    var cur = anchor
    var tok = start
    while not tok.bol:
        let copied = try copy(tok)
        cur.next = copied
        cur = copied
        tok = tok.next else break
    cur.next = try new Tok(kind = Kind.eof, text = "", file = start.file, line = start.line, space = false, bol = false, hide = none, next = none)
    return (anchor.next else trap("unreachable"), tok)

## The tokens inside the parentheses opened at `open`, as a list ending in `eof`, and the
## token after the closing one.
pub func group_after(open: Tok*) -> (Tok*, Tok*)!:
    let anchor = try new Tok(kind = Kind.eof, text = "", file = open.file, line = open.line, space = false, bol = false, hide = none, next = none)
    var cur = anchor
    var tok = open.next else trap("unreachable")
    var level: u32 = 0
    while tok.kind != Kind.eof:
        if tok.is_punct(")") and level == 0:
            break
        if tok.is_punct("("):
            level += 1
        elif tok.is_punct(")"):
            level -= 1
        let copied = try copy(tok)
        cur.next = copied
        cur = copied
        tok = tok.next else trap("unreachable")
    if tok.kind == Kind.eof:
        error(failed, try at("", open.line, "a parenthesis is not closed"))
    cur.next = try new Tok(kind = Kind.eof, text = "", file = open.file, line = open.line, space = false, bol = false, hide = none, next = none)
    return (anchor.next else trap("unreachable"), tok.next else trap("unreachable"))

## `<a/b.h>` joined from its tokens, and the token after `>`.
func angle_name(open: Tok*) -> (str, Tok*)!:
    var out = try buffer.Buffer.create(64)
    var tok = open.next else trap("unreachable")
    while not tok.is_punct(">"):
        if tok.bol or tok.kind == Kind.eof:
            error(failed, "an #include name is not closed")
        try out.put(tok.text)
        tok = tok.next else trap("unreachable")
    return (try strings.copy(out.view()), tok.next else trap("unreachable"))

## The spelling of a token list, one space where the source had any.
pub func spelled(head: Tok*) -> str!:
    var out = try buffer.Buffer.create(64)
    var cursor: Tok*? = head
    var first = true
    while let t = cursor:
        if t.kind == Kind.eof:
            break
        if t.space and not first:
            try out.put(" ")
        try out.put(t.text)
        first = false
        cursor = t.next
    return try strings.copy(out.view())

## `#param`: the argument's spelling as a string literal.
func stringize(argument: Tok*, origin: Tok*) -> Tok*!:
    var out = try buffer.Buffer.create(64)
    try out.put("\"")
    var cursor: Tok*? = argument
    var first = true
    while let t = cursor:
        if t.kind == Kind.eof:
            break
        if t.space and not first:
            try out.put(" ")
        if t.kind == Kind.text or t.kind == Kind.character:
            for b in t.text.bytes:
                if b == '"' or b == '\\':
                    try out.put_byte('\\')
                try out.put_byte(b)
        else:
            try out.put(t.text)
        first = false
        cursor = t.next
    try out.put("\"")
    return try new Tok(kind = Kind.text, text = try strings.copy(out.view()), file = origin.file, line = origin.line,
                       space = origin.space, bol = false, hide = none, next = none)

# mark: Tokens =================================================================================

let three: str[3] = ["<<=", ">>=", "..."]
let two: str[19] = ["->", "++", "--", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^="]

func is_ident_start(b: u8) -> bool:
    return (b >= 'a' and b <= 'z') or (b >= 'A' and b <= 'Z') or b == '_' or b == '$' or b >= 128

func is_ident_char(b: u8) -> bool:
    return is_ident_start(b) or (b >= '0' and b <= '9')

func is_digit(b: u8) -> bool:
    return b >= '0' and b <= '9'

func is_punct_char(b: u8) -> bool:
    let set = "[](){}.&*+-~!/%<>^|?:;=,#@"
    for p in set.bytes:
        if p == b:
            return true
    return false

## The text with backslash-newlines removed, and where each was, for line numbers.
func splice(bytes: const u8[]) -> (u8[], usize[])!:
    let out = try new u8[bytes.length] ---
    var splices = try list.List[usize].create(16)
    var n: usize = 0
    var i: usize = 0
    while i < bytes.length:
        if bytes[i] == '\\' and i + 1 < bytes.length and bytes[i + 1] == '\n':
            try splices.push(n)
            i += 2
            continue
        if bytes[i] == '\\' and i + 2 < bytes.length and bytes[i + 1] == '\r' and bytes[i + 2] == '\n':
            try splices.push(n)
            i += 3
            continue
        out[n] = bytes[i]
        n += 1
        i += 1
    return (out[..<n], splices.span())

## The preprocessing tokens of `bytes`, a list ending in `eof`. Comments are spacing.
pub func tokenize(bytes: const u8[], file: u32) -> Tok*!:
    let (text, splices) = try splice(bytes)
    let anchor = try new Tok(kind = Kind.eof, text = "", file = file, line = 0, space = false, bol = true, hide = none, next = none)
    var cur = anchor
    var i: usize = 0
    var line: u32 = 1
    var bol = true
    var space = false
    var spliced: usize = 0
    while i < text.length:
        let b = text[i]
        if b == '\n':
            line += 1
            bol = true
            space = true
            i += 1
            continue
        if b == ' ' or b == '\t' or b == '\r' or b == 12 or b == 11:
            space = true
            i += 1
            continue
        if b == '/' and i + 1 < text.length and text[i + 1] == '/':
            while i < text.length and text[i] != '\n':
                i += 1
            space = true
            continue
        if b == '/' and i + 1 < text.length and text[i + 1] == '*':
            i += 2
            while i < text.length and not (text[i] == '*' and i + 1 < text.length and text[i + 1] == '/'):
                if text[i] == '\n':
                    line += 1
                i += 1
            i += 2
            space = true
            continue
        while spliced < splices.length and splices[spliced] <= i:
            line += 1
            spliced += 1
        let start = i
        var kind = Kind.other
        if is_ident_start(b):
            while i < text.length and is_ident_char(text[i]):
                i += 1
            kind = Kind.ident
            let word = (str)text[start..<i]
            if i < text.length and (text[i] == '"' or text[i] == '\'') and (word == "L" or word == "u" or word == "U" or word == "u8"):
                kind = Kind.text if text[i] == '"' else Kind.character
                i = quoted_end(text, i)
        elif is_digit(b) or (b == '.' and i + 1 < text.length and is_digit(text[i + 1])):
            i += 1
            while i < text.length:
                let d = text[i]
                if (d == '+' or d == '-') and (text[i - 1] == 'e' or text[i - 1] == 'E' or text[i - 1] == 'p' or text[i - 1] == 'P'):
                    i += 1
                elif is_ident_char(d) or d == '.':
                    i += 1
                else:
                    break
            kind = Kind.number
        elif b == '"':
            kind = Kind.text
            i = quoted_end(text, i)
        elif b == '\'':
            kind = Kind.character
            i = quoted_end(text, i)
        elif is_punct_char(b):
            kind = Kind.punct
            i += 1
            if start + 3 <= text.length:
                for p in three:
                    if (str)text[start..<start + 3] == p:
                        i = start + 3
            if i == start + 1 and start + 2 <= text.length:
                for p in two:
                    if (str)text[start..<start + 2] == p:
                        i = start + 2
            if i == start + 1 and b == '#' and start + 1 < text.length and text[start + 1] == '#':
                i = start + 2
        else:
            i += 1
        let made = try new Tok(kind = kind, text = (str)text[start..<i], file = file, line = line, space = space, bol = bol, hide = none, next = none)
        cur.next = made
        cur = made
        bol = false
        space = false
    cur.next = try new Tok(kind = Kind.eof, text = "", file = file, line = line, space = true, bol = true, hide = none, next = none)
    return anchor.next else trap("unreachable")

## Past the closing quote of the literal opening at `open`, or the end of its line.
func quoted_end(text: const u8[], open: usize) -> usize:
    let quote = text[open]
    var i = open + 1
    while i < text.length and text[i] != quote and text[i] != '\n':
        if text[i] == '\\' and i + 1 < text.length:
            i += 1
        i += 1
    if i < text.length and text[i] == quote:
        i += 1
    return i

## How tightly a C binary operator binds, from `||` at 1 to `*` at 10; 0 for text that is
## no binary operator. The `#if` evaluator and the binder's constant expressions share it.
pub func precedence(text: str) -> u32:
    if text == "||": return 1
    if text == "&&": return 2
    if text == "|": return 3
    if text == "^": return 4
    if text == "&": return 5
    if text == "==" or text == "!=": return 6
    if text == "<" or text == "<=" or text == ">" or text == ">=": return 7
    if text == "<<" or text == ">>": return 8
    if text == "+" or text == "-": return 9
    if text == "*" or text == "/" or text == "%": return 10
    return 0

test "tokens keep their spacing and line":
    var backing: u8[65536]
    var arena = memory.FixedBuffer.over(backing)
    with arena:
        let head = try tokenize("a  += 0x1fUL\n  \"s\\\"q\" 'c' ... <<= x##y\n".bytes, 0)
        assert(head.is(Kind.ident, "a") and head.bol)
        let plus = head.next else trap("")
        assert(plus.is_punct("+=") and plus.space)
        let number = plus.next else trap("")
        assert(number.is(Kind.number, "0x1fUL"))
        let s = number.next else trap("")
        assert(s.kind == Kind.text and s.bol and s.line == 2)
        let ch = s.next else trap("")
        assert(ch.is(Kind.character, "'c'"))
        let dots = ch.next else trap("")
        assert(dots.is_punct("..."))
        let shift = dots.next else trap("")
        assert(shift.is_punct("<<="))
        let x = shift.next else trap("")
        let paste_op = x.next else trap("")
        assert(paste_op.is_punct("##") and not paste_op.space)

test "integers and characters evaluate":
    let v = try integer("0x10u") else trap("")
    assert(v.value == 16 and v.unsigned)
    let o = try integer("017") else trap("")
    assert(o.value == 15)
    assert(try integer("1.5") == none)
    assert(character_value("'\\n'") == 10)
    assert(character_value("L'A'") == 65)