Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.46.0

0.46.0 / src / front / token.lucb

#==============================================================================================
#
#   token - Tokens, reserved words, and operator spellings
#
#   DESCRIPTION:
#       The token kinds the lexer produces, the reserved words of base.md §3.6, and the
#       operator spellings of §21 ordered longest first so `+%=` wins over `+%` and `+`.
#       Tokens are spans into the source; nothing here decodes a literal.
#
#==============================================================================================
pub enum Kind as u8:
    name = 1
    keyword = 2
    integer = 3
    float = 4
    character = 5
    text = 6
    bytes = 7
    raw = 8
    format_start = 9
    format_text = 10
    format_end = 11
    operator = 12
    newline = 13
    indent = 14
    dedent = 15
    doc_comment = 16
    raw_line = 17
    eof = 18
    ## A formatted string's field's specification, the text after its `:` (§5.5).
    format_spec = 19

## One lexical token: its kind and its span, line, and column in the source.
pub struct Token:
    ## What kind of token this is.
    pub var kind: Kind
    ## The byte offset where the token begins.
    pub var start: usize
    ## The byte offset just past the token.
    pub var end: usize
    ## The one-based line the token begins on.
    pub var line: u32
    ## The one-based column the token begins at.
    pub var column: u32

    ## The token kind's name, for diagnostics.
    pub func label() -> str:
        match self.kind:
            .name: return "name"
            .keyword: return "keyword"
            .integer: return "integer"
            .float: return "float"
            .character: return "character"
            .text: return "text"
            .bytes: return "bytes"
            .raw: return "raw"
            .format_start: return "format_start"
            .format_text: return "format_text"
            .format_end: return "format_end"
            .operator: return "operator"
            .newline: return "newline"
            .indent: return "indent"
            .dedent: return "dedent"
            .doc_comment: return "doc_comment"
            .raw_line: return "raw_line"
            .format_spec: return "format_spec"
            _: return "eof"

# mark: Reserved words ========================================================================

## The reserved words of the language (§3.6); no name may be one.
pub let keywords: str[43] = [
    "and", "as", "asm", "break", "catch", "const", "continue", "defer",
    "elif", "else", "enum", "errdefer", "extern", "false", "for", "free",
    "from", "func", "if", "import", "in", "interface", "let",
    "match", "new", "none", "not", "or", "pub", "recover", "return",
    "self", "static", "struct", "test", "true", "try", "type",
    "union", "var", "volatile", "while", "with"]

## The core names of §3.5: the language's own functions and type names. No declaration that
## binds a name may take one; a member, reached through its value or type, may, and so may
## the standard modules, which are what these names mean.
pub let core_names: str[26] = [
    "assert", "error", "trap", "print", "bool", "i8", "i16",
    "i32", "i64", "isize", "u8", "u16", "u32", "u64", "usize",
    "f16", "f32", "f64", "char", "str", "unit", "never",
    "void", "fmt", "Error", "ErrorCode"]

## Whether `word` is one of the core names of §3.5.
pub func is_core_name(word: str) -> bool:
    if word.length == 0:
        return false
    for name in core_names:
        if name.length == word.length and name.bytes[0] == word.bytes[0] and name == word:
            return true
    return false

## Whether `word` is a reserved keyword.
pub func is_keyword(word: str) -> bool:
    # every keyword is two to nine lower-case letters: most names are refused here, and
    # the rest are compared only with the keywords of their length and first letter
    if word.length < 2 or word.length > 9 or word.bytes[0] < 'a' or word.bytes[0] > 'z':
        return false
    for k in keywords:
        if k.length == word.length and k.bytes[0] == word.bytes[0] and k == word:
            return true
    return false

# mark: Operators =============================================================================

## Operators, longest first. The lexer takes the first entry that matches.
pub let operators: str[53] = [
    "+%=", "-%=", "*%=", "+|=", "-|=", "*|=", "<<=", ">>=", "//=", "---", "...",
    "..<", "..=", "+%", "-%", "*%", "+|", "-|", "*|", "+?", "-?", "*?", "->", "=>",
    "==", "!=", "<=", ">=", "<<", ">>", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=",
    "//", "..", "+", "-", "*", "/", "%", "&", "|", "^", "~", "<", ">", "=", "("]

## The one-character operators that are not a prefix of any longer one.
pub let singles: str[13] = [")", "[", "]", "{", "}", ",", ":", ".", "?", "!", "@", ";", "\\"]

test "keywords are recognised":
    assert(is_keyword("func"))
    assert(not is_keyword("local"))
    assert(not is_keyword("mutating"))
    assert(not is_keyword("function"))