dymokomi/luce-base
Source of release 0.46.0
0.46.0 / src / front / token.lucb
#==============================================================================================
#
# token - Tokens, reserved words, and operator spellings
#
# DESCRIPTION:
# The token kinds the lexer produces, the reserved words of base.md §3.6, and the
# operator spellings of §21 ordered longest first so `+%=` wins over `+%` and `+`.
# Tokens are spans into the source; nothing here decodes a literal.
#
#==============================================================================================
pub enum Kind as u8:
name = 1
keyword = 2
integer = 3
float = 4
character = 5
text = 6
bytes = 7
raw = 8
format_start = 9
format_text = 10
format_end = 11
operator = 12
newline = 13
indent = 14
dedent = 15
doc_comment = 16
raw_line = 17
eof = 18
## A formatted string's field's specification, the text after its `:` (§5.5).
format_spec = 19
## One lexical token: its kind and its span, line, and column in the source.
pub struct Token:
## What kind of token this is.
pub var kind: Kind
## The byte offset where the token begins.
pub var start: usize
## The byte offset just past the token.
pub var end: usize
## The one-based line the token begins on.
pub var line: u32
## The one-based column the token begins at.
pub var column: u32
## The token kind's name, for diagnostics.
pub func label() -> str:
match self.kind:
.name: return "name"
.keyword: return "keyword"
.integer: return "integer"
.float: return "float"
.character: return "character"
.text: return "text"
.bytes: return "bytes"
.raw: return "raw"
.format_start: return "format_start"
.format_text: return "format_text"
.format_end: return "format_end"
.operator: return "operator"
.newline: return "newline"
.indent: return "indent"
.dedent: return "dedent"
.doc_comment: return "doc_comment"
.raw_line: return "raw_line"
.format_spec: return "format_spec"
_: return "eof"
# mark: Reserved words ========================================================================
## The reserved words of the language (§3.6); no name may be one.
pub let keywords: str[43] = [
"and", "as", "asm", "break", "catch", "const", "continue", "defer",
"elif", "else", "enum", "errdefer", "extern", "false", "for", "free",
"from", "func", "if", "import", "in", "interface", "let",
"match", "new", "none", "not", "or", "pub", "recover", "return",
"self", "static", "struct", "test", "true", "try", "type",
"union", "var", "volatile", "while", "with"]
## The core names of §3.5: the language's own functions and type names. No declaration that
## binds a name may take one; a member, reached through its value or type, may, and so may
## the standard modules, which are what these names mean.
pub let core_names: str[26] = [
"assert", "error", "trap", "print", "bool", "i8", "i16",
"i32", "i64", "isize", "u8", "u16", "u32", "u64", "usize",
"f16", "f32", "f64", "char", "str", "unit", "never",
"void", "fmt", "Error", "ErrorCode"]
## Whether `word` is one of the core names of §3.5.
pub func is_core_name(word: str) -> bool:
if word.length == 0:
return false
for name in core_names:
if name.length == word.length and name.bytes[0] == word.bytes[0] and name == word:
return true
return false
## Whether `word` is a reserved keyword.
pub func is_keyword(word: str) -> bool:
# every keyword is two to nine lower-case letters: most names are refused here, and
# the rest are compared only with the keywords of their length and first letter
if word.length < 2 or word.length > 9 or word.bytes[0] < 'a' or word.bytes[0] > 'z':
return false
for k in keywords:
if k.length == word.length and k.bytes[0] == word.bytes[0] and k == word:
return true
return false
# mark: Operators =============================================================================
## Operators, longest first. The lexer takes the first entry that matches.
pub let operators: str[53] = [
"+%=", "-%=", "*%=", "+|=", "-|=", "*|=", "<<=", ">>=", "//=", "---", "...",
"..<", "..=", "+%", "-%", "*%", "+|", "-|", "*|", "+?", "-?", "*?", "->", "=>",
"==", "!=", "<=", ">=", "<<", ">>", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=",
"//", "..", "+", "-", "*", "/", "%", "&", "|", "^", "~", "<", ">", "=", "("]
## The one-character operators that are not a prefix of any longer one.
pub let singles: str[13] = [")", "[", "]", "{", "}", ",", ":", ".", "?", "!", "@", ";", "\\"]
test "keywords are recognised":
assert(is_keyword("func"))
assert(not is_keyword("local"))
assert(not is_keyword("mutating"))
assert(not is_keyword("function"))