Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-prism

Source of release 0.6.1

0.6.1 / src / serialization / lexer.lucb

## Serialization lexer.
from storage import Buffer, List, close_names
from types import invalid, limit_exceeded, max_bytes

pub struct Parser:
    pub var source: const u8[]
    pub var position: usize
    pub var token_start: usize
    pub var token: str = ""
    pub var token_kind: u8
    pub var decoded: List[Buffer]
    pub var string_budget: usize = max_bytes

    pub func peek() -> str:
        return self.token

    pub func kind() -> u8:
        return self.token_kind

    pub func remaining() -> usize:
        return self.source.length - self.token_start

    pub func take() -> str!:
        if self.kind() == 0:
            try Parser.fail("unexpected end of input")
        let text = self.peek()
        try self.advance()
        return text

    pub func at(text: str) -> bool:
        if self.peek() != text:
            return false
        if text.length == 1 and punctuation(text.bytes[0]):
            return self.kind() == text.bytes[0]
        return self.kind() == 1

    pub func accept(text: str) -> bool!:
        if not self.at(text):
            return false
        try self.advance()
        return true

    pub func expect(text: str) -> !:
        if not (try self.accept(text)):
            try Parser.fail("unexpected token")

    pub static func fail(message: str) -> !:
        error(invalid, message)

    pub func name() -> str!:
        if self.kind() != 1 and self.kind() != 2:
            try Parser.fail("expected a name")
        return try self.take()

    pub func quoted() -> str!:
        if self.kind() != 2:
            try Parser.fail("expected a quoted string")
        return try self.take()

    pub func path() -> str!:
        if self.kind() != 4:
            try Parser.fail("expected a <path>")
        return try self.take()

    pub func advance() -> !:
        let bytes = self.source
        var pos = self.position
        while pos < bytes.length:
            let byte = bytes[pos]
            if byte == ' ' or byte == '\t' or byte == '\r' or byte == '\n':
                pos += 1
            elif byte == '#':
                while pos < bytes.length and bytes[pos] != '\n':
                    pos += 1
            else:
                break
        self.token_start = pos
        self.token = ""
        self.token_kind = 0
        if pos == bytes.length:
            self.position = pos
            return
        let byte = bytes[pos]
        if byte == '"' or byte == '\'' or byte == '<':
            let end: u8 = '>' if byte == '<' else byte
            self.token_kind = 4 if byte == '<' else (2 if byte == '"' else 3)
            pos += 1
            let start = pos
            var escaped = false
            var closed = false
            while pos < bytes.length:
                let current = bytes[pos]
                if current == end:
                    closed = true
                    break
                pos += 1
                if current == '\\' and byte != '<':
                    if pos == bytes.length:
                        break
                    escaped = true
                    pos += 1
            if not closed:
                error(invalid, "unterminated Prism literal")
            let raw = bytes[start..<pos]
            if escaped:
                var decoded: Buffer
                errdefer decoded.close()
                var i: usize = 0
                while i < raw.length:
                    var current = raw[i]
                    i += 1
                    if current == '\\':
                        current = raw[i]
                        i += 1
                        current = '\n' if current == 'n' else ('\t' if current == 't' else current)
                    try decoded.byte(current)
                let cost = decoded.used + 128
                if cost > self.string_budget:
                    error(limit_exceeded, "Prism decoded string budget exceeded")
                try self.decoded.push(decoded)
                self.string_budget -= cost
                self.token = decoded.view()
            else:
                self.token = (str)raw
            if self.token_kind == 3 and self.token.length != 1:
                error(invalid, "invalid Prism byte character")
            pos += 1
        elif punctuation(byte):
            self.token_kind = byte
            self.token = (str)bytes[pos..<pos + 1]
            pos += 1
        else:
            self.token_kind = 1
            let start = pos
            while pos < bytes.length:
                let current = bytes[pos]
                if current <= 32 or punctuation(current) or current == '"' or current == '\'' or current == '<' or current == '#' or current == '>':
                    break
                pos += 1
            if pos == start:
                error(invalid, "invalid Prism text byte")
            self.token = (str)bytes[start..<pos]
        self.position = pos

    pub func close():
        close_names(&self.decoded)

pub func punctuation(byte: u8) -> bool:
    return byte == '{' or byte == '}' or byte == '[' or byte == ']' or byte == '(' or byte == ')' or byte == '=' or byte == ',' or byte == ':'

pub func tokenize(bytes: const u8[]) -> Parser!:
    var parser = Parser(source = bytes)
    errdefer parser.close()
    try parser.advance()
    return parser