Luce packages
LuceLuce BaseLuciaOS

dymokomi/luced

Source of release 0.4.1

0.4.1 / src / language / tm.luc

## Highlighting driven by a TextMate grammar. On any change it re-tokenizes the whole document
## through the luce-base tokenizer — fast enough now to do wholesale — and colors each scoped
## span through the theme. A wall-clock budget inside the tokenizer leaves a very large file's
## tail plain rather than blocking on open. Folding for grammar languages is a later pass; this
## module publishes highlights only.
from luce_ui.ui import TextEditor, Highlight
from language.appearance import SyntaxTheme
from luce_textmate.textmate import Grammar, Tokenizer

pub class TmHighlighter:
    let editor: TextEditor
    let tok: Tokenizer
    var theme: SyntaxTheme
    var version: int

    pub func init(self, editor: TextEditor, grammar: Grammar, theme: SyntaxTheme? = none) -> unit!:
        self.editor = editor
        self.tok = Tokenizer(grammar)
        self.theme = theme else SyntaxTheme({})
        self.version = -1

    pub func set_theme(self, theme: SyntaxTheme) -> unit!:
        self.theme = theme
        self.version = -1
        self.update()

    pub func update(self) -> unit!:
        let revision = self.editor.version()
        if revision == self.version:
            return
        let length = self.editor.length()
        let text = self.editor.text_range(0, length)
        # Re-tokenize incrementally when this is exactly the next revision after the one we have,
        # so the tokenizer can reuse the unchanged prefix and suffix; otherwise tokenize afresh.
        let change = self.editor.change()
        var count = 0
        if self.version >= 0 and revision == self.version + 1 and change.version == revision:
            count = self.tok.tokenize_edit(text, change.start, change.old_end, change.new_end)
        else:
            count = self.tok.tokenize(text)
        var spans: list[Highlight] = []
        var i = 0
        while i < count:
            if let color = self.theme.color_for_scope(self.tok.span_scope(i)):
                spans.append(Highlight(self.tok.span_start(i), self.tok.span_end(i), color))
            i += 1
        if self.editor.update_highlights(0, length, spans, revision):
            self.version = revision