Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-prism

Source of release 0.6.1

0.6.1 / src / indexes / full_text.lucb

## Inverted text index with legacy ASCII tokenization and smoothed tf-idf ranking.
import interop
from luce_std import math
import indexes.ranked
from collections.index import Index
from query.results import Selection
from document import Document, assemble_element, walk_document
from model import Element
from animation import resolve_property
from storage import List, Buffer, Reader
from types import DType
import strings

struct Terms:
    var words: List[Buffer]

    func close():
        for i in 0..<self.words.count:
            self.words.items[i].close()
        self.words.close()

func tokenize(text: str) -> Terms!:
    var result: Terms
    errdefer result.close()
    var current: Buffer
    defer current.close()
    for byte in text.bytes:
        if (byte >= 'a' and byte <= 'z') or (byte >= 'A' and byte <= 'Z') or (byte >= '0' and byte <= '9'):
            try current.byte(byte + 32 if byte >= 'A' and byte <= 'Z' else byte)
        elif current.used > 0:
            try result.words.push(current)
            current = Buffer()
    if current.used > 0:
        try result.words.push(current)
        current = Buffer()
    return result

struct Posting:
    var row: usize
    var frequency: usize

pub struct TextIndex:
    var rows: Selection = Selection()
    var vocabulary: Index
    var postings: List[Posting]

    pub func init(doc: interop.Reference[Document], field: str) -> !:
        self.rows = Selection()
        self.vocabulary = Index()
        self.postings = List[Posting]()
        errdefer self.close()
        try self.build(doc, field)

    func consider(element: const Element*, field: str) -> !:
        let position = element.property(field) else return
        var value = try resolve_property(&element.properties.items[position], 0.0)
        defer value.close()
        if value.kind != DType.string or value.count() == 0:
            return
        var reader = Reader(data = value.payload.bytes())
        let text = try reader.text()
        if text.length == 0:
            return
        let row = self.rows.count()
        try self.rows.append(element)
        var words = try tokenize(text)
        defer words.close()
        var frequencies: Index
        defer frequencies.close()
        for word in words.words.span():
            _ = try frequencies.add(word.view(), 0)
        for word in frequencies.buckets.span():
            _ = try self.vocabulary.add(word.key.view(), self.postings.count)
            try self.postings.push(Posting(row = row, frequency = word.rows.count))

    func build(doc: interop.Reference[Document], field: str) -> !:
        let view = try doc.get()
        if view.is_layered():
            var paths = try walk_document(view)
            defer paths.close()
            for path in paths.span():
                var element = try assemble_element(view, path)
                defer element.close()
                try self.consider(&element, field)
            return
        for element in view.view_data().elements.span():
            try self.consider(&element, field)

    pub func terms() -> usize:
        return self.vocabulary.buckets.count

    pub func documents() -> usize:
        return self.rows.count()

    func search_terms(query: str, all: bool) -> interop.Reference[Selection]!:
        var words = try tokenize(query)
        defer words.close()
        var candidates: Index
        defer candidates.close()
        var scores: List[f64]
        defer scores.close()
        let count = words.words.count if all else (1 if words.words.count > 0 else 0)
        var absent = false
        for i in 0..<count:
            let word = words.words.items[i].view()
            let position = self.vocabulary.find(word)
            if position == none:
                absent = true
                break
            let bucket = &self.vocabulary.buckets.items[position else 0]
            let idf = math.log(1.0 + ((f64)self.rows.count() + 1.0) / ((f64)bucket.rows.count + 1.0))
            for index in bucket.rows.span():
                let posting = self.postings.items[index]
                var scratch: u8[32]
                let candidate = try candidates.add(try strings.format(scratch, f"{posting.row}"), posting.row)
                if candidate == scores.count:
                    try scores.push(0.0)
                scores.items[candidate] += (f64)posting.frequency * idf
        var hits: List[ranked.Hit]
        defer hits.close()
        if not absent:
            for i in 0..<candidates.buckets.count:
                let bucket = &candidates.buckets.items[i]
                if bucket.rows.count == count:
                    try hits.push(ranked.Hit(row = bucket.rows.items[0], score = scores.items[i]))
        return try ranked.select(self.rows.view_data(), hits.span(), hits.count)

    pub func search(term: str) -> interop.Reference[Selection]!:
        return try self.search_terms(term, false)

    pub func search_all(query: str) -> interop.Reference[Selection]!:
        return try self.search_terms(query, true)

    pub func close():
        self.rows.close()
        self.vocabulary.close()
        self.postings.close()

pub let text_index_type = interop.Type[TextIndex]("Prism text index", TextIndex.close, closeable = true)