Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-prism

Source of release 0.6.0

0.6.0 / src / indexes / vector.lucb

## Exact vector search over float32 arrays, with cosine, dot and L2 metrics.
import interop
from luce_std import math
import indexes.ranked
from query.results import Selection
from document import Document, assemble_element, walk_document
from model import Element
from animation import resolve_property
from storage import List
from types import DType

pub enum Metric as u8:
    cosine = 0
    dot = 1
    l2 = 2

struct Entry:
    var vector: List[f32]
    var norm: f64

    func close():
        self.vector.close()

pub struct VectorIndex:
    var rows: Selection = Selection()
    var entries: List[Entry]
    var dimension: usize
    var metric: Metric

    pub func init(doc: interop.Reference[Document], field: str, metric: Metric = Metric.cosine) -> !:
        self.rows = Selection()
        self.entries = List[Entry]()
        self.dimension = 0
        self.metric = metric
        errdefer self.close()
        try self.build(doc, field)

    func consider(element: const Element*, field: str) -> !:
        let position = element.property(field) else return
        var value = try resolve_property(&element.properties.items[position], 0.0)
        defer value.close()
        if value.kind != DType.float32 or value.shape.count != 1 or value.count() == 0:
            return
        if self.dimension != 0 and value.count() != self.dimension:
            return
        self.dimension = value.count()
        try self.entries.reserve(self.entries.count + 1)
        var entry: Entry
        errdefer entry.close()
        try entry.vector.reserve(self.dimension)
        for i in 0..<self.dimension:
            let number = try value.number(i)
            try entry.vector.push((f32)number)
            entry.norm += number * number
        entry.norm = math.sqrt(entry.norm)
        try self.rows.append(element)
        try self.entries.push(entry)

    func build(doc: interop.Reference[Document], field: str) -> !:
        let view = try doc.get()
        if view.is_layered():
            var paths = try walk_document(view)
            defer paths.close()
            for path in paths.span():
                var element = try assemble_element(view, path)
                defer element.close()
                try self.consider(&element, field)
            return
        for element in view.view_data().elements.span():
            try self.consider(&element, field)

    pub func size() -> usize:
        return self.entries.count

    pub func dimensions() -> usize:
        return self.dimension

    pub func nearest(query: const f64[], count: usize) -> interop.Reference[Selection]!:
        var hits: List[ranked.Hit]
        defer hits.close()
        if query.length == self.dimension and count > 0:
            var norm: f64 = 0.0
            for value in query:
                let number = (f64)(f32)value
                norm += number * number
            norm = math.sqrt(norm)
            try hits.reserve(self.entries.count)
            for row in 0..<self.entries.count:
                let entry = &self.entries.items[row]
                var dot: f64 = 0.0
                var l2: f64 = 0.0
                for i in 0..<self.dimension:
                    let query_value = (f64)(f32)query[i]
                    let value = (f64)entry.vector.items[i]
                    dot += value * query_value
                    l2 += (value - query_value) * (value - query_value)
                var score = dot
                if self.metric == Metric.l2:
                    score = -l2
                elif self.metric == Metric.cosine:
                    score = dot / (entry.norm * norm) if entry.norm > 0.0 and norm > 0.0 else 0.0
                try hits.push(ranked.Hit(row = row, score = score))
        return try ranked.select(self.rows.view_data(), hits.span(), count)

    pub func close():
        self.rows.close()
        for i in 0..<self.entries.count:
            self.entries.items[i].close()
        self.entries.close()

pub let vector_index_type = interop.Type[VectorIndex]("Prism vector index", VectorIndex.close, closeable = true)