Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.41.0

0.41.0 / src / support / modules.lucb

#==============================================================================================
#
#   modules - Resolving package modules
#
#   DESCRIPTION:
#       A package's modules are the files under its source root, named by their paths there
#       (§16.3, §16.4). Inside the package a module is imported by that path, `painter`;
#       another package imports it behind the package's identifier, `luce_ui.painter`, when
#       the package lists it as public. In a build, the modules of the package being built
#       keep their bare names and every other package's carry its identifier, so two
#       packages' `ui` stay two modules. Both compilers resolve through here.
#
#==============================================================================================

import c
import luce_std.files
import memory
import luce_std.paths
import strings
import support.hashed
import support.list
import support.manifest
import support.cache as cache
import support.in_arena
# mark: Resolutions and packages ==============================================================

## A manifest or dependency layout is malformed.
pub let invalid: ErrorCode = ErrorCode.package(28)

## A resolved module: its name, its source file, and its package root.
pub struct Resolution:
    ## The module's name.
    pub let name: str
    ## The path to its source file.
    pub let path: str
    ## The package's source root.
    pub let root: str

struct Mapping:
    let name: str
    let value: str

struct Package:
    var name: str
    var root: str
    var source: str
    var dependencies: list.List[Mapping]
    ## The modules other packages may import, by their paths under the source root.
    var public: list.List[str]

## Paths and manifest text live in the compiler's arena. Canonical file paths
## identify one module even if more than one public export names it.
##
## Each path is resolved once a run: on macOS, `realpath` lists every directory on the way to
## learn each name's true case, and a program's imports ask about the same paths again and
## again (luced-3d's took 1.4 s of directory listings). The answer is in the caller's
## allocator, a copy of the remembered one.
pub func canonical(path: str) -> str!:
    if let known = resolved_before(path):
        return try strings.copy(known)
    let terminated = try strings.copy(path)
    defer strings.release(terminated)
    let resolved = try in_arena.canonical((c.str)terminated)
    remember(path, resolved)
    return resolved

## The paths resolved so far, each with the index of its answer in `resolved_answers`,
## in the heap; made at the first path.
var resolved_paths: hashed.Table[str]? = none
var resolved_answers: list.List[str]? = none

func resolved_before(path: str) -> str?:
    let table = resolved_paths else return none
    let answers = resolved_answers else return none
    let index = table.find(path) else return none
    return answers.get(index)

## Keep `answer` for `path`; a memo that cannot grow keeps nothing, which costs time only.
func remember(path: str, answer: str):
    with memory.heap:
        keep(path, answer) catch ignored:
            _ = ignored

func keep(path: str, answer: str) -> !:
    var table = resolved_paths else try hashed.Table[str].create(256)
    var answers = resolved_answers else try list.List[str].create(256)
    try answers.push(try strings.copy(answer))
    _ = try table.add(try strings.copy(path), answers.count - 1)
    resolved_paths = table
    resolved_answers = answers

## Whether a file is at `path`; an absence is a build input (`cache.exists`).
func present(path: str) -> bool!:
    return cache.exists((c.str)try strings.copy(path))

func contained(path: str, root: str) -> bool:
    return path == root or (strings.starts_with(path, root) and path.length > root.length and path.bytes[root.length] == '/')

func identifier(name: str) -> bool:
    if name.length == 0:
        return false
    var first = true
    for byte in name.bytes:
        if byte == '.':
            if first:
                return false
            first = true
        elif byte == '_' or (byte >= 'a' and byte <= 'z') or (byte >= 'A' and byte <= 'Z') or (not first and byte >= '0' and byte <= '9'):
            first = false
        else:
            return false
    return not first

## A package module setting's value, which is never empty.
func quoted(value: str) -> str!:
    if value.length == 0:
        error(invalid, "package module settings are quoted strings")
    return value

func mapping(entries: list.List[Mapping]*, name: str, value: str) -> !:
    if not identifier(name):
        error(invalid, "an import or export name must be a dotted module name")
    for previous in entries.span():
        if previous.name == name:
            error(invalid, "a package module setting is declared twice")
    try entries.push(Mapping(name, value))

## The manifests read so far and their packages, in the heap: a program's imports ask for the
## same few manifests again and again, and a compiled workspace's manifest names every module
## (luced-3d's has 451 module packages, and each read checked them against each other).
var read_paths: hashed.Table[str]? = none
var read_packages: list.List[Package]? = none

## Read every manifest again from here on: a host that checks for a long time
## (`embed.Library`) calls this when one changed. The packages read so far stay where they
## are, in the heap, for what still holds them.
pub func forget_manifests():
    read_paths = none
    read_packages = none

## The package of the manifest at `path`, read once a run; the caller only reads it.
func read_package(path: str) -> Package!:
    if let table = read_paths:
        if let index = table.find(path):
            let packages = read_packages else trap("unreachable")
            return packages.get(index)
    with memory.heap:
        # in the heap, since the kept package outlives whatever arena asked for it
        let package = try parse_package(path)
        var table = read_paths else try hashed.Table[str].create(16)
        var packages = read_packages else try list.List[Package].create(16)
        try packages.push(package)
        _ = try table.add(try strings.copy(path), packages.count - 1)
        read_paths = table
        read_packages = packages
        return package

func parse_package(path: str) -> Package!:
    let root = try canonical(paths.directory(path))
    let bytes = try in_arena.read((c.str)try strings.copy(path))
    # a dependency's manifest says what it exports, so it is an input of every build
    cache.note_input(path, bytes)
    let contents = (str)bytes
    var package = Package(name = "", root = root, source = "src",
                          dependencies = try list.List[Mapping].create(4), public = try list.List[str].create(4))
    var named = false
    var sourced = false
    for setting in try manifest.settings(contents, path):
        let section = setting.section
        let key = setting.key
        if section != "[package]" and section != "[dependencies]":
            continue
        if section == "[package]" and key != "name" and key != "source" and key != "public":
            continue
        if section == "[package]" and key == "public":
            for module in try manifest.strings_of(setting.value):
                if not identifier(module):
                    error(invalid, "a public module is a dotted path under the source root")
                try package.public.push(module)
            continue
        let value = try quoted(setting.value)
        if section == "[dependencies]":
            try mapping(&package.dependencies, key, value)
        elif key == "name":
            if named:
                error(invalid, "the package name is declared twice")
            named = true
            package.name = value
        elif key == "source":
            if sourced:
                error(invalid, "the package source root is declared twice")
            sourced = true
            package.source = value
    package.source = try in_arena.join(root, package.source)
    return package

## Where a dependency lives: its declared checkout path, or `.luc/deps/<name>` at the package
## root or the nearest directory above it, where luc unpacks registry packages. A package
## that itself sits under `.luc/deps/` came from the registry, so its own checkout paths are
## its author's and every dependency is looked up the same way; so is a checkout path that
## is not there, as when an application published with its author's sibling checkouts is
## installed elsewhere.
func dependency_directory(package: Package, dependency: Mapping) -> str!:
    let synced = strings.starts_with(dependency.value, ".luc/deps/") or strings.contains(package.root, "/.luc/deps/")
    if not synced:
        let declared = try in_arena.join(package.root, dependency.value)
        if try present(declared):
            return declared
    var relative = try in_arena.join(".luc/deps", dependency.name)
    var directory = package.root
    while true:
        let candidate = try in_arena.join(directory, relative)
        if try present(candidate):
            return candidate
        let parent = paths.directory(directory)
        if parent == directory or parent.length == 0:
            error(invalid, "a registry dependency is not synced; run luc sync")
        directory = parent

## The source of every module of the package that owns the file at `path`, canonical and in
## the order of their paths: each `.lucb` file under its source root, and each directory
## there whose `ORDER` makes it one module (§16.1); nothing in a hidden directory, in one
## holding a `package.prisma` of its own, or in `skip` (the standard library, when the
## package is the compiler's own). None outside a package. What a test of the whole package
## loads (§16.5).
pub func package_modules(path: str, skip: str) -> list.List[str]!:
    var found = try list.List[str].create(16)
    let package = try enclosing(try canonical(path)) else return found
    if package.name.length == 0 or not try present(package.source):
        return found
    try gather_modules(try canonical(package.source), skip, &found)
    # gathered directory by directory; the modules go in their paths' order
    for k in 1..<found.count:
        let moved = found.get(k)
        var j = k
        while j > 0 and moved < found.get(j - 1):
            found.set(j, found.get(j - 1))
            j -= 1
        found.set(j, moved)
    return found

## The modules under `directory` into `found`: its `.lucb` files, and each directory below
## it that is one module by its `ORDER`, or else the modules under it.
func gather_modules(directory: str, skip: str, found: list.List[str]*) -> !:
    let listed = try files.list((c.str)directory)
    defer listed.release()
    for name in listed.value:
        if name.length == 0 or name.bytes[0] == '.':
            continue
        let path = try in_arena.join(directory, name)
        if strings.ends_with(name, ".lucb"):
            try found.push(path)
            continue
        if path == skip or try present(try in_arena.join(path, "package.prisma")):
            continue
        # a directory with an `ORDER` is one module, its files the fragments
        if try present(try in_arena.join(path, "ORDER")):
            try found.push(path)
            continue
        # anything else is searched when it is a directory, and is not a module when not
        gather_modules(path, skip, found) catch failure:
            _ = failure

## The package that owns the file at `path` and its root directory, when a manifest
## encloses it; the root is canonical, as module paths are.
pub func owner_root(path: str) -> (str, str)?!:
    let package = try enclosing(try canonical(path)) else return none
    if package.name.length == 0:
        return none
    return (package.name, package.root)

## The directory of test fragments for the module at `path` (§16.5): under the package's
## `tests/`, the module's path below the source root, when a `TESTS` file there lists them;
## none otherwise. `luce-base test` adds them to the module under test; a build never reads
## them. The list is not an `ORDER`, so the directory is never taken for a module that an
## import from `tests/` would resolve to.
pub func test_fragments_of(path: str) -> str?!:
    let module = try canonical(path)
    let package = try enclosing(module) else return none
    # the root is canonical, so the source root under it is spelled as `module` is
    let source = package.source
    if module.length <= source.length + 1 or not strings.starts_with(module, source) or (module.bytes[source.length] != '/' and module.bytes[source.length] != '\\'):
        return none
    var relative = (str)module.bytes[source.length + 1..]
    if strings.ends_with(relative, ".lucb"):
        relative = (str)relative.bytes[..<relative.length - 5]
    let directory = try in_arena.join(try in_arena.join(package.root, "tests"), relative)
    if not try present(try in_arena.join(directory, "TESTS")):
        return none
    return directory

func enclosing(path: str) -> Package?!:
    var directory = paths.directory(path)
    while directory.length > 0:
        let candidate = try in_arena.join(directory, "package.prisma")
        if try present(candidate):
            return try read_package(candidate)
        let parent = paths.directory(directory)
        if parent == directory:
            break
        directory = parent
    return none

func source_file(root: str, name: str, base_only: bool) -> str?!:
    let relative = try new u8[name.length] ---
    for index in 0..<name.length:
        relative[index] = '/' if name.bytes[index] == '.' else name.bytes[index]
    let prefix = try in_arena.join(root, (str)relative)
    if not base_only:
        let luce = try strings.join([prefix, ".luc"], "")
        if try present(luce):
            return try canonical(luce)
    let base = try strings.join([prefix, ".lucb"], "")
    if try present(base):
        return try canonical(base)
    # a directory whose `ORDER` lists the module's fragments stands for the file
    let order = try strings.join([prefix, "/ORDER"], "")
    if try present(order):
        return try canonical(prefix)
    return none

## The name module `module` (its dotted path under the source root) of `package` has in a
## build of package `building`: bare in that package, else behind its package's identifier
## (§16.3).
func build_name(package: Package, module: str, building: str) -> str!:
    if package.name == building or package.name.length == 0:
        return module
    return try strings.join([package.name, ".", module], "")

## Whether `package` lists `module` among its public modules.
func is_public(package: Package, module: str) -> bool:
    for listed in package.public.span():
        if listed == module:
            return true
    return false

## Whether `name` begins with a package's identifier, the importer's own or a dependency's,
## and what follows it: `luce_ui.painter` is `("luce_ui", "painter")`.
func split_package(name: str) -> (str, str):
    let dot = strings.find(name, ".") else return (name, "")
    return ((str)name.bytes[..<dot], (str)name.bytes[dot + 1..])

## The package `head` names for a file of `package`: the package itself or one it depends on.
func named_package(package: Package, head: str) -> Package?!:
    if head == package.name:
        return package
    var found: Package? = none
    for dependency in package.dependencies.span():
        # every dependency is checked, not only the one named: a misnamed one is an error
        # wherever it is imported from
        let other = try read_package(try in_arena.join(try dependency_directory(package, dependency), "package.prisma"))
        if other.name != dependency.name:
            error(invalid, "a dependency key must match its package name")
        if dependency.name == head:
            found = other
    return found

# mark: Resolving imports =====================================================================

## A module named behind a package's identifier (§16.3): `luce_ui.painter`, module `painter`
## of `luce_ui`, which the importer's package depends on, or the importer's own package
## spelled out. Another package's module must be among its `public` ones. None when the
## first part names no package, or names the importer's own package alone and the package
## has a module of its name (`import mylib` inside mylib, its starter module): the import
## is the importer's package's own module. `building` is the package being built, which
## names its modules bare.
pub func package_import(importer: str, name: str, building: str, base_only: bool = true) -> Resolution?!:
    if not identifier(name):
        error(invalid, "an import must name a dotted module")
    let input = try canonical(importer)
    let package = try enclosing(input) else return none
    let (head, module) = split_package(name)
    let owner = try named_package(package, head) else return none
    let own_named = try source_file(package.source, head, base_only) != none
    if owner.name == package.name:
        if module.length == 0 and own_named:
            return none
    elif own_named:
        error(invalid, "a module of this package takes the name of a package it imports from")
    if module.length == 0:
        error(invalid, "a package is not a module: import one of its modules, `import pkg.module` or `from pkg import module` (§16.3)")
    if owner.name != package.name and not is_public(owner, module):
        error(invalid, "the module is not among the package's `public` ones (§16.4)")
    let root = try canonical(owner.source)
    let path = try source_file(root, module, base_only) else error(invalid, "the package has no module of this name")
    return Resolution(try build_name(owner, module, building), path, root)

## Whether the names `from name import ...` brings are modules (§16.3): `name` is a package,
## the importer's own or one it depends on (`from luce_ui import ui`), or a directory of
## modules in one (`from luce_geocore.core import parallel`) or in the importer's own source
## (`from core import parallel`), rather than a module whose declarations they would be. The
## importer's own package's module of the package's name is a module (`from mylib import x`
## inside mylib brings its declaration `x`).
pub func names_modules(importer: str, name: str) -> bool!:
    let package = try enclosing(try canonical(importer)) else return false
    let (head, rest) = split_package(name)
    var owner = package
    var path = name
    if let named = try named_package(package, head):
        if rest.length == 0:
            if named.name == package.name and try present(package.source):
                return try source_file(try canonical(package.source), head, false) == none
            return true
        owner = named
        path = rest
    if not try present(owner.source):
        return false
    let root = try canonical(owner.source)
    if try source_file(root, path, false) != none:
        return false
    var directory = try new u8[root.length + 1 + path.length] ---
    let spelled = try strings.format(directory, f"{root}/{path}")
    for i in root.length + 1..<spelled.length:
        if spelled.bytes[i] == '.': directory[i] = '/'
    return try present(spelled)

## Whether the importer's package has a module of its own named `name`, which a standard
## module's name would hide (§16.6).
pub func has_local_module(importer: str, name: str) -> bool!:
    let package = try enclosing(try canonical(importer)) else return false
    if not try present(package.source):
        return false
    return try source_file(try canonical(package.source), name, true) != none

## The identifier of the package a build with source root `source_root` builds, whose
## modules keep their bare names (§16.3); empty outside a package.
pub func building_package(source_root: str) -> str!:
    if not try present(source_root):
        return ""
    let built = try enclosing(try in_arena.join(try canonical(source_root), "x")) else return ""
    return built.name

## Where import `name` of the file `importer` leads, for every compiler that resolves
## imports (the Base checker and Luce), `source_root` being where the importer's search
## stands (base.md §16.3, §16.4). In order:
## - a package's module behind its identifier, `luce_ui.ui`, public unless it is the
##   importer's own package's;
## - for a file inside a package's source root, the module at that path under the root;
## - for a file of a package outside its source root, a test under `tests/`, the package's
##   module first, then as for a file in no package;
## - for a file in no package, a module beside it (never the file itself), then under
##   `source_root` and each directory enclosing it, where a directory holding a
##   `package.prisma` offers that package's modules.
## `base_only` leaves `.luc` modules out. `built` names the package being built, whose
## modules keep bare names; empty, it is the package enclosing `source_root`. The
## resolution's root is where the module was found.
pub func resolve(importer: str, source_root: str, name: str, base_only: bool = false, built: str = "") -> Resolution!:
    let building = built if built.length > 0 else try building_package(source_root)
    if let external = try package_import(importer, name, building, base_only):
        return external
    let input = try canonical(importer)
    let here = try canonical(paths.directory(input))
    if let package = try enclosing(input):
        if try present(package.source):
            let source = try canonical(package.source)
            if let path = try source_file(source, name, base_only):
                return Resolution(try build_name(package, name, building), path, source)
            if contained(input, source):
                error(invalid, "the module is neither this package's nor a public module of a package it depends on")
    # a file in no package, or one of a package outside its source root (a program under
    # `tests/`): beside it, then up from where the search stands
    if let path = try beside(here, name, base_only, input):
        return Resolution(name, path, here)
    var root = try canonical(source_root if source_root.length > 0 else ".")
    while true:
        if let path = try beside(root, name, base_only, input):
            return Resolution(name, path, root)
        let manifest_path = try in_arena.join(root, "package.prisma")
        if try present(manifest_path):
            let package = try read_package(manifest_path)
            if let path = try source_file(package.source, name, base_only):
                return Resolution(name, path, try canonical(package.source))
        let parent = paths.directory(root)
        if parent == root or parent.length == 0:
            break
        root = parent
    # nothing else of the name: a file importing itself, which the loader reports as the
    # cycle it is
    if let path = try source_file(here, name, base_only):
        return Resolution(name, path, here)
    error(invalid, "the module is neither this package's nor a public module of a package it depends on")

# mark: A sealed root ========================================================================

## Where import `name` of the file `importer` leads under a sealed root, an embedded
## checker's (docs/EMBEDDING.md): the module at that path under `root`, a `.lucb` file or a
## directory of fragments, and nowhere else. A directory directly under the root is a
## package, as `luce_std/` is: its modules import each other by their paths in it first, as
## a package's own modules do (`resolve`), and such a module is named by its path under the
## root, `luce_std.math`, the name `from luce_std import math` gives it. No manifest is read.
pub func resolve_sealed(root: str, importer: str, name: str) -> Resolution!:
    if not identifier(name):
        error(invalid, "an import must name a dotted module")
    if let package = sealed_package(root, importer):
        if let path = try source_file(try in_arena.join(root, package), name, true):
            return Resolution(try strings.join([package, ".", name], ""), path, root)
    let path = try source_file(root, name, true) else error(invalid, "the checked directory has no module of this name")
    return Resolution(name, path, root)

## Whether module `name`, as the file `importer` would import it, is in a sealed root: in
## the importer's package there, or at its path under the root.
pub func sealed_has(root: str, importer: str, name: str) -> bool!:
    if not identifier(name):
        return false
    if let package = sealed_package(root, importer):
        if try source_file(try in_arena.join(root, package), name, true) != none:
            return true
    return try source_file(root, name, true) != none

## Whether `from name import ...` in the file `importer` under a sealed root brings modules:
## `name` is a directory of modules there (`from luce_kernel import lib`), not a module of
## its own; in the importer's package first, as `resolve_sealed` looks.
pub func sealed_names_modules(root: str, importer: str, name: str) -> bool!:
    if not identifier(name):
        return false
    if let package = sealed_package(root, importer):
        let own = try in_arena.join(root, package)
        if try source_file(own, name, true) != none:
            return false
        if try present(try directory_of(own, name)):
            return true
    if try source_file(root, name, true) != none:
        return false
    return try present(try directory_of(root, name))

## The package under a sealed root that the file `importer` belongs to: the directory
## directly under `root` that holds it (`luce_std` for `root/luce_std/math32.lucb`); none
## for a module at the root's top, or a file outside it.
func sealed_package(root: str, importer: str) -> str?:
    if not contained(importer, root) or importer.length == root.length:
        return none
    let relative = (str)importer.bytes[root.length + 1..]
    let slash = strings.find(relative, "/") else return none
    return (str)relative.bytes[..<slash]

## The directory a dotted `name` spells under `root`: `root/core/parallel` for `core.parallel`.
func directory_of(root: str, name: str) -> str!:
    var directory = try new u8[root.length + 1 + name.length] ---
    let spelled = try strings.format(directory, f"{root}/{name}")
    for i in root.length + 1..<spelled.length:
        if spelled.bytes[i] == '.': directory[i] = '/'
    return spelled

## `path`, a canonical module path, spelled as `importer` is: relative to the working
## directory when the importer's path is relative and `path` lies below it, else as is.
pub func spelled_like(path: str, importer: str) -> str!:
    if importer.length == 0 or paths.is_absolute(importer):
        return path
    let here = try canonical(".")
    if contained(path, here) and path.length > here.length:
        return (str)path.bytes[here.length + 1..]
    return path

## Module `name` under `directory` unless it is the file `importer` itself: an import of
## its own name looks past it first (`tests/area.lucb` importing `area`).
func beside(directory: str, name: str, base_only: bool, importer: str) -> str?!:
    let path = try source_file(directory, name, base_only) else return none
    if path == importer:
        return none
    return path

## The module's canonical dotted name inside its own package source root.
pub func name_of(path: str) -> str!:
    if not (try present(path)):
        return paths.stem(path)
    let input = try canonical(path)
    if let package = try enclosing(input):
        if try present(package.source):
            let root = try canonical(package.source)
            if contained(input, root) and input.length > root.length:
                return try dotted((str)input.bytes[root.length + 1..])
    return paths.stem(input)

## The name of the program whose entry is `path`: its module name, except for an entry
## outside its package's source root named like one of the package's modules, a test
## `tests/area.lucb` of module `area`, which is named by its path under the package root
## (`tests.area`), so that it and the module it imports stay two modules.
pub func entry_name(path: str) -> str!:
    let name = try name_of(path)
    if not (try present(path)):
        return name
    let input = try canonical(path)
    let package = try enclosing(input) else return name
    if not try present(package.source):
        return name
    let source = try canonical(package.source)
    if contained(input, source) or try source_file(source, name, true) == none:
        return name
    let top = try canonical(package.root)
    return try dotted((str)input.bytes[top.length + 1..])

## `relative`, a file's path below a root, as a dotted module name without its extension.
func dotted(relative: str) -> str!:
    let extension = paths.extension(relative)
    let size = relative.length - extension.length
    let name = try new u8[size] ---
    for index in 0..<size:
        name[index] = '.' if relative.bytes[index] == '/' else relative.bytes[index]
    return (str)name

## The identifier of the package that owns the file at `path`: the nearest manifest's, or
## `fallback` outside a package. It is the identity of the module's error codes (§11.3).
pub func package_of(path: str, fallback: str) -> str!:
    if let package = try enclosing(try canonical(path)):
        if package.name.length > 0:
            return package.name
    return fallback