dymokomi/luce-base
Source of release 0.44.0
0.44.0 / src / support / modules.lucb
#==============================================================================================
#
# modules - Resolving package modules
#
# DESCRIPTION:
# A package's modules are the files under its source root, named by their paths there
# (§16.3, §16.4). Inside the package a module is imported by that path, `painter`;
# another package imports it behind the package's identifier, `luce_ui.painter`, when
# the package lists it as public. In a build, the modules of the package being built
# keep their bare names and every other package's carry its identifier, so two
# packages' `ui` stay two modules. Both compilers resolve through here.
#
#==============================================================================================
import c
import luce_std.files
import memory
import luce_std.paths
import platform
import strings
import support.hashed
import support.list
import support.manifest
import support.cache as cache
import support.in_arena
# mark: Resolutions and packages ==============================================================
## A manifest or dependency layout is malformed.
pub let invalid: ErrorCode = ErrorCode.package(28)
## A resolved module: its name, its source file, and its package root.
pub struct Resolution:
## The module's name.
pub let name: str
## The path to its source file.
pub let path: str
## The package's source root.
pub let root: str
struct Mapping:
let name: str
let value: str
struct Package:
var name: str
var root: str
var source: str
var dependencies: list.List[Mapping]
## The modules other packages may import, by their paths under the source root.
var public: list.List[str]
## Paths and manifest text live in the compiler's arena. Canonical file paths
## identify one module even if more than one public export names it.
##
## Each path is resolved once a run: on macOS, `realpath` lists every directory on the way to
## learn each name's true case, and a program's imports ask about the same paths again and
## again (luced-3d's took 1.4 s of directory listings). The answer is in the caller's
## allocator, a copy of the remembered one.
pub func canonical(path: str) -> str!:
if let known = resolved_before(path):
return try strings.copy(known)
let terminated = try strings.copy(path)
defer strings.release(terminated)
let resolved = try in_arena.canonical((c.str)terminated)
remember(path, resolved)
return resolved
## `path` spelled as the compiler spells the paths it composes: on Windows with `/` between
## its directories, so a source the command line names with `\` reads in a position
## (§11.3) as a module it imports does; elsewhere, where `\` is a byte of a name, `path`
## itself. A copy in the current allocator when it differs.
pub func portable(path: c.str) -> c.str!:
let text = (str)path
if not platform.windows or strings.find(text, "\\") == none:
return path
let room = try new u8[text.length + 1] ---
for (k, byte) in text.bytes.indexed():
room[k] = '/' if byte == '\\' else byte
room[text.length] = 0
return (c.str)(str)room[..<text.length]
## The paths resolved so far, each with the index of its answer in `resolved_answers`,
## in the heap; made at the first path.
var resolved_paths: hashed.Table[str]? = none
var resolved_answers: list.List[str]? = none
func resolved_before(path: str) -> str?:
let table = resolved_paths else return none
let answers = resolved_answers else return none
let index = table.find(path) else return none
return answers.get(index)
## Keep `answer` for `path`; a memo that cannot grow keeps nothing, which costs time only.
func remember(path: str, answer: str):
with memory.heap:
keep(path, answer) catch ignored:
_ = ignored
func keep(path: str, answer: str) -> !:
var table = resolved_paths else try hashed.Table[str].create(256)
var answers = resolved_answers else try list.List[str].create(256)
try answers.push(try strings.copy(answer))
_ = try table.add(try strings.copy(path), answers.count - 1)
resolved_paths = table
resolved_answers = answers
## Whether a file is at `path`; an absence is a build input (`cache.exists`).
func present(path: str) -> bool!:
return cache.exists((c.str)try strings.copy(path))
func contained(path: str, root: str) -> bool:
return path == root or (strings.starts_with(path, root) and path.length > root.length and path.bytes[root.length] == '/')
func identifier(name: str) -> bool:
if name.length == 0:
return false
var first = true
for byte in name.bytes:
if byte == '.':
if first:
return false
first = true
elif byte == '_' or (byte >= 'a' and byte <= 'z') or (byte >= 'A' and byte <= 'Z') or (not first and byte >= '0' and byte <= '9'):
first = false
else:
return false
return not first
## A package module setting's value, which is never empty.
func quoted(value: str) -> str!:
if value.length == 0:
error(invalid, "package module settings are quoted strings")
return value
func mapping(entries: list.List[Mapping]*, name: str, value: str) -> !:
if not identifier(name):
error(invalid, "an import or export name must be a dotted module name")
for previous in entries.span():
if previous.name == name:
error(invalid, "a package module setting is declared twice")
try entries.push(Mapping(name, value))
## The manifests read so far and their packages, in the heap: a program's imports ask for the
## same few manifests again and again, and a compiled workspace's manifest names every module
## (luced-3d's has 451 module packages, and each read checked them against each other).
var read_paths: hashed.Table[str]? = none
var read_packages: list.List[Package]? = none
## Read every manifest again from here on: a host that checks for a long time
## (`embed.Library`) calls this when one changed. The packages read so far stay where they
## are, in the heap, for what still holds them.
pub func forget_manifests():
read_paths = none
read_packages = none
## The package of the manifest at `path`, read once a run; the caller only reads it.
func read_package(path: str) -> Package!:
if let table = read_paths:
if let index = table.find(path):
let packages = read_packages else trap("unreachable")
return packages.get(index)
with memory.heap:
# in the heap, since the kept package outlives whatever arena asked for it
let package = try parse_package(path)
var table = read_paths else try hashed.Table[str].create(16)
var packages = read_packages else try list.List[Package].create(16)
try packages.push(package)
_ = try table.add(try strings.copy(path), packages.count - 1)
read_paths = table
read_packages = packages
return package
func parse_package(path: str) -> Package!:
let root = try canonical(paths.directory(path))
let bytes = try in_arena.read((c.str)try strings.copy(path))
# a dependency's manifest says what it exports, so it is an input of every build
cache.note_input(path, bytes)
let contents = (str)bytes
var package = Package(name = "", root = root, source = "src",
dependencies = try list.List[Mapping].create(4), public = try list.List[str].create(4))
var named = false
var sourced = false
for setting in try manifest.settings(contents, path):
let section = setting.section
let key = setting.key
if section != "[package]" and section != "[dependencies]":
continue
if section == "[package]" and key != "name" and key != "source" and key != "public":
continue
if section == "[package]" and key == "public":
for module in try manifest.strings_of(setting.value):
if not identifier(module):
error(invalid, "a public module is a dotted path under the source root")
try package.public.push(module)
continue
let value = try quoted(setting.value)
if section == "[dependencies]":
try mapping(&package.dependencies, key, value)
elif key == "name":
if named:
error(invalid, "the package name is declared twice")
named = true
package.name = value
elif key == "source":
if sourced:
error(invalid, "the package source root is declared twice")
sourced = true
package.source = value
package.source = try in_arena.join(root, package.source)
return package
## Where a dependency lives: its declared checkout path, or `.luc/deps/<name>` at the package
## root or the nearest directory above it, where luc unpacks registry packages. A package
## that itself sits under `.luc/deps/` came from the registry, so its own checkout paths are
## its author's and every dependency is looked up the same way; so is a checkout path that
## is not there, as when an application published with its author's sibling checkouts is
## installed elsewhere.
func dependency_directory(package: Package, dependency: Mapping) -> str!:
let synced = strings.starts_with(dependency.value, ".luc/deps/") or strings.contains(package.root, "/.luc/deps/")
if not synced:
let declared = try in_arena.join(package.root, dependency.value)
if try present(declared):
return declared
var relative = try in_arena.join(".luc/deps", dependency.name)
var directory = package.root
while true:
let candidate = try in_arena.join(directory, relative)
if try present(candidate):
return candidate
let parent = paths.directory(directory)
if parent == directory or parent.length == 0:
error(invalid, "a registry dependency is not synced; run luc sync")
directory = parent
## The source of every module of the package that owns the file at `path`, canonical and in
## the order of their paths: each `.lucb` file under its source root, and each directory
## there whose `ORDER` makes it one module (§16.1); nothing in a hidden directory, in one
## holding a `package.prisma` of its own, or in `skip` (the standard library, when the
## package is the compiler's own). The compiler's package, luce-base, keeps the standard
## library's source in its `std` directory, which is the standard modules and none of its
## own, whichever compiler reads it. None outside a package. What a test or check of the
## whole package loads (§16.5).
pub func package_modules(path: str, skip: str) -> list.List[str]!:
var found = try list.List[str].create(16)
let package = try enclosing(try canonical(path)) else return found
if package.name.length == 0 or not try present(package.source):
return found
let source = try canonical(package.source)
# the manifest's name, in its identifier form (§16.3)
let own_standard = try in_arena.join(source, "std") if package.name == "luce_base" else ""
try gather_modules(source, [skip, own_standard], &found)
# gathered directory by directory; the modules go in their paths' order
for k in 1..<found.count:
let moved = found.get(k)
var j = k
while j > 0 and moved < found.get(j - 1):
found.set(j, found.get(j - 1))
j -= 1
found.set(j, moved)
return found
## The modules under `directory` into `found`: its `.lucb` files, and each directory below
## it that is one module by its `ORDER`, or else the modules under it, but none of the
## directories `skips` names.
func gather_modules(directory: str, skips: const str[], found: list.List[str]*) -> !:
let listed = try files.list((c.str)directory)
defer listed.release()
for name in listed.value:
if name.length == 0 or name.bytes[0] == '.':
continue
let path = try in_arena.join(directory, name)
if strings.ends_with(name, ".lucb"):
try found.push(path)
continue
if skipped(path, skips) or try present(try in_arena.join(path, "package.prisma")):
continue
# a directory with an `ORDER` is one module, its files the fragments
if try present(try in_arena.join(path, "ORDER")):
try found.push(path)
continue
# anything else is searched when it is a directory, and is not a module when not
gather_modules(path, skips, found) catch failure:
_ = failure
## Whether `path` is one of the nonempty `skips`.
func skipped(path: str, skips: const str[]) -> bool:
for skip in skips:
if skip.length > 0 and path == skip: return true
return false
## The package that owns the file at `path` and its root directory, when a manifest
## encloses it; the root is canonical, as module paths are.
pub func owner_root(path: str) -> (str, str)?!:
let package = try enclosing(try canonical(path)) else return none
if package.name.length == 0:
return none
return (package.name, package.root)
## The directory of test fragments for the module at `path` (§16.5): under the package's
## `tests/`, the module's path below the source root, when a `TESTS` file there lists them;
## none otherwise. `luce-base test` adds them to the module under test; a build never reads
## them. The list is not an `ORDER`, so the directory is never taken for a module that an
## import from `tests/` would resolve to.
pub func test_fragments_of(path: str) -> str?!:
let module = try canonical(path)
let package = try enclosing(module) else return none
# the root is canonical, so the source root under it is spelled as `module` is
let source = package.source
if module.length <= source.length + 1 or not strings.starts_with(module, source) or (module.bytes[source.length] != '/' and module.bytes[source.length] != '\\'):
return none
var relative = (str)module.bytes[source.length + 1..]
if strings.ends_with(relative, ".lucb"):
relative = (str)relative.bytes[..<relative.length - 5]
let directory = try in_arena.join(try in_arena.join(package.root, "tests"), relative)
if not try present(try in_arena.join(directory, "TESTS")):
return none
return directory
func enclosing(path: str) -> Package?!:
var directory = paths.directory(path)
while directory.length > 0:
let candidate = try in_arena.join(directory, "package.prisma")
if try present(candidate):
return try read_package(candidate)
let parent = paths.directory(directory)
if parent == directory:
break
directory = parent
return none
func source_file(root: str, name: str, base_only: bool) -> str?!:
let relative = try new u8[name.length] ---
for index in 0..<name.length:
relative[index] = '/' if name.bytes[index] == '.' else name.bytes[index]
let prefix = try in_arena.join(root, (str)relative)
if not base_only:
let luce = try strings.join([prefix, ".luc"], "")
if try present(luce):
return try canonical(luce)
let base = try strings.join([prefix, ".lucb"], "")
if try present(base):
return try canonical(base)
# a directory whose `ORDER` lists the module's fragments stands for the file
let order = try strings.join([prefix, "/ORDER"], "")
if try present(order):
return try canonical(prefix)
return none
## The name module `module` (its dotted path under the source root) of `package` has in a
## build of package `building`: bare in that package, else behind its package's identifier
## (§16.3).
func build_name(package: Package, module: str, building: str) -> str!:
if package.name == building or package.name.length == 0:
return module
return try strings.join([package.name, ".", module], "")
## Whether `package` lists `module` among its public modules.
func is_public(package: Package, module: str) -> bool:
for listed in package.public.span():
if listed == module:
return true
return false
## Whether `name` begins with a package's identifier, the importer's own or a dependency's,
## and what follows it: `luce_ui.painter` is `("luce_ui", "painter")`.
func split_package(name: str) -> (str, str):
let dot = strings.find(name, ".") else return (name, "")
return ((str)name.bytes[..<dot], (str)name.bytes[dot + 1..])
## The package `head` names for a file of `package`: the package itself or one it depends on.
func named_package(package: Package, head: str) -> Package?!:
if head == package.name:
return package
var found: Package? = none
for dependency in package.dependencies.span():
# every dependency is checked, not only the one named: a misnamed one is an error
# wherever it is imported from
let other = try read_package(try in_arena.join(try dependency_directory(package, dependency), "package.prisma"))
if other.name != dependency.name:
error(invalid, "a dependency key must match its package name")
if dependency.name == head:
found = other
return found
# mark: Resolving imports =====================================================================
## A module named behind a package's identifier (§16.3): `luce_ui.painter`, module `painter`
## of `luce_ui`, which the importer's package depends on, or the importer's own package
## spelled out. Another package's module must be among its `public` ones. None when the
## first part names no package, or names the importer's own package alone and the package
## has a module of its name (`import mylib` inside mylib, its starter module): the import
## is the importer's package's own module. `building` is the package being built, which
## names its modules bare.
pub func package_import(importer: str, name: str, building: str, base_only: bool = true) -> Resolution?!:
if not identifier(name):
error(invalid, "an import must name a dotted module")
let input = try canonical(importer)
let package = try enclosing(input) else return none
let (head, module) = split_package(name)
let owner = try named_package(package, head) else return none
let own_named = try source_file(package.source, head, base_only) != none
if owner.name == package.name:
if module.length == 0 and own_named:
return none
elif own_named:
error(invalid, "a module of this package takes the name of a package it imports from")
if module.length == 0:
error(invalid, "a package is not a module: import one of its modules, `import pkg.module` or `from pkg import module` (§16.3)")
if owner.name != package.name and not is_public(owner, module):
error(invalid, "the module is not among the package's `public` ones (§16.4)")
let root = try canonical(owner.source)
let path = try source_file(root, module, base_only) else error(invalid, "the package has no module of this name")
return Resolution(try build_name(owner, module, building), path, root)
## Whether the names `from name import ...` brings are modules (§16.3): `name` is a package,
## the importer's own or one it depends on (`from luce_ui import ui`), or a directory of
## modules in one (`from luce_geocore.core import parallel`) or in the importer's own source
## (`from core import parallel`), rather than a module whose declarations they would be. The
## importer's own package's module of the package's name is a module (`from mylib import x`
## inside mylib brings its declaration `x`).
pub func names_modules(importer: str, name: str) -> bool!:
let package = try enclosing(try canonical(importer)) else return false
let (head, rest) = split_package(name)
var owner = package
var path = name
if let named = try named_package(package, head):
if rest.length == 0:
if named.name == package.name and try present(package.source):
return try source_file(try canonical(package.source), head, false) == none
return true
owner = named
path = rest
if not try present(owner.source):
return false
let root = try canonical(owner.source)
if try source_file(root, path, false) != none:
return false
var directory = try new u8[root.length + 1 + path.length] ---
let spelled = try strings.format(directory, f"{root}/{path}")
for i in root.length + 1..<spelled.length:
if spelled.bytes[i] == '.': directory[i] = '/'
return try present(spelled)
## Whether the importer's package has a module of its own named `name`, which a standard
## module's name would hide (§16.6).
pub func has_local_module(importer: str, name: str) -> bool!:
let package = try enclosing(try canonical(importer)) else return false
if not try present(package.source):
return false
return try source_file(try canonical(package.source), name, true) != none
## The identifier of the package a build with source root `source_root` builds, whose
## modules keep their bare names (§16.3); empty outside a package.
pub func building_package(source_root: str) -> str!:
if not try present(source_root):
return ""
let built = try enclosing(try in_arena.join(try canonical(source_root), "x")) else return ""
return built.name
## Where import `name` of the file `importer` leads, for every compiler that resolves
## imports (the Base checker and Luce), `source_root` being where the importer's search
## stands (base.md §16.3, §16.4). In order:
## - a package's module behind its identifier, `luce_ui.ui`, public unless it is the
## importer's own package's;
## - for a file inside a package's source root, the module at that path under the root;
## - for a file of a package outside its source root, a test under `tests/`, the package's
## module first, then as for a file in no package;
## - for a file in no package, a module beside it (never the file itself), then under
## `source_root` and each directory enclosing it, where a directory holding a
## `package.prisma` offers that package's modules.
## `base_only` leaves `.luc` modules out. `built` names the package being built, whose
## modules keep bare names; empty, it is the package enclosing `source_root`. The
## resolution's root is where the module was found.
pub func resolve(importer: str, source_root: str, name: str, base_only: bool = false, built: str = "") -> Resolution!:
let building = built if built.length > 0 else try building_package(source_root)
if let external = try package_import(importer, name, building, base_only):
return external
let input = try canonical(importer)
let here = try canonical(paths.directory(input))
if let package = try enclosing(input):
if try present(package.source):
let source = try canonical(package.source)
if let path = try source_file(source, name, base_only):
return Resolution(try build_name(package, name, building), path, source)
if contained(input, source):
error(invalid, "the module is neither this package's nor a public module of a package it depends on")
# a file in no package, or one of a package outside its source root (a program under
# `tests/`): beside it, then up from where the search stands
if let path = try beside(here, name, base_only, input):
return Resolution(name, path, here)
var root = try canonical(source_root if source_root.length > 0 else ".")
while true:
if let path = try beside(root, name, base_only, input):
return Resolution(name, path, root)
let manifest_path = try in_arena.join(root, "package.prisma")
if try present(manifest_path):
let package = try read_package(manifest_path)
if let path = try source_file(package.source, name, base_only):
return Resolution(name, path, try canonical(package.source))
let parent = paths.directory(root)
if parent == root or parent.length == 0:
break
root = parent
# nothing else of the name: a file importing itself, which the loader reports as the
# cycle it is
if let path = try source_file(here, name, base_only):
return Resolution(name, path, here)
error(invalid, "the module is neither this package's nor a public module of a package it depends on")
# mark: A sealed root ========================================================================
## Where import `name` of the file `importer` leads under a sealed root, an embedded
## checker's (docs/EMBEDDING.md): the module at that path under `root`, a `.lucb` file or a
## directory of fragments, and nowhere else. A directory directly under the root is a
## package, as `luce_std/` is: its modules import each other by their paths in it first, as
## a package's own modules do (`resolve`), and such a module is named by its path under the
## root, `luce_std.math`, the name `from luce_std import math` gives it. No manifest is read.
pub func resolve_sealed(root: str, importer: str, name: str) -> Resolution!:
if not identifier(name):
error(invalid, "an import must name a dotted module")
if let package = sealed_package(root, importer):
if let path = try source_file(try in_arena.join(root, package), name, true):
return Resolution(try strings.join([package, ".", name], ""), path, root)
let path = try source_file(root, name, true) else error(invalid, "the checked directory has no module of this name")
return Resolution(name, path, root)
## Whether module `name`, as the file `importer` would import it, is in a sealed root: in
## the importer's package there, or at its path under the root.
pub func sealed_has(root: str, importer: str, name: str) -> bool!:
if not identifier(name):
return false
if let package = sealed_package(root, importer):
if try source_file(try in_arena.join(root, package), name, true) != none:
return true
return try source_file(root, name, true) != none
## Whether `from name import ...` in the file `importer` under a sealed root brings modules:
## `name` is a directory of modules there (`from luce_kernel import lib`), not a module of
## its own; in the importer's package first, as `resolve_sealed` looks.
pub func sealed_names_modules(root: str, importer: str, name: str) -> bool!:
if not identifier(name):
return false
if let package = sealed_package(root, importer):
let own = try in_arena.join(root, package)
if try source_file(own, name, true) != none:
return false
if try present(try directory_of(own, name)):
return true
if try source_file(root, name, true) != none:
return false
return try present(try directory_of(root, name))
## The package under a sealed root that the file `importer` belongs to: the directory
## directly under `root` that holds it (`luce_std` for `root/luce_std/math32.lucb`); none
## for a module at the root's top, or a file outside it.
func sealed_package(root: str, importer: str) -> str?:
if not contained(importer, root) or importer.length == root.length:
return none
let relative = (str)importer.bytes[root.length + 1..]
let slash = strings.find(relative, "/") else return none
return (str)relative.bytes[..<slash]
## The directory a dotted `name` spells under `root`: `root/core/parallel` for `core.parallel`.
func directory_of(root: str, name: str) -> str!:
var directory = try new u8[root.length + 1 + name.length] ---
let spelled = try strings.format(directory, f"{root}/{name}")
for i in root.length + 1..<spelled.length:
if spelled.bytes[i] == '.': directory[i] = '/'
return spelled
## `path`, a canonical module path, spelled as `importer` is: relative to the working
## directory when the importer's path is relative and `path` lies below it, else as is.
pub func spelled_like(path: str, importer: str) -> str!:
if importer.length == 0 or paths.is_absolute(importer):
return path
let here = try canonical(".")
if contained(path, here) and path.length > here.length:
return (str)path.bytes[here.length + 1..]
return path
## Module `name` under `directory` unless it is the file `importer` itself: an import of
## its own name looks past it first (`tests/area.lucb` importing `area`).
func beside(directory: str, name: str, base_only: bool, importer: str) -> str?!:
let path = try source_file(directory, name, base_only) else return none
if path == importer:
return none
return path
## The module's canonical dotted name inside its own package source root.
pub func name_of(path: str) -> str!:
if not (try present(path)):
return paths.stem(path)
let input = try canonical(path)
if let package = try enclosing(input):
if try present(package.source):
let root = try canonical(package.source)
if contained(input, root) and input.length > root.length:
return try dotted((str)input.bytes[root.length + 1..])
return paths.stem(input)
## The name of the program whose entry is `path`: its module name, except for an entry
## outside its package's source root named like one of the package's modules, a test
## `tests/area.lucb` of module `area`, which is named by its path under the package root
## (`tests.area`), so that it and the module it imports stay two modules.
pub func entry_name(path: str) -> str!:
let name = try name_of(path)
if not (try present(path)):
return name
let input = try canonical(path)
let package = try enclosing(input) else return name
if not try present(package.source):
return name
let source = try canonical(package.source)
if contained(input, source) or try source_file(source, name, true) == none:
return name
let top = try canonical(package.root)
return try dotted((str)input.bytes[top.length + 1..])
## `relative`, a file's path below a root, as a dotted module name without its extension.
func dotted(relative: str) -> str!:
let extension = paths.extension(relative)
let size = relative.length - extension.length
let name = try new u8[size] ---
for index in 0..<size:
name[index] = '.' if relative.bytes[index] == '/' else relative.bytes[index]
return (str)name
## The identifier of the package that owns the file at `path`: the nearest manifest's, or
## `fallback` outside a package. It is the identity of the module's error codes (§11.3).
pub func package_of(path: str, fallback: str) -> str!:
if let package = try enclosing(try canonical(path)):
if package.name.length > 0:
return package.name
return fallback