Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-js

Source of release 0.6.0

0.6.0 / src / engine / tests_tokenizer.lucb

#==============================================================================================
#
#   tests_tokenizer - Unit tests of the tokenizer and emitter helpers that need no runtime
#
#   DESCRIPTION:
#       Line and column numbers, the keyword token order, the look-ahead scanner
#       (simple_next_token), skip_shebang and JS_DetectModule, the variable and scope
#       lookups on a FunctionDef laid out by hand, and the byte emitters on a bare
#       bytecode buffer. Nothing here makes a runtime: the FunctionDef's context is never
#       read by the functions tested.
#
#==============================================================================================

# mark: Source positions ========================================================================

test "get_line_col counts lines and UTF-8 characters":
    let text = "ab\ncd\u{e9}\nf"
    var col: i32 = -1
    assert(get_line_col(&col, text.bytes.data, 0) == 0 and col == 0)
    assert(get_line_col(&col, text.bytes.data, 2) == 0 and col == 2)
    assert(get_line_col(&col, text.bytes.data, 3) == 1 and col == 0)
    # 'é' is two bytes and one column
    assert(get_line_col(&col, text.bytes.data, 7) == 1 and col == 3)
    assert(get_line_col(&col, text.bytes.data, text.length) == 2 and col == 1)

test "get_line_col_cached agrees with get_line_col in both directions":
    let text = "one\ntwo \u{e9}\n\nfour five"
    let start = text.bytes.data
    var cache = GetLineColCache(ptr = start, line_num = 0, col_num = 0, buf_start = start)
    let offsets: usize[8] = [5, 12, 13, 20, 2, 9, 0, 17]
    for offset in offsets:
        var col: i32 = -1
        var expect_col: i32 = -2
        let line = get_line_col_cached(&cache, &col, start + offset)
        let expect_line = get_line_col(&expect_col, start, offset)
        assert(line == expect_line)
        assert(col == expect_col)

# mark: Tokens ==================================================================================

test "keyword tokens follow the keyword atoms":
    let first = (i32)tok_first_keyword
    assert((TokenType)((i32)atom_null_keyword - 1 + first) == TokenType.null)
    assert((TokenType)((i32)atom_with - 1 + first) == TokenType.with_)
    assert((TokenType)((i32)atom_super - 1 + first) == TokenType.super)
    assert((TokenType)((i32)atom_last_keyword - 1 + first) == TokenType.super)
    assert((TokenType)((i32)atom_yield - 1 + first) == TokenType.yield)
    assert((TokenType)((i32)atom_await - 1 + first) == tok_last_keyword)
    assert(tok_is_keyword(TokenType.null) and tok_is_keyword(TokenType.await))
    assert(not tok_is_keyword(TokenType.eof) and not tok_is_keyword(TokenType.of))
    assert(not tok_is_keyword(TokenType.lparen))

test "one-character tokens are their character":
    assert((i32)TokenType.lparen == (i32)'(')
    assert((i32)TokenType.rbrace == (i32)'}')
    assert((i32)TokenType.semicolon == (i32)';')
    assert((i32)TokenType.newline == (i32)'\n')
    assert(tok_char(TokenType.colon) == ':')

# The token simple_next_token answers at the start of `text`.
func simple_token(text: str, no_line_terminator: bool) -> TokenType:
    var p = text.bytes.data
    return simple_next_token(&p, no_line_terminator)

test "simple_next_token skips spaces and comments":
    assert(simple_token("  /* c */ => x", false) == TokenType.arrow)
    assert(simple_token("\n\t// line\n  :", false) == TokenType.colon)
    assert(simple_token(" // line\n:", true) == TokenType.newline)
    assert(simple_token("/* a\nb */ x", true) == TokenType.newline)
    assert(simple_token("/* a b */ x", true) == TokenType.ident)
    assert(simple_token("\u{2028}x", true) == TokenType.newline)
    assert(simple_token("\u{a0}x", true) == TokenType.ident)
    assert(simple_token("= 1", false) == TokenType.assign)
    assert(simple_token("/ 2", false) == TokenType.slash)
    assert(simple_token("", false) == (TokenType)0)

test "simple_next_token recognizes its words":
    assert(simple_token(" in x", false) == TokenType.in_)
    assert(simple_token("inner", false) == TokenType.ident)
    assert(simple_token("of", false) == TokenType.of)
    assert(simple_token("export {}", false) == TokenType.export_)
    assert(simple_token("function f", false) == TokenType.function)
    assert(simple_token("functional", false) == TokenType.ident)
    assert(simple_token("\u{e9}t\u{e9}", false) == TokenType.ident)
    var p = "import x".bytes.data
    assert(simple_next_token(&p, false) == TokenType.import_)
    assert(*p == ' ')

test "match_identifier needs the whole word":
    assert(match_identifier("n x".bytes.data, "n"))
    assert(match_identifier("n".bytes.data, "n"))
    assert(not match_identifier("nx".bytes.data, "n"))
    assert(not match_identifier("n\u{e9}".bytes.data, "n"))
    assert(not match_identifier("m".bytes.data, "n"))

test "skip_shebang skips the first line only when it starts with #!":
    let text = "#!/usr/bin/env qjs\nlet x"
    var p = text.bytes.data
    skip_shebang(&p, text.bytes.data + text.length)
    assert(*p == '\n')
    let other = "# not\nx"
    var q = other.bytes.data
    skip_shebang(&q, other.bytes.data + other.length)
    assert(q == other.bytes.data)

# Whether js_detect_module finds a module in `text`.
func detect(text: str) -> bool:
    return js_detect_module(text.bytes.data, text.length)

test "JS_DetectModule finds import and export":
    assert(detect("import x from 'y'"))
    assert(detect("/* c */ import {a} from 'b'"))
    assert(detect("export default 1"))
    assert(detect("#!/usr/bin/env qjs\n// c\nexport {}"))
    assert(not detect("import('x')"))
    assert(not detect("import.meta"))
    assert(not detect("var x = 1"))
    assert(not detect(""))

# mark: Variables and scopes ====================================================================

# A zeroed FunctionDef over `vars` and `scope_count` scopes; the caller frees it with
# cutils.c_free.
func test_function_def(vars: VarDef*?, var_count: i32, scope_count: i32) -> FunctionDef*:
    let raw = cutils.malloc(memory.size_of(FunctionDef)) else trap("out of memory")
    _ = cutils.memset(raw, 0, memory.size_of(FunctionDef))
    let fd = (FunctionDef*)raw
    fd.scopes = &fd.def_scope_array[0]
    fd.scope_size = 4
    fd.scope_count = scope_count
    fd.vars = vars
    fd.var_count = var_count
    fd.var_size = var_count
    fd.func_var_idx = -1
    return fd

# Give `fd` the `arg_count` arguments at `args`.
func set_test_args(fd: FunctionDef*, args: VarDef*, arg_count: i32):
    fd.args = args
    fd.arg_count = arg_count
    fd.arg_size = arg_count

# Give `fd` the `count` global variables at `globals`.
func set_test_globals(fd: FunctionDef*, globals: GlobalVar*, count: i32):
    fd.global_vars = globals
    fd.global_var_count = count
    fd.global_var_size = count

test "variables are found by scope":
    # scope 0: the function; scope 1: a block in it; scope 2: a block in scope 1
    var vars: VarDef[4]
    vars[0] = VarDef(var_name = atom_length, scope_level = 0, scope_next = 1)
    vars[1] = VarDef(var_name = atom_name, scope_level = 1, scope_next = -1,
                     is_lexical = true)
    vars[2] = VarDef(var_name = atom_length, scope_level = 2, scope_next = 1,
                     is_lexical = true)
    vars[3] = VarDef(var_name = atom_value, scope_level = 0, scope_next = 2)
    let fd = test_function_def(&vars[0], 4, 3)
    defer cutils.c_free(fd)
    fd_scope(fd, 0).parent = -1
    fd_scope(fd, 0).first = -1
    fd_scope(fd, 1).parent = 0
    fd_scope(fd, 1).first = 1
    fd_scope(fd, 2).parent = 1
    fd_scope(fd, 2).first = 2

    assert(find_var(fd.ctx, fd, atom_length) == 0)
    assert(find_var(fd.ctx, fd, atom_name) == -1)
    assert(find_var_in_scope(fd.ctx, fd, atom_name, 1) == 1)
    assert(find_var_in_scope(fd.ctx, fd, atom_length, 2) == 2)
    assert(find_var_in_scope(fd.ctx, fd, atom_name, 2) == -1)
    assert(is_child_scope(fd.ctx, fd, 2, 0) and is_child_scope(fd.ctx, fd, 1, 1))
    assert(not is_child_scope(fd.ctx, fd, 0, 1))
    # 'value' is a var of scope 2 (scope_next holds its scope)
    assert(find_var_in_child_scope(fd.ctx, fd, atom_value, 1) == 3)
    assert(find_var_in_child_scope(fd.ctx, fd, atom_value, 0) == 3)
    assert(find_var_in_child_scope(fd.ctx, fd, atom_length, 2) == -1)
    assert(find_lexical_decl(fd.ctx, fd, atom_name, 2, false) == 1)
    assert(find_lexical_decl(fd.ctx, fd, atom_value, 2, false) == -1)
    assert(get_first_lexical_var(fd, 2) == 2)
    assert(get_first_lexical_var(fd, 0) == -1)

test "arguments come after the variables and carry the argument offset":
    var args: VarDef[2]
    args[0] = VarDef(var_name = atom_length)
    args[1] = VarDef(var_name = atom_name)
    let fd = test_function_def(none, 0, 1)
    defer cutils.c_free(fd)
    set_test_args(fd, &args[0], 2)
    assert(find_arg(fd.ctx, fd, atom_name) == (1 | argument_var_offset))
    assert(find_var(fd.ctx, fd, atom_length) == (0 | argument_var_offset))
    assert(find_arg(fd.ctx, fd, atom_value) == -1)

test "global variables":
    var globals: GlobalVar[2]
    globals[0] = GlobalVar(cpool_idx = -1, var_name = atom_length)
    globals[1] = GlobalVar(cpool_idx = -1, var_name = atom_name, is_lexical = true)
    let fd = test_function_def(none, 0, 1)
    defer cutils.c_free(fd)
    set_test_globals(fd, &globals[0], 2)
    assert(find_global_var(fd, atom_length) == &globals[0])
    assert(find_lexical_global_var(fd, atom_length) == none)
    assert(find_lexical_global_var(fd, atom_name) == &globals[1])
    fd.is_eval = true
    fd.eval_type = eval_type_global
    assert(find_lexical_decl(fd.ctx, fd, atom_name, -1, false) == global_var_offset)

# mark: Emitting ================================================================================

test "emitted bytes and the live code test":
    let fd = test_function_def(none, 0, 1)
    defer cutils.c_free(fd)
    cutils.dbuf_init(&fd.byte_code)
    defer cutils.dbuf_free(&fd.byte_code)
    fd.last_opcode_pos = -1
    let source = "x = 1"
    let raw = cutils.malloc(memory.size_of(ParseState)) else trap("out of memory")
    defer cutils.c_free(raw)
    _ = cutils.memset(raw, 0, memory.size_of(ParseState))
    let s = (ParseState*)raw
    s.cur_func = fd
    s.buf_start = source.bytes.data

    assert(get_prev_opcode(fd) == (u8)Op.invalid)
    assert(js_is_live_code(s))
    emit_op(s, (u8)Op.push_i32)
    emit_u32(s, 0x01020304)
    emit_u16(s, 0x0506)
    emit_u8(s, 7)
    assert(fd.byte_code.size == 8)
    assert(get_prev_opcode(fd) == (u8)Op.push_i32)
    let buf = fd.byte_code.buf else trap("no bytecode")
    assert(cutils.get_u32(buf + 1) == 0x01020304)
    assert(cutils.get_u16(buf + 5) == 0x0506)
    assert(buf[7] == 7)
    emit_op(s, (u8)Op.return_)
    assert(not js_is_live_code(s))
    emit_op(s, (u8)Op.drop)
    assert(js_is_live_code(s))

    # a line number opcode only when the source position moves
    emit_source_pos(s, source.bytes.data + 2)
    assert(fd.byte_code.size == 15)
    emit_source_pos(s, source.bytes.data + 2)
    assert(fd.byte_code.size == 15)
    let buf2 = fd.byte_code.buf else trap("no bytecode")
    assert(buf2[10] == (u8)TempOp.line_num)
    assert(cutils.get_u32(buf2 + 11) == 2)