parser_spec.lua (15347B)
1 local t = require('test.unit.testutil') 2 local itp = t.gen_itp(it) 3 local t_viml = require('test.unit.viml.testutil') 4 5 local make_enum_conv_tab = t.make_enum_conv_tab 6 local child_call_once = t.child_call_once 7 local alloc_log_new = t.alloc_log_new 8 local kvi_destroy = t.kvi_destroy 9 local conv_enum = t.conv_enum 10 local debug_log = t.debug_log 11 local ptr2key = t.ptr2key 12 local cimport = t.cimport 13 local ffi = t.ffi 14 local neq = t.neq 15 local eq = t.eq 16 local mergedicts_copy = t.mergedicts_copy 17 local format_string = require('test.format_string').format_string 18 local format_luav = require('test.format_string').format_luav 19 local intchar2lua = t.intchar2lua 20 local dictdiff = t.dictdiff 21 22 local conv_ccs = t_viml.conv_ccs 23 local new_pstate = t_viml.new_pstate 24 local conv_cmp_type = t_viml.conv_cmp_type 25 local pstate_set_str = t_viml.pstate_set_str 26 local conv_expr_asgn_type = t_viml.conv_expr_asgn_type 27 28 local lib = cimport('./src/nvim/viml/parser/expressions.h', './src/nvim/syntax.h') 29 30 local alloc_log = alloc_log_new() 31 32 local predefined_hl_defs = { 33 -- From highlight_init_both 34 Conceal = true, 35 Cursor = true, 36 lCursor = true, 37 DiffText = true, 38 ErrorMsg = true, 39 IncSearch = true, 40 ModeMsg = true, 41 NonText = true, 42 PmenuSbar = true, 43 StatusLine = true, 44 StatusLineNC = true, 45 TabLineFill = true, 46 TabLineSel = true, 47 TermCursor = true, 48 VertSplit = true, 49 WildMenu = true, 50 WinSeparator = true, 51 EndOfBuffer = true, 52 QuickFixLine = true, 53 Substitute = true, 54 Whitespace = true, 55 Error = true, 56 Todo = true, 57 String = true, 58 Character = true, 59 Number = true, 60 Boolean = true, 61 Float = true, 62 Function = true, 63 Conditional = true, 64 Repeat = true, 65 Label = true, 66 Operator = true, 67 Keyword = true, 68 Exception = true, 69 Include = true, 70 Define = true, 71 Macro = true, 72 PreCondit = true, 73 StorageClass = true, 74 Structure = true, 75 Typedef = true, 76 Tag = true, 77 SpecialChar = true, 78 Delimiter = true, 79 SpecialComment = true, 80 Debug = true, 81 82 -- From highlight_init_(dark|light) 83 ColorColumn = true, 84 CursorColumn = true, 85 CursorLine = true, 86 CursorLineNr = true, 87 DiffAdd = true, 88 DiffChange = true, 89 DiffDelete = true, 90 Directory = true, 91 FoldColumn = true, 92 Folded = true, 93 LineNr = true, 94 MatchParen = true, 95 MoreMsg = true, 96 Pmenu = true, 97 PmenuSel = true, 98 PmenuThumb = true, 99 Question = true, 100 Search = true, 101 SignColumn = true, 102 SpecialKey = true, 103 SpellBad = true, 104 SpellCap = true, 105 SpellLocal = true, 106 SpellRare = true, 107 TabLine = true, 108 Title = true, 109 Visual = true, 110 WarningMsg = true, 111 Normal = true, 112 Comment = true, 113 Constant = true, 114 Special = true, 115 Identifier = true, 116 Statement = true, 117 PreProc = true, 118 Type = true, 119 Underlined = true, 120 Ignore = true, 121 } 122 123 local nvim_hl_defs = {} 124 125 child_call_once(function() 126 local i = 0 127 while lib.highlight_init_cmdline[i] ~= nil do 128 local hl_args = lib.highlight_init_cmdline[i] 129 local s = ffi.string(hl_args) 130 local err, msg = pcall(function() 131 if s:sub(1, 13) == 'default link ' then 132 local new_grp, grp_link = s:match('^default link (%w+) (%w+)$') 133 neq(nil, new_grp) 134 -- Note: group to link to must be already defined at the time of 135 -- linking, otherwise it will be created as cleared. So existence 136 -- of the group is checked here and not in the next pass over 137 -- nvim_hl_defs. 138 eq(true, not not (nvim_hl_defs[grp_link] or predefined_hl_defs[grp_link])) 139 eq(false, not not (nvim_hl_defs[new_grp] or predefined_hl_defs[new_grp])) 140 nvim_hl_defs[new_grp] = { 'link', grp_link } 141 else 142 local new_grp, grp_args = s:match('^(%w+) (.*)') 143 neq(nil, new_grp) 144 eq(false, not not (nvim_hl_defs[new_grp] or predefined_hl_defs[new_grp])) 145 nvim_hl_defs[new_grp] = { 'definition', grp_args } 146 end 147 end) 148 if not err then 149 msg = format_string('Error while processing string %s at position %u:\n%s', s, i, msg) 150 error(msg) 151 end 152 i = i + 1 153 end 154 for k, _ in ipairs(nvim_hl_defs) do 155 eq('Nvim', k:sub(1, 4)) 156 -- NvimInvalid 157 -- 12345678901 158 local err, msg = pcall(function() 159 if k:sub(5, 11) == 'Invalid' then 160 neq(nil, nvim_hl_defs['Nvim' .. k:sub(12)]) 161 else 162 neq(nil, nvim_hl_defs['NvimInvalid' .. k:sub(5)]) 163 end 164 end) 165 if not err then 166 msg = format_string('Error while processing group %s:\n%s', k, msg) 167 error(msg) 168 end 169 end 170 end) 171 172 local function hls_to_hl_fs(hls) 173 local ret = {} 174 local next_col = 0 175 for i, v in ipairs(hls) do 176 local group, line, col, str = v:match('^Nvim([a-zA-Z]+):(%d+):(%d+):(.*)$') 177 col = tonumber(col) 178 line = tonumber(line) 179 assert(line == 0) 180 local col_shift = col - next_col 181 assert(col_shift >= 0) 182 next_col = col + #str 183 ret[i] = format_string( 184 'hl(%r, %r%s)', 185 group, 186 str, 187 (col_shift == 0 and '' or (', %u'):format(col_shift)) 188 ) 189 end 190 return ret 191 end 192 193 local function format_check(expr, format_check_data, opts) 194 -- That forces specific order. 195 local zflags = opts.flags[1] 196 local zdata = format_check_data[zflags] 197 local dig_len 198 if opts.funcname then 199 print(format_string('\n%s(%r, {', opts.funcname, expr)) 200 dig_len = #opts.funcname + 2 201 else 202 print(format_string('\n_check_parsing(%r, %r, {', opts, expr)) 203 dig_len = #"_check_parsing(, '" + #(format_string('%r', opts)) 204 end 205 local digits = ' --' .. (' '):rep(dig_len - #' --') 206 local digits2 = digits:sub(1, -10) 207 for i = 0, #expr - 1 do 208 if i % 10 == 0 then 209 digits2 = ('%s%10u'):format(digits2, i / 10) 210 end 211 digits = ('%s%u'):format(digits, i % 10) 212 end 213 print(digits) 214 if #expr > 10 then 215 print(digits2) 216 end 217 if zdata.ast.len then 218 print((' len = %u,'):format(zdata.ast.len)) 219 end 220 print(' ast = ' .. format_luav(zdata.ast.ast, ' ') .. ',') 221 if zdata.ast.err then 222 print(' err = {') 223 print(' arg = ' .. format_luav(zdata.ast.err.arg) .. ',') 224 print(' msg = ' .. format_luav(zdata.ast.err.msg) .. ',') 225 print(' },') 226 end 227 print('}, {') 228 for _, v in ipairs(zdata.hl_fs) do 229 print(' ' .. v .. ',') 230 end 231 local diffs = {} 232 local diffs_num = 0 233 for flags, v in pairs(format_check_data) do 234 if flags ~= zflags then 235 diffs[flags] = dictdiff(zdata, v) 236 if diffs[flags] then 237 if flags == 3 + zflags then 238 if 239 dictdiff(format_check_data[1 + zflags], format_check_data[3 + zflags]) == nil 240 or dictdiff(format_check_data[2 + zflags], format_check_data[3 + zflags]) == nil 241 then 242 diffs[flags] = nil 243 else 244 diffs_num = diffs_num + 1 245 end 246 else 247 diffs_num = diffs_num + 1 248 end 249 end 250 end 251 end 252 if diffs_num ~= 0 then 253 print('}, {') 254 local flags = 1 255 while diffs_num ~= 0 do 256 if diffs[flags] then 257 diffs_num = diffs_num - 1 258 local diff = diffs[flags] 259 print((' [%u] = {'):format(flags)) 260 if diff.ast then 261 print(' ast = ' .. format_luav(diff.ast, ' ') .. ',') 262 end 263 if diff.hl_fs then 264 print(' hl_fs = ' .. format_luav(diff.hl_fs, ' ', { 265 literal_strings = true, 266 }) .. ',') 267 end 268 print(' },') 269 end 270 flags = flags + 1 271 end 272 end 273 print('})') 274 end 275 276 local east_node_type_tab 277 make_enum_conv_tab( 278 lib, 279 { 280 'kExprNodeMissing', 281 'kExprNodeOpMissing', 282 'kExprNodeTernary', 283 'kExprNodeTernaryValue', 284 'kExprNodeRegister', 285 'kExprNodeSubscript', 286 'kExprNodeListLiteral', 287 'kExprNodeUnaryPlus', 288 'kExprNodeBinaryPlus', 289 'kExprNodeNested', 290 'kExprNodeCall', 291 'kExprNodePlainIdentifier', 292 'kExprNodePlainKey', 293 'kExprNodeComplexIdentifier', 294 'kExprNodeUnknownFigure', 295 'kExprNodeLambda', 296 'kExprNodeDictLiteral', 297 'kExprNodeCurlyBracesIdentifier', 298 'kExprNodeComma', 299 'kExprNodeColon', 300 'kExprNodeArrow', 301 'kExprNodeComparison', 302 'kExprNodeConcat', 303 'kExprNodeConcatOrSubscript', 304 'kExprNodeInteger', 305 'kExprNodeFloat', 306 'kExprNodeSingleQuotedString', 307 'kExprNodeDoubleQuotedString', 308 'kExprNodeOr', 309 'kExprNodeAnd', 310 'kExprNodeUnaryMinus', 311 'kExprNodeBinaryMinus', 312 'kExprNodeNot', 313 'kExprNodeMultiplication', 314 'kExprNodeDivision', 315 'kExprNodeMod', 316 'kExprNodeOption', 317 'kExprNodeEnvironment', 318 'kExprNodeAssignment', 319 }, 320 'kExprNode', 321 function(ret) 322 east_node_type_tab = ret 323 end 324 ) 325 326 local function conv_east_node_type(typ) 327 return conv_enum(east_node_type_tab, typ) 328 end 329 330 local eastnodelist2lua 331 332 local function eastnode2lua(pstate, eastnode, checked_nodes) 333 local key = ptr2key(eastnode) 334 if checked_nodes[key] then 335 checked_nodes[key].duplicate_key = key 336 return { duplicate = key } 337 end 338 local typ = conv_east_node_type(eastnode.type) 339 local ret = {} 340 checked_nodes[key] = ret 341 ret.children = eastnodelist2lua(pstate, eastnode.children, checked_nodes) 342 local str = pstate_set_str(pstate, eastnode.start, eastnode.len) 343 local ret_str 344 if str.error then 345 ret_str = 'error:' .. str.error 346 else 347 ret_str = ('%u:%u:%s'):format(str.start.line, str.start.col, str.str) 348 end 349 if typ == 'Register' then 350 typ = typ .. ('(name=%s)'):format(tostring(intchar2lua(eastnode.data.reg.name))) 351 elseif typ == 'PlainIdentifier' then 352 typ = typ 353 .. ('(scope=%s,ident=%s)'):format( 354 tostring(intchar2lua(eastnode.data.var.scope)), 355 ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len) 356 ) 357 elseif typ == 'PlainKey' then 358 typ = typ 359 .. ('(key=%s)'):format(ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) 360 elseif 361 typ == 'UnknownFigure' 362 or typ == 'DictLiteral' 363 or typ == 'CurlyBracesIdentifier' 364 or typ == 'Lambda' 365 then 366 typ = typ 367 .. ('(%s)'):format( 368 (eastnode.data.fig.type_guesses.allow_lambda and '\\' or '-') 369 .. (eastnode.data.fig.type_guesses.allow_dict and 'd' or '-') 370 .. (eastnode.data.fig.type_guesses.allow_ident and 'i' or '-') 371 ) 372 elseif typ == 'Comparison' then 373 typ = typ 374 .. ('(type=%s,inv=%u,ccs=%s)'):format( 375 conv_cmp_type(eastnode.data.cmp.type), 376 eastnode.data.cmp.inv and 1 or 0, 377 conv_ccs(eastnode.data.cmp.ccs) 378 ) 379 elseif typ == 'Integer' then 380 typ = typ .. ('(val=%u)'):format(tonumber(eastnode.data.num.value)) 381 elseif typ == 'Float' then 382 typ = typ .. format_string('(val=%e)', tonumber(eastnode.data.flt.value)) 383 elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then 384 if eastnode.data.str.value == nil then 385 typ = typ .. '(val=NULL)' 386 else 387 local s = ffi.string(eastnode.data.str.value, eastnode.data.str.size) 388 typ = format_string('%s(val=%q)', typ, s) 389 end 390 elseif typ == 'Option' then 391 typ = ('%s(scope=%s,ident=%s)'):format( 392 typ, 393 tostring(intchar2lua(eastnode.data.opt.scope)), 394 ffi.string(eastnode.data.opt.ident, eastnode.data.opt.ident_len) 395 ) 396 elseif typ == 'Environment' then 397 typ = ('%s(ident=%s)'):format( 398 typ, 399 ffi.string(eastnode.data.env.ident, eastnode.data.env.ident_len) 400 ) 401 elseif typ == 'Assignment' then 402 typ = ('%s(%s)'):format(typ, conv_expr_asgn_type(eastnode.data.ass.type)) 403 end 404 ret_str = typ .. ':' .. ret_str 405 local can_simplify = not ret.children 406 if can_simplify then 407 ret = ret_str 408 else 409 ret[1] = ret_str 410 end 411 return ret 412 end 413 414 eastnodelist2lua = function(pstate, eastnode, checked_nodes) 415 local ret = {} 416 while eastnode ~= nil do 417 ret[#ret + 1] = eastnode2lua(pstate, eastnode, checked_nodes) 418 eastnode = eastnode.next 419 end 420 if #ret == 0 then 421 ret = nil 422 end 423 return ret 424 end 425 426 local function east2lua(str, pstate, east) 427 local checked_nodes = {} 428 local len = tonumber(pstate.pos.col) 429 if pstate.pos.line == 1 then 430 len = tonumber(pstate.reader.lines.items[0].size) 431 end 432 if type(str) == 'string' and len == #str then 433 len = nil 434 end 435 return { 436 err = east.err.msg ~= nil and { 437 msg = ffi.string(east.err.msg), 438 arg = ffi.string(east.err.arg, east.err.arg_len), 439 } or nil, 440 len = len, 441 ast = eastnodelist2lua(pstate, east.root, checked_nodes), 442 } 443 end 444 445 local function phl2lua(pstate) 446 local ret = {} 447 for i = 0, (tonumber(pstate.colors.size) - 1) do 448 local chunk = pstate.colors.items[i] 449 local chunk_tbl = pstate_set_str(pstate, chunk.start, chunk.end_col - chunk.start.col, { 450 group = ffi.string(chunk.group), 451 }) 452 ret[i + 1] = ('%s:%u:%u:%s'):format( 453 chunk_tbl.group, 454 chunk_tbl.start.line, 455 chunk_tbl.start.col, 456 chunk_tbl.str 457 ) 458 end 459 return ret 460 end 461 462 describe('Expressions parser', function() 463 local function _check_parsing(opts, str, exp_ast, exp_highlighting_fs, nz_flags_exps) 464 local zflags = opts.flags[1] 465 nz_flags_exps = nz_flags_exps or {} 466 local format_check_data = {} 467 for _, flags in ipairs(opts.flags) do 468 debug_log(('Running test case (%s, %u)'):format(str, flags)) 469 local err, msg = pcall(function() 470 if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then 471 print(str, flags) 472 end 473 alloc_log:check({}) 474 475 local pstate = new_pstate({ str }) 476 local east = lib.viml_pexpr_parse(pstate, flags) 477 local ast = east2lua(str, pstate, east) 478 local hls = phl2lua(pstate) 479 if exp_ast == nil then 480 format_check_data[flags] = { ast = ast, hl_fs = hls_to_hl_fs(hls) } 481 else 482 local exps = { 483 ast = exp_ast, 484 hl_fs = exp_highlighting_fs, 485 } 486 local add_exps = nz_flags_exps[flags] 487 if not add_exps and flags == 3 + zflags then 488 add_exps = nz_flags_exps[1 + zflags] or nz_flags_exps[2 + zflags] 489 end 490 if add_exps then 491 if add_exps.ast then 492 exps.ast = mergedicts_copy(exps.ast, add_exps.ast) 493 end 494 if add_exps.hl_fs then 495 exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs) 496 end 497 end 498 eq(exps.ast, ast) 499 if exp_highlighting_fs then 500 local exp_highlighting = {} 501 local next_col = 0 502 for i, h in ipairs(exps.hl_fs) do 503 exp_highlighting[i], next_col = h(next_col) 504 end 505 eq(exp_highlighting, hls) 506 end 507 end 508 lib.viml_pexpr_free_ast(east) 509 kvi_destroy(pstate.colors) 510 alloc_log:clear_tmp_allocs(true) 511 alloc_log:check({}) 512 end) 513 if not err then 514 msg = format_string('Error while processing test (%r, %u):\n%s', str, flags, msg) 515 error(msg) 516 end 517 end 518 if exp_ast == nil then 519 format_check(str, format_check_data, opts) 520 end 521 end 522 local function hl(group, str, shift) 523 return function(next_col) 524 if nvim_hl_defs['Nvim' .. group] == nil then 525 error(('Unknown group: Nvim%s'):format(group)) 526 end 527 local col = next_col + (shift or 0) 528 return (('%s:%u:%u:%s'):format('Nvim' .. group, 0, col, str)), (col + #str) 529 end 530 end 531 local function fmtn(typ, args, rest) 532 return ('%s(%s)%s'):format(typ, args, rest) 533 end 534 require('test.unit.viml.expressions.parser_tests')(itp, _check_parsing, hl, fmtn) 535 end)