commit b75a68ff06f805b2a46554038c3bd504c882c248
parent a4c9247a1ad1cac7ea7739de9b5c189005a68c12
Author: bfredl <bjorn.linse@gmail.com>
Date: Wed, 29 Oct 2025 11:37:14 +0100
Merge pull request #36367 from bfredl/metadata_iii
refactor(build): split out metadata step
Diffstat:
4 files changed, 141 insertions(+), 114 deletions(-)
diff --git a/src/gen/gen_api_dispatch.lua b/src/gen/gen_api_dispatch.lua
@@ -1,30 +1,21 @@
-- Generates C code to bridge API <=> Lua.
---
--- Example (manual) invocation:
---
--- make
--- cp build/nvim_version.lua src/nvim/
--- cd src/nvim
--- nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h"
+-- to obtain how the script is invoked, look in build/build.ninja and grep for
+-- "gen_api_dispatch.lua"
local hashy = require 'gen.hashy'
local c_grammar = require('gen.c_grammar')
-- output h file with generated dispatch functions (dispatch_wrappers.generated.h)
local dispatch_outputf = arg[1]
--- output h file with packed metadata (api_metadata.generated.h)
-local api_metadata_outputf = arg[2]
--- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack)
-local mpack_outputf = arg[3]
+-- output file with exported functions metadata
+local exported_funcs_metadata_outputf = arg[2]
+-- output mpack file with raw metadata, for use by gen_eval.lua (funcs_metadata.mpack)
+local eval_funcs_metadata_outputf = arg[3]
local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c
local keysets_outputf = arg[5] -- keysets_defs.generated.h
-local ui_metadata_inputf = arg[6] -- ui events metadata
-local git_version_inputf = arg[7] -- git version header
-local nvim_version_inputf = arg[8] -- nvim version
-local dump_bin_array_inputf = arg[9]
-local dispatch_deprecated_inputf = arg[10]
-local pre_args = 10
-assert(#arg >= pre_args)
+local dispatch_deprecated_inputf = arg[6]
+local pre_args = 6
+assert(#arg >= 6)
local function real_type(type, exported)
local ptype = c_grammar.typed_container:match(type)
@@ -165,8 +156,6 @@ local function add_keyset(val)
}
end
-local ui_options_text = nil
-
-- read each input file, parse and append to the api metadata
for i = pre_args + 1, #arg do
local full_path = arg[i]
@@ -190,12 +179,9 @@ for i = pre_args + 1, #arg do
end
end
- ui_options_text = ui_options_text or text:match('ui_ext_names%[][^{]+{([^}]+)}')
input:close()
end
---- @cast ui_options_text string
-
--- @generic T: table
--- @param orig T
--- @return T
@@ -303,77 +289,11 @@ for _, f in ipairs(functions) do
end
end
-local ui_options = { 'rgb' }
-for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do
- table.insert(ui_options, x)
-end
+local metadata_output = assert(io.open(exported_funcs_metadata_outputf, 'wb'))
+metadata_output:write(vim.mpack.encode(exported_functions))
+metadata_output:close()
--- @type integer[]
-local version = loadfile(nvim_version_inputf)()
-local git_version = io.open(git_version_inputf):read '*a'
-local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
-
-local pieces = {} --- @type string[]
-
--- Naively using mpack.encode({foo=x, bar=y}) will make the build
--- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
-local function fixdict(num)
- if num > 15 then
- error 'implement more dict codes'
- end
- pieces[#pieces + 1] = string.char(128 + num)
-end
-
-local function put(item, item2)
- table.insert(pieces, vim.mpack.encode(item))
- if item2 ~= nil then
- table.insert(pieces, vim.mpack.encode(item2))
- end
-end
-
-fixdict(6)
-
-put('version')
-fixdict(1 + #version)
-for _, item in ipairs(version) do
- -- NB: all items are mandatory. But any error will be less confusing
- -- with placeholder vim.NIL (than invalid mpack data)
- local val = item[2] == nil and vim.NIL or item[2]
- put(item[1], val)
-end
-put('build', version_build)
-
-put('functions', exported_functions)
-put('ui_events')
-table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
-put('ui_options', ui_options)
-
-put('error_types')
-fixdict(2)
-put('Exception', { id = 0 })
-put('Validation', { id = 1 })
-
-put('types')
-local types =
- { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
-fixdict(#types)
-for i, item in ipairs(types) do
- put(item[1])
- fixdict(2)
- put('id', i - 1)
- put('prefix', item[2])
-end
-
-local packed = table.concat(pieces)
---- @type fun(api_metadata: file*, name: string, packed: string)
-local dump_bin_array = loadfile(dump_bin_array_inputf)()
-
--- serialize the API metadata using msgpack and embed into the resulting
--- binary for easy querying by clients
-local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
-dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
-api_metadata_output:close()
-
-- start building the dispatch wrapper output
local output = assert(io.open(dispatch_outputf, 'wb'))
@@ -757,7 +677,7 @@ output:close()
--- @cast functions {[integer]: gen_api_dispatch.Function, keysets: gen_api_dispatch.Keyset[]}
functions.keysets = keysets
-local mpack_output = assert(io.open(mpack_outputf, 'wb'))
+local mpack_output = assert(io.open(eval_funcs_metadata_outputf, 'wb'))
mpack_output:write(vim.mpack.encode(functions))
mpack_output:close()
diff --git a/src/gen/gen_api_metadata.lua b/src/gen/gen_api_metadata.lua
@@ -0,0 +1,83 @@
+local mpack = vim.mpack
+
+assert(#arg == 7)
+local funcs_metadata_inputf = arg[1] -- exported functions metadata
+local ui_metadata_inputf = arg[2] -- ui events metadata
+local ui_options_inputf = arg[3] -- for ui options
+local git_version_inputf = arg[4] -- git version header
+local nvim_version_inputf = arg[5] -- nvim version
+local dump_bin_array_inputf = arg[6]
+local api_metadata_outputf = arg[7]
+
+local version = loadfile(nvim_version_inputf)()
+local git_version = io.open(git_version_inputf):read '*a'
+local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
+
+local text = io.open(ui_options_inputf):read '*a'
+local ui_options_text = text:match('ui_ext_names%[][^{]+{([^}]+)}')
+local ui_options = { 'rgb' }
+for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do
+ table.insert(ui_options, x)
+end
+
+local pieces = {} --- @type string[]
+
+-- Naively using mpack.encode({foo=x, bar=y}) will make the build
+-- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
+local function fixdict(num)
+ if num > 15 then
+ error 'implement more dict codes'
+ end
+ pieces[#pieces + 1] = string.char(128 + num)
+end
+
+local function put(item, item2)
+ table.insert(pieces, mpack.encode(item))
+ if item2 ~= nil then
+ table.insert(pieces, mpack.encode(item2))
+ end
+end
+
+fixdict(6)
+
+put('version')
+fixdict(1 + #version)
+for _, item in ipairs(version) do
+ -- NB: all items are mandatory. But any error will be less confusing
+ -- with placeholder vim.NIL (than invalid mpack data)
+ local val = item[2] == nil and vim.NIL or item[2]
+ put(item[1], val)
+end
+put('build', version_build)
+
+put('functions')
+table.insert(pieces, io.open(funcs_metadata_inputf, 'rb'):read('*all'))
+put('ui_events')
+table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
+put('ui_options', ui_options)
+
+put('error_types')
+fixdict(2)
+put('Exception', { id = 0 })
+put('Validation', { id = 1 })
+
+put('types')
+local types =
+ { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
+fixdict(#types)
+for i, item in ipairs(types) do
+ put(item[1])
+ fixdict(2)
+ put('id', i - 1)
+ put('prefix', item[2])
+end
+
+local packed = table.concat(pieces)
+--- @type fun(api_metadata: file*, name: string, packed: string)
+local dump_bin_array = loadfile(dump_bin_array_inputf)()
+
+-- serialize the API metadata using msgpack and embed into the resulting
+-- binary for easy querying by clients
+local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
+dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
+api_metadata_output:close()
diff --git a/src/gen/gen_steps.zig b/src/gen/gen_steps.zig
@@ -100,32 +100,40 @@ pub fn nvim_gen_sources(
break :ui_step ui_metadata;
};
- const funcs_metadata = api_step: {
+ const eval_funcs_metadata, const exported_funcs_metadata = dispatch_step: {
const gen_step = b.addRunArtifact(nlua0);
gen_step.addFileArg(b.path("src/gen/gen_api_dispatch.lua"));
_ = try gen_header_with_header(b, gen_step, "api/private/dispatch_wrappers.generated.h", nlua0, gen_headers);
- _ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers);
- const funcs_metadata = gen_step.addOutputFileArg("funcs_metadata.mpack");
+ const exported_funcs_metadata = gen_step.addOutputFileArg("exported_funcs_metadata.mpack");
+ const eval_funcs_metadata = gen_step.addOutputFileArg("eval_funcs_metadata.mpack");
_ = gen_header(b, gen_step, "lua_api_c_bindings.generated.h", gen_headers);
_ = gen_header(b, gen_step, "keysets_defs.generated.h", gen_headers);
- gen_step.addFileArg(ui_metadata);
- gen_step.addFileArg(versiondef_git);
- gen_step.addFileArg(version_lua);
- gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua"));
gen_step.addFileArg(b.path("src/nvim/api/dispatch_deprecated.lua"));
// now follows all .h files with exported functions
for (api_headers.items) |h| {
gen_step.addFileArg(h);
}
- break :api_step funcs_metadata;
+ break :dispatch_step .{ eval_funcs_metadata, exported_funcs_metadata };
};
+ {
+ const gen_step = b.addRunArtifact(nlua0);
+ gen_step.addFileArg(b.path("src/gen/gen_api_metadata.lua"));
+ gen_step.addFileArg(exported_funcs_metadata);
+ gen_step.addFileArg(ui_metadata);
+ gen_step.addFileArg(b.path("src/nvim/api/ui.h"));
+ gen_step.addFileArg(versiondef_git);
+ gen_step.addFileArg(version_lua);
+ gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua"));
+ _ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers);
+ }
+
const funcs_data = eval_step: {
const gen_step = b.addRunArtifact(nlua0);
gen_step.addFileArg(b.path("src/gen/gen_eval.lua"));
_ = gen_header(b, gen_step, "funcs.generated.h", gen_headers);
- gen_step.addFileArg(funcs_metadata);
+ gen_step.addFileArg(eval_funcs_metadata);
const funcs_data = gen_step.addOutputFileArg("funcs_data.mpack");
gen_step.addFileArg(b.path("src/nvim/eval.lua"));
break :eval_step funcs_data;
diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
@@ -285,7 +285,8 @@ endif()
# Variables
#-------------------------------------------------------------------------------
-set(FUNCS_METADATA ${PROJECT_BINARY_DIR}/funcs_metadata.mpack)
+set(EVAL_FUNCS_METADATA ${PROJECT_BINARY_DIR}/eval_funcs_metadata.mpack)
+set(EXPORTED_FUNCS_METADATA ${PROJECT_BINARY_DIR}/exported_funcs_metadata.mpack)
set(UI_METADATA ${PROJECT_BINARY_DIR}/ui_metadata.mpack)
set(BINARY_LIB_DIR ${PROJECT_BINARY_DIR}/lib/nvim)
set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
@@ -298,6 +299,7 @@ set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime)
# GENERATOR_DIR
set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua)
set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua)
+set(API_METADATA_GENERATOR ${GENERATOR_DIR}/gen_api_metadata.lua)
set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua)
set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua)
set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
@@ -564,16 +566,13 @@ set(NVIM_VERSION_LUA ${PROJECT_BINARY_DIR}/nvim_version.lua)
configure_file(${GENERATOR_DIR}/nvim_version.lua.in ${NVIM_VERSION_LUA})
add_custom_command(
- OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_API_METADATA}
- ${FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS}
+ OUTPUT ${GENERATED_API_DISPATCH} ${EXPORTED_FUNCS_METADATA}
+ ${EVAL_FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS}
COMMAND ${LUA_GEN} ${API_DISPATCH_GENERATOR}
${GENERATED_API_DISPATCH}
- ${GENERATED_API_METADATA} ${FUNCS_METADATA}
+ ${EXPORTED_FUNCS_METADATA} ${EVAL_FUNCS_METADATA}
${LUA_API_C_BINDINGS}
${GENERATED_KEYSETS_DEFS}
- ${UI_METADATA}
- ${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA}
- ${GENERATOR_DIR}/dump_bin_array.lua
${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
${API_HEADERS}
@@ -584,11 +583,27 @@ add_custom_command(
${API_DISPATCH_GENERATOR}
${GENERATOR_C_GRAMMAR}
${GENERATOR_HASHY}
+ ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
+)
+
+add_custom_command(
+ OUTPUT ${GENERATED_API_METADATA}
+ COMMAND ${LUA_GEN} ${API_METADATA_GENERATOR}
+ ${EXPORTED_FUNCS_METADATA}
+ ${UI_METADATA}
+ ${CMAKE_CURRENT_LIST_DIR}/api/ui.h
+ ${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA}
+ ${GENERATOR_DIR}/dump_bin_array.lua
+ ${GENERATED_API_METADATA}
+
+ DEPENDS
+ ${LUA_GEN_DEPS}
+ ${API_METADATA_GENERATOR}
${GENERATOR_DIR}/dump_bin_array.lua
${UI_METADATA}
+ ${EXPORTED_FUNCS_METADATA}
${NVIM_VERSION_LUA}
${NVIM_VERSION_GIT_H}
- ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
)
add_custom_command(
@@ -654,8 +669,8 @@ add_custom_command(OUTPUT ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS}
)
add_custom_command(OUTPUT ${GENERATED_FUNCS} ${FUNCS_DATA}
- COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua
- DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${FUNCS_METADATA}
+ COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${EVAL_FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua
+ DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${EVAL_FUNCS_METADATA}
)
add_custom_command(OUTPUT ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP}
@@ -690,6 +705,7 @@ list(APPEND NVIM_GENERATED_FOR_SOURCES
"${GENERATED_OPTIONS}"
"${GENERATED_OPTIONS_MAP}"
"${VIM_MODULE_FILE}"
+ "${GENERATED_API_METADATA}"
"${PROJECT_BINARY_DIR}/cmake.config/auto/pathdef.h"
)
@@ -960,10 +976,10 @@ add_target(doc-vim
)
add_target(doc-eval
- COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags
+ COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${EVAL_FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags
DEPENDS
nvim
- ${FUNCS_METADATA}
+ ${EVAL_FUNCS_METADATA}
${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
${PROJECT_SOURCE_DIR}/src/nvim/eval.lua
${PROJECT_SOURCE_DIR}/src/nvim/options.lua
