commit 829a5dc83eb87de401256c4907fb153d548c9180
parent 9875a9b69141e0096767d8ff187fd219c3905429
Author: bfredl <bjorn.linse@gmail.com>
Date: Tue, 28 Oct 2025 11:43:30 +0100
refactor(build): split out metadata step
This was part of an attempt to add "git describe" to zig build
without re-building too much. This doesn't yet work as zig upstream
changes are are also needed, but I think this was a sensible refactor
even in isolation, so breaking it out.
"API dispatch" doesn't depend on any ui event stuff nor on
version, that was just an accident of the metadata collection
being crammed into the same file.
Remove "manual invocation" instruction. this is going to bitrot anyway.
Both cmake and build.zig allows you to extract the command line of a
step, so you can debug it separately.
Diffstat:
4 files changed, 141 insertions(+), 114 deletions(-)
diff --git a/src/gen/gen_api_dispatch.lua b/src/gen/gen_api_dispatch.lua
@@ -1,30 +1,21 @@
-- Generates C code to bridge API <=> Lua.
---
--- Example (manual) invocation:
---
--- make
--- cp build/nvim_version.lua src/nvim/
--- cd src/nvim
--- nvim -l generators/gen_api_dispatch.lua "../../build/src/nvim/auto/api/private/dispatch_wrappers.generated.h" "../../build/src/nvim/auto/api/private/api_metadata.generated.h" "../../build/funcs_metadata.mpack" "../../build/src/nvim/auto/lua_api_c_bindings.generated.h" "../../build/src/nvim/auto/keysets_defs.generated.h" "../../build/ui_metadata.mpack" "../../build/cmake.config/auto/versiondef_git.h" "./api/autocmd.h" "./api/buffer.h" "./api/command.h" "./api/deprecated.h" "./api/extmark.h" "./api/keysets_defs.h" "./api/options.h" "./api/tabpage.h" "./api/ui.h" "./api/vim.h" "./api/vimscript.h" "./api/win_config.h" "./api/window.h" "../../build/include/api/autocmd.h.generated.h" "../../build/include/api/buffer.h.generated.h" "../../build/include/api/command.h.generated.h" "../../build/include/api/deprecated.h.generated.h" "../../build/include/api/extmark.h.generated.h" "../../build/include/api/options.h.generated.h" "../../build/include/api/tabpage.h.generated.h" "../../build/include/api/ui.h.generated.h" "../../build/include/api/vim.h.generated.h" "../../build/include/api/vimscript.h.generated.h" "../../build/include/api/win_config.h.generated.h" "../../build/include/api/window.h.generated.h"
+-- to obtain how the script is invoked, look in build/build.ninja and grep for
+-- "gen_api_dispatch.lua"
local hashy = require 'gen.hashy'
local c_grammar = require('gen.c_grammar')
-- output h file with generated dispatch functions (dispatch_wrappers.generated.h)
local dispatch_outputf = arg[1]
--- output h file with packed metadata (api_metadata.generated.h)
-local api_metadata_outputf = arg[2]
--- output metadata mpack file, for use by other build scripts (funcs_metadata.mpack)
-local mpack_outputf = arg[3]
+-- output file with exported functions metadata
+local exported_funcs_metadata_outputf = arg[2]
+-- output mpack file with raw metadata, for use by gen_eval.lua (funcs_metadata.mpack)
+local eval_funcs_metadata_outputf = arg[3]
local lua_c_bindings_outputf = arg[4] -- lua_api_c_bindings.generated.c
local keysets_outputf = arg[5] -- keysets_defs.generated.h
-local ui_metadata_inputf = arg[6] -- ui events metadata
-local git_version_inputf = arg[7] -- git version header
-local nvim_version_inputf = arg[8] -- nvim version
-local dump_bin_array_inputf = arg[9]
-local dispatch_deprecated_inputf = arg[10]
-local pre_args = 10
-assert(#arg >= pre_args)
+local dispatch_deprecated_inputf = arg[6]
+local pre_args = 6
+assert(#arg >= 6)
local function real_type(type, exported)
local ptype = c_grammar.typed_container:match(type)
@@ -165,8 +156,6 @@ local function add_keyset(val)
}
end
-local ui_options_text = nil
-
-- read each input file, parse and append to the api metadata
for i = pre_args + 1, #arg do
local full_path = arg[i]
@@ -190,12 +179,9 @@ for i = pre_args + 1, #arg do
end
end
- ui_options_text = ui_options_text or text:match('ui_ext_names%[][^{]+{([^}]+)}')
input:close()
end
---- @cast ui_options_text string
-
--- @generic T: table
--- @param orig T
--- @return T
@@ -303,77 +289,11 @@ for _, f in ipairs(functions) do
end
end
-local ui_options = { 'rgb' }
-for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do
- table.insert(ui_options, x)
-end
+local metadata_output = assert(io.open(exported_funcs_metadata_outputf, 'wb'))
+metadata_output:write(vim.mpack.encode(exported_functions))
+metadata_output:close()
--- @type integer[]
-local version = loadfile(nvim_version_inputf)()
-local git_version = io.open(git_version_inputf):read '*a'
-local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
-
-local pieces = {} --- @type string[]
-
--- Naively using mpack.encode({foo=x, bar=y}) will make the build
--- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
-local function fixdict(num)
- if num > 15 then
- error 'implement more dict codes'
- end
- pieces[#pieces + 1] = string.char(128 + num)
-end
-
-local function put(item, item2)
- table.insert(pieces, vim.mpack.encode(item))
- if item2 ~= nil then
- table.insert(pieces, vim.mpack.encode(item2))
- end
-end
-
-fixdict(6)
-
-put('version')
-fixdict(1 + #version)
-for _, item in ipairs(version) do
- -- NB: all items are mandatory. But any error will be less confusing
- -- with placeholder vim.NIL (than invalid mpack data)
- local val = item[2] == nil and vim.NIL or item[2]
- put(item[1], val)
-end
-put('build', version_build)
-
-put('functions', exported_functions)
-put('ui_events')
-table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
-put('ui_options', ui_options)
-
-put('error_types')
-fixdict(2)
-put('Exception', { id = 0 })
-put('Validation', { id = 1 })
-
-put('types')
-local types =
- { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
-fixdict(#types)
-for i, item in ipairs(types) do
- put(item[1])
- fixdict(2)
- put('id', i - 1)
- put('prefix', item[2])
-end
-
-local packed = table.concat(pieces)
---- @type fun(api_metadata: file*, name: string, packed: string)
-local dump_bin_array = loadfile(dump_bin_array_inputf)()
-
--- serialize the API metadata using msgpack and embed into the resulting
--- binary for easy querying by clients
-local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
-dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
-api_metadata_output:close()
-
-- start building the dispatch wrapper output
local output = assert(io.open(dispatch_outputf, 'wb'))
@@ -757,7 +677,7 @@ output:close()
--- @cast functions {[integer]: gen_api_dispatch.Function, keysets: gen_api_dispatch.Keyset[]}
functions.keysets = keysets
-local mpack_output = assert(io.open(mpack_outputf, 'wb'))
+local mpack_output = assert(io.open(eval_funcs_metadata_outputf, 'wb'))
mpack_output:write(vim.mpack.encode(functions))
mpack_output:close()
diff --git a/src/gen/gen_api_metadata.lua b/src/gen/gen_api_metadata.lua
@@ -0,0 +1,83 @@
+local mpack = vim.mpack
+
+assert(#arg == 7)
+local funcs_metadata_inputf = arg[1] -- exported functions metadata
+local ui_metadata_inputf = arg[2] -- ui events metadata
+local ui_options_inputf = arg[3] -- for ui options
+local git_version_inputf = arg[4] -- git version header
+local nvim_version_inputf = arg[5] -- nvim version
+local dump_bin_array_inputf = arg[6]
+local api_metadata_outputf = arg[7]
+
+local version = loadfile(nvim_version_inputf)()
+local git_version = io.open(git_version_inputf):read '*a'
+local version_build = git_version:match('#define NVIM_VERSION_BUILD "([^"]+)"') or vim.NIL
+
+local text = io.open(ui_options_inputf):read '*a'
+local ui_options_text = text:match('ui_ext_names%[][^{]+{([^}]+)}')
+local ui_options = { 'rgb' }
+for x in ui_options_text:gmatch('"([a-z][a-z_]+)"') do
+ table.insert(ui_options, x)
+end
+
+local pieces = {} --- @type string[]
+
+-- Naively using mpack.encode({foo=x, bar=y}) will make the build
+-- "non-reproducible". Emit maps directly as FIXDICT(2) "foo" x "bar" y instead
+local function fixdict(num)
+ if num > 15 then
+ error 'implement more dict codes'
+ end
+ pieces[#pieces + 1] = string.char(128 + num)
+end
+
+local function put(item, item2)
+ table.insert(pieces, mpack.encode(item))
+ if item2 ~= nil then
+ table.insert(pieces, mpack.encode(item2))
+ end
+end
+
+fixdict(6)
+
+put('version')
+fixdict(1 + #version)
+for _, item in ipairs(version) do
+ -- NB: all items are mandatory. But any error will be less confusing
+ -- with placeholder vim.NIL (than invalid mpack data)
+ local val = item[2] == nil and vim.NIL or item[2]
+ put(item[1], val)
+end
+put('build', version_build)
+
+put('functions')
+table.insert(pieces, io.open(funcs_metadata_inputf, 'rb'):read('*all'))
+put('ui_events')
+table.insert(pieces, io.open(ui_metadata_inputf, 'rb'):read('*all'))
+put('ui_options', ui_options)
+
+put('error_types')
+fixdict(2)
+put('Exception', { id = 0 })
+put('Validation', { id = 1 })
+
+put('types')
+local types =
+ { { 'Buffer', 'nvim_buf_' }, { 'Window', 'nvim_win_' }, { 'Tabpage', 'nvim_tabpage_' } }
+fixdict(#types)
+for i, item in ipairs(types) do
+ put(item[1])
+ fixdict(2)
+ put('id', i - 1)
+ put('prefix', item[2])
+end
+
+local packed = table.concat(pieces)
+--- @type fun(api_metadata: file*, name: string, packed: string)
+local dump_bin_array = loadfile(dump_bin_array_inputf)()
+
+-- serialize the API metadata using msgpack and embed into the resulting
+-- binary for easy querying by clients
+local api_metadata_output = assert(io.open(api_metadata_outputf, 'wb'))
+dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
+api_metadata_output:close()
diff --git a/src/gen/gen_steps.zig b/src/gen/gen_steps.zig
@@ -100,32 +100,40 @@ pub fn nvim_gen_sources(
break :ui_step ui_metadata;
};
- const funcs_metadata = api_step: {
+ const eval_funcs_metadata, const exported_funcs_metadata = dispatch_step: {
const gen_step = b.addRunArtifact(nlua0);
gen_step.addFileArg(b.path("src/gen/gen_api_dispatch.lua"));
_ = try gen_header_with_header(b, gen_step, "api/private/dispatch_wrappers.generated.h", nlua0, gen_headers);
- _ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers);
- const funcs_metadata = gen_step.addOutputFileArg("funcs_metadata.mpack");
+ const exported_funcs_metadata = gen_step.addOutputFileArg("exported_funcs_metadata.mpack");
+ const eval_funcs_metadata = gen_step.addOutputFileArg("eval_funcs_metadata.mpack");
_ = gen_header(b, gen_step, "lua_api_c_bindings.generated.h", gen_headers);
_ = gen_header(b, gen_step, "keysets_defs.generated.h", gen_headers);
- gen_step.addFileArg(ui_metadata);
- gen_step.addFileArg(versiondef_git);
- gen_step.addFileArg(version_lua);
- gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua"));
gen_step.addFileArg(b.path("src/nvim/api/dispatch_deprecated.lua"));
// now follows all .h files with exported functions
for (api_headers.items) |h| {
gen_step.addFileArg(h);
}
- break :api_step funcs_metadata;
+ break :dispatch_step .{ eval_funcs_metadata, exported_funcs_metadata };
};
+ {
+ const gen_step = b.addRunArtifact(nlua0);
+ gen_step.addFileArg(b.path("src/gen/gen_api_metadata.lua"));
+ gen_step.addFileArg(exported_funcs_metadata);
+ gen_step.addFileArg(ui_metadata);
+ gen_step.addFileArg(b.path("src/nvim/api/ui.h"));
+ gen_step.addFileArg(versiondef_git);
+ gen_step.addFileArg(version_lua);
+ gen_step.addFileArg(b.path("src/gen/dump_bin_array.lua"));
+ _ = gen_header(b, gen_step, "api/private/api_metadata.generated.h", gen_headers);
+ }
+
const funcs_data = eval_step: {
const gen_step = b.addRunArtifact(nlua0);
gen_step.addFileArg(b.path("src/gen/gen_eval.lua"));
_ = gen_header(b, gen_step, "funcs.generated.h", gen_headers);
- gen_step.addFileArg(funcs_metadata);
+ gen_step.addFileArg(eval_funcs_metadata);
const funcs_data = gen_step.addOutputFileArg("funcs_data.mpack");
gen_step.addFileArg(b.path("src/nvim/eval.lua"));
break :eval_step funcs_data;
diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt
@@ -285,7 +285,8 @@ endif()
# Variables
#-------------------------------------------------------------------------------
-set(FUNCS_METADATA ${PROJECT_BINARY_DIR}/funcs_metadata.mpack)
+set(EVAL_FUNCS_METADATA ${PROJECT_BINARY_DIR}/eval_funcs_metadata.mpack)
+set(EXPORTED_FUNCS_METADATA ${PROJECT_BINARY_DIR}/exported_funcs_metadata.mpack)
set(UI_METADATA ${PROJECT_BINARY_DIR}/ui_metadata.mpack)
set(BINARY_LIB_DIR ${PROJECT_BINARY_DIR}/lib/nvim)
set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
@@ -298,6 +299,7 @@ set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime)
# GENERATOR_DIR
set(API_DISPATCH_GENERATOR ${GENERATOR_DIR}/gen_api_dispatch.lua)
set(API_UI_EVENTS_GENERATOR ${GENERATOR_DIR}/gen_api_ui_events.lua)
+set(API_METADATA_GENERATOR ${GENERATOR_DIR}/gen_api_metadata.lua)
set(CHAR_BLOB_GENERATOR ${GENERATOR_DIR}/gen_char_blob.lua)
set(EVENTS_GENERATOR ${GENERATOR_DIR}/gen_events.lua)
set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
@@ -564,16 +566,13 @@ set(NVIM_VERSION_LUA ${PROJECT_BINARY_DIR}/nvim_version.lua)
configure_file(${GENERATOR_DIR}/nvim_version.lua.in ${NVIM_VERSION_LUA})
add_custom_command(
- OUTPUT ${GENERATED_API_DISPATCH} ${GENERATED_API_METADATA}
- ${FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS}
+ OUTPUT ${GENERATED_API_DISPATCH} ${EXPORTED_FUNCS_METADATA}
+ ${EVAL_FUNCS_METADATA} ${LUA_API_C_BINDINGS} ${GENERATED_KEYSETS_DEFS}
COMMAND ${LUA_GEN} ${API_DISPATCH_GENERATOR}
${GENERATED_API_DISPATCH}
- ${GENERATED_API_METADATA} ${FUNCS_METADATA}
+ ${EXPORTED_FUNCS_METADATA} ${EVAL_FUNCS_METADATA}
${LUA_API_C_BINDINGS}
${GENERATED_KEYSETS_DEFS}
- ${UI_METADATA}
- ${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA}
- ${GENERATOR_DIR}/dump_bin_array.lua
${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
${API_HEADERS}
@@ -584,11 +583,27 @@ add_custom_command(
${API_DISPATCH_GENERATOR}
${GENERATOR_C_GRAMMAR}
${GENERATOR_HASHY}
+ ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
+)
+
+add_custom_command(
+ OUTPUT ${GENERATED_API_METADATA}
+ COMMAND ${LUA_GEN} ${API_METADATA_GENERATOR}
+ ${EXPORTED_FUNCS_METADATA}
+ ${UI_METADATA}
+ ${CMAKE_CURRENT_LIST_DIR}/api/ui.h
+ ${NVIM_VERSION_GIT_H} ${NVIM_VERSION_LUA}
+ ${GENERATOR_DIR}/dump_bin_array.lua
+ ${GENERATED_API_METADATA}
+
+ DEPENDS
+ ${LUA_GEN_DEPS}
+ ${API_METADATA_GENERATOR}
${GENERATOR_DIR}/dump_bin_array.lua
${UI_METADATA}
+ ${EXPORTED_FUNCS_METADATA}
${NVIM_VERSION_LUA}
${NVIM_VERSION_GIT_H}
- ${CMAKE_CURRENT_LIST_DIR}/api/dispatch_deprecated.lua
)
add_custom_command(
@@ -654,8 +669,8 @@ add_custom_command(OUTPUT ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS}
)
add_custom_command(OUTPUT ${GENERATED_FUNCS} ${FUNCS_DATA}
- COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua
- DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${FUNCS_METADATA}
+ COMMAND ${LUA_GEN} ${FUNCS_GENERATOR} ${GENERATED_FUNCS} ${EVAL_FUNCS_METADATA} ${FUNCS_DATA} ${CMAKE_CURRENT_LIST_DIR}/eval.lua
+ DEPENDS ${LUA_GEN_DEPS} ${FUNCS_GENERATOR} ${GENERATOR_HASHY} ${CMAKE_CURRENT_LIST_DIR}/eval.lua ${EVAL_FUNCS_METADATA}
)
add_custom_command(OUTPUT ${GENERATED_EVENTS_ENUM} ${GENERATED_EVENTS_NAMES_MAP}
@@ -690,6 +705,7 @@ list(APPEND NVIM_GENERATED_FOR_SOURCES
"${GENERATED_OPTIONS}"
"${GENERATED_OPTIONS_MAP}"
"${VIM_MODULE_FILE}"
+ "${GENERATED_API_METADATA}"
"${PROJECT_BINARY_DIR}/cmake.config/auto/pathdef.h"
)
@@ -960,10 +976,10 @@ add_target(doc-vim
)
add_target(doc-eval
- COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags
+ COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua ${EVAL_FUNCS_METADATA} ${PROJECT_BINARY_DIR}/runtime/doc/tags
DEPENDS
nvim
- ${FUNCS_METADATA}
+ ${EVAL_FUNCS_METADATA}
${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
${PROJECT_SOURCE_DIR}/src/nvim/eval.lua
${PROJECT_SOURCE_DIR}/src/nvim/options.lua
