Converting lua tables to token lists and back

We can measure the dimensions of a box almost entirely from Lua. The only place where we need to escape out to TeX is to typeset and save the text into a box, then we can measure the box from Lua.

\documentclass{article} %%%%%%%%%%%%%%%%%%%%%% %%% Implementation %%% %%%%%%%%%%%%%%%%%%%%%% \makeatletter %% The same as \newbox, but using LaTeX syntax. \newsavebox{\example@tmp@box} \makeatother \usepackage{luacode} \begin{luacode*} -- Define a table to hold all Lua functions defined by this module. example = example or {} -- Constants local bgroup = token.create(utf8.codepoint "{", token.command_id "left_brace") local box_index = luatexbase.registernumber("example@tmp@box") local box_token = token.create("example@tmp@box") local document_catcodes = luatexbase.registernumber("c_document_cctab") local egroup = token.create(utf8.codepoint "}", token.command_id "right_brace") local sbox_token = token.create("sbox") local sp_to_pt = tex.sp("1pt") -- Typesets a string into a box. function example.string_to_box(str) -- `tex.runtoks` forces the code to run now. Otherwise, `tex.sprint` -- just pushes some tokens onto the input stream, so it will return -- before the code is actually executed, so inspecting the box will give -- us old data. tex.runtoks(function() -- We use `document_catcodes` here to ensure that the string is -- processed using a normal catcode regime. Otherwise, if this Lua -- code happened to run inside of `\ExplSyntaxOn`/`\ExplSyntaxOff`, -- all the spaces would disappear. See -- -- https://tex.stackexchange.com/a/747040/ -- -- for further discussion. tex.sprint(document_catcodes, { -- `\sbox` is the LaTeX equivalent of TeX's `\setbox` command, -- except it is “colour safe” (which means that it will properly -- obey grouping when used with colour commands). sbox_token, -- We need to use `box_token` here since our box's csname -- contains an `@`, which is not a letter when using the -- document catcodes. bgroup, box_token, egroup, bgroup, str, egroup, }) end) -- We need to copy the node list here, because when TeX overwrites the -- box later, it will free the node list, and using a freed node list -- gives a fatal error with an unhelpful error message. return node.copy_list(tex.box[box_index]) end -- Gets the dimensions of a box. function example.get_box_dimensions(box) -- All TeX dimensions are internally stored in scaled points (sp), so -- we need to convert them to points (pt) for easier reading. local width = box.width / sp_to_pt local height = box.height / sp_to_pt local depth = box.depth / sp_to_pt return { width = width, height = height, depth = depth, } end \end{luacode*} %%%%%%%%%%%%%%%%%%%%% %%% Demonstration %%% %%%%%%%%%%%%%%%%%%%%% \usepackage{tensor} %% Define the command used to print each example. \makeatletter %% We _could_ write this all in Lua instead of using a TeX helper macro, %% but it's generally a bad idea to mix TeX with Lua, so it's best to do any %% formatting/typesetting from TeX. \NewDocumentCommand{\example@print}{m m m}{% \texttt{#1} & \shortstack[l]{#2 \\ #3} \\ \hline } \makeatother \begin{luacode*} -- Constants local bgroup = token.create(utf8.codepoint "{", token.command_id "left_brace") local egroup = token.create(utf8.codepoint "}", token.command_id "right_brace") local print_token = token.create("example@print") local sp_to_pt = tex.sp("1pt") local verbatim_catcodes = luatexbase.registernumber("c_str_cctab") -- Generic helper function that exposes a Lua function as a TeX macro. local function register_tex_cmd(name, func) -- See the following links -- -- https://tex.stackexchange.com/a/747876 -- -- https://github.com/gucci-on-fleek/luatools/blob/b910432c/source/luatools.lua#L2055-L2131 -- -- for extended versions of this function. local index = luatexbase.new_luafunction(name) lua.get_functions_table()[index] = func token.set_lua(name, index, "global", "protected") end -- The strings that we want to measure. local my_strings = { -- Standard Lua trick: you can use [[bracketed]] strings to avoid having -- to escape special characters like backslashes. [[$\tensor{A}{^{34}^{66}_{123}}$]], "Hello, world!", [[\textbf{Bold} and \textit{italic}.]], [[$\displaystyle \int_0^1 \ln x dx = -1$]], } -- Creates a rule with the given dimensions in points. local function create_rule(width, height, depth) local rule = node.new("rule") rule.width = width * sp_to_pt rule.height = height * sp_to_pt rule.depth = depth * sp_to_pt return rule end -- Another standard Lua trick: the "\z" escape gobbles any whitespace -- (including newlines) until the next non-whitespace character, so you can -- indent multi-line strings nicely. -- -- "string.formatters" returns a function that formats its contents like -- the standard "string.format" function. See page 94 of the CLD manual -- -- https://www.pragma-ade.nl/general/manuals/cld-mkiv.pdf#page=96 -- -- for more details. local fmt = string.formatters["\z \n\z String: %s\n\z Width: %6.2fpt\n\z Height: %6.2fpt\n\z Depth: %6.2fpt\n\z "] -- Define a TeX command that measures each string and prints the results. register_tex_cmd("example", function() -- Loop over each string in the list. for _, str in ipairs(my_strings) do local box = example.string_to_box(str) local dimensions = example.get_box_dimensions(box) local rule = create_rule( dimensions.width, dimensions.height, dimensions.depth ) -- Print the results to the console of the TeX engine. texio.write_nl(fmt( str, dimensions.width, dimensions.height, dimensions.depth )) -- As you can see, `tex.sprint` is a fairly magical function. Here, -- we're passing it userdata-type tokens (`print_token`, `bgroup`, -- etc.) that it will flush to TeX exactly, node lists (`rule` and -- `box`) that it will splice into the output at the current point, -- and strings (`str`) that it will tokenize using the given catcode -- table (`verbatim_catcodes`). This lets us simultaneously pass the -- verbatim input string and its typeset result to a TeX macro, in a -- way that is completely safe and robust. tex.sprint(verbatim_catcodes, { print_token, bgroup, str, egroup, bgroup, rule, egroup, bgroup, box, egroup, }) end end) \end{luacode*} \pagestyle{empty} \setlength{\parindent}{0pt} %% Typeset the examples in a table. \begin{document} \begin{tabular}{rl} \hline \example \end{tabular} \end{document} 

String: $\tensor{A}{^{34}^{66}_{123}}$ Width: 35.90pt Height: 8.14pt Depth: 2.48pt String: Hello, world! Width: 55.59pt Height: 7.16pt Depth: 1.93pt String: \textbf{Bold} and \textit{italic}. Width: 70.80pt Height: 6.94pt Depth: 0.11pt String: $\displaystyle \int_0^1 \ln x dx = -1$ Width: 68.90pt Height: 15.65pt Depth: 9.11pt