Bug#1009196: texlive-binaries: Reproducible content of .fmt files

[Norbert Preining <norbert@preining.info>]

Hi Luigi, hi all luatex devs,

here at Debian we got a bug report about reproducability of luatex
format dumps. It contains a patch to make the hyphenation exception list
sorted. (I attach the patch)

Could you please take a look whether this is still relevant for the
latest release of luatex.

Thanks

Norbert

On Fri, 08 Apr 2022, Roland Clobus wrote:
> Hello maintainers of texlive-binaries,
> 
> While working on the “reproducible builds” effort [1], I have noticed that the
> live image for Cinnamon in bookworm is no longer reproducible [2].
> 
> The attached patch ensures that the output of the function 'exception_strings'
> always uses the same order of the hyphenation exceptions.
> I've written the solution in C, perhaps someone more versed in lua could
> rewrite it more elegantly.
> (The lua manual says for the 'next' function: 'The order in which the indices
> are enumerated is not specified' [3])
> 
> With the attached patch applied, I'm able (with the help of FORCE_SOURCE_DATE=1
> and SOURCE_DATE_EPOCH) to reproducibly rebuild the .fmt files, as created by
> 'fmtutil --sys --all'.
> 
> Small test case to reproduce:
> export FORCE_SOURCE_DATE=1
> export SOURCE_DATE_EPOCH=$(date +%s)
> for i in `seq 1 10`; do luahbtex -ini -jobname=luahbtex -progname=luabhtex
> luatex.ini > /dev/null; md5sum luahbtex.*; done
> 
> With kind regards,
> Roland Clobus
> 
>  [1]: https://wiki.debian.org/ReproducibleBuilds
>  [2]:
> https://jenkins.debian.net/view/live/job/reproducible_debian_live_build_cinnamon_bookworm/
>  [3]: http://www.lua.org/manual/5.4/manual.html#pdf-next
> 

--
PREINING Norbert                              https://www.preining.info
Mercari Inc.     +     IFMGA Guide     +     TU Wien     +     TeX Live
GPG: 0x860CDC13   fp: F7D8 A928 26E3 16A1 9FA0 ACF0 6CAC A448 860C DC13

diff --git a/texk/web2c/luatexdir/lang/texlang.c b/texk/web2c/luatexdir/lang/texlang.c
index ba7614ff..ccc0ec90 100644
--- a/texk/web2c/luatexdir/lang/texlang.c
+++ b/texk/web2c/luatexdir/lang/texlang.c
@@ -498,10 +498,20 @@ static char *hyphenation_exception(int exceptions, char *w)
     return ret;
 }
 
+char *unsorted_buffer = NULL;
+size_t *indexes = NULL;
+
+static int sort_func(const void *a, const void *b) {
+    size_t ia = *(size_t*)a;
+    size_t ib = *(size_t*)b;
+    return strcmp(&unsorted_buffer[ia], &unsorted_buffer[ib]);
+}
+
 char *exception_strings(struct tex_language *lang)
 {
     const char *value;
     size_t size = 0, current = 0;
+    size_t num_bytes = 0;
     size_t l = 0;
     char *ret = NULL;
     if (lang->exceptions == 0)
@@ -509,19 +519,42 @@ char *exception_strings(struct tex_language *lang)
     lua_checkstack(Luas, 2);
     lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions);
     if (lua_istable(Luas, -1)) {
-        /*tex Iterate and join. */
+        /*tex Determine required memory. */
         lua_pushnil(Luas);
         while (lua_next(Luas, -2) != 0) {
             value = lua_tolstring(Luas, -1, &l);
-            if (current + 2 + l > size) {
-                ret = xrealloc(ret, (unsigned) ((size + size / 5) + current + l + 1024));
-                size = (size + size / 5) + current + l + 1024;
-            }
-            *(ret + current) = ' ';
-            strcpy(ret + current + 1, value);
+            num_bytes += l + 1;
+            size++;
+            lua_pop(Luas, 1);
+        }
+        unsorted_buffer = xmalloc(num_bytes);
+        indexes = xmalloc(sizeof(size_t)*size);
+
+        /*tex Fetch values. */
+        current = 0;
+        size = 0;
+        lua_pushnil(Luas);
+        while (lua_next(Luas, -2) != 0) {
+            value = lua_tolstring(Luas, -1, &l);
+            strcpy(unsorted_buffer + current, value);
+            indexes[size++] = current;
             current += l + 1;
             lua_pop(Luas, 1);
         }
+        /*tex Sort and join. */
+        qsort(indexes, size, sizeof(size_t), sort_func);
+        ret = xmalloc(num_bytes);
+        current = 0;
+        for (l = 0; l < size; l++) {
+           strcpy(ret + current, &unsorted_buffer[indexes[l]]);
+           current += strlen(&unsorted_buffer[indexes[l]]);
+           ret[current] = ' ';
+           current += 1;
+        }
+        ret[current - 1] = '\0';
+
+        free(unsorted_buffer);
+        free(indexes);
     }
     return ret;
 }