Bug#828540: sendmail: FTBFS with openssl 1.1.0

[Andreas Beckmann <anbe@debian.org>]

On 2016-09-12 09:02, Sebastian Andrzej Siewior wrote:
> could you please try if the patch attached works? It compiles :)

Many thanks, so let's just try it :-)

It FTBFS on the non-linux architectures:
https://buildd.debian.org/status/package.php?p=sendmail&suite=unstable

It's this part of your patch:

@@ -1002,9 +1047,23 @@ inittls(ctx, req, options, srv, certfile, keyfile, cacertpath, cacertfile, dhpar

        if (bitset(TLS_I_RSA_TMP, req)
 #  if SM_CONF_SHM
-           && ShmId != SM_SHM_NO_ID &&
-           (rsa_tmp = RSA_generate_key(RSA_KEYLENGTH, RSA_F4, NULL,
-                                       NULL)) == NULL
+           && ShmId != SM_SHM_NO_ID) {
+               BIGNUM *bn;
+
+               bn = BN_new();
+               rsa_tmp = RSA_new();
+               if (!bn || !rsa_tmp || !BN_set_word(bn, RSA_F4)) {
+                       RSA_free(rsa_tmp);
+                       rsa_tmp = NULL;
+               }
+               if (rsa_tmp) {
+                       if (!RSA_generate_key_ex(rsa_tmp, RSA_KEYLENGTH, bn, NULL)) {
+                               RSA_free(rsa_tmp);
+                               rsa_tmp = NULL;
+                       }
+               }
+               BN_free(bn);
+               if (!rsa_tmp
 #  else /* SM_CONF_SHM */
            && 0        /* no shared memory: no need to generate key now */
 #  endif /* SM_CONF_SHM */
@@ -1020,6 +1079,7 @@ inittls(ctx, req, options, srv, certfile, keyfile, cacertpath, cacertfile, dhpar
                }
                return false;
        }
+       }
 # endif /* !TLS_NO_RSA */

        /*

The braces don't match the #if scope (opened within SM_CONF_SHM, closed outside).
kfreebsd doesn't seem to have shmem.


Andreas