net-libs/libvncserver: fix openssl crypto backend

Bug: https://bugs.gentoo.org/893608
Signed-off-by: Alexander Tsoy <alexander@tsoy.me>
Closes: https://github.com/gentoo/gentoo/pull/30311
Signed-off-by: Sam James <sam@gentoo.org>

1 files changed, 59 insertions, 0 deletions

diff --git a/net-libs/libvncserver/files/libvncserver-0.9.14-crypto-openssl-fix.patch b/net-libs/libvncserver/files/libvncserver-0.9.14-crypto-openssl-fix.patch
new file mode 100644
index 000000000000..d4c76773b02c
--- /dev/null
+++ b/net-libs/libvncserver/files/libvncserver-0.9.14-crypto-openssl-fix.patch
@@ -0,0 +1,59 @@
+From b686f379c34114cf938fe88291f58014337558f6 Mon Sep 17 00:00:00 2001
+From: Gaurav Ujjwal <gujjwal00@gmail.com>
+Date: Mon, 23 Jan 2023 00:03:03 +0530
+Subject: [PATCH] common/crypto_openssl: pad DH key buffers with leading zeros
+ for smaller keys
+
+Re: https://github.com/LibVNC/libvncserver/issues/493
+Re: https://github.com/bk138/multivnc/issues/202
+---
+ common/crypto_openssl.c | 18 ++++++++++++++++--
+ 1 file changed, 16 insertions(+), 2 deletions(-)
+
+diff --git a/common/crypto_openssl.c b/common/crypto_openssl.c
+index 50e8073a..dc1ee093 100644
+--- a/common/crypto_openssl.c
++++ b/common/crypto_openssl.c
+@@ -156,6 +156,15 @@ int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], cons
+     return result;
+ }
+ 
++static void pad_leading_zeros(uint8_t *out, const size_t current_len, const size_t expected_len) {
++    if (current_len >= expected_len || expected_len < 1)
++        return;
++
++    size_t diff = expected_len - current_len;
++    memmove(out + diff, out, current_len);
++    memset(out, 0, diff);
++}
++
+ int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen, const size_t gen_len, const uint8_t *prime, const size_t keylen)
+ {
+     int result = 0;
+@@ -184,6 +193,9 @@ int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen,
+ 	goto out;
+     if(BN_bn2bin(dh->pub_key, pub_out) == 0)
+ 	goto out;
++
++    pad_leading_zeros(priv_out, BN_num_bytes(dh->priv_key), keylen);
++    pad_leading_zeros(pub_out, BN_num_bytes(dh->pub_key), keylen);
+ #else
+     DH_get0_key(dh, &pub_key, &priv_key);
+     if(BN_bn2binpad(priv_key, priv_out, keylen) == -1)
+@@ -216,9 +228,11 @@ int dh_compute_shared_key(uint8_t *shared_out, const uint8_t *priv, const uint8_
+     if(!DH_set0_key(dh, NULL, BN_bin2bn(priv, keylen, NULL)))
+ 	goto out;
+ #endif
+-    if(DH_compute_key(shared_out, BN_bin2bn(pub, keylen, NULL), dh) == -1)
+-	goto out;
++    int shared_len = DH_compute_key(shared_out, BN_bin2bn(pub, keylen, NULL), dh);
++    if(shared_len == -1)
++        goto out;
+ 
++    pad_leading_zeros(shared_out, shared_len, keylen);
+     result = 1;
+ 
+  out:
+-- 
+2.39.2
+