Use libnettle instead of libcrypto (from openssl) for MD5 and HMAC(MD5).

The HMAC_ and EVP_MD_ API:s changed in OpenSSL 1.1 in a way that made
it unfeasable to support both older and newer OpenSSL.

Radsecproxy already depends on libnettle for Fticks.
Moving away from libcrypto makes it easier to add support for other
TLS libraries than OpenSSL.

1 files changed, 31 insertions, 69 deletions

diff --git a/radsecproxy.c b/radsecproxy.c
index cbf3cdf..d568acd 100644
--- a/radsecproxy.c
+++ b/radsecproxy.c
@@ -58,7 +58,7 @@
 #include <openssl/ssl.h>
 #include <openssl/rand.h>
 #include <openssl/err.h>
-#include <openssl/md5.h>
+#include <nettle/md5.h>
 #include "debug.h"
 #include "hash.h"
 #include "util.h"
@@ -545,27 +545,19 @@ void sendreply(struct request *rq) {
 }
 
 static int pwdcrypt(char encrypt_flag, uint8_t *in, uint8_t len, char *shared, uint8_t sharedlen, uint8_t *auth) {
-    int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned char hash[EVP_MAX_MD_SIZE], *input;
-    unsigned int md_len;
+    struct md5_ctx mdctx;
+    unsigned char hash[MD5_DIGEST_SIZE], *input;
     uint8_t i, offset = 0, out[128];
 
     pthread_mutex_lock(&lock);
-    mdctx = EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
 
+    md5_init(&mdctx);
     input = auth;
     for (;;) {
-	if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	    !EVP_DigestUpdate(mdctx, (uint8_t *)shared, sharedlen) ||
-	    !EVP_DigestUpdate(mdctx, input, 16) ||
-	    !EVP_DigestFinal_ex(mdctx, hash, &md_len) ||
-	    md_len != 16) {
-            goto out;
-        }
+	md5_update(&mdctx, sharedlen, (uint8_t *) shared);
+        md5_update(&mdctx, 16, input);
+        md5_digest(&mdctx, sizeof(hash), hash);
 	for (i = 0; i < 16; i++)
 	    out[offset + i] = hash[i] ^ in[offset + i];
 	if (encrypt_flag)
@@ -577,25 +569,19 @@ static int pwdcrypt(char encrypt_flag, uint8_t *in, uint8_t len, char *shared, u
 	    break;
     }
     memcpy(in, out, len);
-    result = 1;
-out:
-    EVP_MD_CTX_free(mdctx);
+
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sharedlen, uint8_t *auth, uint8_t *salt) {
-    int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned char hash[EVP_MAX_MD_SIZE];
-    unsigned int md_len;
+    struct md5_ctx mdctx;
+    unsigned char hash[MD5_DIGEST_SIZE];
     uint8_t i, offset;
 
     pthread_mutex_lock(&lock);
-    mdctx = EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
+    md5_init(&mdctx);
 
 #if 0
     printfchars(NULL, "msppencrypt auth in", "%02x ", auth, 16);
@@ -603,13 +589,10 @@ static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
     printfchars(NULL, "msppencrypt in", "%02x ", text, len);
 #endif
 
-    if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	!EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	!EVP_DigestUpdate(mdctx, auth, 16) ||
-	!EVP_DigestUpdate(mdctx, salt, 2) ||
-	!EVP_DigestFinal_ex(mdctx, hash, &md_len)) {
-        goto out;
-    }
+    md5_update(&mdctx, sharedlen, shared);
+    md5_update(&mdctx, 16, auth);
+    md5_update(&mdctx, 2, salt);
+    md5_digest(&mdctx, sizeof(hash), hash);
 
 #if 0
     printfchars(NULL, "msppencrypt hash", "%02x ", hash, 16);
@@ -623,13 +606,9 @@ static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
 	printf("text + offset - 16 c(%d): ", offset / 16);
 	printfchars(NULL, NULL, "%02x ", text + offset - 16, 16);
 #endif
-	if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	    !EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	    !EVP_DigestUpdate(mdctx, text + offset - 16, 16) ||
-	    !EVP_DigestFinal_ex(mdctx, hash, &md_len) ||
-	    md_len != 16) {
-            goto out;
-	}
+        md5_update(&mdctx, sharedlen, shared);
+        md5_update(&mdctx, 16, text + offset - 16);
+        md5_digest(&mdctx, sizeof(hash), hash);
 #if 0
 	printfchars(NULL, "msppencrypt hash", "%02x ", hash, 16);
 #endif
@@ -637,31 +616,24 @@ static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
 	for (i = 0; i < 16; i++)
 	    text[offset + i] ^= hash[i];
     }
-    result = 1;
 
 #if 0
     printfchars(NULL, "msppencrypt out", "%02x ", text, len);
 #endif
 
-out:
-    EVP_MD_CTX_free(mdctx);
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sharedlen, uint8_t *auth, uint8_t *salt) {
-    int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned char hash[EVP_MAX_MD_SIZE];
-    unsigned int md_len;
+    struct md5_ctx mdctx;
+    unsigned char hash[MD5_DIGEST_SIZE];
     uint8_t i, offset;
     char plain[255];
 
     pthread_mutex_lock(&lock);
-    mdctx= EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
+    md5_init(&mdctx);
 
 #if 0
     printfchars(NULL, "msppdecrypt auth in", "%02x ", auth, 16);
@@ -669,13 +641,10 @@ static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
     printfchars(NULL, "msppdecrypt in", "%02x ", text, len);
 #endif
 
-    if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	!EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	!EVP_DigestUpdate(mdctx, auth, 16) ||
-	!EVP_DigestUpdate(mdctx, salt, 2) ||
-	!EVP_DigestFinal_ex(mdctx, hash, &md_len)) {
-        goto out;
-    }
+    md5_update(&mdctx, sharedlen, shared);
+    md5_update(&mdctx, 16, auth);
+    md5_update(&mdctx, 2, salt);
+    md5_digest(&mdctx, sizeof(hash), hash);
 
 #if 0
     printfchars(NULL, "msppdecrypt hash", "%02x ", hash, 16);
@@ -689,13 +658,9 @@ static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
 	printf("text + offset - 16 c(%d): ", offset / 16);
 	printfchars(NULL, NULL, "%02x ", text + offset - 16, 16);
 #endif
-	if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	    !EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	    !EVP_DigestUpdate(mdctx, text + offset - 16, 16) ||
-	    !EVP_DigestFinal_ex(mdctx, hash, &md_len) ||
-	    md_len != 16) {
-            goto out;
-	}
+        md5_update(&mdctx, sharedlen, shared);
+        md5_update(&mdctx, 16, text + offset - 16);
+        md5_digest(&mdctx, sizeof(hash), hash);
 #if 0
 	printfchars(NULL, "msppdecrypt hash", "%02x ", hash, 16);
 #endif
@@ -705,15 +670,12 @@ static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
     }
 
     memcpy(text, plain, len);
-    result = 1;
 #if 0
     printfchars(NULL, "msppdecrypt out", "%02x ", text, len);
 #endif
 
-out:
-    EVP_MD_CTX_free(mdctx);
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 struct realm *newrealmref(struct realm *r) {