Use libnettle instead of libcrypto (from openssl) for MD5 and HMAC(MD5).

The HMAC_ and EVP_MD_ API:s changed in OpenSSL 1.1 in a way that made
it unfeasable to support both older and newer OpenSSL.

Radsecproxy already depends on libnettle for Fticks.
Moving away from libcrypto makes it easier to add support for other
TLS libraries than OpenSSL.

3 files changed, 75 insertions, 137 deletions

diff --git a/radmsg.c b/radmsg.c
index 514d5d3..1a0f912 100644
--- a/radmsg.c
+++ b/radmsg.c
@@ -14,7 +14,7 @@
 #include "radmsg.h"
 #include "debug.h"
 #include <pthread.h>
-#include <openssl/hmac.h>
+#include <nettle/hmac.h>
 #include <openssl/rand.h>
 
 #define RADLEN(x) ntohs(((uint16_t *)(x))[1])
@@ -124,115 +124,86 @@ int radmsg_copy_attrs(struct radmsg *dst,
 int _checkmsgauth(unsigned char *rad, uint8_t *authattr, uint8_t *secret) {
     int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    HMAC_CTX *hmacctx = NULL;
-    unsigned int md_len;
-    uint8_t auth[16], hash[EVP_MAX_MD_SIZE];
+    struct hmac_md5_ctx hmacctx;
+    uint8_t auth[16], hash[MD5_DIGEST_SIZE];
 
     pthread_mutex_lock(&lock);
 
+   /* FIXME: Why clearing authattr during hashing? */
     memcpy(auth, authattr, 16);
     memset(authattr, 0, 16);
-    md_len = 0;
-    hmacctx = HMAC_CTX_new();
-    if (!hmacctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
-    HMAC_Init_ex(hmacctx, secret, strlen((char *)secret), EVP_md5(), NULL);
-    HMAC_Update(hmacctx, rad, RADLEN(rad));
-    HMAC_Final(hmacctx, hash, &md_len);
+
+    hmac_md5_set_key(&hmacctx, strlen((char *) secret), secret);
+    hmac_md5_update(&hmacctx, RADLEN(rad), rad);
+    hmac_md5_digest(&hmacctx, sizeof(hash), hash);
+
     memcpy(authattr, auth, 16);
-    if (md_len != 16) {
-	debug(DBG_WARN, "message auth computation failed");
-        goto out;
-    }
 
     if (memcmp(auth, hash, 16)) {
 	debug(DBG_WARN, "message authenticator, wrong value");
         goto out;
     }
+    result = 1;                 /* Success. */
 
-    result = 1;
 out:
-    HMAC_CTX_free(hmacctx);
     pthread_mutex_unlock(&lock);
     return result;
 }
 
 int _validauth(unsigned char *rad, unsigned char *reqauth, unsigned char *sec) {
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned char hash[EVP_MAX_MD_SIZE];
-    unsigned int len;
-    int result;
+    struct md5_ctx mdctx;
+    unsigned char hash[MD5_DIGEST_SIZE];
+    const unsigned int len = RADLEN(rad);
+    int result = 0;             /* Fail. */
 
     pthread_mutex_lock(&lock);
-    mdctx = EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
-
-    len = RADLEN(rad);
-
-    result = (EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) &&
-	      EVP_DigestUpdate(mdctx, rad, 4) &&
-	      EVP_DigestUpdate(mdctx, reqauth, 16) &&
-	      (len <= 20 || EVP_DigestUpdate(mdctx, rad + 20, len - 20)) &&
-	      EVP_DigestUpdate(mdctx, sec, strlen((char *)sec)) &&
-	      EVP_DigestFinal_ex(mdctx, hash, &len) &&
-	      len == 16 &&
-	      !memcmp(hash, rad + 4, 16));
-    EVP_MD_CTX_free(mdctx);
+    md5_init(&mdctx);
+
+    md5_update(&mdctx, 4, rad);
+    md5_update(&mdctx, 16, reqauth);
+    if (len > 20)
+        md5_update(&mdctx, len - 20, rad + 20);
+    md5_update(&mdctx, strlen((char *) sec), sec);
+    md5_digest(&mdctx, sizeof(hash), hash);
+
+    result = !memcmp(hash, rad + 4, 16);
+
     pthread_mutex_unlock(&lock);
     return result;
 }
 
 int _createmessageauth(unsigned char *rad, unsigned char *authattrval, uint8_t *secret) {
-    int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    static HMAC_CTX *hmacctx = NULL;
-    unsigned int md_len;
+    struct hmac_md5_ctx hmacctx;
 
     if (!authattrval)
 	return 1;
 
     pthread_mutex_lock(&lock);
-    hmacctx = HMAC_CTX_new();
-    if (!hmacctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
 
     memset(authattrval, 0, 16);
-    md_len = 0;
-    HMAC_Init_ex(hmacctx, secret, strlen((char *)secret), EVP_md5(), NULL);
-    HMAC_Update(hmacctx, rad, RADLEN(rad));
-    HMAC_Final(hmacctx, authattrval, &md_len);
-    if (md_len != 16) {
-	debug(DBG_WARN, "message auth computation failed");
-        goto out;
-    }
-    result = 1;
-out:
-    HMAC_CTX_free(hmacctx);
+    hmac_md5_set_key(&hmacctx, strlen((char *) secret), secret);
+    hmac_md5_update(&hmacctx, RADLEN(rad), rad);
+    hmac_md5_digest(&hmacctx, MD5_DIGEST_SIZE, authattrval);
+
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 int _radsign(unsigned char *rad, unsigned char *sec) {
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned int md_len;
-    int result;
+    struct md5_ctx mdctx;
 
     pthread_mutex_lock(&lock);
-    mdctx = EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
-
-    result = (EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) &&
-	      EVP_DigestUpdate(mdctx, rad, RADLEN(rad)) &&
-	      EVP_DigestUpdate(mdctx, sec, strlen((char *)sec)) &&
-	      EVP_DigestFinal_ex(mdctx, rad + 4, &md_len) &&
-	      md_len == 16);
-    EVP_MD_CTX_free(mdctx);
+
+    md5_init(&mdctx);
+    md5_update(&mdctx, RADLEN(rad), rad);
+    md5_update(&mdctx, strlen((char *) sec), sec);
+    md5_digest(&mdctx, MD5_DIGEST_SIZE, rad + 4);
+
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 uint8_t *radmsg2buf(struct radmsg *msg, uint8_t *secret) {
diff --git a/radsecproxy.c b/radsecproxy.c
index cbf3cdf..d568acd 100644
--- a/radsecproxy.c
+++ b/radsecproxy.c
@@ -58,7 +58,7 @@
 #include <openssl/ssl.h>
 #include <openssl/rand.h>
 #include <openssl/err.h>
-#include <openssl/md5.h>
+#include <nettle/md5.h>
 #include "debug.h"
 #include "hash.h"
 #include "util.h"
@@ -545,27 +545,19 @@ void sendreply(struct request *rq) {
 }
 
 static int pwdcrypt(char encrypt_flag, uint8_t *in, uint8_t len, char *shared, uint8_t sharedlen, uint8_t *auth) {
-    int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned char hash[EVP_MAX_MD_SIZE], *input;
-    unsigned int md_len;
+    struct md5_ctx mdctx;
+    unsigned char hash[MD5_DIGEST_SIZE], *input;
     uint8_t i, offset = 0, out[128];
 
     pthread_mutex_lock(&lock);
-    mdctx = EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
 
+    md5_init(&mdctx);
     input = auth;
     for (;;) {
-	if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	    !EVP_DigestUpdate(mdctx, (uint8_t *)shared, sharedlen) ||
-	    !EVP_DigestUpdate(mdctx, input, 16) ||
-	    !EVP_DigestFinal_ex(mdctx, hash, &md_len) ||
-	    md_len != 16) {
-            goto out;
-        }
+	md5_update(&mdctx, sharedlen, (uint8_t *) shared);
+        md5_update(&mdctx, 16, input);
+        md5_digest(&mdctx, sizeof(hash), hash);
 	for (i = 0; i < 16; i++)
 	    out[offset + i] = hash[i] ^ in[offset + i];
 	if (encrypt_flag)
@@ -577,25 +569,19 @@ static int pwdcrypt(char encrypt_flag, uint8_t *in, uint8_t len, char *shared, u
 	    break;
     }
     memcpy(in, out, len);
-    result = 1;
-out:
-    EVP_MD_CTX_free(mdctx);
+
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sharedlen, uint8_t *auth, uint8_t *salt) {
-    int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned char hash[EVP_MAX_MD_SIZE];
-    unsigned int md_len;
+    struct md5_ctx mdctx;
+    unsigned char hash[MD5_DIGEST_SIZE];
     uint8_t i, offset;
 
     pthread_mutex_lock(&lock);
-    mdctx = EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
+    md5_init(&mdctx);
 
 #if 0
     printfchars(NULL, "msppencrypt auth in", "%02x ", auth, 16);
@@ -603,13 +589,10 @@ static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
     printfchars(NULL, "msppencrypt in", "%02x ", text, len);
 #endif
 
-    if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	!EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	!EVP_DigestUpdate(mdctx, auth, 16) ||
-	!EVP_DigestUpdate(mdctx, salt, 2) ||
-	!EVP_DigestFinal_ex(mdctx, hash, &md_len)) {
-        goto out;
-    }
+    md5_update(&mdctx, sharedlen, shared);
+    md5_update(&mdctx, 16, auth);
+    md5_update(&mdctx, 2, salt);
+    md5_digest(&mdctx, sizeof(hash), hash);
 
 #if 0
     printfchars(NULL, "msppencrypt hash", "%02x ", hash, 16);
@@ -623,13 +606,9 @@ static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
 	printf("text + offset - 16 c(%d): ", offset / 16);
 	printfchars(NULL, NULL, "%02x ", text + offset - 16, 16);
 #endif
-	if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	    !EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	    !EVP_DigestUpdate(mdctx, text + offset - 16, 16) ||
-	    !EVP_DigestFinal_ex(mdctx, hash, &md_len) ||
-	    md_len != 16) {
-            goto out;
-	}
+        md5_update(&mdctx, sharedlen, shared);
+        md5_update(&mdctx, 16, text + offset - 16);
+        md5_digest(&mdctx, sizeof(hash), hash);
 #if 0
 	printfchars(NULL, "msppencrypt hash", "%02x ", hash, 16);
 #endif
@@ -637,31 +616,24 @@ static int msmppencrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
 	for (i = 0; i < 16; i++)
 	    text[offset + i] ^= hash[i];
     }
-    result = 1;
 
 #if 0
     printfchars(NULL, "msppencrypt out", "%02x ", text, len);
 #endif
 
-out:
-    EVP_MD_CTX_free(mdctx);
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sharedlen, uint8_t *auth, uint8_t *salt) {
-    int result = 0;             /* Fail. */
     static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-    EVP_MD_CTX *mdctx = NULL;
-    unsigned char hash[EVP_MAX_MD_SIZE];
-    unsigned int md_len;
+    struct md5_ctx mdctx;
+    unsigned char hash[MD5_DIGEST_SIZE];
     uint8_t i, offset;
     char plain[255];
 
     pthread_mutex_lock(&lock);
-    mdctx= EVP_MD_CTX_new();
-    if (!mdctx)
-        debugx(1, DBG_ERR, "%s: malloc failed", __func__);
+    md5_init(&mdctx);
 
 #if 0
     printfchars(NULL, "msppdecrypt auth in", "%02x ", auth, 16);
@@ -669,13 +641,10 @@ static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
     printfchars(NULL, "msppdecrypt in", "%02x ", text, len);
 #endif
 
-    if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	!EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	!EVP_DigestUpdate(mdctx, auth, 16) ||
-	!EVP_DigestUpdate(mdctx, salt, 2) ||
-	!EVP_DigestFinal_ex(mdctx, hash, &md_len)) {
-        goto out;
-    }
+    md5_update(&mdctx, sharedlen, shared);
+    md5_update(&mdctx, 16, auth);
+    md5_update(&mdctx, 2, salt);
+    md5_digest(&mdctx, sizeof(hash), hash);
 
 #if 0
     printfchars(NULL, "msppdecrypt hash", "%02x ", hash, 16);
@@ -689,13 +658,9 @@ static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
 	printf("text + offset - 16 c(%d): ", offset / 16);
 	printfchars(NULL, NULL, "%02x ", text + offset - 16, 16);
 #endif
-	if (!EVP_DigestInit_ex(mdctx, EVP_md5(), NULL) ||
-	    !EVP_DigestUpdate(mdctx, shared, sharedlen) ||
-	    !EVP_DigestUpdate(mdctx, text + offset - 16, 16) ||
-	    !EVP_DigestFinal_ex(mdctx, hash, &md_len) ||
-	    md_len != 16) {
-            goto out;
-	}
+        md5_update(&mdctx, sharedlen, shared);
+        md5_update(&mdctx, 16, text + offset - 16);
+        md5_digest(&mdctx, sizeof(hash), hash);
 #if 0
 	printfchars(NULL, "msppdecrypt hash", "%02x ", hash, 16);
 #endif
@@ -705,15 +670,12 @@ static int msmppdecrypt(uint8_t *text, uint8_t len, uint8_t *shared, uint8_t sha
     }
 
     memcpy(text, plain, len);
-    result = 1;
 #if 0
     printfchars(NULL, "msppdecrypt out", "%02x ", text, len);
 #endif
 
-out:
-    EVP_MD_CTX_free(mdctx);
     pthread_mutex_unlock(&lock);
-    return result;
+    return 1;
 }
 
 struct realm *newrealmref(struct realm *r) {
diff --git a/tlscommon.h b/tlscommon.h
index 4f6eadc..de990ef 100644
--- a/tlscommon.h
+++ b/tlscommon.h
@@ -3,6 +3,11 @@
 
 #include <openssl/ssl.h>
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+#define ASN1_STRING_get0_data(o) ((o)->data)
+#define ASN1_STRING_length(o) ((o)->length)
+#endif
+
 struct tls {
     char *name;
     char *cacertfile;