From dd3a59c098be7c427eb2c2f7e4d17e6cf9a788e8 Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Fri, 13 Dec 2024 14:44:13 +1000 Subject: [PATCH 1/3] DTLS decryption threaded Add support for decryption in threads for DTLS. --- src/dtls13.c | 5 +- src/internal.c | 618 ++++++++++++++++++++++++++++----------------- src/ssl.c | 68 ++++- src/tls13.c | 7 +- tests/api.c | 2 +- wolfssl/internal.h | 12 +- wolfssl/ssl.h | 6 + 7 files changed, 471 insertions(+), 247 deletions(-) diff --git a/src/dtls13.c b/src/dtls13.c index 5011f7d85b..0ca7138aa1 100644 --- a/src/dtls13.c +++ b/src/dtls13.c @@ -2666,9 +2666,6 @@ int DoDtls13Ack(WOLFSSL* ssl, const byte* input, word32 inputSize, const byte* ackMessage; w64wrapper epoch, seq; word16 length; -#ifndef WOLFSSL_RW_THREADED - int ret; -#endif int i; if (inputSize < OPAQUE16_LEN) @@ -2702,7 +2699,7 @@ int DoDtls13Ack(WOLFSSL* ssl, const byte* input, word32 inputSize, #ifndef WOLFSSL_RW_THREADED if (ssl->dtls13WaitKeyUpdateAck) { - ret = DoDtls13KeyUpdateAck(ssl); + int ret = DoDtls13KeyUpdateAck(ssl); if (ret != 0) return ret; } diff --git a/src/internal.c b/src/internal.c index 756f2812fc..ef53cf8d1a 100644 --- a/src/internal.c +++ b/src/internal.c @@ -7350,6 +7350,9 @@ int InitSSL(WOLFSSL* ssl, WOLFSSL_CTX* ctx, int writeDup) for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { ssl->buffers.encrypt[i].avail = 1; } + for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { + ssl->buffers.decrypt[i].avail = 1; + } } #endif @@ -8234,7 +8237,21 @@ void wolfSSL_ResourceFree(WOLFSSL* ssl) bufferStatic* buff = &ssl->buffers.encrypt[i].buffer; ssl->buffers.encrypt[i].stop = 1; - FreeCiphersSide(&ssl->buffers.encrypt[i].encrypt, ssl->heap); + FreeCiphersSide(&ssl->buffers.encrypt[i].cipher, ssl->heap); + if (buff->dynamicFlag) { + XFREE(buff->buffer - buff->offset, ssl->heap, + DYNAMIC_TYPE_OUT_BUFFER); + buff->buffer = buff->staticBuffer; + buff->bufferSize = STATIC_BUFFER_LEN; + buff->offset = 0; + buff->dynamicFlag = 0; + } + } + for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { + bufferStatic* buff = &ssl->buffers.decrypt[i].buffer; + + ssl->buffers.decrypt[i].stop = 1; + FreeCiphersSide(&ssl->buffers.decrypt[i].cipher, ssl->heap); if (buff->dynamicFlag) { XFREE(buff->buffer - buff->offset, ssl->heap, DYNAMIC_TYPE_OUT_BUFFER); @@ -10761,7 +10778,6 @@ int SendBuffered(WOLFSSL* ssl) return 0; } -#ifdef WOLFSSL_THREADED_CRYPT static WC_INLINE int GrowAnOutputBuffer(WOLFSSL* ssl, bufferStatic* outputBuffer, int size) { @@ -10773,6 +10789,7 @@ static WC_INLINE int GrowAnOutputBuffer(WOLFSSL* ssl, #else const byte align = WOLFSSL_GENERAL_ALIGNMENT; #endif + word32 newSz; #if WOLFSSL_GENERAL_ALIGNMENT > 0 /* the encrypted data will be offset from the front of the buffer by @@ -10783,8 +10800,14 @@ static WC_INLINE int GrowAnOutputBuffer(WOLFSSL* ssl, align *= 2; #endif - tmp = (byte*)XMALLOC(size + outputBuffer->length + align, - ssl->heap, DYNAMIC_TYPE_OUT_BUFFER); + if (! WC_SAFE_SUM_WORD32(outputBuffer->idx, outputBuffer->length, newSz)) + return BUFFER_E; + if (! WC_SAFE_SUM_WORD32(newSz, (word32)size, newSz)) + return BUFFER_E; + if (! WC_SAFE_SUM_WORD32(newSz, align, newSz)) + return BUFFER_E; + tmp = (byte*)XMALLOC(newSz, ssl->heap, DYNAMIC_TYPE_OUT_BUFFER); + newSz -= align; WOLFSSL_MSG("growing output buffer"); if (tmp == NULL) @@ -10798,7 +10821,7 @@ static WC_INLINE int GrowAnOutputBuffer(WOLFSSL* ssl, #ifdef WOLFSSL_STATIC_MEMORY /* can be from IO memory pool which does not need copy if same buffer */ if (outputBuffer->length && tmp == outputBuffer->buffer) { - outputBuffer->bufferSize = size + outputBuffer->length; + outputBuffer->bufferSize = newSz; return 0; } #endif @@ -10810,6 +10833,7 @@ static WC_INLINE int GrowAnOutputBuffer(WOLFSSL* ssl, XFREE(outputBuffer->buffer - outputBuffer->offset, ssl->heap, DYNAMIC_TYPE_OUT_BUFFER); } + outputBuffer->dynamicFlag = 1; #if WOLFSSL_GENERAL_ALIGNMENT > 0 if (align) @@ -10819,11 +10843,9 @@ static WC_INLINE int GrowAnOutputBuffer(WOLFSSL* ssl, outputBuffer->offset = 0; outputBuffer->buffer = tmp; - outputBuffer->dynamicFlag = 1; - outputBuffer->bufferSize = size + outputBuffer->length; + outputBuffer->bufferSize = newSz; return 0; } -#endif /* returns the current location in the output buffer to start writing to */ byte* GetOutputBuffer(WOLFSSL* ssl) @@ -10836,80 +10858,13 @@ byte* GetOutputBuffer(WOLFSSL* ssl) /* Grow the output buffer */ static WC_INLINE int GrowOutputBuffer(WOLFSSL* ssl, int size) { - byte* tmp; -#if WOLFSSL_GENERAL_ALIGNMENT > 0 - byte hdrSz = ssl->options.dtls ? DTLS_RECORD_HEADER_SZ : - RECORD_HEADER_SZ; - byte align = WOLFSSL_GENERAL_ALIGNMENT; -#else - const byte align = WOLFSSL_GENERAL_ALIGNMENT; -#endif - word32 newSz; - -#if WOLFSSL_GENERAL_ALIGNMENT > 0 - /* the encrypted data will be offset from the front of the buffer by - the header, if the user wants encrypted alignment they need - to define their alignment requirement */ - - while (align < hdrSz) - align *= 2; -#endif - - if (! WC_SAFE_SUM_WORD32(ssl->buffers.outputBuffer.idx, - ssl->buffers.outputBuffer.length, newSz)) - return BUFFER_E; - if (! WC_SAFE_SUM_WORD32(newSz, (word32)size, newSz)) - return BUFFER_E; - if (! WC_SAFE_SUM_WORD32(newSz, align, newSz)) - return BUFFER_E; - tmp = (byte*)XMALLOC(newSz, ssl->heap, DYNAMIC_TYPE_OUT_BUFFER); - newSz -= align; - WOLFSSL_MSG("growing output buffer"); - - if (tmp == NULL) - return MEMORY_E; - -#if WOLFSSL_GENERAL_ALIGNMENT > 0 - if (align) - tmp += align - hdrSz; -#endif - -#ifdef WOLFSSL_STATIC_MEMORY - /* can be from IO memory pool which does not need copy if same buffer */ - if (ssl->buffers.outputBuffer.length && - tmp == ssl->buffers.outputBuffer.buffer) { - ssl->buffers.outputBuffer.bufferSize = newSz; - return 0; - } -#endif - - if (ssl->buffers.outputBuffer.length) - XMEMCPY(tmp, ssl->buffers.outputBuffer.buffer, - ssl->buffers.outputBuffer.idx + - ssl->buffers.outputBuffer.length); - - if (ssl->buffers.outputBuffer.dynamicFlag) { - XFREE(ssl->buffers.outputBuffer.buffer - - ssl->buffers.outputBuffer.offset, ssl->heap, - DYNAMIC_TYPE_OUT_BUFFER); - } - ssl->buffers.outputBuffer.dynamicFlag = 1; - -#if WOLFSSL_GENERAL_ALIGNMENT > 0 - if (align) - ssl->buffers.outputBuffer.offset = align - hdrSz; - else -#endif - ssl->buffers.outputBuffer.offset = 0; - - ssl->buffers.outputBuffer.buffer = tmp; - ssl->buffers.outputBuffer.bufferSize = newSz; - return 0; + return GrowAnOutputBuffer(ssl, &ssl->buffers.outputBuffer, size); } /* Grow the input buffer, should only be to read cert or big app data */ -int GrowInputBuffer(WOLFSSL* ssl, int size, int usedLength) +static int GrowAnInputBuffer(WOLFSSL* ssl, bufferStatic* inputBuffer, int size, + int usedLength) { byte* tmp; #if defined(WOLFSSL_DTLS) || WOLFSSL_GENERAL_ALIGNMENT > 0 @@ -10950,43 +10905,46 @@ int GrowInputBuffer(WOLFSSL* ssl, int size, int usedLength) #ifdef WOLFSSL_STATIC_MEMORY /* can be from IO memory pool which does not need copy if same buffer */ - if (usedLength && tmp == ssl->buffers.inputBuffer.buffer) { - ssl->buffers.inputBuffer.bufferSize = size + usedLength; - ssl->buffers.inputBuffer.idx = 0; - ssl->buffers.inputBuffer.length = usedLength; + if (usedLength && tmp == inputBuffer->buffer) { + inputBuffer->bufferSize = size + usedLength; + inputBuffer->idx = 0; + inputBuffer->length = usedLength; return 0; } #endif if (usedLength) - XMEMCPY(tmp, ssl->buffers.inputBuffer.buffer + - ssl->buffers.inputBuffer.idx, usedLength); + XMEMCPY(tmp, inputBuffer->buffer + inputBuffer->idx, usedLength); - if (ssl->buffers.inputBuffer.dynamicFlag) { + if (inputBuffer->dynamicFlag) { if (IsEncryptionOn(ssl, 1)) { - ForceZero(ssl->buffers.inputBuffer.buffer, - ssl->buffers.inputBuffer.length); + ForceZero(inputBuffer->buffer, inputBuffer->length); } - XFREE(ssl->buffers.inputBuffer.buffer - ssl->buffers.inputBuffer.offset, - ssl->heap, DYNAMIC_TYPE_IN_BUFFER); + XFREE(inputBuffer->buffer - inputBuffer->offset, ssl->heap, + DYNAMIC_TYPE_IN_BUFFER); } - ssl->buffers.inputBuffer.dynamicFlag = 1; + inputBuffer->dynamicFlag = 1; #if defined(WOLFSSL_DTLS) || WOLFSSL_GENERAL_ALIGNMENT > 0 if (align) - ssl->buffers.inputBuffer.offset = align - hdrSz; + inputBuffer->offset = align - hdrSz; else #endif - ssl->buffers.inputBuffer.offset = 0; + inputBuffer->offset = 0; - ssl->buffers.inputBuffer.buffer = tmp; - ssl->buffers.inputBuffer.bufferSize = (word32)(size + usedLength); - ssl->buffers.inputBuffer.idx = 0; - ssl->buffers.inputBuffer.length = (word32)usedLength; + inputBuffer->buffer = tmp; + inputBuffer->bufferSize = (word32)(size + usedLength); + inputBuffer->idx = 0; + inputBuffer->length = (word32)usedLength; return 0; } +/* Grow the input buffer, should only be to read cert or big app data */ +int GrowInputBuffer(WOLFSSL* ssl, int size, int usedLength) +{ + return GrowAnInputBuffer(ssl, &ssl->buffers.inputBuffer, size, usedLength); +} /* Check available size into output buffer, make room if needed. * This function needs to be called before anything gets put @@ -21331,6 +21289,77 @@ static int removeMsgInnerPadding(WOLFSSL* ssl) } #endif +#ifdef WOLFSSL_THREADED_CRYPT +static int ReceiveAsyncData(WOLFSSL* ssl) +{ + int i; + int cnt; + + do { + cnt = 0; + for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { + ThreadCrypt* decrypt = &ssl->buffers.decrypt[i]; + + if (decrypt->avail) { + cnt++; + } + else if (decrypt->done) { + int ret; + int error; + + + /* Parse record header again. */ + GrowInputBuffer(ssl, decrypt->recordHdrLen, 0); + XMEMCPY(ssl->buffers.inputBuffer.buffer, decrypt->recordHdr, + decrypt->recordHdrLen); + ssl->buffers.inputBuffer.length = decrypt->recordHdrLen; + ssl->buffers.inputBuffer.idx = 0; + ret = GetRecordHeader(ssl, &ssl->buffers.inputBuffer.idx, + &ssl->curRL, &ssl->curSize); + if (ret != 0) { + return ret; + } + + GrowInputBuffer(ssl, decrypt->buffer.length, 0); + XMEMCPY(ssl->buffers.inputBuffer.buffer, decrypt->buffer.buffer, + decrypt->buffer.length); + ssl->buffers.inputBuffer.length = decrypt->buffer.length; + ssl->buffers.inputBuffer.idx = RECORD_HEADER_SZ; + + decrypt->done = 0; + decrypt->avail = 1; + + ssl->curStartIdx = ssl->buffers.inputBuffer.idx; + ssl->options.processReply = runProcessingOneRecord; + if ((error = ProcessReply(ssl)) < 0) { + if (error == WC_NO_ERR_TRACE(ZERO_RETURN)) { + ssl->error = error; + WOLFSSL_MSG("Zero return, no more data coming"); + return 0; /* no more data coming */ + } + if (error == WC_NO_ERR_TRACE(SOCKET_ERROR_E)) { + if (ssl->options.connReset || ssl->options.isClosed) { + WOLFSSL_MSG( + "Peer reset or closed, connection done"); + error = SOCKET_PEER_CLOSED_E; + ssl->error = error; + WOLFSSL_ERROR(error); + return 0; /* peer reset or closed */ + } + } + ssl->error = error; + WOLFSSL_ERROR(error); + return error; + } + } + } + } + while (cnt == 0); + + return 0; +} +#endif + int ProcessReply(WOLFSSL* ssl) { return ProcessReplyEx(ssl, 0); @@ -21347,6 +21376,13 @@ int ProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) #if defined(WOLFSSL_DTLS) int used; #endif +#ifdef WOLFSSL_THREADED_CRYPT + int recordHdrIdx; + byte recordHdr[DTLS_RECORD_HEADER_MAX_SZ]; + int recordHdrLen = 0; + byte origRecordHdr[DTLS_RECORD_HEADER_MAX_SZ + 16]; + int origRecordHdrLen = 0; +#endif #ifdef ATOMIC_USER if (ssl->ctx->DecryptVerifyCb) @@ -21530,6 +21566,15 @@ int ProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) /* get the record layer header */ case getRecordLayerHeader: +#ifdef WOLFSSL_THREADED_CRYPT + recordHdrIdx = ssl->buffers.inputBuffer.idx; + origRecordHdrLen = min(ssl->buffers.inputBuffer.length, + DTLS_RECORD_HEADER_MAX_SZ + 16); + XMEMCPY(origRecordHdr, + ssl->buffers.inputBuffer.buffer + recordHdrIdx, + origRecordHdrLen); +#endif + /* DTLSv1.3 record numbers in the header are encrypted, and AAD * uses the unencrypted form. Because of this we need to modify the * header, decrypting the numbers inside @@ -21538,6 +21583,15 @@ int ProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) ret = GetRecordHeader(ssl, &ssl->buffers.inputBuffer.idx, &ssl->curRL, &ssl->curSize); +#ifdef WOLFSSL_THREADED_CRYPT + if (ret == 0) { + recordHdrLen = ssl->buffers.inputBuffer.idx - recordHdrIdx; + XMEMCPY(recordHdr, + ssl->buffers.inputBuffer.buffer + recordHdrIdx, + recordHdrLen); + } +#endif + #ifdef WOLFSSL_DTLS if (ssl->options.dtls && DtlsShouldDrop(ssl, ret)) { ssl->options.processReply = doProcessInit; @@ -21697,6 +21751,75 @@ int ProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) return ret; } +#ifdef WOLFSSL_THREADED_CRYPT + if (ssl->options.handShakeDone && + ssl->buffers.decryptSignalRegistered) { + int i; + ThreadCrypt* decrypt = NULL; + byte *aad = (byte*)&ssl->curRL; + word16 aad_size = RECORD_HEADER_SZ; + #ifdef WOLFSSL_DTLS13 + if (ssl->options.dtls) { + /* aad now points to the record header */ + aad = ssl->dtls13CurRL; + aad_size = ssl->dtls13CurRlLength; + } + #endif /* WOLFSSL_DTLS13 */ + + WOLFSSL_MSG("Not decrypting\n"); + + for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { + if (ssl->buffers.decrypt[i].avail) { + decrypt = &ssl->buffers.decrypt[i]; + break; + } + } + decrypt->done = 0; + decrypt->avail = 0; + + GrowAnInputBuffer(ssl, &decrypt->buffer, + recordHdrLen + ssl->curSize, 0); + + XMEMCPY(decrypt->buffer.buffer, recordHdr, recordHdrLen); + decrypt->offset = recordHdrLen; + XMEMCPY(decrypt->recordHdr, origRecordHdr, + origRecordHdrLen); + decrypt->recordHdrLen = origRecordHdrLen; + XMEMCPY(decrypt->buffer.buffer + decrypt->offset, + in->buffer + in->idx, ssl->curSize); + decrypt->buffer.length = recordHdrLen + ssl->curSize; + decrypt->cryptLen = ssl->curSize; + + if (!decrypt->init) { + SetKeys(NULL, &decrypt->cipher, &ssl->keys, &ssl->specs, + ssl->options.side, ssl->heap, ssl->devId, ssl->rng, + ssl->options.tls1_3); + decrypt->init = 1; + } + #ifdef HAVE_TRUNCATED_HMAC + if (ssl->truncated_hmac) { + decrypt->cryptLen -= min(TRUNCATED_HMAC_SZ, + ssl->specs.hash_size); + } + else + #endif + { + decrypt->cryptLen -= ssl->specs.aead_mac_size; + } + + XMEMCPY(decrypt->additional, aad, aad_size); + BuildTls13Nonce(ssl, decrypt->nonce, + ssl->keys.aead_dec_imp_IV, PEER_ORDER); + + if (decrypt->signal != NULL) { + decrypt->signal(decrypt->signalCtx, ssl); + } + + ssl->options.processReply = doProcessInit; + in->idx += ssl->curSize; + return ret; + } +#endif if (atomicUser) { #ifdef ATOMIC_USER #if defined(HAVE_ENCRYPT_THEN_MAC) && !defined(WOLFSSL_AEAD_ONLY) @@ -23292,51 +23415,51 @@ int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, const byte* input, } #endif -#ifdef WOLFSSL_THREADED_CRYPT - if (asyncOkay) { - WOLFSSL_MSG("Not encrypting\n"); - /* make sure build message state is reset */ - ssl->options.buildMsgState = BUILD_MSG_BEGIN; + #ifdef WOLFSSL_THREADED_CRYPT + if (asyncOkay && ssl->buffers.encryptSignalRegistered) { + WOLFSSL_MSG("Not encrypting\n"); + /* make sure build message state is reset */ + ssl->options.buildMsgState = BUILD_MSG_BEGIN; - /* return sz on success */ - if (ret == 0) { - ret = args->sz; - } - else { - WOLFSSL_ERROR_VERBOSE(ret); - } + /* return sz on success */ + if (ret == 0) { + ret = args->sz; + } + else { + WOLFSSL_ERROR_VERBOSE(ret); + } - /* Final cleanup */ - FreeBuildMsgArgs(ssl, args); + /* Final cleanup */ + FreeBuildMsgArgs(ssl, args); - return ret; - } - else -#endif - { - #if defined(HAVE_ENCRYPT_THEN_MAC) && !defined(WOLFSSL_AEAD_ONLY) - if (ssl->options.startedETMWrite) { - ret = Encrypt(ssl, output + args->headerSz, - output + args->headerSz, - (word16)(args->size - args->digestSz), - asyncOkay, args->type); + return ret; } else - #endif + #endif { - ret = Encrypt(ssl, output + args->headerSz, - output + args->headerSz, args->size, asyncOkay, - args->type); - } - #if defined(HAVE_SECURE_RENEGOTIATION) && defined(WOLFSSL_DTLS) - /* Restore sequence numbers */ - if (swap_seq) { - ssl->keys.dtls_epoch = dtls_epoch; - ssl->keys.dtls_sequence_number_hi = dtls_sequence_number_hi; - ssl->keys.dtls_sequence_number_lo = dtls_sequence_number_lo; + #if defined(HAVE_ENCRYPT_THEN_MAC) && !defined(WOLFSSL_AEAD_ONLY) + if (ssl->options.startedETMWrite) { + ret = Encrypt(ssl, output + args->headerSz, + output + args->headerSz, + (word16)(args->size - args->digestSz), + asyncOkay, args->type); + } + else + #endif + { + ret = Encrypt(ssl, output + args->headerSz, + output + args->headerSz, args->size, + asyncOkay, args->type); + } + #if defined(HAVE_SECURE_RENEGOTIATION) && defined(WOLFSSL_DTLS) + /* Restore sequence numbers */ + if (swap_seq) { + ssl->keys.dtls_epoch = dtls_epoch; + ssl->keys.dtls_sequence_number_hi = dtls_sequence_number_hi; + ssl->keys.dtls_sequence_number_lo = dtls_sequence_number_lo; + } + #endif } - #endif - } } if (ret != 0) { @@ -24982,15 +25105,17 @@ int SendData(WOLFSSL* ssl, const void* data, int sz) } #ifdef WOLFSSL_THREADED_CRYPT - ret = SendAsyncData(ssl); - if (ret != 0) { - ssl->error = ret; - return WOLFSSL_FATAL_ERROR; - } - if (ssl->dtls13WaitKeyUpdateAck) { - ret = DoDtls13KeyUpdateAck(ssl); - if (ret != 0) - return ret; + if (ssl->buffers.encryptSignalRegistered) { + ret = SendAsyncData(ssl); + if (ret != 0) { + ssl->error = ret; + return WOLFSSL_FATAL_ERROR; + } + if (ssl->dtls13WaitKeyUpdateAck) { + ret = DoDtls13KeyUpdateAck(ssl); + if (ret != 0) + return ret; + } } #endif @@ -25093,30 +25218,34 @@ int SendData(WOLFSSL* ssl, const void* data, int sz) return (ssl->error = ret); /* get output buffer */ -#ifndef WOLFSSL_THREADED_CRYPT - out = GetOutputBuffer(ssl); -#else - do { - for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { - if (ssl->buffers.encrypt[i].avail) { - encrypt = &ssl->buffers.encrypt[i]; - break; +#ifdef WOLFSSL_THREADED_CRYPT + if (ssl->buffers.encryptSignalRegistered) { + do { + for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { + if (ssl->buffers.encrypt[i].avail) { + encrypt = &ssl->buffers.encrypt[i]; + break; + } } - } - if (encrypt == NULL) { - ret = SendAsyncData(ssl); - if (ret != 0) { - ssl->error = ret; - return WOLFSSL_FATAL_ERROR; + if (encrypt == NULL) { + ret = SendAsyncData(ssl); + if (ret != 0) { + ssl->error = ret; + return WOLFSSL_FATAL_ERROR; + } } } + while (encrypt == NULL); + encrypt->done = 0; + encrypt->avail = 0; + GrowAnOutputBuffer(ssl, &encrypt->buffer, outputSz); + out = encrypt->buffer.buffer; } - while (encrypt == NULL); - encrypt->done = 0; - encrypt->avail = 0; - GrowAnOutputBuffer(ssl, &encrypt->buffer, outputSz); - out = encrypt->buffer.buffer; + else #endif + { + out = GetOutputBuffer(ssl); + } #ifdef HAVE_LIBZ if (ssl->options.usingCompression) { @@ -25161,79 +25290,86 @@ int SendData(WOLFSSL* ssl, const void* data, int sz) FreeAsyncCtx(ssl, 0); #endif #ifdef WOLFSSL_THREADED_CRYPT - if (!encrypt->init) { - SetKeys(&encrypt->encrypt, NULL, &ssl->keys, &ssl->specs, - ssl->options.side, ssl->heap, ssl->devId, ssl->rng, - ssl->options.tls1_3); - encrypt->init = 1; - } - encrypt->buffer.length = sendSz; - encrypt->offset = RECORD_HEADER_SZ; - if (ssl->options.dtls) { - encrypt->offset += DTLS_RECORD_EXTRA; - } - encrypt->cryptLen = outputSz - encrypt->offset; - #ifdef HAVE_TRUNCATED_HMAC - if (ssl->truncated_hmac) { - encrypt->cryptLen -= min(TRUNCATED_HMAC_SZ, ssl->specs.hash_size); - } - else - #endif - { - encrypt->cryptLen -= ssl->specs.hash_size; - } - -#if !defined(NO_PUBLIC_GCM_SET_IV) && \ - ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ - (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) - XMEMCPY(encrypt->nonce, ssl->keys.aead_enc_imp_IV, AESGCM_IMP_IV_SZ); - XMEMCPY(encrypt->nonce + AESGCM_IMP_IV_SZ, ssl->keys.aead_exp_IV, - AESGCM_EXP_IV_SZ); -#endif - XMEMSET(encrypt->additional, 0, AEAD_AUTH_DATA_SZ); - WriteSEQ(ssl, CUR_ORDER, encrypt->additional); - XMEMCPY(encrypt->additional + AEAD_TYPE_OFFSET, encrypt->buffer.buffer, - 3); - c16toa(sendSz - encrypt->offset - AESGCM_EXP_IV_SZ - - ssl->specs.aead_mac_size, encrypt->additional + AEAD_LEN_OFFSET); + if (ssl->buffers.encryptSignalRegistered) { + if (!encrypt->init) { + SetKeys(&encrypt->cipher, NULL, &ssl->keys, &ssl->specs, + ssl->options.side, ssl->heap, ssl->devId, ssl->rng, + ssl->options.tls1_3); + encrypt->init = 1; + } + encrypt->buffer.length = sendSz; + encrypt->offset = RECORD_HEADER_SZ; + if (ssl->options.dtls) { + encrypt->offset += DTLS_RECORD_EXTRA; + } + encrypt->cryptLen = outputSz - encrypt->offset; + #ifdef HAVE_TRUNCATED_HMAC + if (ssl->truncated_hmac) { + encrypt->cryptLen -= min(TRUNCATED_HMAC_SZ, + ssl->specs.hash_size); + } + else + #endif + { + encrypt->cryptLen -= ssl->specs.hash_size; + } - #ifdef WOLFSSL_DTLS - if (ssl->options.dtls) - DtlsSEQIncrement(ssl, CUR_ORDER); + #if !defined(NO_PUBLIC_GCM_SET_IV) && \ + ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ + (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) + XMEMCPY(encrypt->nonce, ssl->keys.aead_enc_imp_IV, + AESGCM_IMP_IV_SZ); + XMEMCPY(encrypt->nonce + AESGCM_IMP_IV_SZ, ssl->keys.aead_exp_IV, + AESGCM_EXP_IV_SZ); #endif + XMEMSET(encrypt->additional, 0, AEAD_AUTH_DATA_SZ); + WriteSEQ(ssl, CUR_ORDER, encrypt->additional); + XMEMCPY(encrypt->additional + AEAD_TYPE_OFFSET, + encrypt->buffer.buffer, 3); + c16toa(sendSz - encrypt->offset - AESGCM_EXP_IV_SZ - + ssl->specs.aead_mac_size, + encrypt->additional + AEAD_LEN_OFFSET); - if (encrypt->signal != NULL) { - encrypt->signal(encrypt->signalCtx, ssl); + #ifdef WOLFSSL_DTLS + if (ssl->options.dtls) + DtlsSEQIncrement(ssl, CUR_ORDER); + #endif + + if (encrypt->signal != NULL) { + encrypt->signal(encrypt->signalCtx, ssl); + } + return sendSz; } - return sendSz; -#else - ssl->buffers.outputBuffer.length += (word32)sendSz; + else +#endif + { + ssl->buffers.outputBuffer.length += (word32)sendSz; - if ( (error = SendBuffered(ssl)) < 0) { - ssl->error = error; - WOLFSSL_ERROR(error); - /* store for next call if WANT_WRITE or user embedSend() that - doesn't present like WANT_WRITE */ - ssl->buffers.plainSz = buffSz; - ssl->buffers.prevSent = sent; - if (error == WC_NO_ERR_TRACE(SOCKET_ERROR_E) && - (ssl->options.connReset || ssl->options.isClosed)) { - error = SOCKET_PEER_CLOSED_E; - ssl->error = SOCKET_PEER_CLOSED_E; + if ( (error = SendBuffered(ssl)) < 0) { + ssl->error = error; WOLFSSL_ERROR(error); - return 0; /* peer reset or closed */ + /* store for next call if WANT_WRITE or user embedSend() that + doesn't present like WANT_WRITE */ + ssl->buffers.plainSz = buffSz; + ssl->buffers.prevSent = sent; + if (error == WC_NO_ERR_TRACE(SOCKET_ERROR_E) && + (ssl->options.connReset || ssl->options.isClosed)) { + error = SOCKET_PEER_CLOSED_E; + ssl->error = SOCKET_PEER_CLOSED_E; + WOLFSSL_ERROR(error); + return 0; /* peer reset or closed */ + } + return error; } - return error; - } - sent += buffSz; + sent += buffSz; - /* only one message per attempt */ - if (ssl->options.partialWrite == 1) { - WOLFSSL_MSG("Partial Write on, only sending one record"); - break; + /* only one message per attempt */ + if (ssl->options.partialWrite == 1) { + WOLFSSL_MSG("Partial Write on, only sending one record"); + break; + } } -#endif } return sent; @@ -25286,19 +25422,17 @@ int ReceiveData(WOLFSSL* ssl, byte* output, int sz, int peek) } else #endif - { - if (ssl_in_handshake(ssl, 0)) { - int err; - WOLFSSL_MSG("Handshake not complete, trying to finish"); - if ( (err = wolfSSL_negotiate(ssl)) != WOLFSSL_SUCCESS) { - #ifdef WOLFSSL_ASYNC_CRYPT - /* if async would block return WANT_WRITE */ - if (ssl->error == WC_NO_ERR_TRACE(WC_PENDING_E)) { - return WOLFSSL_CBIO_ERR_WANT_READ; - } - #endif - return err; + if (ssl_in_handshake(ssl, 0)) { + int err; + WOLFSSL_MSG("Handshake not complete, trying to finish"); + if ( (err = wolfSSL_negotiate(ssl)) != WOLFSSL_SUCCESS) { + #ifdef WOLFSSL_ASYNC_CRYPT + /* if async would block return WANT_WRITE */ + if (ssl->error == WC_NO_ERR_TRACE(WC_PENDING_E)) { + return WOLFSSL_CBIO_ERR_WANT_READ; } + #endif + return err; } } @@ -25315,6 +25449,16 @@ int ReceiveData(WOLFSSL* ssl, byte* output, int sz, int peek) #endif while (ssl->buffers.clearOutputBuffer.length == 0) { + #ifdef WOLFSSL_THREADED_CRYPT + if (ssl->buffers.decryptSignalRegistered && !ssl_in_handshake(ssl, 0)) { + error = ReceiveAsyncData(ssl); + if (error != 0) { + ssl->error = error; + WOLFSSL_ERROR(error); + return WOLFSSL_FATAL_ERROR; + } + } + #endif if ( (error = ProcessReply(ssl)) < 0) { if (error == WC_NO_ERR_TRACE(ZERO_RETURN)) { ssl->error = error; diff --git a/src/ssl.c b/src/ssl.c index b11ed59a7e..0afddab226 100644 --- a/src/ssl.c +++ b/src/ssl.c @@ -23930,7 +23930,7 @@ int wolfSSL_AsyncEncrypt(WOLFSSL* ssl, int idx) #else wc_AesGcmEncrypt #endif - (encrypt->encrypt.aes, + (encrypt->cipher.aes, out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ, encSz - AESGCM_EXP_IV_SZ - ssl->specs.aead_mac_size, encrypt->nonce, AESGCM_NONCE_SZ, @@ -23959,6 +23959,72 @@ int wolfSSL_AsyncEncryptSetSignal(WOLFSSL* ssl, int idx, else { ssl->buffers.encrypt[idx].signal = signal; ssl->buffers.encrypt[idx].signalCtx = ctx; + + ssl->buffers.encryptSignalRegistered = 1; + } + + return ret; +} + + +int wolfSSL_AsyncDecryptReady(WOLFSSL* ssl, int idx) +{ + ThreadCrypt* decrypt; + + if (ssl == NULL) { + return 0; + } + + decrypt = &ssl->buffers.decrypt[idx]; + return (decrypt->avail == 0) && (decrypt->done == 0); +} + +int wolfSSL_AsyncDecryptStop(WOLFSSL* ssl, int idx) +{ + ThreadCrypt* decrypt; + + if (ssl == NULL) { + return 1; + } + + decrypt = &ssl->buffers.decrypt[idx]; + return decrypt->stop; +} + +int wolfSSL_AsyncDecrypt(WOLFSSL* ssl, int idx) +{ + int ret = WC_NO_ERR_TRACE(NOT_COMPILED_IN); + ThreadCrypt* decrypt = &ssl->buffers.decrypt[idx]; + + if (ssl->specs.bulk_cipher_algorithm == wolfssl_aes_gcm) { + unsigned char* out = decrypt->buffer.buffer + decrypt->offset; + unsigned char* input = decrypt->buffer.buffer + decrypt->offset; + unsigned char* tag = input + decrypt->cryptLen; + + ret = wc_AesGcmDecrypt(decrypt->cipher.aes, out, input, + decrypt->cryptLen, + decrypt->nonce, AESGCM_NONCE_SZ, + tag, ssl->specs.aead_mac_size, + decrypt->additional, RECORD_HEADER_SZ); + decrypt->done = 1; + } + + return ret; +} + +int wolfSSL_AsyncDecryptSetSignal(WOLFSSL* ssl, int idx, + WOLFSSL_THREAD_SIGNAL signal, void* ctx) +{ + int ret = 0; + + if (ssl == NULL) { + ret = BAD_FUNC_ARG; + } + else { + ssl->buffers.decrypt[idx].signal = signal; + ssl->buffers.decrypt[idx].signalCtx = ctx; + + ssl->buffers.decryptSignalRegistered = 1; } return ret; diff --git a/src/tls13.c b/src/tls13.c index 0d5a8b9365..96c9ae9c48 100644 --- a/src/tls13.c +++ b/src/tls13.c @@ -2425,8 +2425,10 @@ static WC_INLINE void WriteSEQTls13(WOLFSSL* ssl, int verifyOrder, byte* out) * iv The derived IV. * order The side on which the message is to be or was sent. */ -static WC_INLINE void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, - int order) +#ifndef WOLFSSL_THREADED_CRYPT +static WC_INLINE +#endif +void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, int order) { int i; @@ -2655,7 +2657,6 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, if (ret == WC_NO_ERR_TRACE(NOT_COMPILED_IN)) #endif { - #if ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) ret = wc_AesGcmEncrypt(ssl->encrypt.aes, output, input, diff --git a/tests/api.c b/tests/api.c index e7b64124a6..46e7579c5d 100644 --- a/tests/api.c +++ b/tests/api.c @@ -90683,6 +90683,7 @@ static void test_AEAD_limit_client(WOLFSSL* ssl) /* Key update should be sent and negotiated */ ret = wolfSSL_read(ssl, msgBuf, sizeof(msgBuf)); AssertIntGT(ret, 0); + ret = wolfSSL_write(ssl, msgBuf, sizeof(msgBuf)); /* Epoch after one key update is 4 */ if (w64Equal(ssl->dtls13PeerEpoch, w64From32(0, 4)) && w64Equal(Dtls13GetEpoch(ssl, ssl->dtls13PeerEpoch)->dropCount, counter)) { @@ -90751,7 +90752,6 @@ static void test_AEAD_limit_server(WOLFSSL* ssl) test_AEAD_seq_num = 0; } (void)wolfSSL_read(ssl, msgBuf, sizeof(msgBuf)); - ret = wolfSSL_write(ssl, msgBuf, sizeof(msgBuf)); nanosleep(&delay, NULL); } } diff --git a/wolfssl/internal.h b/wolfssl/internal.h index f5ce5b02ef..dc9afda97b 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -4743,10 +4743,12 @@ enum AcceptStateTls13 { #include typedef struct ThreadCrypt { - Ciphers encrypt; + Ciphers cipher; bufferStatic buffer; unsigned char nonce[AESGCM_NONCE_SZ]; unsigned char additional[AEAD_AUTH_DATA_SZ]; + unsigned char recordHdr[DTLS_RECORD_HEADER_MAX_SZ + 16]; + word32 recordHdrLen; int init; int offset; int cryptLen; @@ -4764,7 +4766,10 @@ typedef struct Buffers { bufferStatic inputBuffer; bufferStatic outputBuffer; #ifdef WOLFSSL_THREADED_CRYPT + int encryptSignalRegistered; ThreadCrypt encrypt[WOLFSSL_THREADED_CRYPT_CNT]; + int decryptSignalRegistered; + ThreadCrypt decrypt[WOLFSSL_THREADED_CRYPT_CNT]; #endif buffer domainName; /* for client check */ buffer clearOutputBuffer; @@ -6867,6 +6872,11 @@ WOLFSSL_LOCAL int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, int sizeOnly, int asyncOkay, int epochOrder); #ifdef WOLFSSL_TLS13 +#ifdef WOLFSSL_THREADED_CRYPT +WOLFSSL_LOCAL void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, + int order); +#endif + /* Use WOLFSSL_API to use this function in tests/api.c */ WOLFSSL_API int BuildTls13Message(WOLFSSL* ssl, byte* output, int outSz, const byte* input, int inSz, int type, int hashOutput, int sizeOnly, int asyncOkay); diff --git a/wolfssl/ssl.h b/wolfssl/ssl.h index 8989f52044..07f4ed1598 100644 --- a/wolfssl/ssl.h +++ b/wolfssl/ssl.h @@ -3718,6 +3718,12 @@ WOLFSSL_API int wolfSSL_AsyncEncryptStop(WOLFSSL* ssl, int idx); WOLFSSL_API int wolfSSL_AsyncEncrypt(WOLFSSL* ssl, int idx); WOLFSSL_API int wolfSSL_AsyncEncryptSetSignal(WOLFSSL* ssl, int idx, WOLFSSL_THREAD_SIGNAL signal, void* ctx); + +WOLFSSL_API int wolfSSL_AsyncDecryptReady(WOLFSSL* ssl, int idx); +WOLFSSL_API int wolfSSL_AsyncDecryptStop(WOLFSSL* ssl, int idx); +WOLFSSL_API int wolfSSL_AsyncDecrypt(WOLFSSL* ssl, int idx); +WOLFSSL_API int wolfSSL_AsyncDecryptSetSignal(WOLFSSL* ssl, int idx, + WOLFSSL_THREAD_SIGNAL signal, void* ctx); #endif From 3133e2c4b504891a20b45e63cddf5681eecce199 Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Tue, 17 Dec 2024 17:22:58 +1000 Subject: [PATCH 2/3] Multithreaded encrypt: improvements Split out encryption in software for TLSv13. Call software encryption in async encrypt. Support ChaCha20-Poly1305. --- src/internal.c | 77 ++++++------- src/keys.c | 5 +- src/ssl.c | 43 +++---- src/tls13.c | 277 ++++++++++++++++++++++++--------------------- tests/api.c | 2 +- wolfssl/internal.h | 10 ++ 6 files changed, 218 insertions(+), 196 deletions(-) diff --git a/src/internal.c b/src/internal.c index ef53cf8d1a..9608fc4737 100644 --- a/src/internal.c +++ b/src/internal.c @@ -8234,10 +8234,17 @@ void wolfSSL_ResourceFree(WOLFSSL* ssl) { int i; for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { - bufferStatic* buff = &ssl->buffers.encrypt[i].buffer; - - ssl->buffers.encrypt[i].stop = 1; - FreeCiphersSide(&ssl->buffers.encrypt[i].cipher, ssl->heap); + ThreadCrypt* encrypt = &ssl->buffers.encrypt[i]; + bufferStatic* buff = &encrypt->buffer; + + encrypt->stop = 1; + FreeCiphersSide(&encrypt->cipher, ssl->heap); + #if defined(HAVE_POLY1305) && defined(HAVE_ONE_TIME_AUTH) + if (encrypt->auth.poly1305) + ForceZero(encrypt->auth.poly1305, sizeof(Poly1305)); + XFREE(encrypt->auth.poly1305, ssl->heap, DYNAMIC_TYPE_CIPHER); + encrypt->auth.poly1305 = NULL; + #endif if (buff->dynamicFlag) { XFREE(buff->buffer - buff->offset, ssl->heap, DYNAMIC_TYPE_OUT_BUFFER); @@ -8248,10 +8255,17 @@ void wolfSSL_ResourceFree(WOLFSSL* ssl) } } for (i = 0; i < WOLFSSL_THREADED_CRYPT_CNT; i++) { - bufferStatic* buff = &ssl->buffers.decrypt[i].buffer; - - ssl->buffers.decrypt[i].stop = 1; - FreeCiphersSide(&ssl->buffers.decrypt[i].cipher, ssl->heap); + ThreadCrypt* decrypt = &ssl->buffers.decrypt[i]; + bufferStatic* buff = &decrypt->buffer; + + decrypt->stop = 1; + FreeCiphersSide(&decrypt->cipher, ssl->heap); + #if defined(HAVE_POLY1305) && defined(HAVE_ONE_TIME_AUTH) + if (decrypt->auth.poly1305) + ForceZero(decrypt->auth.poly1305, sizeof(Poly1305)); + XFREE(decrypt->auth.poly1305, ssl->heap, DYNAMIC_TYPE_CIPHER); + decrypt->auth.poly1305 = NULL; + #endif if (buff->dynamicFlag) { XFREE(buff->buffer - buff->offset, ssl->heap, DYNAMIC_TYPE_OUT_BUFFER); @@ -25295,40 +25309,27 @@ int SendData(WOLFSSL* ssl, const void* data, int sz) SetKeys(&encrypt->cipher, NULL, &ssl->keys, &ssl->specs, ssl->options.side, ssl->heap, ssl->devId, ssl->rng, ssl->options.tls1_3); + #ifdef HAVE_ONE_TIME_AUTH + if (ssl->specs.bulk_cipher_algorithm == wolfssl_chacha) { + ret = SetAuthKeys(&encrypt->auth, &ssl->keys, &ssl->specs, + ssl->heap, ssl->devId); + if (ret != 0) + return ret; + } + #endif + encrypt->init = 1; } + encrypt->buffer.length = sendSz; encrypt->offset = RECORD_HEADER_SZ; - if (ssl->options.dtls) { - encrypt->offset += DTLS_RECORD_EXTRA; - } - encrypt->cryptLen = outputSz - encrypt->offset; - #ifdef HAVE_TRUNCATED_HMAC - if (ssl->truncated_hmac) { - encrypt->cryptLen -= min(TRUNCATED_HMAC_SZ, - ssl->specs.hash_size); - } - else - #endif - { - encrypt->cryptLen -= ssl->specs.hash_size; - } - - #if !defined(NO_PUBLIC_GCM_SET_IV) && \ - ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ - (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) - XMEMCPY(encrypt->nonce, ssl->keys.aead_enc_imp_IV, - AESGCM_IMP_IV_SZ); - XMEMCPY(encrypt->nonce + AESGCM_IMP_IV_SZ, ssl->keys.aead_exp_IV, - AESGCM_EXP_IV_SZ); - #endif - XMEMSET(encrypt->additional, 0, AEAD_AUTH_DATA_SZ); - WriteSEQ(ssl, CUR_ORDER, encrypt->additional); - XMEMCPY(encrypt->additional + AEAD_TYPE_OFFSET, - encrypt->buffer.buffer, 3); - c16toa(sendSz - encrypt->offset - AESGCM_EXP_IV_SZ - - ssl->specs.aead_mac_size, - encrypt->additional + AEAD_LEN_OFFSET); + encrypt->buffer.idx = 0; + encrypt->cryptLen = sendSz - RECORD_HEADER_SZ; + + BuildTls13Nonce(ssl, encrypt->nonce, ssl->keys.aead_enc_imp_IV, + CUR_ORDER); + XMEMCPY(encrypt->additional, encrypt->buffer.buffer, + encrypt->offset); #ifdef WOLFSSL_DTLS if (ssl->options.dtls) diff --git a/src/keys.c b/src/keys.c index 693e6b1333..231516bdd8 100644 --- a/src/keys.c +++ b/src/keys.c @@ -3403,7 +3403,10 @@ int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs, #ifdef HAVE_ONE_TIME_AUTH /* set one time authentication keys */ -static int SetAuthKeys(OneTimeAuth* authentication, Keys* keys, +#ifndef WOLFSSL_THREADED_CRYPT +static +#endif +int SetAuthKeys(OneTimeAuth* authentication, Keys* keys, CipherSpecs* specs, void* heap, int devId) { diff --git a/src/ssl.c b/src/ssl.c index 0afddab226..98cafc3f6c 100644 --- a/src/ssl.c +++ b/src/ssl.c @@ -23914,36 +23914,27 @@ int wolfSSL_AsyncEncryptStop(WOLFSSL* ssl, int idx) int wolfSSL_AsyncEncrypt(WOLFSSL* ssl, int idx) { - int ret = WC_NO_ERR_TRACE(NOT_COMPILED_IN); + int ret; ThreadCrypt* encrypt = &ssl->buffers.encrypt[idx]; + unsigned char* out = encrypt->buffer.buffer + encrypt->offset; + word32 dataSz = encrypt->cryptLen - ssl->specs.aead_mac_size; - if (ssl->specs.bulk_cipher_algorithm == wolfssl_aes_gcm) { - unsigned char* out = encrypt->buffer.buffer + encrypt->offset; - unsigned char* input = encrypt->buffer.buffer + encrypt->offset; - word32 encSz = encrypt->buffer.length - encrypt->offset; - - ret = -#if !defined(NO_GCM_ENCRYPT_EXTRA) && \ - ((!defined(HAVE_FIPS) && !defined(HAVE_SELFTEST)) || \ - (defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2))) - wc_AesGcmEncrypt_ex + ret = EncryptTls13Sw(ssl->specs.bulk_cipher_algorithm, &encrypt->cipher, +#ifdef HAVE_ONE_TIME_AUTH + &encrypt->auth, #else - wc_AesGcmEncrypt -#endif - (encrypt->cipher.aes, - out + AESGCM_EXP_IV_SZ, input + AESGCM_EXP_IV_SZ, - encSz - AESGCM_EXP_IV_SZ - ssl->specs.aead_mac_size, - encrypt->nonce, AESGCM_NONCE_SZ, - out + encSz - ssl->specs.aead_mac_size, - ssl->specs.aead_mac_size, - encrypt->additional, AEAD_AUTH_DATA_SZ); -#if !defined(NO_PUBLIC_GCM_SET_IV) && \ - ((!defined(HAVE_FIPS) && !defined(HAVE_SELFTEST)) || \ - (defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2))) - XMEMCPY(out, encrypt->nonce + AESGCM_IMP_IV_SZ, AESGCM_EXP_IV_SZ); -#endif - encrypt->done = 1; + NULL, +#endif + out, out, dataSz, encrypt->nonce, encrypt->additional, RECORD_HEADER_SZ, + ssl->specs.aead_mac_size, 1); +#ifdef WOLFSSL_DTLS13 + if (ret == 0 && ssl->options.dtls) { + ret = Dtls13EncryptRecordNumber(ssl, encrypt->buffer.buffer, + (word16)encrypt->buffer.length); } +#endif /* WOLFSSL_DTLS13 */ + + encrypt->done = 1; return ret; } diff --git a/src/tls13.c b/src/tls13.c index 96c9ae9c48..563ea0e822 100644 --- a/src/tls13.c +++ b/src/tls13.c @@ -2454,9 +2454,10 @@ void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, int order) * tag The authentication tag buffer. * returns 0 on success, otherwise failure. */ -static int ChaCha20Poly1305_Encrypt(WOLFSSL* ssl, byte* output, - const byte* input, word16 sz, byte* nonce, - const byte* aad, word16 aadSz, byte* tag) +static int ChaCha20Poly1305_Encrypt(Ciphers* encrypt, OneTimeAuth* auth, + byte* output, const byte* input, word16 sz, + byte* nonce, const byte* aad, word16 aadSz, + byte* tag) { int ret = 0; byte poly[CHACHA20_256_KEY_SIZE]; @@ -2465,21 +2466,21 @@ static int ChaCha20Poly1305_Encrypt(WOLFSSL* ssl, byte* output, XMEMSET(poly, 0, sizeof(poly)); /* Set the nonce for ChaCha and get Poly1305 key. */ - ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 0); + ret = wc_Chacha_SetIV(encrypt->chacha, nonce, 0); if (ret != 0) return ret; /* Create Poly1305 key using ChaCha20 keystream. */ - ret = wc_Chacha_Process(ssl->encrypt.chacha, poly, poly, sizeof(poly)); + ret = wc_Chacha_Process(encrypt->chacha, poly, poly, sizeof(poly)); if (ret != 0) return ret; #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Add("ChaCha20Poly1305_Encrypt poly", poly, sizeof(poly)); #endif - ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 1); + ret = wc_Chacha_SetIV(encrypt->chacha, nonce, 1); if (ret != 0) return ret; /* Encrypt the plain text. */ - ret = wc_Chacha_Process(ssl->encrypt.chacha, output, input, sz); + ret = wc_Chacha_Process(encrypt->chacha, output, input, sz); if (ret != 0) { ForceZero(poly, sizeof(poly)); #ifdef WOLFSSL_CHECK_MEM_ZERO @@ -2489,7 +2490,7 @@ static int ChaCha20Poly1305_Encrypt(WOLFSSL* ssl, byte* output, } /* Set key for Poly1305. */ - ret = wc_Poly1305SetKey(ssl->auth.poly1305, poly, sizeof(poly)); + ret = wc_Poly1305SetKey(auth->poly1305, poly, sizeof(poly)); ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */ #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(poly, sizeof(poly)); @@ -2497,7 +2498,7 @@ static int ChaCha20Poly1305_Encrypt(WOLFSSL* ssl, byte* output, if (ret != 0) return ret; /* Add authentication code of encrypted data to end. */ - ret = wc_Poly1305_MAC(ssl->auth.poly1305, aad, aadSz, output, sz, tag, + ret = wc_Poly1305_MAC(auth->poly1305, aad, aadSz, output, sz, tag, POLY1305_AUTH_SZ); return ret; @@ -2518,7 +2519,7 @@ static int ChaCha20Poly1305_Encrypt(WOLFSSL* ssl, byte* output, * tag The authentication tag buffer. * returns 0 on success, otherwise failure. */ -static int Tls13IntegrityOnly_Encrypt(WOLFSSL* ssl, byte* output, +static int Tls13IntegrityOnly_Encrypt(Ciphers* encrypt, byte* output, const byte* input, word16 sz, const byte* nonce, const byte* aad, word16 aadSz, byte* tag) @@ -2526,13 +2527,13 @@ static int Tls13IntegrityOnly_Encrypt(WOLFSSL* ssl, byte* output, int ret; /* HMAC: nonce | aad | input */ - ret = wc_HmacUpdate(ssl->encrypt.hmac, nonce, HMAC_NONCE_SZ); + ret = wc_HmacUpdate(encrypt->hmac, nonce, HMAC_NONCE_SZ); if (ret == 0) - ret = wc_HmacUpdate(ssl->encrypt.hmac, aad, aadSz); + ret = wc_HmacUpdate(encrypt->hmac, aad, aadSz); if (ret == 0) - ret = wc_HmacUpdate(ssl->encrypt.hmac, input, sz); + ret = wc_HmacUpdate(encrypt->hmac, input, sz); if (ret == 0) - ret = wc_HmacFinal(ssl->encrypt.hmac, tag); + ret = wc_HmacFinal(encrypt->hmac, tag); /* Copy the input to output if not the same buffer */ if (ret == 0 && output != input) XMEMCPY(output, input, sz); @@ -2540,6 +2541,103 @@ static int Tls13IntegrityOnly_Encrypt(WOLFSSL* ssl, byte* output, } #endif +/* + */ +int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, + const byte* input, word16 dataSz, byte* nonce, + const byte* aad, word16 aadSz, word32 macSz, int async) +{ + int ret; +#ifdef HAVE_ONE_TIME_AUTH + OneTimeAuth* auth = authPtr; +#endif + + (void)encrypt; + (void)authPtr; + (void)output; + (void)input; + (void)dataSz; + (void)nonce; + (void)aad; + (void)aadSz; + (void)macSz; + (void)async; + + switch (alg) { + #ifdef BUILD_AESGCM + case wolfssl_aes_gcm: + #if ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ + (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) + ret = wc_AesGcmEncrypt(encrypt->aes, output, input, + dataSz, nonce, AEAD_NONCE_SZ, + output + dataSz, macSz, aad, aadSz); + #else + ret = wc_AesGcmSetExtIV(encrypt->aes, nonce, AEAD_NONCE_SZ); + if (ret == 0) { + ret = wc_AesGcmEncrypt_ex(encrypt->aes, output, + input, dataSz, nonce, AEAD_NONCE_SZ, + output + dataSz, macSz, aad, aadSz); + } + #endif + break; + #endif + + #ifdef HAVE_AESCCM + case wolfssl_aes_ccm: + #if ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ + (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) + ret = wc_AesCcmEncrypt(encrypt->aes, output, input, + dataSz, nonce, AEAD_NONCE_SZ, + output + dataSz, macSz, aad, aadSz); + #else + ret = wc_AesCcmSetNonce(encrypt->aes, nonce, AEAD_NONCE_SZ); + if (ret == 0) { + ret = wc_AesCcmEncrypt_ex(encrypt->aes, output, + input, dataSz, nonce, AEAD_NONCE_SZ, + output + dataSz, macSz, aad, aadSz); + } + #endif + break; + #endif + + #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) + case wolfssl_chacha: + ret = ChaCha20Poly1305_Encrypt(encrypt, auth, output, + input, dataSz, nonce, aad, aadSz, output + dataSz); + break; + #endif + + #ifdef WOLFSSL_SM4_GCM + case wolfssl_sm4_gcm: + ret = wc_Sm4GcmEncrypt(&encrypt->sm4, output, input, + dataSz, nonce, AEAD_NONCE_SZ, output + dataSz, + macSz, aad, aadSz); + break; + #endif + + #ifdef WOLFSSL_SM4_CCM + case wolfssl_sm4_ccm: + ret = wc_Sm4CcmEncrypt(encrypt->sm4, output, input, + dataSz, nonce, AEAD_NONCE_SZ, output + dataSz, + macSz, aad, aadSz); + break; + #endif + + #ifdef HAVE_NULL_CIPHER + case wolfssl_cipher_null: + ret = Tls13IntegrityOnly_Encrypt(encrypt, output, input, dataSz, + nonce, aad, aadSz, output + dataSz); + break; + #endif + + default: + WOLFSSL_MSG("wolfSSL Encrypt programming error"); + ret = ENCRYPT_ERROR; + } + + return ret; +} + /* Encrypt data for TLS v1.3. * * ssl The SSL/TLS object. @@ -2558,7 +2656,6 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, int ret = 0; word16 dataSz = sz - ssl->specs.aead_mac_size; word16 macSz = ssl->specs.aead_mac_size; - word32 nonceSz = 0; #ifdef WOLFSSL_ASYNC_CRYPT WC_ASYNC_DEV* asyncDev = NULL; word32 event_flags = WC_ASYNC_FLAG_CALL_AGAIN; @@ -2572,7 +2669,6 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, (void)dataSz; (void)macSz; (void)asyncOkay; - (void)nonceSz; #ifdef WOLFSSL_ASYNC_CRYPT if (ssl->error == WC_NO_ERR_TRACE(WC_PENDING_E)) { @@ -2632,125 +2728,40 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, case CIPHER_STATE_DO: { - switch (ssl->specs.bulk_cipher_algorithm) { - #ifdef BUILD_AESGCM - case wolfssl_aes_gcm: - #ifdef WOLFSSL_ASYNC_CRYPT - /* initialize event */ - asyncDev = &ssl->encrypt.aes->asyncDev; - ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags); - if (ret != 0) - break; - #endif - - nonceSz = AESGCM_NONCE_SZ; - - #if defined(HAVE_PK_CALLBACKS) - ret = NOT_COMPILED_IN; - if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { - ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 1, - output, input, dataSz, - ssl->encrypt.nonce, nonceSz, - output + dataSz, macSz, - aad, aadSz); - } - if (ret == WC_NO_ERR_TRACE(NOT_COMPILED_IN)) - #endif - { - #if ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ - (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) - ret = wc_AesGcmEncrypt(ssl->encrypt.aes, output, input, - dataSz, ssl->encrypt.nonce, nonceSz, - output + dataSz, macSz, aad, aadSz); - #else - ret = wc_AesGcmSetExtIV(ssl->encrypt.aes, - ssl->encrypt.nonce, nonceSz); - if (ret == 0) { - ret = wc_AesGcmEncrypt_ex(ssl->encrypt.aes, output, - input, dataSz, ssl->encrypt.nonce, nonceSz, - output + dataSz, macSz, aad, aadSz); - } - #endif - } - break; - #endif - - #ifdef HAVE_AESCCM - case wolfssl_aes_ccm: - #ifdef WOLFSSL_ASYNC_CRYPT - /* initialize event */ - asyncDev = &ssl->encrypt.aes->asyncDev; - ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags); - if (ret != 0) - break; - #endif - - nonceSz = AESCCM_NONCE_SZ; - #if defined(HAVE_PK_CALLBACKS) - ret = NOT_COMPILED_IN; - if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { - ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 1, - output, input, dataSz, - ssl->encrypt.nonce, nonceSz, - output + dataSz, macSz, - aad, aadSz); - } - if (ret == WC_NO_ERR_TRACE(NOT_COMPILED_IN)) - #endif - { - #if ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ - (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) - ret = wc_AesCcmEncrypt(ssl->encrypt.aes, output, input, - dataSz, ssl->encrypt.nonce, nonceSz, - output + dataSz, macSz, aad, aadSz); - #else - ret = wc_AesCcmSetNonce(ssl->encrypt.aes, - ssl->encrypt.nonce, nonceSz); - if (ret == 0) { - ret = wc_AesCcmEncrypt_ex(ssl->encrypt.aes, output, - input, dataSz, ssl->encrypt.nonce, nonceSz, - output + dataSz, macSz, aad, aadSz); - } - #endif - } - break; - #endif - - #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) - case wolfssl_chacha: - ret = ChaCha20Poly1305_Encrypt(ssl, output, input, dataSz, - ssl->encrypt.nonce, aad, aadSz, output + dataSz); + if ((ssl->specs.bulk_cipher_algorithm == wolfssl_aes_gcm || + ssl->specs.bulk_cipher_algorithm == wolfssl_aes_ccm)) { + #ifdef WOLFSSL_ASYNC_CRYPT + /* initialize event */ + asyncDev = &ssl->encrypt.aes->asyncDev; + ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags); + if (ret != 0) break; #endif - - #ifdef WOLFSSL_SM4_GCM - case wolfssl_sm4_gcm: - nonceSz = SM4_GCM_NONCE_SZ; - ret = wc_Sm4GcmEncrypt(ssl->encrypt.sm4, output, input, - dataSz, ssl->encrypt.nonce, nonceSz, output + dataSz, - macSz, aad, aadSz); - break; - #endif - - #ifdef WOLFSSL_SM4_CCM - case wolfssl_sm4_ccm: - nonceSz = SM4_CCM_NONCE_SZ; - ret = wc_Sm4CcmEncrypt(ssl->encrypt.sm4, output, input, - dataSz, ssl->encrypt.nonce, nonceSz, output + dataSz, - macSz, aad, aadSz); + #if defined(HAVE_PK_CALLBACKS) + ret = NOT_COMPILED_IN; + if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { + ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 1, + output, input, dataSz, + ssl->encrypt.nonce, AEAD_NONCE_SZ, + output + dataSz, macSz, + aad, aadSz); + } + if (ret != WC_NO_ERR_TRACE(NOT_COMPILED_IN)) break; + ret = 0; #endif + } - #ifdef HAVE_NULL_CIPHER - case wolfssl_cipher_null: - ret = Tls13IntegrityOnly_Encrypt(ssl, output, input, dataSz, - ssl->encrypt.nonce, aad, aadSz, output + dataSz); - break; + if (ret == 0) { + ret = EncryptTls13Sw(ssl->specs.bulk_cipher_algorithm, + &ssl->encrypt, + #ifdef HAVE_ONE_TIME_AUTH + &ssl->auth, + #else + NULL, #endif - - default: - WOLFSSL_MSG("wolfSSL Encrypt programming error"); - return ENCRYPT_ERROR; + output, input, dataSz, ssl->encrypt.nonce, + aad, aadSz, macSz, 0); } /* Advance state */ @@ -3061,7 +3072,6 @@ int DecryptTls13(WOLFSSL* ssl, byte* output, const byte* input, word16 sz, ret = wc_AesGcmDecrypt(ssl->decrypt.aes, output, input, dataSz, ssl->decrypt.nonce, nonceSz, input + dataSz, macSz, aad, aadSz); - #ifdef WOLFSSL_ASYNC_CRYPT if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { ret = wolfSSL_AsyncPush(ssl, @@ -3385,6 +3395,13 @@ int BuildTls13Message(WOLFSSL* ssl, byte* output, int outSz, const byte* input, output, output, args->size, ssl->MacEncryptCtx); } else + #endif + #ifdef WOLFSSL_THREADED_CRYPT + if (ssl->buffers.encryptSignalRegistered && + type == application_data) { + ret = args->sz; + } + else #endif { const byte* aad = output; diff --git a/tests/api.c b/tests/api.c index 46e7579c5d..43ef1a6c26 100644 --- a/tests/api.c +++ b/tests/api.c @@ -90683,7 +90683,6 @@ static void test_AEAD_limit_client(WOLFSSL* ssl) /* Key update should be sent and negotiated */ ret = wolfSSL_read(ssl, msgBuf, sizeof(msgBuf)); AssertIntGT(ret, 0); - ret = wolfSSL_write(ssl, msgBuf, sizeof(msgBuf)); /* Epoch after one key update is 4 */ if (w64Equal(ssl->dtls13PeerEpoch, w64From32(0, 4)) && w64Equal(Dtls13GetEpoch(ssl, ssl->dtls13PeerEpoch)->dropCount, counter)) { @@ -90752,6 +90751,7 @@ static void test_AEAD_limit_server(WOLFSSL* ssl) test_AEAD_seq_num = 0; } (void)wolfSSL_read(ssl, msgBuf, sizeof(msgBuf)); + (void)wolfSSL_write(ssl, msgBuf, sizeof(msgBuf)); nanosleep(&delay, NULL); } } diff --git a/wolfssl/internal.h b/wolfssl/internal.h index dc9afda97b..37a51f70b2 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -4744,6 +4744,9 @@ enum AcceptStateTls13 { typedef struct ThreadCrypt { Ciphers cipher; +#ifdef HAVE_ONE_TIME_AUTH + OneTimeAuth auth; +#endif bufferStatic buffer; unsigned char nonce[AESGCM_NONCE_SZ]; unsigned char additional[AEAD_AUTH_DATA_SZ]; @@ -6822,6 +6825,10 @@ WOLFSSL_LOCAL int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs, int side, void* heap, int devId, WC_RNG* rng, int tls13); WOLFSSL_LOCAL int SetKeysSide(WOLFSSL* ssl, enum encrypt_side side); +#ifdef WOLFSSL_THREADED_CRYPT +WOLFSSL_LOCAL int SetAuthKeys(OneTimeAuth* authentication, Keys* keys, + CipherSpecs* specs, void* heap, int devId); +#endif /* Set*Internal and Set*External functions */ #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL) @@ -6876,6 +6883,9 @@ WOLFSSL_LOCAL int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, WOLFSSL_LOCAL void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, int order); #endif +int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, + const byte* input, word16 dataSz, byte* nonce, + const byte* aad, word16 aadSz, word32 macSz, int async); /* Use WOLFSSL_API to use this function in tests/api.c */ WOLFSSL_API int BuildTls13Message(WOLFSSL* ssl, byte* output, int outSz, const byte* input, From ed2606e61cc6134770d95d06649b69f9696df0de Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Wed, 18 Dec 2024 11:14:28 +1000 Subject: [PATCH 3/3] Multithreaded decrypt: improvements Split out decryption in software for TLSv13. Call software decryption in async decrypt. Support ChaCha20-Poly1305. --- src/dtls13.c | 19 ++- src/internal.c | 13 +- src/keys.c | 8 +- src/ssl.c | 31 ++-- src/tls13.c | 389 +++++++++++++++++++++++++-------------------- src/wolfio.c | 2 +- wolfssl/internal.h | 12 +- 7 files changed, 281 insertions(+), 193 deletions(-) diff --git a/src/dtls13.c b/src/dtls13.c index 0ca7138aa1..781c79072a 100644 --- a/src/dtls13.c +++ b/src/dtls13.c @@ -956,8 +956,18 @@ static int Dtls13SendOneFragmentRtx(WOLFSSL* ssl, handshakeType, hashOutput, Dtls13SendNow(ssl, handshakeType)); if (rtxRecord != NULL) { - if (ret == 0 || ret == WC_NO_ERR_TRACE(WANT_WRITE)) + if (ret == 0 || ret == WC_NO_ERR_TRACE(WANT_WRITE)) { + #ifdef WOLFSSL_RW_THREADED + int lockRet = wc_LockMutex(&ssl->dtls13Rtx.mutex); + if (lockRet < 0) { + return lockRet; + } + #endif Dtls13RtxAddRecord(&ssl->dtls13Rtx, rtxRecord); + #ifdef WOLFSSL_RW_THREADED + wc_UnLockMutex(&ssl->dtls13Rtx.mutex); + #endif + } else Dtls13FreeRtxBufferRecord(ssl, rtxRecord); } @@ -1534,8 +1544,15 @@ static void Dtls13RtxMoveToEndOfList(WOLFSSL* ssl, Dtls13RtxRecord** prevNext, return; Dtls13RtxRecordUnlink(ssl, prevNext, r); +#ifdef WOLFSSL_RW_THREADED + if (wc_LockMutex(&ssl->dtls13Rtx.mutex) != 0) + return; +#endif /* add to the end */ Dtls13RtxAddRecord(&ssl->dtls13Rtx, r); +#ifdef WOLFSSL_RW_THREADED + wc_UnLockMutex(&ssl->dtls13Rtx.mutex); +#endif } static int Dtls13RtxSendBuffered(WOLFSSL* ssl) diff --git a/src/internal.c b/src/internal.c index 9608fc4737..6473bf8045 100644 --- a/src/internal.c +++ b/src/internal.c @@ -2844,11 +2844,17 @@ void InitCiphers(WOLFSSL* ssl) #endif #if defined(HAVE_POLY1305) && defined(HAVE_ONE_TIME_AUTH) ssl->auth.poly1305 = NULL; +#ifdef WOLFSSL_RW_THREADED + ssl->decAuth.poly1305 = NULL; +#endif #endif ssl->encrypt.setup = 0; ssl->decrypt.setup = 0; #ifdef HAVE_ONE_TIME_AUTH ssl->auth.setup = 0; +#ifdef WOLFSSL_RW_THREADED + ssl->decAuth.setup = 0; +#endif #endif #ifdef WOLFSSL_DTLS13 @@ -2926,6 +2932,12 @@ void FreeCiphers(WOLFSSL* ssl) ForceZero(ssl->auth.poly1305, sizeof(Poly1305)); XFREE(ssl->auth.poly1305, ssl->heap, DYNAMIC_TYPE_CIPHER); ssl->auth.poly1305 = NULL; +#ifdef WOLFSSL_RW_THREADED + if (ssl->decAuth.poly1305) + ForceZero(ssl->decAuth.poly1305, sizeof(Poly1305)); + XFREE(ssl->decAuth.poly1305, ssl->heap, DYNAMIC_TYPE_CIPHER); + ssl->decAuth.poly1305 = NULL; +#endif #endif #ifdef WOLFSSL_DTLS13 @@ -21321,7 +21333,6 @@ static int ReceiveAsyncData(WOLFSSL* ssl) int ret; int error; - /* Parse record header again. */ GrowInputBuffer(ssl, decrypt->recordHdrLen, 0); XMEMCPY(ssl->buffers.inputBuffer.buffer, decrypt->recordHdr, diff --git a/src/keys.c b/src/keys.c index 231516bdd8..62dc802a86 100644 --- a/src/keys.c +++ b/src/keys.c @@ -3550,7 +3550,13 @@ int SetKeysSide(WOLFSSL* ssl, enum encrypt_side side) if (!ssl->auth.setup && ssl->specs.bulk_cipher_algorithm == wolfssl_chacha){ ret = SetAuthKeys(&ssl->auth, keys, &ssl->specs, ssl->heap, ssl->devId); if (ret != 0) - return ret; + return ret; + #ifdef WOLFSSL_RW_THREADED + ret = SetAuthKeys(&ssl->decAuth, keys, &ssl->specs, ssl->heap, + ssl->devId); + if (ret != 0) + return ret; + #endif } #endif diff --git a/src/ssl.c b/src/ssl.c index 98cafc3f6c..16974e3a18 100644 --- a/src/ssl.c +++ b/src/ssl.c @@ -23920,13 +23920,13 @@ int wolfSSL_AsyncEncrypt(WOLFSSL* ssl, int idx) word32 dataSz = encrypt->cryptLen - ssl->specs.aead_mac_size; ret = EncryptTls13Sw(ssl->specs.bulk_cipher_algorithm, &encrypt->cipher, -#ifdef HAVE_ONE_TIME_AUTH + #ifdef HAVE_ONE_TIME_AUTH &encrypt->auth, -#else + #else NULL, -#endif + #endif out, out, dataSz, encrypt->nonce, encrypt->additional, RECORD_HEADER_SZ, - ssl->specs.aead_mac_size, 1); + ssl->specs.aead_mac_size); #ifdef WOLFSSL_DTLS13 if (ret == 0 && ssl->options.dtls) { ret = Dtls13EncryptRecordNumber(ssl, encrypt->buffer.buffer, @@ -23984,21 +23984,20 @@ int wolfSSL_AsyncDecryptStop(WOLFSSL* ssl, int idx) int wolfSSL_AsyncDecrypt(WOLFSSL* ssl, int idx) { - int ret = WC_NO_ERR_TRACE(NOT_COMPILED_IN); + int ret; ThreadCrypt* decrypt = &ssl->buffers.decrypt[idx]; + unsigned char* out = decrypt->buffer.buffer + decrypt->offset; - if (ssl->specs.bulk_cipher_algorithm == wolfssl_aes_gcm) { - unsigned char* out = decrypt->buffer.buffer + decrypt->offset; - unsigned char* input = decrypt->buffer.buffer + decrypt->offset; - unsigned char* tag = input + decrypt->cryptLen; + ret = DecryptTls13Sw(ssl->specs.bulk_cipher_algorithm, &decrypt->cipher, + #ifdef HAVE_ONE_TIME_AUTH + &decrypt->auth, + #else + NULL, + #endif + out, out, decrypt->cryptLen, decrypt->nonce, decrypt->additional, + RECORD_HEADER_SZ, ssl->specs.aead_mac_size, ssl->specs.hash_size); - ret = wc_AesGcmDecrypt(decrypt->cipher.aes, out, input, - decrypt->cryptLen, - decrypt->nonce, AESGCM_NONCE_SZ, - tag, ssl->specs.aead_mac_size, - decrypt->additional, RECORD_HEADER_SZ); - decrypt->done = 1; - } + decrypt->done = 1; return ret; } diff --git a/src/tls13.c b/src/tls13.c index 563ea0e822..aa58d886d5 100644 --- a/src/tls13.c +++ b/src/tls13.c @@ -2443,15 +2443,16 @@ void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, int order) #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) /* Encrypt with ChaCha20 and create authentication tag with Poly1305. * - * ssl The SSL/TLS object. - * output The buffer to write encrypted data and authentication tag into. - * May be the same pointer as input. - * input The data to encrypt. - * sz The number of bytes to encrypt. - * nonce The nonce to use with ChaCha20. - * aad The additional authentication data. - * aadSz The size of the addition authentication data. - * tag The authentication tag buffer. + * encrypt Encryption object. + * auth Authentication object. + * output The buffer to write encrypted data and authentication tag into. + * May be the same pointer as input. + * input The data to encrypt. + * sz The number of bytes to encrypt. + * nonce The nonce to use with ChaCha20. + * aad The additional authentication data. + * aadSz The size of the addition authentication data. + * tag The authentication tag buffer. * returns 0 on success, otherwise failure. */ static int ChaCha20Poly1305_Encrypt(Ciphers* encrypt, OneTimeAuth* auth, @@ -2508,15 +2509,15 @@ static int ChaCha20Poly1305_Encrypt(Ciphers* encrypt, OneTimeAuth* auth, #ifdef HAVE_NULL_CIPHER /* Create authentication tag and copy data over input. * - * ssl The SSL/TLS object. - * output The buffer to copy data into. - * May be the same pointer as input. - * input The data. - * sz The number of bytes of data. - * nonce The nonce to use with authentication. - * aad The additional authentication data. - * aadSz The size of the addition authentication data. - * tag The authentication tag buffer. + * encrypt Encryption object. + * output The buffer to copy data into. + * May be the same pointer as input. + * input The data. + * sz The number of bytes of data. + * nonce The nonce to use with authentication. + * aad The additional authentication data. + * aadSz The size of the addition authentication data. + * tag The authentication tag buffer. * returns 0 on success, otherwise failure. */ static int Tls13IntegrityOnly_Encrypt(Ciphers* encrypt, byte* output, @@ -2541,19 +2542,37 @@ static int Tls13IntegrityOnly_Encrypt(Ciphers* encrypt, byte* output, } #endif -/* +/* Encrypt data for TLS v1.3 using software cryptography implementation. + * + * encrypt Encryption object. + * authPtr Authentication object. + * output The buffer to write encrypted data and authentication tag into. + * May be the same pointer as input. + * input The record header and data to encrypt. + * dataSz The number of bytes to encrypt. + * nonce Nonce data for GCM, CCM and NULL ciphers. + * aad The additional authentication data. + * aadSz The size of the addition authentication data. + * macSz Size of MAC to produce. + * returns 0 on success, otherwise failure. */ +#ifndef WOLFSSL_THREADED_CRYPT +static +#endif int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, const byte* input, word16 dataSz, byte* nonce, - const byte* aad, word16 aadSz, word32 macSz, int async) + const byte* aad, word16 aadSz, word16 macSz) { int ret; #ifdef HAVE_ONE_TIME_AUTH - OneTimeAuth* auth = authPtr; + OneTimeAuth* auth = (OneTimeAuth*)authPtr; #endif (void)encrypt; (void)authPtr; +#ifdef HAVE_ONE_TIME_AUTH + (void)auth; +#endif (void)output; (void)input; (void)dataSz; @@ -2561,7 +2580,6 @@ int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, (void)aad; (void)aadSz; (void)macSz; - (void)async; switch (alg) { #ifdef BUILD_AESGCM @@ -2569,13 +2587,13 @@ int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, #if ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) ret = wc_AesGcmEncrypt(encrypt->aes, output, input, - dataSz, nonce, AEAD_NONCE_SZ, + dataSz, nonce, AESGCM_NONCE_SZ, output + dataSz, macSz, aad, aadSz); #else ret = wc_AesGcmSetExtIV(encrypt->aes, nonce, AEAD_NONCE_SZ); if (ret == 0) { ret = wc_AesGcmEncrypt_ex(encrypt->aes, output, - input, dataSz, nonce, AEAD_NONCE_SZ, + input, dataSz, nonce, AESGCM_NONCE_SZ, output + dataSz, macSz, aad, aadSz); } #endif @@ -2587,13 +2605,13 @@ int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, #if ((defined(HAVE_FIPS) || defined(HAVE_SELFTEST)) && \ (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION < 2))) ret = wc_AesCcmEncrypt(encrypt->aes, output, input, - dataSz, nonce, AEAD_NONCE_SZ, + dataSz, nonce, AESCCM_NONCE_SZ, output + dataSz, macSz, aad, aadSz); #else ret = wc_AesCcmSetNonce(encrypt->aes, nonce, AEAD_NONCE_SZ); if (ret == 0) { ret = wc_AesCcmEncrypt_ex(encrypt->aes, output, - input, dataSz, nonce, AEAD_NONCE_SZ, + input, dataSz, nonce, AESCCM_NONCE_SZ, output + dataSz, macSz, aad, aadSz); } #endif @@ -2610,7 +2628,7 @@ int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, #ifdef WOLFSSL_SM4_GCM case wolfssl_sm4_gcm: ret = wc_Sm4GcmEncrypt(&encrypt->sm4, output, input, - dataSz, nonce, AEAD_NONCE_SZ, output + dataSz, + dataSz, nonce, SM4_GCM_NONCE_SZ, output + dataSz, macSz, aad, aadSz); break; #endif @@ -2618,7 +2636,7 @@ int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, #ifdef WOLFSSL_SM4_CCM case wolfssl_sm4_ccm: ret = wc_Sm4CcmEncrypt(encrypt->sm4, output, input, - dataSz, nonce, AEAD_NONCE_SZ, output + dataSz, + dataSz, nonce, SM4_CCM_NONCE_SZ, output + dataSz, macSz, aad, aadSz); break; #endif @@ -2761,7 +2779,7 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, NULL, #endif output, input, dataSz, ssl->encrypt.nonce, - aad, aadSz, macSz, 0); + aad, aadSz, macSz); } /* Advance state */ @@ -2834,20 +2852,21 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) /* Decrypt with ChaCha20 and check authentication tag with Poly1305. * - * ssl The SSL/TLS object. - * output The buffer to write decrypted data into. - * May be the same pointer as input. - * input The data to decrypt. - * sz The number of bytes to decrypt. - * nonce The nonce to use with ChaCha20. - * aad The additional authentication data. - * aadSz The size of the addition authentication data. - * tagIn The authentication tag data from packet. + * decrypt Decryption object. + * authPtr Authentication object. + * output The buffer to write decrypted data into. + * May be the same pointer as input. + * input The data to decrypt. + * sz The number of bytes to decrypt. + * nonce The nonce to use with ChaCha20. + * aad The additional authentication data. + * aadSz The size of the addition authentication data. + * tagIn The authentication tag data from packet. * returns 0 on success, otherwise failure. */ -static int ChaCha20Poly1305_Decrypt(WOLFSSL* ssl, byte* output, - const byte* input, word16 sz, byte* nonce, - const byte* aad, word16 aadSz, +static int ChaCha20Poly1305_Decrypt(Ciphers* decrypt, OneTimeAuth* auth, + byte* output, const byte* input, word16 sz, + byte* nonce, const byte* aad, word16 aadSz, const byte* tagIn) { int ret; @@ -2858,17 +2877,17 @@ static int ChaCha20Poly1305_Decrypt(WOLFSSL* ssl, byte* output, XMEMSET(poly, 0, sizeof(poly)); /* Set nonce and get Poly1305 key. */ - ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 0); + ret = wc_Chacha_SetIV(decrypt->chacha, nonce, 0); if (ret != 0) return ret; /* Use ChaCha20 keystream to get Poly1305 key for tag. */ - ret = wc_Chacha_Process(ssl->decrypt.chacha, poly, poly, sizeof(poly)); + ret = wc_Chacha_Process(decrypt->chacha, poly, poly, sizeof(poly)); if (ret != 0) return ret; #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Add("ChaCha20Poly1305_Decrypt poly", poly, sizeof(poly)); #endif - ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 1); + ret = wc_Chacha_SetIV(decrypt->chacha, nonce, 1); if (ret != 0) { ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */ #ifdef WOLFSSL_CHECK_MEM_ZERO @@ -2878,7 +2897,7 @@ static int ChaCha20Poly1305_Decrypt(WOLFSSL* ssl, byte* output, } /* Set key for Poly1305. */ - ret = wc_Poly1305SetKey(ssl->auth.poly1305, poly, sizeof(poly)); + ret = wc_Poly1305SetKey(auth->poly1305, poly, sizeof(poly)); ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */ #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(poly, sizeof(poly)); @@ -2886,7 +2905,7 @@ static int ChaCha20Poly1305_Decrypt(WOLFSSL* ssl, byte* output, if (ret != 0) return ret; /* Generate authentication tag for encrypted data. */ - if ((ret = wc_Poly1305_MAC(ssl->auth.poly1305, aad, aadSz, input, sz, tag, + if ((ret = wc_Poly1305_MAC(auth->poly1305, aad, aadSz, input, sz, tag, sizeof(tag))) != 0) { return ret; } @@ -2898,7 +2917,7 @@ static int ChaCha20Poly1305_Decrypt(WOLFSSL* ssl, byte* output, } /* If the tag was good decrypt message. */ - ret = wc_Chacha_Process(ssl->decrypt.chacha, output, input, sz); + ret = wc_Chacha_Process(decrypt->chacha, output, input, sz); return ret; } @@ -2907,36 +2926,36 @@ static int ChaCha20Poly1305_Decrypt(WOLFSSL* ssl, byte* output, #ifdef HAVE_NULL_CIPHER /* Check HMAC tag and copy over input. * - * ssl The SSL/TLS object. - * output The buffer to copy data into. - * May be the same pointer as input. - * input The data. - * sz The number of bytes of data. - * nonce The nonce to use with authentication. - * aad The additional authentication data. - * aadSz The size of the addition authentication data. - * tagIn The authentication tag data from packet. + * decrypt Decryption object. + * output The buffer to copy data into. + * May be the same pointer as input. + * input The data. + * sz The number of bytes of data. + * nonce The nonce to use with authentication. + * aad The additional authentication data. + * aadSz The size of the addition authentication data. + * tagIn The authentication tag data from packet. * returns 0 on success, otherwise failure. */ -static int Tls13IntegrityOnly_Decrypt(WOLFSSL* ssl, byte* output, +static int Tls13IntegrityOnly_Decrypt(Ciphers* decrypt, byte* output, const byte* input, word16 sz, const byte* nonce, const byte* aad, word16 aadSz, - const byte* tagIn) + const byte* tagIn, word16 hashSz) { int ret; byte hmac[WC_MAX_DIGEST_SIZE]; /* HMAC: nonce | aad | input */ - ret = wc_HmacUpdate(ssl->decrypt.hmac, nonce, HMAC_NONCE_SZ); + ret = wc_HmacUpdate(decrypt->hmac, nonce, HMAC_NONCE_SZ); if (ret == 0) - ret = wc_HmacUpdate(ssl->decrypt.hmac, aad, aadSz); + ret = wc_HmacUpdate(decrypt->hmac, aad, aadSz); if (ret == 0) - ret = wc_HmacUpdate(ssl->decrypt.hmac, input, sz); + ret = wc_HmacUpdate(decrypt->hmac, input, sz); if (ret == 0) - ret = wc_HmacFinal(ssl->decrypt.hmac, hmac); + ret = wc_HmacFinal(decrypt->hmac, hmac); /* Check authentication tag matches */ - if (ret == 0 && ConstantCompare(tagIn, hmac, ssl->specs.hash_size) != 0) + if (ret == 0 && ConstantCompare(tagIn, hmac, hashSz) != 0) ret = DECRYPT_ERROR; /* Copy the input to output if not the same buffer */ if (ret == 0 && output != input) @@ -2945,6 +2964,102 @@ static int Tls13IntegrityOnly_Decrypt(WOLFSSL* ssl, byte* output, } #endif +/* Decrypt data for TLS v1.3 using software cryptography implementation. + * + * decrypt Decryption object. + * authPtr Authentication object. + * output The buffer to write decrypted data into. + * May be the same pointer as input. + * input The data to decrypt and authentication tag. + * dataSz The length of the encrypted data. + * nonce Nonce data for GCM, CCM and NULL ciphers. + * aad The additional authentication data. + * aadSz The size of the addition authentication data. + * macSz Size of MAC to use. + * hashSz Size of hash to with NULL cipher. + * returns 0 on success, otherwise failure. + */ +#ifndef WOLFSSL_THREADED_CRYPT +static +#endif +int DecryptTls13Sw(byte alg, Ciphers* decrypt, void* authPtr, byte* output, + const byte* input, word16 dataSz, byte* nonce, + const byte* aad, word16 aadSz, word16 macSz, word16 hashSz) +{ + int ret; +#ifdef HAVE_ONE_TIME_AUTH + OneTimeAuth* auth = (OneTimeAuth*)authPtr; +#endif + + (void)decrypt; + (void)authPtr; +#ifdef HAVE_ONE_TIME_AUTH + (void)auth; +#endif + (void)output; + (void)input; + (void)dataSz; + (void)nonce; + (void)aad; + (void)aadSz; + (void)macSz; + (void)hashSz; + + switch (alg) { + #ifdef BUILD_AESGCM + case wolfssl_aes_gcm: + ret = wc_AesGcmDecrypt(decrypt->aes, output, input, + dataSz, nonce, AESGCM_NONCE_SZ, + input + dataSz, macSz, aad, aadSz); + break; + #endif + + #ifdef HAVE_AESCCM + case wolfssl_aes_ccm: + ret = wc_AesCcmDecrypt(decrypt->aes, output, input, + dataSz, nonce, AESCCM_NONCE_SZ, + input + dataSz, macSz, aad, aadSz); + break; + #endif + + #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) + case wolfssl_chacha: + ret = ChaCha20Poly1305_Decrypt(decrypt, auth, output, input, dataSz, + decrypt->nonce, aad, aadSz, input + dataSz); + break; + #endif + + #ifdef WOLFSSL_SM4_GCM + case wolfssl_sm4_gcm: + ret = wc_Sm4GcmDecrypt(decrypt->sm4, output, input, + dataSz, nonce, SM4_GCM_NONCE_SZ, output + dataSz, + macSz, aad, aadSz); + break; + #endif + + #ifdef WOLFSSL_SM4_CCM + case wolfssl_sm4_ccm: + ret = wc_Sm4CcmDecrypt(decrypt->sm4, output, input, + dataSz, nonce, SM4_CCM_NONCE_SZ, output + dataSz, + macSz, aad, aadSz); + break; + #endif + + #ifdef HAVE_NULL_CIPHER + case wolfssl_cipher_null: + ret = Tls13IntegrityOnly_Decrypt(decrypt, output, input, dataSz, + nonce, aad, aadSz, input + dataSz, hashSz); + break; + #endif + + default: + WOLFSSL_MSG("wolfSSL Decrypt programming error"); + return DECRYPT_ERROR; + } + + return ret; +} + /* Decrypt data for TLS v1.3. * * ssl The SSL/TLS object. @@ -2962,7 +3077,7 @@ int DecryptTls13(WOLFSSL* ssl, byte* output, const byte* input, word16 sz, int ret = 0; word16 dataSz = sz - ssl->specs.aead_mac_size; word16 macSz = ssl->specs.aead_mac_size; - word32 nonceSz = 0; + word32 nonceSz = AEAD_NONCE_SZ; WOLFSSL_ENTER("DecryptTls13"); @@ -3043,118 +3158,52 @@ int DecryptTls13(WOLFSSL* ssl, byte* output, const byte* input, word16 sz, case CIPHER_STATE_DO: { - switch (ssl->specs.bulk_cipher_algorithm) { - #ifdef BUILD_AESGCM - case wolfssl_aes_gcm: - #ifdef WOLFSSL_ASYNC_CRYPT - /* initialize event */ - ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.aes->asyncDev, - WC_ASYNC_FLAG_NONE); - if (ret != 0) - break; - #endif - - nonceSz = AESGCM_NONCE_SZ; - - #if defined(HAVE_PK_CALLBACKS) - ret = NOT_COMPILED_IN; - if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { - ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 0, - output, input, dataSz, - ssl->decrypt.nonce, nonceSz, - (byte *)(input + dataSz), macSz, - aad, aadSz); - } - if (ret == WC_NO_ERR_TRACE(NOT_COMPILED_IN)) - #endif - { - - ret = wc_AesGcmDecrypt(ssl->decrypt.aes, output, input, - dataSz, ssl->decrypt.nonce, nonceSz, - input + dataSz, macSz, aad, aadSz); - #ifdef WOLFSSL_ASYNC_CRYPT - if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { - ret = wolfSSL_AsyncPush(ssl, - &ssl->decrypt.aes->asyncDev); - } - #endif - - } - break; - #endif - - #ifdef HAVE_AESCCM - case wolfssl_aes_ccm: - #ifdef WOLFSSL_ASYNC_CRYPT - /* initialize event */ - ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.aes->asyncDev, - WC_ASYNC_FLAG_NONE); - if (ret != 0) - break; - #endif - - nonceSz = AESCCM_NONCE_SZ; - #if defined(HAVE_PK_CALLBACKS) - ret = NOT_COMPILED_IN; - if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { - ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 0, - output, input, dataSz, - ssl->decrypt.nonce, nonceSz, - (byte *)(input + dataSz), macSz, - aad, aadSz); - } - if (ret == WC_NO_ERR_TRACE(NOT_COMPILED_IN)) - #endif - { - ret = wc_AesCcmDecrypt(ssl->decrypt.aes, output, input, - dataSz, ssl->decrypt.nonce, nonceSz, - input + dataSz, macSz, aad, aadSz); - #ifdef WOLFSSL_ASYNC_CRYPT - if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { - ret = wolfSSL_AsyncPush(ssl, - &ssl->decrypt.aes->asyncDev); - } - #endif - } - break; - #endif - - #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) - case wolfssl_chacha: - ret = ChaCha20Poly1305_Decrypt(ssl, output, input, dataSz, - ssl->decrypt.nonce, aad, aadSz, input + dataSz); - break; - #endif - - #ifdef WOLFSSL_SM4_GCM - case wolfssl_sm4_gcm: - nonceSz = SM4_GCM_NONCE_SZ; - ret = wc_Sm4GcmDecrypt(ssl->decrypt.sm4, output, input, - dataSz, ssl->decrypt.nonce, nonceSz, output + dataSz, - macSz, aad, aadSz); + if (ssl->specs.bulk_cipher_algorithm == wolfssl_aes_gcm || + ssl->specs.bulk_cipher_algorithm == wolfssl_aes_ccm) { + #ifdef WOLFSSL_ASYNC_CRYPT + /* initialize event */ + ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.aes->asyncDev, + WC_ASYNC_FLAG_NONE); + if (ret != 0) break; #endif - - #ifdef WOLFSSL_SM4_CCM - case wolfssl_sm4_ccm: - nonceSz = SM4_CCM_NONCE_SZ; - ret = wc_Sm4CcmDecrypt(ssl->decrypt.sm4, output, input, - dataSz, ssl->decrypt.nonce, nonceSz, output + dataSz, - macSz, aad, aadSz); + #if defined(HAVE_PK_CALLBACKS) + ret = NOT_COMPILED_IN; + if (ssl->ctx && ssl->ctx->PerformTlsRecordProcessingCb) { + ret = ssl->ctx->PerformTlsRecordProcessingCb(ssl, 0, + output, input, dataSz, + ssl->decrypt.nonce, nonceSz, + (byte *)(input + dataSz), macSz, + aad, aadSz); + } + if (ret != WC_NO_ERR_TRACE(NOT_COMPILED_IN)) { break; + } + ret = 0; #endif + } - #ifdef HAVE_NULL_CIPHER - case wolfssl_cipher_null: - ret = Tls13IntegrityOnly_Decrypt(ssl, output, input, dataSz, - ssl->decrypt.nonce, aad, aadSz, input + dataSz); - break; + if (ret == 0) { + ret = DecryptTls13Sw(ssl->specs.bulk_cipher_algorithm, + &ssl->decrypt, + #ifdef HAVE_ONE_TIME_AUTH + #ifndef WOLFSSL_RW_THREADED + &ssl->auth, + #else + &ssl->decAuth, + #endif + #else + NULL, #endif - default: - WOLFSSL_MSG("wolfSSL Decrypt programming error"); - return DECRYPT_ERROR; + output, input, dataSz, ssl->decrypt.nonce, + aad, aadSz, macSz, ssl->specs.hash_size); } - + #ifdef WOLFSSL_ASYNC_CRYPT + if (ret == WC_NO_ERR_TRACE(WC_PENDING_E)) { + ret = wolfSSL_AsyncPush(ssl, + &ssl->decrypt.aes->asyncDev); + } + #endif /* Advance state */ ssl->decrypt.state = CIPHER_STATE_END; @@ -3398,7 +3447,7 @@ int BuildTls13Message(WOLFSSL* ssl, byte* output, int outSz, const byte* input, #endif #ifdef WOLFSSL_THREADED_CRYPT if (ssl->buffers.encryptSignalRegistered && - type == application_data) { + ssl->options.handShakeDone && type == application_data) { ret = args->sz; } else diff --git a/src/wolfio.c b/src/wolfio.c index 8d0b2f089b..70650cccf8 100644 --- a/src/wolfio.c +++ b/src/wolfio.c @@ -703,7 +703,6 @@ int EmbedReceiveFrom(WOLFSSL *ssl, char *buf, int sz, void *ctx) #ifdef WOLFSSL_DTLS13 if (ssl->options.dtls && IsAtLeastTLSv1_3(ssl->version)) { - doDtlsTimeout = doDtlsTimeout || ssl->dtls13Rtx.rtxRecords != NULL; #ifdef WOLFSSL_RW_THREADED { int ret = wc_LockMutex(&ssl->dtls13Rtx.mutex); @@ -712,6 +711,7 @@ int EmbedReceiveFrom(WOLFSSL *ssl, char *buf, int sz, void *ctx) } } #endif + doDtlsTimeout = doDtlsTimeout || ssl->dtls13Rtx.rtxRecords != NULL; doDtlsTimeout = doDtlsTimeout || (ssl->dtls13FastTimeout && ssl->dtls13Rtx.seenRecords != NULL); #ifdef WOLFSSL_RW_THREADED diff --git a/wolfssl/internal.h b/wolfssl/internal.h index 37a51f70b2..1fe4c870d4 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -6055,7 +6055,10 @@ struct WOLFSSL { int devId; /* async device id to use */ #ifdef HAVE_ONE_TIME_AUTH OneTimeAuth auth; +#ifdef WOLFSSL_RW_THREADED + OneTimeAuth decAuth; #endif +#endif /* HAVE_ONE_TIME_AUTH */ #ifdef HAVE_TLS_EXTENSIONS TLSX* extensions; /* RFC 6066 TLS Extensions data */ #ifdef HAVE_MAX_FRAGMENT @@ -6882,10 +6885,13 @@ WOLFSSL_LOCAL int BuildMessage(WOLFSSL* ssl, byte* output, int outSz, #ifdef WOLFSSL_THREADED_CRYPT WOLFSSL_LOCAL void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, int order); +WOLFSSL_LOCAL int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, + byte* output, const byte* input, word16 dataSz, byte* nonce, + const byte* aad, word16 aadSz, word16 macSz); +WOLFSSL_LOCAL int DecryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, + byte* output, const byte* input, word16 dataSz, byte* nonce, + const byte* aad, word16 aadSz, word16 macSz, word16 hashSz); #endif -int EncryptTls13Sw(byte alg, Ciphers* encrypt, void* authPtr, byte* output, - const byte* input, word16 dataSz, byte* nonce, - const byte* aad, word16 aadSz, word32 macSz, int async); /* Use WOLFSSL_API to use this function in tests/api.c */ WOLFSSL_API int BuildTls13Message(WOLFSSL* ssl, byte* output, int outSz, const byte* input,