From 312fc25c668d30183bee4c95158a1980af558aaa Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Sat, 18 Jan 2025 19:04:11 +0000 Subject: [PATCH] Add crc algorithms to DigestStream Since openssl does not support crc as a digest algorithm I extended our support through a new self implemented class and took the initial implementation out to it's own class. crc32 support is implemented using zlib's implementation. crc32c and crc64nvme are supported using our own implementation to generate the crc table at compiletime. --- src/workerd/api/crypto/crc-impl.c++ | 85 +++++++++ src/workerd/api/crypto/crc-impl.h | 9 + src/workerd/api/crypto/crypto.c++ | 125 +++++++++++-- src/workerd/api/crypto/crypto.h | 8 +- src/workerd/api/crypto/endianness.c++ | 186 +++++++++++++++++++ src/workerd/api/crypto/endianness.h | 21 +++ src/workerd/api/tests/crypto-streams-test.js | 51 +++++ 7 files changed, 470 insertions(+), 15 deletions(-) create mode 100644 src/workerd/api/crypto/crc-impl.c++ create mode 100644 src/workerd/api/crypto/crc-impl.h create mode 100644 src/workerd/api/crypto/endianness.c++ create mode 100644 src/workerd/api/crypto/endianness.h diff --git a/src/workerd/api/crypto/crc-impl.c++ b/src/workerd/api/crypto/crc-impl.c++ new file mode 100644 index 00000000000..8f380bfd385 --- /dev/null +++ b/src/workerd/api/crypto/crc-impl.c++ @@ -0,0 +1,85 @@ +// Copyright (c) 2017-2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "crc-impl.h" + +#include +#include + +namespace { +constexpr auto crcTableSize = 256; +template +constexpr T reverse(T value) { + static_assert(std::is_same::value || std::is_same::value, + "value must be uint32_t or uint64_t"); + if constexpr (std::is_same::value) { + value = ((value & 0xaaaaaaaa) >> 1) | ((value & 0x55555555) << 1); + value = ((value & 0xcccccccc) >> 2) | ((value & 0x33333333) << 2); + value = ((value & 0xf0f0f0f0) >> 4) | ((value & 0x0f0f0f0f) << 4); + value = ((value & 0xff00ff00) >> 8) | ((value & 0x00ff00ff) << 8); + value = (value >> 16) | (value << 16); + return value; + } else { + value = ((value & 0xaaaaaaaaaaaaaaaa) >> 1) | ((value & 0x5555555555555555) << 1); + value = ((value & 0xcccccccccccccccc) >> 2) | ((value & 0x3333333333333333) << 2); + value = ((value & 0xf0f0f0f0f0f0f0f0) >> 4) | ((value & 0x0f0f0f0f0f0f0f0f) << 4); + value = ((value & 0xff00ff00ff00ff00) >> 8) | ((value & 0x00ff00ff00ff00ff) << 8); + value = ((value & 0xffff0000ffff0000) >> 16) | ((value & 0x0000ffff0000ffff) << 16); + value = (value >> 32) | (value << 32); + return value; + } +} + +template +constexpr std::array gen_crc_table(T polynomial, bool reflectIn, bool reflectOut) { + static_assert(std::is_same::value || std::is_same::value, + "polynomial must be uint32_t or uint64_t"); + constexpr auto numIterations = sizeof(polynomial) * 8; // number of bits in polynomial + auto crcTable = std::array{}; + + for (T byte = 0u; byte < crcTableSize; ++byte) { + T crc = (reflectIn ? (reverse(T(byte)) >> (numIterations - 8)) : byte); + + for (int i = 0; i < numIterations; ++i) { + if (crc & (static_cast(1) << (numIterations - 1))) { + crc = (crc << 1) ^ polynomial; + } else { + crc <<= 1; + } + } + + crcTable[byte] = (reflectOut ? reverse(crc) : crc); + } + + return crcTable; +} + +// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi +constexpr auto crc32c_table = gen_crc_table(static_cast(0x1edc6f41), true, true); +// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme +constexpr auto crc64nvme_table = + gen_crc_table(static_cast(0xad93d23594c93659), true, true); +} // namespace + +uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) { + if (data == nullptr) { + return 0; + } + crc ^= 0xffffffff; + while (length--) { + crc = crc32c_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8); + } + return crc ^ 0xffffffff; +} + +uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length) { + if (data == nullptr) { + return 0; + } + crc ^= 0xffffffffffffffff; + while (length--) { + crc = crc64nvme_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8); + } + return crc ^ 0xffffffffffffffff; +} diff --git a/src/workerd/api/crypto/crc-impl.h b/src/workerd/api/crypto/crc-impl.h new file mode 100644 index 00000000000..7fac35b385a --- /dev/null +++ b/src/workerd/api/crypto/crc-impl.h @@ -0,0 +1,9 @@ +// Copyright (c) 2017-2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once +#include + +uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length); +uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length); diff --git a/src/workerd/api/crypto/crypto.c++ b/src/workerd/api/crypto/crypto.c++ index 04c4eacf8c8..1b3d92d212e 100644 --- a/src/workerd/api/crypto/crypto.c++ +++ b/src/workerd/api/crypto/crypto.c++ @@ -6,6 +6,8 @@ #include "impl.h" +#include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include +#include #include #include @@ -676,13 +679,113 @@ kj::String Crypto::randomUUID() { // ======================================================================================= // Crypto Streams implementation +class CRC32DigestContext final: public DigestContext { + public: + CRC32DigestContext(): value(crc32(0, Z_NULL, 0)) {} + virtual ~CRC32DigestContext() = default; + + void write(kj::ArrayPtr buffer) { + value = crc32(value, buffer.begin(), buffer.size()); + } + + kj::Array close() { + auto beValue = htobe32(value); + static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + auto digest = kj::heapArray(sizeof(beValue)); + KJ_DASSERT(digest.size() == sizeof(beValue)); + memcpy(digest.begin(), &beValue, sizeof(beValue)); + return digest; + } + + private: + uint32_t value; +}; + +class CRC32CDigestContext final: public DigestContext { + public: + CRC32CDigestContext(): value(crc32c(0, nullptr, 0)) {} + virtual ~CRC32CDigestContext() = default; + + void write(kj::ArrayPtr buffer) { + value = crc32c(value, buffer.begin(), buffer.size()); + } + + kj::Array close() { + auto beValue = htobe32(value); + static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + auto digest = kj::heapArray(sizeof(beValue)); + KJ_DASSERT(digest.size() == sizeof(beValue)); + memcpy(digest.begin(), &beValue, sizeof(beValue)); + return digest; + } + + private: + uint32_t value; +}; + +class CRC64NVMEDigestContext final: public DigestContext { + public: + CRC64NVMEDigestContext(): value(crc64nvme(0, nullptr, 0)) {} + virtual ~CRC64NVMEDigestContext() = default; + + void write(kj::ArrayPtr buffer) { + value = crc64nvme(value, buffer.begin(), buffer.size()); + } + + kj::Array close() { + auto beValue = htobe64(value); + static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + auto digest = kj::heapArray(sizeof(beValue)); + KJ_DASSERT(digest.size() == sizeof(beValue)); + memcpy(digest.begin(), &beValue, sizeof(beValue)); + return digest; + } + + private: + uint64_t value; +}; + +class OpenSSLDigestContext final: public DigestContext { + public: + OpenSSLDigestContext(kj::StringPtr algorithm): algorithm(kj::str(algorithm)) { + auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm); + auto type = lookupDigestAlgorithm(algorithm).second; + auto opensslContext = kj::disposeWith(EVP_MD_CTX_new()); + KJ_ASSERT(opensslContext.get() != nullptr); + OSSLCALL(EVP_DigestInit_ex(opensslContext.get(), type, nullptr)); + context = kj::mv(opensslContext); + } + virtual ~OpenSSLDigestContext() = default; + + void write(kj::ArrayPtr buffer) { + auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm); + OSSLCALL(EVP_DigestUpdate(context.get(), buffer.begin(), buffer.size())); + } + + kj::Array close() { + auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm); + uint size = 0; + auto digest = kj::heapArray(EVP_MD_CTX_size(context.get())); + OSSLCALL(EVP_DigestFinal_ex(context.get(), digest.begin(), &size)); + KJ_ASSERT(size, digest.size()); + return kj::mv(digest); + } + + private: + kj::String algorithm; + kj::Own context; +}; + DigestStream::DigestContextPtr DigestStream::initContext(SubtleCrypto::HashAlgorithm& algorithm) { - auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm.name); - auto type = lookupDigestAlgorithm(algorithm.name).second; - auto context = kj::disposeWith(EVP_MD_CTX_new()); - KJ_ASSERT(context.get() != nullptr); - OSSLCALL(EVP_DigestInit_ex(context.get(), type, nullptr)); - return kj::mv(context); + if (algorithm.name == "crc32") { + return kj::heap(); + } else if (algorithm.name == "crc32c") { + return kj::heap(); + } else if (algorithm.name == "crc64nvme") { + return kj::heap(); + } else { + return kj::heap(algorithm.name); + } } DigestStream::DigestStream(kj::Own controller, @@ -726,8 +829,7 @@ kj::Maybe DigestStream::write(jsg::Lock& js, kj::ArrayPtr return errored.addRef(js); } KJ_CASE_ONEOF(ready, Ready) { - auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name); - OSSLCALL(EVP_DigestUpdate(ready.context.get(), buffer.begin(), buffer.size())); + ready.context->write(buffer); return kj::none; } } @@ -743,12 +845,7 @@ kj::Maybe DigestStream::close(jsg::Lock& js) { return errored.addRef(js); } KJ_CASE_ONEOF(ready, Ready) { - auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name); - uint size = 0; - auto digest = kj::heapArray(EVP_MD_CTX_size(ready.context.get())); - OSSLCALL(EVP_DigestFinal_ex(ready.context.get(), digest.begin(), &size)); - KJ_ASSERT(size, digest.size()); - ready.resolver.resolve(js, kj::mv(digest)); + ready.resolver.resolve(js, ready.context->close()); state.init(); return kj::none; } diff --git a/src/workerd/api/crypto/crypto.h b/src/workerd/api/crypto/crypto.h index f183a34d20a..75b38b8b4cc 100644 --- a/src/workerd/api/crypto/crypto.h +++ b/src/workerd/api/crypto/crypto.h @@ -666,9 +666,15 @@ class SubtleCrypto: public jsg::Object { // DigestStream is a non-standard extension that provides a way of generating // a hash digest from streaming data. It combines Web Crypto concepts into a // WritableStream and is compatible with both APIs. +class DigestContext { + public: + virtual void write(kj::ArrayPtr buffer) = 0; + virtual kj::Array close() = 0; +}; + class DigestStream: public WritableStream { public: - using DigestContextPtr = kj::Own; + using DigestContextPtr = kj::Own; using Algorithm = kj::OneOf; explicit DigestStream(kj::Own controller, diff --git a/src/workerd/api/crypto/endianness.c++ b/src/workerd/api/crypto/endianness.c++ new file mode 100644 index 00000000000..f38b23a7c3d --- /dev/null +++ b/src/workerd/api/crypto/endianness.c++ @@ -0,0 +1,186 @@ +// Copyright (c) 2017-2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "endianness.h" + +#if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) + +#define __WINDOWS__ + +#endif + +#if defined(__linux__) || defined(__CYGWIN__) + +#include + +#elif defined(__APPLE__) + +#include + +uint16_t htobe16(uint16_t x) { + return OSSwapHostToBigInt16(x); +} +uint16_t htole16(uint16_t x) { + return OSSwapHostToLittleInt16(x); +} +uint16_t be16toh(uint16_t x) { + return OSSwapBigToHostInt16(x); +} +uint16_t le16toh(uint16_t x) { + return OSSwapLittleToHostInt16(x); +} + +uint32_t htobe32(uint32_t x) { + return OSSwapHostToBigInt32(x); +} +uint32_t htole32(uint32_t x) { + return OSSwapHostToLittleInt32(x); +} +uint32_t be32toh(uint32_t x) { + return OSSwapBigToHostInt32(x); +} +uint32_t le32toh(uint32_t x) { + return OSSwapLittleToHostInt32(x); +} + +uint64_t htobe64(uint64_t x) { + return OSSwapHostToBigInt64(x); +} +uint64_t htole64(uint64_t x) { + return OSSwapHostToLittleInt64(x); +} +uint64_t be64toh(uint64_t x) { + return OSSwapBigToHostInt64(x); +} +uint64_t le64toh(uint64_t x) { + return OSSwapLittleToHostInt64(x); +} + +#elif defined(__OpenBSD__) + +#include + +#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) + +#include + +uint16_t be16toh(uint16_t x) { + return betoh16(x); +} +uint16_t le16toh(uint16_t x) { + return letoh16(x); +} + +uint32_t be32toh(uint32_t x) { + return betoh32(x); +} +uint32_t le32toh(uint32_t x) { + return letoh32(x); +} + +uint64_t be64toh(uint64_t x) { + return betoh64(x); +} +uint64_t le64toh(uint64_t x) { + return letoh64(x); +} + +#elif defined(__WINDOWS__) + +#include + +#if BYTE_ORDER == LITTLE_ENDIAN + +uint16_t htobe16(uint16_t x) { + return htons(x); +} +uint16_t htole16(uint16_t x) { + return x; +} +uint16_t be16toh(uint16_t x) { + return ntohs(x); +} +uint16_t le16toh(uint16_t x) { + return x; +} + +uint32_t htobe32(uint32_t x) { + return htonl(x); +} +uint32_t htole32(uint32_t x) { + return x; +} +uint32_t be32toh(uint32_t x) { + return ntohl(x); +} +uint32_t le32toh(uint32_t x) { + return x; +} + +uint64_t htobe64(uint64_t x) { + return htonll(x); +} +uint64_t htole64(uint64_t x) { + return x; +} +uint64_t be64toh(uint64_t x) { + return ntohll(x); +} +uint64_t le64toh(uint64_t x) { + return x; +} + +#elif BYTE_ORDER == BIG_ENDIAN + +/* that would be xbox 360 */ +uint16_t htobe16(uint16_t x) { + return x; +} +uint16_t htole16(uint16_t x) { + return __builtin_bswap16(x); +} +uint16_t be16toh(uint16_t x) { + return x; +} +uint16_t le16toh(uint16_t x) { + return __builtin_bswap16(x); +} + +uint32_t htobe32(uint32_t x) { + return x; +} +uint32_t htole32(uint32_t x) { + return __builtin_bswap32(x); +} +uint32_t be32toh(uint32_t x) { + return x; +} +uint32_t le32toh(uint32_t x) { + return __builtin_bswap32(x); +} + +uint64_t htobe64(uint64_t x) { + return x; +} +uint64_t htole64(uint64_t x) { + return __builtin_bswap64(x); +} +uint64_t be64toh(uint64_t x) { + return x; +} +uint64_t le64toh(uint64_t x) { + return __builtin_bswap64(x); +} + +#else + +#error byte order not supported + +#endif + +#else + +#error platform not supported + +#endif diff --git a/src/workerd/api/crypto/endianness.h b/src/workerd/api/crypto/endianness.h new file mode 100644 index 00000000000..e29ffc5da4f --- /dev/null +++ b/src/workerd/api/crypto/endianness.h @@ -0,0 +1,21 @@ +// Copyright (c) 2017-2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once +#include + +uint16_t htobe16(uint16_t x); +uint16_t htole16(uint16_t x); +uint16_t be16toh(uint16_t x); +uint16_t le16toh(uint16_t x); + +uint32_t htobe32(uint32_t x); +uint32_t htole32(uint32_t x); +uint32_t be32toh(uint32_t x); +uint32_t le32toh(uint32_t x); + +uint64_t htobe64(uint64_t x); +uint64_t htole64(uint64_t x); +uint64_t be64toh(uint64_t x); +uint64_t le64toh(uint64_t x); diff --git a/src/workerd/api/tests/crypto-streams-test.js b/src/workerd/api/tests/crypto-streams-test.js index 1ce4171abdd..d92efa4cb8e 100644 --- a/src/workerd/api/tests/crypto-streams-test.js +++ b/src/workerd/api/tests/crypto-streams-test.js @@ -1,4 +1,5 @@ import { strictEqual, deepStrictEqual, rejects, throws } from 'node:assert'; +import { Buffer } from 'node:buffer'; export const digeststream = { async test() { @@ -38,6 +39,7 @@ export const digeststream = { new crypto.DigestStream('SHA-256'); new crypto.DigestStream('SHA-384'); new crypto.DigestStream('SHA-512'); + new crypto.DigestStream('crc32'); // But fails for unknown digest names... throws(() => new crypto.DigestStream('foo')); @@ -107,6 +109,16 @@ export const digeststream = { deepStrictEqual(digest, check); } + { + const check = new Uint8Array([176, 224, 34, 147]); + const digestStream = new crypto.DigestStream('crc32'); + const writer = digestStream.getWriter(); + await writer.write(new Uint32Array([1, 2, 3])); + await writer.close(); + const digest = new Uint8Array(await digestStream.digest); + deepStrictEqual(digest, check); + } + { const digestStream = new crypto.DigestStream('md5'); const writer = digestStream.getWriter(); @@ -123,6 +135,45 @@ export const digeststream = { } } + // AWS CRC tests, source: + // https://github.com/aws/aws-sdk-js-v3/blob/c3f3d0a1c652c88fef5859881c9a12cfc8df61c1/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts#L21 + const testCases = [ + ['', 'crc32', 'AAAAAA=='], + ['abc', 'crc32', 'NSRBwg=='], + ['Hello world', 'crc32', 'i9aeUg=='], + + ['', 'crc32c', 'AAAAAA=='], + ['abc', 'crc32c', 'Nks/tw=='], + ['Hello world', 'crc32c', 'crUfeA=='], + + ['', 'crc64nvme', 'AAAAAAAAAAA='], + ['abc', 'crc64nvme', 'BeXKuz/B+us='], + ['Hello world', 'crc64nvme', 'OOJZ0D8xKts='], + + ['', 'SHA-1', '2jmj7l5rSw0yVb/vlWAYkK/YBwk='], + ['abc', 'SHA-1', 'qZk+NkcGgWq6PiVxeFDCbJzQ2J0='], + ['Hello world', 'SHA-1', 'e1AsOh9IyGCa4hLN+2Od7jlnP14='], + + ['', 'SHA-256', '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='], + ['abc', 'SHA-256', 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0='], + [ + 'Hello world', + 'SHA-256', + 'ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=', + ], + ]; + { + for (const [body, algorithm, expected] of testCases) { + const digestStream = new crypto.DigestStream(algorithm); + const writer = digestStream.getWriter(); + const enc = new TextEncoder(); + writer.write(enc.encode(body)); + writer.close(); + const digest = await digestStream.digest; + deepStrictEqual(digest, Buffer.from(expected, 'base64').buffer); + } + } + // Creating and not using a digest stream doesn't crash new crypto.DigestStream('SHA-1'); },