From d78071d63733a4978ef7258ea349e18e0293c8f6 Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Sat, 18 Jan 2025 19:04:11 +0000 Subject: [PATCH] Add crc algorithms to DigestStream Since openssl does not support crc as a digest algorithm I extended our support through a new self implemented class and took the initial implementation out to it's own class. crc32 support is implemented using zlib's implementation. crc32c and crc64nvme are supported using our own implementation to generate the crc table at compiletime. --- src/workerd/api/crypto/crc-impl.c++ | 85 +++++++++++++ src/workerd/api/crypto/crc-impl.h | 9 ++ src/workerd/api/crypto/crypto.c++ | 124 ++++++++++++++++--- src/workerd/api/crypto/crypto.h | 9 +- src/workerd/api/tests/crypto-streams-test.js | 51 ++++++++ 5 files changed, 263 insertions(+), 15 deletions(-) create mode 100644 src/workerd/api/crypto/crc-impl.c++ create mode 100644 src/workerd/api/crypto/crc-impl.h diff --git a/src/workerd/api/crypto/crc-impl.c++ b/src/workerd/api/crypto/crc-impl.c++ new file mode 100644 index 00000000000..8f380bfd385 --- /dev/null +++ b/src/workerd/api/crypto/crc-impl.c++ @@ -0,0 +1,85 @@ +// Copyright (c) 2017-2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "crc-impl.h" + +#include +#include + +namespace { +constexpr auto crcTableSize = 256; +template +constexpr T reverse(T value) { + static_assert(std::is_same::value || std::is_same::value, + "value must be uint32_t or uint64_t"); + if constexpr (std::is_same::value) { + value = ((value & 0xaaaaaaaa) >> 1) | ((value & 0x55555555) << 1); + value = ((value & 0xcccccccc) >> 2) | ((value & 0x33333333) << 2); + value = ((value & 0xf0f0f0f0) >> 4) | ((value & 0x0f0f0f0f) << 4); + value = ((value & 0xff00ff00) >> 8) | ((value & 0x00ff00ff) << 8); + value = (value >> 16) | (value << 16); + return value; + } else { + value = ((value & 0xaaaaaaaaaaaaaaaa) >> 1) | ((value & 0x5555555555555555) << 1); + value = ((value & 0xcccccccccccccccc) >> 2) | ((value & 0x3333333333333333) << 2); + value = ((value & 0xf0f0f0f0f0f0f0f0) >> 4) | ((value & 0x0f0f0f0f0f0f0f0f) << 4); + value = ((value & 0xff00ff00ff00ff00) >> 8) | ((value & 0x00ff00ff00ff00ff) << 8); + value = ((value & 0xffff0000ffff0000) >> 16) | ((value & 0x0000ffff0000ffff) << 16); + value = (value >> 32) | (value << 32); + return value; + } +} + +template +constexpr std::array gen_crc_table(T polynomial, bool reflectIn, bool reflectOut) { + static_assert(std::is_same::value || std::is_same::value, + "polynomial must be uint32_t or uint64_t"); + constexpr auto numIterations = sizeof(polynomial) * 8; // number of bits in polynomial + auto crcTable = std::array{}; + + for (T byte = 0u; byte < crcTableSize; ++byte) { + T crc = (reflectIn ? (reverse(T(byte)) >> (numIterations - 8)) : byte); + + for (int i = 0; i < numIterations; ++i) { + if (crc & (static_cast(1) << (numIterations - 1))) { + crc = (crc << 1) ^ polynomial; + } else { + crc <<= 1; + } + } + + crcTable[byte] = (reflectOut ? reverse(crc) : crc); + } + + return crcTable; +} + +// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi +constexpr auto crc32c_table = gen_crc_table(static_cast(0x1edc6f41), true, true); +// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme +constexpr auto crc64nvme_table = + gen_crc_table(static_cast(0xad93d23594c93659), true, true); +} // namespace + +uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) { + if (data == nullptr) { + return 0; + } + crc ^= 0xffffffff; + while (length--) { + crc = crc32c_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8); + } + return crc ^ 0xffffffff; +} + +uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length) { + if (data == nullptr) { + return 0; + } + crc ^= 0xffffffffffffffff; + while (length--) { + crc = crc64nvme_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8); + } + return crc ^ 0xffffffffffffffff; +} diff --git a/src/workerd/api/crypto/crc-impl.h b/src/workerd/api/crypto/crc-impl.h new file mode 100644 index 00000000000..7fac35b385a --- /dev/null +++ b/src/workerd/api/crypto/crc-impl.h @@ -0,0 +1,9 @@ +// Copyright (c) 2017-2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once +#include + +uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length); +uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length); diff --git a/src/workerd/api/crypto/crypto.c++ b/src/workerd/api/crypto/crypto.c++ index 04c4eacf8c8..59066c52bd2 100644 --- a/src/workerd/api/crypto/crypto.c++ +++ b/src/workerd/api/crypto/crypto.c++ @@ -6,6 +6,7 @@ #include "impl.h" +#include #include #include #include @@ -14,6 +15,7 @@ #include #include +#include #include #include @@ -676,13 +678,113 @@ kj::String Crypto::randomUUID() { // ======================================================================================= // Crypto Streams implementation +class CRC32DigestContext final: public DigestContext { + public: + CRC32DigestContext(): value(crc32(0, Z_NULL, 0)) {} + virtual ~CRC32DigestContext() = default; + + void write(kj::ArrayPtr buffer) { + value = crc32(value, buffer.begin(), buffer.size()); + } + + kj::Array close() { + auto beValue = htobe32(value); + static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + auto digest = kj::heapArray(sizeof(beValue)); + KJ_DASSERT(digest.size() == sizeof(beValue)); + memcpy(digest.begin(), &beValue, sizeof(beValue)); + return digest; + } + + private: + uint32_t value; +}; + +class CRC32CDigestContext final: public DigestContext { + public: + CRC32CDigestContext(): value(crc32c(0, nullptr, 0)) {} + virtual ~CRC32CDigestContext() = default; + + void write(kj::ArrayPtr buffer) { + value = crc32c(value, buffer.begin(), buffer.size()); + } + + kj::Array close() { + auto beValue = htobe32(value); + static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + auto digest = kj::heapArray(sizeof(beValue)); + KJ_DASSERT(digest.size() == sizeof(beValue)); + memcpy(digest.begin(), &beValue, sizeof(beValue)); + return digest; + } + + private: + uint32_t value; +}; + +class CRC64NVMEDigestContext final: public DigestContext { + public: + CRC64NVMEDigestContext(): value(crc64nvme(0, nullptr, 0)) {} + virtual ~CRC64NVMEDigestContext() = default; + + void write(kj::ArrayPtr buffer) { + value = crc64nvme(value, buffer.begin(), buffer.size()); + } + + kj::Array close() { + auto beValue = htobe64(value); + static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + auto digest = kj::heapArray(sizeof(beValue)); + KJ_DASSERT(digest.size() == sizeof(beValue)); + memcpy(digest.begin(), &beValue, sizeof(beValue)); + return digest; + } + + private: + uint64_t value; +}; + +class OpenSSLDigestContext final: public DigestContext { + public: + OpenSSLDigestContext(kj::StringPtr algorithm): algorithm(kj::str(algorithm)) { + auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm); + auto type = lookupDigestAlgorithm(algorithm).second; + auto opensslContext = kj::disposeWith(EVP_MD_CTX_new()); + KJ_ASSERT(opensslContext.get() != nullptr); + OSSLCALL(EVP_DigestInit_ex(opensslContext.get(), type, nullptr)); + context = kj::mv(opensslContext); + } + virtual ~OpenSSLDigestContext() = default; + + void write(kj::ArrayPtr buffer) { + auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm); + OSSLCALL(EVP_DigestUpdate(context.get(), buffer.begin(), buffer.size())); + } + + kj::Array close() { + auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm); + uint size = 0; + auto digest = kj::heapArray(EVP_MD_CTX_size(context.get())); + OSSLCALL(EVP_DigestFinal_ex(context.get(), digest.begin(), &size)); + KJ_ASSERT(size, digest.size()); + return kj::mv(digest); + } + + private: + kj::String algorithm; + kj::Own context; +}; + DigestStream::DigestContextPtr DigestStream::initContext(SubtleCrypto::HashAlgorithm& algorithm) { - auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm.name); - auto type = lookupDigestAlgorithm(algorithm.name).second; - auto context = kj::disposeWith(EVP_MD_CTX_new()); - KJ_ASSERT(context.get() != nullptr); - OSSLCALL(EVP_DigestInit_ex(context.get(), type, nullptr)); - return kj::mv(context); + if (algorithm.name == "crc32") { + return kj::heap(); + } else if (algorithm.name == "crc32c") { + return kj::heap(); + } else if (algorithm.name == "crc64nvme") { + return kj::heap(); + } else { + return kj::heap(algorithm.name); + } } DigestStream::DigestStream(kj::Own controller, @@ -726,8 +828,7 @@ kj::Maybe DigestStream::write(jsg::Lock& js, kj::ArrayPtr return errored.addRef(js); } KJ_CASE_ONEOF(ready, Ready) { - auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name); - OSSLCALL(EVP_DigestUpdate(ready.context.get(), buffer.begin(), buffer.size())); + ready.context->write(buffer); return kj::none; } } @@ -743,12 +844,7 @@ kj::Maybe DigestStream::close(jsg::Lock& js) { return errored.addRef(js); } KJ_CASE_ONEOF(ready, Ready) { - auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name); - uint size = 0; - auto digest = kj::heapArray(EVP_MD_CTX_size(ready.context.get())); - OSSLCALL(EVP_DigestFinal_ex(ready.context.get(), digest.begin(), &size)); - KJ_ASSERT(size, digest.size()); - ready.resolver.resolve(js, kj::mv(digest)); + ready.resolver.resolve(js, ready.context->close()); state.init(); return kj::none; } diff --git a/src/workerd/api/crypto/crypto.h b/src/workerd/api/crypto/crypto.h index f183a34d20a..099a529eb0c 100644 --- a/src/workerd/api/crypto/crypto.h +++ b/src/workerd/api/crypto/crypto.h @@ -666,9 +666,16 @@ class SubtleCrypto: public jsg::Object { // DigestStream is a non-standard extension that provides a way of generating // a hash digest from streaming data. It combines Web Crypto concepts into a // WritableStream and is compatible with both APIs. +class DigestContext { + public: + virtual ~DigestContext() = default; + virtual void write(kj::ArrayPtr buffer) = 0; + virtual kj::Array close() = 0; +}; + class DigestStream: public WritableStream { public: - using DigestContextPtr = kj::Own; + using DigestContextPtr = kj::Own; using Algorithm = kj::OneOf; explicit DigestStream(kj::Own controller, diff --git a/src/workerd/api/tests/crypto-streams-test.js b/src/workerd/api/tests/crypto-streams-test.js index 1ce4171abdd..d92efa4cb8e 100644 --- a/src/workerd/api/tests/crypto-streams-test.js +++ b/src/workerd/api/tests/crypto-streams-test.js @@ -1,4 +1,5 @@ import { strictEqual, deepStrictEqual, rejects, throws } from 'node:assert'; +import { Buffer } from 'node:buffer'; export const digeststream = { async test() { @@ -38,6 +39,7 @@ export const digeststream = { new crypto.DigestStream('SHA-256'); new crypto.DigestStream('SHA-384'); new crypto.DigestStream('SHA-512'); + new crypto.DigestStream('crc32'); // But fails for unknown digest names... throws(() => new crypto.DigestStream('foo')); @@ -107,6 +109,16 @@ export const digeststream = { deepStrictEqual(digest, check); } + { + const check = new Uint8Array([176, 224, 34, 147]); + const digestStream = new crypto.DigestStream('crc32'); + const writer = digestStream.getWriter(); + await writer.write(new Uint32Array([1, 2, 3])); + await writer.close(); + const digest = new Uint8Array(await digestStream.digest); + deepStrictEqual(digest, check); + } + { const digestStream = new crypto.DigestStream('md5'); const writer = digestStream.getWriter(); @@ -123,6 +135,45 @@ export const digeststream = { } } + // AWS CRC tests, source: + // https://github.com/aws/aws-sdk-js-v3/blob/c3f3d0a1c652c88fef5859881c9a12cfc8df61c1/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts#L21 + const testCases = [ + ['', 'crc32', 'AAAAAA=='], + ['abc', 'crc32', 'NSRBwg=='], + ['Hello world', 'crc32', 'i9aeUg=='], + + ['', 'crc32c', 'AAAAAA=='], + ['abc', 'crc32c', 'Nks/tw=='], + ['Hello world', 'crc32c', 'crUfeA=='], + + ['', 'crc64nvme', 'AAAAAAAAAAA='], + ['abc', 'crc64nvme', 'BeXKuz/B+us='], + ['Hello world', 'crc64nvme', 'OOJZ0D8xKts='], + + ['', 'SHA-1', '2jmj7l5rSw0yVb/vlWAYkK/YBwk='], + ['abc', 'SHA-1', 'qZk+NkcGgWq6PiVxeFDCbJzQ2J0='], + ['Hello world', 'SHA-1', 'e1AsOh9IyGCa4hLN+2Od7jlnP14='], + + ['', 'SHA-256', '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='], + ['abc', 'SHA-256', 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0='], + [ + 'Hello world', + 'SHA-256', + 'ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=', + ], + ]; + { + for (const [body, algorithm, expected] of testCases) { + const digestStream = new crypto.DigestStream(algorithm); + const writer = digestStream.getWriter(); + const enc = new TextEncoder(); + writer.write(enc.encode(body)); + writer.close(); + const digest = await digestStream.digest; + deepStrictEqual(digest, Buffer.from(expected, 'base64').buffer); + } + } + // Creating and not using a digest stream doesn't crash new crypto.DigestStream('SHA-1'); },