Add crc algorithms to DigestStream

Since openssl does not support crc as a digest algorithm I extended our support through a new self implemented class and took the initial implementation out to it's own class. crc32 support is implemented using zlib's implementation. crc32c and crc64nvme are supported using our own implementation to generate the crc table at compiletime.
cloudflare · Jan 20, 2025 · eddc015 · eddc015
1 parent c134deb
commit eddc015
Show file tree

Hide file tree

Showing 5 changed files with 262 additions and 15 deletions.
diff --git a/src/workerd/api/crypto/crc-impl.c++ b/src/workerd/api/crypto/crc-impl.c++
@@ -0,0 +1,85 @@
+// Copyright (c) 2017-2025 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+#include "crc-impl.h"
+
+#include <array>
+#include <type_traits>
+
+namespace {
+constexpr auto crcTableSize = 256;
+template <typename T>
+constexpr T reverse(T value) {
+  static_assert(std::is_same<T, uint32_t>::value || std::is_same<T, uint64_t>::value,
+      "value must be uint32_t or uint64_t");
+  if constexpr (std::is_same<T, uint32_t>::value) {
+    value = ((value & 0xaaaaaaaa) >> 1) | ((value & 0x55555555) << 1);
+    value = ((value & 0xcccccccc) >> 2) | ((value & 0x33333333) << 2);
+    value = ((value & 0xf0f0f0f0) >> 4) | ((value & 0x0f0f0f0f) << 4);
+    value = ((value & 0xff00ff00) >> 8) | ((value & 0x00ff00ff) << 8);
+    value = (value >> 16) | (value << 16);
+    return value;
+  } else {
+    value = ((value & 0xaaaaaaaaaaaaaaaa) >> 1) | ((value & 0x5555555555555555) << 1);
+    value = ((value & 0xcccccccccccccccc) >> 2) | ((value & 0x3333333333333333) << 2);
+    value = ((value & 0xf0f0f0f0f0f0f0f0) >> 4) | ((value & 0x0f0f0f0f0f0f0f0f) << 4);
+    value = ((value & 0xff00ff00ff00ff00) >> 8) | ((value & 0x00ff00ff00ff00ff) << 8);
+    value = ((value & 0xffff0000ffff0000) >> 16) | ((value & 0x0000ffff0000ffff) << 16);
+    value = (value >> 32) | (value << 32);
+    return value;
+  }
+}
+
+template <typename T>
+constexpr std::array<T, crcTableSize> gen_crc_table(T polynomial, bool reflectIn, bool reflectOut) {
+  static_assert(std::is_same<T, uint32_t>::value || std::is_same<T, uint64_t>::value,
+      "polynomial must be uint32_t or uint64_t");
+  constexpr auto numIterations = sizeof(polynomial) * 8;  // number of bits in polynomial
+  auto crcTable = std::array<T, crcTableSize>{};
+
+  for (T byte = 0u; byte < crcTableSize; ++byte) {
+    T crc = (reflectIn ? (reverse(T(byte)) >> (numIterations - 8)) : byte);
+
+    for (int i = 0; i < numIterations; ++i) {
+      if (crc & (static_cast<T>(1) << (numIterations - 1))) {
+        crc = (crc << 1) ^ polynomial;
+      } else {
+        crc <<= 1;
+      }
+    }
+
+    crcTable[byte] = (reflectOut ? reverse(crc) : crc);
+  }
+
+  return crcTable;
+}
+
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi
+constexpr auto crc32c_table = gen_crc_table(static_cast<uint32_t>(0x1edc6f41), true, true);
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme
+constexpr auto crc64nvme_table =
+    gen_crc_table(static_cast<uint64_t>(0xad93d23594c93659), true, true);
+}  // namespace
+
+uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) {
+  if (data == nullptr) {
+    return 0;
+  }
+  crc ^= 0xffffffff;
+  while (length--) {
+    crc = crc32c_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
+  }
+  return crc ^ 0xffffffff;
+}
+
+uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length) {
+  if (data == nullptr) {
+    return 0;
+  }
+  crc ^= 0xffffffffffffffff;
+  while (length--) {
+    crc = crc64nvme_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
+  }
+  return crc ^ 0xffffffffffffffff;
+}
diff --git a/src/workerd/api/crypto/crc-impl.h b/src/workerd/api/crypto/crc-impl.h
@@ -0,0 +1,9 @@
+// Copyright (c) 2017-2025 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+#pragma once
+#include <stdint.h>
+
+uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length);
+uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length);
diff --git a/src/workerd/api/crypto/crypto.c++ b/src/workerd/api/crypto/crypto.c++
@@ -6,6 +6,7 @@
 
 #include "impl.h"
 
+#include <workerd/api/crypto/crc-impl.h>
 #include <workerd/api/streams/standard.h>
 #include <workerd/api/util.h>
 #include <workerd/io/io-context.h>
@@ -14,6 +15,7 @@
 
 #include <openssl/digest.h>
 #include <openssl/mem.h>
+#include <zlib.h>
 
 #include <algorithm>
 #include <array>
@@ -676,13 +678,113 @@ kj::String Crypto::randomUUID() {
 // =======================================================================================
 // Crypto Streams implementation
 
+class CRC32DigestContext final: public DigestContext {
+ public:
+  CRC32DigestContext(): value(crc32(0, Z_NULL, 0)) {}
+  virtual ~CRC32DigestContext() = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) {
+    value = crc32(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() {
+    auto beValue = htobe32(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint32_t value;
+};
+
+class CRC32CDigestContext final: public DigestContext {
+ public:
+  CRC32CDigestContext(): value(crc32c(0, nullptr, 0)) {}
+  virtual ~CRC32CDigestContext() = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) {
+    value = crc32c(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() {
+    auto beValue = htobe32(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint32_t value;
+};
+
+class CRC64NVMEDigestContext final: public DigestContext {
+ public:
+  CRC64NVMEDigestContext(): value(crc64nvme(0, nullptr, 0)) {}
+  virtual ~CRC64NVMEDigestContext() = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) {
+    value = crc64nvme(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() {
+    auto beValue = htobe64(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint64_t value;
+};
+
+class OpenSSLDigestContext final: public DigestContext {
+ public:
+  OpenSSLDigestContext(kj::StringPtr algorithm): algorithm(kj::str(algorithm)) {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    auto type = lookupDigestAlgorithm(algorithm).second;
+    auto opensslContext = kj::disposeWith<EVP_MD_CTX_free>(EVP_MD_CTX_new());
+    KJ_ASSERT(opensslContext.get() != nullptr);
+    OSSLCALL(EVP_DigestInit_ex(opensslContext.get(), type, nullptr));
+    context = kj::mv(opensslContext);
+  }
+  virtual ~OpenSSLDigestContext() = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    OSSLCALL(EVP_DigestUpdate(context.get(), buffer.begin(), buffer.size()));
+  }
+
+  kj::Array<kj::byte> close() {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    uint size = 0;
+    auto digest = kj::heapArray<kj::byte>(EVP_MD_CTX_size(context.get()));
+    OSSLCALL(EVP_DigestFinal_ex(context.get(), digest.begin(), &size));
+    KJ_ASSERT(size, digest.size());
+    return kj::mv(digest);
+  }
+
+ private:
+  kj::String algorithm;
+  kj::Own<EVP_MD_CTX> context;
+};
+
 DigestStream::DigestContextPtr DigestStream::initContext(SubtleCrypto::HashAlgorithm& algorithm) {
-  auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm.name);
-  auto type = lookupDigestAlgorithm(algorithm.name).second;
-  auto context = kj::disposeWith<EVP_MD_CTX_free>(EVP_MD_CTX_new());
-  KJ_ASSERT(context.get() != nullptr);
-  OSSLCALL(EVP_DigestInit_ex(context.get(), type, nullptr));
-  return kj::mv(context);
+  if (algorithm.name == "crc32") {
+    return kj::heap<CRC32DigestContext>();
+  } else if (algorithm.name == "crc32c") {
+    return kj::heap<CRC32CDigestContext>();
+  } else if (algorithm.name == "crc64nvme") {
+    return kj::heap<CRC64NVMEDigestContext>();
+  } else {
+    return kj::heap<OpenSSLDigestContext>(algorithm.name);
+  }
 }
 
 DigestStream::DigestStream(kj::Own<WritableStreamController> controller,
@@ -726,8 +828,7 @@ kj::Maybe<StreamStates::Errored> DigestStream::write(jsg::Lock& js, kj::ArrayPtr
       return errored.addRef(js);
     }
     KJ_CASE_ONEOF(ready, Ready) {
-      auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name);
-      OSSLCALL(EVP_DigestUpdate(ready.context.get(), buffer.begin(), buffer.size()));
+      ready.context->write(buffer);
       return kj::none;
     }
   }
@@ -743,12 +844,7 @@ kj::Maybe<StreamStates::Errored> DigestStream::close(jsg::Lock& js) {
       return errored.addRef(js);
     }
     KJ_CASE_ONEOF(ready, Ready) {
-      auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name);
-      uint size = 0;
-      auto digest = kj::heapArray<kj::byte>(EVP_MD_CTX_size(ready.context.get()));
-      OSSLCALL(EVP_DigestFinal_ex(ready.context.get(), digest.begin(), &size));
-      KJ_ASSERT(size, digest.size());
-      ready.resolver.resolve(js, kj::mv(digest));
+      ready.resolver.resolve(js, ready.context->close());
       state.init<StreamStates::Closed>();
       return kj::none;
     }

diff --git a/src/workerd/api/crypto/crypto.h b/src/workerd/api/crypto/crypto.h
@@ -666,9 +666,15 @@ class SubtleCrypto: public jsg::Object {
 // DigestStream is a non-standard extension that provides a way of generating
 // a hash digest from streaming data. It combines Web Crypto concepts into a
 // WritableStream and is compatible with both APIs.
+class DigestContext {
+ public:
+  virtual void write(kj::ArrayPtr<kj::byte> buffer) = 0;
+  virtual kj::Array<kj::byte> close() = 0;
+};
+
 class DigestStream: public WritableStream {
  public:
-  using DigestContextPtr = kj::Own<EVP_MD_CTX>;
+  using DigestContextPtr = kj::Own<DigestContext>;
   using Algorithm = kj::OneOf<kj::String, SubtleCrypto::HashAlgorithm>;
 
   explicit DigestStream(kj::Own<WritableStreamController> controller,

diff --git a/src/workerd/api/tests/crypto-streams-test.js b/src/workerd/api/tests/crypto-streams-test.js
@@ -1,4 +1,5 @@
 import { strictEqual, deepStrictEqual, rejects, throws } from 'node:assert';
+import { Buffer } from 'node:buffer';
 
 export const digeststream = {
   async test() {
@@ -38,6 +39,7 @@ export const digeststream = {
       new crypto.DigestStream('SHA-256');
       new crypto.DigestStream('SHA-384');
       new crypto.DigestStream('SHA-512');
+      new crypto.DigestStream('crc32');
 
       // But fails for unknown digest names...
       throws(() => new crypto.DigestStream('foo'));
@@ -107,6 +109,16 @@ export const digeststream = {
       deepStrictEqual(digest, check);
     }
 
+    {
+      const check = new Uint8Array([176, 224, 34, 147]);
+      const digestStream = new crypto.DigestStream('crc32');
+      const writer = digestStream.getWriter();
+      await writer.write(new Uint32Array([1, 2, 3]));
+      await writer.close();
+      const digest = new Uint8Array(await digestStream.digest);
+      deepStrictEqual(digest, check);
+    }
+
     {
       const digestStream = new crypto.DigestStream('md5');
       const writer = digestStream.getWriter();
@@ -123,6 +135,45 @@ export const digeststream = {
       }
     }
 
+    // AWS CRC tests, source:
+    // https://github.com/aws/aws-sdk-js-v3/blob/c3f3d0a1c652c88fef5859881c9a12cfc8df61c1/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts#L21
+    const testCases = [
+      ['', 'crc32', 'AAAAAA=='],
+      ['abc', 'crc32', 'NSRBwg=='],
+      ['Hello world', 'crc32', 'i9aeUg=='],
+
+      ['', 'crc32c', 'AAAAAA=='],
+      ['abc', 'crc32c', 'Nks/tw=='],
+      ['Hello world', 'crc32c', 'crUfeA=='],
+
+      ['', 'crc64nvme', 'AAAAAAAAAAA='],
+      ['abc', 'crc64nvme', 'BeXKuz/B+us='],
+      ['Hello world', 'crc64nvme', 'OOJZ0D8xKts='],
+
+      ['', 'SHA-1', '2jmj7l5rSw0yVb/vlWAYkK/YBwk='],
+      ['abc', 'SHA-1', 'qZk+NkcGgWq6PiVxeFDCbJzQ2J0='],
+      ['Hello world', 'SHA-1', 'e1AsOh9IyGCa4hLN+2Od7jlnP14='],
+
+      ['', 'SHA-256', '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='],
+      ['abc', 'SHA-256', 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0='],
+      [
+        'Hello world',
+        'SHA-256',
+        'ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=',
+      ],
+    ];
+    {
+      for (const [body, algorithm, expected] of testCases) {
+        const digestStream = new crypto.DigestStream(algorithm);
+        const writer = digestStream.getWriter();
+        const enc = new TextEncoder();
+        writer.write(enc.encode(body));
+        writer.close();
+        const digest = await digestStream.digest;
+        deepStrictEqual(digest, Buffer.from(expected, 'base64').buffer);
+      }
+    }
+
     // Creating and not using a digest stream doesn't crash
     new crypto.DigestStream('SHA-1');
   },