Add crc algorithms to DigestStream

Since openssl does not support crc as a digest algorithm I extended our support through a new self implemented class and took the initial implementation out to it's own class. crc32 support is implemented using zlib's implementation. crc32c and crc64nvme are supported using our own implementation to generate the crc table at compiletime.
cloudflare · Jan 21, 2025 · fe6cfac · fe6cfac
1 parent c134deb
commit fe6cfac
Show file tree

Hide file tree

Showing 7 changed files with 480 additions and 15 deletions.
diff --git a/src/workerd/api/crypto/crc-impl.c++ b/src/workerd/api/crypto/crc-impl.c++
@@ -0,0 +1,85 @@
+// Copyright (c) 2017-2025 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+#include "crc-impl.h"
+
+#include <array>
+#include <type_traits>
+
+namespace {
+constexpr auto crcTableSize = 256;
+
+template <typename T>
+concept Uint64OrUint32 = std::unsigned_integral<T> && (sizeof(T) == 8 || sizeof(T) == 4);
+
+template <Uint64OrUint32 T>
+constexpr T reverse(T value) {
+  if constexpr (sizeof(T) == 4) {
+    value = ((value & 0xaaaaaaaa) >> 1) | ((value & 0x55555555) << 1);
+    value = ((value & 0xcccccccc) >> 2) | ((value & 0x33333333) << 2);
+    value = ((value & 0xf0f0f0f0) >> 4) | ((value & 0x0f0f0f0f) << 4);
+    value = ((value & 0xff00ff00) >> 8) | ((value & 0x00ff00ff) << 8);
+    value = (value >> 16) | (value << 16);
+    return value;
+  } else {
+    value = ((value & 0xaaaaaaaaaaaaaaaa) >> 1) | ((value & 0x5555555555555555) << 1);
+    value = ((value & 0xcccccccccccccccc) >> 2) | ((value & 0x3333333333333333) << 2);
+    value = ((value & 0xf0f0f0f0f0f0f0f0) >> 4) | ((value & 0x0f0f0f0f0f0f0f0f) << 4);
+    value = ((value & 0xff00ff00ff00ff00) >> 8) | ((value & 0x00ff00ff00ff00ff) << 8);
+    value = ((value & 0xffff0000ffff0000) >> 16) | ((value & 0x0000ffff0000ffff) << 16);
+    value = (value >> 32) | (value << 32);
+    return value;
+  }
+}
+
+template <Uint64OrUint32 T>
+constexpr std::array<T, crcTableSize> gen_crc_table(T polynomial, bool reflectIn, bool reflectOut) {
+  constexpr auto numIterations = sizeof(polynomial) * 8;  // number of bits in polynomial
+  auto crcTable = std::array<T, crcTableSize>{};
+
+  for (T byte = 0u; byte < crcTableSize; ++byte) {
+    T crc = (reflectIn ? (reverse(T(byte)) >> (numIterations - 8)) : byte);
+
+    for (int i = 0; i < numIterations; ++i) {
+      if (crc & (static_cast<T>(1) << (numIterations - 1))) {
+        crc = (crc << 1) ^ polynomial;
+      } else {
+        crc <<= 1;
+      }
+    }
+
+    crcTable[byte] = (reflectOut ? reverse(crc) : crc);
+  }
+
+  return crcTable;
+}
+
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi
+constexpr auto crc32c_table = gen_crc_table(static_cast<uint32_t>(0x1edc6f41), true, true);
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme
+constexpr auto crc64nvme_table =
+    gen_crc_table(static_cast<uint64_t>(0xad93d23594c93659), true, true);
+}  // namespace
+
+uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) {
+  if (data == nullptr) {
+    return 0;
+  }
+  crc ^= 0xffffffff;
+  while (length--) {
+    crc = crc32c_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
+  }
+  return crc ^ 0xffffffff;
+}
+
+uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length) {
+  if (data == nullptr) {
+    return 0;
+  }
+  crc ^= 0xffffffffffffffff;
+  while (length--) {
+    crc = crc64nvme_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
+  }
+  return crc ^ 0xffffffffffffffff;
+}
diff --git a/src/workerd/api/crypto/crc-impl.h b/src/workerd/api/crypto/crc-impl.h
@@ -0,0 +1,14 @@
+// Copyright (c) 2017-2025 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+#pragma once
+#include <stdint.h>
+
+// crc32-c implementation according to the spec:
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi
+uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length);
+
+// crc64-nvme implementation according to the spec:
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme
+uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length);
diff --git a/src/workerd/api/crypto/crypto.c++ b/src/workerd/api/crypto/crypto.c++
@@ -6,6 +6,8 @@
 
 #include "impl.h"
 
+#include <workerd/api/crypto/crc-impl.h>
+#include <workerd/api/crypto/endianness.h>
 #include <workerd/api/streams/standard.h>
 #include <workerd/api/util.h>
 #include <workerd/io/io-context.h>
@@ -14,6 +16,7 @@
 
 #include <openssl/digest.h>
 #include <openssl/mem.h>
+#include <zlib.h>
 
 #include <algorithm>
 #include <array>
@@ -676,13 +679,113 @@ kj::String Crypto::randomUUID() {
 // =======================================================================================
 // Crypto Streams implementation
 
+class CRC32DigestContext final: public DigestContext {
+ public:
+  CRC32DigestContext(): value(crc32(0, Z_NULL, 0)) {}
+  ~CRC32DigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    value = crc32(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto beValue = htobe32(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint32_t value;
+};
+
+class CRC32CDigestContext final: public DigestContext {
+ public:
+  CRC32CDigestContext(): value(crc32c(0, nullptr, 0)) {}
+  ~CRC32CDigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    value = crc32c(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto beValue = htobe32(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint32_t value;
+};
+
+class CRC64NVMEDigestContext final: public DigestContext {
+ public:
+  CRC64NVMEDigestContext(): value(crc64nvme(0, nullptr, 0)) {}
+  ~CRC64NVMEDigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    value = crc64nvme(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto beValue = htobe64(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint64_t value;
+};
+
+class OpenSSLDigestContext final: public DigestContext {
+ public:
+  OpenSSLDigestContext(kj::StringPtr algorithm): algorithm(kj::str(algorithm)) {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    auto type = lookupDigestAlgorithm(algorithm).second;
+    auto opensslContext = kj::disposeWith<EVP_MD_CTX_free>(EVP_MD_CTX_new());
+    KJ_ASSERT(opensslContext.get() != nullptr);
+    OSSLCALL(EVP_DigestInit_ex(opensslContext.get(), type, nullptr));
+    context = kj::mv(opensslContext);
+  }
+  ~OpenSSLDigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    OSSLCALL(EVP_DigestUpdate(context.get(), buffer.begin(), buffer.size()));
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    uint size = 0;
+    auto digest = kj::heapArray<kj::byte>(EVP_MD_CTX_size(context.get()));
+    OSSLCALL(EVP_DigestFinal_ex(context.get(), digest.begin(), &size));
+    KJ_ASSERT(size, digest.size());
+    return kj::mv(digest);
+  }
+
+ private:
+  kj::String algorithm;
+  kj::Own<EVP_MD_CTX> context;
+};
+
 DigestStream::DigestContextPtr DigestStream::initContext(SubtleCrypto::HashAlgorithm& algorithm) {
-  auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm.name);
-  auto type = lookupDigestAlgorithm(algorithm.name).second;
-  auto context = kj::disposeWith<EVP_MD_CTX_free>(EVP_MD_CTX_new());
-  KJ_ASSERT(context.get() != nullptr);
-  OSSLCALL(EVP_DigestInit_ex(context.get(), type, nullptr));
-  return kj::mv(context);
+  if (algorithm.name == "crc32") {
+    return kj::heap<CRC32DigestContext>();
+  } else if (algorithm.name == "crc32c") {
+    return kj::heap<CRC32CDigestContext>();
+  } else if (algorithm.name == "crc64nvme") {
+    return kj::heap<CRC64NVMEDigestContext>();
+  } else {
+    return kj::heap<OpenSSLDigestContext>(algorithm.name);
+  }
 }
 
 DigestStream::DigestStream(kj::Own<WritableStreamController> controller,
@@ -726,8 +829,7 @@ kj::Maybe<StreamStates::Errored> DigestStream::write(jsg::Lock& js, kj::ArrayPtr
       return errored.addRef(js);
     }
     KJ_CASE_ONEOF(ready, Ready) {
-      auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name);
-      OSSLCALL(EVP_DigestUpdate(ready.context.get(), buffer.begin(), buffer.size()));
+      ready.context->write(buffer);
       return kj::none;
     }
   }
@@ -743,12 +845,7 @@ kj::Maybe<StreamStates::Errored> DigestStream::close(jsg::Lock& js) {
       return errored.addRef(js);
     }
     KJ_CASE_ONEOF(ready, Ready) {
-      auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name);
-      uint size = 0;
-      auto digest = kj::heapArray<kj::byte>(EVP_MD_CTX_size(ready.context.get()));
-      OSSLCALL(EVP_DigestFinal_ex(ready.context.get(), digest.begin(), &size));
-      KJ_ASSERT(size, digest.size());
-      ready.resolver.resolve(js, kj::mv(digest));
+      ready.resolver.resolve(js, ready.context->close());
       state.init<StreamStates::Closed>();
       return kj::none;
     }

diff --git a/src/workerd/api/crypto/crypto.h b/src/workerd/api/crypto/crypto.h
@@ -666,9 +666,16 @@ class SubtleCrypto: public jsg::Object {
 // DigestStream is a non-standard extension that provides a way of generating
 // a hash digest from streaming data. It combines Web Crypto concepts into a
 // WritableStream and is compatible with both APIs.
+class DigestContext {
+ public:
+  virtual ~DigestContext() noexcept = default;
+  virtual void write(kj::ArrayPtr<kj::byte> buffer) = 0;
+  virtual kj::Array<kj::byte> close() = 0;
+};
+
 class DigestStream: public WritableStream {
  public:
-  using DigestContextPtr = kj::Own<EVP_MD_CTX>;
+  using DigestContextPtr = kj::Own<DigestContext>;
   using Algorithm = kj::OneOf<kj::String, SubtleCrypto::HashAlgorithm>;
 
   explicit DigestStream(kj::Own<WritableStreamController> controller,