Merge pull request #3358 from cloudflare/dlapid/crc32_digeststream

Add crc32 algorithm to DigestStream

src/workerd/api/crypto/crc-impl.c++

@@ -0,0 +1,85 @@
+// Copyright (c) 2017-2025 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+#include "crc-impl.h"
+
+#include <array>
+#include <type_traits>
+
+namespace {
+constexpr auto crcTableSize = 256;
+
+template <typename T>
+concept Uint64OrUint32 = std::unsigned_integral<T> && (sizeof(T) == 8 || sizeof(T) == 4);
+
+template <Uint64OrUint32 T>
+constexpr T reverse(T value) {
+  if constexpr (sizeof(T) == 4) {
+    value = ((value & 0xaaaaaaaa) >> 1) | ((value & 0x55555555) << 1);
+    value = ((value & 0xcccccccc) >> 2) | ((value & 0x33333333) << 2);
+    value = ((value & 0xf0f0f0f0) >> 4) | ((value & 0x0f0f0f0f) << 4);
+    value = ((value & 0xff00ff00) >> 8) | ((value & 0x00ff00ff) << 8);
+    value = (value >> 16) | (value << 16);
+    return value;
+  } else {
+    value = ((value & 0xaaaaaaaaaaaaaaaa) >> 1) | ((value & 0x5555555555555555) << 1);
+    value = ((value & 0xcccccccccccccccc) >> 2) | ((value & 0x3333333333333333) << 2);
+    value = ((value & 0xf0f0f0f0f0f0f0f0) >> 4) | ((value & 0x0f0f0f0f0f0f0f0f) << 4);
+    value = ((value & 0xff00ff00ff00ff00) >> 8) | ((value & 0x00ff00ff00ff00ff) << 8);
+    value = ((value & 0xffff0000ffff0000) >> 16) | ((value & 0x0000ffff0000ffff) << 16);
+    value = (value >> 32) | (value << 32);
+    return value;
+  }
+}
+
+template <Uint64OrUint32 T>
+constexpr std::array<T, crcTableSize> gen_crc_table(T polynomial, bool reflectIn, bool reflectOut) {
+  constexpr auto numIterations = sizeof(polynomial) * 8;  // number of bits in polynomial
+  auto crcTable = std::array<T, crcTableSize>{};
+
+  for (T byte = 0u; byte < crcTableSize; ++byte) {
+    T crc = (reflectIn ? (reverse(T(byte)) >> (numIterations - 8)) : byte);
+
+    for (int i = 0; i < numIterations; ++i) {
+      if (crc & (static_cast<T>(1) << (numIterations - 1))) {
+        crc = (crc << 1) ^ polynomial;
+      } else {
+        crc <<= 1;
+      }
+    }
+
+    crcTable[byte] = (reflectOut ? reverse(crc) : crc);
+  }
+
+  return crcTable;
+}
+
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi
+constexpr auto crc32c_table = gen_crc_table(static_cast<uint32_t>(0x1edc6f41), true, true);
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme
+constexpr auto crc64nvme_table =
+    gen_crc_table(static_cast<uint64_t>(0xad93d23594c93659), true, true);
+}  // namespace
+
+uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) {
+  if (data == nullptr) {
+    return 0;
+  }
+  crc ^= 0xffffffff;
+  while (length--) {
+    crc = crc32c_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
+  }
+  return crc ^ 0xffffffff;
+}
+
+uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length) {
+  if (data == nullptr) {
+    return 0;
+  }
+  crc ^= 0xffffffffffffffff;
+  while (length--) {
+    crc = crc64nvme_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
+  }
+  return crc ^ 0xffffffffffffffff;
+}

src/workerd/api/crypto/crc-impl.h

@@ -0,0 +1,14 @@
+// Copyright (c) 2017-2025 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+#pragma once
+#include <stdint.h>
+
+// crc32-c implementation according to the spec:
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi
+uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length);
+
+// crc64-nvme implementation according to the spec:
+// https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme
+uint64_t crc64nvme(uint64_t crc, const uint8_t *data, unsigned int length);

src/workerd/api/crypto/crypto.c++

@@ -6,6 +6,8 @@
 
 #include "impl.h"
 
+#include <workerd/api/crypto/crc-impl.h>
+#include <workerd/api/crypto/endianness.h>
 #include <workerd/api/streams/standard.h>
 #include <workerd/api/util.h>
 #include <workerd/io/io-context.h>
@@ -14,6 +16,7 @@
 
 #include <openssl/digest.h>
 #include <openssl/mem.h>
+#include <zlib.h>
 
 #include <algorithm>
 #include <array>
@@ -676,13 +679,113 @@ kj::String Crypto::randomUUID() {
 // =======================================================================================
 // Crypto Streams implementation
 
+class CRC32DigestContext final: public DigestContext {
+ public:
+  CRC32DigestContext(): value(crc32(0, Z_NULL, 0)) {}
+  ~CRC32DigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    value = crc32(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto beValue = htobe32(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint32_t value;
+};
+
+class CRC32CDigestContext final: public DigestContext {
+ public:
+  CRC32CDigestContext(): value(crc32c(0, nullptr, 0)) {}
+  ~CRC32CDigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    value = crc32c(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto beValue = htobe32(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint32_t value;
+};
+
+class CRC64NVMEDigestContext final: public DigestContext {
+ public:
+  CRC64NVMEDigestContext(): value(crc64nvme(0, nullptr, 0)) {}
+  ~CRC64NVMEDigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    value = crc64nvme(value, buffer.begin(), buffer.size());
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto beValue = htobe64(value);
+    static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?");
+    auto digest = kj::heapArray<kj::byte>(sizeof(beValue));
+    KJ_DASSERT(digest.size() == sizeof(beValue));
+    memcpy(digest.begin(), &beValue, sizeof(beValue));
+    return digest;
+  }
+
+ private:
+  uint64_t value;
+};
+
+class OpenSSLDigestContext final: public DigestContext {
+ public:
+  OpenSSLDigestContext(kj::StringPtr algorithm): algorithm(kj::str(algorithm)) {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    auto type = lookupDigestAlgorithm(algorithm).second;
+    auto opensslContext = kj::disposeWith<EVP_MD_CTX_free>(EVP_MD_CTX_new());
+    KJ_ASSERT(opensslContext.get() != nullptr);
+    OSSLCALL(EVP_DigestInit_ex(opensslContext.get(), type, nullptr));
+    context = kj::mv(opensslContext);
+  }
+  ~OpenSSLDigestContext() noexcept override = default;
+
+  void write(kj::ArrayPtr<kj::byte> buffer) override {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    OSSLCALL(EVP_DigestUpdate(context.get(), buffer.begin(), buffer.size()));
+  }
+
+  kj::Array<kj::byte> close() override {
+    auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm);
+    uint size = 0;
+    auto digest = kj::heapArray<kj::byte>(EVP_MD_CTX_size(context.get()));
+    OSSLCALL(EVP_DigestFinal_ex(context.get(), digest.begin(), &size));
+    KJ_ASSERT(size, digest.size());
+    return kj::mv(digest);
+  }
+
+ private:
+  kj::String algorithm;
+  kj::Own<EVP_MD_CTX> context;
+};
+
 DigestStream::DigestContextPtr DigestStream::initContext(SubtleCrypto::HashAlgorithm& algorithm) {
-  auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, algorithm.name);
-  auto type = lookupDigestAlgorithm(algorithm.name).second;
-  auto context = kj::disposeWith<EVP_MD_CTX_free>(EVP_MD_CTX_new());
-  KJ_ASSERT(context.get() != nullptr);
-  OSSLCALL(EVP_DigestInit_ex(context.get(), type, nullptr));
-  return kj::mv(context);
+  if (algorithm.name == "crc32") {
+    return kj::heap<CRC32DigestContext>();
+  } else if (algorithm.name == "crc32c") {
+    return kj::heap<CRC32CDigestContext>();
+  } else if (algorithm.name == "crc64nvme") {
+    return kj::heap<CRC64NVMEDigestContext>();
+  } else {
+    return kj::heap<OpenSSLDigestContext>(algorithm.name);
+  }
 }
 
 DigestStream::DigestStream(kj::Own<WritableStreamController> controller,
@@ -726,8 +829,7 @@ kj::Maybe<StreamStates::Errored> DigestStream::write(jsg::Lock& js, kj::ArrayPtr
       return errored.addRef(js);
     }
     KJ_CASE_ONEOF(ready, Ready) {
-      auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name);
-      OSSLCALL(EVP_DigestUpdate(ready.context.get(), buffer.begin(), buffer.size()));
+      ready.context->write(buffer);
       return kj::none;
     }
   }
@@ -743,12 +845,7 @@ kj::Maybe<StreamStates::Errored> DigestStream::close(jsg::Lock& js) {
       return errored.addRef(js);
     }
     KJ_CASE_ONEOF(ready, Ready) {
-      auto checkErrorsOnFinish = webCryptoOperationBegin(__func__, ready.algorithm.name);
-      uint size = 0;
-      auto digest = kj::heapArray<kj::byte>(EVP_MD_CTX_size(ready.context.get()));
-      OSSLCALL(EVP_DigestFinal_ex(ready.context.get(), digest.begin(), &size));
-      KJ_ASSERT(size, digest.size());
-      ready.resolver.resolve(js, kj::mv(digest));
+      ready.resolver.resolve(js, ready.context->close());
       state.init<StreamStates::Closed>();
       return kj::none;
     }

src/workerd/api/crypto/crypto.h

@@ -666,9 +666,16 @@ class SubtleCrypto: public jsg::Object {
 // DigestStream is a non-standard extension that provides a way of generating
 // a hash digest from streaming data. It combines Web Crypto concepts into a
 // WritableStream and is compatible with both APIs.
+class DigestContext {
+ public:
+  virtual ~DigestContext() noexcept = default;
+  virtual void write(kj::ArrayPtr<kj::byte> buffer) = 0;
+  virtual kj::Array<kj::byte> close() = 0;
+};
+
 class DigestStream: public WritableStream {
  public:
-  using DigestContextPtr = kj::Own<EVP_MD_CTX>;
+  using DigestContextPtr = kj::Own<DigestContext>;
   using Algorithm = kj::OneOf<kj::String, SubtleCrypto::HashAlgorithm>;
 
   explicit DigestStream(kj::Own<WritableStreamController> controller,