crc32 : the calculation result of folly is inconsistent with the calculation result of ISAL

[LiuQinfei]

Issue:

crc32 (IEEE) : the calculation result of folly is inconsistent with the calculation result of ISAL;
However, crc32c (iSCSI) is consistent.
If this is expected?

What i did?

[root@client1 folly]# git diff 48227f514017c97d55f0b8b40786e91980e28b9c b89411df15fe1a9bd9cedecf87210157379cf04e folly/hash/Checksum.cpp
diff --git a/folly/hash/Checksum.cpp b/folly/hash/Checksum.cpp
index 5d134821f..5bea35235 100644
--- a/folly/hash/Checksum.cpp
+++ b/folly/hash/Checksum.cpp
@@ -15,6 +15,7 @@
  */

 #include <folly/hash/Checksum.h>
+#include <isa-l.h>

 #include <algorithm>
 #include <stdexcept>
@@ -29,6 +30,8 @@
 #include <nmmintrin.h>
 #endif

+#define ON_ARM (1)
+
 namespace folly {

 namespace detail {
@@ -77,6 +80,24 @@ bool crc32_hw_supported() {
   return id.sse42();
 }

+#elif FOLLY_AARCH64 || (1)
+
+uint32_t crc32_sw(
+    const uint8_t* data, size_t nbytes, uint32_t startingChecksum);
+
+// Fast SIMD implementation of CRC-32 for arm64 with isal
+uint32_t crc32_hw(
+    const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
+  return crc32_ieee(startingChecksum, data, nbytes);
+}
+
+bool crc32c_hw_supported() {
+  return true;
+}
+
+bool crc32_hw_supported() {
+  return true;
+}
 #else

 uint32_t crc32_hw(
[root@client1 folly]# git diff 48227f514017c97d55f0b8b40786e91980e28b9c b89411df15fe1a9bd9cedecf87210157379cf04e folly/hash/detail/Crc32cDetail.cpp
diff --git a/folly/hash/detail/Crc32cDetail.cpp b/folly/hash/detail/Crc32cDetail.cpp
index ada6ac20c..be20caf24 100644
--- a/folly/hash/detail/Crc32cDetail.cpp
+++ b/folly/hash/detail/Crc32cDetail.cpp
@@ -34,6 +34,7 @@
 #include <boost/preprocessor/arithmetic/add.hpp>
 #include <boost/preprocessor/arithmetic/sub.hpp>
 #include <boost/preprocessor/repetition/repeat_from_to.hpp>
+#include <isa-l.h>

 #include <folly/CppAttributes.h>
 #include <folly/hash/detail/ChecksumDetail.h>
@@ -285,6 +286,12 @@ uint32_t crc32c_hw(const uint8_t* buf, size_t len, uint32_t crc) {
   return (uint32_t)crc0;
 }

+#elif FOLLY_AARCH64 || (1)
+uint32_t crc32c_hw(
+    const uint8_t* buf, size_t len, uint32_t crc) {
+  return crc32_iscsi((unsigned char *)buf, (int)len, (unsigned int)crc);
+}
+
 #else

 uint32_t crc32c_hw(

Environment Information

[root@client1 CacheLib]# cat /etc/os-release
NAME="CentOS Stream"
VERSION="8"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="8"
PLATFORM_ID="platform:el8"
PRETTY_NAME="CentOS Stream 8"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:8"
HOME_URL="https://centos.org/"
BUG_REPORT_URL="https://bugzilla.redhat.com/"
REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8"
REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
[root@client1 CacheLib]# uname -r
4.18.0-372.9.1.el8.x86_64

cachelib: tag: v2023.08.21.00

Benchmark result:

[root@client1 CacheLib]# ./build-folly/checksum_test
[==========] Running 12 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 12 tests from Checksum
[ RUN      ] Checksum.crc32c_software
[       OK ] Checksum.crc32c_software (32 ms)
[ RUN      ] Checksum.crc32c_continuation_software
[       OK ] Checksum.crc32c_continuation_software (32 ms)
[ RUN      ] Checksum.crc32c_hardware
[       OK ] Checksum.crc32c_hardware (0 ms)
[ RUN      ] Checksum.crc32c_hardware_eq
[       OK ] Checksum.crc32c_hardware_eq (2 ms)
[ RUN      ] Checksum.crc32c_continuation_hardware
[       OK ] Checksum.crc32c_continuation_hardware (0 ms)
[ RUN      ] Checksum.crc32c_autodetect
[       OK ] Checksum.crc32c_autodetect (0 ms)
[ RUN      ] Checksum.crc32c_continuation_autodetect
[       OK ] Checksum.crc32c_continuation_autodetect (0 ms)
[ RUN      ] Checksum.crc32
/home/liuqinfei/CacheLib20230811/CacheLib/cachelib/external/folly/folly/hash/test/ChecksumTest.cpp:163: Failure
Expected equality of these values:
  sw_res
    Which is: 3943577151
  hw_res
    Which is: 4223187897

[  FAILED  ] Checksum.crc32 (33 ms)
[ RUN      ] Checksum.crc32_continuation
/home/liuqinfei/CacheLib20230811/CacheLib/cachelib/external/folly/folly/hash/test/ChecksumTest.cpp:184: Failure
Expected equality of these values:
  sw_res
    Which is: 3918048003
  hw_res
    Which is: 1564758716

[  FAILED  ] Checksum.crc32_continuation (65 ms)
[ RUN      ] Checksum.crc32_type
/home/liuqinfei/CacheLib20230811/CacheLib/cachelib/external/folly/folly/hash/test/ChecksumTest.cpp:102: Failure
Expected equality of these values:
  follyResult
    Which is: 1246755826
  boostResult
    Which is: 957143474

/home/liuqinfei/CacheLib20230811/CacheLib/cachelib/external/folly/folly/hash/test/ChecksumTest.cpp:210: Failure
Expected equality of these values:
  combined
    Which is: 4282882988
  crcfull
    Which is: 1119910916

[  FAILED  ] Checksum.crc32_combine (1 ms)
[ RUN      ] Checksum.crc32c_combine
/home/liuqinfei/CacheLib20230811/CacheLib/cachelib/external/folly/folly/hash/test/ChecksumTest.cpp:221: Failure
Expected equality of these values:
  combined
    Which is: 1766302386
  crcfull
    Which is: 801318612

[  FAILED  ] Checksum.crc32c_combine (1 ms)
[----------] 12 tests from Checksum (202 ms total)

[----------] Global test environment tear-down
[==========] 12 tests from 1 test suite ran. (202 ms total)
[  PASSED  ] 7 tests.
[  FAILED  ] 5 tests, listed below:
[  FAILED  ] Checksum.crc32
[  FAILED  ] Checksum.crc32_continuation
[  FAILED  ] Checksum.crc32_type
[  FAILED  ] Checksum.crc32_combine
[  FAILED  ] Checksum.crc32c_combine

 5 FAILED TESTS