diff options
| -rw-r--r-- | sys/conf/files.arm64 | 1 | ||||
| -rw-r--r-- | sys/libkern/arm64/crc32c_armv8.S | 78 | ||||
| -rw-r--r-- | sys/libkern/crc32.c | 16 | ||||
| -rw-r--r-- | sys/sys/libkern.h | 3 | ||||
| -rw-r--r-- | tests/sys/kern/Makefile | 8 | ||||
| -rw-r--r-- | tests/sys/kern/libkern_crc32.c | 19 |
6 files changed, 124 insertions, 1 deletions
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 10eb61f..16880f4 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -107,6 +107,7 @@ libkern/flsl.c standard libkern/flsll.c standard libkern/memmove.c standard libkern/memset.c standard +libkern/arm64/crc32c_armv8.S standard cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S optional zfs | dtrace compile-with "${CDDL_C}" cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" diff --git a/sys/libkern/arm64/crc32c_armv8.S b/sys/libkern/arm64/crc32c_armv8.S new file mode 100644 index 0000000..072dfd6 --- /dev/null +++ b/sys/libkern/arm64/crc32c_armv8.S @@ -0,0 +1,78 @@ +/*- + * Copyright (c) 2017 Michael Tuexen + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include <machine/asm.h> +__FBSDID("$FreeBSD$"); + +/* + * uint32_t + * armv8_crc32c(uint32_t crc, const unsigned char *buf, unsigned int len) + */ + +ENTRY(armv8_crc32c) + cbz w2, end + tbz x1, #0x0, half_word_aligned + sub w2, w2, 0x1 + ldr w10, [x1], #0x1 + crc32cb w0, w0, w10 +half_word_aligned: + cmp w2, #0x2 + b.lo last_byte + tbz x1, #0x1, word_aligned + sub w2, w2, 0x2 + ldr w10, [x1], #0x2 + crc32ch w0, w0, w10 +word_aligned: + cmp w2, #0x4 + b.lo last_half_word + tbz x1, #0x2, double_word_aligned + sub w2, w2, 0x4 + ldr w10, [x1], #0x4 + crc32cw w0, w0, w10 +double_word_aligned: + lsr w9, w2, #0x3 + cbz w9, last_word +loop: + ldr x10, [x1], #0x8 + crc32cx w0, w0, x10 + subs w9, w9, #1 + b.ne loop +last_word: + tbz w2, #0x2, last_half_word + ldr w10, [x1], #0x4 + crc32cw w0, w0, w10 +last_half_word: + tbz w2, #0x1, last_byte + ldr w10, [x1], #0x2 + crc32ch w0, w0, w10 +last_byte: + tbz w2, #0x0, end + ldr w10, [x1], #0x1 + crc32cb w0, w0, w10 +end: + ret +END(armv8_crc32c) diff --git a/sys/libkern/crc32.c b/sys/libkern/crc32.c index 281c4b6..b0a8ce0 100644 --- a/sys/libkern/crc32.c +++ b/sys/libkern/crc32.c @@ -54,6 +54,10 @@ __FBSDID("$FreeBSD$"); #include <machine/specialreg.h> #endif +#if defined(__aarch64__) +#include <machine/cpu.h> +#endif + const uint32_t crc32_tab[] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, @@ -760,6 +764,18 @@ calculate_crc32c(uint32_t crc32c, return (sse42_crc32c(crc32c, buffer, length)); } else #endif +#if defined(__aarch64__) + uint64_t reg; + + /* + * We only test for CRC32 support on the CPU with index 0 assuming that + * this applies to all CPUs. + */ + reg = READ_SPECIALREG(id_aa64isar0_el1); + if (ID_AA64ISAR0_CRC32(reg) != ID_AA64ISAR0_CRC32_NONE) { + return (armv8_crc32c(crc32c, buffer, length)); + } else +#endif if (length < 4) { return (singletable_crc32c(crc32c, buffer, length)); } else { diff --git a/sys/sys/libkern.h b/sys/sys/libkern.h index a960df8..b622dd8 100644 --- a/sys/sys/libkern.h +++ b/sys/sys/libkern.h @@ -182,6 +182,9 @@ calculate_crc32c(uint32_t crc32c, const unsigned char *buffer, #if defined(__amd64__) || defined(__i386__) uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned); #endif +#if defined(__aarch64__) +uint32_t armv8_crc32c(uint32_t, const unsigned char *, unsigned int); +#endif #endif diff --git a/tests/sys/kern/Makefile b/tests/sys/kern/Makefile index d78785c..35cc8c7 100644 --- a/tests/sys/kern/Makefile +++ b/tests/sys/kern/Makefile @@ -25,10 +25,16 @@ NETBSD_ATF_TESTS_C+= mqueue_test CFLAGS.mqueue_test+= -I${SRCTOP}/tests LIBADD.mqueue_test+= rt -.if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386" +.if ${MACHINE_ARCH} == "amd64" || \ + ${MACHINE_ARCH} == "i386" || \ + ${MACHINE_ARCH} == "aarch64" ATF_TESTS_C+= libkern_crc32 CFLAGS.libkern_crc32+= -msse4 -DUSERSPACE_TESTING +.if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386" LDADD.libkern_crc32+= ${SRCTOP}/sys/libkern/x86/crc32_sse42.c +.else +LDADD.libkern_crc32+= ${SRCTOP}/sys/libkern/arm64/crc32c_armv8.S +.endif .endif # subr_unit.c contains functions whose prototypes lie in headers that cannot be diff --git a/tests/sys/kern/libkern_crc32.c b/tests/sys/kern/libkern_crc32.c index 5eef045..03d0231 100644 --- a/tests/sys/kern/libkern_crc32.c +++ b/tests/sys/kern/libkern_crc32.c @@ -32,7 +32,13 @@ #include <atf-c.h> +#if defined(__amd64__) || defined(__i386__) extern uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned); +#elif defined(__aarch64__) +extern uint32_t armv8_crc32c(uint32_t, const unsigned char *, unsigned); +#else +#error These tests are not supported on this platform +#endif ATF_TC_WITHOUT_HEAD(crc32c_basic_correctness); ATF_TC_BODY(crc32c_basic_correctness, tc) @@ -79,8 +85,13 @@ ATF_TC_BODY(crc32c_basic_correctness, tc) ATF_REQUIRE(nitems(inputs) == nitems(results)); for (i = 0; i < nitems(inputs); i++) { +#if defined(__amd64__) || defined(__i386__) act = sse42_crc32c(~0, (const void *)&inputs[i], sizeof(inputs[0])); +#else + act = armv8_crc32c(~0, (const void *)&inputs[i], + sizeof(inputs[0])); +#endif ATF_REQUIRE_MSG(act == results[i], "crc32c(0x%jx) = 0x%08x, got 0x%08x", (uintmax_t)inputs[i], results[i], act); @@ -100,7 +111,11 @@ ATF_TC_BODY(crc32c_alignment, tc) for (i = 1; i < 8; i++) { memcpy(&buf[i], &input, sizeof(input)); +#if defined(__amd64__) || defined(__i386__) act = sse42_crc32c(~0, (const void *)&buf[i], sizeof(input)); +#else + act = armv8_crc32c(~0, (const void *)&buf[i], sizeof(input)); +#endif ATF_REQUIRE_MSG(act == result, "crc32c(0x%jx) = 0x%08x, got 0x%08x", (uintmax_t)input, result, act); @@ -117,7 +132,11 @@ ATF_TC_BODY(crc32c_trailing_bytes, tc) const uint32_t result = 0xec638d62; uint32_t act; +#if defined(__amd64__) || defined(__i386__) act = sse42_crc32c(~0, input, sizeof(input)); +#else + act = armv8_crc32c(~0, input, sizeof(input)); +#endif ATF_REQUIRE_MSG(act == result, "expected 0x%08x, got 0x%08x", result, act); } |
