diff --git a/.gitignore b/.gitignore
index 0ef96441751..b444a290425 100644
--- a/.gitignore
+++ b/.gitignore
@@ -239,12 +239,17 @@ linuxkm/linuxkm
linuxkm/src
linuxkm/patches/src
*.nds
+
+# Generated during FreeBSD kernel module build.
bsdkm/export_syms
bsdkm/i386
bsdkm/libwolfssl.ko
bsdkm/machine
bsdkm/opt_global.h
bsdkm/x86
+bsdkm/bus_if.h
+bsdkm/cryptodev_if.h
+bsdkm/device_if.h
# autotools generated
scripts/unit.test
diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras
index 0a681bd403a..3a45f971e31 100644
--- a/.wolfssl_known_macro_extras
+++ b/.wolfssl_known_macro_extras
@@ -36,6 +36,7 @@ BLAKE2B_SELFTEST
BLAKE2S_SELFTEST
BLOCKING
BSDKM_EXPORT_SYMS
+ENABLED_BSDKM_REGISTER
BSP_DEFAULT_IO_CHANNEL_DEFINED
BSP_LED_0
BSP_LED_1
diff --git a/Makefile.am b/Makefile.am
index b7fc4db2fb3..7ae6d99fdec 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -247,8 +247,9 @@ if BUILD_BSDKM
EXTRA_CFLAGS EXTRA_CPPFLAGS EXTRA_CCASFLAGS EXTRA_LDFLAGS \
AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \
AM_CCASFLAGS CCASFLAGS \
- src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS
-
+ src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_BSDKM_REGISTER \
+ ENABLED_ASM ENABLED_INTELASM ENABLED_AESNI ENABLED_AESNI_WITH_AVX \
+ ENABLED_KERNEL_BENCHMARKS
endif
diff --git a/bsdkm/Makefile b/bsdkm/Makefile
index dd6bbcbd775..5a191080f70 100644
--- a/bsdkm/Makefile
+++ b/bsdkm/Makefile
@@ -1,19 +1,27 @@
-# wolfssl kernel module name and source, and root dir.
-KMOD=libwolfssl
-SRCS=wolfkmod.c
-WOLFSSL_DIR=../
+# wolfssl kernel module name and main source, and wolfssl root dir.
+KMOD = libwolfssl
+SRCS = wolfkmod.c
+WOLFSSL_DIR = ../
-CFLAGS+=-I${WOLFSSL_DIR}
-CFLAGS+=-DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER
+CFLAGS += -I${WOLFSSL_DIR}
+CFLAGS += -DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER
#
# debug options
# verbose printing:
-# CFLAGS+=-DWOLFSSL_BSDKM_VERBOSE_DEBUG
+# CFLAGS += -DWOLFSSL_BSDKM_VERBOSE_DEBUG
#
# print memory mallocs / frees:
-# CFLAGS+=-DWOLFSSL_BSDKM_MEMORY_DEBUG
+# CFLAGS += -DWOLFSSL_BSDKM_MEMORY_DEBUG
#
-CFLAGS+=$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
+# print fpu_kern_enter / leave:
+# CFLAGS += WOLFSSL_BSDKM_FPU_DEBUG
+#
+CFLAGS += $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
+
+.if defined(ENABLED_BSDKM_REGISTER)
+ # These device header files are generated during build.
+ SRCS += bus_if.h cryptodev_if.h device_if.h
+.endif
# FreeBSD make does not support GNU make's patsubst and related. Filter
# through sed instead.
@@ -21,19 +29,26 @@ WOLFSSL_OBJS != echo ${src_libwolfssl_la_OBJECTS} | \
sed 's|src_libwolfssl_la-||g' | sed 's|\.lo|.o|g' | \
sed 's|wolfcrypt/src/|${WOLFSSL_DIR}/wolfcrypt/src/|g'
+# wolfcrypt test
.if ${ENABLED_CRYPT_TESTS} == "yes"
WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/test/test.o
.else
- CFLAGS+=-DNO_CRYPT_TEST
+ CFLAGS += -DNO_CRYPT_TEST
+.endif
+
+# wolfcrypt benchmark
+.if ${ENABLED_KERNEL_BENCHMARKS} == "yes"
+ WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/benchmark/benchmark.o
+ CFLAGS += -DWOLFSSL_NO_FLOAT_FMT
.endif
OBJS += ${WOLFSSL_OBJS}
# Export no public symbols by default.
.if !defined(BSDKM_EXPORT_SYMS)
- EXPORT_SYMS=NO
+ EXPORT_SYMS = NO
.else
- EXPORT_SYMS=${BSDKM_EXPORT_SYMS}
+ EXPORT_SYMS = ${BSDKM_EXPORT_SYMS}
.endif
# Default to live kernel src tree makefile at
@@ -45,12 +60,52 @@ OBJS += ${WOLFSSL_OBJS}
.endif
.include "${SYSDIR}/conf/kmod.mk"
+#
+# To use aesni and friends in FreeBSD kernel we need to adjust build flags.
+# See these kernel makefiles for reference:
+# - /usr/src/sys/modules/aesni/Makefile
+# - /usr/src/sys/conf/kern.mk
+#
+WOLFKMOD_SIMD_BASE = -msse -msse2 -msse4.1
+WOLFKMOD_SIMD_AES = -maes -mpclmul
+WOLFKMOD_SIMD_AVX = -mavx -mavx2
+
+.if ${ENABLED_AESNI} == "yes"
+ CFLAGS.aes.c += ${WOLFKMOD_SIMD_BASE}
+ CFLAGS.aes.c += ${WOLFKMOD_SIMD_AES}
+.if ${ENABLED_AESNI_WITH_AVX} == "yes"
+ CFLAGS.aes.c += ${WOLFKMOD_SIMD_AVX}
+.endif # ENABLED_AESNI_WITH_AVX #
+ CFLAGS.aes.c := ${CFLAGS.aes.c:N-nostdinc}
+ CFLAGS.aes.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endif # ENABLED_AESNI
+
+.if ${ENABLED_ASM} == "yes"
+.for f in chacha dilithium poly1305 sha sha256 sha3 sha512
+ CFLAGS.${f}.c += ${WOLFKMOD_SIMD_BASE}
+ CFLAGS.${f}.c += ${WOLFKMOD_SIMD_AVX}
+ CFLAGS.${f}.c := ${CFLAGS.${f}.c:N-nostdinc}
+ CFLAGS.${f}.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endfor
+
+.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endif # ENABLED_ASM == "yes"
+
+# wolfcrypt benchmark always needs simd for the floating point timings.
+.if ${ENABLED_KERNEL_BENCHMARKS} == "yes"
+ CFLAGS.benchmark.c += ${WOLFKMOD_SIMD_BASE}
+ CFLAGS.benchmark.c := ${CFLAGS.benchmark.c:N-nostdinc}
+ CFLAGS.benchmark.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+ .PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
+.endif
+
# Smooth out a few inconsistencies between FreeBSD default compiler flags
# in /usr/src/sys/conf/kern.mk, vs wolfssl harden flags in
# m4/ax_harden_compiler_flags.m4. E.g. some FreeBSD header files shorten
# 64 to 32 bit, and some wolfcrypt functions cast away const.
-CFLAGS+= -Wno-unused-function
-CFLAGS+= -Wno-cast-qual
-CFLAGS+= -Wno-error=cast-qual
-CFLAGS+= -Wno-shorten-64-to-32
-CFLAGS+= -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""
+CFLAGS += -Wno-unused-function
+CFLAGS += -Wno-cast-qual
+CFLAGS += -Wno-error=cast-qual
+CFLAGS += -Wno-shorten-64-to-32
+CFLAGS += -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""
diff --git a/bsdkm/README.md b/bsdkm/README.md
index b84c2588ff7..0c23ceb2517 100644
--- a/bsdkm/README.md
+++ b/bsdkm/README.md
@@ -7,10 +7,10 @@ other loadable modules to link to wolfCrypt.
Supported features:
- wolfCrypt in kernel.
- FIPS-wolfcrypt.
+- crypto acceleration: AES-NI, AVX, etc.
Planned features:
-- crypto acceleration: AES-NI, AVX, etc.
-- kernel opencrypto driver registration.
+- kernel opencrypto driver registration (supported for internal testing presently).
- full wolfSSL in kernel (kernel TLS).
## Building and Installing
@@ -44,10 +44,12 @@ sudo kldunload libwolfssl
### options
-| freebsdkm option | description |
-| :------------------------------- | :--------------------------------------- |
-| --with-bsd-export-syms=LIST | Export list of symbols as global.
. Options are 'all', 'none', or
comma separated list of symbols. |
-| --with-kernel-source=PATH | Path to kernel tree root (default `/usr/src/sys`) |
+| freebsdkm option | description |
+| :--------------------------------- | :--------------------------------------- |
+| --with-bsd-export-syms=LIST | Export list of symbols as global.
. Options are 'all', 'none', or
comma separated list of symbols. |
+| --with-kernel-source=PATH | Path to kernel tree root (default `/usr/src/sys`) |
+| --enable-kernel-benchmarks | Run wolfcrypt benchmark at module load |
+| --enable-freebsdkm-crypto-register | Register with the FreeBSD kernel opencrypto
framework (preliminary, for testing) |
### FIPS
diff --git a/bsdkm/bsdkm_wc_port.h b/bsdkm/bsdkm_wc_port.h
index 2a5524d5f5e..0bd80f092b0 100644
--- a/bsdkm/bsdkm_wc_port.h
+++ b/bsdkm/bsdkm_wc_port.h
@@ -61,13 +61,13 @@ static inline time_t wolfkmod_time(time_t * tloc) {
#define WOLFSSL_DEBUG_PRINTF_FN printf
/* str and char utility functions */
-#define XATOI(s) ({ \
- char * endptr = NULL; \
- long _xatoi_ret = strtol(s, &endptr, 10); \
- if ((s) == endptr || *endptr != '\0') { \
- _xatoi_ret = 0; \
- } \
- (int)_xatoi_ret; \
+#define XATOI(s) ({ \
+ char * endptr = NULL; \
+ long _xatoi_ret = strtol(s, &endptr, 10); \
+ if ((s) == endptr || *endptr != '\0') { \
+ _xatoi_ret = 0; \
+ } \
+ (int)_xatoi_ret; \
})
#if !defined(XMALLOC_OVERRIDE)
@@ -103,6 +103,33 @@ extern struct malloc_type M_WOLFSSL[1];
})
#endif /* WOLFSSL_BSDKM_DEBUG_MEMORY */
+
+#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
+ int wolfkmod_vecreg_init(void);
+ void wolfkmod_vecreg_exit(void);
+ int wolfkmod_vecreg_save(int flags_unused);
+ void wolfkmod_vecreg_restore(void);
+ /* wrapper defines for FPU_KERN(9).
+ * /usr/src/sys/amd64/amd64/fpu.c
+ * /usr/src/sys/amd64/include/pcb.h
+ * */
+ #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+ #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+ #endif
+
+ #define SAVE_VECTOR_REGISTERS(fail_clause) { \
+ int _svr_ret = wolfkmod_vecreg_save(0); \
+ if (_svr_ret != 0) { \
+ fail_clause \
+ } \
+ }
+
+ #define SAVE_VECTOR_REGISTERS2() wolfkmod_vecreg_save(0)
+
+ #define RESTORE_VECTOR_REGISTERS() wolfkmod_vecreg_restore()
+
+#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS */
+
#if !defined(SINGLE_THREADED)
#define WC_MUTEX_OPS_INLINE
@@ -149,7 +176,8 @@ extern struct malloc_type M_WOLFSSL[1];
typedef volatile int wolfSSL_Atomic_Int;
typedef volatile unsigned int wolfSSL_Atomic_Uint;
#define WOLFSSL_ATOMIC_INITIALIZER(x) (x)
- #define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x))
+ #define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x))
+ #define WOLFSSL_ATOMIC_LOAD_UINT(x) atomic_load_acq_int(&(x))
#define WOLFSSL_ATOMIC_STORE(x, v) atomic_store_rel_int(&(x), (v))
#define WOLFSSL_ATOMIC_OPS
diff --git a/bsdkm/include.am b/bsdkm/include.am
index 896a5447c9b..44666b6c084 100644
--- a/bsdkm/include.am
+++ b/bsdkm/include.am
@@ -2,8 +2,10 @@
# included from Top Level Makefile.am
# All paths should be given relative to the root
-EXTRA_DIST += m4/ax_bsdkm.m4 \
- bsdkm/Makefile \
- bsdkm/README.md \
- bsdkm/wolfkmod.c \
+EXTRA_DIST += m4/ax_bsdkm.m4 \
+ bsdkm/Makefile \
+ bsdkm/README.md \
+ bsdkm/wolfkmod.c \
+ bsdkm/wolfkmod_aes.c \
+ bsdkm/x86_vecreg.c \
bsdkm/bsdkm_wc_port.h
diff --git a/bsdkm/wolfkmod.c b/bsdkm/wolfkmod.c
index bb03e8ebbc3..f477954efe6 100644
--- a/bsdkm/wolfkmod.c
+++ b/bsdkm/wolfkmod.c
@@ -26,6 +26,12 @@
#include
#include
+#if defined(BSDKM_CRYPTO_REGISTER)
+ #include
+ #include
+ #include "cryptodev_if.h"
+#endif
+
/* wolf includes */
#include
#ifdef WOLFCRYPT_ONLY
@@ -44,15 +50,42 @@
#if !defined(NO_CRYPT_TEST)
#include
#endif
+#if defined(WOLFSSL_KERNEL_BENCHMARKS)
+ #include
+#endif
#include
MALLOC_DEFINE(M_WOLFSSL, "libwolfssl", "wolfSSL kernel memory");
-static int wolfkmod_init(void);
-static int wolfkmod_cleanup(void);
-static int wolfkmod_load(void);
-static int wolfkmod_unload(void);
+#if defined(BSDKM_CRYPTO_REGISTER)
+ #include "bsdkm/wolfkmod_aes.c"
+#endif
+
+/* common functions. */
+static int wolfkmod_init(void);
+static int wolfkmod_cleanup(void);
+#if !defined(BSDKM_CRYPTO_REGISTER)
+/* functions specific to a pure kernel module library build. */
+static int wolfkmod_load(void);
+static int wolfkmod_unload(void);
+#else
+/* functions specific to a kernel crypto driver module build. */
+static void wolfkdriv_identify(driver_t * driver, device_t parent);
+static int wolfkdriv_probe(device_t dev);
+static int wolfkdriv_attach(device_t dev);
+static int wolfkdriv_detach(device_t dev);
+static int wolfkdriv_probesession(device_t dev,
+ const struct crypto_session_params *csp);
+static int wolfkdriv_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp);
+static void wolfkdriv_freesession(device_t dev, crypto_session_t cses);
+static int wolfkdriv_process(device_t dev, struct cryptop *crp, int hint);
+#endif /* !BSDKM_CRYPTO_REGISTER */
+
+#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
+ #include "bsdkm/x86_vecreg.c"
+#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/
#ifdef HAVE_FIPS
#define WOLFKMOD_FIPS_ERR_MSG(hash) ({ \
@@ -82,6 +115,14 @@ static int wolfkmod_init(void)
{
int error = 0;
+ #if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
+ error = wolfkmod_vecreg_init();
+ if (error != 0) {
+ printf("error: wolfkmod_vecreg_init: %d\n", error);
+ return (ECANCELED);
+ }
+ #endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/
+
#ifdef HAVE_FIPS
error = wolfCrypt_SetCb_fips(wolfkmod_fips_cb);
if (error != 0) {
@@ -174,14 +215,16 @@ static int wolfkmod_cleanup(void)
if (error != 0) {
printf("error: wolfCrypt_Cleanup failed: %s\n",
wc_GetErrorString(error));
- return (ECANCELED);
+ error = ECANCELED;
+ goto wolfkmod_cleanup_out;
}
#else
error = wolfSSL_Cleanup();
if (error != WOLFSSL_SUCCESS) {
printf("error: wolfSSL_Cleanup failed: %s\n",
wc_GetErrorString(error));
- return (ECANCELED);
+ error = ECANCELED;
+ goto wolfkmod_cleanup_out;
}
#endif /* WOLFCRYPT_ONLY */
@@ -189,10 +232,17 @@ static int wolfkmod_cleanup(void)
printf("info: libwolfssl " LIBWOLFSSL_VERSION_STRING
" cleanup complete.\n");
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+ error = 0;
- return (0);
+wolfkmod_cleanup_out:
+ #if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
+ wolfkmod_vecreg_exit();
+ #endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/
+
+ return (error);
}
+#if !defined(BSDKM_CRYPTO_REGISTER)
static int wolfkmod_load(void)
{
int error = 0;
@@ -212,10 +262,15 @@ static int wolfkmod_load(void)
printf("info: wolfCrypt self-test passed.\n");
#endif /* NO_CRYPT_TEST */
- /**
- * todo: register wolfcrypt algs here with crypto_get_driverid
- * and related.
- * */
+ #ifdef WOLFSSL_KERNEL_BENCHMARKS
+ error = benchmark_test(NULL);
+ if (error != 0) {
+ printf("error: wolfcrypt benchmark failed: %d\n", error);
+ (void)wolfkmod_cleanup();
+ return (ECANCELED);
+ }
+ printf("info: wolfCrypt benchmark passed.\n");
+ #endif /* WOLFSSL_KERNEL_BENCHMARKS */
printf("info: libwolfssl loaded\n");
@@ -239,11 +294,6 @@ static int wolfkmod_unload(void)
error = wolfkmod_cleanup();
- /**
- * todo: unregister wolfcrypt algs here with crypto_unregister_all
- * and related.
- * */
-
if (error == 0) {
printf("info: libwolfssl unloaded\n");
}
@@ -294,7 +344,721 @@ wolfkmod_event(struct module * m, int what, void * arg)
return (error);
}
+#endif /* !BSDKM_CRYPTO_REGISTER */
+
+#if defined(BSDKM_CRYPTO_REGISTER)
+/* wolfkdriv device driver software context. */
+struct wolfkdriv_softc {
+ int32_t crid;
+ device_t dev;
+};
+
+struct km_aes_ctx {
+ Aes aes_encrypt;
+ Aes aes_decrypt;
+};
+
+typedef struct km_aes_ctx km_aes_ctx;
+
+struct wolfkdriv_session {
+ km_aes_ctx aes_ctx;
+ int32_t crid;
+ int type;
+ int ivlen;
+ int klen;
+};
+
+typedef struct wolfkdriv_session wolfkdriv_session_t;
+
+static void km_AesFree(Aes * aes) {
+ if (aes == NULL) {
+ return;
+ }
+ wc_AesFree(aes);
+ #if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0)
+ ForceZero(aes, sizeof(*aes));
+ #endif
+}
+
+static void wolfkdriv_aes_ctx_clear(km_aes_ctx * ctx)
+{
+ if (ctx != NULL) {
+ km_AesFree(&ctx->aes_encrypt);
+ km_AesFree(&ctx->aes_decrypt);
+ }
+
+ #ifdef WOLFKM_DEBUG_AES
+ printf("info: exiting km_AesExitCommon\n");
+ #endif /* WOLFKM_DEBUG_AES */
+}
+
+static void wolfkdriv_identify(driver_t * driver, device_t parent)
+{
+ (void)driver;
+
+ /* don't double add wolfkdriv child. */
+ if (device_find_child(parent, "libwolf", -1) != NULL) {
+ return;
+ }
+
+ BUS_ADD_CHILD(parent, 10, "libwolf", -1);
+}
+
+static int wolfkdriv_probe(device_t dev)
+{
+ device_set_desc(dev, "wolfSSL crypto");
+ return (BUS_PROBE_DEFAULT);
+}
+
+/*
+ * unregister libwolfssl crypto driver
+ */
+static void wolfkdriv_unregister(struct wolfkdriv_softc * softc)
+{
+ if (softc && softc->crid >= 0) {
+ crypto_unregister_all(softc->crid);
+ device_printf(softc->dev, "info: crid unregistered: %d\n", softc->crid);
+ softc->crid = -1;
+ }
+
+ return;
+}
+
+static int wolfkdriv_attach(device_t dev)
+{
+ struct wolfkdriv_softc * softc = NULL;
+ int flags = CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
+ CRYPTOCAP_F_ACCEL_SOFTWARE | CRYPTOCAP_F_HARDWARE;
+ int ret = 0;
+ int crid = 0;
+ int error = 0;
+
+ ret = wolfkmod_init();
+ if (ret != 0) {
+ return (ECANCELED);
+ }
+
+ /**
+ * register wolfcrypt algs here with crypto_get_driverid.
+ *
+ * The crid is the literal index into the kernel crypto_drivers array:
+ * - crid >= 0 is valid.
+ * - crid < 0 is error.
+ * */
+ softc = device_get_softc(dev);
+ softc->dev = dev;
+
+ softc->crid = crypto_get_driverid(dev, sizeof(wolfkdriv_session_t), flags);
+ if (softc->crid < 0) {
+ device_printf(dev, "error: crypto_get_driverid failed: %d\n",
+ softc->crid);
+ return (ENXIO);
+ }
+
+ /*
+ * various sanity checks
+ */
+
+ /* 1. we should find ourself by name */
+ crid = crypto_find_driver("libwolf");
+
+ if (crid != softc->crid) {
+ device_printf(dev, "error: attach: got crid %d, expected %d\n", crid,
+ softc->crid);
+ error = ENXIO;
+ goto attach_out;
+ }
+
+ /* 2. test various algs */
+ error = wolfkdriv_test_aes(dev, crid);
+
+ if (error) {
+ device_printf(dev, "error: attach: test_aes: %d\n", error);
+ error = ENXIO;
+ goto attach_out;
+ }
+
+ device_printf(dev, "info: driver loaded: %d\n", crid);
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: exiting attach\n");
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+attach_out:
+ if (error) {
+ wolfkdriv_unregister(softc);
+ error = ENXIO;
+ }
+
+ return (error);
+}
+
+static int wolfkdriv_detach(device_t dev)
+{
+ struct wolfkdriv_softc * softc = NULL;
+ int ret = 0;
+
+ ret = wolfkmod_cleanup();
+
+ if (ret == 0) {
+ /* unregister wolfcrypt algs */
+ softc = device_get_softc(dev);
+ wolfkdriv_unregister(softc);
+ }
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: exiting detach\n");
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+ return (0);
+}
+
+static int wolfkdriv_probesession(device_t dev,
+ const struct crypto_session_params *csp)
+{
+ struct wolfkdriv_softc * softc = NULL;
+ int error = CRYPTODEV_PROBE_ACCEL_SOFTWARE;
+
+ softc = device_get_softc(dev);
+
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ break;
+ default:
+ error = EINVAL;
+ break;
+ }
+ break;
+
+ case CSP_MODE_AEAD:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ break;
+ default:
+ error = EINVAL;
+ break;
+ }
+ break;
+ case CSP_MODE_DIGEST:
+ case CSP_MODE_ETA:
+ default:
+ error = EINVAL;
+ break;
+ }
+
+ (void)softc;
+ (void)csp;
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: probesession: mode=%d, cipher_alg=%d, error=%d\n",
+ csp->csp_mode, csp->csp_cipher_alg, error);
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+ return (error);
+}
+
+static int wolfkdriv_newsession_aes(device_t dev,
+ wolfkdriv_session_t * session,
+ const struct crypto_session_params *csp)
+{
+ int error = 0;
+ int klen = csp->csp_cipher_klen; /* key len in bytes */
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ session->type = CRYPTO_AES_NIST_GCM_16;
+ break;
+ case CRYPTO_AES_CBC:
+ session->type = CRYPTO_AES_CBC;
+ break;
+ default:
+ return (EOPNOTSUPP);
+ }
+
+ if (klen != 16 && klen != 24 && klen != 32) {
+ device_printf(dev, "info: newsession_cipher: invalid klen: %d\n", klen);
+ return (EINVAL);
+ }
+
+ session->klen = klen;
+ session->ivlen = csp->csp_ivlen;
+
+ /* encrypt */
+ error = wc_AesInit(&session->aes_ctx.aes_encrypt, NULL, INVALID_DEVID);
+ if (error) {
+ device_printf(dev, "error: newsession_cipher: aes init: %d\n", error);
+ goto newsession_cipher_out;
+ }
+
+ if (session->type == CRYPTO_AES_CBC) {
+ /* Need a separate decrypt structure for aes-cbc. */
+ error = wc_AesInit(&session->aes_ctx.aes_decrypt, NULL, INVALID_DEVID);
+ if (error) {
+ device_printf(dev, "error: newsession_cipher: aes init: %d\n",
+ error);
+ goto newsession_cipher_out;
+ }
+ }
+
+newsession_cipher_out:
+
+ if (error != 0) {
+ wolfkdriv_aes_ctx_clear(&session->aes_ctx);
+ return (EINVAL);
+ }
+
+ return (error);
+}
+
+static int wolfkdriv_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ wolfkdriv_session_t * session = NULL;
+ int error = 0;
+
+ /* get the wolfkdriv_session_t context */
+ session = crypto_get_driver_session(cses);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ case CSP_MODE_ETA:
+ device_printf(dev, "info: not supported: %d\n", csp->csp_mode);
+ error = EOPNOTSUPP;
+ break;
+ case CSP_MODE_CIPHER:
+ case CSP_MODE_AEAD:
+ error = wolfkdriv_newsession_aes(dev, session, csp);
+ break;
+ default:
+ __assert_unreachable();
+ }
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: newsession: mode=%d, cipher_alg=%d, error=%d\n",
+ csp->csp_mode, csp->csp_cipher_alg, error);
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+ return (error);
+}
+
+static void
+wolfkdriv_freesession(device_t dev, crypto_session_t cses)
+{
+ wolfkdriv_session_t * session = NULL;
+ (void)dev;
+
+ /* get the wolfkdriv_session_t context */
+ session = crypto_get_driver_session(cses);
+
+ /* clean it up */
+ wolfkdriv_aes_ctx_clear(&session->aes_ctx);
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: exiting freesession\n");
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+ return;
+}
+
+static int wolfkdriv_cbc_work(device_t dev, wolfkdriv_session_t * session,
+ struct cryptop * crp,
+ const struct crypto_session_params * csp)
+{
+ struct crypto_buffer_cursor cc_in;
+ struct crypto_buffer_cursor cc_out;
+ const unsigned char * in_block = NULL;
+ const unsigned char * in_seg = NULL;
+ unsigned char * out_block = NULL;
+ unsigned char * out_seg = NULL;
+ Aes aes;
+ uint8_t iv[WC_AES_BLOCK_SIZE];
+ uint8_t block[EALG_MAX_BLOCK_LEN];
+ size_t data_len = 0;
+ size_t seg_len = 0;
+ size_t in_len = 0;
+ size_t out_len = 0;
+ int error = 0;
+ int is_encrypt = 0;
+ int type = AES_ENCRYPTION;
+
+ if (csp->csp_cipher_alg != CRYPTO_AES_CBC) {
+ error = EINVAL;
+ goto cbc_work_out;
+ }
+
+ data_len = crp->crp_payload_length;
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ is_encrypt = 1;
+ type = AES_ENCRYPTION;
+ memcpy(&aes, &session->aes_ctx.aes_encrypt, sizeof(aes));
+ }
+ else {
+ is_encrypt = 0;
+ type = AES_DECRYPTION;
+ memcpy(&aes, &session->aes_ctx.aes_decrypt, sizeof(aes));
+ }
+
+ /* must be multiple of block size */
+ if (data_len % WC_AES_BLOCK_SIZE) {
+ error = EINVAL;
+ goto cbc_work_out;
+ }
+
+ crypto_read_iv(crp, iv);
+ error = wc_AesSetKey(&aes, csp->csp_cipher_key,
+ csp->csp_cipher_klen, iv, type);
+ if (error) {
+ device_printf(dev, "error: wc_AesSetKey: %d\n", error);
+ goto cbc_work_out;
+ }
+
+ /* set up the crypto buffers */
+ crypto_cursor_init(&cc_in, &crp->crp_buf);
+ crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+
+ in_seg = crypto_cursor_segment(&cc_in, &in_len);
+
+ /* handle if the user supplied a separate out buffer. */
+ if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+ crypto_cursor_init(&cc_out, &crp->crp_obuf);
+ crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
+ }
+ else {
+ cc_out = cc_in;
+ }
+
+ out_seg = crypto_cursor_segment(&cc_out, &out_len);
+
+ while (data_len) {
+ /* set up input buffers */
+ if (in_len < WC_AES_BLOCK_SIZE) {
+ /* less than a block in segment */
+ crypto_cursor_copydata(&cc_in, WC_AES_BLOCK_SIZE, block);
+ in_block = block;
+ in_len = WC_AES_BLOCK_SIZE;
+ }
+ else {
+ in_block = in_seg;
+ }
+
+ /* set up output buffers */
+ if (out_len < WC_AES_BLOCK_SIZE) {
+ out_block = block;
+ out_len = WC_AES_BLOCK_SIZE;
+ }
+ else {
+ out_block = out_seg;
+ }
+
+ /* choose which of data_len, in_len, out_len, is shorter.
+ * round down to multiple of aes block size. */
+ seg_len = rounddown(MIN(data_len, MIN(in_len, out_len)),
+ WC_AES_BLOCK_SIZE);
+
+ if (is_encrypt) {
+ error = wc_AesCbcEncrypt(&aes, out_block, in_block, seg_len);
+ if (error) {
+ device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error);
+ goto cbc_work_out;
+ }
+ }
+ else {
+ error = wc_AesCbcDecrypt(&aes, out_block, in_block, seg_len);
+ if (error) {
+ device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error);
+ goto cbc_work_out;
+ }
+ }
+
+ if (out_block == block) {
+ /* we used the block as local output buffer. copy to cc_out,
+ * and grab the next out cursor segment. */
+ crypto_cursor_copyback(&cc_out, WC_AES_BLOCK_SIZE, block);
+ out_seg = crypto_cursor_segment(&cc_out, &out_len);
+ } else {
+ /* we worked directly in cc_out. advance the cursor. */
+ crypto_cursor_advance(&cc_out, seg_len);
+ out_seg += seg_len;
+ out_len -= seg_len;
+ }
+
+ if (in_block == block) {
+ /* grab a new in cursor segment. */
+ in_seg = crypto_cursor_segment(&cc_in, &in_len);
+ } else {
+ /* else advance existing in cursor. */
+ crypto_cursor_advance(&cc_in, seg_len);
+ in_seg += seg_len;
+ in_len -= seg_len;
+ }
+
+ data_len -= seg_len;
+ }
+
+cbc_work_out:
+ /* cleanup. */
+ wc_ForceZero(iv, sizeof(iv));
+ wc_ForceZero(block, sizeof(block));
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: cbc_work: mode=%d, cipher_alg=%d, "
+ "payload_length=%d, error=%d\n",
+ csp->csp_mode, csp->csp_cipher_alg, crp->crp_payload_length,
+ error);
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+ return (error);
+}
+
+static int wolfkdriv_gcm_work(device_t dev, wolfkdriv_session_t * session,
+ struct cryptop * crp,
+ const struct crypto_session_params * csp)
+{
+ struct crypto_buffer_cursor cc_in;
+ struct crypto_buffer_cursor cc_out;
+ const unsigned char * in_seg = NULL;
+ unsigned char * out_seg = NULL;
+ Aes aes;
+ uint8_t iv[WC_AES_BLOCK_SIZE];
+ uint8_t tag[WC_AES_BLOCK_SIZE];
+ size_t data_len = 0;
+ size_t seg_len = 0;
+ size_t in_len = 0;
+ size_t out_len = 0;
+ int error = 0;
+ int is_encrypt = 0;
+
+ memcpy(&aes, &session->aes_ctx.aes_encrypt, sizeof(aes));
+
+ if (csp->csp_cipher_alg != CRYPTO_AES_NIST_GCM_16) {
+ error = EINVAL;
+ goto gcm_work_out;
+ }
+
+ data_len = crp->crp_payload_length;
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ is_encrypt = 1;
+ }
+ else {
+ is_encrypt = 0;
+ }
+
+ error = wc_AesGcmSetKey(&aes, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+ if (error) {
+ device_printf(dev, "error: wc_AesGcmSetKey: %d\n", error);
+ goto gcm_work_out;
+ }
+
+ crypto_read_iv(crp, iv);
+ error = wc_AesGcmInit(&aes, NULL /* key */, 0 /* keylen */,
+ iv, csp->csp_ivlen);
+ if (error) {
+ device_printf(dev, "error: wc_AesGcmInit: %d\n", error);
+ goto gcm_work_out;
+ }
+
+ /*
+ * process aad first
+ */
+ if (crp->crp_aad != NULL) {
+ /* they passed it in separate buffer. */
+ if (is_encrypt) {
+ error = wc_AesGcmEncryptUpdate(&aes, NULL, NULL, 0,
+ crp->crp_aad, crp->crp_aad_length);
+ }
+ else {
+ error = wc_AesGcmDecryptUpdate(&aes, NULL, NULL, 0,
+ crp->crp_aad, crp->crp_aad_length);
+ }
+
+ if (error) {
+ error = EINVAL;
+ }
+ }
+ else {
+ /* we need to pull aad out of crp->crp_buf from crp_aad_start. */
+ size_t aad_len = 0;
+
+ crypto_cursor_init(&cc_in, &crp->crp_buf);
+ crypto_cursor_advance(&cc_in, crp->crp_aad_start);
+
+ for (aad_len = crp->crp_aad_length; aad_len > 0; aad_len -= seg_len) {
+ in_seg = crypto_cursor_segment(&cc_in, &in_len);
+ seg_len = MIN(aad_len, in_len);
+
+ if (is_encrypt) {
+ error = wc_AesGcmEncryptUpdate(&aes, NULL, NULL, 0,
+ in_seg, seg_len);
+ }
+ else {
+ error = wc_AesGcmDecryptUpdate(&aes, NULL, NULL, 0,
+ in_seg, seg_len);
+ }
+
+ if (error) {
+ error = EINVAL;
+ goto gcm_work_out;
+ }
+
+ crypto_cursor_advance(&cc_in, seg_len);
+ }
+ }
+
+ /*
+ * process cipher/plaintext next
+ */
+
+ /* set up the crypto buffers */
+ crypto_cursor_init(&cc_in, &crp->crp_buf);
+ crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+
+ in_seg = crypto_cursor_segment(&cc_in, &in_len);
+
+ /* handle if the user supplied a separate out buffer. */
+ if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
+ crypto_cursor_init(&cc_out, &crp->crp_obuf);
+ crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
+ }
+ else {
+ cc_out = cc_in;
+ }
+
+ out_seg = crypto_cursor_segment(&cc_out, &out_len);
+
+ while (data_len) {
+ /* process through the available segments. */
+ in_seg = crypto_cursor_segment(&cc_in, &in_len);
+ out_seg = crypto_cursor_segment(&cc_out, &out_len);
+ seg_len = MIN(data_len, MIN(in_len, out_len));
+
+ if (is_encrypt) {
+ error = wc_AesGcmEncryptUpdate(&aes, out_seg, in_seg, seg_len,
+ NULL, 0);
+ if (error) {
+ device_printf(dev, "error: wc_AesGcmEncrypt: %d\n", error);
+ goto gcm_work_out;
+ }
+ }
+ else {
+ error = wc_AesGcmDecryptUpdate(&aes, out_seg, in_seg, seg_len,
+ NULL, 0);
+ if (error) {
+ device_printf(dev, "error: wc_AesGcmEncrypt: %d\n", error);
+ goto gcm_work_out;
+ }
+ }
+
+ /* advance the cursors by amount processed */
+ crypto_cursor_advance(&cc_in, seg_len);
+ crypto_cursor_advance(&cc_out, seg_len);
+
+ data_len -= seg_len;
+ }
+
+ /*
+ * process auth tag finally
+ */
+
+ if (is_encrypt) {
+ error = wc_AesGcmEncryptFinal(&aes, tag, WC_AES_BLOCK_SIZE);
+ if (error == 0) {
+ crypto_copyback(crp, crp->crp_digest_start, WC_AES_BLOCK_SIZE, tag);
+ }
+ }
+ else {
+ crypto_copydata(crp, crp->crp_digest_start, WC_AES_BLOCK_SIZE, tag);
+ error = wc_AesGcmDecryptFinal(&aes, tag, WC_AES_BLOCK_SIZE);
+ if (error) {
+ error = EBADMSG;
+ }
+ }
+
+gcm_work_out:
+ /* cleanup. */
+ wc_ForceZero(iv, sizeof(iv));
+ wc_ForceZero(tag, sizeof(tag));
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: gcm_work: mode=%d, cipher_alg=%d, "
+ "payload_length=%d, error=%d\n",
+ csp->csp_mode, csp->csp_cipher_alg, crp->crp_payload_length,
+ error);
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+ return (error);
+}
+
+static int wolfkdriv_process(device_t dev, struct cryptop * crp, int hint)
+{
+ const struct crypto_session_params * csp = NULL;
+ wolfkdriv_session_t * session = NULL;
+ int error = 0;
+ (void)hint;
+
+ session = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
+
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ error = wolfkdriv_cbc_work(dev, session, crp, csp);
+ break;
+ case CSP_MODE_DIGEST:
+ case CSP_MODE_ETA:
+ error = EINVAL;
+ break;
+ case CSP_MODE_AEAD:
+ error = wolfkdriv_gcm_work(dev, session, crp, csp);
+ break;
+ default:
+ __assert_unreachable();
+ }
+
+ crp->crp_etype = error;
+ crypto_done(crp);
+
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: process: mode=%d, cipher_alg=%d, error=%d\n",
+ csp->csp_mode, csp->csp_cipher_alg, error);
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+ return (error);
+}
+
+/*
+ * wolfkmod as a crypto device driver.
+ */
+static device_method_t wolfkdriv_methods[] = {
+ /* device interface methods: called during device setup, etc. */
+ DEVMETHOD(device_identify, wolfkdriv_identify),
+ DEVMETHOD(device_probe, wolfkdriv_probe),
+ DEVMETHOD(device_attach, wolfkdriv_attach),
+ DEVMETHOD(device_detach, wolfkdriv_detach),
+
+ /* crypto device session methods: called during crypto session setup,
+ * work, etc. */
+ DEVMETHOD(cryptodev_probesession, wolfkdriv_probesession),
+ DEVMETHOD(cryptodev_newsession, wolfkdriv_newsession),
+ DEVMETHOD(cryptodev_freesession, wolfkdriv_freesession),
+ DEVMETHOD(cryptodev_process, wolfkdriv_process),
+
+ DEVMETHOD_END
+};
+
+static driver_t wolfkdriv_driver = {
+ .name = "libwolf",
+ .methods = wolfkdriv_methods,
+ .size = sizeof(struct wolfkdriv_softc),
+};
+
+/* on x86, software-only drivers usually attach to nexus bus. */
+DRIVER_MODULE(libwolfssl, nexus, wolfkdriv_driver, NULL, NULL);
+#endif /* BSDKM_CRYPTO_REGISTER */
+
+#if !defined(BSDKM_CRYPTO_REGISTER)
+/*
+ * wolfkmod as a pure kernel module.
+ */
static moduledata_t libwolfmod = {
#ifdef HAVE_FIPS
"libwolfssl_fips", /* module name */
@@ -305,6 +1069,8 @@ static moduledata_t libwolfmod = {
NULL /* extra data, unused */
};
-MODULE_VERSION(libwolfssl, 1);
DECLARE_MODULE(libwolfssl, libwolfmod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
+#endif /* !BSDKM_CRYPTO_REGISTER */
+
+MODULE_VERSION(libwolfssl, 1);
#endif /* WOLFSSL_BSDKM */
diff --git a/bsdkm/wolfkmod_aes.c b/bsdkm/wolfkmod_aes.c
new file mode 100644
index 00000000000..5cbe2871e14
--- /dev/null
+++ b/bsdkm/wolfkmod_aes.c
@@ -0,0 +1,387 @@
+#if !defined(WC_SKIP_INCLUDED_C_FILES) && defined(BSDKM_CRYPTO_REGISTER)
+#include
+
+#if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG)
+static void
+wolfkmod_print_data(const char * what, const uint8_t * data, size_t data_len)
+{
+ size_t i = 0;
+
+ printf("%s:\n", what);
+ for (i = 0; i < data_len; ++i) {
+ printf("0x%02x, ", data[i]);
+ if ((i + 1) % 8 == 0) {
+ printf("\n");
+ }
+ }
+
+ printf("\n");
+}
+#endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */
+
+/*
+ * cryptodev framework always calls a callback, even when CRYPTOCAP_F_SYNC.
+ */
+static int
+wolfkdriv_test_crp_callback(struct cryptop * crp)
+{
+ (void)crp;
+ return (0);
+}
+
+/* Test aes-cbc with a buffer larger than aes block size.
+ * Verify direct wolfcrypt API and opencrypto framework return
+ * same result. */
+static int wolfkdriv_test_aes_cbc_big(device_t dev, int crid)
+{
+ crypto_session_t session = NULL;
+ struct crypto_session_params csp;
+ struct cryptop * crp = NULL;
+ Aes * aes_encrypt = NULL;
+ int error = 0;
+ byte msg[] = {
+ 0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
+ 0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20,
+ 0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
+ 0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20,
+ 0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
+ 0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20
+ };
+ byte work1[WC_AES_BLOCK_SIZE * 3]; /* wolfcrypt buffer */
+ byte work2[WC_AES_BLOCK_SIZE * 3]; /* opencrypto buffer */
+ /* padded to 16-bytes */
+ const byte key[] = "0123456789abcdef ";
+ /* padded to 16-bytes */
+ const byte iv[] = "1234567890abcdef ";
+
+ memset(&csp, 0, sizeof(csp));
+ memcpy(work1, msg, sizeof(msg)); /* wolfcrypt work buffer */
+ memcpy(work2, msg, sizeof(msg)); /* opencrypto work buffer */
+
+ /* wolfcrypt encrypt */
+ aes_encrypt = (Aes *)XMALLOC(sizeof(Aes), NULL, DYNAMIC_TYPE_AES);
+ if (aes_encrypt == NULL) {
+ error = ENOMEM;
+ device_printf(dev, "error: malloc failed\n");
+ goto test_aes_cbc_big_out;
+ }
+
+ error = wc_AesInit(aes_encrypt, NULL, INVALID_DEVID);
+ if (error) {
+ device_printf(dev, "error: newsession_cipher: aes init: %d\n", error);
+ goto test_aes_cbc_big_out;
+ }
+
+ error = wc_AesSetKey(aes_encrypt, key, 16, iv, AES_ENCRYPTION);
+ if (error) {
+ device_printf(dev, "error: wc_AesSetKey: %d\n", error);
+ goto test_aes_cbc_big_out;
+ }
+
+ error = wc_AesCbcEncrypt(aes_encrypt, work1, work1, sizeof(work1));
+ if (error) {
+ device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error);
+ goto test_aes_cbc_big_out;
+ }
+
+ /* opencrypto encrypt */
+ csp.csp_mode = CSP_MODE_CIPHER;
+ csp.csp_cipher_alg = CRYPTO_AES_CBC;
+ csp.csp_ivlen = WC_AES_BLOCK_SIZE;
+ csp.csp_cipher_key = key;
+ csp.csp_cipher_klen = WC_AES_BLOCK_SIZE;
+ error = crypto_newsession(&session, &csp, crid);
+ if (error || session == NULL) {
+ goto test_aes_cbc_big_out;
+ }
+
+ crp = crypto_getreq(session, M_WAITOK);
+ if (crp == NULL) {
+ device_printf(dev, "error: test_aes: crypto_getreq failed\n");
+ goto test_aes_cbc_big_out;
+ }
+
+ crp->crp_callback = wolfkdriv_test_crp_callback;
+ crp->crp_op = CRYPTO_OP_ENCRYPT;
+ crp->crp_flags = CRYPTO_F_IV_SEPARATE;
+
+ memcpy(crp->crp_iv, iv, WC_AES_BLOCK_SIZE);
+
+ crypto_use_buf(crp, work2, sizeof(work2));
+ crp->crp_payload_start = 0;
+ crp->crp_payload_length = sizeof(work2);
+
+ error = crypto_dispatch(crp);
+ if (error) {
+ goto test_aes_cbc_big_out;
+ }
+
+ error = XMEMCMP(work1, work2, sizeof(work2));
+ if (error) {
+ device_printf(dev, "error: test_aes: enc vectors diff: %d\n", error);
+ goto test_aes_cbc_big_out;
+ }
+
+ #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG)
+ wolfkmod_print_data("msg_enc", work2, sizeof(work2));
+ #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */
+
+ /* opencrypto decrypt */
+ crp->crp_op = CRYPTO_OP_DECRYPT;
+
+ error = crypto_dispatch(crp);
+ if (error) {
+ goto test_aes_cbc_big_out;
+ }
+
+ error = XMEMCMP(work2, msg, sizeof(msg));
+ if (error) {
+ device_printf(dev, "error: test_aes: dec vectors diff: %d\n", error);
+ goto test_aes_cbc_big_out;
+ }
+
+ #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG)
+ wolfkmod_print_data("msg_dec", work2, sizeof(work2));
+ #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */
+
+test_aes_cbc_big_out:
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: test_aes_cbc_big: error=%d, session=%p, crp=%p\n",
+ error, (void *)session, (void*)crp);
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+ if (crp != NULL) {
+ crypto_freereq(crp);
+ crp = NULL;
+ }
+
+ if (session != NULL) {
+ crypto_freesession(session);
+ session = NULL;
+ }
+
+ if (aes_encrypt != NULL) {
+ wc_AesFree(aes_encrypt);
+ XFREE(aes_encrypt, NULL, DYNAMIC_TYPE_AES);
+ aes_encrypt = NULL;
+ }
+
+ return (error);
+}
+
+/* Test aes-gcm encrypt and decrypt a small buffer with opencrypto
+ * framework and wolfcrypt.
+ */
+static int wolfkdriv_test_aes_gcm(device_t dev, int crid)
+{
+ crypto_session_t session = NULL;
+ struct crypto_session_params csp;
+ struct cryptop * crp = NULL;
+ Aes * enc = NULL;
+ int error = 0;
+
+ WOLFSSL_SMALL_STACK_STATIC const byte p[] =
+ {
+ 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5,
+ 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a,
+ 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda,
+ 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72,
+ 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53,
+ 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25,
+ 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57,
+ 0xba, 0x63, 0x7b, 0x39
+ };
+
+ WOLFSSL_SMALL_STACK_STATIC const byte c1[] =
+ {
+ 0x52, 0x2d, 0xc1, 0xf0, 0x99, 0x56, 0x7d, 0x07,
+ 0xf4, 0x7f, 0x37, 0xa3, 0x2a, 0x84, 0x42, 0x7d,
+ 0x64, 0x3a, 0x8c, 0xdc, 0xbf, 0xe5, 0xc0, 0xc9,
+ 0x75, 0x98, 0xa2, 0xbd, 0x25, 0x55, 0xd1, 0xaa,
+ 0x8c, 0xb0, 0x8e, 0x48, 0x59, 0x0d, 0xbb, 0x3d,
+ 0xa7, 0xb0, 0x8b, 0x10, 0x56, 0x82, 0x88, 0x38,
+ 0xc5, 0xf6, 0x1e, 0x63, 0x93, 0xba, 0x7a, 0x0a,
+ 0xbc, 0xc9, 0xf6, 0x62
+ };
+
+ WOLFSSL_SMALL_STACK_STATIC byte a[] =
+ {
+ 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef,
+ 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef,
+ 0xab, 0xad, 0xda, 0xd2
+ };
+
+ WOLFSSL_SMALL_STACK_STATIC const byte k1[] =
+ {
+ 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c,
+ 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08,
+ 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c,
+ 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08
+ };
+
+ WOLFSSL_SMALL_STACK_STATIC const byte iv1[] =
+ {
+ 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad,
+ 0xde, 0xca, 0xf8, 0x88
+ };
+
+ WOLFSSL_SMALL_STACK_STATIC const byte t1[] =
+ {
+ 0x76, 0xfc, 0x6e, 0xce, 0x0f, 0x4e, 0x17, 0x68,
+ 0xcd, 0xdf, 0x88, 0x53, 0xbb, 0x2d, 0x55, 0x1b
+ };
+
+ byte resultT[sizeof(t1) + WC_AES_BLOCK_SIZE];
+ byte resultC[sizeof(p) + WC_AES_BLOCK_SIZE];
+ byte resultC2[sizeof(p) + WC_AES_BLOCK_SIZE];
+
+ XMEMSET(resultT, 0, sizeof(resultT));
+ XMEMSET(resultC, 0, sizeof(resultC));
+
+ XMEMSET(resultC2, 0, sizeof(resultC));
+ XMEMCPY(resultC2, p, sizeof(p));
+
+ /* wolfcrypt encrypt */
+ enc = (Aes *)XMALLOC(sizeof(Aes), NULL, DYNAMIC_TYPE_AES);
+ if (enc == NULL) {
+ error = ENOMEM;
+ device_printf(dev, "error: malloc failed\n");
+ goto test_aes_gcm_out;
+ }
+
+ error = wc_AesGcmEncryptInit(enc, k1, sizeof(k1), iv1, sizeof(iv1));
+ if (error) { goto test_aes_gcm_out; }
+
+ error = wc_AesGcmEncryptUpdate(enc, resultC, p, sizeof(p), a, sizeof(a));
+ if (error) { goto test_aes_gcm_out; }
+
+ error = wc_AesGcmEncryptFinal(enc, resultT, sizeof(t1));
+ if (error) { goto test_aes_gcm_out; }
+
+ #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG)
+ wolfkmod_print_data("resultC", resultC, sizeof(p));
+ wolfkmod_print_data("resultT", resultT, sizeof(t1));
+ #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */
+
+ error = XMEMCMP(resultC, c1, sizeof(c1));
+ if (error) { goto test_aes_gcm_out; }
+
+ error = XMEMCMP(resultT, t1, sizeof(t1));
+ if (error) { goto test_aes_gcm_out; }
+
+ /*
+ * opencrypto encrypt
+ * */
+
+ /* set crypto session params */
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_flags |= CSP_F_SEPARATE_AAD;
+ csp.csp_mode = CSP_MODE_AEAD;
+ csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
+ csp.csp_ivlen = sizeof(iv1);
+ csp.csp_cipher_key = k1;
+ csp.csp_cipher_klen = sizeof(k1);
+
+ /* get crypto session handle */
+ error = crypto_newsession(&session, &csp, crid);
+ if (error || session == NULL) {
+ device_printf(dev, "error: test_aes: crypto_newsession: %d, %p\n",
+ error, (void *)session);
+ goto test_aes_gcm_out;
+ }
+
+ /* get a crypto op handle */
+ crp = crypto_getreq(session, M_WAITOK);
+ if (crp == NULL) {
+ device_printf(dev, "error: test_aes: crypto_getreq failed\n");
+ goto test_aes_gcm_out;
+ }
+
+ /* configure it */
+ crp->crp_callback = wolfkdriv_test_crp_callback;
+ crp->crp_op = (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST);
+ crp->crp_flags = CRYPTO_F_IV_SEPARATE;
+
+ memcpy(crp->crp_iv, iv1, sizeof(iv1));
+
+ crypto_use_buf(crp, resultC2, sizeof(resultC2));
+ crp->crp_payload_start = 0;
+ crp->crp_payload_length = sizeof(p);
+
+ crp->crp_aad = a;
+ crp->crp_aad_start = 0;
+ crp->crp_aad_length = sizeof(a);
+ crp->crp_digest_start = crp->crp_payload_start + sizeof(p);
+
+ error = crypto_dispatch(crp);
+ if (error) {
+ goto test_aes_gcm_out;
+ }
+
+ #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG)
+ wolfkmod_print_data("resultC2", resultC2, sizeof(p));
+ wolfkmod_print_data("resultT2", resultC2 + sizeof(p), sizeof(t1));
+ #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */
+
+ error = XMEMCMP(resultC2, c1, sizeof(c1));
+ if (error) { goto test_aes_gcm_out; }
+
+ error = XMEMCMP(resultC2 + sizeof(p), t1, sizeof(t1));
+ if (error) { goto test_aes_gcm_out; }
+
+ /* opencrypto decrypt */
+ crp->crp_op = (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST);
+
+ error = crypto_dispatch(crp);
+ if (error) {
+ goto test_aes_gcm_out;
+ }
+
+ #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG)
+ wolfkmod_print_data("resultC2_dec", resultC2, sizeof(p));
+ #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */
+
+ error = XMEMCMP(resultC2, p, sizeof(p));
+ if (error) { goto test_aes_gcm_out; }
+
+test_aes_gcm_out:
+ #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
+ device_printf(dev, "info: test_aes_gcm: error=%d, session=%p, crp=%p\n",
+ error, (void *)session, (void*)crp);
+ #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
+
+ if (crp != NULL) {
+ crypto_freereq(crp);
+ crp = NULL;
+ }
+
+ if (session != NULL) {
+ crypto_freesession(session);
+ session = NULL;
+ }
+
+ if (enc != NULL) {
+ wc_AesFree(enc);
+ XFREE(enc, NULL, DYNAMIC_TYPE_AES);
+ enc = NULL;
+ }
+
+ return (error);
+}
+
+
+static int wolfkdriv_test_aes(device_t dev, int crid)
+{
+ int error = 0;
+
+ if (error == 0) {
+ error = wolfkdriv_test_aes_cbc_big(dev, crid);
+ }
+
+ if (error == 0) {
+ error = wolfkdriv_test_aes_gcm(dev, crid);
+ }
+
+ return (error);
+}
+#endif /* !WC_SKIP_INCLUDED_C_FILES && BSDKM_CRYPTO_REGISTER */
diff --git a/bsdkm/x86_vecreg.c b/bsdkm/x86_vecreg.c
new file mode 100644
index 00000000000..90febe360a3
--- /dev/null
+++ b/bsdkm/x86_vecreg.c
@@ -0,0 +1,225 @@
+/* x86_vecreg.c -- logic to save and restore vector registers
+ * on amd64 in FreeBSD kernel.
+ *
+ * Copyright (C) 2006-2025 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/* included by bsdkm/wolfkmod.c */
+#ifndef WC_SKIP_INCLUDED_C_FILES
+
+#include
+#include
+#include
+#include
+
+struct wolfkmod_fpu_state_t {
+ volatile lwpid_t td_tid;
+ volatile u_int nest;
+};
+
+typedef struct wolfkmod_fpu_state_t wolfkmod_fpu_state_t;
+
+/* fpu_states array tracks thread id and nesting level of save/restore
+ * and push/pop vector registers macro calls. It is indexed by raw cpu id,
+ * and only accessed after the thread calls fpu_kern_enter(), and before
+ * calling fpu_kern_leave(), and only indexed by the thread's PCPU_GET(cpuid).
+ *
+ * after calling fpu_kern_enter():
+ * - kernel fpu is enabled
+ * - migration is disabled
+ * - soft preempts are disabled
+ * Hard irq are still possible , but hard irq are forbidden from using FPU
+ * in FreeBSD kernel.
+ * */
+static wolfkmod_fpu_state_t * fpu_states = NULL;
+
+/* check for active td_tid with atomic before proceeding.
+ * technically not necessary because fpu_kern_enter() gives thread pinning
+ * to cpu, but just to be safe...
+ * */
+#define wolfkmod_fpu_get_tid() \
+ atomic_load_acq_int(&fpu_states[PCPU_GET(cpuid)].td_tid)
+
+int wolfkmod_vecreg_init(void)
+{
+ if (mp_ncpus <= 0) {
+ printf("error: wolfkmod_vecreg_init: mp_ncpus = %d\n", mp_ncpus);
+ return (EINVAL);
+ }
+
+ fpu_states = malloc(mp_ncpus * sizeof(wolfkmod_fpu_state_t),
+ M_WOLFSSL, M_WAITOK | M_ZERO);
+ if (fpu_states == NULL) {
+ printf("error: wolfkmod_vecreg_init: malloc(%lu) failed\n",
+ mp_ncpus * sizeof(wolfkmod_fpu_state_t));
+ return (ENOMEM);
+ }
+
+ return (0);
+}
+
+void wolfkmod_vecreg_exit(void)
+{
+ int i = 0;
+
+ if (fpu_states == NULL) {
+ return;
+ }
+
+ for (i = 0; i < mp_ncpus; ++i) {
+ #if defined(WOLFSSL_BSDKM_FPU_DEBUG)
+ printf("info: wolfkmod_vecreg_exit: fpu_states[%d] = %d, %d\n",
+ i, fpu_states[i].nest, fpu_states[i].td_tid);
+ #endif /* WOLFSSL_BSDKM_FPU_DEBUG */
+
+ if (fpu_states[i].nest != 0 || fpu_states[i].td_tid != 0) {
+ /* Check for orphaned fpu state. There's nothing we can do
+ * but log the event and zero the nesting level. */
+ printf("error: wolfkmod_vecreg_exit: fpu_states[%d] = %d, %d\n",
+ i, fpu_states[i].nest, fpu_states[i].td_tid);
+ fpu_states[i].nest = 0;
+ }
+ }
+
+ free(fpu_states, M_WOLFSSL);
+ fpu_states = NULL;
+
+ return;
+}
+
+/* fpu_kern_enter() and fpu_kern_leave() wrapper defines.
+ * Build with WOLFSSL_BSDKM_FPU_DEBUG to see verbose FPU logging.
+ */
+#if defined(WOLFSSL_BSDKM_FPU_DEBUG)
+ #define wolfkmod_print_curthread(what) \
+ printf("%s: cpuid = %d, curthread: td_tid = %d, pid = %d (%s), " \
+ " td_critnest = %d, kernfpu = %02x\n", \
+ (what), PCPU_GET(cpuid), curthread->td_tid, \
+ curthread->td_proc ? curthread->td_proc->p_pid : -1, \
+ curthread->td_proc ? curthread->td_proc->p_comm : "noproc", \
+ curthread->td_critnest, \
+ curthread->td_pcb->pcb_flags & PCB_KERNFPU);
+
+ #define wolfkmod_fpu_kern_enter() \
+ wolfkmod_print_curthread("fpu_kern_enter"); \
+ fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
+
+ #define wolfkmod_fpu_kern_leave() \
+ wolfkmod_print_curthread("fpu_kern_leave"); \
+ fpu_kern_leave(curthread, NULL);
+#else
+ #define wolfkmod_fpu_kern_enter() \
+ fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
+
+ #define wolfkmod_fpu_kern_leave() \
+ fpu_kern_leave(curthread, NULL);
+#endif /* WOLFSSL_BSDKM_FPU_DEBUG */
+
+int wolfkmod_vecreg_save(int flags_unused)
+{
+ (void)flags_unused;
+
+ #if defined(WOLFSSL_BSDKM_FPU_DEBUG)
+ wolfkmod_print_curthread("wolfkmod_vecreg_save");
+ #endif
+
+ if (is_fpu_kern_thread(0)) {
+ /* kernel fpu threads are special, do nothing. They own a
+ * persistent, dedicated fpu context. */
+ #if defined(WOLFSSL_BSDKM_FPU_DEBUG)
+ printf("info: wolfkmod_vecreg_save: is fpu kern thread\n");
+ #endif
+ return (0);
+ }
+
+ if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) {
+ /* kern fpu is active for this thread. check td_tid and
+ * increment nesting level. */
+ lwpid_t td_tid = wolfkmod_fpu_get_tid();
+ if (td_tid != curthread->td_tid) {
+ printf("error: wolfkmod_vecreg_save: got tid = %d, expected %d\n",
+ td_tid, curthread->td_tid);
+ return (EINVAL);
+ }
+ fpu_states[PCPU_GET(cpuid)].nest++;
+ }
+ else {
+ /* kern fpu not active for this thread, call fpu_kern_enter().
+ * after calling fpu_kern_enter():
+ * - kernel fpu is enabled
+ * - migration is disabled
+ * - soft preempts are disabled */
+ lwpid_t td_tid = 0;
+ wolfkmod_fpu_kern_enter();
+ td_tid = wolfkmod_fpu_get_tid();
+
+ if (fpu_states[PCPU_GET(cpuid)].nest != 0 || td_tid != 0) {
+ printf("error: wolfkmod_fpu_kern_enter() with nest: %d, %d\n",
+ fpu_states[PCPU_GET(cpuid)].nest, td_tid);
+ return (EINVAL);
+ }
+
+ /* increment nest and save td_tid. */
+ fpu_states[PCPU_GET(cpuid)].nest++;
+ fpu_states[PCPU_GET(cpuid)].td_tid = curthread->td_tid;
+ }
+
+ return (0);
+}
+
+void wolfkmod_vecreg_restore(void)
+{
+ #if defined(WOLFSSL_BSDKM_FPU_DEBUG)
+ wolfkmod_print_curthread("wolfkmod_vecreg_restore");
+ #endif
+
+ if (is_fpu_kern_thread(0)) {
+ /* kernel fpu threads are special, do nothing. They own a
+ * persistent, dedicated fpu context. */
+ #if defined(WOLFSSL_BSDKM_FPU_DEBUG)
+ printf("info: wolfkmod_vecreg_restore: is fpu kern thread\n");
+ #endif
+ return;
+ }
+
+ if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) {
+ /* kern fpu is active for this thread. check tid and nesting level. */
+ lwpid_t td_tid = wolfkmod_fpu_get_tid();
+ if (td_tid != curthread->td_tid) {
+ printf("error: wolfkmod_vecreg_restore: got tid = %d, "
+ "expected %d\n", td_tid, curthread->td_tid);
+ return;
+ }
+
+ /* decrement the nesting level. */
+ if (fpu_states[PCPU_GET(cpuid)].nest > 0) {
+ fpu_states[PCPU_GET(cpuid)].nest--;
+ }
+
+ /* if last level, zero the thread id then call fpu_kern_leave */
+ if (fpu_states[PCPU_GET(cpuid)].nest == 0) {
+ fpu_states[PCPU_GET(cpuid)].td_tid = 0;
+ wolfkmod_fpu_kern_leave();
+ }
+ }
+
+ return;
+}
+
+#endif /* !WC_SKIP_INCLUDED_C_FILES */
diff --git a/configure.ac b/configure.ac
index 0e987002833..012b6c232af 100644
--- a/configure.ac
+++ b/configure.ac
@@ -123,9 +123,18 @@ then
AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_EXPERIMENTAL_SETTINGS"
fi
+# Kernel module benchmark
+ENABLED_KERNEL_BENCHMARKS=""
+AC_ARG_ENABLE([kernel-benchmarks],
+ [AS_HELP_STRING([--enable-kernel-benchmarks],[Enable crypto benchmarking autorun at module load time for kernel module (default: disabled)])],
+ [ENABLED_KERNEL_BENCHMARKS=$enableval])
+if test "$ENABLED_KERNEL_BENCHMARKS" = "yes"
+then
+ AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_KERNEL_BENCHMARKS"
+fi
+AC_SUBST([ENABLED_KERNEL_BENCHMARKS])
# Linux Kernel Module options (more options later)
-
AC_ARG_ENABLE([linuxkm],
[AS_HELP_STRING([--enable-linuxkm],[Enable Linux Kernel Module (default: disabled)])],
[ENABLED_LINUXKM=$enableval],
@@ -145,6 +154,12 @@ AC_ARG_ENABLE([freebsdkm],
[ENABLED_BSDKM=no]
)
+AC_ARG_ENABLE([freebsdkm-crypto-register],
+ [AS_HELP_STRING([--enable-freebsdkm-crypto-register],[Register wolfCrypt implementations with the FreeBSD kernel opencrypto framework. (default: disabled)])],
+ [ENABLED_BSDKM_REGISTER=$enableval],
+ [ENABLED_BSDKM_REGISTER=no]
+ )
+
AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stddef.h time.h sys/ioctl.h sys/socket.h sys/time.h errno.h sys/un.h ctype.h sys/random.h])
AC_CHECK_LIB([network],[socket])
AC_C_BIGENDIAN
@@ -727,10 +742,8 @@ AC_SUBST([ENABLED_LINUXKM_PIE])
AC_ARG_ENABLE([linuxkm-benchmarks],
[AS_HELP_STRING([--enable-linuxkm-benchmarks],[Enable crypto benchmarking autorun at module load time for Linux kernel module (default: disabled)])],
- [ENABLED_KERNEL_BENCHMARKS=$enableval],
- [ENABLED_KERNEL_BENCHMARKS=no]
- )
-if test "$ENABLED_KERNEL_BENCHMARKS" = "yes"
+ [ENABLED_KERNEL_BENCHMARKS=$enableval])
+if test "$ENABLED_LINUXKM" = "yes" && test "$ENABLED_KERNEL_BENCHMARKS" = "yes"
then
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_LINUXKM_BENCHMARKS"
fi
@@ -819,17 +832,15 @@ AC_ARG_WITH([bsd-export-syms],
if test "x$ENABLED_BSDKM" = "xyes"
then
- # wolfcrypt only, no-asm supported for now.
+ # note: bsdkm is wolfcrypt only for now.
HAVE_KERNEL_MODE=yes
KERNEL_MODE_DEFAULTS=yes
ENABLED_NO_LIBRARY=yes
ENABLED_BENCHMARK=no
- ENABLED_ASM=no
output_objdir="$(realpath "$output_objdir")/bsdkm"
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_BSDKM -DWC_SIPHASH_NO_ASM"
- AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM"
AM_CFLAGS="$AM_CFLAGS -DNO_DEV_RANDOM -DNO_WRITEV -DNO_STDIO_FILESYSTEM"
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_SOCK -DWOLFSSL_USER_IO"
AM_CFLAGS="$AM_CFLAGS -DXMALLOC_OVERRIDE -DWOLFCRYPT_ONLY"
@@ -846,7 +857,16 @@ then
fi
AC_SUBST([KERNEL_ROOT])
AC_SUBST([BSDKM_EXPORT_SYMS])
+fi
+
+if test "x$ENABLED_BSDKM_REGISTER" = "xyes"
+then
+ if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$enable_aesgcm_stream" != "no" && (test "$ENABLED_FIPS" = "no" || test $HAVE_FIPS_VERSION -ge 6); then
+ ENABLED_AESGCM_STREAM=yes
+ fi
+ AM_CFLAGS="$AM_CFLAGS -DBSDKM_CRYPTO_REGISTER"
+ AC_SUBST([ENABLED_BSDKM_REGISTER])
fi
# end FreeBSD configure
@@ -3969,6 +3989,8 @@ then
ENABLED_X86_ASM=yes
fi
fi
+AC_SUBST([ENABLED_AESNI])
+AC_SUBST([ENABLED_AESNI_WITH_AVX])
AC_ARG_ENABLE([aligndata],
[AS_HELP_STRING([--enable-aligndata],[align data for ciphers (default: enabled)])],
diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c
index 3e1dcf050fd..f6e2615e99d 100644
--- a/wolfcrypt/benchmark/benchmark.c
+++ b/wolfcrypt/benchmark/benchmark.c
@@ -2683,9 +2683,9 @@ static WC_INLINE void bench_stats_start(int* count, double* start)
#endif
}
-#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
#define bench_stats_start(count, start) do { \
- SAVE_VECTOR_REGISTERS(pr_err( \
+ SAVE_VECTOR_REGISTERS(WOLFSSL_DEBUG_PRINTF( \
"ERROR: SAVE_VECTOR_REGISTERS failed for benchmark run."); \
return; ); \
bench_stats_start(count, start); \
@@ -3161,7 +3161,7 @@ static void bench_stats_sym_finish(const char* desc, int useDeviceID,
(void)useDeviceID;
(void)ret;
-#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
RESTORE_VECTOR_REGISTERS();
#elif defined(WOLFSSL_LINUXKM)
kernel_fpu_end();
@@ -3559,7 +3559,7 @@ static void bench_stats_asym_finish_ex(const char* algo, int strength,
(void)useDeviceID;
(void)ret;
-#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
+#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
RESTORE_VECTOR_REGISTERS();
#elif defined(WOLFSSL_LINUXKM)
kernel_fpu_end();
@@ -16024,6 +16024,20 @@ void bench_sphincsKeySign(byte level, byte optim)
return (double)ns / 1000000000.0;
}
+#elif defined(WOLFSSL_BSDKM)
+
+ #include
+ double current_time(int reset)
+ {
+ (void)reset;
+ struct timespec ts;
+ int64_t result = 0;
+
+ getnanouptime(&ts);
+ result = (int64_t) ts.tv_sec + (int64_t) ts.tv_nsec / NANOSECOND;
+ return (double)result;
+ }
+
#elif defined(WOLFSSL_GAISLER_BCC)
#include
diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c
index a1d09be4c29..c2daec56130 100644
--- a/wolfcrypt/src/aes.c
+++ b/wolfcrypt/src/aes.c
@@ -7466,6 +7466,7 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
#if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT)
#if defined(WOLFSSL_AESNI) && defined(GCM_TABLE_4BIT)
if (aes->use_aesni) {
+ VECTOR_REGISTERS_PUSH;
#if defined(WC_C_DYNAMIC_FALLBACK)
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags)) {
@@ -7483,6 +7484,7 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
GCM_generate_m0_aesni(aes->gcm.H, (byte*)aes->gcm.M0);
}
#endif
+ VECTOR_REGISTERS_POP;
}
else
#endif
diff --git a/wolfcrypt/src/cpuid.c b/wolfcrypt/src/cpuid.c
index 5c3e333ffee..9d9b458e8a9 100644
--- a/wolfcrypt/src/cpuid.c
+++ b/wolfcrypt/src/cpuid.c
@@ -113,7 +113,11 @@
static WC_INLINE void cpuid_set_flags(void)
{
+ #ifdef WOLFSSL_BSDKM
+ if (WOLFSSL_ATOMIC_LOAD_UINT(cpuid_flags) == WC_CPUID_INITIALIZER) {
+ #else
if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
+ #endif
cpuid_flags_t new_cpuid_flags = 0,
old_cpuid_flags = WC_CPUID_INITIALIZER;
if (cpuid_flag(1, 0, ECX, 28)) { new_cpuid_flags |= CPUID_AVX1 ; }
diff --git a/wolfssl/wolfcrypt/cpuid.h b/wolfssl/wolfcrypt/cpuid.h
index 176f99f2922..38b64e8fa7e 100644
--- a/wolfssl/wolfcrypt/cpuid.h
+++ b/wolfssl/wolfcrypt/cpuid.h
@@ -122,7 +122,11 @@ typedef word32 cpuid_flags_t;
* accurate.
*/
static WC_INLINE int cpuid_get_flags_atomic(cpuid_flags_atomic_t *flags) {
+ #ifdef WOLFSSL_BSDKM
+ if (WOLFSSL_ATOMIC_LOAD_UINT(*flags) == WC_CPUID_INITIALIZER) {
+ #else
if (WOLFSSL_ATOMIC_LOAD(*flags) == WC_CPUID_INITIALIZER) {
+ #endif /* WOLFSSL_BSDKM */
cpuid_flags_t old_cpuid_flags = WC_CPUID_INITIALIZER;
return wolfSSL_Atomic_Uint_CompareExchange
(flags, &old_cpuid_flags, cpuid_get_flags());