From 9a6a336c90fabc0d85d74b71f8c777b6a49b3919 Mon Sep 17 00:00:00 2001 From: jordan Date: Mon, 26 Jan 2026 00:22:03 -0600 Subject: [PATCH 1/4] bsdkm: x86 crypto acceleration support. --- .gitignore | 5 + Makefile.am | 5 +- bsdkm/Makefile | 66 ++- bsdkm/bsdkm_wc_port.h | 45 +- bsdkm/include.am | 10 +- bsdkm/wolfkmod.c | 799 +++++++++++++++++++++++++++++++- bsdkm/wolfkmod_aes.c | 387 ++++++++++++++++ bsdkm/x86_vecreg.c | 219 +++++++++ configure.ac | 42 +- wolfcrypt/benchmark/benchmark.c | 34 +- wolfcrypt/src/aes.c | 2 + wolfcrypt/src/cpuid.c | 4 + wolfssl/wolfcrypt/cpuid.h | 4 + 13 files changed, 1564 insertions(+), 58 deletions(-) create mode 100644 bsdkm/wolfkmod_aes.c create mode 100644 bsdkm/x86_vecreg.c diff --git a/.gitignore b/.gitignore index 0ef96441751..b444a290425 100644 --- a/.gitignore +++ b/.gitignore @@ -239,12 +239,17 @@ linuxkm/linuxkm linuxkm/src linuxkm/patches/src *.nds + +# Generated during FreeBSD kernel module build. bsdkm/export_syms bsdkm/i386 bsdkm/libwolfssl.ko bsdkm/machine bsdkm/opt_global.h bsdkm/x86 +bsdkm/bus_if.h +bsdkm/cryptodev_if.h +bsdkm/device_if.h # autotools generated scripts/unit.test diff --git a/Makefile.am b/Makefile.am index b7fc4db2fb3..67c5a4571ca 100644 --- a/Makefile.am +++ b/Makefile.am @@ -247,8 +247,9 @@ if BUILD_BSDKM EXTRA_CFLAGS EXTRA_CPPFLAGS EXTRA_CCASFLAGS EXTRA_LDFLAGS \ AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \ AM_CCASFLAGS CCASFLAGS \ - src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS - + src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_BSDKM_REGISTER \ + ENABLED_ASM ENABLED_AESNI \ + ENABLED_KERNEL_BENCHMARKS endif diff --git a/bsdkm/Makefile b/bsdkm/Makefile index dd6bbcbd775..eeeaecab5d0 100644 --- a/bsdkm/Makefile +++ b/bsdkm/Makefile @@ -1,19 +1,25 @@ # wolfssl kernel module name and source, and root dir. -KMOD=libwolfssl -SRCS=wolfkmod.c -WOLFSSL_DIR=../ +KMOD = libwolfssl +SRCS = wolfkmod.c +WOLFSSL_DIR = ../ CFLAGS+=-I${WOLFSSL_DIR} CFLAGS+=-DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER + # # debug options # verbose printing: -# CFLAGS+=-DWOLFSSL_BSDKM_VERBOSE_DEBUG +# CFLAGS += -DWOLFSSL_BSDKM_VERBOSE_DEBUG # # print memory mallocs / frees: -# CFLAGS+=-DWOLFSSL_BSDKM_MEMORY_DEBUG +# CFLAGS += -DWOLFSSL_BSDKM_MEMORY_DEBUG # -CFLAGS+=$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) +CFLAGS += $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) + +.if defined(ENABLED_BSDKM_REGISTER) + # These device header files are generated during build. + SRCS += bus_if.h cryptodev_if.h device_if.h +.endif # FreeBSD make does not support GNU make's patsubst and related. Filter # through sed instead. @@ -21,19 +27,26 @@ WOLFSSL_OBJS != echo ${src_libwolfssl_la_OBJECTS} | \ sed 's|src_libwolfssl_la-||g' | sed 's|\.lo|.o|g' | \ sed 's|wolfcrypt/src/|${WOLFSSL_DIR}/wolfcrypt/src/|g' +# wolfcrypt test .if ${ENABLED_CRYPT_TESTS} == "yes" WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/test/test.o .else - CFLAGS+=-DNO_CRYPT_TEST + CFLAGS += -DNO_CRYPT_TEST +.endif + +# wolfcrypt benchmark +.if ${ENABLED_KERNEL_BENCHMARKS} == "yes" + WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/benchmark/benchmark.o + CFLAGS+=-DWOLFSSL_NO_FLOAT_FMT .endif OBJS += ${WOLFSSL_OBJS} # Export no public symbols by default. .if !defined(BSDKM_EXPORT_SYMS) - EXPORT_SYMS=NO + EXPORT_SYMS = NO .else - EXPORT_SYMS=${BSDKM_EXPORT_SYMS} + EXPORT_SYMS = ${BSDKM_EXPORT_SYMS} .endif # Default to live kernel src tree makefile at @@ -45,12 +58,37 @@ OBJS += ${WOLFSSL_OBJS} .endif .include "${SYSDIR}/conf/kmod.mk" +# To use aesni in FreeBSD kernel we need to adjust build flags. +# See these for reference: +# - /usr/src/sys/modules/aesni/Makefile +# - /usr/src/sys/conf/kern.mk +.if ${ENABLED_ASM} == "yes" + CFLAGS.aes.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 + CFLAGS.aes.c := ${CFLAGS.aes.c:N-nostdinc} + CFLAGS.aes.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers + + CFLAGS.poly1305.c += -msse -msse2 -msse4.1 -maes -mpclmul + CFLAGS.poly1305.c := ${CFLAGS.aes.c:N-nostdinc} + CFLAGS.poly1305.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers + + CFLAGS.chacha.c += -msse -msse2 -msse4.1 -maes -mpclmul + CFLAGS.chacha.c := ${CFLAGS.aes.c:N-nostdinc} + CFLAGS.chacha.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers + + CFLAGS.sha.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 + CFLAGS.sha256.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 + CFLAGS.benchmark.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 + CFLAGS.benchmark.c := ${CFLAGS.aes.c:N-nostdinc} + CFLAGS.benchmark.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers + .PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers +.endif + # Smooth out a few inconsistencies between FreeBSD default compiler flags # in /usr/src/sys/conf/kern.mk, vs wolfssl harden flags in # m4/ax_harden_compiler_flags.m4. E.g. some FreeBSD header files shorten # 64 to 32 bit, and some wolfcrypt functions cast away const. -CFLAGS+= -Wno-unused-function -CFLAGS+= -Wno-cast-qual -CFLAGS+= -Wno-error=cast-qual -CFLAGS+= -Wno-shorten-64-to-32 -CFLAGS+= -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\"" +CFLAGS += -Wno-unused-function +CFLAGS += -Wno-cast-qual +CFLAGS += -Wno-error=cast-qual +CFLAGS += -Wno-shorten-64-to-32 +CFLAGS += -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\"" diff --git a/bsdkm/bsdkm_wc_port.h b/bsdkm/bsdkm_wc_port.h index 2a5524d5f5e..73d442df68e 100644 --- a/bsdkm/bsdkm_wc_port.h +++ b/bsdkm/bsdkm_wc_port.h @@ -61,13 +61,13 @@ static inline time_t wolfkmod_time(time_t * tloc) { #define WOLFSSL_DEBUG_PRINTF_FN printf /* str and char utility functions */ -#define XATOI(s) ({ \ - char * endptr = NULL; \ - long _xatoi_ret = strtol(s, &endptr, 10); \ - if ((s) == endptr || *endptr != '\0') { \ - _xatoi_ret = 0; \ - } \ - (int)_xatoi_ret; \ +#define XATOI(s) ({ \ + char * endptr = NULL; \ + long _xatoi_ret = strtol(s, &endptr, 10); \ + if ((s) == endptr || *endptr != '\0') { \ + _xatoi_ret = 0; \ + } \ + (int)_xatoi_ret; \ }) #if !defined(XMALLOC_OVERRIDE) @@ -103,6 +103,34 @@ extern struct malloc_type M_WOLFSSL[1]; }) #endif /* WOLFSSL_BSDKM_DEBUG_MEMORY */ + +#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS) + int wolfkmod_vecreg_init(void); + void wolfkmod_vecreg_exit(void); + int wolfkmod_vecreg_save(int flags_unused); + void wolfkmod_vecreg_restore(void); + /* wrapper defines for FPU_KERN(9). + * /usr/src/sys/amd64/amd64/fpu.c + * /usr/src/sys/amd64/include/pcb.h + * */ + #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS + #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS + #endif + + + #define SAVE_VECTOR_REGISTERS(fail_clause) { \ + int _svr_ret = wolfkmod_vecreg_save(0); \ + if (_svr_ret != 0) { \ + fail_clause \ + } \ + } + + #define SAVE_VECTOR_REGISTERS2() wolfkmod_vecreg_save(0) + + #define RESTORE_VECTOR_REGISTERS() wolfkmod_vecreg_restore() + +#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS */ + #if !defined(SINGLE_THREADED) #define WC_MUTEX_OPS_INLINE @@ -149,7 +177,8 @@ extern struct malloc_type M_WOLFSSL[1]; typedef volatile int wolfSSL_Atomic_Int; typedef volatile unsigned int wolfSSL_Atomic_Uint; #define WOLFSSL_ATOMIC_INITIALIZER(x) (x) - #define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x)) + #define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x)) + #define WOLFSSL_ATOMIC_LOAD_UINT(x) atomic_load_acq_int(&(x)) #define WOLFSSL_ATOMIC_STORE(x, v) atomic_store_rel_int(&(x), (v)) #define WOLFSSL_ATOMIC_OPS diff --git a/bsdkm/include.am b/bsdkm/include.am index 896a5447c9b..44666b6c084 100644 --- a/bsdkm/include.am +++ b/bsdkm/include.am @@ -2,8 +2,10 @@ # included from Top Level Makefile.am # All paths should be given relative to the root -EXTRA_DIST += m4/ax_bsdkm.m4 \ - bsdkm/Makefile \ - bsdkm/README.md \ - bsdkm/wolfkmod.c \ +EXTRA_DIST += m4/ax_bsdkm.m4 \ + bsdkm/Makefile \ + bsdkm/README.md \ + bsdkm/wolfkmod.c \ + bsdkm/wolfkmod_aes.c \ + bsdkm/x86_vecreg.c \ bsdkm/bsdkm_wc_port.h diff --git a/bsdkm/wolfkmod.c b/bsdkm/wolfkmod.c index bb03e8ebbc3..f8e5f4d54b2 100644 --- a/bsdkm/wolfkmod.c +++ b/bsdkm/wolfkmod.c @@ -26,6 +26,12 @@ #include #include +#if defined(BSDKM_CRYPTO_REGISTER) + #include + #include + #include "cryptodev_if.h" +#endif + /* wolf includes */ #include #ifdef WOLFCRYPT_ONLY @@ -44,15 +50,42 @@ #if !defined(NO_CRYPT_TEST) #include #endif +#if defined(WOLFSSL_KERNEL_BENCHMARKS) + #include +#endif #include MALLOC_DEFINE(M_WOLFSSL, "libwolfssl", "wolfSSL kernel memory"); -static int wolfkmod_init(void); -static int wolfkmod_cleanup(void); -static int wolfkmod_load(void); -static int wolfkmod_unload(void); +#if defined(BSDKM_CRYPTO_REGISTER) + #include "bsdkm/wolfkmod_aes.c" +#endif + +/* common functions. */ +static int wolfkmod_init(void); +static int wolfkmod_cleanup(void); +#if !defined(BSDKM_CRYPTO_REGISTER) +/* functions specific to a pure kernel module library build. */ +static int wolfkmod_load(void); +static int wolfkmod_unload(void); +#else +/* functions specific to a kernel crypto driver module build. */ +static void wolfkdriv_identify(driver_t * driver, device_t parent); +static int wolfkdriv_probe(device_t dev); +static int wolfkdriv_attach(device_t dev); +static int wolfkdriv_detach(device_t dev); +static int wolfkdriv_probesession(device_t dev, + const struct crypto_session_params *csp); +static int wolfkdriv_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp); +static void wolfkdriv_freesession(device_t dev, crypto_session_t cses); +static int wolfkdriv_process(device_t dev, struct cryptop *crp, int hint); +#endif /* !BSDKM_CRYPTO_REGISTER */ + +#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS) + #include "bsdkm/x86_vecreg.c" +#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/ #ifdef HAVE_FIPS #define WOLFKMOD_FIPS_ERR_MSG(hash) ({ \ @@ -82,6 +115,14 @@ static int wolfkmod_init(void) { int error = 0; + #if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS) + error = wolfkmod_vecreg_init(); + if (error != 0) { + printf("error: wolfkmod_vecreg_init: %d\n", error); + return (ECANCELED); + } + #endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/ + #ifdef HAVE_FIPS error = wolfCrypt_SetCb_fips(wolfkmod_fips_cb); if (error != 0) { @@ -190,9 +231,14 @@ static int wolfkmod_cleanup(void) " cleanup complete.\n"); #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + #if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS) + wolfkmod_vecreg_exit(); + #endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/ + return (0); } +#if !defined(BSDKM_CRYPTO_REGISTER) static int wolfkmod_load(void) { int error = 0; @@ -212,10 +258,15 @@ static int wolfkmod_load(void) printf("info: wolfCrypt self-test passed.\n"); #endif /* NO_CRYPT_TEST */ - /** - * todo: register wolfcrypt algs here with crypto_get_driverid - * and related. - * */ + #ifdef WOLFSSL_KERNEL_BENCHMARKS + error = benchmark_test(NULL); + if (error != 0) { + printf("error: wolfcrypt benchmark failed: %d\n", error); + (void)wolfkmod_cleanup(); + return (ECANCELED); + } + printf("info: wolfCrypt benchmark passed.\n"); + #endif /* WOLFSSL_KERNEL_BENCHMARKS */ printf("info: libwolfssl loaded\n"); @@ -239,11 +290,6 @@ static int wolfkmod_unload(void) error = wolfkmod_cleanup(); - /** - * todo: unregister wolfcrypt algs here with crypto_unregister_all - * and related. - * */ - if (error == 0) { printf("info: libwolfssl unloaded\n"); } @@ -294,7 +340,730 @@ wolfkmod_event(struct module * m, int what, void * arg) return (error); } +#endif /* !BSDKM_CRYPTO_REGISTER */ + +#if defined(BSDKM_CRYPTO_REGISTER) +/* wolfkdriv device driver software context. */ +struct wolfkdriv_softc { + int32_t crid; + device_t dev; +}; + +struct km_aes_ctx { + Aes aes_encrypt; + Aes aes_decrypt; +}; + +typedef struct km_aes_ctx km_aes_ctx; + +struct wolfkdriv_session { + km_aes_ctx aes_ctx; + int32_t crid; + int type; + int ivlen; + int klen; +}; + +typedef struct wolfkdriv_session wolfkdriv_session_t; + +static void km_AesFree(Aes * aes) { + if (aes == NULL) { + return; + } + wc_AesFree(aes); + #if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0) + ForceZero(aes, sizeof(*aes)); + #endif +} + +static void wolfkdriv_aes_ctx_clear(km_aes_ctx * ctx) +{ + if (ctx != NULL) { + km_AesFree(&ctx->aes_encrypt); + km_AesFree(&ctx->aes_decrypt); + } + + #ifdef WOLFKM_DEBUG_AES + printf("info: exiting km_AesExitCommon\n"); + #endif /* WOLFKM_DEBUG_AES */ +} + +static void wolfkdriv_identify(driver_t * driver, device_t parent) +{ + (void)driver; + + /* don't double add wolfkdriv child. */ + if (device_find_child(parent, "libwolf", -1) != NULL) { + return; + } + + BUS_ADD_CHILD(parent, 10, "libwolf", -1); +} + +static int wolfkdriv_probe(device_t dev) +{ + device_set_desc(dev, "wolfSSL crypto"); + return (BUS_PROBE_DEFAULT); +} + +/* + * unregister libwolfssl crypto driver + */ +static void wolfkdriv_unregister(struct wolfkdriv_softc * softc) +{ + if (softc && softc->crid >= 0) { + crypto_unregister_all(softc->crid); + device_printf(softc->dev, "info: crid unregistered: %d\n", softc->crid); + softc->crid = -1; + } + + return; +} + +static int wolfkdriv_attach(device_t dev) +{ + struct wolfkdriv_softc * softc = NULL; + int flags = CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | + CRYPTOCAP_F_ACCEL_SOFTWARE | CRYPTOCAP_F_HARDWARE; + int ret = 0; + int crid = 0; + int error = 0; + + ret = wolfkmod_init(); + if (ret != 0) { + return (ECANCELED); + } + + /** + * register wolfcrypt algs here with crypto_get_driverid. + * + * The crid is the literal index into the kernel crypto_drivers array: + * - crid >= 0 is valid. + * - crid < 0 is error. + * */ + softc = device_get_softc(dev); + softc->dev = dev; + + softc->crid = crypto_get_driverid(dev, sizeof(wolfkdriv_session_t), flags); + if (softc->crid < 0) { + device_printf(dev, "error: crypto_get_driverid failed: %d\n", + softc->crid); + return (ENXIO); + } + + /* + * various sanity checks + */ + + /* 1. we should find ourself by name */ + crid = crypto_find_driver("libwolf"); + + if (crid != softc->crid) { + device_printf(dev, "error: attach: got crid %d, expected %d\n", crid, + softc->crid); + error = ENXIO; + goto attach_out; + } + + /* 2. test various algs */ + error = wolfkdriv_test_aes(dev, crid); + + if (error) { + device_printf(dev, "error: attach: test_aes: %d\n", error); + error = ENXIO; + goto attach_out; + } + + device_printf(dev, "info: driver loaded: %d\n", crid); + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: exiting attach\n"); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + +attach_out: + if (error) { + wolfkdriv_unregister(softc); + error = ENXIO; + } + + return (error); +} + +static int wolfkdriv_detach(device_t dev) +{ + struct wolfkdriv_softc * softc = NULL; + int ret = 0; + + ret = wolfkmod_cleanup(); + + if (ret == 0) { + /* unregister wolfcrypt algs */ + softc = device_get_softc(dev); + wolfkdriv_unregister(softc); + } + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: exiting detach\n"); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + + return (0); +} + +static int wolfkdriv_probesession(device_t dev, + const struct crypto_session_params *csp) +{ + struct wolfkdriv_softc * softc = NULL; + int error = CRYPTODEV_PROBE_ACCEL_SOFTWARE; + + softc = device_get_softc(dev); + + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + break; + default: + error = EINVAL; + break; + } + break; + + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + break; + default: + error = EINVAL; + break; + } + break; + case CSP_MODE_DIGEST: + case CSP_MODE_ETA: + default: + error = EINVAL; + break; + } + + (void)softc; + (void)csp; + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: probesession: mode=%d, cipher_alg=%d, error=%d\n", + csp->csp_mode, csp->csp_cipher_alg, error); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + return (error); +} + +static int wolfkdriv_newsession_cipher(device_t dev, + wolfkdriv_session_t * session, + const struct crypto_session_params *csp) +{ + int error = 0; + int klen = csp->csp_cipher_klen; /* key len in bytes */ + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + session->type = CRYPTO_AES_NIST_GCM_16; + break; + case CRYPTO_AES_CBC: + session->type = CRYPTO_AES_CBC; + break; + default: + return (EOPNOTSUPP); + } + + if (klen != 16 && klen != 24 && klen != 32) { + device_printf(dev, "info: newsession_cipher: invalid klen: %d\n", klen); + return (EINVAL); + } + + session->klen = klen; + session->ivlen = csp->csp_ivlen; + + /* encrypt */ + error = wc_AesInit(&session->aes_ctx.aes_encrypt, NULL, INVALID_DEVID); + if (error) { + device_printf(dev, "error: newsession_cipher: aes init: %d\n", error); + goto newsession_cipher_out; + } + + if (session->type == CRYPTO_AES_CBC) { + /* Need a separate decrypt structure for aes-cbc. */ + error = wc_AesInit(&session->aes_ctx.aes_decrypt, NULL, INVALID_DEVID); + if (error) { + device_printf(dev, "error: newsession_cipher: aes init: %d\n", + error); + goto newsession_cipher_out; + } + } + +newsession_cipher_out: + + if (error != 0) { + wolfkdriv_aes_ctx_clear(&session->aes_ctx); + return (EINVAL); + } + + return (error); +} + +static int wolfkdriv_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + wolfkdriv_session_t * session = NULL; + int error = 0; + + /* get the wolfkdriv_session_t context */ + session = crypto_get_driver_session(cses); + + switch (csp->csp_mode) { + case CSP_MODE_DIGEST: + case CSP_MODE_ETA: + device_printf(dev, "info: not supported: %d\n", csp->csp_mode); + error = EOPNOTSUPP; + break; + case CSP_MODE_CIPHER: + case CSP_MODE_AEAD: + error = wolfkdriv_newsession_cipher(dev, session, csp); + break; + default: + __assert_unreachable(); + } + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: newsession: mode=%d, cipher_alg=%d, error=%d\n", + csp->csp_mode, csp->csp_cipher_alg, error); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + + return (error); +} + +/* + * + */ +static void +wolfkdriv_freesession(device_t dev, crypto_session_t cses) +{ + wolfkdriv_session_t * session = NULL; + (void)dev; + + /* get the wolfkdriv_session_t context */ + session = crypto_get_driver_session(cses); + + /* clean it up */ + wolfkdriv_aes_ctx_clear(&session->aes_ctx); + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: exiting freesession\n"); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + return; +} + +/* + * + */ +static int wolfkdriv_cbc_work(device_t dev, wolfkdriv_session_t * session, + struct cryptop * crp, + const struct crypto_session_params * csp) +{ + struct crypto_buffer_cursor cc_in; + struct crypto_buffer_cursor cc_out; + const unsigned char * in_block = NULL; + const unsigned char * in_seg = NULL; + unsigned char * out_block = NULL; + unsigned char * out_seg = NULL; + Aes aes; + uint8_t iv[WC_AES_BLOCK_SIZE]; + uint8_t block[EALG_MAX_BLOCK_LEN]; + size_t data_len = 0; + size_t seg_len = 0; + size_t in_len = 0; + size_t out_len = 0; + int error = 0; + int is_encrypt = 0; + int type = AES_ENCRYPTION; + + if (csp->csp_cipher_alg != CRYPTO_AES_CBC) { + error = EINVAL; + goto cbc_work_out; + } + + data_len = crp->crp_payload_length; + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + is_encrypt = 1; + type = AES_ENCRYPTION; + memcpy(&aes, &session->aes_ctx.aes_encrypt, sizeof(aes)); + } + else { + is_encrypt = 0; + type = AES_DECRYPTION; + memcpy(&aes, &session->aes_ctx.aes_decrypt, sizeof(aes)); + } + + /* must be multiple of block size */ + if (data_len % WC_AES_BLOCK_SIZE) { + error = EINVAL; + goto cbc_work_out; + } + + crypto_read_iv(crp, iv); + error = wc_AesSetKey(&aes, csp->csp_cipher_key, + csp->csp_cipher_klen, iv, type); + if (error) { + device_printf(dev, "error: wc_AesSetKey: %d\n", error); + goto cbc_work_out; + } + + /* set up the crypto buffers */ + crypto_cursor_init(&cc_in, &crp->crp_buf); + crypto_cursor_advance(&cc_in, crp->crp_payload_start); + + in_seg = crypto_cursor_segment(&cc_in, &in_len); + + /* handle if the user supplied a separate out buffer. */ + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + crypto_cursor_init(&cc_out, &crp->crp_obuf); + crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); + } + else { + cc_out = cc_in; + } + + out_seg = crypto_cursor_segment(&cc_out, &out_len); + + while (data_len) { + /* set up input buffers */ + if (in_len < WC_AES_BLOCK_SIZE) { + /* less than a block in segment */ + crypto_cursor_copydata(&cc_in, WC_AES_BLOCK_SIZE, block); + in_block = block; + in_len = WC_AES_BLOCK_SIZE; + } + else { + in_block = in_seg; + } + + /* set up output buffers */ + if (out_len < WC_AES_BLOCK_SIZE) { + out_block = block; + out_len = WC_AES_BLOCK_SIZE; + } + else { + out_block = out_seg; + } + + /* choose which of data_len, in_len, out_len, is shorter. + * round down to multiple of aes block size. */ + seg_len = rounddown(MIN(data_len, MIN(in_len, out_len)), + WC_AES_BLOCK_SIZE); + + if (is_encrypt) { + error = wc_AesCbcEncrypt(&aes, out_block, in_block, seg_len); + if (error) { + device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error); + goto cbc_work_out; + } + } + else { + error = wc_AesCbcDecrypt(&aes, out_block, in_block, seg_len); + if (error) { + device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error); + goto cbc_work_out; + } + } + + if (out_block == block) { + /* we used the block as local output buffer. copy to cc_out, + * and grab the next out cursor segment. */ + crypto_cursor_copyback(&cc_out, WC_AES_BLOCK_SIZE, block); + out_seg = crypto_cursor_segment(&cc_out, &out_len); + } else { + /* we worked directly in cc_out. advance the cursor. */ + crypto_cursor_advance(&cc_out, seg_len); + out_seg += seg_len; + out_len -= seg_len; + } + + if (in_block == block) { + /* grab a new in cursor segment. */ + in_seg = crypto_cursor_segment(&cc_in, &in_len); + } else { + /* else advance existing in cursor. */ + crypto_cursor_advance(&cc_in, seg_len); + in_seg += seg_len; + in_len -= seg_len; + } + + data_len -= seg_len; + } + +cbc_work_out: + /* cleanup. */ + wc_ForceZero(iv, sizeof(iv)); + wc_ForceZero(block, sizeof(block)); + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: cbc_work: mode=%d, cipher_alg=%d, " + "payload_length=%d, error=%d\n", + csp->csp_mode, csp->csp_cipher_alg, crp->crp_payload_length, + error); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + + return (error); +} + +/* + * todo: skeleton implementation, finish. + */ +static int wolfkdriv_gcm_work(device_t dev, wolfkdriv_session_t * session, + struct cryptop * crp, + const struct crypto_session_params * csp) +{ + struct crypto_buffer_cursor cc_in; + struct crypto_buffer_cursor cc_out; + const unsigned char * in_seg = NULL; + unsigned char * out_seg = NULL; + Aes aes; + uint8_t iv[WC_AES_BLOCK_SIZE]; + uint8_t tag[WC_AES_BLOCK_SIZE]; + size_t data_len = 0; + size_t seg_len = 0; + size_t in_len = 0; + size_t out_len = 0; + int error = 0; + int is_encrypt = 0; + + memcpy(&aes, &session->aes_ctx.aes_encrypt, sizeof(aes)); + + if (csp->csp_cipher_alg != CRYPTO_AES_NIST_GCM_16) { + error = EINVAL; + goto gcm_work_out; + } + + data_len = crp->crp_payload_length; + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + is_encrypt = 1; + } + else { + is_encrypt = 0; + } + + error = wc_AesGcmSetKey(&aes, csp->csp_cipher_key, + csp->csp_cipher_klen); + if (error) { + device_printf(dev, "error: wc_AesGcmSetKey: %d\n", error); + goto gcm_work_out; + } + + crypto_read_iv(crp, iv); + error = wc_AesGcmInit(&aes, NULL /* key */, 0 /* keylen */, + iv, csp->csp_ivlen); + if (error) { + device_printf(dev, "error: wc_AesGcmInit: %d\n", error); + goto gcm_work_out; + } + + /* + * process aad first + */ + if (crp->crp_aad != NULL) { + /* they passed it in separate buffer. */ + if (is_encrypt) { + error = wc_AesGcmEncryptUpdate(&aes, NULL, NULL, 0, + crp->crp_aad, crp->crp_aad_length); + } + else { + error = wc_AesGcmDecryptUpdate(&aes, NULL, NULL, 0, + crp->crp_aad, crp->crp_aad_length); + } + + if (error) { + error = EINVAL; + } + } + else { + /* we need to pull aad out of crp->crp_buf from crp_aad_start. */ + size_t aad_len = 0; + + crypto_cursor_init(&cc_in, &crp->crp_buf); + crypto_cursor_advance(&cc_in, crp->crp_aad_start); + + for (aad_len = crp->crp_aad_length; aad_len > 0; aad_len -= seg_len) { + in_seg = crypto_cursor_segment(&cc_in, &in_len); + seg_len = MIN(aad_len, in_len); + + if (is_encrypt) { + error = wc_AesGcmEncryptUpdate(&aes, NULL, NULL, 0, + in_seg, seg_len); + } + else { + error = wc_AesGcmDecryptUpdate(&aes, NULL, NULL, 0, + in_seg, seg_len); + } + + if (error) { + error = EINVAL; + goto gcm_work_out; + } + + crypto_cursor_advance(&cc_in, seg_len); + } + } + + /* + * process cipher/plaintext next + */ + + /* set up the crypto buffers */ + crypto_cursor_init(&cc_in, &crp->crp_buf); + crypto_cursor_advance(&cc_in, crp->crp_payload_start); + + in_seg = crypto_cursor_segment(&cc_in, &in_len); + + /* handle if the user supplied a separate out buffer. */ + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + crypto_cursor_init(&cc_out, &crp->crp_obuf); + crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); + } + else { + cc_out = cc_in; + } + + out_seg = crypto_cursor_segment(&cc_out, &out_len); + + while (data_len) { + /* process through the available segments. */ + in_seg = crypto_cursor_segment(&cc_in, &in_len); + out_seg = crypto_cursor_segment(&cc_out, &out_len); + seg_len = MIN(data_len, MIN(in_len, out_len)); + + if (is_encrypt) { + error = wc_AesGcmEncryptUpdate(&aes, out_seg, in_seg, seg_len, + NULL, 0); + if (error) { + device_printf(dev, "error: wc_AesGcmEncrypt: %d\n", error); + goto gcm_work_out; + } + } + else { + error = wc_AesGcmDecryptUpdate(&aes, out_seg, in_seg, seg_len, + NULL, 0); + if (error) { + device_printf(dev, "error: wc_AesGcmEncrypt: %d\n", error); + goto gcm_work_out; + } + } + + /* advance the cursors by amount processed */ + crypto_cursor_advance(&cc_in, seg_len); + crypto_cursor_advance(&cc_out, seg_len); + + data_len -= seg_len; + } + + /* + * process auth tag finally + */ + + if (is_encrypt) { + error = wc_AesGcmEncryptFinal(&aes, tag, WC_AES_BLOCK_SIZE); + if (error == 0) { + crypto_copyback(crp, crp->crp_digest_start, WC_AES_BLOCK_SIZE, tag); + } + } + else { + crypto_copydata(crp, crp->crp_digest_start, WC_AES_BLOCK_SIZE, tag); + error = wc_AesGcmDecryptFinal(&aes, tag, WC_AES_BLOCK_SIZE); + if (error) { + error = EBADMSG; + } + } + +gcm_work_out: + /* cleanup. */ + wc_ForceZero(iv, sizeof(iv)); + wc_ForceZero(tag, sizeof(tag)); + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: gcm_work: mode=%d, cipher_alg=%d, " + "payload_length=%d, error=%d\n", + csp->csp_mode, csp->csp_cipher_alg, crp->crp_payload_length, + error); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + + return (error); +} + +static int wolfkdriv_process(device_t dev, struct cryptop * crp, int hint) +{ + const struct crypto_session_params * csp = NULL; + wolfkdriv_session_t * session = NULL; + int error = 0; + (void)hint; + + session = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); + + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + error = wolfkdriv_cbc_work(dev, session, crp, csp); + break; + case CSP_MODE_DIGEST: + case CSP_MODE_ETA: + error = EINVAL; + break; + case CSP_MODE_AEAD: + error = wolfkdriv_gcm_work(dev, session, crp, csp); + break; + default: + __assert_unreachable(); + } + + crp->crp_etype = error; + crypto_done(crp); + + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: process: mode=%d, cipher_alg=%d, error=%d\n", + csp->csp_mode, csp->csp_cipher_alg, error); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + + return (error); +} + +/* + * wolfkmod as a crypto device driver. + */ +static device_method_t wolfkdriv_methods[] = { + /* device interface methods: called during device setup, etc. */ + DEVMETHOD(device_identify, wolfkdriv_identify), + DEVMETHOD(device_probe, wolfkdriv_probe), + DEVMETHOD(device_attach, wolfkdriv_attach), + DEVMETHOD(device_detach, wolfkdriv_detach), + + /* crypto device session methods: called during crypto session setup, + * work, etc. */ + DEVMETHOD(cryptodev_probesession, wolfkdriv_probesession), + DEVMETHOD(cryptodev_newsession, wolfkdriv_newsession), + DEVMETHOD(cryptodev_freesession, wolfkdriv_freesession), + DEVMETHOD(cryptodev_process, wolfkdriv_process), + DEVMETHOD_END +}; + +static driver_t wolfkdriv_driver = { + .name = "libwolf", + .methods = wolfkdriv_methods, + .size = sizeof(struct wolfkdriv_softc), +}; + +/* note: on x86, software-only drivers usually attach to nexus bus. */ +DRIVER_MODULE(libwolfssl, nexus, wolfkdriv_driver, NULL, NULL); +#endif /* BSDKM_CRYPTO_REGISTER */ + +#if !defined(BSDKM_CRYPTO_REGISTER) +/* + * wolfkmod as a pure kernel module. + */ static moduledata_t libwolfmod = { #ifdef HAVE_FIPS "libwolfssl_fips", /* module name */ @@ -305,6 +1074,8 @@ static moduledata_t libwolfmod = { NULL /* extra data, unused */ }; -MODULE_VERSION(libwolfssl, 1); DECLARE_MODULE(libwolfssl, libwolfmod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); +#endif /* !BSDKM_CRYPTO_REGISTER */ + +MODULE_VERSION(libwolfssl, 1); #endif /* WOLFSSL_BSDKM */ diff --git a/bsdkm/wolfkmod_aes.c b/bsdkm/wolfkmod_aes.c new file mode 100644 index 00000000000..5cbe2871e14 --- /dev/null +++ b/bsdkm/wolfkmod_aes.c @@ -0,0 +1,387 @@ +#if !defined(WC_SKIP_INCLUDED_C_FILES) && defined(BSDKM_CRYPTO_REGISTER) +#include + +#if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG) +static void +wolfkmod_print_data(const char * what, const uint8_t * data, size_t data_len) +{ + size_t i = 0; + + printf("%s:\n", what); + for (i = 0; i < data_len; ++i) { + printf("0x%02x, ", data[i]); + if ((i + 1) % 8 == 0) { + printf("\n"); + } + } + + printf("\n"); +} +#endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */ + +/* + * cryptodev framework always calls a callback, even when CRYPTOCAP_F_SYNC. + */ +static int +wolfkdriv_test_crp_callback(struct cryptop * crp) +{ + (void)crp; + return (0); +} + +/* Test aes-cbc with a buffer larger than aes block size. + * Verify direct wolfcrypt API and opencrypto framework return + * same result. */ +static int wolfkdriv_test_aes_cbc_big(device_t dev, int crid) +{ + crypto_session_t session = NULL; + struct crypto_session_params csp; + struct cryptop * crp = NULL; + Aes * aes_encrypt = NULL; + int error = 0; + byte msg[] = { + 0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74, + 0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20, + 0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74, + 0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20, + 0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74, + 0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20 + }; + byte work1[WC_AES_BLOCK_SIZE * 3]; /* wolfcrypt buffer */ + byte work2[WC_AES_BLOCK_SIZE * 3]; /* opencrypto buffer */ + /* padded to 16-bytes */ + const byte key[] = "0123456789abcdef "; + /* padded to 16-bytes */ + const byte iv[] = "1234567890abcdef "; + + memset(&csp, 0, sizeof(csp)); + memcpy(work1, msg, sizeof(msg)); /* wolfcrypt work buffer */ + memcpy(work2, msg, sizeof(msg)); /* opencrypto work buffer */ + + /* wolfcrypt encrypt */ + aes_encrypt = (Aes *)XMALLOC(sizeof(Aes), NULL, DYNAMIC_TYPE_AES); + if (aes_encrypt == NULL) { + error = ENOMEM; + device_printf(dev, "error: malloc failed\n"); + goto test_aes_cbc_big_out; + } + + error = wc_AesInit(aes_encrypt, NULL, INVALID_DEVID); + if (error) { + device_printf(dev, "error: newsession_cipher: aes init: %d\n", error); + goto test_aes_cbc_big_out; + } + + error = wc_AesSetKey(aes_encrypt, key, 16, iv, AES_ENCRYPTION); + if (error) { + device_printf(dev, "error: wc_AesSetKey: %d\n", error); + goto test_aes_cbc_big_out; + } + + error = wc_AesCbcEncrypt(aes_encrypt, work1, work1, sizeof(work1)); + if (error) { + device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error); + goto test_aes_cbc_big_out; + } + + /* opencrypto encrypt */ + csp.csp_mode = CSP_MODE_CIPHER; + csp.csp_cipher_alg = CRYPTO_AES_CBC; + csp.csp_ivlen = WC_AES_BLOCK_SIZE; + csp.csp_cipher_key = key; + csp.csp_cipher_klen = WC_AES_BLOCK_SIZE; + error = crypto_newsession(&session, &csp, crid); + if (error || session == NULL) { + goto test_aes_cbc_big_out; + } + + crp = crypto_getreq(session, M_WAITOK); + if (crp == NULL) { + device_printf(dev, "error: test_aes: crypto_getreq failed\n"); + goto test_aes_cbc_big_out; + } + + crp->crp_callback = wolfkdriv_test_crp_callback; + crp->crp_op = CRYPTO_OP_ENCRYPT; + crp->crp_flags = CRYPTO_F_IV_SEPARATE; + + memcpy(crp->crp_iv, iv, WC_AES_BLOCK_SIZE); + + crypto_use_buf(crp, work2, sizeof(work2)); + crp->crp_payload_start = 0; + crp->crp_payload_length = sizeof(work2); + + error = crypto_dispatch(crp); + if (error) { + goto test_aes_cbc_big_out; + } + + error = XMEMCMP(work1, work2, sizeof(work2)); + if (error) { + device_printf(dev, "error: test_aes: enc vectors diff: %d\n", error); + goto test_aes_cbc_big_out; + } + + #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG) + wolfkmod_print_data("msg_enc", work2, sizeof(work2)); + #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */ + + /* opencrypto decrypt */ + crp->crp_op = CRYPTO_OP_DECRYPT; + + error = crypto_dispatch(crp); + if (error) { + goto test_aes_cbc_big_out; + } + + error = XMEMCMP(work2, msg, sizeof(msg)); + if (error) { + device_printf(dev, "error: test_aes: dec vectors diff: %d\n", error); + goto test_aes_cbc_big_out; + } + + #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG) + wolfkmod_print_data("msg_dec", work2, sizeof(work2)); + #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */ + +test_aes_cbc_big_out: + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: test_aes_cbc_big: error=%d, session=%p, crp=%p\n", + error, (void *)session, (void*)crp); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + + if (crp != NULL) { + crypto_freereq(crp); + crp = NULL; + } + + if (session != NULL) { + crypto_freesession(session); + session = NULL; + } + + if (aes_encrypt != NULL) { + wc_AesFree(aes_encrypt); + XFREE(aes_encrypt, NULL, DYNAMIC_TYPE_AES); + aes_encrypt = NULL; + } + + return (error); +} + +/* Test aes-gcm encrypt and decrypt a small buffer with opencrypto + * framework and wolfcrypt. + */ +static int wolfkdriv_test_aes_gcm(device_t dev, int crid) +{ + crypto_session_t session = NULL; + struct crypto_session_params csp; + struct cryptop * crp = NULL; + Aes * enc = NULL; + int error = 0; + + WOLFSSL_SMALL_STACK_STATIC const byte p[] = + { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 + }; + + WOLFSSL_SMALL_STACK_STATIC const byte c1[] = + { + 0x52, 0x2d, 0xc1, 0xf0, 0x99, 0x56, 0x7d, 0x07, + 0xf4, 0x7f, 0x37, 0xa3, 0x2a, 0x84, 0x42, 0x7d, + 0x64, 0x3a, 0x8c, 0xdc, 0xbf, 0xe5, 0xc0, 0xc9, + 0x75, 0x98, 0xa2, 0xbd, 0x25, 0x55, 0xd1, 0xaa, + 0x8c, 0xb0, 0x8e, 0x48, 0x59, 0x0d, 0xbb, 0x3d, + 0xa7, 0xb0, 0x8b, 0x10, 0x56, 0x82, 0x88, 0x38, + 0xc5, 0xf6, 0x1e, 0x63, 0x93, 0xba, 0x7a, 0x0a, + 0xbc, 0xc9, 0xf6, 0x62 + }; + + WOLFSSL_SMALL_STACK_STATIC byte a[] = + { + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xab, 0xad, 0xda, 0xd2 + }; + + WOLFSSL_SMALL_STACK_STATIC const byte k1[] = + { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }; + + WOLFSSL_SMALL_STACK_STATIC const byte iv1[] = + { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 + }; + + WOLFSSL_SMALL_STACK_STATIC const byte t1[] = + { + 0x76, 0xfc, 0x6e, 0xce, 0x0f, 0x4e, 0x17, 0x68, + 0xcd, 0xdf, 0x88, 0x53, 0xbb, 0x2d, 0x55, 0x1b + }; + + byte resultT[sizeof(t1) + WC_AES_BLOCK_SIZE]; + byte resultC[sizeof(p) + WC_AES_BLOCK_SIZE]; + byte resultC2[sizeof(p) + WC_AES_BLOCK_SIZE]; + + XMEMSET(resultT, 0, sizeof(resultT)); + XMEMSET(resultC, 0, sizeof(resultC)); + + XMEMSET(resultC2, 0, sizeof(resultC)); + XMEMCPY(resultC2, p, sizeof(p)); + + /* wolfcrypt encrypt */ + enc = (Aes *)XMALLOC(sizeof(Aes), NULL, DYNAMIC_TYPE_AES); + if (enc == NULL) { + error = ENOMEM; + device_printf(dev, "error: malloc failed\n"); + goto test_aes_gcm_out; + } + + error = wc_AesGcmEncryptInit(enc, k1, sizeof(k1), iv1, sizeof(iv1)); + if (error) { goto test_aes_gcm_out; } + + error = wc_AesGcmEncryptUpdate(enc, resultC, p, sizeof(p), a, sizeof(a)); + if (error) { goto test_aes_gcm_out; } + + error = wc_AesGcmEncryptFinal(enc, resultT, sizeof(t1)); + if (error) { goto test_aes_gcm_out; } + + #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG) + wolfkmod_print_data("resultC", resultC, sizeof(p)); + wolfkmod_print_data("resultT", resultT, sizeof(t1)); + #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */ + + error = XMEMCMP(resultC, c1, sizeof(c1)); + if (error) { goto test_aes_gcm_out; } + + error = XMEMCMP(resultT, t1, sizeof(t1)); + if (error) { goto test_aes_gcm_out; } + + /* + * opencrypto encrypt + * */ + + /* set crypto session params */ + memset(&csp, 0, sizeof(csp)); + csp.csp_flags |= CSP_F_SEPARATE_AAD; + csp.csp_mode = CSP_MODE_AEAD; + csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; + csp.csp_ivlen = sizeof(iv1); + csp.csp_cipher_key = k1; + csp.csp_cipher_klen = sizeof(k1); + + /* get crypto session handle */ + error = crypto_newsession(&session, &csp, crid); + if (error || session == NULL) { + device_printf(dev, "error: test_aes: crypto_newsession: %d, %p\n", + error, (void *)session); + goto test_aes_gcm_out; + } + + /* get a crypto op handle */ + crp = crypto_getreq(session, M_WAITOK); + if (crp == NULL) { + device_printf(dev, "error: test_aes: crypto_getreq failed\n"); + goto test_aes_gcm_out; + } + + /* configure it */ + crp->crp_callback = wolfkdriv_test_crp_callback; + crp->crp_op = (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST); + crp->crp_flags = CRYPTO_F_IV_SEPARATE; + + memcpy(crp->crp_iv, iv1, sizeof(iv1)); + + crypto_use_buf(crp, resultC2, sizeof(resultC2)); + crp->crp_payload_start = 0; + crp->crp_payload_length = sizeof(p); + + crp->crp_aad = a; + crp->crp_aad_start = 0; + crp->crp_aad_length = sizeof(a); + crp->crp_digest_start = crp->crp_payload_start + sizeof(p); + + error = crypto_dispatch(crp); + if (error) { + goto test_aes_gcm_out; + } + + #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG) + wolfkmod_print_data("resultC2", resultC2, sizeof(p)); + wolfkmod_print_data("resultT2", resultC2 + sizeof(p), sizeof(t1)); + #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */ + + error = XMEMCMP(resultC2, c1, sizeof(c1)); + if (error) { goto test_aes_gcm_out; } + + error = XMEMCMP(resultC2 + sizeof(p), t1, sizeof(t1)); + if (error) { goto test_aes_gcm_out; } + + /* opencrypto decrypt */ + crp->crp_op = (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST); + + error = crypto_dispatch(crp); + if (error) { + goto test_aes_gcm_out; + } + + #if defined(WOLFSSL_BSDKM_AES_VERBOSE_DEBUG) + wolfkmod_print_data("resultC2_dec", resultC2, sizeof(p)); + #endif /* WOLFSSL_BSDKM_AES_VERBOSE_DEBUG */ + + error = XMEMCMP(resultC2, p, sizeof(p)); + if (error) { goto test_aes_gcm_out; } + +test_aes_gcm_out: + #if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG) + device_printf(dev, "info: test_aes_gcm: error=%d, session=%p, crp=%p\n", + error, (void *)session, (void*)crp); + #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + + if (crp != NULL) { + crypto_freereq(crp); + crp = NULL; + } + + if (session != NULL) { + crypto_freesession(session); + session = NULL; + } + + if (enc != NULL) { + wc_AesFree(enc); + XFREE(enc, NULL, DYNAMIC_TYPE_AES); + enc = NULL; + } + + return (error); +} + + +static int wolfkdriv_test_aes(device_t dev, int crid) +{ + int error = 0; + + if (error == 0) { + error = wolfkdriv_test_aes_cbc_big(dev, crid); + } + + if (error == 0) { + error = wolfkdriv_test_aes_gcm(dev, crid); + } + + return (error); +} +#endif /* !WC_SKIP_INCLUDED_C_FILES && BSDKM_CRYPTO_REGISTER */ diff --git a/bsdkm/x86_vecreg.c b/bsdkm/x86_vecreg.c new file mode 100644 index 00000000000..d7444fff5d9 --- /dev/null +++ b/bsdkm/x86_vecreg.c @@ -0,0 +1,219 @@ +/* x86_vecreg.c -- logic to save and restore vector registers + * on amd64 in FreeBSD kernel. + * + * Copyright (C) 2006-2025 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* included by bsdkm/wolfkmod.c */ +#ifndef WC_SKIP_INCLUDED_C_FILES + +#include +#include +#include +#include + +struct wolfkmod_fpu_state_t { + volatile lwpid_t td_tid; + volatile u_int nest; +}; + +typedef struct wolfkmod_fpu_state_t wolfkmod_fpu_state_t; + +/* fpu_states array tracks thread id and nesting level of save/restore + * and push/pop vector registers macro calls. It is indexed by raw cpu id, + * and only accessed after the thread calls fpu_kern_enter(), and before + * calling fpu_kern_leave(), and only indexed by the thread's PCPU_GET(cpuid). + * + * after calling fpu_kern_enter(): + * - kernel fpu is enabled + * - migration is disabled + * - soft preempts are disabled + * Hard irq are still possible , but hard irq are forbidden from using FPU + * in FreeBSD kernel. + * */ +static wolfkmod_fpu_state_t * fpu_states = NULL; + +/* check for active td_tid with atomic before proceeding. + * technically not necessary because fpu_kern_enter() gives thread pinning + * to cpu, but just to be safe... + * */ +#define wolfkmod_fpu_get_tid() \ + atomic_load_acq_int(&fpu_states[PCPU_GET(cpuid)].td_tid) + +int wolfkmod_vecreg_init(void) +{ + if (mp_ncpus <= 0) { + printf("error: wolfkmod_vecreg_init: mp_ncpus = %d\n", mp_ncpus); + return (EINVAL); + } + + fpu_states = malloc(mp_ncpus * sizeof(wolfkmod_fpu_state_t), + M_WOLFSSL, M_WAITOK | M_ZERO); + if (fpu_states == NULL) { + printf("error: wolfkmod_vecreg_init: malloc(%lu) failed\n", + mp_ncpus * sizeof(wolfkmod_fpu_state_t)); + return (ENOMEM); + } + + return (0); +} + +void wolfkmod_vecreg_exit(void) +{ + int i = 0; + + if (fpu_states == NULL) { + return; + } + + for (i = 0; i < mp_ncpus; ++i) { + #if defined(WOLFSSL_BSDKM_FPU_DEBUG) + printf("info: wolfkmod_vecreg_exit: fpu_states[%d] = %d, %d\n", + i, fpu_states[i].nest, fpu_states[i].td_tid); + #endif /* WOLFSSL_BSDKM_FPU_DEBUG */ + + if (fpu_states[i].nest != 0 || fpu_states[i].td_tid != 0) { + /* Check for orphaned fpu state. There's nothing we can do + * but log the event and zero the nesting level. */ + printf("error: wolfkmod_vecreg_exit: fpu_states[%d] = %d, %d\n", + i, fpu_states[i].nest, fpu_states[i].td_tid); + fpu_states[i].nest = 0; + } + } + + free(fpu_states, M_WOLFSSL); + fpu_states = NULL; + + return; +} + +/* fpu_kern_enter() and fpu_kern_leave() wrapper defines. + * Build with WOLFSSL_BSDKM_FPU_DEBUG to see verbose FPU logging. + */ +#if defined(WOLFSSL_BSDKM_FPU_DEBUG) + #define wolfkmod_print_curthread(what) \ + printf("%s: cpuid = %d, curthread: td_tid = %d, pid = %d (%s), " \ + " td_critnest = %d, kernfpu = %02x\n", \ + (what), PCPU_GET(cpuid), curthread->td_tid, \ + curthread->td_proc ? curthread->td_proc->p_pid : -1, \ + curthread->td_proc ? curthread->td_proc->p_comm : "noproc", \ + curthread->td_critnest, \ + curthread->td_pcb->pcb_flags & PCB_KERNFPU); + + #define wolfkmod_fpu_kern_enter() \ + wolfkmod_print_curthread("fpu_kern_enter"); \ + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); + + #define wolfkmod_fpu_kern_leave() \ + wolfkmod_print_curthread("fpu_kern_leave"); \ + fpu_kern_leave(curthread, NULL); +#else + #define wolfkmod_fpu_kern_enter() \ + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); + + #define wolfkmod_fpu_kern_leave() \ + fpu_kern_leave(curthread, NULL); +#endif /* WOLFSSL_BSDKM_FPU_DEBUG */ + +int wolfkmod_vecreg_save(int flags_unused) +{ + (void)flags_unused; + + #if defined(WOLFSSL_BSDKM_FPU_DEBUG) + wolfkmod_print_curthread("wolfkmod_vecreg_save"); + #endif + + if (is_fpu_kern_thread(0)) { + /* kernel fpu threads are special, do nothing. They own a + * persistent, dedicated fpu context. */ + printf("info: wolfkmod_vecreg_save: is fpu kern thread\n"); + return (0); + } + + if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) { + /* fpu context already active. check td_tid and + * increment nesting level. */ + lwpid_t td_tid = wolfkmod_fpu_get_tid(); + if (td_tid != curthread->td_tid) { + printf("error: wolfkmod_vecreg_save: got tid = %d, expected %d\n", + td_tid, curthread->td_tid); + return (EINVAL); + } + fpu_states[PCPU_GET(cpuid)].nest++; + } + else { + /* fpu context not active, call fpu_kern_enter(). + * after calling fpu_kern_enter(): + * - kernel fpu is enabled + * - migration is disabled + * - soft preempts are disabled */ + lwpid_t td_tid = 0; + wolfkmod_fpu_kern_enter(); + td_tid = wolfkmod_fpu_get_tid(); + + if (fpu_states[PCPU_GET(cpuid)].nest != 0 || td_tid != 0) { + printf("error: wolfkmod_fpu_kern_enter() with nest: %d, %d\n", + fpu_states[PCPU_GET(cpuid)].nest, td_tid); + return (EINVAL); + } + + /* increment nest and save td_tid. */ + fpu_states[PCPU_GET(cpuid)].nest++; + fpu_states[PCPU_GET(cpuid)].td_tid = curthread->td_tid; + } + + return (0); +} + +void wolfkmod_vecreg_restore(void) +{ + #if defined(WOLFSSL_BSDKM_FPU_DEBUG) + wolfkmod_print_curthread("wolfkmod_vecreg_restore"); + #endif + + if (is_fpu_kern_thread(0)) { + /* kernel fpu threads are special, do nothing. They own a + * persistent, dedicated fpu context. */ + printf("info: wolfkmod_vecreg_restore: is fpu kern thread\n"); + return; + } + + if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) { + if (fpu_states[PCPU_GET(cpuid)].td_tid != curthread->td_tid) { + printf("error: wolfkmod_vecreg_restore: got tid = %d, expected %d\n", + fpu_states[PCPU_GET(cpuid)].td_tid, curthread->td_tid); + return; + } + + /* decrement the nesting level. */ + if (fpu_states[PCPU_GET(cpuid)].nest > 0) { + fpu_states[PCPU_GET(cpuid)].nest--; + } + + /* if last level, zero the thread id and call fpu_kern_leave */ + if (fpu_states[PCPU_GET(cpuid)].nest == 0) { + fpu_states[PCPU_GET(cpuid)].td_tid = 0; + wolfkmod_fpu_kern_leave(); + } + } + + return; +} + +#endif /* !WC_SKIP_INCLUDED_C_FILES */ diff --git a/configure.ac b/configure.ac index 0e987002833..d74402d4b67 100644 --- a/configure.ac +++ b/configure.ac @@ -123,9 +123,18 @@ then AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_EXPERIMENTAL_SETTINGS" fi +# Kernel module benchmark +ENABLED_KERNEL_BENCHMARKS="" +AC_ARG_ENABLE([kernel-benchmarks], + [AS_HELP_STRING([--enable-kernel-benchmarks],[Enable crypto benchmarking autorun at module load time for kernel module (default: disabled)])], + [ENABLED_KERNEL_BENCHMARKS=$enableval]) +if test "$ENABLED_KERNEL_BENCHMARKS" = "yes" +then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_KERNEL_BENCHMARKS" +fi +AC_SUBST([ENABLED_KERNEL_BENCHMARKS]) # Linux Kernel Module options (more options later) - AC_ARG_ENABLE([linuxkm], [AS_HELP_STRING([--enable-linuxkm],[Enable Linux Kernel Module (default: disabled)])], [ENABLED_LINUXKM=$enableval], @@ -145,6 +154,12 @@ AC_ARG_ENABLE([freebsdkm], [ENABLED_BSDKM=no] ) +AC_ARG_ENABLE([freebsdkm-crypto-register], + [AS_HELP_STRING([--enable-freebsdkm-crypto-register],[Register wolfCrypt implementations with the FreeBSD kernel opencrypto framework. (default: disabled)])], + [ENABLED_BSDKM_REGISTER=$enableval], + [ENABLED_BSDKM_REGISTER=no] + ) + AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stddef.h time.h sys/ioctl.h sys/socket.h sys/time.h errno.h sys/un.h ctype.h sys/random.h]) AC_CHECK_LIB([network],[socket]) AC_C_BIGENDIAN @@ -727,10 +742,8 @@ AC_SUBST([ENABLED_LINUXKM_PIE]) AC_ARG_ENABLE([linuxkm-benchmarks], [AS_HELP_STRING([--enable-linuxkm-benchmarks],[Enable crypto benchmarking autorun at module load time for Linux kernel module (default: disabled)])], - [ENABLED_KERNEL_BENCHMARKS=$enableval], - [ENABLED_KERNEL_BENCHMARKS=no] - ) -if test "$ENABLED_KERNEL_BENCHMARKS" = "yes" + [ENABLED_KERNEL_BENCHMARKS=$enableval]) +if test "$ENABLED_LINUXKM" = "yes" && test "$ENABLED_KERNEL_BENCHMARKS" = "yes" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_LINUXKM_BENCHMARKS" fi @@ -819,17 +832,20 @@ AC_ARG_WITH([bsd-export-syms], if test "x$ENABLED_BSDKM" = "xyes" then - # wolfcrypt only, no-asm supported for now. + #AX_SIMD_CC_COMPILER_FLAGS + # wolfcrypt only for now. HAVE_KERNEL_MODE=yes KERNEL_MODE_DEFAULTS=yes ENABLED_NO_LIBRARY=yes ENABLED_BENCHMARK=no - ENABLED_ASM=no + + # todo: remove, leaving for notes for now. + #ENABLED_ASM=no + #AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM" output_objdir="$(realpath "$output_objdir")/bsdkm" AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_BSDKM -DWC_SIPHASH_NO_ASM" - AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM" AM_CFLAGS="$AM_CFLAGS -DNO_DEV_RANDOM -DNO_WRITEV -DNO_STDIO_FILESYSTEM" AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_SOCK -DWOLFSSL_USER_IO" AM_CFLAGS="$AM_CFLAGS -DXMALLOC_OVERRIDE -DWOLFCRYPT_ONLY" @@ -848,6 +864,16 @@ then AC_SUBST([BSDKM_EXPORT_SYMS]) fi + +if test "x$ENABLED_BSDKM_REGISTER" = "xyes" +then + if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$enable_aesgcm_stream" != "no" && (test "$ENABLED_FIPS" = "no" || test $HAVE_FIPS_VERSION -ge 6); then + ENABLED_AESGCM_STREAM=yes + fi + + AM_CFLAGS="$AM_CFLAGS -DBSDKM_CRYPTO_REGISTER" + AC_SUBST([ENABLED_BSDKM_REGISTER]) +fi # end FreeBSD configure # MATH LIBRARY SELECTION diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 3e1dcf050fd..0b16b665386 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -1995,11 +1995,15 @@ static const char* bench_result_words3[][5] = { #pragma warning(disable: 4996) #endif -#ifdef WOLFSSL_CURRTIME_REMAP - #define current_time WOLFSSL_CURRTIME_REMAP +#if defined(WOLFSSL_BSDKM) + int64_t current_time(int reset); #else - double current_time(int reset); -#endif + #ifdef WOLFSSL_CURRTIME_REMAP + #define current_time WOLFSSL_CURRTIME_REMAP + #else + double current_time(int reset); + #endif +#endif /* WOLFSSL_BSDKM */ #ifdef LINUX_RUSAGE_UTIME static void check_for_excessive_stime(const char *algo, @@ -2683,9 +2687,9 @@ static WC_INLINE void bench_stats_start(int* count, double* start) #endif } -#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) #define bench_stats_start(count, start) do { \ - SAVE_VECTOR_REGISTERS(pr_err( \ + SAVE_VECTOR_REGISTERS(WOLFSSL_DEBUG_PRINTF( \ "ERROR: SAVE_VECTOR_REGISTERS failed for benchmark run."); \ return; ); \ bench_stats_start(count, start); \ @@ -3161,7 +3165,7 @@ static void bench_stats_sym_finish(const char* desc, int useDeviceID, (void)useDeviceID; (void)ret; -#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) RESTORE_VECTOR_REGISTERS(); #elif defined(WOLFSSL_LINUXKM) kernel_fpu_end(); @@ -3559,7 +3563,7 @@ static void bench_stats_asym_finish_ex(const char* algo, int strength, (void)useDeviceID; (void)ret; -#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) RESTORE_VECTOR_REGISTERS(); #elif defined(WOLFSSL_LINUXKM) kernel_fpu_end(); @@ -16024,6 +16028,20 @@ void bench_sphincsKeySign(byte level, byte optim) return (double)ns / 1000000000.0; } +#elif defined(WOLFSSL_BSDKM) + + #include + int64_t current_time(int reset) + { + (void)reset; + struct timespec ts; + int64_t result = 0; + + getnanouptime(&ts); + result = (int64_t) ts.tv_sec + (int64_t) ts.tv_nsec / NANOSECOND; + return result; + } + #elif defined(WOLFSSL_GAISLER_BCC) #include diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index a1d09be4c29..c2daec56130 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -7466,6 +7466,7 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) #if defined(WOLFSSL_AESNI) && defined(GCM_TABLE_4BIT) if (aes->use_aesni) { + VECTOR_REGISTERS_PUSH; #if defined(WC_C_DYNAMIC_FALLBACK) #ifdef HAVE_INTEL_AVX2 if (IS_INTEL_AVX2(intel_flags)) { @@ -7483,6 +7484,7 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) GCM_generate_m0_aesni(aes->gcm.H, (byte*)aes->gcm.M0); } #endif + VECTOR_REGISTERS_POP; } else #endif diff --git a/wolfcrypt/src/cpuid.c b/wolfcrypt/src/cpuid.c index 5c3e333ffee..9d9b458e8a9 100644 --- a/wolfcrypt/src/cpuid.c +++ b/wolfcrypt/src/cpuid.c @@ -113,7 +113,11 @@ static WC_INLINE void cpuid_set_flags(void) { + #ifdef WOLFSSL_BSDKM + if (WOLFSSL_ATOMIC_LOAD_UINT(cpuid_flags) == WC_CPUID_INITIALIZER) { + #else if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) { + #endif cpuid_flags_t new_cpuid_flags = 0, old_cpuid_flags = WC_CPUID_INITIALIZER; if (cpuid_flag(1, 0, ECX, 28)) { new_cpuid_flags |= CPUID_AVX1 ; } diff --git a/wolfssl/wolfcrypt/cpuid.h b/wolfssl/wolfcrypt/cpuid.h index 176f99f2922..38b64e8fa7e 100644 --- a/wolfssl/wolfcrypt/cpuid.h +++ b/wolfssl/wolfcrypt/cpuid.h @@ -122,7 +122,11 @@ typedef word32 cpuid_flags_t; * accurate. */ static WC_INLINE int cpuid_get_flags_atomic(cpuid_flags_atomic_t *flags) { + #ifdef WOLFSSL_BSDKM + if (WOLFSSL_ATOMIC_LOAD_UINT(*flags) == WC_CPUID_INITIALIZER) { + #else if (WOLFSSL_ATOMIC_LOAD(*flags) == WC_CPUID_INITIALIZER) { + #endif /* WOLFSSL_BSDKM */ cpuid_flags_t old_cpuid_flags = WC_CPUID_INITIALIZER; return wolfSSL_Atomic_Uint_CompareExchange (flags, &old_cpuid_flags, cpuid_get_flags()); From 66c2fb76ef6f5647320f50e3e9a6d75a8e8aec91 Mon Sep 17 00:00:00 2001 From: jordan Date: Tue, 27 Jan 2026 11:36:34 -0600 Subject: [PATCH 2/4] bsdkm: cleanup. --- .wolfssl_known_macro_extras | 1 + Makefile.am | 2 +- bsdkm/Makefile | 55 +++++++++++++++++++++------------ bsdkm/bsdkm_wc_port.h | 1 - configure.ac | 2 ++ wolfcrypt/benchmark/benchmark.c | 16 ++++------ 6 files changed, 46 insertions(+), 31 deletions(-) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 0a681bd403a..3a45f971e31 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -36,6 +36,7 @@ BLAKE2B_SELFTEST BLAKE2S_SELFTEST BLOCKING BSDKM_EXPORT_SYMS +ENABLED_BSDKM_REGISTER BSP_DEFAULT_IO_CHANNEL_DEFINED BSP_LED_0 BSP_LED_1 diff --git a/Makefile.am b/Makefile.am index 67c5a4571ca..7ae6d99fdec 100644 --- a/Makefile.am +++ b/Makefile.am @@ -248,7 +248,7 @@ if BUILD_BSDKM AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \ AM_CCASFLAGS CCASFLAGS \ src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_BSDKM_REGISTER \ - ENABLED_ASM ENABLED_AESNI \ + ENABLED_ASM ENABLED_INTELASM ENABLED_AESNI ENABLED_AESNI_WITH_AVX \ ENABLED_KERNEL_BENCHMARKS endif diff --git a/bsdkm/Makefile b/bsdkm/Makefile index eeeaecab5d0..5a191080f70 100644 --- a/bsdkm/Makefile +++ b/bsdkm/Makefile @@ -1,11 +1,10 @@ -# wolfssl kernel module name and source, and root dir. +# wolfssl kernel module name and main source, and wolfssl root dir. KMOD = libwolfssl SRCS = wolfkmod.c WOLFSSL_DIR = ../ -CFLAGS+=-I${WOLFSSL_DIR} -CFLAGS+=-DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER - +CFLAGS += -I${WOLFSSL_DIR} +CFLAGS += -DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER # # debug options # verbose printing: @@ -14,6 +13,9 @@ CFLAGS+=-DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER # print memory mallocs / frees: # CFLAGS += -DWOLFSSL_BSDKM_MEMORY_DEBUG # +# print fpu_kern_enter / leave: +# CFLAGS += WOLFSSL_BSDKM_FPU_DEBUG +# CFLAGS += $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) .if defined(ENABLED_BSDKM_REGISTER) @@ -37,7 +39,7 @@ WOLFSSL_OBJS != echo ${src_libwolfssl_la_OBJECTS} | \ # wolfcrypt benchmark .if ${ENABLED_KERNEL_BENCHMARKS} == "yes" WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/benchmark/benchmark.o - CFLAGS+=-DWOLFSSL_NO_FLOAT_FMT + CFLAGS += -DWOLFSSL_NO_FLOAT_FMT .endif OBJS += ${WOLFSSL_OBJS} @@ -58,27 +60,42 @@ OBJS += ${WOLFSSL_OBJS} .endif .include "${SYSDIR}/conf/kmod.mk" -# To use aesni in FreeBSD kernel we need to adjust build flags. -# See these for reference: +# +# To use aesni and friends in FreeBSD kernel we need to adjust build flags. +# See these kernel makefiles for reference: # - /usr/src/sys/modules/aesni/Makefile # - /usr/src/sys/conf/kern.mk -.if ${ENABLED_ASM} == "yes" - CFLAGS.aes.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 +# +WOLFKMOD_SIMD_BASE = -msse -msse2 -msse4.1 +WOLFKMOD_SIMD_AES = -maes -mpclmul +WOLFKMOD_SIMD_AVX = -mavx -mavx2 + +.if ${ENABLED_AESNI} == "yes" + CFLAGS.aes.c += ${WOLFKMOD_SIMD_BASE} + CFLAGS.aes.c += ${WOLFKMOD_SIMD_AES} +.if ${ENABLED_AESNI_WITH_AVX} == "yes" + CFLAGS.aes.c += ${WOLFKMOD_SIMD_AVX} +.endif # ENABLED_AESNI_WITH_AVX # CFLAGS.aes.c := ${CFLAGS.aes.c:N-nostdinc} CFLAGS.aes.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers +.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers +.endif # ENABLED_AESNI - CFLAGS.poly1305.c += -msse -msse2 -msse4.1 -maes -mpclmul - CFLAGS.poly1305.c := ${CFLAGS.aes.c:N-nostdinc} - CFLAGS.poly1305.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers +.if ${ENABLED_ASM} == "yes" +.for f in chacha dilithium poly1305 sha sha256 sha3 sha512 + CFLAGS.${f}.c += ${WOLFKMOD_SIMD_BASE} + CFLAGS.${f}.c += ${WOLFKMOD_SIMD_AVX} + CFLAGS.${f}.c := ${CFLAGS.${f}.c:N-nostdinc} + CFLAGS.${f}.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers +.endfor - CFLAGS.chacha.c += -msse -msse2 -msse4.1 -maes -mpclmul - CFLAGS.chacha.c := ${CFLAGS.aes.c:N-nostdinc} - CFLAGS.chacha.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers +.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers +.endif # ENABLED_ASM == "yes" - CFLAGS.sha.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 - CFLAGS.sha256.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 - CFLAGS.benchmark.c += -msse -msse2 -msse4.1 -maes -mpclmul -mavx -mavx2 - CFLAGS.benchmark.c := ${CFLAGS.aes.c:N-nostdinc} +# wolfcrypt benchmark always needs simd for the floating point timings. +.if ${ENABLED_KERNEL_BENCHMARKS} == "yes" + CFLAGS.benchmark.c += ${WOLFKMOD_SIMD_BASE} + CFLAGS.benchmark.c := ${CFLAGS.benchmark.c:N-nostdinc} CFLAGS.benchmark.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers .PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers .endif diff --git a/bsdkm/bsdkm_wc_port.h b/bsdkm/bsdkm_wc_port.h index 73d442df68e..0bd80f092b0 100644 --- a/bsdkm/bsdkm_wc_port.h +++ b/bsdkm/bsdkm_wc_port.h @@ -117,7 +117,6 @@ extern struct malloc_type M_WOLFSSL[1]; #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS #endif - #define SAVE_VECTOR_REGISTERS(fail_clause) { \ int _svr_ret = wolfkmod_vecreg_save(0); \ if (_svr_ret != 0) { \ diff --git a/configure.ac b/configure.ac index d74402d4b67..0f51bb8a154 100644 --- a/configure.ac +++ b/configure.ac @@ -3995,6 +3995,8 @@ then ENABLED_X86_ASM=yes fi fi +AC_SUBST([ENABLED_AESNI]) +AC_SUBST([ENABLED_AESNI_WITH_AVX]) AC_ARG_ENABLE([aligndata], [AS_HELP_STRING([--enable-aligndata],[align data for ciphers (default: enabled)])], diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 0b16b665386..91d0c4c4aed 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -1995,15 +1995,11 @@ static const char* bench_result_words3[][5] = { #pragma warning(disable: 4996) #endif -#if defined(WOLFSSL_BSDKM) - int64_t current_time(int reset); +#ifdef WOLFSSL_CURRTIME_REMAP + #define current_time WOLFSSL_CURRTIME_REMAP #else - #ifdef WOLFSSL_CURRTIME_REMAP - #define current_time WOLFSSL_CURRTIME_REMAP - #else - double current_time(int reset); - #endif -#endif /* WOLFSSL_BSDKM */ + double current_time(int reset); +#endif #ifdef LINUX_RUSAGE_UTIME static void check_for_excessive_stime(const char *algo, @@ -16031,7 +16027,7 @@ void bench_sphincsKeySign(byte level, byte optim) #elif defined(WOLFSSL_BSDKM) #include - int64_t current_time(int reset) + double current_time(int reset) { (void)reset; struct timespec ts; @@ -16039,7 +16035,7 @@ void bench_sphincsKeySign(byte level, byte optim) getnanouptime(&ts); result = (int64_t) ts.tv_sec + (int64_t) ts.tv_nsec / NANOSECOND; - return result; + return (double)result; } #elif defined(WOLFSSL_GAISLER_BCC) From 10bd57ca217ad4b446b678d614b00072865857a8 Mon Sep 17 00:00:00 2001 From: jordan Date: Tue, 27 Jan 2026 23:44:20 -0600 Subject: [PATCH 3/4] bsdkm: fix exit error handling. --- bsdkm/README.md | 2 +- bsdkm/wolfkmod.c | 29 ++++++++++++----------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/bsdkm/README.md b/bsdkm/README.md index b84c2588ff7..c9492e37db0 100644 --- a/bsdkm/README.md +++ b/bsdkm/README.md @@ -7,9 +7,9 @@ other loadable modules to link to wolfCrypt. Supported features: - wolfCrypt in kernel. - FIPS-wolfcrypt. +- crypto acceleration: AES-NI, AVX, etc. Planned features: -- crypto acceleration: AES-NI, AVX, etc. - kernel opencrypto driver registration. - full wolfSSL in kernel (kernel TLS). diff --git a/bsdkm/wolfkmod.c b/bsdkm/wolfkmod.c index f8e5f4d54b2..f477954efe6 100644 --- a/bsdkm/wolfkmod.c +++ b/bsdkm/wolfkmod.c @@ -215,14 +215,16 @@ static int wolfkmod_cleanup(void) if (error != 0) { printf("error: wolfCrypt_Cleanup failed: %s\n", wc_GetErrorString(error)); - return (ECANCELED); + error = ECANCELED; + goto wolfkmod_cleanup_out; } #else error = wolfSSL_Cleanup(); if (error != WOLFSSL_SUCCESS) { printf("error: wolfSSL_Cleanup failed: %s\n", wc_GetErrorString(error)); - return (ECANCELED); + error = ECANCELED; + goto wolfkmod_cleanup_out; } #endif /* WOLFCRYPT_ONLY */ @@ -230,12 +232,14 @@ static int wolfkmod_cleanup(void) printf("info: libwolfssl " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n"); #endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */ + error = 0; +wolfkmod_cleanup_out: #if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS) wolfkmod_vecreg_exit(); #endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/ - return (0); + return (error); } #if !defined(BSDKM_CRYPTO_REGISTER) @@ -554,9 +558,9 @@ static int wolfkdriv_probesession(device_t dev, return (error); } -static int wolfkdriv_newsession_cipher(device_t dev, - wolfkdriv_session_t * session, - const struct crypto_session_params *csp) +static int wolfkdriv_newsession_aes(device_t dev, + wolfkdriv_session_t * session, + const struct crypto_session_params *csp) { int error = 0; int klen = csp->csp_cipher_klen; /* key len in bytes */ @@ -624,7 +628,7 @@ static int wolfkdriv_newsession(device_t dev, crypto_session_t cses, break; case CSP_MODE_CIPHER: case CSP_MODE_AEAD: - error = wolfkdriv_newsession_cipher(dev, session, csp); + error = wolfkdriv_newsession_aes(dev, session, csp); break; default: __assert_unreachable(); @@ -638,9 +642,6 @@ static int wolfkdriv_newsession(device_t dev, crypto_session_t cses, return (error); } -/* - * - */ static void wolfkdriv_freesession(device_t dev, crypto_session_t cses) { @@ -659,9 +660,6 @@ wolfkdriv_freesession(device_t dev, crypto_session_t cses) return; } -/* - * - */ static int wolfkdriv_cbc_work(device_t dev, wolfkdriv_session_t * session, struct cryptop * crp, const struct crypto_session_params * csp) @@ -812,9 +810,6 @@ static int wolfkdriv_cbc_work(device_t dev, wolfkdriv_session_t * session, return (error); } -/* - * todo: skeleton implementation, finish. - */ static int wolfkdriv_gcm_work(device_t dev, wolfkdriv_session_t * session, struct cryptop * crp, const struct crypto_session_params * csp) @@ -1056,7 +1051,7 @@ static driver_t wolfkdriv_driver = { .size = sizeof(struct wolfkdriv_softc), }; -/* note: on x86, software-only drivers usually attach to nexus bus. */ +/* on x86, software-only drivers usually attach to nexus bus. */ DRIVER_MODULE(libwolfssl, nexus, wolfkdriv_driver, NULL, NULL); #endif /* BSDKM_CRYPTO_REGISTER */ From 1ef0761f272f104f62e1f6a7ba50937066dc69a0 Mon Sep 17 00:00:00 2001 From: jordan Date: Wed, 28 Jan 2026 11:22:15 -0600 Subject: [PATCH 4/4] bsdkm: update readme, cleanup overlong lines. --- bsdkm/README.md | 12 +++++++----- bsdkm/x86_vecreg.c | 18 ++++++++++++------ configure.ac | 8 +------- wolfcrypt/benchmark/benchmark.c | 2 +- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/bsdkm/README.md b/bsdkm/README.md index c9492e37db0..0c23ceb2517 100644 --- a/bsdkm/README.md +++ b/bsdkm/README.md @@ -10,7 +10,7 @@ Supported features: - crypto acceleration: AES-NI, AVX, etc. Planned features: -- kernel opencrypto driver registration. +- kernel opencrypto driver registration (supported for internal testing presently). - full wolfSSL in kernel (kernel TLS). ## Building and Installing @@ -44,10 +44,12 @@ sudo kldunload libwolfssl ### options -| freebsdkm option | description | -| :------------------------------- | :--------------------------------------- | -| --with-bsd-export-syms=LIST | Export list of symbols as global.
. Options are 'all', 'none', or
comma separated list of symbols. | -| --with-kernel-source=PATH | Path to kernel tree root (default `/usr/src/sys`) | +| freebsdkm option | description | +| :--------------------------------- | :--------------------------------------- | +| --with-bsd-export-syms=LIST | Export list of symbols as global.
. Options are 'all', 'none', or
comma separated list of symbols. | +| --with-kernel-source=PATH | Path to kernel tree root (default `/usr/src/sys`) | +| --enable-kernel-benchmarks | Run wolfcrypt benchmark at module load | +| --enable-freebsdkm-crypto-register | Register with the FreeBSD kernel opencrypto
framework (preliminary, for testing) | ### FIPS diff --git a/bsdkm/x86_vecreg.c b/bsdkm/x86_vecreg.c index d7444fff5d9..90febe360a3 100644 --- a/bsdkm/x86_vecreg.c +++ b/bsdkm/x86_vecreg.c @@ -142,12 +142,14 @@ int wolfkmod_vecreg_save(int flags_unused) if (is_fpu_kern_thread(0)) { /* kernel fpu threads are special, do nothing. They own a * persistent, dedicated fpu context. */ + #if defined(WOLFSSL_BSDKM_FPU_DEBUG) printf("info: wolfkmod_vecreg_save: is fpu kern thread\n"); + #endif return (0); } if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) { - /* fpu context already active. check td_tid and + /* kern fpu is active for this thread. check td_tid and * increment nesting level. */ lwpid_t td_tid = wolfkmod_fpu_get_tid(); if (td_tid != curthread->td_tid) { @@ -158,7 +160,7 @@ int wolfkmod_vecreg_save(int flags_unused) fpu_states[PCPU_GET(cpuid)].nest++; } else { - /* fpu context not active, call fpu_kern_enter(). + /* kern fpu not active for this thread, call fpu_kern_enter(). * after calling fpu_kern_enter(): * - kernel fpu is enabled * - migration is disabled @@ -190,14 +192,18 @@ void wolfkmod_vecreg_restore(void) if (is_fpu_kern_thread(0)) { /* kernel fpu threads are special, do nothing. They own a * persistent, dedicated fpu context. */ + #if defined(WOLFSSL_BSDKM_FPU_DEBUG) printf("info: wolfkmod_vecreg_restore: is fpu kern thread\n"); + #endif return; } if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) { - if (fpu_states[PCPU_GET(cpuid)].td_tid != curthread->td_tid) { - printf("error: wolfkmod_vecreg_restore: got tid = %d, expected %d\n", - fpu_states[PCPU_GET(cpuid)].td_tid, curthread->td_tid); + /* kern fpu is active for this thread. check tid and nesting level. */ + lwpid_t td_tid = wolfkmod_fpu_get_tid(); + if (td_tid != curthread->td_tid) { + printf("error: wolfkmod_vecreg_restore: got tid = %d, " + "expected %d\n", td_tid, curthread->td_tid); return; } @@ -206,7 +212,7 @@ void wolfkmod_vecreg_restore(void) fpu_states[PCPU_GET(cpuid)].nest--; } - /* if last level, zero the thread id and call fpu_kern_leave */ + /* if last level, zero the thread id then call fpu_kern_leave */ if (fpu_states[PCPU_GET(cpuid)].nest == 0) { fpu_states[PCPU_GET(cpuid)].td_tid = 0; wolfkmod_fpu_kern_leave(); diff --git a/configure.ac b/configure.ac index 0f51bb8a154..012b6c232af 100644 --- a/configure.ac +++ b/configure.ac @@ -832,17 +832,12 @@ AC_ARG_WITH([bsd-export-syms], if test "x$ENABLED_BSDKM" = "xyes" then - #AX_SIMD_CC_COMPILER_FLAGS - # wolfcrypt only for now. + # note: bsdkm is wolfcrypt only for now. HAVE_KERNEL_MODE=yes KERNEL_MODE_DEFAULTS=yes ENABLED_NO_LIBRARY=yes ENABLED_BENCHMARK=no - # todo: remove, leaving for notes for now. - #ENABLED_ASM=no - #AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM" - output_objdir="$(realpath "$output_objdir")/bsdkm" AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_BSDKM -DWC_SIPHASH_NO_ASM" @@ -862,7 +857,6 @@ then fi AC_SUBST([KERNEL_ROOT]) AC_SUBST([BSDKM_EXPORT_SYMS]) - fi if test "x$ENABLED_BSDKM_REGISTER" = "xyes" diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 91d0c4c4aed..f6e2615e99d 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -2685,7 +2685,7 @@ static WC_INLINE void bench_stats_start(int* count, double* start) #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) #define bench_stats_start(count, start) do { \ - SAVE_VECTOR_REGISTERS(WOLFSSL_DEBUG_PRINTF( \ + SAVE_VECTOR_REGISTERS(WOLFSSL_DEBUG_PRINTF( \ "ERROR: SAVE_VECTOR_REGISTERS failed for benchmark run."); \ return; ); \ bench_stats_start(count, start); \