| version 1.1.2.1, 2015/06/03 04:20:02 |
version 1.1.2.2, 2015/06/06 14:40:08 |
|
|
| |
/* $NetBSD$ */ |
| |
/* |
| |
* Copyright (c) 2015 Internet Initiative Japan Inc. |
| |
* All rights reserved. |
| |
* |
| |
* Redistribution and use in source and binary forms, with or without |
| |
* modification, are permitted provided that the following conditions |
| |
* are met: |
| |
* 1. Redistributions of source code must retain the above copyright |
| |
* notice, this list of conditions and the following disclaimer. |
| |
* 2. Redistributions in binary form must reproduce the above copyright |
| |
* notice, this list of conditions and the following disclaimer in the |
| |
* documentation and/or other materials provided with the distribution. |
| |
* |
| |
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| |
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| |
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| |
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, |
| |
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| |
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| |
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| |
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
| |
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
| |
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| |
* POSSIBILITY OF SUCH DAMAGE. |
| |
*/ |
| |
/* |
| |
* Cryptographic Engine and Security Accelerator(MVXPSEC) |
| |
*/ |
| |
#include <sys/cdefs.h> |
| |
#include <sys/param.h> |
| |
#include <sys/types.h> |
| |
#include <sys/kernel.h> |
| |
#include <sys/queue.h> |
| |
#include <sys/conf.h> |
| |
#include <sys/proc.h> |
| |
#include <sys/bus.h> |
| |
#include <sys/evcnt.h> |
| |
#include <sys/device.h> |
| |
#include <sys/endian.h> |
| |
#include <sys/errno.h> |
| |
#include <sys/kmem.h> |
| |
#include <sys/mbuf.h> |
| |
#include <sys/callout.h> |
| |
#include <sys/pool.h> |
| |
#include <sys/cprng.h> |
| |
#include <sys/syslog.h> |
| |
#include <sys/mutex.h> |
| |
#include <sys/kthread.h> |
| |
#include <sys/atomic.h> |
| |
#include <sys/sha1.h> |
| |
#include <sys/md5.h> |
| |
|
| |
#include <uvm/uvm_extern.h> |
| |
|
| |
#include <crypto/rijndael/rijndael.h> |
| |
|
| |
#include <opencrypto/cryptodev.h> |
| |
#include <opencrypto/xform.h> |
| |
|
| |
#include <net/net_stats.h> |
| |
|
| |
#include <netinet/in_systm.h> |
| |
#include <netinet/in.h> |
| |
#include <netinet/ip.h> |
| |
#include <netinet/ip6.h> |
| |
|
| |
#include <netipsec/esp_var.h> |
| |
|
| |
#include <arm/cpufunc.h> |
| |
#include <arm/marvell/mvsocvar.h> |
| |
#include <arm/marvell/armadaxpreg.h> |
| |
#include <dev/marvell/marvellreg.h> |
| |
#include <dev/marvell/marvellvar.h> |
| |
#include <dev/marvell/mvxpsecreg.h> |
| |
#include <dev/marvell/mvxpsecvar.h> |
| |
|
| |
#ifdef DEBUG |
| |
#define STATIC __attribute__ ((noinline)) extern |
| |
#define _STATIC __attribute__ ((noinline)) extern |
| |
#define INLINE __attribute__ ((noinline)) extern |
| |
#define _INLINE __attribute__ ((noinline)) extern |
| |
#else |
| |
#define STATIC static |
| |
#define _STATIC __attribute__ ((unused)) static |
| |
#define INLINE static inline |
| |
#define _INLINE __attribute__ ((unused)) static inline |
| |
#endif |
| |
|
| |
/* |
| |
* IRQ and SRAM spaces for each of unit |
| |
* XXX: move to attach_args |
| |
*/ |
| |
struct { |
| |
int err_int; |
| |
} mvxpsec_config[] = { |
| |
{ .err_int = ARMADAXP_IRQ_CESA0_ERR, }, /* unit 0 */ |
| |
{ .err_int = ARMADAXP_IRQ_CESA1_ERR, }, /* unit 1 */ |
| |
}; |
| |
#define MVXPSEC_ERR_INT(sc) \ |
| |
mvxpsec_config[device_unit((sc)->sc_dev)].err_int |
| |
|
| |
/* |
| |
* AES |
| |
*/ |
| |
#define MAXBC (128/32) |
| |
#define MAXKC (256/32) |
| |
#define MAXROUNDS 14 |
| |
STATIC int mv_aes_ksched(uint8_t[4][MAXKC], int, |
| |
uint8_t[MAXROUNDS+1][4][MAXBC]); |
| |
STATIC int mv_aes_deckey(uint8_t *, uint8_t *, int); |
| |
|
| |
/* |
| |
* device driver autoconf interface |
| |
*/ |
| |
STATIC int mvxpsec_match(device_t, cfdata_t, void *); |
| |
STATIC void mvxpsec_attach(device_t, device_t, void *); |
| |
STATIC void mvxpsec_evcnt_attach(struct mvxpsec_softc *); |
| |
|
| |
/* |
| |
* register setup |
| |
*/ |
| |
STATIC int mvxpsec_wininit(struct mvxpsec_softc *, enum marvell_tags *); |
| |
|
| |
/* |
| |
* timer(callout) interface |
| |
* |
| |
* XXX: callout is not MP safe... |
| |
*/ |
| |
STATIC void mvxpsec_timer(void *); |
| |
|
| |
/* |
| |
* interrupt interface |
| |
*/ |
| |
STATIC int mvxpsec_intr(void *); |
| |
INLINE void mvxpsec_intr_cleanup(struct mvxpsec_softc *); |
| |
STATIC int mvxpsec_eintr(void *); |
| |
STATIC uint32_t mvxpsec_intr_ack(struct mvxpsec_softc *); |
| |
STATIC uint32_t mvxpsec_eintr_ack(struct mvxpsec_softc *); |
| |
INLINE void mvxpsec_intr_cnt(struct mvxpsec_softc *, int); |
| |
|
| |
/* |
| |
* memory allocators and VM management |
| |
*/ |
| |
STATIC struct mvxpsec_devmem *mvxpsec_alloc_devmem(struct mvxpsec_softc *, |
| |
paddr_t, int); |
| |
STATIC int mvxpsec_init_sram(struct mvxpsec_softc *); |
| |
|
| |
/* |
| |
* Low-level DMA interface |
| |
*/ |
| |
STATIC int mvxpsec_init_dma(struct mvxpsec_softc *, |
| |
struct marvell_attach_args *); |
| |
INLINE int mvxpsec_dma_wait(struct mvxpsec_softc *); |
| |
INLINE int mvxpsec_acc_wait(struct mvxpsec_softc *); |
| |
INLINE struct mvxpsec_descriptor_handle *mvxpsec_dma_getdesc(struct mvxpsec_softc *); |
| |
_INLINE void mvxpsec_dma_putdesc(struct mvxpsec_softc *, struct mvxpsec_descriptor_handle *); |
| |
INLINE void mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *, |
| |
uint32_t, uint32_t, uint32_t); |
| |
INLINE void mvxpsec_dma_cat(struct mvxpsec_softc *, |
| |
struct mvxpsec_descriptor_handle *, struct mvxpsec_descriptor_handle *); |
| |
|
| |
/* |
| |
* High-level DMA interface |
| |
*/ |
| |
INLINE int mvxpsec_dma_copy0(struct mvxpsec_softc *, |
| |
mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t); |
| |
INLINE int mvxpsec_dma_copy(struct mvxpsec_softc *, |
| |
mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t); |
| |
INLINE int mvxpsec_dma_acc_activate(struct mvxpsec_softc *, |
| |
mvxpsec_dma_ring *); |
| |
INLINE void mvxpsec_dma_finalize(struct mvxpsec_softc *, |
| |
mvxpsec_dma_ring *); |
| |
INLINE void mvxpsec_dma_free(struct mvxpsec_softc *, |
| |
mvxpsec_dma_ring *); |
| |
INLINE int mvxpsec_dma_copy_packet(struct mvxpsec_softc *, struct mvxpsec_packet *); |
| |
INLINE int mvxpsec_dma_sync_packet(struct mvxpsec_softc *, struct mvxpsec_packet *); |
| |
|
| |
/* |
| |
* Session management interface (OpenCrypto) |
| |
*/ |
| |
#define MVXPSEC_SESSION(sid) ((sid) & 0x0fffffff) |
| |
#define MVXPSEC_SID(crd, sesn) (((crd) << 28) | ((sesn) & 0x0fffffff)) |
| |
/* pool management */ |
| |
STATIC int mvxpsec_session_ctor(void *, void *, int); |
| |
STATIC void mvxpsec_session_dtor(void *, void *); |
| |
STATIC int mvxpsec_packet_ctor(void *, void *, int); |
| |
STATIC void mvxpsec_packet_dtor(void *, void *); |
| |
|
| |
/* session management */ |
| |
STATIC struct mvxpsec_session *mvxpsec_session_alloc(struct mvxpsec_softc *); |
| |
STATIC void mvxpsec_session_dealloc(struct mvxpsec_session *); |
| |
INLINE struct mvxpsec_session *mvxpsec_session_lookup(struct mvxpsec_softc *, int); |
| |
INLINE int mvxpsec_session_ref(struct mvxpsec_session *); |
| |
INLINE void mvxpsec_session_unref(struct mvxpsec_session *); |
| |
|
| |
/* packet management */ |
| |
STATIC struct mvxpsec_packet *mvxpsec_packet_alloc(struct mvxpsec_session *); |
| |
INLINE void mvxpsec_packet_enqueue(struct mvxpsec_packet *); |
| |
STATIC void mvxpsec_packet_dealloc(struct mvxpsec_packet *); |
| |
STATIC int mvxpsec_done_packet(struct mvxpsec_packet *); |
| |
|
| |
/* session header manegement */ |
| |
STATIC int mvxpsec_header_finalize(struct mvxpsec_packet *); |
| |
|
| |
/* packet queue management */ |
| |
INLINE void mvxpsec_drop(struct mvxpsec_softc *, struct cryptop *, struct mvxpsec_packet *, int); |
| |
STATIC int mvxpsec_dispatch_queue(struct mvxpsec_softc *); |
| |
|
| |
/* opencrypto opration */ |
| |
INLINE int mvxpsec_parse_crd(struct mvxpsec_packet *, struct cryptodesc *); |
| |
INLINE int mvxpsec_parse_crp(struct mvxpsec_packet *); |
| |
|
| |
/* payload data management */ |
| |
INLINE int mvxpsec_packet_setcrp(struct mvxpsec_packet *, struct cryptop *); |
| |
STATIC int mvxpsec_packet_setdata(struct mvxpsec_packet *, void *, uint32_t); |
| |
STATIC int mvxpsec_packet_setmbuf(struct mvxpsec_packet *, struct mbuf *); |
| |
STATIC int mvxpsec_packet_setuio(struct mvxpsec_packet *, struct uio *); |
| |
STATIC int mvxpsec_packet_rdata(struct mvxpsec_packet *, int, int, void *); |
| |
_STATIC int mvxpsec_packet_wdata(struct mvxpsec_packet *, int, int, void *); |
| |
STATIC int mvxpsec_packet_write_iv(struct mvxpsec_packet *, void *, int); |
| |
STATIC int mvxpsec_packet_copy_iv(struct mvxpsec_packet *, int, int); |
| |
|
| |
/* key pre-computation */ |
| |
STATIC int mvxpsec_key_precomp(int, void *, int, void *, void *); |
| |
STATIC int mvxpsec_hmac_precomp(int, void *, int, void *, void *); |
| |
|
| |
/* crypto operation management */ |
| |
INLINE void mvxpsec_packet_reset_op(struct mvxpsec_packet *); |
| |
INLINE void mvxpsec_packet_update_op_order(struct mvxpsec_packet *, int); |
| |
|
| |
/* |
| |
* parameter converters |
| |
*/ |
| |
INLINE uint32_t mvxpsec_alg2acc(uint32_t alg); |
| |
INLINE uint32_t mvxpsec_aesklen(int klen); |
| |
|
| |
/* |
| |
* string formatters |
| |
*/ |
| |
_STATIC const char *s_ctrlreg(uint32_t); |
| |
_STATIC const char *s_winreg(uint32_t); |
| |
_STATIC const char *s_errreg(uint32_t); |
| |
_STATIC const char *s_xpsecintr(uint32_t); |
| |
_STATIC const char *s_ctlalg(uint32_t); |
| |
_STATIC const char *s_xpsec_op(uint32_t); |
| |
_STATIC const char *s_xpsec_enc(uint32_t); |
| |
_STATIC const char *s_xpsec_mac(uint32_t); |
| |
_STATIC const char *s_xpsec_frag(uint32_t); |
| |
|
| |
/* |
| |
* debugging supports |
| |
*/ |
| |
#ifdef MVXPSEC_DEBUG |
| |
_STATIC void mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *); |
| |
_STATIC void mvxpsec_dump_reg(struct mvxpsec_softc *); |
| |
_STATIC void mvxpsec_dump_sram(const char *, struct mvxpsec_softc *, size_t); |
| |
_STATIC void mvxpsec_dump_data(const char *, void *, size_t); |
| |
|
| |
_STATIC void mvxpsec_dump_packet(const char *, struct mvxpsec_packet *); |
| |
_STATIC void mvxpsec_dump_packet_data(const char *, struct mvxpsec_packet *); |
| |
_STATIC void mvxpsec_dump_packet_desc(const char *, struct mvxpsec_packet *); |
| |
|
| |
_STATIC void mvxpsec_dump_acc_config(const char *, uint32_t); |
| |
_STATIC void mvxpsec_dump_acc_encdata(const char *, uint32_t, uint32_t); |
| |
_STATIC void mvxpsec_dump_acc_enclen(const char *, uint32_t); |
| |
_STATIC void mvxpsec_dump_acc_enckey(const char *, uint32_t); |
| |
_STATIC void mvxpsec_dump_acc_enciv(const char *, uint32_t); |
| |
_STATIC void mvxpsec_dump_acc_macsrc(const char *, uint32_t); |
| |
_STATIC void mvxpsec_dump_acc_macdst(const char *, uint32_t); |
| |
_STATIC void mvxpsec_dump_acc_maciv(const char *, uint32_t); |
| |
#endif |
| |
|
| |
/* |
| |
* global configurations, params, work spaces, ... |
| |
* |
| |
* XXX: use sysctl for global configurations |
| |
*/ |
| |
/* waiting for device */ |
| |
static int mvxpsec_wait_interval = 10; /* usec */ |
| |
static int mvxpsec_wait_retry = 100; /* times = wait for 1 [msec] */ |
| |
#ifdef MVXPSEC_DEBUG |
| |
static uint32_t mvxpsec_debug = MVXPSEC_DEBUG; /* debug level */ |
| |
#endif |
| |
|
| |
/* |
| |
* Register accessors |
| |
*/ |
| |
#define MVXPSEC_WRITE(sc, off, val) \ |
| |
bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (off), (val)) |
| |
#define MVXPSEC_READ(sc, off) \ |
| |
bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (off)) |
| |
|
| |
/* |
| |
* device driver autoconf interface |
| |
*/ |
| |
CFATTACH_DECL2_NEW(mvxpsec_mbus, sizeof(struct mvxpsec_softc), |
| |
mvxpsec_match, mvxpsec_attach, NULL, NULL, NULL, NULL); |
| |
|
| |
STATIC int |
| |
mvxpsec_match(device_t dev, cfdata_t match, void *aux) |
| |
{ |
| |
struct marvell_attach_args *mva = aux; |
| |
uint32_t tag; |
| |
int window; |
| |
|
| |
if (strcmp(mva->mva_name, match->cf_name) != 0) |
| |
return 0; |
| |
if (mva->mva_offset == MVA_OFFSET_DEFAULT) |
| |
return 0; |
| |
|
| |
switch (mva->mva_unit) { |
| |
case 0: |
| |
tag = ARMADAXP_TAG_CRYPT0; |
| |
break; |
| |
case 1: |
| |
tag = ARMADAXP_TAG_CRYPT1; |
| |
break; |
| |
default: |
| |
aprint_error_dev(dev, |
| |
"unit %d is not supported\n", mva->mva_unit); |
| |
return 0; |
| |
} |
| |
|
| |
window = mvsoc_target(tag, NULL, NULL, NULL, NULL); |
| |
if (window >= nwindow) { |
| |
aprint_error_dev(dev, |
| |
"Security Accelerator SRAM is not configured.\n"); |
| |
return 0; |
| |
} |
| |
|
| |
return 1; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_attach(device_t parent, device_t self, void *aux) |
| |
{ |
| |
struct marvell_attach_args *mva = aux; |
| |
struct mvxpsec_softc *sc = device_private(self); |
| |
int v; |
| |
int i; |
| |
|
| |
sc->sc_dev = self; |
| |
|
| |
aprint_normal(": Marvell Crypto Engines and Security Accelerator\n"); |
| |
aprint_naive("\n"); |
| |
#ifdef MVXPSEC_MULTI_PACKET |
| |
aprint_normal_dev(sc->sc_dev, "multi-packet chained mode enabled.\n"); |
| |
#else |
| |
aprint_normal_dev(sc->sc_dev, "multi-packet chained mode disabled.\n"); |
| |
#endif |
| |
aprint_normal_dev(sc->sc_dev, |
| |
"Max %d sessions.\n", MVXPSEC_MAX_SESSIONS); |
| |
|
| |
/* mutex */ |
| |
mutex_init(&sc->sc_session_mtx, MUTEX_DEFAULT, IPL_NET); |
| |
mutex_init(&sc->sc_dma_mtx, MUTEX_DEFAULT, IPL_NET); |
| |
mutex_init(&sc->sc_queue_mtx, MUTEX_DEFAULT, IPL_NET); |
| |
|
| |
/* Packet queue */ |
| |
SIMPLEQ_INIT(&sc->sc_wait_queue); |
| |
SIMPLEQ_INIT(&sc->sc_run_queue); |
| |
SLIST_INIT(&sc->sc_free_list); |
| |
sc->sc_wait_qlen = 0; |
| |
#ifdef MVXPSEC_MULTI_PACKET |
| |
sc->sc_wait_qlimit = 16; |
| |
#else |
| |
sc->sc_wait_qlimit = 0; |
| |
#endif |
| |
sc->sc_free_qlen = 0; |
| |
|
| |
/* Timer */ |
| |
callout_init(&sc->sc_timeout, 0); /* XXX: use CALLOUT_MPSAFE */ |
| |
callout_setfunc(&sc->sc_timeout, mvxpsec_timer, sc); |
| |
|
| |
/* I/O */ |
| |
sc->sc_iot = mva->mva_iot; |
| |
if (bus_space_subregion(mva->mva_iot, mva->mva_ioh, |
| |
mva->mva_offset, mva->mva_size, &sc->sc_ioh)) { |
| |
aprint_error_dev(self, "Cannot map registers\n"); |
| |
return; |
| |
} |
| |
|
| |
/* DMA */ |
| |
sc->sc_dmat = mva->mva_dmat; |
| |
if (mvxpsec_init_dma(sc, mva) < 0) |
| |
return; |
| |
|
| |
/* SRAM */ |
| |
if (mvxpsec_init_sram(sc) < 0) |
| |
return; |
| |
|
| |
/* Registers */ |
| |
mvxpsec_wininit(sc, mva->mva_tags); |
| |
|
| |
/* INTR */ |
| |
MVXPSEC_WRITE(sc, MVXPSEC_INT_MASK, MVXPSEC_DEFAULT_INT); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_ERR_MASK, MVXPSEC_DEFAULT_ERR); |
| |
sc->sc_done_ih = |
| |
marvell_intr_establish(mva->mva_irq, IPL_NET, mvxpsec_intr, sc); |
| |
/* XXX: sould pass error IRQ using mva */ |
| |
sc->sc_error_ih = marvell_intr_establish(MVXPSEC_ERR_INT(sc), |
| |
IPL_NET, mvxpsec_eintr, sc); |
| |
aprint_normal_dev(self, |
| |
"Error Reporting IRQ %d\n", MVXPSEC_ERR_INT(sc)); |
| |
|
| |
/* Initialize TDMA (It's enabled here, but waiting for SA) */ |
| |
if (mvxpsec_dma_wait(sc) < 0) |
| |
panic("%s: DMA DEVICE not responding\n", __func__); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_DST, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0); |
| |
v = MVXPSEC_READ(sc, MV_TDMA_CONTROL); |
| |
v |= MV_TDMA_CONTROL_ENABLE; |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, v); |
| |
|
| |
/* Initialize SA */ |
| |
if (mvxpsec_acc_wait(sc) < 0) |
| |
panic("%s: MVXPSEC not responding\n", __func__); |
| |
v = MVXPSEC_READ(sc, MV_ACC_CONFIG); |
| |
v &= ~MV_ACC_CONFIG_STOP_ON_ERR; |
| |
v |= MV_ACC_CONFIG_MULT_PKT; |
| |
v |= MV_ACC_CONFIG_WAIT_TDMA; |
| |
v |= MV_ACC_CONFIG_ACT_TDMA; |
| |
MVXPSEC_WRITE(sc, MV_ACC_CONFIG, v); |
| |
MVXPSEC_WRITE(sc, MV_ACC_DESC, 0); |
| |
MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP); |
| |
|
| |
/* Session */ |
| |
sc->sc_session_pool = |
| |
pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0, |
| |
"mvxpsecpl", NULL, IPL_NET, |
| |
mvxpsec_session_ctor, mvxpsec_session_dtor, sc); |
| |
pool_cache_sethiwat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS); |
| |
pool_cache_setlowat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS / 2); |
| |
sc->sc_last_session = NULL; |
| |
|
| |
/* Pakcet */ |
| |
sc->sc_packet_pool = |
| |
pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0, |
| |
"mvxpsec_pktpl", NULL, IPL_NET, |
| |
mvxpsec_packet_ctor, mvxpsec_packet_dtor, sc); |
| |
pool_cache_sethiwat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS); |
| |
pool_cache_setlowat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS / 2); |
| |
|
| |
/* Register to EVCNT framework */ |
| |
mvxpsec_evcnt_attach(sc); |
| |
|
| |
/* Register to Opencrypto */ |
| |
for (i = 0; i < MVXPSEC_MAX_SESSIONS; i++) { |
| |
sc->sc_sessions[i] = NULL; |
| |
} |
| |
if (mvxpsec_register(sc)) |
| |
panic("cannot initialize OpenCrypto module.\n"); |
| |
|
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_evcnt_attach(struct mvxpsec_softc *sc) |
| |
{ |
| |
struct mvxpsec_evcnt *sc_ev = &sc->sc_ev; |
| |
|
| |
evcnt_attach_dynamic(&sc_ev->intr_all, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "Main Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_auth, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "Auth Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_des, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "DES Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_aes_enc, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "AES-Encrypt Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_aes_dec, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "AES-Decrypt Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_enc, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "Crypto Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_sa, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "SA Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_acctdma, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "AccTDMA Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_comp, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "TDMA-Complete Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_own, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "TDMA-Ownership Intr."); |
| |
evcnt_attach_dynamic(&sc_ev->intr_acctdma_cont, EVCNT_TYPE_INTR, |
| |
NULL, device_xname(sc->sc_dev), "AccTDMA-Continue Intr."); |
| |
|
| |
evcnt_attach_dynamic(&sc_ev->session_new, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "New-Session"); |
| |
evcnt_attach_dynamic(&sc_ev->session_free, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Free-Session"); |
| |
|
| |
evcnt_attach_dynamic(&sc_ev->packet_ok, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Packet-OK"); |
| |
evcnt_attach_dynamic(&sc_ev->packet_err, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Packet-ERR"); |
| |
|
| |
evcnt_attach_dynamic(&sc_ev->dispatch_packets, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Packet-Dispatch"); |
| |
evcnt_attach_dynamic(&sc_ev->dispatch_queue, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Queue-Dispatch"); |
| |
evcnt_attach_dynamic(&sc_ev->queue_full, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Queue-Full"); |
| |
evcnt_attach_dynamic(&sc_ev->max_dispatch, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Max-Dispatch"); |
| |
evcnt_attach_dynamic(&sc_ev->max_done, EVCNT_TYPE_MISC, |
| |
NULL, device_xname(sc->sc_dev), "Max-Done"); |
| |
} |
| |
|
| |
/* |
| |
* Register setup |
| |
*/ |
| |
STATIC int mvxpsec_wininit(struct mvxpsec_softc *sc, enum marvell_tags *tags) |
| |
{ |
| |
device_t pdev = device_parent(sc->sc_dev); |
| |
uint64_t base; |
| |
uint32_t size, reg; |
| |
int window, target, attr, rv, i; |
| |
|
| |
/* disable all window */ |
| |
for (window = 0; window < MV_TDMA_NWINDOW; window++) |
| |
{ |
| |
MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), 0); |
| |
} |
| |
|
| |
for (window = 0, i = 0; |
| |
tags[i] != MARVELL_TAG_UNDEFINED && window < MV_TDMA_NWINDOW; i++) { |
| |
rv = marvell_winparams_by_tag(pdev, tags[i], |
| |
&target, &attr, &base, &size); |
| |
if (rv != 0 || size == 0) |
| |
continue; |
| |
|
| |
if (base > 0xffffffffULL) { |
| |
aprint_error_dev(sc->sc_dev, |
| |
"can't remap window %d\n", window); |
| |
continue; |
| |
} |
| |
|
| |
reg = MV_TDMA_BAR_BASE(base); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), reg); |
| |
|
| |
reg = MV_TDMA_ATTR_TARGET(target); |
| |
reg |= MV_TDMA_ATTR_ATTR(attr); |
| |
reg |= MV_TDMA_ATTR_SIZE(size); |
| |
reg |= MV_TDMA_ATTR_ENABLE; |
| |
MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), reg); |
| |
|
| |
window++; |
| |
} |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Timer handling |
| |
*/ |
| |
STATIC void |
| |
mvxpsec_timer(void *aux) |
| |
{ |
| |
struct mvxpsec_softc *sc = aux; |
| |
struct mvxpsec_packet *mv_p; |
| |
uint32_t reg; |
| |
int ndone; |
| |
int refill; |
| |
int s; |
| |
|
| |
/* IPL_SOFTCLOCK */ |
| |
|
| |
log(LOG_ERR, "%s: device timeout.\n", __func__); |
| |
#ifdef MVXPSEC_DEBUG |
| |
mvxpsec_dump_reg(sc); |
| |
#endif |
| |
|
| |
s = splnet(); |
| |
/* stop security accelerator */ |
| |
MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP); |
| |
|
| |
/* stop TDMA */ |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, 0); |
| |
|
| |
/* cleanup packet queue */ |
| |
mutex_enter(&sc->sc_queue_mtx); |
| |
ndone = 0; |
| |
while ( (mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue)) != NULL) { |
| |
SIMPLEQ_REMOVE_HEAD(&sc->sc_run_queue, queue); |
| |
|
| |
mv_p->crp->crp_etype = EINVAL; |
| |
mvxpsec_done_packet(mv_p); |
| |
ndone++; |
| |
} |
| |
MVXPSEC_EVCNT_MAX(sc, max_done, ndone); |
| |
sc->sc_flags &= ~HW_RUNNING; |
| |
refill = (sc->sc_wait_qlen > 0) ? 1 : 0; |
| |
mutex_exit(&sc->sc_queue_mtx); |
| |
|
| |
/* reenable TDMA */ |
| |
if (mvxpsec_dma_wait(sc) < 0) |
| |
panic("%s: failed to reset DMA DEVICE. give up.", __func__); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_DST, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0); |
| |
MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0); |
| |
reg = MV_TDMA_DEFAULT_CONTROL; |
| |
reg |= MV_TDMA_CONTROL_ENABLE; |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, reg); |
| |
|
| |
if (mvxpsec_acc_wait(sc) < 0) |
| |
panic("%s: failed to reset MVXPSEC. give up.", __func__); |
| |
reg = MV_ACC_CONFIG_MULT_PKT; |
| |
reg |= MV_ACC_CONFIG_WAIT_TDMA; |
| |
reg |= MV_ACC_CONFIG_ACT_TDMA; |
| |
MVXPSEC_WRITE(sc, MV_ACC_CONFIG, reg); |
| |
MVXPSEC_WRITE(sc, MV_ACC_DESC, 0); |
| |
|
| |
if (refill) { |
| |
mutex_enter(&sc->sc_queue_mtx); |
| |
mvxpsec_dispatch_queue(sc); |
| |
mutex_exit(&sc->sc_queue_mtx); |
| |
} |
| |
|
| |
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); |
| |
splx(s); |
| |
} |
| |
|
| |
/* |
| |
* DMA handling |
| |
*/ |
| |
|
| |
/* |
| |
* Allocate kernel devmem and DMA safe memory with bus_dma API |
| |
* used for DMA descriptors. |
| |
* |
| |
* if phys != 0, assume phys is a DMA safe memory and bypass |
| |
* allocator. |
| |
*/ |
| |
STATIC struct mvxpsec_devmem * |
| |
mvxpsec_alloc_devmem(struct mvxpsec_softc *sc, paddr_t phys, int size) |
| |
{ |
| |
struct mvxpsec_devmem *devmem; |
| |
bus_dma_segment_t seg; |
| |
int rseg; |
| |
int err; |
| |
|
| |
if (sc == NULL) |
| |
return NULL; |
| |
|
| |
devmem = kmem_alloc(sizeof(*devmem), KM_NOSLEEP); |
| |
if (devmem == NULL) { |
| |
aprint_error_dev(sc->sc_dev, "can't alloc kmem\n"); |
| |
return NULL; |
| |
} |
| |
|
| |
devmem->size = size; |
| |
|
| |
if (phys) { |
| |
seg.ds_addr = phys; |
| |
seg.ds_len = devmem->size; |
| |
rseg = 1; |
| |
err = 0; |
| |
} |
| |
else { |
| |
err = bus_dmamem_alloc(sc->sc_dmat, |
| |
devmem->size, PAGE_SIZE, 0, |
| |
&seg, MVXPSEC_DMA_MAX_SEGS, &rseg, BUS_DMA_NOWAIT); |
| |
} |
| |
if (err) { |
| |
aprint_error_dev(sc->sc_dev, "can't alloc DMA buffer\n"); |
| |
goto fail_kmem_free; |
| |
} |
| |
|
| |
err = bus_dmamem_map(sc->sc_dmat, &seg, rseg, |
| |
devmem->size, &devmem->kva, BUS_DMA_NOWAIT); |
| |
if (err) { |
| |
aprint_error_dev(sc->sc_dev, "can't map DMA buffer\n"); |
| |
goto fail_dmamem_free; |
| |
} |
| |
|
| |
err = bus_dmamap_create(sc->sc_dmat, |
| |
size, 1, size, 0, BUS_DMA_NOWAIT, &devmem->map); |
| |
if (err) { |
| |
aprint_error_dev(sc->sc_dev, "can't create DMA map\n"); |
| |
goto fail_unmap; |
| |
} |
| |
|
| |
err = bus_dmamap_load(sc->sc_dmat, |
| |
devmem->map, devmem->kva, devmem->size, NULL, |
| |
BUS_DMA_NOWAIT); |
| |
if (err) { |
| |
aprint_error_dev(sc->sc_dev, |
| |
"can't load DMA buffer VA:%p PA:0x%08x\n", |
| |
devmem->kva, (int)seg.ds_addr); |
| |
goto fail_destroy; |
| |
} |
| |
|
| |
return devmem; |
| |
|
| |
fail_destroy: |
| |
bus_dmamap_destroy(sc->sc_dmat, devmem->map); |
| |
fail_unmap: |
| |
bus_dmamem_unmap(sc->sc_dmat, devmem->kva, devmem->size); |
| |
fail_dmamem_free: |
| |
bus_dmamem_free(sc->sc_dmat, &seg, rseg); |
| |
fail_kmem_free: |
| |
kmem_free(devmem, sizeof(*devmem)); |
| |
|
| |
return NULL; |
| |
} |
| |
|
| |
/* |
| |
* Get DMA Descriptor from (DMA safe) descriptor pool. |
| |
*/ |
| |
INLINE struct mvxpsec_descriptor_handle * |
| |
mvxpsec_dma_getdesc(struct mvxpsec_softc *sc) |
| |
{ |
| |
struct mvxpsec_descriptor_handle *entry; |
| |
|
| |
/* must called with sc->sc_dma_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_dma_mtx)); |
| |
|
| |
if (sc->sc_desc_ring_prod == sc->sc_desc_ring_cons) |
| |
return NULL; |
| |
|
| |
entry = &sc->sc_desc_ring[sc->sc_desc_ring_prod]; |
| |
sc->sc_desc_ring_prod++; |
| |
if (sc->sc_desc_ring_prod >= sc->sc_desc_ring_size) |
| |
sc->sc_desc_ring_prod -= sc->sc_desc_ring_size; |
| |
|
| |
return entry; |
| |
} |
| |
|
| |
/* |
| |
* Put DMA Descriptor to descriptor pool. |
| |
*/ |
| |
_INLINE void |
| |
mvxpsec_dma_putdesc(struct mvxpsec_softc *sc, |
| |
struct mvxpsec_descriptor_handle *dh) |
| |
{ |
| |
/* must called with sc->sc_dma_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_dma_mtx)); |
| |
|
| |
sc->sc_desc_ring_cons++; |
| |
if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size) |
| |
sc->sc_desc_ring_cons -= sc->sc_desc_ring_size; |
| |
|
| |
return; |
| |
} |
| |
|
| |
/* |
| |
* Setup DMA Descriptor |
| |
* copy from 'src' to 'dst' by 'size' bytes. |
| |
* 'src' or 'dst' must be SRAM address. |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *dh, |
| |
uint32_t dst, uint32_t src, uint32_t size) |
| |
{ |
| |
struct mvxpsec_descriptor *desc; |
| |
|
| |
desc = (struct mvxpsec_descriptor *)dh->_desc; |
| |
|
| |
desc->tdma_dst = dst; |
| |
desc->tdma_src = src; |
| |
desc->tdma_word0 = size; |
| |
if (size != 0) |
| |
desc->tdma_word0 |= MV_TDMA_CNT_OWN; |
| |
/* size == 0 is owned by ACC, not TDMA */ |
| |
|
| |
#ifdef MVXPSEC_DEBUG |
| |
mvxpsec_dump_dmaq(dh); |
| |
#endif |
| |
|
| |
} |
| |
|
| |
/* |
| |
* Concat 2 DMA |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_dma_cat(struct mvxpsec_softc *sc, |
| |
struct mvxpsec_descriptor_handle *dh1, |
| |
struct mvxpsec_descriptor_handle *dh2) |
| |
{ |
| |
((struct mvxpsec_descriptor*)dh1->_desc)->tdma_nxt = dh2->phys_addr; |
| |
MVXPSEC_SYNC_DESC(sc, dh1, BUS_DMASYNC_PREWRITE); |
| |
} |
| |
|
| |
/* |
| |
* Schedule DMA Copy |
| |
*/ |
| |
INLINE int |
| |
mvxpsec_dma_copy0(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r, |
| |
uint32_t dst, uint32_t src, uint32_t size) |
| |
{ |
| |
struct mvxpsec_descriptor_handle *dh; |
| |
|
| |
dh = mvxpsec_dma_getdesc(sc); |
| |
if (dh == NULL) { |
| |
log(LOG_ERR, "%s: descriptor full\n", __func__); |
| |
return -1; |
| |
} |
| |
|
| |
mvxpsec_dma_setup(dh, dst, src, size); |
| |
if (r->dma_head == NULL) { |
| |
r->dma_head = dh; |
| |
r->dma_last = dh; |
| |
r->dma_size = 1; |
| |
} |
| |
else { |
| |
mvxpsec_dma_cat(sc, r->dma_last, dh); |
| |
r->dma_last = dh; |
| |
r->dma_size++; |
| |
} |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
INLINE int |
| |
mvxpsec_dma_copy(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r, |
| |
uint32_t dst, uint32_t src, uint32_t size) |
| |
{ |
| |
if (size == 0) /* 0 is very special descriptor */ |
| |
return 0; |
| |
|
| |
return mvxpsec_dma_copy0(sc, r, dst, src, size); |
| |
} |
| |
|
| |
/* |
| |
* Schedule ACC Activate |
| |
*/ |
| |
INLINE int |
| |
mvxpsec_dma_acc_activate(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r) |
| |
{ |
| |
return mvxpsec_dma_copy0(sc, r, 0, 0, 0); |
| |
} |
| |
|
| |
/* |
| |
* Finalize DMA setup |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_dma_finalize(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r) |
| |
{ |
| |
struct mvxpsec_descriptor_handle *dh; |
| |
|
| |
dh = r->dma_last; |
| |
((struct mvxpsec_descriptor*)dh->_desc)->tdma_nxt = 0; |
| |
MVXPSEC_SYNC_DESC(sc, dh, BUS_DMASYNC_PREWRITE); |
| |
} |
| |
|
| |
/* |
| |
* Free entire DMA ring |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_dma_free(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r) |
| |
{ |
| |
sc->sc_desc_ring_cons += r->dma_size; |
| |
if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size) |
| |
sc->sc_desc_ring_cons -= sc->sc_desc_ring_size; |
| |
r->dma_head = NULL; |
| |
r->dma_last = NULL; |
| |
r->dma_size = 0; |
| |
} |
| |
|
| |
/* |
| |
* create DMA descriptor chain for the packet |
| |
*/ |
| |
INLINE int |
| |
mvxpsec_dma_copy_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p) |
| |
{ |
| |
struct mvxpsec_session *mv_s = mv_p->mv_s; |
| |
uint32_t src, dst, len; |
| |
uint32_t pkt_off, pkt_off_r; |
| |
int err; |
| |
int i; |
| |
|
| |
/* must called with sc->sc_dma_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_dma_mtx)); |
| |
|
| |
/* |
| |
* set offset for mem->device copy |
| |
* |
| |
* typical packet image: |
| |
* |
| |
* enc_ivoff |
| |
* mac_off |
| |
* | |
| |
* | enc_off |
| |
* | | |
| |
* v v |
| |
* +----+--------... |
| |
* |IV |DATA |
| |
* +----+--------... |
| |
*/ |
| |
pkt_off = 0; |
| |
if (mv_p->mac_off > 0) |
| |
pkt_off = mv_p->mac_off; |
| |
if ((mv_p->flags & CRP_EXT_IV) == 0 && pkt_off > mv_p->enc_ivoff) |
| |
pkt_off = mv_p->enc_ivoff; |
| |
if (mv_p->enc_off > 0 && pkt_off > mv_p->enc_off) |
| |
pkt_off = mv_p->enc_off; |
| |
pkt_off_r = pkt_off; |
| |
|
| |
/* make DMA descriptors to copy packet header: DRAM -> SRAM */ |
| |
dst = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc); |
| |
src = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr; |
| |
len = sizeof(mv_p->pkt_header); |
| |
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
|
| |
/* |
| |
* make DMA descriptors to copy session header: DRAM -> SRAM |
| |
* we can reuse session header on SRAM if session is not changed. |
| |
*/ |
| |
if (sc->sc_last_session != mv_s) { |
| |
dst = (uint32_t)MVXPSEC_SRAM_SESS_HDR_PA(sc); |
| |
src = (uint32_t)mv_s->session_header_map->dm_segs[0].ds_addr; |
| |
len = sizeof(mv_s->session_header); |
| |
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
sc->sc_last_session = mv_s; |
| |
} |
| |
|
| |
/* make DMA descriptor to copy payload data: DRAM -> SRAM */ |
| |
dst = MVXPSEC_SRAM_PAYLOAD_PA(sc, 0); |
| |
for (i = 0; i < mv_p->data_map->dm_nsegs; i++) { |
| |
src = mv_p->data_map->dm_segs[i].ds_addr; |
| |
len = mv_p->data_map->dm_segs[i].ds_len; |
| |
if (pkt_off) { |
| |
if (len <= pkt_off) { |
| |
/* ignore the segment */ |
| |
dst += len; |
| |
pkt_off -= len; |
| |
continue; |
| |
} |
| |
/* copy from the middle of the segment */ |
| |
dst += pkt_off; |
| |
src += pkt_off; |
| |
len -= pkt_off; |
| |
pkt_off = 0; |
| |
} |
| |
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
dst += len; |
| |
} |
| |
|
| |
/* make special descriptor to activate security accelerator */ |
| |
err = mvxpsec_dma_acc_activate(sc, &mv_p->dma_ring); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
|
| |
/* make DMA descriptors to copy payload: SRAM -> DRAM */ |
| |
src = (uint32_t)MVXPSEC_SRAM_PAYLOAD_PA(sc, 0); |
| |
for (i = 0; i < mv_p->data_map->dm_nsegs; i++) { |
| |
dst = (uint32_t)mv_p->data_map->dm_segs[i].ds_addr; |
| |
len = (uint32_t)mv_p->data_map->dm_segs[i].ds_len; |
| |
if (pkt_off_r) { |
| |
if (len <= pkt_off_r) { |
| |
/* ignore the segment */ |
| |
src += len; |
| |
pkt_off_r -= len; |
| |
continue; |
| |
} |
| |
/* copy from the middle of the segment */ |
| |
src += pkt_off_r; |
| |
dst += pkt_off_r; |
| |
len -= pkt_off_r; |
| |
pkt_off_r = 0; |
| |
} |
| |
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
src += len; |
| |
} |
| |
KASSERT(pkt_off == 0); |
| |
KASSERT(pkt_off_r == 0); |
| |
|
| |
/* |
| |
* make DMA descriptors to copy packet header: SRAM->DRAM |
| |
* if IV is present in the payload, no need to copy. |
| |
*/ |
| |
if (mv_p->flags & CRP_EXT_IV) { |
| |
dst = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr; |
| |
src = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc); |
| |
len = sizeof(mv_p->pkt_header); |
| |
err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
} |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
INLINE int |
| |
mvxpsec_dma_sync_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p) |
| |
{ |
| |
/* sync packet header */ |
| |
bus_dmamap_sync(sc->sc_dmat, |
| |
mv_p->pkt_header_map, 0, sizeof(mv_p->pkt_header), |
| |
BUS_DMASYNC_PREWRITE); |
| |
|
| |
#ifdef MVXPSEC_DEBUG |
| |
/* sync session header */ |
| |
if (mvxpsec_debug != 0) { |
| |
struct mvxpsec_session *mv_s = mv_p->mv_s; |
| |
|
| |
/* only debug code touch the session header after newsession */ |
| |
bus_dmamap_sync(sc->sc_dmat, |
| |
mv_s->session_header_map, |
| |
0, sizeof(mv_s->session_header), |
| |
BUS_DMASYNC_PREWRITE); |
| |
} |
| |
#endif |
| |
|
| |
/* sync packet buffer */ |
| |
bus_dmamap_sync(sc->sc_dmat, |
| |
mv_p->data_map, 0, mv_p->data_len, |
| |
BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Initialize MVXPSEC Internal SRAM |
| |
* |
| |
* - must be called after DMA initizlization. |
| |
* - make VM mapping for SRAM area on MBus. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_init_sram(struct mvxpsec_softc *sc) |
| |
{ |
| |
uint32_t tag, target, attr, base, size; |
| |
vaddr_t va; |
| |
int window; |
| |
|
| |
switch (sc->sc_dev->dv_unit) { |
| |
case 0: |
| |
tag = ARMADAXP_TAG_CRYPT0; |
| |
break; |
| |
case 1: |
| |
tag = ARMADAXP_TAG_CRYPT1; |
| |
break; |
| |
default: |
| |
aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n"); |
| |
return -1; |
| |
} |
| |
|
| |
window = mvsoc_target(tag, &target, &attr, &base, &size); |
| |
if (window >= nwindow) { |
| |
aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n"); |
| |
return -1; |
| |
} |
| |
|
| |
if (sizeof(struct mvxpsec_crypt_sram) > size) { |
| |
aprint_error_dev(sc->sc_dev, |
| |
"SRAM Data Structure Excceeds SRAM window size.\n"); |
| |
return -1; |
| |
} |
| |
|
| |
aprint_normal_dev(sc->sc_dev, |
| |
"internal SRAM window at 0x%08x-0x%08x", |
| |
base, base + size - 1); |
| |
sc->sc_sram_pa = base; |
| |
|
| |
/* get vmspace to read/write device internal SRAM */ |
| |
va = uvm_km_alloc(kernel_map, PAGE_SIZE, PAGE_SIZE, |
| |
UVM_KMF_VAONLY | UVM_KMF_NOWAIT); |
| |
if (va == 0) { |
| |
aprint_error_dev(sc->sc_dev, "cannot map SRAM window\n"); |
| |
sc->sc_sram_va = NULL; |
| |
aprint_normal("\n"); |
| |
return 0; |
| |
} |
| |
/* XXX: not working. PMAP_NOCACHE is not affected? */ |
| |
pmap_kenter_pa(va, base, VM_PROT_READ|VM_PROT_WRITE, PMAP_NOCACHE); |
| |
pmap_update(pmap_kernel()); |
| |
sc->sc_sram_va = (void *)va; |
| |
aprint_normal(" va %p\n", sc->sc_sram_va); |
| |
memset(sc->sc_sram_va, 0xff, MV_ACC_SRAM_SIZE); |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Initialize TDMA engine. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_init_dma(struct mvxpsec_softc *sc, struct marvell_attach_args *mva) |
| |
{ |
| |
struct mvxpsec_descriptor_handle *dh; |
| |
uint8_t *va; |
| |
paddr_t pa; |
| |
off_t va_off, pa_off; |
| |
int i, n, seg, ndh; |
| |
|
| |
/* Init Deviced's control parameters (disabled yet) */ |
| |
MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, MV_TDMA_DEFAULT_CONTROL); |
| |
|
| |
/* Init Software DMA Handlers */ |
| |
sc->sc_devmem_desc = |
| |
mvxpsec_alloc_devmem(sc, 0, PAGE_SIZE * MVXPSEC_DMA_DESC_PAGES); |
| |
if (sc->sc_devmem_desc == NULL) |
| |
panic("Cannot allocate memory\n"); |
| |
ndh = (PAGE_SIZE / sizeof(struct mvxpsec_descriptor)) |
| |
* MVXPSEC_DMA_DESC_PAGES; |
| |
sc->sc_desc_ring = |
| |
kmem_alloc(sizeof(struct mvxpsec_descriptor_handle) * ndh, |
| |
KM_NOSLEEP); |
| |
if (sc->sc_desc_ring == NULL) |
| |
panic("Cannot allocate memory\n"); |
| |
aprint_normal_dev(sc->sc_dev, "%d DMA handles in %zu bytes array\n", |
| |
ndh, sizeof(struct mvxpsec_descriptor_handle) * ndh); |
| |
|
| |
ndh = 0; |
| |
for (seg = 0; seg < devmem_nseg(sc->sc_devmem_desc); seg++) { |
| |
va = devmem_va(sc->sc_devmem_desc); |
| |
pa = devmem_pa(sc->sc_devmem_desc, seg); |
| |
n = devmem_palen(sc->sc_devmem_desc, seg) / |
| |
sizeof(struct mvxpsec_descriptor); |
| |
va_off = (PAGE_SIZE * seg); |
| |
pa_off = 0; |
| |
for (i = 0; i < n; i++) { |
| |
dh = &sc->sc_desc_ring[ndh]; |
| |
dh->map = devmem_map(sc->sc_devmem_desc); |
| |
dh->off = va_off + pa_off; |
| |
dh->_desc = (void *)(va + va_off + pa_off); |
| |
dh->phys_addr = pa + pa_off; |
| |
pa_off += sizeof(struct mvxpsec_descriptor); |
| |
ndh++; |
| |
} |
| |
} |
| |
sc->sc_desc_ring_size = ndh; |
| |
sc->sc_desc_ring_prod = 0; |
| |
sc->sc_desc_ring_cons = sc->sc_desc_ring_size - 1; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Wait for TDMA controller become idle |
| |
*/ |
| |
INLINE int |
| |
mvxpsec_dma_wait(struct mvxpsec_softc *sc) |
| |
{ |
| |
int retry = 0; |
| |
|
| |
while (MVXPSEC_READ(sc, MV_TDMA_CONTROL) & MV_TDMA_CONTROL_ACT) { |
| |
delay(mvxpsec_wait_interval); |
| |
if (retry++ >= mvxpsec_wait_retry) |
| |
return -1; |
| |
} |
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Wait for Security Accelerator become idle |
| |
*/ |
| |
INLINE int |
| |
mvxpsec_acc_wait(struct mvxpsec_softc *sc) |
| |
{ |
| |
int retry = 0; |
| |
|
| |
while (MVXPSEC_READ(sc, MV_ACC_COMMAND) & MV_ACC_COMMAND_ACT) { |
| |
delay(mvxpsec_wait_interval); |
| |
if (++retry >= mvxpsec_wait_retry) |
| |
return -1; |
| |
} |
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Entry of interrupt handler |
| |
* |
| |
* register this to kernel via marvell_intr_establish() |
| |
*/ |
| |
int |
| |
mvxpsec_intr(void *arg) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
uint32_t v; |
| |
|
| |
/* IPL_NET */ |
| |
while ((v = mvxpsec_intr_ack(sc)) != 0) { |
| |
mvxpsec_intr_cnt(sc, v); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "MVXPSEC Intr 0x%08x\n", v); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "%s\n", s_xpsecintr(v)); |
| |
#ifdef MVXPSEC_DEBUG |
| |
mvxpsec_dump_reg(sc); |
| |
#endif |
| |
|
| |
/* call high-level handlers */ |
| |
if (v & MVXPSEC_INT_ACCTDMA) |
| |
mvxpsec_done(sc); |
| |
} |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
INLINE void |
| |
mvxpsec_intr_cleanup(struct mvxpsec_softc *sc) |
| |
{ |
| |
struct mvxpsec_packet *mv_p; |
| |
|
| |
/* must called with sc->sc_dma_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_dma_mtx)); |
| |
|
| |
/* |
| |
* there is only one intr for run_queue. |
| |
* no one touch sc_run_queue. |
| |
*/ |
| |
SIMPLEQ_FOREACH(mv_p, &sc->sc_run_queue, queue) |
| |
mvxpsec_dma_free(sc, &mv_p->dma_ring); |
| |
} |
| |
|
| |
/* |
| |
* Acknowledge to interrupt |
| |
* |
| |
* read cause bits, clear it, and return it. |
| |
* NOTE: multiple cause bits may be returned at once. |
| |
*/ |
| |
STATIC uint32_t |
| |
mvxpsec_intr_ack(struct mvxpsec_softc *sc) |
| |
{ |
| |
uint32_t reg; |
| |
|
| |
reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE); |
| |
reg &= MVXPSEC_DEFAULT_INT; |
| |
MVXPSEC_WRITE(sc, MVXPSEC_INT_CAUSE, ~reg); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg)); |
| |
|
| |
return reg; |
| |
} |
| |
|
| |
/* |
| |
* Entry of TDMA error interrupt handler |
| |
* |
| |
* register this to kernel via marvell_intr_establish() |
| |
*/ |
| |
int |
| |
mvxpsec_eintr(void *arg) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
uint32_t err; |
| |
|
| |
/* IPL_NET */ |
| |
again: |
| |
err = mvxpsec_eintr_ack(sc); |
| |
if (err == 0) |
| |
goto done; |
| |
|
| |
log(LOG_ERR, "%s: DMA Error Interrupt: %s\n", __func__, |
| |
s_errreg(err)); |
| |
#ifdef MVXPSEC_DEBUG |
| |
mvxpsec_dump_reg(sc); |
| |
#endif |
| |
|
| |
goto again; |
| |
done: |
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Acknowledge to TDMA error interrupt |
| |
* |
| |
* read cause bits, clear it, and return it. |
| |
* NOTE: multiple cause bits may be returned at once. |
| |
*/ |
| |
STATIC uint32_t |
| |
mvxpsec_eintr_ack(struct mvxpsec_softc *sc) |
| |
{ |
| |
uint32_t reg; |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE); |
| |
reg &= MVXPSEC_DEFAULT_ERR; |
| |
MVXPSEC_WRITE(sc, MV_TDMA_ERR_CAUSE, ~reg); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg)); |
| |
|
| |
return reg; |
| |
} |
| |
|
| |
/* |
| |
* Interrupt statistics |
| |
* |
| |
* this is NOT a statistics of how may times the events 'occured'. |
| |
* this ONLY means how many times the events 'handled'. |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_intr_cnt(struct mvxpsec_softc *sc, int cause) |
| |
{ |
| |
MVXPSEC_EVCNT_INCR(sc, intr_all); |
| |
if (cause & MVXPSEC_INT_AUTH) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_auth); |
| |
if (cause & MVXPSEC_INT_DES) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_des); |
| |
if (cause & MVXPSEC_INT_AES_ENC) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_aes_enc); |
| |
if (cause & MVXPSEC_INT_AES_DEC) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_aes_dec); |
| |
if (cause & MVXPSEC_INT_ENC) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_enc); |
| |
if (cause & MVXPSEC_INT_SA) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_sa); |
| |
if (cause & MVXPSEC_INT_ACCTDMA) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_acctdma); |
| |
if (cause & MVXPSEC_INT_TDMA_COMP) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_comp); |
| |
if (cause & MVXPSEC_INT_TDMA_OWN) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_own); |
| |
if (cause & MVXPSEC_INT_ACCTDMA_CONT) |
| |
MVXPSEC_EVCNT_INCR(sc, intr_acctdma_cont); |
| |
} |
| |
|
| |
/* |
| |
* Setup MVXPSEC header structure. |
| |
* |
| |
* the header contains descriptor of security accelerator, |
| |
* key material of chiphers, iv of ciphers and macs, ... |
| |
* |
| |
* the header is transfered to MVXPSEC Internal SRAM by TDMA, |
| |
* and parsed by MVXPSEC H/W. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_header_finalize(struct mvxpsec_packet *mv_p) |
| |
{ |
| |
struct mvxpsec_acc_descriptor *desc = &mv_p->pkt_header.desc; |
| |
int enc_start, enc_len, iv_offset; |
| |
int mac_start, mac_len, mac_offset; |
| |
|
| |
/* offset -> device address */ |
| |
enc_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_off); |
| |
enc_len = mv_p->enc_len; |
| |
if (mv_p->flags & CRP_EXT_IV) |
| |
iv_offset = mv_p->enc_ivoff; |
| |
else |
| |
iv_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_ivoff); |
| |
mac_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_off); |
| |
mac_len = mv_p->mac_len; |
| |
mac_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_dst); |
| |
|
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"PAYLOAD at 0x%08x\n", (int)MVXPSEC_SRAM_PAYLOAD_OFF); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"ENC from 0x%08x\n", enc_start); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"MAC from 0x%08x\n", mac_start); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"MAC to 0x%08x\n", mac_offset); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"ENC IV at 0x%08x\n", iv_offset); |
| |
|
| |
/* setup device addresses in Security Accelerator Descriptors */ |
| |
desc->acc_encdata = MV_ACC_DESC_ENC_DATA(enc_start, enc_start); |
| |
desc->acc_enclen = MV_ACC_DESC_ENC_LEN(enc_len); |
| |
if (desc->acc_config & MV_ACC_CRYPTO_DECRYPT) |
| |
desc->acc_enckey = |
| |
MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_D_DA); |
| |
else |
| |
desc->acc_enckey = |
| |
MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_DA); |
| |
desc->acc_enciv = |
| |
MV_ACC_DESC_ENC_IV(MVXPSEC_SRAM_IV_WORK_DA, iv_offset); |
| |
|
| |
desc->acc_macsrc = MV_ACC_DESC_MAC_SRC(mac_start, mac_len); |
| |
desc->acc_macdst = MV_ACC_DESC_MAC_DST(mac_offset, mac_len); |
| |
desc->acc_maciv = |
| |
MV_ACC_DESC_MAC_IV(MVXPSEC_SRAM_MIV_IN_DA, |
| |
MVXPSEC_SRAM_MIV_OUT_DA); |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* constractor of session structure. |
| |
* |
| |
* this constrator will be called by pool_cache framework. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_session_ctor(void *arg, void *obj, int flags) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_session *mv_s = obj; |
| |
|
| |
/* pool is owned by softc */ |
| |
mv_s->sc = sc; |
| |
|
| |
/* Create and load DMA map for session header */ |
| |
mv_s->session_header_map = 0; |
| |
if (bus_dmamap_create(sc->sc_dmat, |
| |
sizeof(mv_s->session_header), 1, |
| |
sizeof(mv_s->session_header), 0, |
| |
BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, |
| |
&mv_s->session_header_map)) { |
| |
log(LOG_ERR, "%s: cannot create DMA map\n", __func__); |
| |
goto fail; |
| |
} |
| |
if (bus_dmamap_load(sc->sc_dmat, mv_s->session_header_map, |
| |
&mv_s->session_header, sizeof(mv_s->session_header), |
| |
NULL, BUS_DMA_NOWAIT)) { |
| |
log(LOG_ERR, "%s: cannot load header\n", __func__); |
| |
goto fail; |
| |
} |
| |
|
| |
return 0; |
| |
fail: |
| |
if (mv_s->session_header_map) |
| |
bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map); |
| |
return ENOMEM; |
| |
} |
| |
|
| |
/* |
| |
* destractor of session structure. |
| |
* |
| |
* this destrator will be called by pool_cache framework. |
| |
*/ |
| |
STATIC void |
| |
mvxpsec_session_dtor(void *arg, void *obj) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_session *mv_s = obj; |
| |
|
| |
if (mv_s->sc != sc) |
| |
panic("inconsitent context\n"); |
| |
|
| |
bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map); |
| |
} |
| |
|
| |
/* |
| |
* constructor of packet structure. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_packet_ctor(void *arg, void *obj, int flags) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_packet *mv_p = obj; |
| |
|
| |
mv_p->dma_ring.dma_head = NULL; |
| |
mv_p->dma_ring.dma_last = NULL; |
| |
mv_p->dma_ring.dma_size = 0; |
| |
|
| |
/* Create and load DMA map for packet header */ |
| |
mv_p->pkt_header_map = 0; |
| |
if (bus_dmamap_create(sc->sc_dmat, |
| |
sizeof(mv_p->pkt_header), 1, sizeof(mv_p->pkt_header), 0, |
| |
BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, |
| |
&mv_p->pkt_header_map)) { |
| |
log(LOG_ERR, "%s: cannot create DMA map\n", __func__); |
| |
goto fail; |
| |
} |
| |
if (bus_dmamap_load(sc->sc_dmat, mv_p->pkt_header_map, |
| |
&mv_p->pkt_header, sizeof(mv_p->pkt_header), |
| |
NULL, BUS_DMA_NOWAIT)) { |
| |
log(LOG_ERR, "%s: cannot load header\n", __func__); |
| |
goto fail; |
| |
} |
| |
|
| |
/* Create DMA map for session data. */ |
| |
mv_p->data_map = 0; |
| |
if (bus_dmamap_create(sc->sc_dmat, |
| |
MVXPSEC_DMA_MAX_SIZE, MVXPSEC_DMA_MAX_SEGS, MVXPSEC_DMA_MAX_SIZE, |
| |
0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mv_p->data_map)) { |
| |
log(LOG_ERR, "%s: cannot create DMA map\n", __func__); |
| |
goto fail; |
| |
} |
| |
|
| |
return 0; |
| |
fail: |
| |
if (mv_p->pkt_header_map) |
| |
bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map); |
| |
if (mv_p->data_map) |
| |
bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map); |
| |
return ENOMEM; |
| |
} |
| |
|
| |
/* |
| |
* destractor of packet structure. |
| |
*/ |
| |
STATIC void |
| |
mvxpsec_packet_dtor(void *arg, void *obj) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_packet *mv_p = obj; |
| |
|
| |
mutex_enter(&sc->sc_dma_mtx); |
| |
mvxpsec_dma_free(sc, &mv_p->dma_ring); |
| |
mutex_exit(&sc->sc_dma_mtx); |
| |
bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map); |
| |
bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map); |
| |
} |
| |
|
| |
/* |
| |
* allocate new session struture. |
| |
*/ |
| |
STATIC struct mvxpsec_session * |
| |
mvxpsec_session_alloc(struct mvxpsec_softc *sc) |
| |
{ |
| |
struct mvxpsec_session *mv_s; |
| |
|
| |
mv_s = pool_cache_get(sc->sc_session_pool, 0); |
| |
if (mv_s == NULL) { |
| |
log(LOG_ERR, "%s: cannot allocate memory\n", __func__); |
| |
return NULL; |
| |
} |
| |
mv_s->refs = 1; /* 0 means session is alredy invalid */ |
| |
mv_s->sflags = 0; |
| |
|
| |
return mv_s; |
| |
} |
| |
|
| |
/* |
| |
* deallocate session structure. |
| |
*/ |
| |
STATIC void |
| |
mvxpsec_session_dealloc(struct mvxpsec_session *mv_s) |
| |
{ |
| |
struct mvxpsec_softc *sc = mv_s->sc; |
| |
|
| |
mv_s->sflags |= DELETED; |
| |
mvxpsec_session_unref(mv_s); |
| |
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); |
| |
|
| |
return; |
| |
} |
| |
|
| |
STATIC int |
| |
mvxpsec_session_ref(struct mvxpsec_session *mv_s) |
| |
{ |
| |
uint32_t refs; |
| |
|
| |
if (mv_s->sflags & DELETED) { |
| |
log(LOG_ERR, |
| |
"%s: session is already deleted.\n", __func__); |
| |
return -1; |
| |
} |
| |
|
| |
refs = atomic_inc_32_nv(&mv_s->refs); |
| |
if (refs == 1) { |
| |
/* |
| |
* a session with refs == 0 is |
| |
* already invalidated. revert it. |
| |
* XXX: use CAS ? |
| |
*/ |
| |
atomic_dec_32(&mv_s->refs); |
| |
log(LOG_ERR, |
| |
"%s: session is already invalidated.\n", __func__); |
| |
return -1; |
| |
} |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_session_unref(struct mvxpsec_session *mv_s) |
| |
{ |
| |
uint32_t refs; |
| |
|
| |
refs = atomic_dec_32_nv(&mv_s->refs); |
| |
if (refs == 0) |
| |
pool_cache_put(mv_s->sc->sc_session_pool, mv_s); |
| |
} |
| |
|
| |
/* |
| |
* look for session is exist or not |
| |
*/ |
| |
INLINE struct mvxpsec_session * |
| |
mvxpsec_session_lookup(struct mvxpsec_softc *sc, int sid) |
| |
{ |
| |
struct mvxpsec_session *mv_s; |
| |
int session; |
| |
|
| |
/* must called sc->sc_session_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_session_mtx)); |
| |
|
| |
session = MVXPSEC_SESSION(sid); |
| |
if (__predict_false(session > MVXPSEC_MAX_SESSIONS)) { |
| |
log(LOG_ERR, "%s: session number too large %d\n", |
| |
__func__, session); |
| |
return NULL; |
| |
} |
| |
if (__predict_false( (mv_s = sc->sc_sessions[session]) == NULL)) { |
| |
log(LOG_ERR, "%s: invalid session %d\n", |
| |
__func__, session); |
| |
return NULL; |
| |
} |
| |
|
| |
KASSERT(mv_s->sid == session); |
| |
|
| |
return mv_s; |
| |
} |
| |
|
| |
/* |
| |
* allocation new packet structure. |
| |
*/ |
| |
STATIC struct mvxpsec_packet * |
| |
mvxpsec_packet_alloc(struct mvxpsec_session *mv_s) |
| |
{ |
| |
struct mvxpsec_softc *sc = mv_s->sc; |
| |
struct mvxpsec_packet *mv_p; |
| |
|
| |
/* must be called mv_queue_mtx held. */ |
| |
KASSERT(mutex_owned(&sc->sc_queue_mtx)); |
| |
/* must be called mv_session_mtx held. */ |
| |
KASSERT(mutex_owned(&sc->sc_session_mtx)); |
| |
|
| |
if (mvxpsec_session_ref(mv_s) < 0) { |
| |
log(LOG_ERR, "%s: invalid session.\n", __func__); |
| |
return NULL; |
| |
} |
| |
|
| |
if ( (mv_p = SLIST_FIRST(&sc->sc_free_list)) != NULL) { |
| |
SLIST_REMOVE_HEAD(&sc->sc_free_list, free_list); |
| |
sc->sc_free_qlen--; |
| |
} |
| |
else { |
| |
mv_p = pool_cache_get(sc->sc_packet_pool, 0); |
| |
if (mv_p == NULL) { |
| |
log(LOG_ERR, "%s: cannot allocate memory\n", |
| |
__func__); |
| |
mvxpsec_session_unref(mv_s); |
| |
return NULL; |
| |
} |
| |
} |
| |
mv_p->mv_s = mv_s; |
| |
mv_p->flags = 0; |
| |
mv_p->data_ptr = NULL; |
| |
|
| |
return mv_p; |
| |
} |
| |
|
| |
/* |
| |
* free packet structure. |
| |
*/ |
| |
STATIC void |
| |
mvxpsec_packet_dealloc(struct mvxpsec_packet *mv_p) |
| |
{ |
| |
struct mvxpsec_session *mv_s = mv_p->mv_s; |
| |
struct mvxpsec_softc *sc = mv_s->sc; |
| |
|
| |
/* must called with sc->sc_queue_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_queue_mtx)); |
| |
|
| |
if (mv_p->dma_ring.dma_size != 0) { |
| |
sc->sc_desc_ring_cons += mv_p->dma_ring.dma_size; |
| |
} |
| |
mv_p->dma_ring.dma_head = NULL; |
| |
mv_p->dma_ring.dma_last = NULL; |
| |
mv_p->dma_ring.dma_size = 0; |
| |
|
| |
if (mv_p->data_map) { |
| |
if (mv_p->flags & RDY_DATA) { |
| |
bus_dmamap_unload(sc->sc_dmat, mv_p->data_map); |
| |
mv_p->flags &= ~RDY_DATA; |
| |
} |
| |
} |
| |
|
| |
if (sc->sc_free_qlen > sc->sc_wait_qlimit) |
| |
pool_cache_put(sc->sc_packet_pool, mv_p); |
| |
else { |
| |
SLIST_INSERT_HEAD(&sc->sc_free_list, mv_p, free_list); |
| |
sc->sc_free_qlen++; |
| |
} |
| |
mvxpsec_session_unref(mv_s); |
| |
} |
| |
|
| |
INLINE void |
| |
mvxpsec_packet_enqueue(struct mvxpsec_packet *mv_p) |
| |
{ |
| |
struct mvxpsec_softc *sc = mv_p->mv_s->sc; |
| |
struct mvxpsec_packet *last_packet; |
| |
struct mvxpsec_descriptor_handle *cur_dma, *prev_dma; |
| |
|
| |
/* must called with sc->sc_queue_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_queue_mtx)); |
| |
|
| |
if (sc->sc_wait_qlen == 0) { |
| |
SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue); |
| |
sc->sc_wait_qlen++; |
| |
mv_p->flags |= SETUP_DONE; |
| |
return; |
| |
} |
| |
|
| |
last_packet = SIMPLEQ_LAST(&sc->sc_wait_queue, mvxpsec_packet, queue); |
| |
SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue); |
| |
sc->sc_wait_qlen++; |
| |
|
| |
/* chain the DMA */ |
| |
cur_dma = mv_p->dma_ring.dma_head; |
| |
prev_dma = last_packet->dma_ring.dma_last; |
| |
mvxpsec_dma_cat(sc, prev_dma, cur_dma); |
| |
mv_p->flags |= SETUP_DONE; |
| |
} |
| |
|
| |
/* |
| |
* called by interrupt handler |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_done_packet(struct mvxpsec_packet *mv_p) |
| |
{ |
| |
struct mvxpsec_session *mv_s = mv_p->mv_s; |
| |
struct mvxpsec_softc *sc = mv_s->sc; |
| |
|
| |
KASSERT((mv_p->flags & RDY_DATA)); |
| |
KASSERT((mv_p->flags & SETUP_DONE)); |
| |
|
| |
/* unload data */ |
| |
bus_dmamap_sync(sc->sc_dmat, mv_p->data_map, |
| |
0, mv_p->data_len, |
| |
BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); |
| |
bus_dmamap_unload(sc->sc_dmat, mv_p->data_map); |
| |
mv_p->flags &= ~RDY_DATA; |
| |
|
| |
#ifdef MVXPSEC_DEBUG |
| |
if (mvxpsec_debug != 0) { |
| |
int s; |
| |
|
| |
bus_dmamap_sync(sc->sc_dmat, mv_p->pkt_header_map, |
| |
0, sizeof(mv_p->pkt_header), |
| |
BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); |
| |
bus_dmamap_sync(sc->sc_dmat, mv_s->session_header_map, |
| |
0, sizeof(mv_s->session_header), |
| |
BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); |
| |
|
| |
if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) { |
| |
char buf[1500]; |
| |
struct mbuf *m; |
| |
struct uio *uio; |
| |
size_t len; |
| |
|
| |
switch (mv_p->data_type) { |
| |
case MVXPSEC_DATA_MBUF: |
| |
m = mv_p->data_mbuf; |
| |
len = m->m_pkthdr.len; |
| |
if (len > sizeof(buf)) |
| |
len = sizeof(buf); |
| |
m_copydata(m, 0, len, buf); |
| |
break; |
| |
case MVXPSEC_DATA_UIO: |
| |
uio = mv_p->data_uio; |
| |
len = uio->uio_resid; |
| |
if (len > sizeof(buf)) |
| |
len = sizeof(buf); |
| |
cuio_copydata(uio, 0, len, buf); |
| |
break; |
| |
default: |
| |
len = 0; |
| |
} |
| |
if (len > 0) |
| |
mvxpsec_dump_data(__func__, buf, len); |
| |
} |
| |
|
| |
if (mvxpsec_debug & MVXPSEC_DEBUG_PAYLOAD) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD, |
| |
"%s: session_descriptor:\n", __func__); |
| |
mvxpsec_dump_packet_desc(__func__, mv_p); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD, |
| |
"%s: session_data:\n", __func__); |
| |
mvxpsec_dump_packet_data(__func__, mv_p); |
| |
} |
| |
|
| |
if (mvxpsec_debug & MVXPSEC_DEBUG_SRAM) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_SRAM, |
| |
"%s: SRAM\n", __func__); |
| |
mvxpsec_dump_sram(__func__, sc, 2000); |
| |
} |
| |
|
| |
s = MVXPSEC_READ(sc, MV_ACC_STATUS); |
| |
if (s & MV_ACC_STATUS_MAC_ERR) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, |
| |
"%s: Message Authentication Failed.\n", __func__); |
| |
} |
| |
} |
| |
#endif |
| |
|
| |
/* copy back IV */ |
| |
if (mv_p->flags & CRP_EXT_IV) { |
| |
memcpy(mv_p->ext_iv, |
| |
&mv_p->pkt_header.crp_iv_ext, mv_p->ext_ivlen); |
| |
mv_p->ext_iv = NULL; |
| |
mv_p->ext_ivlen = 0; |
| |
} |
| |
|
| |
/* notify opencrypto */ |
| |
mv_p->crp->crp_etype = 0; |
| |
crypto_done(mv_p->crp); |
| |
mv_p->crp = NULL; |
| |
|
| |
/* unblock driver */ |
| |
mvxpsec_packet_dealloc(mv_p); |
| |
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); |
| |
|
| |
MVXPSEC_EVCNT_INCR(sc, packet_ok); |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
|
| |
/* |
| |
* Opencrypto API registration |
| |
*/ |
| |
int |
| |
mvxpsec_register(struct mvxpsec_softc *sc) |
| |
{ |
| |
int oplen = SRAM_PAYLOAD_SIZE; |
| |
int flags = 0; |
| |
int err; |
| |
|
| |
sc->sc_nsessions = 0; |
| |
sc->sc_cid = crypto_get_driverid(0); |
| |
if (sc->sc_cid < 0) { |
| |
log(LOG_ERR, |
| |
"%s: crypto_get_driverid() failed.\n", __func__); |
| |
err = EINVAL; |
| |
goto done; |
| |
} |
| |
|
| |
/* Ciphers */ |
| |
err = crypto_register(sc->sc_cid, CRYPTO_DES_CBC, oplen, flags, |
| |
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); |
| |
if (err) |
| |
goto done; |
| |
|
| |
err = crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, oplen, flags, |
| |
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); |
| |
if (err) |
| |
goto done; |
| |
|
| |
err = crypto_register(sc->sc_cid, CRYPTO_AES_CBC, oplen, flags, |
| |
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); |
| |
if (err) |
| |
goto done; |
| |
|
| |
/* MACs */ |
| |
err = crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, |
| |
oplen, flags, |
| |
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); |
| |
if (err) |
| |
goto done; |
| |
|
| |
err = crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, |
| |
oplen, flags, |
| |
mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); |
| |
if (err) |
| |
goto done; |
| |
|
| |
#ifdef DEBUG |
| |
log(LOG_DEBUG, |
| |
"%s: registered to opencrypto(max data = %d bytes)\n", |
| |
device_xname(sc->sc_dev), oplen); |
| |
#endif |
| |
|
| |
err = 0; |
| |
done: |
| |
return err; |
| |
} |
| |
|
| |
/* |
| |
* Create new opencrypto session |
| |
* |
| |
* - register cipher key, mac key. |
| |
* - initialize mac internal state. |
| |
*/ |
| |
int |
| |
mvxpsec_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_session *mv_s = NULL; |
| |
struct cryptoini *c; |
| |
static int hint = 0; |
| |
int session = -1; |
| |
int sid; |
| |
int err; |
| |
int i; |
| |
|
| |
/* allocate driver session context */ |
| |
mv_s = mvxpsec_session_alloc(sc); |
| |
if (mv_s == NULL) |
| |
return ENOMEM; |
| |
|
| |
/* |
| |
* lookup opencrypto session table |
| |
* |
| |
* we have sc_session_mtx after here. |
| |
*/ |
| |
mutex_enter(&sc->sc_session_mtx); |
| |
if (sc->sc_nsessions >= MVXPSEC_MAX_SESSIONS) { |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
log(LOG_ERR, "%s: too many IPsec SA(max %d)\n", |
| |
__func__, MVXPSEC_MAX_SESSIONS); |
| |
mvxpsec_session_dealloc(mv_s); |
| |
return ENOMEM; |
| |
} |
| |
for (i = hint; i < MVXPSEC_MAX_SESSIONS; i++) { |
| |
if (sc->sc_sessions[i]) |
| |
continue; |
| |
session = i; |
| |
hint = session + 1; |
| |
break; |
| |
} |
| |
if (session < 0) { |
| |
for (i = 0; i < hint; i++) { |
| |
if (sc->sc_sessions[i]) |
| |
continue; |
| |
session = i; |
| |
hint = session + 1; |
| |
break; |
| |
} |
| |
if (session < 0) { |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
/* session full */ |
| |
log(LOG_ERR, "%s: too many IPsec SA(max %d)\n", |
| |
__func__, MVXPSEC_MAX_SESSIONS); |
| |
mvxpsec_session_dealloc(mv_s); |
| |
hint = 0; |
| |
return ENOMEM; |
| |
} |
| |
} |
| |
if (hint >= MVXPSEC_MAX_SESSIONS) |
| |
hint = 0; |
| |
sc->sc_nsessions++; |
| |
sc->sc_sessions[session] = mv_s; |
| |
#ifdef DEBUG |
| |
log(LOG_DEBUG, "%s: new session %d allocated\n", __func__, session); |
| |
#endif |
| |
|
| |
sid = MVXPSEC_SID(device_unit(sc->sc_dev), session); |
| |
mv_s->sid = sid; |
| |
|
| |
/* setup the session key ... */ |
| |
for (c = cri; c; c = c->cri_next) { |
| |
switch (c->cri_alg) { |
| |
case CRYPTO_DES_CBC: |
| |
case CRYPTO_3DES_CBC: |
| |
case CRYPTO_AES_CBC: |
| |
/* key */ |
| |
if (mvxpsec_key_precomp(c->cri_alg, |
| |
c->cri_key, c->cri_klen, |
| |
&mv_s->session_header.crp_key, |
| |
&mv_s->session_header.crp_key_d)) { |
| |
log(LOG_ERR, |
| |
"%s: Invalid HMAC key for %s.\n", |
| |
__func__, s_ctlalg(c->cri_alg)); |
| |
err = EINVAL; |
| |
goto fail; |
| |
} |
| |
if (mv_s->sflags & RDY_CRP_KEY) { |
| |
log(LOG_WARNING, |
| |
"%s: overwrite cipher: %s->%s.\n", |
| |
__func__, |
| |
s_ctlalg(mv_s->cipher_alg), |
| |
s_ctlalg(c->cri_alg)); |
| |
} |
| |
mv_s->sflags |= RDY_CRP_KEY; |
| |
mv_s->enc_klen = c->cri_klen; |
| |
mv_s->cipher_alg = c->cri_alg; |
| |
/* create per session IV (compatible with KAME IPsec) */ |
| |
cprng_fast(&mv_s->session_iv, sizeof(mv_s->session_iv)); |
| |
mv_s->sflags |= RDY_CRP_IV; |
| |
break; |
| |
case CRYPTO_SHA1_HMAC_96: |
| |
case CRYPTO_MD5_HMAC_96: |
| |
/* key */ |
| |
if (mvxpsec_hmac_precomp(c->cri_alg, |
| |
c->cri_key, c->cri_klen, |
| |
(uint32_t *)&mv_s->session_header.miv_in, |
| |
(uint32_t *)&mv_s->session_header.miv_out)) { |
| |
log(LOG_ERR, |
| |
"%s: Invalid MAC key\n", __func__); |
| |
err = EINVAL; |
| |
goto fail; |
| |
} |
| |
if (mv_s->sflags & RDY_MAC_KEY || |
| |
mv_s->sflags & RDY_MAC_IV) { |
| |
log(LOG_ERR, |
| |
"%s: overwrite HMAC: %s->%s.\n", |
| |
__func__, s_ctlalg(mv_s->hmac_alg), |
| |
s_ctlalg(c->cri_alg)); |
| |
} |
| |
mv_s->sflags |= RDY_MAC_KEY; |
| |
mv_s->sflags |= RDY_MAC_IV; |
| |
|
| |
mv_s->mac_klen = c->cri_klen; |
| |
mv_s->hmac_alg = c->cri_alg; |
| |
break; |
| |
default: |
| |
log(LOG_ERR, "%s: Unknown algorithm %d\n", |
| |
__func__, c->cri_alg); |
| |
err = EINVAL; |
| |
goto fail; |
| |
} |
| |
} |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"H/W Crypto session (id:%u) added.\n", session); |
| |
|
| |
*sidp = sid; |
| |
MVXPSEC_EVCNT_INCR(sc, session_new); |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
|
| |
/* sync session header(it's never touched after here) */ |
| |
bus_dmamap_sync(sc->sc_dmat, |
| |
mv_s->session_header_map, |
| |
0, sizeof(mv_s->session_header), |
| |
BUS_DMASYNC_PREWRITE); |
| |
|
| |
return 0; |
| |
|
| |
fail: |
| |
sc->sc_nsessions--; |
| |
sc->sc_sessions[session] = NULL; |
| |
hint = session; |
| |
if (mv_s) |
| |
mvxpsec_session_dealloc(mv_s); |
| |
log(LOG_WARNING, |
| |
"%s: Failed to add H/W crypto sessoin (id:%u): err=%d\n", |
| |
__func__, session, err); |
| |
|
| |
mutex_exit(&sc->sc_session_mtx); |
| |
return err; |
| |
} |
| |
|
| |
/* |
| |
* remove opencrypto session |
| |
*/ |
| |
int |
| |
mvxpsec_freesession(void *arg, uint64_t tid) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_session *mv_s; |
| |
int session; |
| |
uint32_t sid = ((uint32_t)tid) & 0xffffffff; |
| |
|
| |
session = MVXPSEC_SESSION(sid); |
| |
if (session < 0 || session >= MVXPSEC_MAX_SESSIONS) { |
| |
log(LOG_ERR, "%s: invalid session (id:%u)\n", |
| |
__func__, session); |
| |
return EINVAL; |
| |
} |
| |
|
| |
mutex_enter(&sc->sc_session_mtx); |
| |
if ( (mv_s = sc->sc_sessions[session]) == NULL) { |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
#ifdef DEBUG |
| |
log(LOG_DEBUG, "%s: session %d already inactivated\n", |
| |
__func__, session); |
| |
#endif |
| |
return ENOENT; |
| |
} |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: inactivate session %d\n", __func__, session); |
| |
|
| |
/* inactivate mvxpsec session */ |
| |
sc->sc_sessions[session] = NULL; |
| |
sc->sc_nsessions--; |
| |
sc->sc_last_session = NULL; |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
|
| |
KASSERT(sc->sc_nsessions >= 0); |
| |
KASSERT(mv_s->sid == sid); |
| |
|
| |
mvxpsec_session_dealloc(mv_s); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"H/W Crypto session (id: %d) deleted.\n", session); |
| |
|
| |
/* force unblock opencrypto */ |
| |
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); |
| |
|
| |
MVXPSEC_EVCNT_INCR(sc, session_free); |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* process data with existing session |
| |
*/ |
| |
int |
| |
mvxpsec_dispatch(void *arg, struct cryptop *crp, int hint) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_session *mv_s; |
| |
struct mvxpsec_packet *mv_p; |
| |
int q_full; |
| |
int running; |
| |
int err; |
| |
|
| |
mutex_enter(&sc->sc_queue_mtx); |
| |
|
| |
/* |
| |
* lookup session |
| |
*/ |
| |
mutex_enter(&sc->sc_session_mtx); |
| |
mv_s = mvxpsec_session_lookup(sc, crp->crp_sid); |
| |
if (__predict_false(mv_s == NULL)) { |
| |
err = EINVAL; |
| |
mv_p = NULL; |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
goto fail; |
| |
} |
| |
mv_p = mvxpsec_packet_alloc(mv_s); |
| |
if (__predict_false(mv_p == NULL)) { |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
mutex_exit(&sc->sc_queue_mtx); |
| |
return ERESTART; /* => queued in opencrypto layer */ |
| |
} |
| |
mutex_exit(&sc->sc_session_mtx); |
| |
|
| |
/* |
| |
* check queue status |
| |
*/ |
| |
#ifdef MVXPSEC_MULTI_PACKET |
| |
q_full = (sc->sc_wait_qlen >= sc->sc_wait_qlimit) ? 1 : 0; |
| |
#else |
| |
q_full = (sc->sc_wait_qlen != 0) ? 1 : 0; |
| |
#endif |
| |
running = (sc->sc_flags & HW_RUNNING) ? 1: 0; |
| |
if (q_full) { |
| |
/* input queue is full. */ |
| |
if (!running && sc->sc_wait_qlen > 0) |
| |
mvxpsec_dispatch_queue(sc); |
| |
MVXPSEC_EVCNT_INCR(sc, queue_full); |
| |
mvxpsec_packet_dealloc(mv_p); |
| |
mutex_exit(&sc->sc_queue_mtx); |
| |
return ERESTART; /* => queued in opencrypto layer */ |
| |
} |
| |
|
| |
/* |
| |
* Load and setup packet data |
| |
*/ |
| |
err = mvxpsec_packet_setcrp(mv_p, crp); |
| |
if (__predict_false(err)) |
| |
goto fail; |
| |
|
| |
/* |
| |
* Setup DMA descriptor chains |
| |
*/ |
| |
mutex_enter(&sc->sc_dma_mtx); |
| |
err = mvxpsec_dma_copy_packet(sc, mv_p); |
| |
mutex_exit(&sc->sc_dma_mtx); |
| |
if (__predict_false(err)) |
| |
goto fail; |
| |
|
| |
#ifdef MVXPSEC_DEBUG |
| |
mvxpsec_dump_packet(__func__, mv_p); |
| |
#endif |
| |
|
| |
/* |
| |
* Sync/inval the data cache |
| |
*/ |
| |
err = mvxpsec_dma_sync_packet(sc, mv_p); |
| |
if (__predict_false(err)) |
| |
goto fail; |
| |
|
| |
/* |
| |
* Enqueue the packet |
| |
*/ |
| |
MVXPSEC_EVCNT_INCR(sc, dispatch_packets); |
| |
#ifdef MVXPSEC_MULTI_PACKET |
| |
mvxpsec_packet_enqueue(mv_p); |
| |
if (!running) |
| |
mvxpsec_dispatch_queue(sc); |
| |
#else |
| |
SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue); |
| |
sc->sc_wait_qlen++; |
| |
mv_p->flags |= SETUP_DONE; |
| |
if (!running) |
| |
mvxpsec_dispatch_queue(sc); |
| |
#endif |
| |
mutex_exit(&sc->sc_queue_mtx); |
| |
return 0; |
| |
|
| |
fail: |
| |
/* Drop the incoming packet */ |
| |
mvxpsec_drop(sc, crp, mv_p, err); |
| |
mutex_exit(&sc->sc_queue_mtx); |
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* back the packet to the IP stack |
| |
*/ |
| |
void |
| |
mvxpsec_done(void *arg) |
| |
{ |
| |
struct mvxpsec_softc *sc = arg; |
| |
struct mvxpsec_packet *mv_p; |
| |
mvxpsec_queue_t ret_queue; |
| |
int ndone; |
| |
|
| |
mutex_enter(&sc->sc_queue_mtx); |
| |
|
| |
/* stop wdog timer */ |
| |
callout_stop(&sc->sc_timeout); |
| |
|
| |
/* refill MVXPSEC */ |
| |
ret_queue = sc->sc_run_queue; |
| |
SIMPLEQ_INIT(&sc->sc_run_queue); |
| |
sc->sc_flags &= ~HW_RUNNING; |
| |
if (sc->sc_wait_qlen > 0) |
| |
mvxpsec_dispatch_queue(sc); |
| |
|
| |
ndone = 0; |
| |
while ( (mv_p = SIMPLEQ_FIRST(&ret_queue)) != NULL) { |
| |
SIMPLEQ_REMOVE_HEAD(&ret_queue, queue); |
| |
mvxpsec_dma_free(sc, &mv_p->dma_ring); |
| |
mvxpsec_done_packet(mv_p); |
| |
ndone++; |
| |
} |
| |
MVXPSEC_EVCNT_MAX(sc, max_done, ndone); |
| |
|
| |
mutex_exit(&sc->sc_queue_mtx); |
| |
} |
| |
|
| |
/* |
| |
* drop the packet |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_drop(struct mvxpsec_softc *sc, struct cryptop *crp, |
| |
struct mvxpsec_packet *mv_p, int err) |
| |
{ |
| |
/* must called with sc->sc_queue_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_queue_mtx)); |
| |
|
| |
if (mv_p) |
| |
mvxpsec_packet_dealloc(mv_p); |
| |
if (err < 0) |
| |
err = EINVAL; |
| |
crp->crp_etype = err; |
| |
crypto_done(crp); |
| |
MVXPSEC_EVCNT_INCR(sc, packet_err); |
| |
|
| |
/* dispatch other packets in queue */ |
| |
if (sc->sc_wait_qlen > 0 && |
| |
!(sc->sc_flags & HW_RUNNING)) |
| |
mvxpsec_dispatch_queue(sc); |
| |
|
| |
/* unblock driver for dropped packet */ |
| |
crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); |
| |
} |
| |
|
| |
/* move wait queue entry to run queue */ |
| |
STATIC int |
| |
mvxpsec_dispatch_queue(struct mvxpsec_softc *sc) |
| |
{ |
| |
struct mvxpsec_packet *mv_p; |
| |
paddr_t head; |
| |
int ndispatch = 0; |
| |
|
| |
/* must called with sc->sc_queue_mtx held */ |
| |
KASSERT(mutex_owned(&sc->sc_queue_mtx)); |
| |
|
| |
/* check there is any task */ |
| |
if (__predict_false(sc->sc_flags & HW_RUNNING)) { |
| |
log(LOG_WARNING, |
| |
"%s: another packet already exist.\n", __func__); |
| |
return 0; |
| |
} |
| |
if (__predict_false(SIMPLEQ_EMPTY(&sc->sc_wait_queue))) { |
| |
log(LOG_WARNING, |
| |
"%s: no waiting packet yet(qlen=%d).\n", |
| |
__func__, sc->sc_wait_qlen); |
| |
return 0; |
| |
} |
| |
|
| |
/* move queue */ |
| |
sc->sc_run_queue = sc->sc_wait_queue; |
| |
sc->sc_flags |= HW_RUNNING; /* dropped by intr or timeout */ |
| |
SIMPLEQ_INIT(&sc->sc_wait_queue); |
| |
ndispatch = sc->sc_wait_qlen; |
| |
sc->sc_wait_qlen = 0; |
| |
|
| |
/* get 1st DMA descriptor */ |
| |
mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue); |
| |
head = mv_p->dma_ring.dma_head->phys_addr; |
| |
|
| |
/* terminate last DMA descriptor */ |
| |
mv_p = SIMPLEQ_LAST(&sc->sc_run_queue, mvxpsec_packet, queue); |
| |
mvxpsec_dma_finalize(sc, &mv_p->dma_ring); |
| |
|
| |
/* configure TDMA */ |
| |
if (mvxpsec_dma_wait(sc) < 0) { |
| |
log(LOG_ERR, "%s: DMA DEVICE not responding", __func__); |
| |
callout_schedule(&sc->sc_timeout, hz); |
| |
return 0; |
| |
} |
| |
MVXPSEC_WRITE(sc, MV_TDMA_NXT, head); |
| |
|
| |
/* trigger ACC */ |
| |
if (mvxpsec_acc_wait(sc) < 0) { |
| |
log(LOG_ERR, "%s: MVXPSEC not responding", __func__); |
| |
callout_schedule(&sc->sc_timeout, hz); |
| |
return 0; |
| |
} |
| |
MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_ACT); |
| |
|
| |
MVXPSEC_EVCNT_MAX(sc, max_dispatch, ndispatch); |
| |
MVXPSEC_EVCNT_INCR(sc, dispatch_queue); |
| |
callout_schedule(&sc->sc_timeout, hz); |
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* process opencrypto operations(cryptop) for packets. |
| |
*/ |
| |
INLINE int |
| |
mvxpsec_parse_crd(struct mvxpsec_packet *mv_p, struct cryptodesc *crd) |
| |
{ |
| |
int ivlen; |
| |
|
| |
KASSERT(mv_p->flags & RDY_DATA); |
| |
|
| |
/* MAC & Ciphers: set data location and operation */ |
| |
switch (crd->crd_alg) { |
| |
case CRYPTO_SHA1_HMAC_96: |
| |
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96; |
| |
/* fall through */ |
| |
case CRYPTO_SHA1_HMAC: |
| |
mv_p->mac_dst = crd->crd_inject; |
| |
mv_p->mac_off = crd->crd_skip; |
| |
mv_p->mac_len = crd->crd_len; |
| |
MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config, |
| |
MV_ACC_CRYPTO_MAC_HMAC_SHA1); |
| |
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC); |
| |
/* No more setup for MAC */ |
| |
return 0; |
| |
case CRYPTO_MD5_HMAC_96: |
| |
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96; |
| |
/* fall through */ |
| |
case CRYPTO_MD5_HMAC: |
| |
mv_p->mac_dst = crd->crd_inject; |
| |
mv_p->mac_off = crd->crd_skip; |
| |
mv_p->mac_len = crd->crd_len; |
| |
MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config, |
| |
MV_ACC_CRYPTO_MAC_HMAC_MD5); |
| |
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC); |
| |
/* No more setup for MAC */ |
| |
return 0; |
| |
case CRYPTO_DES_CBC: |
| |
mv_p->enc_ivoff = crd->crd_inject; |
| |
mv_p->enc_off = crd->crd_skip; |
| |
mv_p->enc_len = crd->crd_len; |
| |
ivlen = 8; |
| |
MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config, |
| |
MV_ACC_CRYPTO_ENC_DES); |
| |
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC; |
| |
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC); |
| |
break; |
| |
case CRYPTO_3DES_CBC: |
| |
mv_p->enc_ivoff = crd->crd_inject; |
| |
mv_p->enc_off = crd->crd_skip; |
| |
mv_p->enc_len = crd->crd_len; |
| |
ivlen = 8; |
| |
MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config, |
| |
MV_ACC_CRYPTO_ENC_3DES); |
| |
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC; |
| |
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_3DES_EDE; |
| |
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC); |
| |
break; |
| |
case CRYPTO_AES_CBC: |
| |
mv_p->enc_ivoff = crd->crd_inject; |
| |
mv_p->enc_off = crd->crd_skip; |
| |
mv_p->enc_len = crd->crd_len; |
| |
ivlen = 16; |
| |
MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config, |
| |
MV_ACC_CRYPTO_ENC_AES); |
| |
MV_ACC_CRYPTO_AES_KLEN_SET( |
| |
mv_p->pkt_header.desc.acc_config, |
| |
mvxpsec_aesklen(mv_p->mv_s->enc_klen)); |
| |
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC; |
| |
mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC); |
| |
break; |
| |
default: |
| |
log(LOG_ERR, "%s: Unknown algorithm %d\n", |
| |
__func__, crd->crd_alg); |
| |
return EINVAL; |
| |
} |
| |
|
| |
/* Operations only for Cipher, not MAC */ |
| |
if (crd->crd_flags & CRD_F_ENCRYPT) { |
| |
/* Ciphers: Originate IV for Encryption.*/ |
| |
mv_p->pkt_header.desc.acc_config &= ~MV_ACC_CRYPTO_DECRYPT; |
| |
mv_p->flags |= DIR_ENCRYPT; |
| |
|
| |
if (crd->crd_flags & CRD_F_IV_EXPLICIT) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "EXPLICIT IV\n"); |
| |
mv_p->flags |= CRP_EXT_IV; |
| |
mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen); |
| |
mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF; |
| |
} |
| |
else if (crd->crd_flags & CRD_F_IV_PRESENT) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "IV is present\n"); |
| |
mvxpsec_packet_copy_iv(mv_p, crd->crd_inject, ivlen); |
| |
} |
| |
else { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "Create New IV\n"); |
| |
mvxpsec_packet_write_iv(mv_p, NULL, ivlen); |
| |
} |
| |
} |
| |
else { |
| |
/* Ciphers: IV is loadded from crd_inject when it's present */ |
| |
mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_DECRYPT; |
| |
mv_p->flags |= DIR_DECRYPT; |
| |
|
| |
if (crd->crd_flags & CRD_F_IV_EXPLICIT) { |
| |
#ifdef MVXPSEC_DEBUG |
| |
if (mvxpsec_debug & MVXPSEC_DEBUG_ENC_IV) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, |
| |
"EXPLICIT IV(Decrypt)\n"); |
| |
mvxpsec_dump_data(__func__, crd->crd_iv, ivlen); |
| |
} |
| |
#endif |
| |
mv_p->flags |= CRP_EXT_IV; |
| |
mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen); |
| |
mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF; |
| |
} |
| |
} |
| |
|
| |
KASSERT(!((mv_p->flags & DIR_ENCRYPT) && (mv_p->flags & DIR_DECRYPT))); |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
INLINE int |
| |
mvxpsec_parse_crp(struct mvxpsec_packet *mv_p) |
| |
{ |
| |
struct cryptop *crp = mv_p->crp; |
| |
struct cryptodesc *crd; |
| |
int err; |
| |
|
| |
KASSERT(crp); |
| |
|
| |
mvxpsec_packet_reset_op(mv_p); |
| |
|
| |
for (crd = crp->crp_desc; crd; crd = crd->crd_next) { |
| |
err = mvxpsec_parse_crd(mv_p, crd); |
| |
if (err) |
| |
return err; |
| |
} |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
INLINE int |
| |
mvxpsec_packet_setcrp(struct mvxpsec_packet *mv_p, struct cryptop *crp) |
| |
{ |
| |
int err = EINVAL; |
| |
|
| |
/* regiseter crp to the MVXPSEC packet */ |
| |
if (crp->crp_flags & CRYPTO_F_IMBUF) { |
| |
err = mvxpsec_packet_setmbuf(mv_p, |
| |
(struct mbuf *)crp->crp_buf); |
| |
mv_p->crp = crp; |
| |
} |
| |
else if (crp->crp_flags & CRYPTO_F_IOV) { |
| |
err = mvxpsec_packet_setuio(mv_p, |
| |
(struct uio *)crp->crp_buf); |
| |
mv_p->crp = crp; |
| |
} |
| |
else { |
| |
err = mvxpsec_packet_setdata(mv_p, |
| |
(struct mbuf *)crp->crp_buf, crp->crp_ilen); |
| |
mv_p->crp = crp; |
| |
} |
| |
if (__predict_false(err)) |
| |
return err; |
| |
|
| |
/* parse crp and setup MVXPSEC registers/descriptors */ |
| |
err = mvxpsec_parse_crp(mv_p); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
|
| |
/* fixup data offset to fit MVXPSEC internal SRAM */ |
| |
err = mvxpsec_header_finalize(mv_p); |
| |
if (__predict_false(err)) |
| |
return err; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* load data for encrypt/decrypt/authentication |
| |
* |
| |
* data is raw kernel memory area. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_packet_setdata(struct mvxpsec_packet *mv_p, |
| |
void *data, uint32_t data_len) |
| |
{ |
| |
struct mvxpsec_session *mv_s = mv_p->mv_s; |
| |
struct mvxpsec_softc *sc = mv_s->sc; |
| |
|
| |
if (bus_dmamap_load(sc->sc_dmat, mv_p->data_map, data, data_len, |
| |
NULL, BUS_DMA_NOWAIT)) { |
| |
log(LOG_ERR, "%s: cannot load data\n", __func__); |
| |
return -1; |
| |
} |
| |
mv_p->data_type = MVXPSEC_DATA_RAW; |
| |
mv_p->data_raw = data; |
| |
mv_p->data_len = data_len; |
| |
mv_p->flags |= RDY_DATA; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* load data for encrypt/decrypt/authentication |
| |
* |
| |
* data is mbuf based network data. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_packet_setmbuf(struct mvxpsec_packet *mv_p, struct mbuf *m) |
| |
{ |
| |
struct mvxpsec_session *mv_s = mv_p->mv_s; |
| |
struct mvxpsec_softc *sc = mv_s->sc; |
| |
size_t pktlen = 0; |
| |
|
| |
if (__predict_true(m->m_flags & M_PKTHDR)) |
| |
pktlen = m->m_pkthdr.len; |
| |
else { |
| |
struct mbuf *mp = m; |
| |
|
| |
while (mp != NULL) { |
| |
pktlen += m->m_len; |
| |
mp = mp->m_next; |
| |
} |
| |
} |
| |
if (pktlen > SRAM_PAYLOAD_SIZE) { |
| |
extern percpu_t *espstat_percpu; |
| |
/* XXX: |
| |
* layer violation. opencrypto knows our max packet size |
| |
* from crypto_register(9) API. |
| |
*/ |
| |
|
| |
_NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG); |
| |
log(LOG_ERR, |
| |
"%s: ESP Packet too large: %zu [oct.] > %zu [oct.]\n", |
| |
device_xname(sc->sc_dev), |
| |
(size_t)pktlen, SRAM_PAYLOAD_SIZE); |
| |
mv_p->data_type = MVXPSEC_DATA_NONE; |
| |
mv_p->data_mbuf = NULL; |
| |
return -1; |
| |
} |
| |
|
| |
if (bus_dmamap_load_mbuf(sc->sc_dmat, mv_p->data_map, m, |
| |
BUS_DMA_NOWAIT)) { |
| |
mv_p->data_type = MVXPSEC_DATA_NONE; |
| |
mv_p->data_mbuf = NULL; |
| |
log(LOG_ERR, "%s: cannot load mbuf\n", __func__); |
| |
return -1; |
| |
} |
| |
|
| |
/* set payload buffer */ |
| |
mv_p->data_type = MVXPSEC_DATA_MBUF; |
| |
mv_p->data_mbuf = m; |
| |
if (m->m_flags & M_PKTHDR) { |
| |
mv_p->data_len = m->m_pkthdr.len; |
| |
} |
| |
else { |
| |
mv_p->data_len = 0; |
| |
while (m) { |
| |
mv_p->data_len += m->m_len; |
| |
m = m->m_next; |
| |
} |
| |
} |
| |
mv_p->flags |= RDY_DATA; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
STATIC int |
| |
mvxpsec_packet_setuio(struct mvxpsec_packet *mv_p, struct uio *uio) |
| |
{ |
| |
struct mvxpsec_session *mv_s = mv_p->mv_s; |
| |
struct mvxpsec_softc *sc = mv_s->sc; |
| |
|
| |
if (uio->uio_resid > SRAM_PAYLOAD_SIZE) { |
| |
extern percpu_t *espstat_percpu; |
| |
/* XXX: |
| |
* layer violation. opencrypto knows our max packet size |
| |
* from crypto_register(9) API. |
| |
*/ |
| |
|
| |
_NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG); |
| |
log(LOG_ERR, |
| |
"%s: uio request too large: %zu [oct.] > %zu [oct.]\n", |
| |
device_xname(sc->sc_dev), |
| |
uio->uio_resid, SRAM_PAYLOAD_SIZE); |
| |
mv_p->data_type = MVXPSEC_DATA_NONE; |
| |
mv_p->data_mbuf = NULL; |
| |
return -1; |
| |
} |
| |
|
| |
if (bus_dmamap_load_uio(sc->sc_dmat, mv_p->data_map, uio, |
| |
BUS_DMA_NOWAIT)) { |
| |
mv_p->data_type = MVXPSEC_DATA_NONE; |
| |
mv_p->data_mbuf = NULL; |
| |
log(LOG_ERR, "%s: cannot load uio buf\n", __func__); |
| |
return -1; |
| |
} |
| |
|
| |
/* set payload buffer */ |
| |
mv_p->data_type = MVXPSEC_DATA_UIO; |
| |
mv_p->data_uio = uio; |
| |
mv_p->data_len = uio->uio_resid; |
| |
mv_p->flags |= RDY_DATA; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
STATIC int |
| |
mvxpsec_packet_rdata(struct mvxpsec_packet *mv_p, |
| |
int off, int len, void *cp) |
| |
{ |
| |
uint8_t *p; |
| |
|
| |
if (mv_p->data_type == MVXPSEC_DATA_RAW) { |
| |
p = (uint8_t *)mv_p->data_raw + off; |
| |
memcpy(cp, p, len); |
| |
} |
| |
else if (mv_p->data_type == MVXPSEC_DATA_MBUF) { |
| |
m_copydata(mv_p->data_mbuf, off, len, cp); |
| |
} |
| |
else if (mv_p->data_type == MVXPSEC_DATA_UIO) { |
| |
cuio_copydata(mv_p->data_uio, off, len, cp); |
| |
} |
| |
else |
| |
return -1; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
STATIC int |
| |
mvxpsec_packet_wdata(struct mvxpsec_packet *mv_p, |
| |
int off, int len, void *cp) |
| |
{ |
| |
uint8_t *p; |
| |
|
| |
if (mv_p->data_type == MVXPSEC_DATA_RAW) { |
| |
p = (uint8_t *)mv_p->data_raw + off; |
| |
memcpy(p, cp, len); |
| |
} |
| |
else if (mv_p->data_type == MVXPSEC_DATA_MBUF) { |
| |
m_copyback(mv_p->data_mbuf, off, len, cp); |
| |
} |
| |
else if (mv_p->data_type == MVXPSEC_DATA_UIO) { |
| |
cuio_copyback(mv_p->data_uio, off, len, cp); |
| |
} |
| |
else |
| |
return -1; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Set initial vector of cipher to the session. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_packet_write_iv(struct mvxpsec_packet *mv_p, void *iv, int ivlen) |
| |
{ |
| |
uint8_t ivbuf[16]; |
| |
|
| |
KASSERT(ivlen == 8 || ivlen == 16); |
| |
|
| |
if (iv == NULL) { |
| |
if (mv_p->mv_s->sflags & RDY_CRP_IV) { |
| |
/* use per session IV (compatible with KAME IPsec) */ |
| |
mv_p->pkt_header.crp_iv_work = mv_p->mv_s->session_iv; |
| |
mv_p->flags |= RDY_CRP_IV; |
| |
return 0; |
| |
} |
| |
cprng_fast(ivbuf, ivlen); |
| |
iv = ivbuf; |
| |
} |
| |
memcpy(&mv_p->pkt_header.crp_iv_work, iv, ivlen); |
| |
if (mv_p->flags & CRP_EXT_IV) { |
| |
memcpy(&mv_p->pkt_header.crp_iv_ext, iv, ivlen); |
| |
mv_p->ext_iv = iv; |
| |
mv_p->ext_ivlen = ivlen; |
| |
} |
| |
mv_p->flags |= RDY_CRP_IV; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
STATIC int |
| |
mvxpsec_packet_copy_iv(struct mvxpsec_packet *mv_p, int off, int ivlen) |
| |
{ |
| |
mvxpsec_packet_rdata(mv_p, off, ivlen, |
| |
&mv_p->pkt_header.crp_iv_work); |
| |
mv_p->flags |= RDY_CRP_IV; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* set a encryption or decryption key to the session |
| |
* |
| |
* Input key material is big endian. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_key_precomp(int alg, void *keymat, int kbitlen, |
| |
void *key_encrypt, void *key_decrypt) |
| |
{ |
| |
uint32_t *kp = keymat; |
| |
uint32_t *ekp = key_encrypt; |
| |
uint32_t *dkp = key_decrypt; |
| |
int i; |
| |
|
| |
switch (alg) { |
| |
case CRYPTO_DES_CBC: |
| |
if (kbitlen < 64 || (kbitlen % 8) != 0) { |
| |
log(LOG_WARNING, |
| |
"mvxpsec: invalid DES keylen %d\n", kbitlen); |
| |
return EINVAL; |
| |
} |
| |
for (i = 0; i < 2; i++) |
| |
dkp[i] = ekp[i] = kp[i]; |
| |
for (; i < 8; i++) |
| |
dkp[i] = ekp[i] = 0; |
| |
break; |
| |
case CRYPTO_3DES_CBC: |
| |
if (kbitlen < 192 || (kbitlen % 8) != 0) { |
| |
log(LOG_WARNING, |
| |
"mvxpsec: invalid 3DES keylen %d\n", kbitlen); |
| |
return EINVAL; |
| |
} |
| |
for (i = 0; i < 8; i++) |
| |
dkp[i] = ekp[i] = kp[i]; |
| |
break; |
| |
case CRYPTO_AES_CBC: |
| |
if (kbitlen < 128) { |
| |
log(LOG_WARNING, |
| |
"mvxpsec: invalid AES keylen %d\n", kbitlen); |
| |
return EINVAL; |
| |
} |
| |
else if (kbitlen < 192) { |
| |
/* AES-128 */ |
| |
for (i = 0; i < 4; i++) |
| |
ekp[i] = kp[i]; |
| |
for (; i < 8; i++) |
| |
ekp[i] = 0; |
| |
} |
| |
else if (kbitlen < 256) { |
| |
/* AES-192 */ |
| |
for (i = 0; i < 6; i++) |
| |
ekp[i] = kp[i]; |
| |
for (; i < 8; i++) |
| |
ekp[i] = 0; |
| |
} |
| |
else { |
| |
/* AES-256 */ |
| |
for (i = 0; i < 8; i++) |
| |
ekp[i] = kp[i]; |
| |
} |
| |
/* make decryption key */ |
| |
mv_aes_deckey((uint8_t *)dkp, (uint8_t *)ekp, kbitlen); |
| |
break; |
| |
default: |
| |
for (i = 0; i < 8; i++) |
| |
ekp[0] = dkp[0] = 0; |
| |
break; |
| |
} |
| |
|
| |
#ifdef MVXPSEC_DEBUG |
| |
if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: keyregistered\n", __func__); |
| |
mvxpsec_dump_data(__func__, ekp, 32); |
| |
} |
| |
#endif |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* set MAC key to the session |
| |
* |
| |
* MAC engine has no register for key itself, but the engine has |
| |
* inner and outer IV register. software must compute IV before |
| |
* enable the engine. |
| |
* |
| |
* IV is a hash of ipad/opad. these are defined by FIPS-198a |
| |
* standard. |
| |
*/ |
| |
STATIC int |
| |
mvxpsec_hmac_precomp(int alg, void *key, int kbitlen, |
| |
void *iv_inner, void *iv_outer) |
| |
{ |
| |
SHA1_CTX sha1; |
| |
MD5_CTX md5; |
| |
uint8_t *key8 = key; |
| |
uint8_t kbuf[64]; |
| |
uint8_t ipad[64]; |
| |
uint8_t opad[64]; |
| |
uint32_t *iv_in = iv_inner; |
| |
uint32_t *iv_out = iv_outer; |
| |
int kbytelen; |
| |
int i; |
| |
#define HMAC_IPAD 0x36 |
| |
#define HMAC_OPAD 0x5c |
| |
|
| |
kbytelen = kbitlen / 8; |
| |
KASSERT(kbitlen == kbytelen * 8); |
| |
if (kbytelen > 64) { |
| |
SHA1Init(&sha1); |
| |
SHA1Update(&sha1, key, kbytelen); |
| |
SHA1Final(kbuf, &sha1); |
| |
key8 = kbuf; |
| |
kbytelen = 64; |
| |
} |
| |
|
| |
/* make initial 64 oct. string */ |
| |
switch (alg) { |
| |
case CRYPTO_SHA1_HMAC_96: |
| |
case CRYPTO_SHA1_HMAC: |
| |
case CRYPTO_MD5_HMAC_96: |
| |
case CRYPTO_MD5_HMAC: |
| |
for (i = 0; i < kbytelen; i++) { |
| |
ipad[i] = (key8[i] ^ HMAC_IPAD); |
| |
opad[i] = (key8[i] ^ HMAC_OPAD); |
| |
} |
| |
for (; i < 64; i++) { |
| |
ipad[i] = HMAC_IPAD; |
| |
opad[i] = HMAC_OPAD; |
| |
} |
| |
break; |
| |
default: |
| |
break; |
| |
} |
| |
#ifdef MVXPSEC_DEBUG |
| |
if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: HMAC-KEY Pre-comp:\n", __func__); |
| |
mvxpsec_dump_data(__func__, key, 64); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: ipad:\n", __func__); |
| |
mvxpsec_dump_data(__func__, ipad, sizeof(ipad)); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: opad:\n", __func__); |
| |
mvxpsec_dump_data(__func__, opad, sizeof(opad)); |
| |
} |
| |
#endif |
| |
|
| |
/* make iv from string */ |
| |
switch (alg) { |
| |
case CRYPTO_SHA1_HMAC_96: |
| |
case CRYPTO_SHA1_HMAC: |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: Generate iv_in(SHA1)\n", __func__); |
| |
SHA1Init(&sha1); |
| |
SHA1Update(&sha1, ipad, 64); |
| |
/* XXX: private state... (LE) */ |
| |
iv_in[0] = htobe32(sha1.state[0]); |
| |
iv_in[1] = htobe32(sha1.state[1]); |
| |
iv_in[2] = htobe32(sha1.state[2]); |
| |
iv_in[3] = htobe32(sha1.state[3]); |
| |
iv_in[4] = htobe32(sha1.state[4]); |
| |
|
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: Generate iv_out(SHA1)\n", __func__); |
| |
SHA1Init(&sha1); |
| |
SHA1Update(&sha1, opad, 64); |
| |
/* XXX: private state... (LE) */ |
| |
iv_out[0] = htobe32(sha1.state[0]); |
| |
iv_out[1] = htobe32(sha1.state[1]); |
| |
iv_out[2] = htobe32(sha1.state[2]); |
| |
iv_out[3] = htobe32(sha1.state[3]); |
| |
iv_out[4] = htobe32(sha1.state[4]); |
| |
break; |
| |
case CRYPTO_MD5_HMAC_96: |
| |
case CRYPTO_MD5_HMAC: |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: Generate iv_in(MD5)\n", __func__); |
| |
MD5Init(&md5); |
| |
MD5Update(&md5, ipad, sizeof(ipad)); |
| |
/* XXX: private state... (LE) */ |
| |
iv_in[0] = htobe32(md5.state[0]); |
| |
iv_in[1] = htobe32(md5.state[1]); |
| |
iv_in[2] = htobe32(md5.state[2]); |
| |
iv_in[3] = htobe32(md5.state[3]); |
| |
iv_in[4] = 0; |
| |
|
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, |
| |
"%s: Generate iv_out(MD5)\n", __func__); |
| |
MD5Init(&md5); |
| |
MD5Update(&md5, opad, sizeof(opad)); |
| |
/* XXX: private state... (LE) */ |
| |
iv_out[0] = htobe32(md5.state[0]); |
| |
iv_out[1] = htobe32(md5.state[1]); |
| |
iv_out[2] = htobe32(md5.state[2]); |
| |
iv_out[3] = htobe32(md5.state[3]); |
| |
iv_out[4] = 0; |
| |
break; |
| |
default: |
| |
break; |
| |
} |
| |
|
| |
#ifdef MVXPSEC_DEBUG |
| |
if (mvxpsec_debug & MVXPSEC_DEBUG_HASH_IV) { |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV, |
| |
"%s: HMAC IV-IN\n", __func__); |
| |
mvxpsec_dump_data(__func__, (uint8_t *)iv_in, 20); |
| |
MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV, |
| |
"%s: HMAC IV-OUT\n", __func__); |
| |
mvxpsec_dump_data(__func__, (uint8_t *)iv_out, 20); |
| |
} |
| |
#endif |
| |
|
| |
return 0; |
| |
#undef HMAC_IPAD |
| |
#undef HMAC_OPAD |
| |
} |
| |
|
| |
/* |
| |
* AES Support routine |
| |
*/ |
| |
static uint8_t AES_SBOX[256] = { |
| |
99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, |
| |
171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, |
| |
156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, |
| |
229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154, |
| |
7, 18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110, |
| |
90, 160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237, |
| |
32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239, |
| |
170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168, |
| |
81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, |
| |
243, 210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, |
| |
100, 93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, |
| |
184, 20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92, |
| |
194, 211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213, |
| |
78, 169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46, |
| |
28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62, |
| |
181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158, |
| |
225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, |
| |
40, 223, 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, |
| |
176, 84, 187, 22 |
| |
}; |
| |
|
| |
static uint32_t AES_RCON[30] = { |
| |
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, |
| |
0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, |
| |
0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91 |
| |
}; |
| |
|
| |
STATIC int |
| |
mv_aes_ksched(uint8_t k[4][MAXKC], int keyBits, |
| |
uint8_t W[MAXROUNDS+1][4][MAXBC]) |
| |
{ |
| |
int KC, BC, ROUNDS; |
| |
int i, j, t, rconpointer = 0; |
| |
uint8_t tk[4][MAXKC]; |
| |
|
| |
switch (keyBits) { |
| |
case 128: |
| |
ROUNDS = 10; |
| |
KC = 4; |
| |
break; |
| |
case 192: |
| |
ROUNDS = 12; |
| |
KC = 6; |
| |
break; |
| |
case 256: |
| |
ROUNDS = 14; |
| |
KC = 8; |
| |
break; |
| |
default: |
| |
return (-1); |
| |
} |
| |
BC = 4; /* 128 bits */ |
| |
|
| |
for(j = 0; j < KC; j++) |
| |
for(i = 0; i < 4; i++) |
| |
tk[i][j] = k[i][j]; |
| |
t = 0; |
| |
|
| |
/* copy values into round key array */ |
| |
for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++) |
| |
for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j]; |
| |
|
| |
while (t < (ROUNDS+1)*BC) { /* while not enough round key material calculated */ |
| |
/* calculate new values */ |
| |
for(i = 0; i < 4; i++) |
| |
tk[i][0] ^= AES_SBOX[tk[(i+1)%4][KC-1]]; |
| |
tk[0][0] ^= AES_RCON[rconpointer++]; |
| |
|
| |
if (KC != 8) |
| |
for(j = 1; j < KC; j++) |
| |
for(i = 0; i < 4; i++) |
| |
tk[i][j] ^= tk[i][j-1]; |
| |
else { |
| |
for(j = 1; j < KC/2; j++) |
| |
for(i = 0; i < 4; i++) |
| |
tk[i][j] ^= tk[i][j-1]; |
| |
for(i = 0; i < 4; i++) |
| |
tk[i][KC/2] ^= AES_SBOX[tk[i][KC/2 - 1]]; |
| |
for(j = KC/2 + 1; j < KC; j++) |
| |
for(i = 0; i < 4; i++) |
| |
tk[i][j] ^= tk[i][j-1]; |
| |
} |
| |
/* copy values into round key array */ |
| |
for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++) |
| |
for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j]; |
| |
} |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
STATIC int |
| |
mv_aes_deckey(uint8_t *expandedKey, uint8_t *keyMaterial, int keyLen) |
| |
{ |
| |
uint8_t W[MAXROUNDS+1][4][MAXBC]; |
| |
uint8_t k[4][MAXKC]; |
| |
uint8_t j; |
| |
int i, rounds, KC; |
| |
|
| |
if (expandedKey == NULL) |
| |
return -1; |
| |
|
| |
if (!((keyLen == 128) || (keyLen == 192) || (keyLen == 256))) |
| |
return -1; |
| |
|
| |
if (keyMaterial == NULL) |
| |
return -1; |
| |
|
| |
/* initialize key schedule: */ |
| |
for (i=0; i<keyLen/8; i++) { |
| |
j = keyMaterial[i]; |
| |
k[i % 4][i / 4] = j; |
| |
} |
| |
|
| |
mv_aes_ksched(k, keyLen, W); |
| |
switch (keyLen) { |
| |
case 128: |
| |
rounds = 10; |
| |
KC = 4; |
| |
break; |
| |
case 192: |
| |
rounds = 12; |
| |
KC = 6; |
| |
break; |
| |
case 256: |
| |
rounds = 14; |
| |
KC = 8; |
| |
break; |
| |
default: |
| |
return -1; |
| |
} |
| |
|
| |
for(i=0; i<MAXBC; i++) |
| |
for(j=0; j<4; j++) |
| |
expandedKey[i*4+j] = W[rounds][j][i]; |
| |
for(; i<KC; i++) |
| |
for(j=0; j<4; j++) |
| |
expandedKey[i*4+j] = W[rounds-1][j][i+MAXBC-KC]; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* Clear cipher/mac operation state |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_packet_reset_op(struct mvxpsec_packet *mv_p) |
| |
{ |
| |
mv_p->pkt_header.desc.acc_config = 0; |
| |
mv_p->enc_off = mv_p->enc_ivoff = mv_p->enc_len = 0; |
| |
mv_p->mac_off = mv_p->mac_dst = mv_p->mac_len = 0; |
| |
} |
| |
|
| |
/* |
| |
* update MVXPSEC operation order |
| |
*/ |
| |
INLINE void |
| |
mvxpsec_packet_update_op_order(struct mvxpsec_packet *mv_p, int op) |
| |
{ |
| |
struct mvxpsec_acc_descriptor *acc_desc = &mv_p->pkt_header.desc; |
| |
uint32_t cur_op = acc_desc->acc_config & MV_ACC_CRYPTO_OP_MASK; |
| |
|
| |
KASSERT(op == MV_ACC_CRYPTO_OP_MAC || op == MV_ACC_CRYPTO_OP_ENC); |
| |
KASSERT((op & MV_ACC_CRYPTO_OP_MASK) == op); |
| |
|
| |
if (cur_op == 0) |
| |
acc_desc->acc_config |= op; |
| |
else if (cur_op == MV_ACC_CRYPTO_OP_MAC && op == MV_ACC_CRYPTO_OP_ENC) { |
| |
acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK; |
| |
acc_desc->acc_config |= MV_ACC_CRYPTO_OP_MACENC; |
| |
/* MAC then ENC (= decryption) */ |
| |
} |
| |
else if (cur_op == MV_ACC_CRYPTO_OP_ENC && op == MV_ACC_CRYPTO_OP_MAC) { |
| |
acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK; |
| |
acc_desc->acc_config |= MV_ACC_CRYPTO_OP_ENCMAC; |
| |
/* ENC then MAC (= encryption) */ |
| |
} |
| |
else { |
| |
log(LOG_ERR, "%s: multiple %s algorithm is not supported.\n", |
| |
__func__, |
| |
(op == MV_ACC_CRYPTO_OP_ENC) ? "encryption" : "authentication"); |
| |
} |
| |
} |
| |
|
| |
/* |
| |
* Parameter Conversions |
| |
*/ |
| |
INLINE uint32_t |
| |
mvxpsec_alg2acc(uint32_t alg) |
| |
{ |
| |
uint32_t reg; |
| |
|
| |
switch (alg) { |
| |
case CRYPTO_DES_CBC: |
| |
reg = MV_ACC_CRYPTO_ENC_DES; |
| |
reg |= MV_ACC_CRYPTO_CBC; |
| |
break; |
| |
case CRYPTO_3DES_CBC: |
| |
reg = MV_ACC_CRYPTO_ENC_3DES; |
| |
reg |= MV_ACC_CRYPTO_3DES_EDE; |
| |
reg |= MV_ACC_CRYPTO_CBC; |
| |
break; |
| |
case CRYPTO_AES_CBC: |
| |
reg = MV_ACC_CRYPTO_ENC_AES; |
| |
reg |= MV_ACC_CRYPTO_CBC; |
| |
break; |
| |
case CRYPTO_SHA1_HMAC_96: |
| |
reg = MV_ACC_CRYPTO_MAC_HMAC_SHA1; |
| |
reg |= MV_ACC_CRYPTO_MAC_96; |
| |
break; |
| |
case CRYPTO_MD5_HMAC_96: |
| |
reg = MV_ACC_CRYPTO_MAC_HMAC_MD5; |
| |
reg |= MV_ACC_CRYPTO_MAC_96; |
| |
break; |
| |
default: |
| |
reg = 0; |
| |
break; |
| |
} |
| |
|
| |
return reg; |
| |
} |
| |
|
| |
INLINE uint32_t |
| |
mvxpsec_aesklen(int klen) |
| |
{ |
| |
if (klen < 128) |
| |
return 0; |
| |
else if (klen < 192) |
| |
return MV_ACC_CRYPTO_AES_KLEN_128; |
| |
else if (klen < 256) |
| |
return MV_ACC_CRYPTO_AES_KLEN_192; |
| |
else |
| |
return MV_ACC_CRYPTO_AES_KLEN_256; |
| |
|
| |
return 0; |
| |
} |
| |
|
| |
/* |
| |
* String Conversions |
| |
*/ |
| |
STATIC const char * |
| |
s_errreg(uint32_t v) |
| |
{ |
| |
static char buf[80]; |
| |
|
| |
snprintf(buf, sizeof(buf), |
| |
"%sMiss %sDoubleHit %sBothHit %sDataError", |
| |
(v & MV_TDMA_ERRC_MISS) ? "+" : "-", |
| |
(v & MV_TDMA_ERRC_DHIT) ? "+" : "-", |
| |
(v & MV_TDMA_ERRC_BHIT) ? "+" : "-", |
| |
(v & MV_TDMA_ERRC_DERR) ? "+" : "-"); |
| |
|
| |
return (const char *)buf; |
| |
} |
| |
|
| |
STATIC const char * |
| |
s_winreg(uint32_t v) |
| |
{ |
| |
static char buf[80]; |
| |
|
| |
snprintf(buf, sizeof(buf), |
| |
"%s TGT 0x%x ATTR 0x%02x size %u(0x%04x)[64KB]", |
| |
(v & MV_TDMA_ATTR_ENABLE) ? "EN" : "DIS", |
| |
MV_TDMA_ATTR_GET_TARGET(v), MV_TDMA_ATTR_GET_ATTR(v), |
| |
MV_TDMA_ATTR_GET_SIZE(v), MV_TDMA_ATTR_GET_SIZE(v)); |
| |
|
| |
return (const char *)buf; |
| |
} |
| |
|
| |
STATIC const char * |
| |
s_ctrlreg(uint32_t reg) |
| |
{ |
| |
static char buf[80]; |
| |
|
| |
snprintf(buf, sizeof(buf), |
| |
"%s: %sFETCH DBURST-%u SBURST-%u %sOUTS %sCHAIN %sBSWAP %sACT", |
| |
(reg & MV_TDMA_CONTROL_ENABLE) ? "ENABLE" : "DISABLE", |
| |
(reg & MV_TDMA_CONTROL_FETCH) ? "+" : "-", |
| |
MV_TDMA_CONTROL_GET_DST_BURST(reg), |
| |
MV_TDMA_CONTROL_GET_SRC_BURST(reg), |
| |
(reg & MV_TDMA_CONTROL_OUTS_EN) ? "+" : "-", |
| |
(reg & MV_TDMA_CONTROL_CHAIN_DIS) ? "-" : "+", |
| |
(reg & MV_TDMA_CONTROL_BSWAP_DIS) ? "-" : "+", |
| |
(reg & MV_TDMA_CONTROL_ACT) ? "+" : "-"); |
| |
|
| |
return (const char *)buf; |
| |
} |
| |
|
| |
_STATIC const char * |
| |
s_xpsecintr(uint32_t v) |
| |
{ |
| |
static char buf[160]; |
| |
|
| |
snprintf(buf, sizeof(buf), |
| |
"%sAuth %sDES %sAES-ENC %sAES-DEC %sENC %sSA %sAccAndTDMA " |
| |
"%sTDMAComp %sTDMAOwn %sAccAndTDMA_Cont", |
| |
(v & MVXPSEC_INT_AUTH) ? "+" : "-", |
| |
(v & MVXPSEC_INT_DES) ? "+" : "-", |
| |
(v & MVXPSEC_INT_AES_ENC) ? "+" : "-", |
| |
(v & MVXPSEC_INT_AES_DEC) ? "+" : "-", |
| |
(v & MVXPSEC_INT_ENC) ? "+" : "-", |
| |
(v & MVXPSEC_INT_SA) ? "+" : "-", |
| |
(v & MVXPSEC_INT_ACCTDMA) ? "+" : "-", |
| |
(v & MVXPSEC_INT_TDMA_COMP) ? "+" : "-", |
| |
(v & MVXPSEC_INT_TDMA_OWN) ? "+" : "-", |
| |
(v & MVXPSEC_INT_ACCTDMA_CONT) ? "+" : "-"); |
| |
|
| |
return (const char *)buf; |
| |
} |
| |
|
| |
STATIC const char * |
| |
s_ctlalg(uint32_t alg) |
| |
{ |
| |
switch (alg) { |
| |
case CRYPTO_SHA1_HMAC_96: |
| |
return "HMAC-SHA1-96"; |
| |
case CRYPTO_SHA1_HMAC: |
| |
return "HMAC-SHA1"; |
| |
case CRYPTO_SHA1: |
| |
return "SHA1"; |
| |
case CRYPTO_MD5_HMAC_96: |
| |
return "HMAC-MD5-96"; |
| |
case CRYPTO_MD5_HMAC: |
| |
return "HMAC-MD5"; |
| |
case CRYPTO_MD5: |
| |
return "MD5"; |
| |
case CRYPTO_DES_CBC: |
| |
return "DES-CBC"; |
| |
case CRYPTO_3DES_CBC: |
| |
return "3DES-CBC"; |
| |
case CRYPTO_AES_CBC: |
| |
return "AES-CBC"; |
| |
default: |
| |
break; |
| |
} |
| |
|
| |
return "Unknown"; |
| |
} |
| |
|
| |
STATIC const char * |
| |
s_xpsec_op(uint32_t reg) |
| |
{ |
| |
reg &= MV_ACC_CRYPTO_OP_MASK; |
| |
switch (reg) { |
| |
case MV_ACC_CRYPTO_OP_ENC: |
| |
return "ENC"; |
| |
case MV_ACC_CRYPTO_OP_MAC: |
| |
return "MAC"; |
| |
case MV_ACC_CRYPTO_OP_ENCMAC: |
| |
return "ENC-MAC"; |
| |
case MV_ACC_CRYPTO_OP_MACENC: |
| |
return "MAC-ENC"; |
| |
default: |
| |
break; |
| |
} |
| |
|
| |
return "Unknown"; |
| |
|
| |
} |
| |
|
| |
STATIC const char * |
| |
s_xpsec_enc(uint32_t alg) |
| |
{ |
| |
alg <<= MV_ACC_CRYPTO_ENC_SHIFT; |
| |
switch (alg) { |
| |
case MV_ACC_CRYPTO_ENC_DES: |
| |
return "DES"; |
| |
case MV_ACC_CRYPTO_ENC_3DES: |
| |
return "3DES"; |
| |
case MV_ACC_CRYPTO_ENC_AES: |
| |
return "AES"; |
| |
default: |
| |
break; |
| |
} |
| |
|
| |
return "Unknown"; |
| |
} |
| |
|
| |
STATIC const char * |
| |
s_xpsec_mac(uint32_t alg) |
| |
{ |
| |
alg <<= MV_ACC_CRYPTO_MAC_SHIFT; |
| |
switch (alg) { |
| |
case MV_ACC_CRYPTO_MAC_NONE: |
| |
return "Disabled"; |
| |
case MV_ACC_CRYPTO_MAC_MD5: |
| |
return "MD5"; |
| |
case MV_ACC_CRYPTO_MAC_SHA1: |
| |
return "SHA1"; |
| |
case MV_ACC_CRYPTO_MAC_HMAC_MD5: |
| |
return "HMAC-MD5"; |
| |
case MV_ACC_CRYPTO_MAC_HMAC_SHA1: |
| |
return "HMAC-SHA1"; |
| |
default: |
| |
break; |
| |
} |
| |
|
| |
return "Unknown"; |
| |
} |
| |
|
| |
STATIC const char * |
| |
s_xpsec_frag(uint32_t frag) |
| |
{ |
| |
frag <<= MV_ACC_CRYPTO_FRAG_SHIFT; |
| |
switch (frag) { |
| |
case MV_ACC_CRYPTO_NOFRAG: |
| |
return "NoFragment"; |
| |
case MV_ACC_CRYPTO_FRAG_FIRST: |
| |
return "FirstFragment"; |
| |
case MV_ACC_CRYPTO_FRAG_MID: |
| |
return "MiddleFragment"; |
| |
case MV_ACC_CRYPTO_FRAG_LAST: |
| |
return "LastFragment"; |
| |
default: |
| |
break; |
| |
} |
| |
|
| |
return "Unknown"; |
| |
} |
| |
|
| |
#ifdef MVXPSEC_DEBUG |
| |
void |
| |
mvxpsec_dump_reg(struct mvxpsec_softc *sc) |
| |
{ |
| |
uint32_t reg; |
| |
int i; |
| |
|
| |
if ((mvxpsec_debug & MVXPSEC_DEBUG_DESC) == 0) |
| |
return; |
| |
|
| |
printf("--- Interrupt Registers ---\n"); |
| |
reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE); |
| |
printf("MVXPSEC INT CAUSE: 0x%08x\n", reg); |
| |
printf("MVXPSEC INT CAUSE: %s\n", s_xpsecintr(reg)); |
| |
reg = MVXPSEC_READ(sc, MVXPSEC_INT_MASK); |
| |
printf("MVXPSEC INT MASK: 0x%08x\n", reg); |
| |
printf("MVXPSEC INT MASKE: %s\n", s_xpsecintr(reg)); |
| |
|
| |
printf("--- DMA Configuration Registers ---\n"); |
| |
for (i = 0; i < MV_TDMA_NWINDOW; i++) { |
| |
reg = MVXPSEC_READ(sc, MV_TDMA_BAR(i)); |
| |
printf("TDMA BAR%d: 0x%08x\n", i, reg); |
| |
reg = MVXPSEC_READ(sc, MV_TDMA_ATTR(i)); |
| |
printf("TDMA ATTR%d: 0x%08x\n", i, reg); |
| |
printf(" -> %s\n", s_winreg(reg)); |
| |
} |
| |
|
| |
printf("--- DMA Control Registers ---\n"); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL); |
| |
printf("TDMA CONTROL: 0x%08x\n", reg); |
| |
printf(" -> %s\n", s_ctrlreg(reg)); |
| |
|
| |
printf("--- DMA Current Command Descriptors ---\n"); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE); |
| |
printf("TDMA ERR CAUSE: 0x%08x\n", reg); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_ERR_MASK); |
| |
printf("TDMA ERR MASK: 0x%08x\n", reg); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_CNT); |
| |
printf("TDMA DATA OWNER: %s\n", |
| |
(reg & MV_TDMA_CNT_OWN) ? "DMAC" : "CPU"); |
| |
printf("TDMA DATA COUNT: %d(0x%x)\n", |
| |
(reg & ~MV_TDMA_CNT_OWN), (reg & ~MV_TDMA_CNT_OWN)); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_SRC); |
| |
printf("TDMA DATA SRC: 0x%08x\n", reg); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_DST); |
| |
printf("TDMA DATA DST: 0x%08x\n", reg); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_NXT); |
| |
printf("TDMA DATA NXT: 0x%08x\n", reg); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_TDMA_CUR); |
| |
printf("TDMA DATA CUR: 0x%08x\n", reg); |
| |
|
| |
printf("--- ACC Command Register ---\n"); |
| |
reg = MVXPSEC_READ(sc, MV_ACC_COMMAND); |
| |
printf("ACC COMMAND: 0x%08x\n", reg); |
| |
printf("ACC: %sACT %sSTOP\n", |
| |
(reg & MV_ACC_COMMAND_ACT) ? "+" : "-", |
| |
(reg & MV_ACC_COMMAND_STOP) ? "+" : "-"); |
| |
|
| |
reg = MVXPSEC_READ(sc, MV_ACC_CONFIG); |
| |
printf("ACC CONFIG: 0x%08x\n", reg); |
| |
reg = MVXPSEC_READ(sc, MV_ACC_DESC); |
| |
printf("ACC DESC: 0x%08x\n", reg); |
| |
|
| |
printf("--- DES Key Register ---\n"); |
| |
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0L); |
| |
printf("DES KEY0 Low: 0x%08x\n", reg); |
| |
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0H); |
| |
printf("DES KEY0 High: 0x%08x\n", reg); |
| |
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1L); |
| |
printf("DES KEY1 Low: 0x%08x\n", reg); |
| |
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1H); |
| |
printf("DES KEY1 High: 0x%08x\n", reg); |
| |
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2L); |
| |
printf("DES KEY2 Low: 0x%08x\n", reg); |
| |
reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2H); |
| |
printf("DES KEY2 High: 0x%08x\n", reg); |
| |
|
| |
printf("--- AES Key Register ---\n"); |
| |
for (i = 0; i < 8; i++) { |
| |
reg = MVXPSEC_READ(sc, MV_CE_AES_EKEY(i)); |
| |
printf("AES ENC KEY COL%d: %08x\n", i, reg); |
| |
} |
| |
for (i = 0; i < 8; i++) { |
| |
reg = MVXPSEC_READ(sc, MV_CE_AES_DKEY(i)); |
| |
printf("AES DEC KEY COL%d: %08x\n", i, reg); |
| |
} |
| |
|
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_sram(const char *name, struct mvxpsec_softc *sc, size_t len) |
| |
{ |
| |
uint32_t reg; |
| |
|
| |
if (sc->sc_sram_va == NULL) |
| |
return; |
| |
|
| |
if (len == 0) { |
| |
printf("\n%s NO DATA(len=0)\n", name); |
| |
return; |
| |
} |
| |
else if (len > MV_ACC_SRAM_SIZE) |
| |
len = MV_ACC_SRAM_SIZE; |
| |
|
| |
mutex_enter(&sc->sc_dma_mtx); |
| |
reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL); |
| |
if (reg & MV_TDMA_CONTROL_ACT) { |
| |
printf("TDMA is active, cannot access SRAM\n"); |
| |
mutex_exit(&sc->sc_dma_mtx); |
| |
return; |
| |
} |
| |
reg = MVXPSEC_READ(sc, MV_ACC_COMMAND); |
| |
if (reg & MV_ACC_COMMAND_ACT) { |
| |
printf("SA is active, cannot access SRAM\n"); |
| |
mutex_exit(&sc->sc_dma_mtx); |
| |
return; |
| |
} |
| |
|
| |
printf("%s: dump SRAM, %zu bytes\n", name, len); |
| |
mvxpsec_dump_data(name, sc->sc_sram_va, len); |
| |
mutex_exit(&sc->sc_dma_mtx); |
| |
return; |
| |
} |
| |
|
| |
|
| |
_STATIC void |
| |
mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *dh) |
| |
{ |
| |
struct mvxpsec_descriptor *d = |
| |
(struct mvxpsec_descriptor *)dh->_desc; |
| |
|
| |
printf("--- DMA Command Descriptor ---\n"); |
| |
printf("DESC: VA=%p PA=0x%08x\n", |
| |
d, (uint32_t)dh->phys_addr); |
| |
printf("DESC: WORD0 = 0x%08x\n", d->tdma_word0); |
| |
printf("DESC: SRC = 0x%08x\n", d->tdma_src); |
| |
printf("DESC: DST = 0x%08x\n", d->tdma_dst); |
| |
printf("DESC: NXT = 0x%08x\n", d->tdma_nxt); |
| |
|
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_data(const char *name, void *p, size_t len) |
| |
{ |
| |
uint8_t *data = p; |
| |
off_t off; |
| |
|
| |
printf("%s: dump %p, %zu bytes", name, p, len); |
| |
if (p == NULL || len == 0) { |
| |
printf("\n%s: NO DATA\n", name); |
| |
return; |
| |
} |
| |
for (off = 0; off < len; off++) { |
| |
if ((off % 16) == 0) { |
| |
printf("\n%s: 0x%08x:", name, (uint32_t)off); |
| |
} |
| |
if ((off % 4) == 0) { |
| |
printf(" "); |
| |
} |
| |
printf("%02x", data[off]); |
| |
} |
| |
printf("\n"); |
| |
|
| |
return; |
| |
} |
| |
|
| |
_STATIC void |
| |
mvxpsec_dump_packet(const char *name, struct mvxpsec_packet *mv_p) |
| |
{ |
| |
struct mvxpsec_softc *sc = mv_p->mv_s->sc; |
| |
|
| |
printf("%s: packet_data:\n", name); |
| |
mvxpsec_dump_packet_data(name, mv_p); |
| |
|
| |
printf("%s: SRAM:\n", name); |
| |
mvxpsec_dump_sram(name, sc, 2000); |
| |
|
| |
printf("%s: packet_descriptor:\n", name); |
| |
mvxpsec_dump_packet_desc(name, mv_p); |
| |
} |
| |
|
| |
_STATIC void |
| |
mvxpsec_dump_packet_data(const char *name, struct mvxpsec_packet *mv_p) |
| |
{ |
| |
static char buf[1500]; |
| |
int len; |
| |
|
| |
if (mv_p->data_type == MVXPSEC_DATA_MBUF) { |
| |
struct mbuf *m; |
| |
|
| |
m = mv_p->data.mbuf; |
| |
len = m->m_pkthdr.len; |
| |
if (len > sizeof(buf)) |
| |
len = sizeof(buf); |
| |
m_copydata(m, 0, len, buf); |
| |
} |
| |
else if (mv_p->data_type == MVXPSEC_DATA_UIO) { |
| |
struct uio *uio; |
| |
|
| |
uio = mv_p->data.uio; |
| |
len = uio->uio_resid; |
| |
if (len > sizeof(buf)) |
| |
len = sizeof(buf); |
| |
cuio_copydata(uio, 0, len, buf); |
| |
} |
| |
else if (mv_p->data_type == MVXPSEC_DATA_RAW) { |
| |
len = mv_p->data_len; |
| |
if (len > sizeof(buf)) |
| |
len = sizeof(buf); |
| |
memcpy(buf, mv_p->data.raw, len); |
| |
} |
| |
else |
| |
return; |
| |
mvxpsec_dump_data(name, buf, len); |
| |
|
| |
return; |
| |
} |
| |
|
| |
_STATIC void |
| |
mvxpsec_dump_packet_desc(const char *name, struct mvxpsec_packet *mv_p) |
| |
{ |
| |
uint32_t *words; |
| |
|
| |
if (mv_p == NULL) |
| |
return; |
| |
|
| |
words = &mv_p->pkt_header.desc.acc_desc_dword0; |
| |
mvxpsec_dump_acc_config(name, words[0]); |
| |
mvxpsec_dump_acc_encdata(name, words[1], words[2]); |
| |
mvxpsec_dump_acc_enclen(name, words[2]); |
| |
mvxpsec_dump_acc_enckey(name, words[3]); |
| |
mvxpsec_dump_acc_enciv(name, words[4]); |
| |
mvxpsec_dump_acc_macsrc(name, words[5]); |
| |
mvxpsec_dump_acc_macdst(name, words[6]); |
| |
mvxpsec_dump_acc_maciv(name, words[7]); |
| |
|
| |
return; |
| |
} |
| |
|
| |
_STATIC void |
| |
mvxpsec_dump_acc_config(const char *name, uint32_t w) |
| |
{ |
| |
/* SA: Dword 0 */ |
| |
printf("%s: Dword0=0x%08x\n", name, w); |
| |
printf("%s: OP = %s\n", name, |
| |
s_xpsec_op(MV_ACC_CRYPTO_OP(w))); |
| |
printf("%s: MAC = %s\n", name, |
| |
s_xpsec_mac(MV_ACC_CRYPTO_MAC(w))); |
| |
printf("%s: MAC_LEN = %s\n", name, |
| |
w & MV_ACC_CRYPTO_MAC_96 ? "96-bit" : "full-bit"); |
| |
printf("%s: ENC = %s\n", name, |
| |
s_xpsec_enc(MV_ACC_CRYPTO_ENC(w))); |
| |
printf("%s: DIR = %s\n", name, |
| |
w & MV_ACC_CRYPTO_DECRYPT ? "decryption" : "encryption"); |
| |
printf("%s: CHAIN = %s\n", name, |
| |
w & MV_ACC_CRYPTO_CBC ? "CBC" : "ECB"); |
| |
printf("%s: 3DES = %s\n", name, |
| |
w & MV_ACC_CRYPTO_3DES_EDE ? "EDE" : "EEE"); |
| |
printf("%s: FRAGMENT = %s\n", name, |
| |
s_xpsec_frag(MV_ACC_CRYPTO_FRAG(w))); |
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_acc_encdata(const char *name, uint32_t w, uint32_t w2) |
| |
{ |
| |
/* SA: Dword 1 */ |
| |
printf("%s: Dword1=0x%08x\n", name, w); |
| |
printf("%s: ENC SRC = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w)); |
| |
printf("%s: ENC DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w)); |
| |
printf("%s: ENC RANGE = 0x%x - 0x%x\n", name, |
| |
MV_ACC_DESC_GET_VAL_1(w), |
| |
MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_1(w2) - 1); |
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_acc_enclen(const char *name, uint32_t w) |
| |
{ |
| |
/* SA: Dword 2 */ |
| |
printf("%s: Dword2=0x%08x\n", name, w); |
| |
printf("%s: ENC LEN = %d\n", name, |
| |
MV_ACC_DESC_GET_VAL_1(w)); |
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_acc_enckey(const char *name, uint32_t w) |
| |
{ |
| |
/* SA: Dword 3 */ |
| |
printf("%s: Dword3=0x%08x\n", name, w); |
| |
printf("%s: EKEY = 0x%x\n", name, |
| |
MV_ACC_DESC_GET_VAL_1(w)); |
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_acc_enciv(const char *name, uint32_t w) |
| |
{ |
| |
/* SA: Dword 4 */ |
| |
printf("%s: Dword4=0x%08x\n", name, w); |
| |
printf("%s: EIV = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w)); |
| |
printf("%s: EIV_BUF = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w)); |
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_acc_macsrc(const char *name, uint32_t w) |
| |
{ |
| |
/* SA: Dword 5 */ |
| |
printf("%s: Dword5=0x%08x\n", name, w); |
| |
printf("%s: MAC_SRC = 0x%x\n", name, |
| |
MV_ACC_DESC_GET_VAL_1(w)); |
| |
printf("%s: MAC_TOTAL_LEN = %d\n", name, |
| |
MV_ACC_DESC_GET_VAL_3(w)); |
| |
printf("%s: MAC_RANGE = 0x%0x - 0x%0x\n", name, |
| |
MV_ACC_DESC_GET_VAL_1(w), |
| |
MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_3(w) - 1); |
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_acc_macdst(const char *name, uint32_t w) |
| |
{ |
| |
/* SA: Dword 6 */ |
| |
printf("%s: Dword6=0x%08x\n", name, w); |
| |
printf("%s: MAC_DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w)); |
| |
printf("%s: MAC_BLOCK_LEN = %d\n", name, |
| |
MV_ACC_DESC_GET_VAL_2(w)); |
| |
return; |
| |
} |
| |
|
| |
STATIC void |
| |
mvxpsec_dump_acc_maciv(const char *name, uint32_t w) |
| |
{ |
| |
/* SA: Dword 7 */ |
| |
printf("%s: Dword7=0x%08x\n", name, w); |
| |
printf("%s: MAC_INNER_IV = 0x%x\n", name, |
| |
MV_ACC_DESC_GET_VAL_1(w)); |
| |
printf("%s: MAC_OUTER_IV = 0x%x\n", name, |
| |
MV_ACC_DESC_GET_VAL_2(w)); |
| |
return; |
| |
} |
| |
#endif |
| |
|
![[BACK]](/icons/back.gif){kind=icon}
Return to mvxpsec.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / dev / marvell