AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-12-27 17:45:09 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity
b8c5d882c8
linux_dsm_epyc7002/drivers/crypto/inside-secure/safexcel.c
Pascal van Leeuwen b8c5d882c8 crypto: inside-secure - Fix stability issue with Macchiatobin 
This patch corrects an error in the Transform Record Cache initialization
code that was causing intermittent stability problems on the Macchiatobin
board.

Unfortunately, due to HW platform specifics, the problem could not happen
on the main development platform, being the VCU118 Xilinx development
board. And since it was a problem with hash table access, it was very
dependent on the actual physical context record DMA buffers being used,
i.e. with some (bad) luck it could seemingly work quit stable for a while.

Signed-off-by: Pascal van Leeuwen <pvanleeuwen@verimatrix.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2019-10-05 01:06:14 +10:00

1883 lines
54 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2017 Marvell
*
* Antoine Tenart <antoine.tenart@free-electrons.com>
*/
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/firmware.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/of_platform.h>
#include <linux/of_irq.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/workqueue.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include "safexcel.h"
static u32 max_rings = EIP197_MAX_RINGS;
module_param(max_rings, uint, 0644);
MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
{
int i;
/*
* Map all interfaces/rings to register index 0
* so they can share contexts. Without this, the EIP197 will
* assume each interface/ring to be in its own memory domain
* i.e. have its own subset of UNIQUE memory addresses.
* Which would cause records with the SAME memory address to
* use DIFFERENT cache buffers, causing both poor cache utilization
* AND serious coherence/invalidation issues.
*/
for (i = 0; i < 4; i++)
writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));
/*
* Initialize other virtualization regs for cache
* These may not be in their reset state ...
*/
for (i = 0; i < priv->config.rings; i++) {
writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
writel(EIP197_FLUE_CONFIG_MAGIC,
priv->base + EIP197_FLUE_CONFIG(i));
}
writel(0, priv->base + EIP197_FLUE_OFFSETS);
writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
}
static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
u32 addrmid, int *actbank)
{
u32 val;
int curbank;
curbank = addrmid >> 16;
if (curbank != *actbank) {
val = readl(priv->base + EIP197_CS_RAM_CTRL);
val = (val & ~EIP197_CS_BANKSEL_MASK) |
(curbank << EIP197_CS_BANKSEL_OFS);
writel(val, priv->base + EIP197_CS_RAM_CTRL);
*actbank = curbank;
}
}
static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
int maxbanks, u32 probemask)
{
u32 val, addrhi, addrlo, addrmid;
int actbank;
/*
* And probe the actual size of the physically attached cache data RAM
* Using a binary subdivision algorithm downto 32 byte cache lines.
*/
addrhi = 1 << (16 + maxbanks);
addrlo = 0;
actbank = min(maxbanks - 1, 0);
while ((addrhi - addrlo) > 32) {
/* write marker to lowest address in top half */
addrmid = (addrhi + addrlo) >> 1;
eip197_trc_cache_banksel(priv, addrmid, &actbank);
writel((addrmid | (addrlo << 16)) & probemask,
priv->base + EIP197_CLASSIFICATION_RAMS +
(addrmid & 0xffff));
/* write marker to lowest address in bottom half */
eip197_trc_cache_banksel(priv, addrlo, &actbank);
writel((addrlo | (addrhi << 16)) & probemask,
priv->base + EIP197_CLASSIFICATION_RAMS +
(addrlo & 0xffff));
/* read back marker from top half */
eip197_trc_cache_banksel(priv, addrmid, &actbank);
val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
(addrmid & 0xffff));
if (val == ((addrmid | (addrlo << 16)) & probemask)) {
/* read back correct, continue with top half */
addrlo = addrmid;
} else {
/* not read back correct, continue with bottom half */
addrhi = addrmid;
}
}
return addrhi;
}
static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
int cs_rc_max, int cs_ht_wc)
{
int i;
u32 htable_offset, val, offset;
/* Clear all records in administration RAM */
for (i = 0; i < cs_rc_max; i++) {
offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
EIP197_CS_RC_PREV(EIP197_RC_NULL),
priv->base + offset);
val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
if (i == 0)
val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
else if (i == cs_rc_max - 1)
val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
writel(val, priv->base + offset + 4);
/* must also initialize the address key due to ECC! */
writel(0, priv->base + offset + 8);
writel(0, priv->base + offset + 12);
}
/* Clear the hash table entries */
htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
for (i = 0; i < cs_ht_wc; i++)
writel(GENMASK(29, 0),
priv->base + EIP197_CLASSIFICATION_RAMS +
htable_offset + i * sizeof(u32));
}
static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
{
u32 val, dsize, asize;
int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
int cs_rc_abs_max, cs_ht_sz;
int maxbanks;
/* Setup (dummy) virtualization for cache */
eip197_trc_cache_setupvirt(priv);
/*
* Enable the record cache memory access and
* probe the bank select width
*/
val = readl(priv->base + EIP197_CS_RAM_CTRL);
val &= ~EIP197_TRC_ENABLE_MASK;
val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
writel(val, priv->base + EIP197_CS_RAM_CTRL);
val = readl(priv->base + EIP197_CS_RAM_CTRL);
maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;
/* Clear all ECC errors */
writel(0, priv->base + EIP197_TRC_ECCCTRL);
/*
* Make sure the cache memory is accessible by taking record cache into
* reset. Need data memory access here, not admin access.
*/
val = readl(priv->base + EIP197_TRC_PARAMS);
val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
writel(val, priv->base + EIP197_TRC_PARAMS);
/* Probed data RAM size in bytes */
dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff);
/*
* Now probe the administration RAM size pretty much the same way
* Except that only the lower 30 bits are writable and we don't need
* bank selects
*/
val = readl(priv->base + EIP197_TRC_PARAMS);
/* admin access now */
val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
writel(val, priv->base + EIP197_TRC_PARAMS);
/* Probed admin RAM size in admin words */
asize = eip197_trc_cache_probe(priv, 0, 0xbfffffff) >> 4;
/* Clear any ECC errors detected while probing! */
writel(0, priv->base + EIP197_TRC_ECCCTRL);
/*
* Determine optimal configuration from RAM sizes
* Note that we assume that the physical RAM configuration is sane
* Therefore, we don't do any parameter error checking here ...
*/
/* For now, just use a single record format covering everything */
cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;
/*
* Step #1: How many records will physically fit?
* Hard upper limit is 1023!
*/
cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
/* Step #2: Need at least 2 words in the admin RAM per record */
cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
/* Step #3: Determine log2 of hash table size */
cs_ht_sz = __fls(asize - cs_rc_max) - 2;
/* Step #4: determine current size of hash table in dwords */
cs_ht_wc = 16 << cs_ht_sz; /* dwords, not admin words */
/* Step #5: add back excess words and see if we can fit more records */
cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2));
/* Clear the cache RAMs */
eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);
/* Disable the record cache memory access */
val = readl(priv->base + EIP197_CS_RAM_CTRL);
val &= ~EIP197_TRC_ENABLE_MASK;
writel(val, priv->base + EIP197_CS_RAM_CTRL);
/* Write head and tail pointers of the record free chain */
val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
writel(val, priv->base + EIP197_TRC_FREECHAIN);
/* Configure the record cache #1 */
val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
writel(val, priv->base + EIP197_TRC_PARAMS2);
/* Configure the record cache #2 */
val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
writel(val, priv->base + EIP197_TRC_PARAMS);
dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
}
static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
{
int pe, i;
u32 val;
for (pe = 0; pe < priv->config.pes; pe++) {
/* Configure the token FIFO's */
writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));
/* Clear the ICE scratchpad memory */
val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
/* clear the scratchpad RAM using 32 bit writes only */
for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
writel(0, EIP197_PE(priv) +
EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));
/* Reset the IFPP engine to make its program mem accessible */
writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
/* Reset the IPUE engine to make its program mem accessible */
writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
/* Enable access to all IFPP program memories */
writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
}
}
static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
const struct firmware *fw)
{
const u32 *data = (const u32 *)fw->data;
int i;
/* Write the firmware */
for (i = 0; i < fw->size / sizeof(u32); i++)
writel(be32_to_cpu(data[i]),
priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
/* Exclude final 2 NOPs from size */
return i - EIP197_FW_TERMINAL_NOPS;
}
/*
* If FW is actual production firmware, then poll for its initialization
* to complete and check if it is good for the HW, otherwise just return OK.
*/
static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
{
int pe, pollcnt;
u32 base, pollofs;
if (fpp)
pollofs = EIP197_FW_FPP_READY;
else
pollofs = EIP197_FW_PUE_READY;
for (pe = 0; pe < priv->config.pes; pe++) {
base = EIP197_PE_ICE_SCRATCH_RAM(pe);
pollcnt = EIP197_FW_START_POLLCNT;
while (pollcnt &&
(readl_relaxed(EIP197_PE(priv) + base +
pollofs) != 1)) {
pollcnt--;
}
if (!pollcnt) {
dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
fpp, pe);
return false;
}
}
return true;
}
static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
int ipuesz, int ifppsz, int minifw)
{
int pe;
u32 val;
for (pe = 0; pe < priv->config.pes; pe++) {
/* Disable access to all program memory */
writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
/* Start IFPP microengines */
if (minifw)
val = 0;
else
val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
EIP197_PE_ICE_UENG_DEBUG_RESET;
writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
/* Start IPUE microengines */
if (minifw)
val = 0;
else
val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
EIP197_PE_ICE_UENG_DEBUG_RESET;
writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
}
/* For miniFW startup, there is no initialization, so always succeed */
if (minifw)
return true;
/* Wait until all the firmwares have properly started up */
if (!poll_fw_ready(priv, 1))
return false;
if (!poll_fw_ready(priv, 0))
return false;
return true;
}
static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
{
const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
const struct firmware *fw[FW_NB];
char fw_path[37], *dir = NULL;
int i, j, ret = 0, pe;
int ipuesz, ifppsz, minifw = 0;
if (priv->version == EIP197D_MRVL)
dir = "eip197d";
else if (priv->version == EIP197B_MRVL ||
priv->version == EIP197_DEVBRD)
dir = "eip197b";
else
return -ENODEV;
retry_fw:
for (i = 0; i < FW_NB; i++) {
snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
if (ret) {
if (minifw || priv->version != EIP197B_MRVL)
goto release_fw;
/* Fallback to the old firmware location for the
* EIP197b.
*/
ret = firmware_request_nowarn(&fw[i], fw_name[i],
priv->dev);
if (ret)
goto release_fw;
}
}
eip197_init_firmware(priv);
ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);
/* Enable access to IPUE program memories */
for (pe = 0; pe < priv->config.pes; pe++)
writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);
if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
dev_dbg(priv->dev, "Firmware loaded successfully\n");
return 0;
}
ret = -ENODEV;
release_fw:
for (j = 0; j < i; j++)
release_firmware(fw[j]);
if (!minifw) {
/* Retry with minifw path */
dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
dir = "eip197_minifw";
minifw = 1;
goto retry_fw;
}
dev_dbg(priv->dev, "Firmware load failed.\n");
return ret;
}
static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
{
u32 cd_size_rnd, val;
int i, cd_fetch_cnt;
cd_size_rnd = (priv->config.cd_size +
(BIT(priv->hwconfig.hwdataw) - 1)) >>
priv->hwconfig.hwdataw;
/* determine number of CD's we can fetch into the CD FIFO as 1 block */
if (priv->flags & SAFEXCEL_HW_EIP197) {
/* EIP197: try to fetch enough in 1 go to keep all pipes busy */
cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
(priv->config.pes * EIP197_FETCH_DEPTH));
} else {
/* for the EIP97, just fetch all that fits minus 1 */
cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
cd_size_rnd) - 1;
}
for (i = 0; i < priv->config.rings; i++) {
/* ring base address */
writel(lower_32_bits(priv->ring[i].cdr.base_dma),
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
writel(upper_32_bits(priv->ring[i].cdr.base_dma),
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
priv->config.cd_size,
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
writel(((cd_fetch_cnt *
(cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
(cd_fetch_cnt * priv->config.cd_offset),
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
/* Configure DMA tx control */
val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
/* clear any pending interrupt */
writel(GENMASK(5, 0),
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
}
return 0;
}
static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
{
u32 rd_size_rnd, val;
int i, rd_fetch_cnt;
/* determine number of RD's we can fetch into the FIFO as one block */
rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
(BIT(priv->hwconfig.hwdataw) - 1)) >>
priv->hwconfig.hwdataw;
if (priv->flags & SAFEXCEL_HW_EIP197) {
/* EIP197: try to fetch enough in 1 go to keep all pipes busy */
rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
(priv->config.pes * EIP197_FETCH_DEPTH));
} else {
/* for the EIP97, just fetch all that fits minus 1 */
rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
rd_size_rnd) - 1;
}
for (i = 0; i < priv->config.rings; i++) {
/* ring base address */
writel(lower_32_bits(priv->ring[i].rdr.base_dma),
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
writel(upper_32_bits(priv->ring[i].rdr.base_dma),
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
priv->config.rd_size,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
writel(((rd_fetch_cnt *
(rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
(rd_fetch_cnt * priv->config.rd_offset),
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
/* Configure DMA tx control */
val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
writel(val,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
/* clear any pending interrupt */
writel(GENMASK(7, 0),
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
/* enable ring interrupt */
val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
val |= EIP197_RDR_IRQ(i);
writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
}
return 0;
}
static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
{
u32 val;
int i, ret, pe;
dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
priv->config.pes, priv->config.rings);
/*
* For EIP197's only set maximum number of TX commands to 2^5 = 32
* Skip for the EIP97 as it does not have this field.
*/
if (priv->flags & SAFEXCEL_HW_EIP197) {
val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
}
/* Configure wr/rd cache values */
writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
/* Interrupts reset */
/* Disable all global interrupts */
writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
/* Clear any pending interrupt */
writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
/* Processing Engine configuration */
for (pe = 0; pe < priv->config.pes; pe++) {
/* Data Fetch Engine configuration */
/* Reset all DFE threads */
writel(EIP197_DxE_THR_CTRL_RESET_PE,
EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
if (priv->flags & SAFEXCEL_HW_EIP197)
/* Reset HIA input interface arbiter (EIP197 only) */
writel(EIP197_HIA_RA_PE_CTRL_RESET,
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
/* DMA transfer size to use */
val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
/* Leave the DFE threads reset state */
writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
/* Configure the processing engine thresholds */
writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
EIP197_PE_IN_xBUF_THRES_MAX(9),
EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
EIP197_PE_IN_xBUF_THRES_MAX(7),
EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
if (priv->flags & SAFEXCEL_HW_EIP197)
/* enable HIA input interface arbiter and rings */
writel(EIP197_HIA_RA_PE_CTRL_EN |
GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
/* Data Store Engine configuration */
/* Reset all DSE threads */
writel(EIP197_DxE_THR_CTRL_RESET_PE,
EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
/* Wait for all DSE threads to complete */
while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
GENMASK(15, 12)) != GENMASK(15, 12))
;
/* DMA transfer size to use */
val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) |
EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
/* FIXME: instability issues can occur for EIP97 but disabling
* it impacts performance.
*/
if (priv->flags & SAFEXCEL_HW_EIP197)
val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
/* Leave the DSE threads reset state */
writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
/* Configure the procesing engine thresholds */
writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) |
EIP197_PE_OUT_DBUF_THRES_MAX(8),
EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
/* Processing Engine configuration */
/* Token & context configuration */
val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
/* H/W capabilities selection: just enable everything */
writel(EIP197_FUNCTION_ALL,
EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
writel(EIP197_FUNCTION_ALL,
EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
}
/* Command Descriptor Rings prepare */
for (i = 0; i < priv->config.rings; i++) {
/* Clear interrupts for this ring */
writel(GENMASK(31, 0),
EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
/* Disable external triggering */
writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
/* Clear the pending prepared counter */
writel(EIP197_xDR_PREP_CLR_COUNT,
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
/* Clear the pending processed counter */
writel(EIP197_xDR_PROC_CLR_COUNT,
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
writel(0,
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
writel(0,
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
}
/* Result Descriptor Ring prepare */
for (i = 0; i < priv->config.rings; i++) {
/* Disable external triggering*/
writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
/* Clear the pending prepared counter */
writel(EIP197_xDR_PREP_CLR_COUNT,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
/* Clear the pending processed counter */
writel(EIP197_xDR_PROC_CLR_COUNT,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
writel(0,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
writel(0,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
/* Ring size */
writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
}
for (pe = 0; pe < priv->config.pes; pe++) {
/* Enable command descriptor rings */
writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
/* Enable result descriptor rings */
writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
}
/* Clear any HIA interrupt */
writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
if (priv->flags & SAFEXCEL_HW_EIP197) {
eip197_trc_cache_init(priv);
priv->flags |= EIP197_TRC_CACHE;
ret = eip197_load_firmwares(priv);
if (ret)
return ret;
}
safexcel_hw_setup_cdesc_rings(priv);
safexcel_hw_setup_rdesc_rings(priv);
return 0;
}
/* Called with ring's lock taken */
static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
int ring)
{
int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
if (!coal)
return;
/* Configure when we want an interrupt */
writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
}
void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
{
struct crypto_async_request *req, *backlog;
struct safexcel_context *ctx;
int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
/* If a request wasn't properly dequeued because of a lack of resources,
* proceeded it first,
*/
req = priv->ring[ring].req;
backlog = priv->ring[ring].backlog;
if (req)
goto handle_req;
while (true) {
spin_lock_bh(&priv->ring[ring].queue_lock);
backlog = crypto_get_backlog(&priv->ring[ring].queue);
req = crypto_dequeue_request(&priv->ring[ring].queue);
spin_unlock_bh(&priv->ring[ring].queue_lock);
if (!req) {
priv->ring[ring].req = NULL;
priv->ring[ring].backlog = NULL;
goto finalize;
}
handle_req:
ctx = crypto_tfm_ctx(req->tfm);
ret = ctx->send(req, ring, &commands, &results);
if (ret)
goto request_failed;
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
/* In case the send() helper did not issue any command to push
* to the engine because the input data was cached, continue to
* dequeue other requests as this is valid and not an error.
*/
if (!commands && !results)
continue;
cdesc += commands;
rdesc += results;
nreq++;
}
request_failed:
/* Not enough resources to handle all the requests. Bail out and save
* the request and the backlog for the next dequeue call (per-ring).
*/
priv->ring[ring].req = req;
priv->ring[ring].backlog = backlog;
finalize:
if (!nreq)
return;
spin_lock_bh(&priv->ring[ring].lock);
priv->ring[ring].requests += nreq;
if (!priv->ring[ring].busy) {
safexcel_try_push_requests(priv, ring);
priv->ring[ring].busy = true;
}
spin_unlock_bh(&priv->ring[ring].lock);
/* let the RDR know we have pending descriptors */
writel((rdesc * priv->config.rd_offset) << 2,
EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
/* let the CDR know we have pending descriptors */
writel((cdesc * priv->config.cd_offset) << 2,
EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
}
inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
struct safexcel_result_desc *rdesc)
{
if (likely((!rdesc->descriptor_overflow) &&
(!rdesc->buffer_overflow) &&
(!rdesc->result_data.error_code)))
return 0;
if (rdesc->descriptor_overflow)
dev_err(priv->dev, "Descriptor overflow detected");
if (rdesc->buffer_overflow)
dev_err(priv->dev, "Buffer overflow detected");
if (rdesc->result_data.error_code & 0x4066) {
/* Fatal error (bits 1,2,5,6 & 14) */
dev_err(priv->dev,
"result descriptor error (%x)",
rdesc->result_data.error_code);
return -EIO;
} else if (rdesc->result_data.error_code &
(BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
/*
* Give priority over authentication fails:
* Blocksize, length & overflow errors,
* something wrong with the input!
*/
return -EINVAL;
} else if (rdesc->result_data.error_code & BIT(9)) {
/* Authentication failed */
return -EBADMSG;
}
/* All other non-fatal errors */
return -EINVAL;
}
inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc,
struct crypto_async_request *req)
{
int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
priv->ring[ring].rdr_req[i] = req;
}
inline struct crypto_async_request *
safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
{
int i = safexcel_ring_first_rdr_index(priv, ring);
return priv->ring[ring].rdr_req[i];
}
void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
{
struct safexcel_command_desc *cdesc;
/* Acknowledge the command descriptors */
do {
cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
if (IS_ERR(cdesc)) {
dev_err(priv->dev,
"Could not retrieve the command descriptor\n");
return;
}
} while (!cdesc->last_seg);
}
void safexcel_inv_complete(struct crypto_async_request *req, int error)
{
struct safexcel_inv_result *result = req->data;
if (error == -EINPROGRESS)
return;
result->error = error;
complete(&result->completion);
}
int safexcel_invalidate_cache(struct crypto_async_request *async,
struct safexcel_crypto_priv *priv,
dma_addr_t ctxr_dma, int ring)
{
struct safexcel_command_desc *cdesc;
struct safexcel_result_desc *rdesc;
int ret = 0;
/* Prepare command descriptor */
cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
if (IS_ERR(cdesc))
return PTR_ERR(cdesc);
cdesc->control_data.type = EIP197_TYPE_EXTENDED;
cdesc->control_data.options = 0;
cdesc->control_data.refresh = 0;
cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
/* Prepare result descriptor */
rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
if (IS_ERR(rdesc)) {
ret = PTR_ERR(rdesc);
goto cdesc_rollback;
}
safexcel_rdr_req_set(priv, ring, rdesc, async);
return ret;
cdesc_rollback:
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
return ret;
}
static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
int ring)
{
struct crypto_async_request *req;
struct safexcel_context *ctx;
int ret, i, nreq, ndesc, tot_descs, handled = 0;
bool should_complete;
handle_results:
tot_descs = 0;
nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
if (!nreq)
goto requests_left;
for (i = 0; i < nreq; i++) {
req = safexcel_rdr_req_get(priv, ring);
ctx = crypto_tfm_ctx(req->tfm);
ndesc = ctx->handle_result(priv, ring, req,
&should_complete, &ret);
if (ndesc < 0) {
dev_err(priv->dev, "failed to handle result (%d)\n",
ndesc);
goto acknowledge;
}
if (should_complete) {
local_bh_disable();
req->complete(req, ret);
local_bh_enable();
}
tot_descs += ndesc;
handled++;
}
acknowledge:
if (i)
writel(EIP197_xDR_PROC_xD_PKT(i) |
EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
/* If the number of requests overflowed the counter, try to proceed more
* requests.
*/
if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
goto handle_results;
requests_left:
spin_lock_bh(&priv->ring[ring].lock);
priv->ring[ring].requests -= handled;
safexcel_try_push_requests(priv, ring);
if (!priv->ring[ring].requests)
priv->ring[ring].busy = false;
spin_unlock_bh(&priv->ring[ring].lock);
}
static void safexcel_dequeue_work(struct work_struct *work)
{
struct safexcel_work_data *data =
container_of(work, struct safexcel_work_data, work);
safexcel_dequeue(data->priv, data->ring);
}
struct safexcel_ring_irq_data {
struct safexcel_crypto_priv *priv;
int ring;
};
static irqreturn_t safexcel_irq_ring(int irq, void *data)
{
struct safexcel_ring_irq_data *irq_data = data;
struct safexcel_crypto_priv *priv = irq_data->priv;
int ring = irq_data->ring, rc = IRQ_NONE;
u32 status, stat;
status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
if (!status)
return rc;
/* RDR interrupts */
if (status & EIP197_RDR_IRQ(ring)) {
stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
if (unlikely(stat & EIP197_xDR_ERR)) {
/*
* Fatal error, the RDR is unusable and must be
* reinitialized. This should not happen under
* normal circumstances.
*/
dev_err(priv->dev, "RDR: fatal error.\n");
} else if (likely(stat & EIP197_xDR_THRESH)) {
rc = IRQ_WAKE_THREAD;
}
/* ACK the interrupts */
writel(stat & 0xff,
EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
}
/* ACK the interrupts */
writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
return rc;
}
static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
{
struct safexcel_ring_irq_data *irq_data = data;
struct safexcel_crypto_priv *priv = irq_data->priv;
int ring = irq_data->ring;
safexcel_handle_result_descriptor(priv, ring);
queue_work(priv->ring[ring].workqueue,
&priv->ring[ring].work_data.work);
return IRQ_HANDLED;
}
static int safexcel_request_ring_irq(void *pdev, int irqid,
int is_pci_dev,
irq_handler_t handler,
irq_handler_t threaded_handler,
struct safexcel_ring_irq_data *ring_irq_priv)
{
int ret, irq;
struct device *dev;
if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
struct pci_dev *pci_pdev = pdev;
dev = &pci_pdev->dev;
irq = pci_irq_vector(pci_pdev, irqid);
if (irq < 0) {
dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
irqid, irq);
return irq;
}
} else if (IS_ENABLED(CONFIG_OF)) {
struct platform_device *plf_pdev = pdev;
char irq_name[6] = {0}; /* "ringX\0" */
snprintf(irq_name, 6, "ring%d", irqid);
dev = &plf_pdev->dev;
irq = platform_get_irq_byname(plf_pdev, irq_name);
if (irq < 0) {
dev_err(dev, "unable to get IRQ '%s' (err %d)\n",
irq_name, irq);
return irq;
}
}
ret = devm_request_threaded_irq(dev, irq, handler,
threaded_handler, IRQF_ONESHOT,
dev_name(dev), ring_irq_priv);
if (ret) {
dev_err(dev, "unable to request IRQ %d\n", irq);
return ret;
}
return irq;
}
static struct safexcel_alg_template *safexcel_algs[] = {
&safexcel_alg_ecb_des,
&safexcel_alg_cbc_des,
&safexcel_alg_ecb_des3_ede,
&safexcel_alg_cbc_des3_ede,
&safexcel_alg_ecb_aes,
&safexcel_alg_cbc_aes,
&safexcel_alg_cfb_aes,
&safexcel_alg_ofb_aes,
&safexcel_alg_ctr_aes,
&safexcel_alg_md5,
&safexcel_alg_sha1,
&safexcel_alg_sha224,
&safexcel_alg_sha256,
&safexcel_alg_sha384,
&safexcel_alg_sha512,
&safexcel_alg_hmac_md5,
&safexcel_alg_hmac_sha1,
&safexcel_alg_hmac_sha224,
&safexcel_alg_hmac_sha256,
&safexcel_alg_hmac_sha384,
&safexcel_alg_hmac_sha512,
&safexcel_alg_authenc_hmac_sha1_cbc_aes,
&safexcel_alg_authenc_hmac_sha224_cbc_aes,
&safexcel_alg_authenc_hmac_sha256_cbc_aes,
&safexcel_alg_authenc_hmac_sha384_cbc_aes,
&safexcel_alg_authenc_hmac_sha512_cbc_aes,
&safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
&safexcel_alg_authenc_hmac_sha1_ctr_aes,
&safexcel_alg_authenc_hmac_sha224_ctr_aes,
&safexcel_alg_authenc_hmac_sha256_ctr_aes,
&safexcel_alg_authenc_hmac_sha384_ctr_aes,
&safexcel_alg_authenc_hmac_sha512_ctr_aes,
&safexcel_alg_xts_aes,
&safexcel_alg_gcm,
&safexcel_alg_ccm,
&safexcel_alg_crc32,
&safexcel_alg_cbcmac,
&safexcel_alg_xcbcmac,
&safexcel_alg_cmac,
&safexcel_alg_chacha20,
&safexcel_alg_chachapoly,
&safexcel_alg_chachapoly_esp,
&safexcel_alg_sm3,
&safexcel_alg_hmac_sm3,
&safexcel_alg_ecb_sm4,
&safexcel_alg_cbc_sm4,
&safexcel_alg_ofb_sm4,
&safexcel_alg_cfb_sm4,
&safexcel_alg_ctr_sm4,
&safexcel_alg_authenc_hmac_sha1_cbc_sm4,
&safexcel_alg_authenc_hmac_sm3_cbc_sm4,
&safexcel_alg_authenc_hmac_sha1_ctr_sm4,
&safexcel_alg_authenc_hmac_sm3_ctr_sm4,
&safexcel_alg_sha3_224,
&safexcel_alg_sha3_256,
&safexcel_alg_sha3_384,
&safexcel_alg_sha3_512,
&safexcel_alg_hmac_sha3_224,
&safexcel_alg_hmac_sha3_256,
&safexcel_alg_hmac_sha3_384,
&safexcel_alg_hmac_sha3_512,
&safexcel_alg_authenc_hmac_sha1_cbc_des,
&safexcel_alg_authenc_hmac_sha256_cbc_des3_ede,
&safexcel_alg_authenc_hmac_sha224_cbc_des3_ede,
&safexcel_alg_authenc_hmac_sha512_cbc_des3_ede,
&safexcel_alg_authenc_hmac_sha384_cbc_des3_ede,
&safexcel_alg_authenc_hmac_sha256_cbc_des,
&safexcel_alg_authenc_hmac_sha224_cbc_des,
&safexcel_alg_authenc_hmac_sha512_cbc_des,
&safexcel_alg_authenc_hmac_sha384_cbc_des,
};
static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
{
int i, j, ret = 0;
for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
safexcel_algs[i]->priv = priv;
/* Do we have all required base algorithms available? */
if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
safexcel_algs[i]->algo_mask)
/* No, so don't register this ciphersuite */
continue;
if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
else
ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
if (ret)
goto fail;
}
return 0;
fail:
for (j = 0; j < i; j++) {
/* Do we have all required base algorithms available? */
if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
safexcel_algs[j]->algo_mask)
/* No, so don't unregister this ciphersuite */
continue;
if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
else
crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
}
return ret;
}
static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
{
int i;
for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
/* Do we have all required base algorithms available? */
if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
safexcel_algs[i]->algo_mask)
/* No, so don't unregister this ciphersuite */
continue;
if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
else
crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
}
}
static void safexcel_configure(struct safexcel_crypto_priv *priv)
{
u32 val, mask = 0;
val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
/* Read number of PEs from the engine */
if (priv->flags & SAFEXCEL_HW_EIP197)
/* Wider field width for all EIP197 type engines */
mask = EIP197_N_PES_MASK;
else
/* Narrow field width for EIP97 type engine */
mask = EIP97_N_PES_MASK;
priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask;
priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
val = (val & GENMASK(27, 25)) >> 25;
mask = BIT(val) - 1;
priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
}
static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
{
struct safexcel_register_offsets *offsets = &priv->offsets;
if (priv->flags & SAFEXCEL_HW_EIP197) {
offsets->hia_aic = EIP197_HIA_AIC_BASE;
offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;
offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;
offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE;
offsets->hia_dfe = EIP197_HIA_DFE_BASE;
offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE;
offsets->hia_dse = EIP197_HIA_DSE_BASE;
offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;
offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;
offsets->pe = EIP197_PE_BASE;
offsets->global = EIP197_GLOBAL_BASE;
} else {
offsets->hia_aic = EIP97_HIA_AIC_BASE;
offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;
offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;
offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE;
offsets->hia_dfe = EIP97_HIA_DFE_BASE;
offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE;
offsets->hia_dse = EIP97_HIA_DSE_BASE;
offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;
offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;
offsets->pe = EIP97_PE_BASE;
offsets->global = EIP97_GLOBAL_BASE;
}
}
/*
* Generic part of probe routine, shared by platform and PCI driver
*
* Assumes IO resources have been mapped, private data mem has been allocated,
* clocks have been enabled, device pointer has been assigned etc.
*
*/
static int safexcel_probe_generic(void *pdev,
struct safexcel_crypto_priv *priv,
int is_pci_dev)
{
struct device *dev = priv->dev;
u32 peid, version, mask, val, hiaopt;
int i, ret, hwctg;
priv->context_pool = dmam_pool_create("safexcel-context", dev,
sizeof(struct safexcel_context_record),
1, 0);
if (!priv->context_pool)
return -ENOMEM;
/*
* First try the EIP97 HIA version regs
* For the EIP197, this is guaranteed to NOT return any of the test
* values
*/
version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);
mask = 0; /* do not swap */
if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
} else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
/* read back byte-swapped, so complement byte swap bits */
mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
} else {
/* So it wasn't an EIP97 ... maybe it's an EIP197? */
version = readl(priv->base + EIP197_HIA_AIC_BASE +
EIP197_HIA_VERSION);
if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
priv->flags |= SAFEXCEL_HW_EIP197;
} else if (EIP197_REG_HI16(version) ==
EIP197_HIA_VERSION_BE) {
/* read back byte-swapped, so complement swap bits */
mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
priv->flags |= SAFEXCEL_HW_EIP197;
} else {
return -ENODEV;
}
}
/* Now initialize the reg offsets based on the probing info so far */
safexcel_init_register_offsets(priv);
/*
* If the version was read byte-swapped, we need to flip the device
* swapping Keep in mind here, though, that what we write will also be
* byte-swapped ...
*/
if (mask) {
val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
val = val ^ (mask >> 24); /* toggle byte swap bits */
writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
}
/*
* We're not done probing yet! We may fall through to here if no HIA
* was found at all. So, with the endianness presumably correct now and
* the offsets setup, *really* probe for the EIP97/EIP197.
*/
version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
if (((priv->flags & SAFEXCEL_HW_EIP197) &&
(EIP197_REG_LO16(version) != EIP197_VERSION_LE)) ||
((!(priv->flags & SAFEXCEL_HW_EIP197) &&
(EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
/*
* We did not find the device that matched our initial probing
* (or our initial probing failed) Report appropriate error.
*/
return -ENODEV;
}
priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
hwctg = version >> 28;
peid = version & 255;
/* Detect EIP96 packet engine and version */
version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
return -ENODEV;
}
priv->hwconfig.pever = EIP197_VERSION_MASK(version);
hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
if (priv->flags & SAFEXCEL_HW_EIP197) {
/* EIP197 */
priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
EIP197_HWDATAW_MASK;
priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
EIP197_CFSIZE_MASK) +
EIP197_CFSIZE_ADJUST;
priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
EIP197_RFSIZE_MASK) +
EIP197_RFSIZE_ADJUST;
} else {
/* EIP97 */
priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
EIP97_HWDATAW_MASK;
priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
EIP97_CFSIZE_MASK;
priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
EIP97_RFSIZE_MASK;
}
/* Get supported algorithms from EIP96 transform engine */
priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
EIP197_PE_EIP96_OPTIONS(0));
/* Print single info line describing what we just detected */
dev_info(priv->dev, "EIP%d:%x(%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n",
peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hiaver,
priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize,
priv->hwconfig.hwrfsize, priv->hwconfig.pever,
priv->hwconfig.algo_flags);
safexcel_configure(priv);
if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) {
/*
* Request MSI vectors for global + 1 per ring -
* or just 1 for older dev images
*/
struct pci_dev *pci_pdev = pdev;
ret = pci_alloc_irq_vectors(pci_pdev,
priv->config.rings + 1,
priv->config.rings + 1,
PCI_IRQ_MSI | PCI_IRQ_MSIX);
if (ret < 0) {
dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
return ret;
}
}
/* Register the ring IRQ handlers and configure the rings */
priv->ring = devm_kcalloc(dev, priv->config.rings,
sizeof(*priv->ring),
GFP_KERNEL);
if (!priv->ring)
return -ENOMEM;
for (i = 0; i < priv->config.rings; i++) {
char wq_name[9] = {0};
int irq;
struct safexcel_ring_irq_data *ring_irq;
ret = safexcel_init_ring_descriptors(priv,
&priv->ring[i].cdr,
&priv->ring[i].rdr);
if (ret) {
dev_err(dev, "Failed to initialize rings\n");
return ret;
}
priv->ring[i].rdr_req = devm_kcalloc(dev,
EIP197_DEFAULT_RING_SIZE,
sizeof(priv->ring[i].rdr_req),
GFP_KERNEL);
if (!priv->ring[i].rdr_req)
return -ENOMEM;
ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
if (!ring_irq)
return -ENOMEM;
ring_irq->priv = priv;
ring_irq->ring = i;
irq = safexcel_request_ring_irq(pdev,
EIP197_IRQ_NUMBER(i, is_pci_dev),
is_pci_dev,
safexcel_irq_ring,
safexcel_irq_ring_thread,
ring_irq);
if (irq < 0) {
dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
return irq;
}
priv->ring[i].work_data.priv = priv;
priv->ring[i].work_data.ring = i;
INIT_WORK(&priv->ring[i].work_data.work,
safexcel_dequeue_work);
snprintf(wq_name, 9, "wq_ring%d", i);
priv->ring[i].workqueue =
create_singlethread_workqueue(wq_name);
if (!priv->ring[i].workqueue)
return -ENOMEM;
priv->ring[i].requests = 0;
priv->ring[i].busy = false;
crypto_init_queue(&priv->ring[i].queue,
EIP197_DEFAULT_RING_SIZE);
spin_lock_init(&priv->ring[i].lock);
spin_lock_init(&priv->ring[i].queue_lock);
}
atomic_set(&priv->ring_used, 0);
ret = safexcel_hw_init(priv);
if (ret) {
dev_err(dev, "HW init failed (%d)\n", ret);
return ret;
}
ret = safexcel_register_algorithms(priv);
if (ret) {
dev_err(dev, "Failed to register algorithms (%d)\n", ret);
return ret;
}
return 0;
}
static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
{
int i;
for (i = 0; i < priv->config.rings; i++) {
/* clear any pending interrupt */
writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
/* Reset the CDR base address */
writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
/* Reset the RDR base address */
writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
}
}
#if IS_ENABLED(CONFIG_OF)
/* for Device Tree platform driver */
static int safexcel_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct safexcel_crypto_priv *priv;
int ret;
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
priv->dev = dev;
priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
platform_set_drvdata(pdev, priv);
priv->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(priv->base)) {
dev_err(dev, "failed to get resource\n");
return PTR_ERR(priv->base);
}
priv->clk = devm_clk_get(&pdev->dev, NULL);
ret = PTR_ERR_OR_ZERO(priv->clk);
/* The clock isn't mandatory */
if (ret != -ENOENT) {
if (ret)
return ret;
ret = clk_prepare_enable(priv->clk);
if (ret) {
dev_err(dev, "unable to enable clk (%d)\n", ret);
return ret;
}
}
priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
ret = PTR_ERR_OR_ZERO(priv->reg_clk);
/* The clock isn't mandatory */
if (ret != -ENOENT) {
if (ret)
goto err_core_clk;
ret = clk_prepare_enable(priv->reg_clk);
if (ret) {
dev_err(dev, "unable to enable reg clk (%d)\n", ret);
goto err_core_clk;
}
}
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
if (ret)
goto err_reg_clk;
/* Generic EIP97/EIP197 device probing */
ret = safexcel_probe_generic(pdev, priv, 0);
if (ret)
goto err_reg_clk;
return 0;
err_reg_clk:
clk_disable_unprepare(priv->reg_clk);
err_core_clk:
clk_disable_unprepare(priv->clk);
return ret;
}
static int safexcel_remove(struct platform_device *pdev)
{
struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
int i;
safexcel_unregister_algorithms(priv);
safexcel_hw_reset_rings(priv);
clk_disable_unprepare(priv->clk);
for (i = 0; i < priv->config.rings; i++)
destroy_workqueue(priv->ring[i].workqueue);
return 0;
}
static const struct of_device_id safexcel_of_match_table[] = {
{
.compatible = "inside-secure,safexcel-eip97ies",
.data = (void *)EIP97IES_MRVL,
},
{
.compatible = "inside-secure,safexcel-eip197b",
.data = (void *)EIP197B_MRVL,
},
{
.compatible = "inside-secure,safexcel-eip197d",
.data = (void *)EIP197D_MRVL,
},
/* For backward compatibility and intended for generic use */
{
.compatible = "inside-secure,safexcel-eip97",
.data = (void *)EIP97IES_MRVL,
},
{
.compatible = "inside-secure,safexcel-eip197",
.data = (void *)EIP197B_MRVL,
},
{},
};
static struct platform_driver crypto_safexcel = {
.probe = safexcel_probe,
.remove = safexcel_remove,
.driver = {
.name = "crypto-safexcel",
.of_match_table = safexcel_of_match_table,
},
};
#endif
#if IS_ENABLED(CONFIG_PCI)
/* PCIE devices - i.e. Inside Secure development boards */
static int safexcel_pci_probe(struct pci_dev *pdev,
const struct pci_device_id *ent)
{
struct device *dev = &pdev->dev;
struct safexcel_crypto_priv *priv;
void __iomem *pciebase;
int rc;
u32 val;
dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
ent->vendor, ent->device, ent->subvendor,
ent->subdevice, ent->driver_data);
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
priv->dev = dev;
priv->version = (enum safexcel_eip_version)ent->driver_data;
pci_set_drvdata(pdev, priv);
/* enable the device */
rc = pcim_enable_device(pdev);
if (rc) {
dev_err(dev, "Failed to enable PCI device\n");
return rc;
}
/* take ownership of PCI BAR0 */
rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
if (rc) {
dev_err(dev, "Failed to map IO region for BAR0\n");
return rc;
}
priv->base = pcim_iomap_table(pdev)[0];
if (priv->version == EIP197_DEVBRD) {
dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");
rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
if (rc) {
dev_err(dev, "Failed to map IO region for BAR4\n");
return rc;
}
pciebase = pcim_iomap_table(pdev)[2];
val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
(val & 0xff));
/* Setup MSI identity map mapping */
writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);
/* Enable all device interrupts */
writel(GENMASK(31, 0),
pciebase + EIP197_XLX_USER_INT_ENB_MSK);
} else {
dev_err(dev, "Unrecognised IRQ block identifier %x\n",
val);
return -ENODEV;
}
/* HW reset FPGA dev board */
/* assert reset */
writel(1, priv->base + EIP197_XLX_GPIO_BASE);
wmb(); /* maintain strict ordering for accesses here */
/* deassert reset */
writel(0, priv->base + EIP197_XLX_GPIO_BASE);
wmb(); /* maintain strict ordering for accesses here */
}
/* enable bus mastering */
pci_set_master(pdev);
/* Generic EIP97/EIP197 device probing */
rc = safexcel_probe_generic(pdev, priv, 1);
return rc;
}
void safexcel_pci_remove(struct pci_dev *pdev)
{
struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
int i;
safexcel_unregister_algorithms(priv);
for (i = 0; i < priv->config.rings; i++)
destroy_workqueue(priv->ring[i].workqueue);
safexcel_hw_reset_rings(priv);
}
static const struct pci_device_id safexcel_pci_ids[] = {
{
PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
0x16ae, 0xc522),
.driver_data = EIP197_DEVBRD,
},
{},
};
MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);
static struct pci_driver safexcel_pci_driver = {
.name = "crypto-safexcel",
.id_table = safexcel_pci_ids,
.probe = safexcel_pci_probe,
.remove = safexcel_pci_remove,
};
#endif
/* Unfortunately, we have to resort to global variables here */
#if IS_ENABLED(CONFIG_PCI)
int pcireg_rc = -EINVAL; /* Default safe value */
#endif
#if IS_ENABLED(CONFIG_OF)
int ofreg_rc = -EINVAL; /* Default safe value */
#endif
static int __init safexcel_init(void)
{
#if IS_ENABLED(CONFIG_PCI)
/* Register PCI driver */
pcireg_rc = pci_register_driver(&safexcel_pci_driver);
#endif
#if IS_ENABLED(CONFIG_OF)
/* Register platform driver */
ofreg_rc = platform_driver_register(&crypto_safexcel);
#if IS_ENABLED(CONFIG_PCI)
/* Return success if either PCI or OF registered OK */
return pcireg_rc ? ofreg_rc : 0;
#else
return ofreg_rc;
#endif
#else
#if IS_ENABLED(CONFIG_PCI)
return pcireg_rc;
#else
return -EINVAL;
#endif
#endif
}
static void __exit safexcel_exit(void)
{
#if IS_ENABLED(CONFIG_OF)
/* Unregister platform driver */
if (!ofreg_rc)
platform_driver_unregister(&crypto_safexcel);
#endif
#if IS_ENABLED(CONFIG_PCI)
/* Unregister PCI driver if successfully registered before */
if (!pcireg_rc)
pci_unregister_driver(&safexcel_pci_driver);
#endif
}
module_init(safexcel_init);
module_exit(safexcel_exit);
MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
MODULE_LICENSE("GPL v2");
Reference in New Issue View Git Blame Copy Permalink