linux_dsm_epyc7002/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
Zain Wang 9a42e4eed3 crypto: rockchip - return the err code when unable dequeue the crypto request
Sometime we would unable to dequeue the crypto request, in this case,
we should finish crypto and return the err code.

Signed-off-by: zain wang <wzz@rock-chips.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2017-08-03 14:00:23 +08:00

527 lines
15 KiB
C

/*
* Crypto acceleration support for Rockchip RK3288
*
* Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
*
* Author: Zain Wang <zain.wang@rock-chips.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* Some ideas are from marvell-cesa.c and s5p-sss.c driver.
*/
#include "rk3288_crypto.h"
#define RK_CRYPTO_DEC BIT(0)
static void rk_crypto_complete(struct rk_crypto_info *dev, int err)
{
if (dev->ablk_req->base.complete)
dev->ablk_req->base.complete(&dev->ablk_req->base, err);
}
static int rk_handle_req(struct rk_crypto_info *dev,
struct ablkcipher_request *req)
{
unsigned long flags;
struct crypto_async_request *async_req, *backlog;
int err;
if (!IS_ALIGNED(req->nbytes, dev->align_size))
return -EINVAL;
dev->left_bytes = req->nbytes;
dev->total = req->nbytes;
dev->sg_src = req->src;
dev->first = req->src;
dev->nents = sg_nents(req->src);
dev->sg_dst = req->dst;
dev->aligned = 1;
dev->ablk_req = req;
spin_lock_irqsave(&dev->lock, flags);
err = ablkcipher_enqueue_request(&dev->queue, req);
backlog = crypto_get_backlog(&dev->queue);
async_req = crypto_dequeue_request(&dev->queue);
spin_unlock_irqrestore(&dev->lock, flags);
if (!async_req) {
dev_err(dev->dev, "async_req is NULL !!\n");
return err;
}
if (backlog) {
backlog->complete(backlog, -EINPROGRESS);
backlog = NULL;
}
dev->ablk_req = ablkcipher_request_cast(async_req);
tasklet_schedule(&dev->queue_task);
return err;
}
static int rk_aes_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
keylen != AES_KEYSIZE_256) {
crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
ctx->keylen = keylen;
memcpy_toio(ctx->dev->reg + RK_CRYPTO_AES_KEY_0, key, keylen);
return 0;
}
static int rk_tdes_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
u32 tmp[DES_EXPKEY_WORDS];
if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) {
crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
if (keylen == DES_KEY_SIZE) {
if (!des_ekey(tmp, key) &&
(tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
return -EINVAL;
}
}
ctx->keylen = keylen;
memcpy_toio(ctx->dev->reg + RK_CRYPTO_TDES_KEY1_0, key, keylen);
return 0;
}
static int rk_aes_ecb_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_AES_ECB_MODE;
return rk_handle_req(dev, req);
}
static int rk_aes_ecb_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;
return rk_handle_req(dev, req);
}
static int rk_aes_cbc_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_AES_CBC_MODE;
return rk_handle_req(dev, req);
}
static int rk_aes_cbc_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;
return rk_handle_req(dev, req);
}
static int rk_des_ecb_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = 0;
return rk_handle_req(dev, req);
}
static int rk_des_ecb_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_DEC;
return rk_handle_req(dev, req);
}
static int rk_des_cbc_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;
return rk_handle_req(dev, req);
}
static int rk_des_cbc_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;
return rk_handle_req(dev, req);
}
static int rk_des3_ede_ecb_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_TDES_SELECT;
return rk_handle_req(dev, req);
}
static int rk_des3_ede_ecb_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;
return rk_handle_req(dev, req);
}
static int rk_des3_ede_cbc_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;
return rk_handle_req(dev, req);
}
static int rk_des3_ede_cbc_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct rk_crypto_info *dev = ctx->dev;
dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |
RK_CRYPTO_DEC;
return rk_handle_req(dev, req);
}
static void rk_ablk_hw_init(struct rk_crypto_info *dev)
{
struct crypto_ablkcipher *cipher =
crypto_ablkcipher_reqtfm(dev->ablk_req);
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher);
u32 ivsize, block, conf_reg = 0;
block = crypto_tfm_alg_blocksize(tfm);
ivsize = crypto_ablkcipher_ivsize(cipher);
if (block == DES_BLOCK_SIZE) {
dev->mode |= RK_CRYPTO_TDES_FIFO_MODE |
RK_CRYPTO_TDES_BYTESWAP_KEY |
RK_CRYPTO_TDES_BYTESWAP_IV;
CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, dev->mode);
memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0,
dev->ablk_req->info, ivsize);
conf_reg = RK_CRYPTO_DESSEL;
} else {
dev->mode |= RK_CRYPTO_AES_FIFO_MODE |
RK_CRYPTO_AES_KEY_CHANGE |
RK_CRYPTO_AES_BYTESWAP_KEY |
RK_CRYPTO_AES_BYTESWAP_IV;
if (ctx->keylen == AES_KEYSIZE_192)
dev->mode |= RK_CRYPTO_AES_192BIT_key;
else if (ctx->keylen == AES_KEYSIZE_256)
dev->mode |= RK_CRYPTO_AES_256BIT_key;
CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, dev->mode);
memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0,
dev->ablk_req->info, ivsize);
}
conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO |
RK_CRYPTO_BYTESWAP_BRFIFO;
CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg);
CRYPTO_WRITE(dev, RK_CRYPTO_INTENA,
RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA);
}
static void crypto_dma_start(struct rk_crypto_info *dev)
{
CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, dev->addr_in);
CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, dev->count / 4);
CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, dev->addr_out);
CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START |
_SBF(RK_CRYPTO_BLOCK_START, 16));
}
static int rk_set_data_start(struct rk_crypto_info *dev)
{
int err;
err = dev->load_data(dev, dev->sg_src, dev->sg_dst);
if (!err)
crypto_dma_start(dev);
return err;
}
static int rk_ablk_start(struct rk_crypto_info *dev)
{
unsigned long flags;
int err;
spin_lock_irqsave(&dev->lock, flags);
rk_ablk_hw_init(dev);
err = rk_set_data_start(dev);
spin_unlock_irqrestore(&dev->lock, flags);
return err;
}
static void rk_iv_copyback(struct rk_crypto_info *dev)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(dev->ablk_req);
u32 ivsize = crypto_ablkcipher_ivsize(tfm);
if (ivsize == DES_BLOCK_SIZE)
memcpy_fromio(dev->ablk_req->info,
dev->reg + RK_CRYPTO_TDES_IV_0, ivsize);
else if (ivsize == AES_BLOCK_SIZE)
memcpy_fromio(dev->ablk_req->info,
dev->reg + RK_CRYPTO_AES_IV_0, ivsize);
}
/* return:
* true some err was occurred
* fault no err, continue
*/
static int rk_ablk_rx(struct rk_crypto_info *dev)
{
int err = 0;
dev->unload_data(dev);
if (!dev->aligned) {
if (!sg_pcopy_from_buffer(dev->ablk_req->dst, dev->nents,
dev->addr_vir, dev->count,
dev->total - dev->left_bytes -
dev->count)) {
err = -EINVAL;
goto out_rx;
}
}
if (dev->left_bytes) {
if (dev->aligned) {
if (sg_is_last(dev->sg_src)) {
dev_err(dev->dev, "[%s:%d] Lack of data\n",
__func__, __LINE__);
err = -ENOMEM;
goto out_rx;
}
dev->sg_src = sg_next(dev->sg_src);
dev->sg_dst = sg_next(dev->sg_dst);
}
err = rk_set_data_start(dev);
} else {
rk_iv_copyback(dev);
/* here show the calculation is over without any err */
dev->complete(dev, 0);
}
out_rx:
return err;
}
static int rk_ablk_cra_init(struct crypto_tfm *tfm)
{
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_alg *alg = tfm->__crt_alg;
struct rk_crypto_tmp *algt;
algt = container_of(alg, struct rk_crypto_tmp, alg.crypto);
ctx->dev = algt->dev;
ctx->dev->align_size = crypto_tfm_alg_alignmask(tfm) + 1;
ctx->dev->start = rk_ablk_start;
ctx->dev->update = rk_ablk_rx;
ctx->dev->complete = rk_crypto_complete;
ctx->dev->addr_vir = (char *)__get_free_page(GFP_KERNEL);
return ctx->dev->addr_vir ? ctx->dev->enable_clk(ctx->dev) : -ENOMEM;
}
static void rk_ablk_cra_exit(struct crypto_tfm *tfm)
{
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
free_page((unsigned long)ctx->dev->addr_vir);
ctx->dev->disable_clk(ctx->dev);
}
struct rk_crypto_tmp rk_ecb_aes_alg = {
.type = ALG_TYPE_CIPHER,
.alg.crypto = {
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-rk",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct rk_cipher_ctx),
.cra_alignmask = 0x0f,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = rk_ablk_cra_init,
.cra_exit = rk_ablk_cra_exit,
.cra_u.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = rk_aes_setkey,
.encrypt = rk_aes_ecb_encrypt,
.decrypt = rk_aes_ecb_decrypt,
}
}
};
struct rk_crypto_tmp rk_cbc_aes_alg = {
.type = ALG_TYPE_CIPHER,
.alg.crypto = {
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-rk",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct rk_cipher_ctx),
.cra_alignmask = 0x0f,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = rk_ablk_cra_init,
.cra_exit = rk_ablk_cra_exit,
.cra_u.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = rk_aes_setkey,
.encrypt = rk_aes_cbc_encrypt,
.decrypt = rk_aes_cbc_decrypt,
}
}
};
struct rk_crypto_tmp rk_ecb_des_alg = {
.type = ALG_TYPE_CIPHER,
.alg.crypto = {
.cra_name = "ecb(des)",
.cra_driver_name = "ecb-des-rk",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct rk_cipher_ctx),
.cra_alignmask = 0x07,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = rk_ablk_cra_init,
.cra_exit = rk_ablk_cra_exit,
.cra_u.ablkcipher = {
.min_keysize = DES_KEY_SIZE,
.max_keysize = DES_KEY_SIZE,
.setkey = rk_tdes_setkey,
.encrypt = rk_des_ecb_encrypt,
.decrypt = rk_des_ecb_decrypt,
}
}
};
struct rk_crypto_tmp rk_cbc_des_alg = {
.type = ALG_TYPE_CIPHER,
.alg.crypto = {
.cra_name = "cbc(des)",
.cra_driver_name = "cbc-des-rk",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct rk_cipher_ctx),
.cra_alignmask = 0x07,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = rk_ablk_cra_init,
.cra_exit = rk_ablk_cra_exit,
.cra_u.ablkcipher = {
.min_keysize = DES_KEY_SIZE,
.max_keysize = DES_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.setkey = rk_tdes_setkey,
.encrypt = rk_des_cbc_encrypt,
.decrypt = rk_des_cbc_decrypt,
}
}
};
struct rk_crypto_tmp rk_ecb_des3_ede_alg = {
.type = ALG_TYPE_CIPHER,
.alg.crypto = {
.cra_name = "ecb(des3_ede)",
.cra_driver_name = "ecb-des3-ede-rk",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct rk_cipher_ctx),
.cra_alignmask = 0x07,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = rk_ablk_cra_init,
.cra_exit = rk_ablk_cra_exit,
.cra_u.ablkcipher = {
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.setkey = rk_tdes_setkey,
.encrypt = rk_des3_ede_ecb_encrypt,
.decrypt = rk_des3_ede_ecb_decrypt,
}
}
};
struct rk_crypto_tmp rk_cbc_des3_ede_alg = {
.type = ALG_TYPE_CIPHER,
.alg.crypto = {
.cra_name = "cbc(des3_ede)",
.cra_driver_name = "cbc-des3-ede-rk",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct rk_cipher_ctx),
.cra_alignmask = 0x07,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = rk_ablk_cra_init,
.cra_exit = rk_ablk_cra_exit,
.cra_u.ablkcipher = {
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.setkey = rk_tdes_setkey,
.encrypt = rk_des3_ede_cbc_encrypt,
.decrypt = rk_des3_ede_cbc_decrypt,
}
}
};