mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-22 18:32:14 +07:00
db91af0fbe
Instead of unconditionally forcing 4 byte alignment for all generic chaining modes that rely on crypto_xor() or crypto_inc() (which may result in unnecessary copying of data when the underlying hardware can perform unaligned accesses efficiently), make those functions deal with unaligned input explicitly, but only if the Kconfig symbol HAVE_EFFICIENT_UNALIGNED_ACCESS is set. This will allow us to drop the alignmasks from the CBC, CMAC, CTR, CTS, PCBC and SEQIV drivers. For crypto_inc(), this simply involves making the 4-byte stride conditional on HAVE_EFFICIENT_UNALIGNED_ACCESS being set, given that it typically operates on 16 byte buffers. For crypto_xor(), an algorithm is implemented that simply runs through the input using the largest strides possible if unaligned accesses are allowed. If they are not, an optimal sequence of memory accesses is emitted that takes the relative alignment of the input buffers into account, e.g., if the relative misalignment of dst and src is 4 bytes, the entire xor operation will be completed using 4 byte loads and stores (modulo unaligned bits at the start and end). Note that all expressions involving misalign are simply eliminated by the compiler when HAVE_EFFICIENT_UNALIGNED_ACCESS is defined. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
226 lines
5.4 KiB
C
226 lines
5.4 KiB
C
/*
|
|
* seqiv: Sequence Number IV Generator
|
|
*
|
|
* This generator generates an IV based on a sequence number by xoring it
|
|
* with a salt. This algorithm is mainly useful for CTR and similar modes.
|
|
*
|
|
* Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License as published by the Free
|
|
* Software Foundation; either version 2 of the License, or (at your option)
|
|
* any later version.
|
|
*
|
|
*/
|
|
|
|
#include <crypto/internal/geniv.h>
|
|
#include <crypto/scatterwalk.h>
|
|
#include <crypto/skcipher.h>
|
|
#include <linux/err.h>
|
|
#include <linux/init.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/string.h>
|
|
|
|
static void seqiv_free(struct crypto_instance *inst);
|
|
|
|
static void seqiv_aead_encrypt_complete2(struct aead_request *req, int err)
|
|
{
|
|
struct aead_request *subreq = aead_request_ctx(req);
|
|
struct crypto_aead *geniv;
|
|
|
|
if (err == -EINPROGRESS)
|
|
return;
|
|
|
|
if (err)
|
|
goto out;
|
|
|
|
geniv = crypto_aead_reqtfm(req);
|
|
memcpy(req->iv, subreq->iv, crypto_aead_ivsize(geniv));
|
|
|
|
out:
|
|
kzfree(subreq->iv);
|
|
}
|
|
|
|
static void seqiv_aead_encrypt_complete(struct crypto_async_request *base,
|
|
int err)
|
|
{
|
|
struct aead_request *req = base->data;
|
|
|
|
seqiv_aead_encrypt_complete2(req, err);
|
|
aead_request_complete(req, err);
|
|
}
|
|
|
|
static int seqiv_aead_encrypt(struct aead_request *req)
|
|
{
|
|
struct crypto_aead *geniv = crypto_aead_reqtfm(req);
|
|
struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv);
|
|
struct aead_request *subreq = aead_request_ctx(req);
|
|
crypto_completion_t compl;
|
|
void *data;
|
|
u8 *info;
|
|
unsigned int ivsize = 8;
|
|
int err;
|
|
|
|
if (req->cryptlen < ivsize)
|
|
return -EINVAL;
|
|
|
|
aead_request_set_tfm(subreq, ctx->child);
|
|
|
|
compl = req->base.complete;
|
|
data = req->base.data;
|
|
info = req->iv;
|
|
|
|
if (req->src != req->dst) {
|
|
SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
|
|
|
|
skcipher_request_set_tfm(nreq, ctx->sknull);
|
|
skcipher_request_set_callback(nreq, req->base.flags,
|
|
NULL, NULL);
|
|
skcipher_request_set_crypt(nreq, req->src, req->dst,
|
|
req->assoclen + req->cryptlen,
|
|
NULL);
|
|
|
|
err = crypto_skcipher_encrypt(nreq);
|
|
if (err)
|
|
return err;
|
|
}
|
|
|
|
if (unlikely(!IS_ALIGNED((unsigned long)info,
|
|
crypto_aead_alignmask(geniv) + 1))) {
|
|
info = kmalloc(ivsize, req->base.flags &
|
|
CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
|
|
GFP_ATOMIC);
|
|
if (!info)
|
|
return -ENOMEM;
|
|
|
|
memcpy(info, req->iv, ivsize);
|
|
compl = seqiv_aead_encrypt_complete;
|
|
data = req;
|
|
}
|
|
|
|
aead_request_set_callback(subreq, req->base.flags, compl, data);
|
|
aead_request_set_crypt(subreq, req->dst, req->dst,
|
|
req->cryptlen - ivsize, info);
|
|
aead_request_set_ad(subreq, req->assoclen + ivsize);
|
|
|
|
crypto_xor(info, ctx->salt, ivsize);
|
|
scatterwalk_map_and_copy(info, req->dst, req->assoclen, ivsize, 1);
|
|
|
|
err = crypto_aead_encrypt(subreq);
|
|
if (unlikely(info != req->iv))
|
|
seqiv_aead_encrypt_complete2(req, err);
|
|
return err;
|
|
}
|
|
|
|
static int seqiv_aead_decrypt(struct aead_request *req)
|
|
{
|
|
struct crypto_aead *geniv = crypto_aead_reqtfm(req);
|
|
struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv);
|
|
struct aead_request *subreq = aead_request_ctx(req);
|
|
crypto_completion_t compl;
|
|
void *data;
|
|
unsigned int ivsize = 8;
|
|
|
|
if (req->cryptlen < ivsize + crypto_aead_authsize(geniv))
|
|
return -EINVAL;
|
|
|
|
aead_request_set_tfm(subreq, ctx->child);
|
|
|
|
compl = req->base.complete;
|
|
data = req->base.data;
|
|
|
|
aead_request_set_callback(subreq, req->base.flags, compl, data);
|
|
aead_request_set_crypt(subreq, req->src, req->dst,
|
|
req->cryptlen - ivsize, req->iv);
|
|
aead_request_set_ad(subreq, req->assoclen + ivsize);
|
|
|
|
scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0);
|
|
|
|
return crypto_aead_decrypt(subreq);
|
|
}
|
|
|
|
static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb)
|
|
{
|
|
struct aead_instance *inst;
|
|
struct crypto_aead_spawn *spawn;
|
|
struct aead_alg *alg;
|
|
int err;
|
|
|
|
inst = aead_geniv_alloc(tmpl, tb, 0, 0);
|
|
|
|
if (IS_ERR(inst))
|
|
return PTR_ERR(inst);
|
|
|
|
spawn = aead_instance_ctx(inst);
|
|
alg = crypto_spawn_aead_alg(spawn);
|
|
|
|
err = -EINVAL;
|
|
if (inst->alg.ivsize != sizeof(u64))
|
|
goto free_inst;
|
|
|
|
inst->alg.encrypt = seqiv_aead_encrypt;
|
|
inst->alg.decrypt = seqiv_aead_decrypt;
|
|
|
|
inst->alg.init = aead_init_geniv;
|
|
inst->alg.exit = aead_exit_geniv;
|
|
|
|
inst->alg.base.cra_ctxsize = sizeof(struct aead_geniv_ctx);
|
|
inst->alg.base.cra_ctxsize += inst->alg.ivsize;
|
|
|
|
err = aead_register_instance(tmpl, inst);
|
|
if (err)
|
|
goto free_inst;
|
|
|
|
out:
|
|
return err;
|
|
|
|
free_inst:
|
|
aead_geniv_free(inst);
|
|
goto out;
|
|
}
|
|
|
|
static int seqiv_create(struct crypto_template *tmpl, struct rtattr **tb)
|
|
{
|
|
struct crypto_attr_type *algt;
|
|
|
|
algt = crypto_get_attr_type(tb);
|
|
if (IS_ERR(algt))
|
|
return PTR_ERR(algt);
|
|
|
|
if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
|
|
return -EINVAL;
|
|
|
|
return seqiv_aead_create(tmpl, tb);
|
|
}
|
|
|
|
static void seqiv_free(struct crypto_instance *inst)
|
|
{
|
|
aead_geniv_free(aead_instance(inst));
|
|
}
|
|
|
|
static struct crypto_template seqiv_tmpl = {
|
|
.name = "seqiv",
|
|
.create = seqiv_create,
|
|
.free = seqiv_free,
|
|
.module = THIS_MODULE,
|
|
};
|
|
|
|
static int __init seqiv_module_init(void)
|
|
{
|
|
return crypto_register_template(&seqiv_tmpl);
|
|
}
|
|
|
|
static void __exit seqiv_module_exit(void)
|
|
{
|
|
crypto_unregister_template(&seqiv_tmpl);
|
|
}
|
|
|
|
module_init(seqiv_module_init);
|
|
module_exit(seqiv_module_exit);
|
|
|
|
MODULE_LICENSE("GPL");
|
|
MODULE_DESCRIPTION("Sequence Number IV Generator");
|
|
MODULE_ALIAS_CRYPTO("seqiv");
|